From 7294e39a44902733241720fecac83644b5552d52 Mon Sep 17 00:00:00 2001 From: "Chaurasiya, Payal" Date: Tue, 13 May 2025 07:18:43 -0700 Subject: [PATCH 1/6] Add workflow federated evaluation tests in pq Signed-off-by: Chaurasiya, Payal --- .github/workflows/pq_pipeline.yml | 15 ++- .github/workflows/wf_federated_evaluation.yml | 65 ++++++++++++ .../test_suites/wf_federated_runtime_tests.py | 100 ++++++++++++++---- 3 files changed, 161 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/wf_federated_evaluation.yml diff --git a/.github/workflows/pq_pipeline.yml b/.github/workflows/pq_pipeline.yml index 41d5ac99c9..f00b18c087 100644 --- a/.github/workflows/pq_pipeline.yml +++ b/.github/workflows/pq_pipeline.yml @@ -157,7 +157,18 @@ jobs: uses: ./.github/workflows/task_runner_flower_e2e.yml with: commit_id: ${{ needs.set_commit_id_for_all_jobs.outputs.commit_id }} - + # wf federated evaluation + wf_federated_evaluation: + if: | + (github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') || + (github.event_name == 'workflow_dispatch') + name: Workflow Federated Evaluation E2E + needs: wf_secagg_e2e + uses: ./.github/workflows/wf_federated_evaluation.yml + with: + commit_id: ${{ needs.set_commit_id_for_all_jobs.outputs.commit_id }} + secrets: inherit + # run code scanning tools run_trivy: if: | (github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') || @@ -206,6 +217,8 @@ jobs: task_runner_dockerized_e2e, task_runner_secret_ssl_e2e, task_runner_flower_app_pytorch, + task_runner_connectivity_e2e, + wf_federated_evaluation, run_trivy, run_bandit ] diff --git a/.github/workflows/wf_federated_evaluation.yml b/.github/workflows/wf_federated_evaluation.yml new file mode 100644 index 0000000000..b538fd3203 --- /dev/null +++ b/.github/workflows/wf_federated_evaluation.yml @@ -0,0 +1,65 @@ +# -------------------------------------------------------- +# It runs the federated evaluation tests +# -------------------------------------------------------- +name: Federated Evaluation E2E +on: + workflow_call: + inputs: + commit_id: + required: false + type: string + workflow_dispatch: + + permissions: + contents: read + +env: + COMMIT_ID: ${{ inputs.commit_id || github.sha }} # use commit_id from the calling workflow + +jobs: + test_federated_evaluation_notebook: + name: WF Federated Evaluation Without TLS + if: | + (github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') || + (github.event_name == 'workflow_dispatch') || + (github.event.pull_request.draft == false) + runs-on: ubuntu-22.04 + timeout-minutes: 20 + env: + PYTHON_VERSION: '3.11' + steps: + - name: Checkout OpenFL repository + uses: actions/checkout@v4 + with: + ref: ${{ env.COMMIT_ID }} # use commit_id from the calling workflow + + - name: Pre test run + uses: ./.github/actions/wf_pre_test_run + if: ${{ always() }} + + - name: Run Federated Runtime 301 MNIST Watermarking via pytest + id: run_tests + run: | + python -m pytest -s tests/end_to_end/test_suites/wf_federated_runtime_tests.py -k test_federated_evaluation + echo "Federated Runtime 301 MNIST Watermarking test run completed" + + - name: Print test summary + id: print_test_summary + if: ${{ always() }} + run: | + export PYTHONPATH="$PYTHONPATH:." + python tests/end_to_end/utils/summary_helper.py --func_name "print_federated_runtime_score" --nb_name "wf_federated_evaluation" + echo "Test summary printed" + + - name: Tar files + if: ${{ always() }} # collect artifacts regardless of failures + run: | + tar -cvf notebook.tar --exclude="__pycache__" $HOME/results --ignore-failed-read + echo "TAR file created" + + - name: Upload Artifacts + uses: actions/upload-artifact@v4 + if: ${{ always() }} # collect artifacts regardless of failures + with: + name: federated_evaluation_${{ github.run_id }} + path: notebook.tar diff --git a/tests/end_to_end/test_suites/wf_federated_runtime_tests.py b/tests/end_to_end/test_suites/wf_federated_runtime_tests.py index 2e798ea0a4..ea15ab2836 100644 --- a/tests/end_to_end/test_suites/wf_federated_runtime_tests.py +++ b/tests/end_to_end/test_suites/wf_federated_runtime_tests.py @@ -12,6 +12,33 @@ log = logging.getLogger(__name__) +def activate_experimental_feature(workspace_path): + """ + Activate the experimental feature. + Args: + workspace_path (str): Path to the workspace + """ + # Activate the experimental feature + cmd = f"fx experimental activate" + error_msg = "Failed to activate the experimental feature" + return_code, output, error = fh.run_command( + cmd, + workspace_path=workspace_path, + error_msg=error_msg, + return_error=True, + ) + + if error: + # Check if the experimental feature is already activated + if [err for err in error if "No such command 'activate'" in err]: + log.info("Experimental feature already activated. Ignore the error.") + else: + log.error(f"{error_msg}: {error}") + raise Exception(error) + + log.info(f"Activated the experimental feature.") + + @pytest.mark.federated_runtime_301_watermarking def test_federated_runtime_301_watermarking(request): """ @@ -141,28 +168,65 @@ def test_federated_runtime_secure_aggregation(request): log.info("Experiment completed successfully") -def activate_experimental_feature(workspace_path): +def test_federated_evaluation(request): """ - Activate the experimental feature. + Test federated evaluation. Args: - workspace_path (str): Path to the workspace + request (Fixture): Pytest fixture """ + envoys = ["Bengaluru", "Portland"] + workspace_path = os.path.join( + os.getcwd(), + "openfl-tutorials/experimental/workflow/FederatedEvaluation", + ) # Activate the experimental feature - cmd = f"fx experimental activate" - error_msg = "Failed to activate the experimental feature" - return_code, output, error = fh.run_command( - cmd, - workspace_path=workspace_path, - error_msg=error_msg, - return_error=True, + activate_experimental_feature(workspace_path) + + # Create result log files for the director and envoys + result_path, participant_res_files = fh.create_federated_runtime_participant_res_files( + request.config.results_dir, envoys, model_name="wf_federated_evaluation" ) - if error: - # Check if the experimental feature is already activated - if [err for err in error if "No such command 'activate'" in err]: - log.info("Experimental feature already activated. Ignore the error.") - else: - log.error(f"{error_msg}: {error}") - raise Exception(error) + # Start the director + fh.start_director(workspace_path, participant_res_files["director"]) - log.info(f"Activated the experimental feature.") + # Start envoys Bangalore and Chandler and connect them to the director + executor = concurrent.futures.ThreadPoolExecutor() + results = [ + executor.submit( + fh.start_envoy, + envoy_name=envoy, + workspace_path=workspace_path, + res_file=participant_res_files[envoy.lower()], + ) + for envoy in envoys + ] + assert all([f.result() for f in results]), "Failed to start one or more envoys" + + # Based on the pattern, the envoys take time to connect to the director + # Hence, adding a sleep of 10 seconds anyways. + time.sleep(10) + nb_workspace_path = os.path.join(workspace_path, "workspace") + notebook_path = nb_workspace_path + "/" + "MNIST_FederatedEvaluation.ipynb" + + assert fh.check_envoys_director_conn_federated_runtime( + notebook_path=notebook_path, expected_envoys=envoys + ), "Envoys are not connected to the director" + + # IMP - Notebook MNIST_Federated_Evaluation.ipynb has hard coded notebook path set, hence changing the directory + # This might not be true for all notebooks, thus keeping it as a separate step + os.chdir(nb_workspace_path) + + assert fh.run_notebook( + notebook_path=notebook_path, + output_notebook_path=result_path + "/" + "MNIST_Federated_Evaluation_output.ipynb" + ), "Notebook run failed" + + # Change the directory back to the original directory + os.chdir(os.getcwd()) + + assert fh.verify_federated_runtime_experiment_completion( + participant_res_files , + expected_envoys=envoys + ), "Experiment failed" + log.info("Experiment completed successfully") \ No newline at end of file From 82f195dbbee5a6e2b92f88e94dd767094c04bdb5 Mon Sep 17 00:00:00 2001 From: "Chaurasiya, Payal" Date: Tue, 13 May 2025 07:22:11 -0700 Subject: [PATCH 2/6] Add workflow federated evaluation tests in pq Signed-off-by: Chaurasiya, Payal --- .github/workflows/wf_federated_evaluation.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wf_federated_evaluation.yml b/.github/workflows/wf_federated_evaluation.yml index b538fd3203..2f6676c546 100644 --- a/.github/workflows/wf_federated_evaluation.yml +++ b/.github/workflows/wf_federated_evaluation.yml @@ -2,16 +2,17 @@ # It runs the federated evaluation tests # -------------------------------------------------------- name: Federated Evaluation E2E + on: workflow_call: inputs: commit_id: required: false type: string - workflow_dispatch: - + workflow_dispatch: + permissions: - contents: read + contents: read env: COMMIT_ID: ${{ inputs.commit_id || github.sha }} # use commit_id from the calling workflow From 06c62796fe64fb93caf794eb9569b75c7f2e1608 Mon Sep 17 00:00:00 2001 From: "Chaurasiya, Payal" Date: Tue, 13 May 2025 07:22:48 -0700 Subject: [PATCH 3/6] Add workflow federated evaluation tests in pq Signed-off-by: Chaurasiya, Payal --- .github/workflows/wf_federated_evaluation.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wf_federated_evaluation.yml b/.github/workflows/wf_federated_evaluation.yml index 2f6676c546..b1a9863ec3 100644 --- a/.github/workflows/wf_federated_evaluation.yml +++ b/.github/workflows/wf_federated_evaluation.yml @@ -11,8 +11,8 @@ on: type: string workflow_dispatch: - permissions: - contents: read +permissions: + contents: read env: COMMIT_ID: ${{ inputs.commit_id || github.sha }} # use commit_id from the calling workflow From 083785d5ba4335da99bf63760cb05c5874270c3b Mon Sep 17 00:00:00 2001 From: "Chaurasiya, Payal" Date: Tue, 13 May 2025 07:24:08 -0700 Subject: [PATCH 4/6] Add workflow federated evaluation tests in pq Signed-off-by: Chaurasiya, Payal --- .github/workflows/wf_federated_evaluation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wf_federated_evaluation.yml b/.github/workflows/wf_federated_evaluation.yml index b1a9863ec3..aa03c3381b 100644 --- a/.github/workflows/wf_federated_evaluation.yml +++ b/.github/workflows/wf_federated_evaluation.yml @@ -9,7 +9,7 @@ on: commit_id: required: false type: string - workflow_dispatch: + workflow_dispatch: permissions: contents: read From bffce286e0ab7162c1fdec2a2e0ff5e039c95fa9 Mon Sep 17 00:00:00 2001 From: "Chaurasiya, Payal" Date: Wed, 14 May 2025 02:24:45 -0700 Subject: [PATCH 5/6] Add marker in pytest.ini Signed-off-by: Chaurasiya, Payal --- tests/end_to_end/pytest.ini | 2 ++ .../test_suites/wf_federated_runtime_tests.py | 3 +-- tests/end_to_end/utils/ssh_helper.py | 11 +++++++---- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/end_to_end/pytest.ini b/tests/end_to_end/pytest.ini index 2e8d4c9d69..6ce851218b 100644 --- a/tests/end_to_end/pytest.ini +++ b/tests/end_to_end/pytest.ini @@ -10,6 +10,8 @@ markers = task_runner_dockerized_ws: mark a test as a task runner dockerized workspace test. task_runner_basic_gandlf: mark a test as a task runner basic for GanDLF test. federated_runtime_301_watermarking: mark a test as a federated runtime 301 watermarking test. + federated_runtime_secure_aggregation: mark a test as a federated runtime secure aggregation test. + federated_runtime_federated_evaluation: mark a test as a federated runtime federated evaluation test. straggler_tests: mark a test as a straggler test. asyncio_mode=auto asyncio_default_fixture_loop_scope="function" diff --git a/tests/end_to_end/test_suites/wf_federated_runtime_tests.py b/tests/end_to_end/test_suites/wf_federated_runtime_tests.py index ea15ab2836..8b460ae616 100644 --- a/tests/end_to_end/test_suites/wf_federated_runtime_tests.py +++ b/tests/end_to_end/test_suites/wf_federated_runtime_tests.py @@ -226,7 +226,6 @@ def test_federated_evaluation(request): os.chdir(os.getcwd()) assert fh.verify_federated_runtime_experiment_completion( - participant_res_files , - expected_envoys=envoys + participant_res_files ), "Experiment failed" log.info("Experiment completed successfully") \ No newline at end of file diff --git a/tests/end_to_end/utils/ssh_helper.py b/tests/end_to_end/utils/ssh_helper.py index 3ccb97730f..2a393a10cc 100644 --- a/tests/end_to_end/utils/ssh_helper.py +++ b/tests/end_to_end/utils/ssh_helper.py @@ -104,11 +104,14 @@ def run_command( cmd, capture_output=True, shell=shell, text=True, cwd=work_dir, check=check, timeout=timeout ) except subprocess.CalledProcessError as e: - log.error(f"Command '{cmd}' failed with return code {e.returncode}") - log.error(f"Error output: {e.stderr}") - if not return_error: + if return_error: + log.warning(f"Command '{cmd}' failed with return code {e.returncode}") + log.warning(f"Output: {e.stderr}") + return e.returncode, [], [e.stderr] + else: + log.error(f"Command '{cmd}' failed with return code {e.returncode}") + log.error(f"Error output: {e.stderr}") raise - return e.returncode, [], [e.stderr] except Exception as e: log.error(f"Failed to execute command '{cmd}': {str(e)}") log.error(f"Error Traceback: {traceback.format_exc()}") From c95bdeb5c429a86787e0357e87c4b7ccd5a3b15f Mon Sep 17 00:00:00 2001 From: "Chaurasiya, Payal" Date: Thu, 15 May 2025 02:44:46 -0700 Subject: [PATCH 6/6] add details Signed-off-by: Chaurasiya, Payal --- tests/end_to_end/test_suites/wf_federated_runtime_tests.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/end_to_end/test_suites/wf_federated_runtime_tests.py b/tests/end_to_end/test_suites/wf_federated_runtime_tests.py index 8b460ae616..5c63aebcda 100644 --- a/tests/end_to_end/test_suites/wf_federated_runtime_tests.py +++ b/tests/end_to_end/test_suites/wf_federated_runtime_tests.py @@ -168,6 +168,7 @@ def test_federated_runtime_secure_aggregation(request): log.info("Experiment completed successfully") +@pytest.mark.federated_runtime_federated_evaluation def test_federated_evaluation(request): """ Test federated evaluation.