TaskRunner E2E Workflow: corrections and enhancements (#1161)

* TaskRunner E2E Workflow: corrections and enhancements Signed-off-by: noopur <[email protected]> * Comment the fork skip part for testing Signed-off-by: noopur <[email protected]> * Let wf run for all repos Signed-off-by: noopur <[email protected]> * Remove the redundant comments Signed-off-by: noopur <[email protected]> --------- Signed-off-by: noopur <[email protected]>
securefederatedai · Nov 20, 2024 · 4ea9eba · 4ea9eba
1 parent 3c983ef
commit 4ea9eba
Show file tree

Hide file tree

Showing 4 changed files with 42 additions and 33 deletions.
diff --git a/.github/workflows/task_runner_e2e.yml b/.github/workflows/task_runner_e2e.yml
@@ -30,7 +30,7 @@ env:
   NUM_COLLABORATORS: ${{ inputs.num_collaborators || '2' }}
 
 jobs:
-  test:
+  test_with_tls:
     name: tr_tls
     runs-on: ubuntu-22.04
     timeout-minutes: 120 # 2 hours
@@ -71,28 +71,30 @@ jobs:
       - name: Run Task Runner E2E tests with TLS
         id: run_tests
         run: |
-          python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -m ${{ env.MODEL_NAME }} --num_rounds $NUM_ROUNDS --num_collaborators $NUM_COLLABORATORS --model_name ${{ env.MODEL_NAME }}
+          python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \
+          -m ${{ env.MODEL_NAME }} --model_name ${{ env.MODEL_NAME }} \
+          --num_rounds $NUM_ROUNDS --num_collaborators $NUM_COLLABORATORS 
           echo "Task runner end to end test run completed"
 
-      - name: Print test summary # Print the test summary only if the tests were run
+      - name: Print test summary
         id: print_test_summary
-        if: steps.run_tests.outcome == 'success' || steps.run_tests.outcome == 'failure'
+        if: ${{ always() }}
         run: |
           export PYTHONPATH="$PYTHONPATH:."
           python tests/end_to_end/utils/summary_helper.py
           echo "Test summary printed"
 
-      - name: Tar files # Tar the test results only if the tests were run
+      - name: Tar files
         id: tar_files
-        if: steps.run_tests.outcome == 'success' || steps.run_tests.outcome == 'failure'
+        if: ${{ always() }}
         run: tar -cvf result.tar results
 
-      - name: Upload Artifacts # Upload the test results only if the tar was created
+      - name: Upload Artifacts
         id: upload_artifacts
         uses: actions/upload-artifact@v4
-        if: steps.tar_files.outcome == 'success'
+        if: ${{ always() }}
         with:
-          name: task_runner_tls_${{ env.MODEL_NAME }}_python${{ env.PYTHON_VERSION }}_${{ github.run_id }}
+          name: tr_tls_${{ env.MODEL_NAME }}_python${{ env.PYTHON_VERSION }}_${{ github.run_id }}
           path: result.tar
 
   test_with_non_tls:
@@ -136,26 +138,28 @@ jobs:
       - name: Run Task Runner E2E tests without TLS
         id: run_tests
         run: |
-          python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py -m ${{ env.MODEL_NAME }} --num_rounds $NUM_ROUNDS --num_collaborators $NUM_COLLABORATORS --disable_tls
+          python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \
+          -m ${{ env.MODEL_NAME }} --model_name ${{ env.MODEL_NAME }} \
+          --num_rounds $NUM_ROUNDS --num_collaborators $NUM_COLLABORATORS --disable_tls
           echo "Task runner end to end test run completed"
 
-      - name: Print test summary # Print the test summary only if the tests were run
+      - name: Print test summary
         id: print_test_summary
-        if: steps.run_tests.outcome == 'success' || steps.run_tests.outcome == 'failure'
+        if: ${{ always() }}
         run: |
           export PYTHONPATH="$PYTHONPATH:."
           python tests/end_to_end/utils/summary_helper.py
           echo "Test summary printed"
 
-      - name: Tar files # Tar the test results only if the tests were run
+      - name: Tar files
         id: tar_files
-        if: steps.run_tests.outcome == 'success' || steps.run_tests.outcome == 'failure'
+        if: ${{ always() }}
         run: tar -cvf result.tar results
 
-      - name: Upload Artifacts # Upload the test results only if the tar was created
+      - name: Upload Artifacts
         id: upload_artifacts
         uses: actions/upload-artifact@v4
-        if: steps.tar_files.outcome == 'success'
+        if: ${{ always() }}
         with:
-          name: task_runner_non_tls_${{ env.MODEL_NAME }}_python${{ env.PYTHON_VERSION }}_${{ github.run_id }}
+          name: tr_non_tls_${{ env.MODEL_NAME }}_python${{ env.PYTHON_VERSION }}_${{ github.run_id }}
           path: result.tar
diff --git a/tests/end_to_end/pytest.ini b/tests/end_to_end/pytest.ini
@@ -1,5 +1,5 @@
 [pytest]
-addopts = -ra -q -s --junitxml=results/results.xml
+addopts = -ra -q -s --junitxml=results/results.xml -x
 testpaths = test_suites
 junit_family = xunit2
 results_dir = results

diff --git a/tests/end_to_end/utils/federation_helper.py b/tests/end_to_end/utils/federation_helper.py
@@ -26,7 +26,7 @@ def setup_pki(fed_obj):
         fed_obj.aggregator.generate_sign_request()
         fed_obj.model_owner.certify_aggregator(fed_obj.aggregator.agg_domain_name)
     except Exception as e:
-        log.error(f"Failed to perform aggregator operations: {e}")
+        log.error(f"Failed to perform PKI setup for aggregator: {e}")
         raise e
 
     # Collaborator and model owner operations
@@ -38,11 +38,11 @@ def setup_pki(fed_obj):
             fed_obj.model_owner.certify_collaborator(collaborator.collaborator_name)
             collaborator.import_pki()
         except Exception as e:
-            log.error(f"Failed to perform collaborator operations: {e}")
+            log.error(f"Failed to perform PKI setup for {collaborator.collaborator_name}: {e}")
             raise e
     success = True
 
-    log.info("CSR operations completed successfully for all participants")
+    log.info("PKI setup successfully for all participants")
     return success
 
 
@@ -93,7 +93,7 @@ def verify_federation_run_completion(fed_obj, results):
     # Result will contain a list of boolean values for all the participants.
     # True - successful completion, False - failed/incomplete
     results = [f.result() for f in futures]
-    log.info(f"Results: {results}")
+    log.info(f"Results from all the participants: {results}")
 
     # If any of the participant failed, return False, else return True
     return all(results)

diff --git a/tests/end_to_end/utils/summary_helper.py b/tests/end_to_end/utils/summary_helper.py
@@ -21,9 +21,10 @@ def get_aggregated_accuracy(agg_log_file):
     Returns:
         agg_accuracy: the aggregated accuracy
     """
+    agg_accuracy = "Not Found"
     if not os.path.exists(agg_log_file):
         print(f"Aggregator log file {agg_log_file} not found. Cannot get aggregated accuracy")
-        return "Not Found"
+        return agg_accuracy
 
     # Example line(s) containing spaces and special characters:
     """
@@ -33,17 +34,17 @@ def get_aggregated_accuracy(agg_log_file):
     try:
         with open(agg_log_file, 'r') as f:
             for line in f:
-                if "metric_origin" in line and "aggregator" in line and "aggregated_model_validation" in line:
+                if "'metric_origin': 'aggregator'" in line and "aggregated_model_validation" in line:
                     line = line.split("aggregator.py:")[0].strip()
                     # If the line does not contain closing bracket "}", then concatenate the next line
                     reqd_line = line if "}" in line else line + next(f).strip()
                     agg_accuracy = eval(reqd_line.split("METRIC")[1].strip('"'))["metric_value"]
-        return agg_accuracy
-
+                    break
     except Exception as e:
         # Do not fail the test if the accuracy cannot be fetched
         print(f"Error while reading aggregator log file: {e}")
-        return "Not Found"
+
+    return agg_accuracy
 
 
 def get_test_status(result):
@@ -54,16 +55,17 @@ def get_test_status(result):
     Returns
         status of the test status
     """
-    status = "FAILED"
+    status, err_msg = "FAILED", "NA"
     if "failure" in result.tag or "error" in result.tag:
         # If the result has a tag "failure", set status as "FAIL"
         status = "FAILED"
+        err_msg = result.get("message").split("\n")[0]
     elif "skipped" in result.tag:
         # If the result has a tag "skipped", set status as "SKIPPED"
         status = "SKIPPED"
     else:
         status = "PASSED"
-    return status
+    return status, err_msg
 
 
 def get_testcase_result():
@@ -84,11 +86,13 @@ def get_testcase_result():
                 # Successful test won't have any result/subtag
                 if len(testcase) == 0:
                     database_dict["result"] = "PASSED"
+                    database_dict["err_msg"] = "NA"
 
                 # Iterate over each result in testsuite
                 for result in testcase:
-                    status = get_test_status(result)
+                    status, err_msg = get_test_status(result)
                     database_dict["result"] = status
+                    database_dict["err_msg"] = err_msg
 
                 # Append the dictionary to database_list
                 database_list.append(database_dict)
@@ -110,6 +114,7 @@ def get_testcase_result():
 
     if not model_name:
         print("MODEL_NAME is not set, cannot find out aggregator logs")
+        agg_accuracy = "Not Found"
     else:
         workspace_name = "workspace_" + model_name
         agg_log_file = os.path.join("results", workspace_name, "aggregator.log")
@@ -118,7 +123,7 @@ def get_testcase_result():
     # Write the results to GitHub step summary
     with open(os.getenv('GITHUB_STEP_SUMMARY'), 'a') as fh:
         # DO NOT change the print statements
-        print("| Name | Time (in seconds) | Result | Collaborators | Rounds to train | Score (if applicable) |", file=fh)
-        print("| ------------- | ------------- | ------------- | ------------- | ------------- | ------------- |", file=fh)
+        print("| Name | Time (in seconds) | Result | Error (if any) | Collaborators | Rounds to train | Score (if applicable) |", file=fh)
+        print("| ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | ------------- |", file=fh)
         for item in result:
-            print(f"| {item['name']} | {item['time']} | {item['result']} | {num_cols} | {num_rounds} | {agg_accuracy} |", file=fh)
+            print(f"| {item['name']} | {item['time']} | {item['result']} | {item['err_msg']} | {num_cols} | {num_rounds} | {agg_accuracy} |", file=fh)