From 4fa1a4b6b36341bc86ce9594734303ba401e0710 Mon Sep 17 00:00:00 2001
From: Irtaza Akram <irtaza.akram@arbisoft.com>
Date: Tue, 25 Jun 2024 16:46:10 +0500
Subject: [PATCH] fix: github action artifact upload issue

---
 .github/actions/unit-tests/action.yml         |  35 ------
 .github/actions/verify-tests-count/action.yml |  47 --------
 .github/dependabot.yml                        |  12 --
 .github/workflows/unit-tests.yml              | 105 ++++++++++++++++--
 4 files changed, 97 insertions(+), 102 deletions(-)
 delete mode 100644 .github/actions/unit-tests/action.yml
 delete mode 100644 .github/actions/verify-tests-count/action.yml

diff --git a/.github/actions/unit-tests/action.yml b/.github/actions/unit-tests/action.yml
deleted file mode 100644
index 7d217cd26f8c..000000000000
--- a/.github/actions/unit-tests/action.yml
+++ /dev/null
@@ -1,35 +0,0 @@
-name: "Run unit tests"
-description: "shared steps to run unit tests on both Github hosted and self hosted runners."
-runs:
-  using: "composite"
-  steps:
-    - name: set settings path
-      shell: bash
-      run: |
-        echo "settings_path=$(python scripts/unit_test_shards_parser.py --shard-name=${{ matrix.shard_name }} --output settings )" >> $GITHUB_ENV
-
-    - name: get unit tests for shard
-      shell: bash
-      run: |
-        echo "unit_test_paths=$(python scripts/unit_test_shards_parser.py --shard-name=${{ matrix.shard_name }} )" >> $GITHUB_ENV
-
-    - name: run tests
-      shell: bash
-      run: |
-        python -Wd -m pytest -p no:randomly --ds=${{ env.settings_path }} ${{ env.unit_test_paths }} --cov=.
-
-    - name: rename warnings json file
-      if: success()
-      shell: bash
-      run: |
-        cd test_root/log
-        mv pytest_warnings.json pytest_warnings_${{ matrix.shard_name }}.json
-
-    - name: save pytest warnings json file
-      if: success()
-      uses: actions/upload-artifact@v4
-      with:
-        name: pytest-warnings-json
-        path: |
-          test_root/log/pytest_warnings*.json
-        overwrite: true
diff --git a/.github/actions/verify-tests-count/action.yml b/.github/actions/verify-tests-count/action.yml
deleted file mode 100644
index d299f1ca11c5..000000000000
--- a/.github/actions/verify-tests-count/action.yml
+++ /dev/null
@@ -1,47 +0,0 @@
-name: "Verify unit tests count"
-description: "shared steps to verify unit tests count on both Github hosted and self hosted runners."
-runs:
-  using: "composite"
-  steps:
-    - name: collect tests from all modules
-      shell: bash
-      run: |
-        echo "root_cms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=cms.envs.test cms/ -q | head -n -2 | wc -l)" >> $GITHUB_ENV
-        echo "root_lms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=lms.envs.test lms/ openedx/ common/djangoapps/ xmodule/ pavelib/ -q | head -n -2 | wc -l)" >> $GITHUB_ENV
-
-    - name: get GHA unit test paths
-      shell: bash
-      run: |
-        echo "cms_unit_test_paths=$(python scripts/gha_unit_tests_collector.py --cms-only)" >> $GITHUB_ENV
-        echo "lms_unit_test_paths=$(python scripts/gha_unit_tests_collector.py --lms-only)" >> $GITHUB_ENV
-
-    - name: collect tests from GHA unit test shards
-      shell: bash
-      run: |
-        echo "cms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=cms.envs.test ${{ env.cms_unit_test_paths }} -q | head -n -2 | wc -l)" >> $GITHUB_ENV
-        echo "lms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=lms.envs.test ${{ env.lms_unit_test_paths }} -q | head -n -2 | wc -l)" >> $GITHUB_ENV
-
-    - name: add unit tests count
-      shell: bash
-      run: |
-        echo "root_all_unit_tests_count=$((${{ env.root_cms_unit_tests_count }}+${{ env.root_lms_unit_tests_count }}))" >> $GITHUB_ENV
-        echo "shards_all_unit_tests_count=$((${{ env.cms_unit_tests_count }}+${{ env.lms_unit_tests_count }}))" >> $GITHUB_ENV
-
-    - name: print unit tests count
-      shell: bash
-      run: |
-        echo CMS unit tests from root: ${{ env.root_cms_unit_tests_count }}
-        echo LMS unit tests from root: ${{ env.root_lms_unit_tests_count }}
-        echo CMS unit tests from shards: ${{ env.cms_unit_tests_count }}
-        echo LMS unit tests from shards: ${{ env.lms_unit_tests_count }}
-        echo All root unit tests count: ${{ env.root_all_unit_tests_count }}
-        echo All shards unit tests count: ${{ env.shards_all_unit_tests_count }}
-
-    - name: fail the check
-      shell: bash
-      if: ${{ env.root_all_unit_tests_count != env.shards_all_unit_tests_count }}
-      run: |
-        echo "::error title='Unit test modules in unit-test-shards.json (unit-tests.yml workflow) are outdated'::unit tests running in unit-tests
-        workflow don't match the count for unit tests for entire edx-platform suite, please update the unit-test-shards.json under .github/workflows
-        to add any missing apps and match the count. for more details please take a look at scripts/gha-shards-readme.md"
-        exit 1
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
index 5345413e561c..d0fde72ac11d 100644
--- a/.github/dependabot.yml
+++ b/.github/dependabot.yml
@@ -7,15 +7,3 @@ updates:
         interval: "weekly"
     reviewers:
       - "openedx/arbi-bom"
-  - package-ecosystem: "github-actions"
-    directory: "/.github/actions/unit-tests/"
-    schedule:
-        interval: "weekly"
-    reviewers:
-      - "openedx/arbi-bom"
-  - package-ecosystem: "github-actions"
-    directory: "/.github/actions/verify-tests-count/"
-    schedule:
-        interval: "weekly"
-    reviewers:
-      - "openedx/arbi-bom"
diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
index c6513baa0fbb..3e442b75d4e7 100644
--- a/.github/workflows/unit-tests.yml
+++ b/.github/workflows/unit-tests.yml
@@ -99,8 +99,36 @@ jobs:
         run: |
           pip freeze
 
-      - name: Setup and run tests
-        uses: ./.github/actions/unit-tests
+      - name: set settings path
+        shell: bash
+        run: |
+          echo "settings_path=$(python scripts/unit_test_shards_parser.py --shard-name=${{ matrix.shard_name }} --output settings )" >> $GITHUB_ENV
+
+      - name: get unit tests for shard
+        shell: bash
+        run: |
+          echo "unit_test_paths=$(python scripts/unit_test_shards_parser.py --shard-name=${{ matrix.shard_name }} )" >> $GITHUB_ENV
+
+      - name: run tests
+        shell: bash
+        run: |
+          python -Wd -m pytest -p no:randomly --ds=${{ env.settings_path }} ${{ env.unit_test_paths }} --cov=.
+
+      - name: rename warnings json file
+        if: success()
+        shell: bash
+        run: |
+          cd test_root/log
+          mv pytest_warnings.json pytest_warnings_${{ matrix.shard_name }}.json
+
+      - name: save pytest warnings json file
+        if: success()
+        uses: actions/upload-artifact@v4
+        with:
+          name: pytest-warnings-json-${{ matrix.shard_name }}
+          path: |
+            test_root/log/pytest_warnings*.json
+          overwrite: true
 
       - name: Renaming coverage data file
         run: |
@@ -109,8 +137,8 @@ jobs:
       - name: Upload coverage
         uses: actions/upload-artifact@v4
         with:
-          name: coverage
-          path: reports/${{matrix.shard_name}}.coverage
+          name: coverage-${{ matrix.shard_name }}
+          path: reports/${{ matrix.shard_name }}.coverage
           overwrite: true
 
   collect-and-verify:
@@ -130,8 +158,49 @@ jobs:
         run: |
           make test-requirements
 
-      - name: verify unit tests count
-        uses: ./.github/actions/verify-tests-count
+      - name: collect tests from all modules
+        shell: bash
+        run: |
+          echo "root_cms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=cms.envs.test cms/ -q | head -n -2 | wc -l)" >> $GITHUB_ENV
+          echo "root_lms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=lms.envs.test lms/ openedx/ common/djangoapps/ xmodule/ pavelib/ -q | head -n -2 | wc -l)" >> $GITHUB_ENV
+
+      - name: get GHA unit test paths
+        shell: bash
+        run: |
+          echo "cms_unit_test_paths=$(python scripts/gha_unit_tests_collector.py --cms-only)" >> $GITHUB_ENV
+          echo "lms_unit_test_paths=$(python scripts/gha_unit_tests_collector.py --lms-only)" >> $GITHUB_ENV
+
+      - name: collect tests from GHA unit test shards
+        shell: bash
+        run: |
+          echo "cms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=cms.envs.test ${{ env.cms_unit_test_paths }} -q | head -n -2 | wc -l)" >> $GITHUB_ENV
+          echo "lms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=lms.envs.test ${{ env.lms_unit_test_paths }} -q | head -n -2 | wc -l)" >> $GITHUB_ENV
+
+      - name: add unit tests count
+        shell: bash
+        run: |
+          echo "root_all_unit_tests_count=$((${{ env.root_cms_unit_tests_count }}+${{ env.root_lms_unit_tests_count }}))" >> $GITHUB_ENV
+          echo "shards_all_unit_tests_count=$((${{ env.cms_unit_tests_count }}+${{ env.lms_unit_tests_count }}))" >> $GITHUB_ENV
+
+      - name: print unit tests count
+        shell: bash
+        run: |
+          echo CMS unit tests from root: ${{ env.root_cms_unit_tests_count }}
+          echo LMS unit tests from root: ${{ env.root_lms_unit_tests_count }}
+          echo CMS unit tests from shards: ${{ env.cms_unit_tests_count }}
+          echo LMS unit tests from shards: ${{ env.lms_unit_tests_count }}
+          echo All root unit tests count: ${{ env.root_all_unit_tests_count }}
+          echo All shards unit tests count: ${{ env.shards_all_unit_tests_count }}
+
+      - name: fail the check
+        shell: bash
+        if: ${{ env.root_all_unit_tests_count != env.shards_all_unit_tests_count }}
+        run: |
+          echo "::error title='Unit test modules in unit-test-shards.json (unit-tests.yml workflow) are outdated'::unit tests running in unit-tests
+          workflow don't match the count for unit tests for entire edx-platform suite, please update the unit-test-shards.json under .github/workflows
+          to add any missing apps and match the count. for more details please take a look at scripts/gha-shards-readme.md"
+          exit 1
+
 
   # This job aggregates test results. It's the required check for branch protection.
   # https://github.com/marketplace/actions/alls-green#why
@@ -156,7 +225,8 @@ jobs:
       - name: collect pytest warnings files
         uses: actions/download-artifact@v4
         with:
-          name: pytest-warnings-json
+          pattern: pytest-warnings-json-*
+          merge-multiple: true
           path: test_root/log
 
       - name: display structure of downloaded files
@@ -175,6 +245,24 @@ jobs:
             reports/pytest_warnings/warning_report_all.html
           overwrite: true
 
+  merge-artifacts:
+    runs-on: ubuntu-20.04
+    needs: [compile-warnings-report]
+    steps:
+      - name: Merge Pytest Warnings JSON Artifacts
+        uses: actions/upload-artifact/merge@v4
+        with:
+          name: pytest-warnings-json
+          pattern: pytest-warnings-json-*
+          delete-merged: true
+
+      - name: Merge Coverage Artifacts
+        uses: actions/upload-artifact/merge@v4
+        with:
+          name: coverage
+          pattern: coverage-*
+          delete-merged: true
+
   # Combine and upload coverage reports.
   coverage:
     if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false))
@@ -196,7 +284,8 @@ jobs:
       - name: Download all artifacts
         uses: actions/download-artifact@v4
         with:
-          name: coverage
+          pattern: coverage-*
+          merge-multiple: true
           path: reports
 
       - name: Install Python dependencies