From e428dd451168654a7e8e7d88d6e6ce68045e9f4b Mon Sep 17 00:00:00 2001
From: Irtaza Akram <irtaza.akram@arbisoft.com>
Date: Wed, 22 May 2024 10:30:33 +0500
Subject: [PATCH 01/10] fix: update github actions & remove edx runner

---
 .github/actions/unit-tests/action.yml         |  3 +-
 .../check-consistent-dependencies.yml         |  4 +-
 .github/workflows/check-for-tutorial-prs.yml  |  2 +-
 .github/workflows/ci-static-analysis.yml      |  6 +--
 .../workflows/compile-python-requirements.yml |  6 +--
 .github/workflows/docker-publish.yml          |  6 +--
 .github/workflows/js-tests.yml                |  9 ++--
 .github/workflows/lint-imports.yml            |  6 +--
 .github/workflows/migrations-check.yml        |  6 +--
 .github/workflows/publish-ci-docker-image.yml |  2 +-
 .github/workflows/pylint-checks.yml           |  6 +--
 .github/workflows/quality-checks.yml          |  9 ++--
 .github/workflows/semgrep.yml                 |  4 +-
 .github/workflows/static-assets-check.yml     |  6 +--
 .github/workflows/unit-tests-gh-hosted.yml    | 12 ++---
 .github/workflows/unit-tests.yml              | 47 +++++++++----------
 .../units-test-scripts-structures-pruning.yml |  2 +-
 .../units-test-scripts-user-retirement.yml    |  2 +-
 .../upgrade-one-python-dependency.yml         |  4 +-
 .github/workflows/verify-dunder-init.yml      |  2 +-
 .../workflows/verify-gha-unit-tests-count.yml | 15 +++---
 21 files changed, 81 insertions(+), 78 deletions(-)

diff --git a/.github/actions/unit-tests/action.yml b/.github/actions/unit-tests/action.yml
index 2a5429baff09..eb7fee0a6ba6 100644
--- a/.github/actions/unit-tests/action.yml
+++ b/.github/actions/unit-tests/action.yml
@@ -27,8 +27,9 @@ runs:
 
     - name: save pytest warnings json file
       if: success()
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: pytest-warnings-json
         path: |
           test_root/log/pytest_warnings*.json
+        overwrite: true
diff --git a/.github/workflows/check-consistent-dependencies.yml b/.github/workflows/check-consistent-dependencies.yml
index 51a4d5f24fe2..1c57fc6f6b9f 100644
--- a/.github/workflows/check-consistent-dependencies.yml
+++ b/.github/workflows/check-consistent-dependencies.yml
@@ -34,10 +34,10 @@ jobs:
               echo "RELEVANT=true" >> "$GITHUB_ENV"
           fi
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         if: ${{ env.RELEVANT == 'true' }}
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         if: ${{ env.RELEVANT == 'true' }}
         with:
           python-version: '3.8'
diff --git a/.github/workflows/check-for-tutorial-prs.yml b/.github/workflows/check-for-tutorial-prs.yml
index 5d64a50573fd..da7995e0c4d3 100644
--- a/.github/workflows/check-for-tutorial-prs.yml
+++ b/.github/workflows/check-for-tutorial-prs.yml
@@ -23,7 +23,7 @@ jobs:
     name: provide helpful bot comment
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Comment PR
         uses: thollander/actions-comment-pull-request@v2
diff --git a/.github/workflows/ci-static-analysis.yml b/.github/workflows/ci-static-analysis.yml
index 2e40e09e0aae..e1c663c08372 100644
--- a/.github/workflows/ci-static-analysis.yml
+++ b/.github/workflows/ci-static-analysis.yml
@@ -13,9 +13,9 @@ jobs:
         os: ['ubuntu-20.04']
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -31,7 +31,7 @@ jobs:
 
       - name: Cache pip dependencies
         id: cache-dependencies
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ${{ steps.pip-cache-dir.outputs.dir }}
           key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }}
diff --git a/.github/workflows/compile-python-requirements.yml b/.github/workflows/compile-python-requirements.yml
index f8f945545d96..85d0eb0882ea 100644
--- a/.github/workflows/compile-python-requirements.yml
+++ b/.github/workflows/compile-python-requirements.yml
@@ -19,12 +19,12 @@ jobs:
 
     steps:
       - name: Check out target branch
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: "${{ inputs.branch }}"
 
       - name: Set up Python environment
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.11"
 
@@ -44,7 +44,7 @@ jobs:
 
       - name: Make a PR
         id: make-pr
-        uses: peter-evans/create-pull-request@v5
+        uses: peter-evans/create-pull-request@v6
         with:
           branch: "${{ github.triggering_actor }}/compile-python-deps"
           branch-suffix: short-commit-hash
diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml
index 978e616ee62a..b0af217e6e70 100644
--- a/.github/workflows/docker-publish.yml
+++ b/.github/workflows/docker-publish.yml
@@ -21,13 +21,13 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
 
       - name: Login to DockerHub
         uses: docker/login-action@v3
diff --git a/.github/workflows/js-tests.yml b/.github/workflows/js-tests.yml
index 8b7c137c8b23..37d825d9ab87 100644
--- a/.github/workflows/js-tests.yml
+++ b/.github/workflows/js-tests.yml
@@ -19,7 +19,7 @@ jobs:
 
     steps:
 
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Fetch master to compare coverage
       run: git fetch --depth=1 origin master
 
@@ -43,7 +43,7 @@ jobs:
       run: sudo apt-get update && sudo apt-get install libxmlsec1-dev ubuntu-restricted-extras xvfb
 
     - name: Setup Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
@@ -54,7 +54,7 @@ jobs:
 
     - name: Cache pip dependencies
       id: cache-dependencies
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ${{ steps.pip-cache-dir.outputs.dir }}
         key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/base.txt') }}
@@ -74,7 +74,7 @@ jobs:
         xvfb-run --auto-servernum ./scripts/all-tests.sh
 
     - name: Save Job Artifacts
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: Build-Artifacts
         path: |
@@ -82,3 +82,4 @@ jobs:
           test_root/log/*.png
           test_root/log/*.log
           **/TEST-*.xml
+        overwrite: true
diff --git a/.github/workflows/lint-imports.yml b/.github/workflows/lint-imports.yml
index 243b44d2039d..f9ae712cfd78 100644
--- a/.github/workflows/lint-imports.yml
+++ b/.github/workflows/lint-imports.yml
@@ -14,10 +14,10 @@ jobs:
 
     steps:
       - name: Check out branch
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.11'
 
@@ -33,7 +33,7 @@ jobs:
 
       - name: Cache pip dependencies
         id: cache-dependencies
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ${{ steps.pip-cache-dir.outputs.dir }}
           key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }}
diff --git a/.github/workflows/migrations-check.yml b/.github/workflows/migrations-check.yml
index 7375583e0400..75d9fde9fda0 100644
--- a/.github/workflows/migrations-check.yml
+++ b/.github/workflows/migrations-check.yml
@@ -70,10 +70,10 @@ jobs:
         mongosh --host 127.0.0.1 --username edxapp --password password --eval 'use edxapp; db.adminCommand("ping");' edxapp
 
     - name: Checkout repo
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
     - name: Setup Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
@@ -89,7 +89,7 @@ jobs:
 
     - name: Cache pip dependencies
       id: cache-dependencies
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ${{ steps.pip-cache-dir.outputs.dir }}
         key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }}
diff --git a/.github/workflows/publish-ci-docker-image.yml b/.github/workflows/publish-ci-docker-image.yml
index 20b238c2380d..d97b205cfa94 100644
--- a/.github/workflows/publish-ci-docker-image.yml
+++ b/.github/workflows/publish-ci-docker-image.yml
@@ -11,7 +11,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       # This has to happen after checkout in order for gh to work.
       - name: "Cancel scheduled job on forks"
diff --git a/.github/workflows/pylint-checks.yml b/.github/workflows/pylint-checks.yml
index 3e3c87568cb4..90e457bf0b35 100644
--- a/.github/workflows/pylint-checks.yml
+++ b/.github/workflows/pylint-checks.yml
@@ -31,13 +31,13 @@ jobs:
     name: pylint ${{ matrix.module-name }}
     steps:
       - name: Check out repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Install required system packages
         run: sudo apt-get update && sudo apt-get install libxmlsec1-dev
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: 3.11
 
@@ -48,7 +48,7 @@ jobs:
 
       - name: Cache pip dependencies
         id: cache-dependencies
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ${{ steps.pip-cache-dir.outputs.dir }}
           key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }}
diff --git a/.github/workflows/quality-checks.yml b/.github/workflows/quality-checks.yml
index 6bf16c9f0b2f..d2264297a0ec 100644
--- a/.github/workflows/quality-checks.yml
+++ b/.github/workflows/quality-checks.yml
@@ -20,7 +20,7 @@ jobs:
 
     steps:
 
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 2
 
@@ -31,7 +31,7 @@ jobs:
       run: sudo apt-get update && sudo apt-get install libxmlsec1-dev
 
     - name: Setup Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
@@ -50,7 +50,7 @@ jobs:
 
     - name: Cache pip dependencies
       id: cache-dependencies
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ${{ steps.pip-cache-dir.outputs.dir }}
         key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/testing.txt') }}
@@ -73,10 +73,11 @@ jobs:
 
     - name: Save Job Artifacts
       if: always()
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: Build-Artifacts
         path: |
           **/reports/**/*
           test_root/log/**/*.log
           *.log
+        overwrite: true
diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml
index ec6b99e7967c..4f01f1112277 100644
--- a/.github/workflows/semgrep.yml
+++ b/.github/workflows/semgrep.yml
@@ -22,11 +22,11 @@ jobs:
           - '3.11'
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 1
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: "${{ matrix.python-version }}"
 
diff --git a/.github/workflows/static-assets-check.yml b/.github/workflows/static-assets-check.yml
index e6837101aad0..09307145be3b 100644
--- a/.github/workflows/static-assets-check.yml
+++ b/.github/workflows/static-assets-check.yml
@@ -35,10 +35,10 @@ jobs:
 
     steps:
     - name: Checkout repo
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
     - name: Setup Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
@@ -62,7 +62,7 @@ jobs:
 
     - name: Cache pip dependencies
       id: cache-dependencies
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ${{ steps.pip-cache-dir.outputs.dir }}
         key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }}
diff --git a/.github/workflows/unit-tests-gh-hosted.yml b/.github/workflows/unit-tests-gh-hosted.yml
index abe49d7d11f7..7fe7c8961b61 100644
--- a/.github/workflows/unit-tests-gh-hosted.yml
+++ b/.github/workflows/unit-tests-gh-hosted.yml
@@ -56,7 +56,7 @@ jobs:
         #     django-version: "pinned"
         #     mongo-version: "4.4"
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Install Required System Packages
         run: sudo apt-get update && sudo apt-get install libxmlsec1-dev lynx
@@ -67,7 +67,7 @@ jobs:
           mongodb-version: ${{ matrix.mongo-version}}
 
       - name: Setup Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -78,7 +78,7 @@ jobs:
 
       - name: Cache pip dependencies
         id: cache-dependencies
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ${{ steps.pip-cache-dir.outputs.dir }}
           key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/testing.txt') }}
@@ -106,13 +106,13 @@ jobs:
         django-version:
           - "pinned"
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Install Required System Packages
         run: sudo apt-get update && sudo apt-get install libxmlsec1-dev
 
       - name: Setup Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -123,7 +123,7 @@ jobs:
 
       - name: Cache pip dependencies
         id: cache-dependencies
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ${{ steps.pip-cache-dir.outputs.dir }}
           key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/testing.txt') }}
diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
index b65e0bcd0b85..6ccf5ae03c88 100644
--- a/.github/workflows/unit-tests.yml
+++ b/.github/workflows/unit-tests.yml
@@ -10,7 +10,7 @@ jobs:
   run-tests:
     name: ${{ matrix.shard_name }}(py=${{ matrix.python-version }},dj=${{ matrix.django-version }},mongo=${{ matrix.mongo-version }})
     if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false))
-    runs-on: [ edx-platform-runner ]
+    runs-on: [ "ubuntu-20.04" ]
     strategy:
       matrix:
         python-version:
@@ -56,14 +56,16 @@ jobs:
         #     mongo-version: "4.4"
 
     steps:
-      - name: sync directory owner
-        run: sudo chown runner:runner -R .*
+      - name: checkout repo
+        uses: actions/checkout@v4
+
+      - name: install system requirements
+        run: |
+          make ubuntu-requirements
 
       - name: install mongo version
         run: |
           if [[ "${{ matrix.mongo-version }}" != "4.4" ]]; then
-            sudo apt-get purge -y "mongodb-org*"
-            sudo apt-get remove -y mongodb-org
             wget -qO - https://www.mongodb.org/static/pgp/server-${{ matrix.mongo-version }}.asc | sudo apt-key add -
             echo "deb https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/${{ matrix.mongo-version }} multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-${{ matrix.mongo-version }}.list
             sudo apt-get update && sudo apt-get install -y mongodb-org="${{ matrix.mongo-version }}.*"
@@ -76,24 +78,21 @@ jobs:
           mongod &
 
       - name: Setup Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: checkout repo
-        uses: actions/checkout@v3
-
       - name: install requirements
         run: |
-          sudo make test-requirements
+          sudo env "PATH=$PATH" make test-requirements
           if [[ "${{ matrix.django-version }}" != "pinned" ]]; then
-            sudo pip install "django~=${{ matrix.django-version }}.0"
-            sudo pip check # fail if this test-reqs/Django combination is broken
+            sudo env "PATH=$PATH" pip install "django~=${{ matrix.django-version }}.0"
+            sudo env "PATH=$PATH" pip check # fail if this test-reqs/Django combination is broken
           fi
 
       - name: list installed package versions
         run: |
-          sudo pip freeze
+          sudo env "PATH=$PATH" pip freeze
 
       - name: Setup and run tests
         uses: ./.github/actions/unit-tests
@@ -103,10 +102,11 @@ jobs:
           mv reports/.coverage reports/${{ matrix.shard_name }}.coverage
 
       - name: Upload coverage
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: coverage
           path: reports/${{matrix.shard_name}}.coverage
+          overwrite: true
 
   # This job aggregates test results. It's the required check for branch protection.
   # https://github.com/marketplace/actions/alls-green#why
@@ -125,14 +125,12 @@ jobs:
           jobs: ${{ toJSON(needs) }}
 
   compile-warnings-report:
-    runs-on: [ edx-platform-runner ]
+    runs-on: [ "ubuntu-20.04" ]
     needs: [ run-tests ]
     steps:
-      - name: sync directory owner
-        run: sudo chown runner:runner -R .*
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: collect pytest warnings files
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: pytest-warnings-json
           path: test_root/log
@@ -146,11 +144,12 @@ jobs:
 
       - name: save warning report
         if: success()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: pytest-warning-report-html
           path: |
             reports/pytest_warnings/warning_report_all.html
+          overwrite: true
 
   # Combine and upload coverage reports.
   coverage:
@@ -162,15 +161,15 @@ jobs:
           - 3.11
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
       - name: Download all artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: coverage
           path: reports
@@ -184,4 +183,4 @@ jobs:
           coverage combine reports/*
           coverage report
           coverage xml
-      - uses: codecov/codecov-action@v3
+      - uses: codecov/codecov-action@v4
diff --git a/.github/workflows/units-test-scripts-structures-pruning.yml b/.github/workflows/units-test-scripts-structures-pruning.yml
index b87d27a884c8..28476ecefdaf 100644
--- a/.github/workflows/units-test-scripts-structures-pruning.yml
+++ b/.github/workflows/units-test-scripts-structures-pruning.yml
@@ -20,7 +20,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
diff --git a/.github/workflows/units-test-scripts-user-retirement.yml b/.github/workflows/units-test-scripts-user-retirement.yml
index 0ab9bfb2da26..a0f1d466c627 100644
--- a/.github/workflows/units-test-scripts-user-retirement.yml
+++ b/.github/workflows/units-test-scripts-user-retirement.yml
@@ -20,7 +20,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 
diff --git a/.github/workflows/upgrade-one-python-dependency.yml b/.github/workflows/upgrade-one-python-dependency.yml
index a5d0aa9e48f9..adb2d2742e14 100644
--- a/.github/workflows/upgrade-one-python-dependency.yml
+++ b/.github/workflows/upgrade-one-python-dependency.yml
@@ -32,12 +32,12 @@ jobs:
 
     steps:
       - name: Check out target branch
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: "${{ inputs.branch }}"
 
       - name: Set up Python environment
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: "3.11"
 
diff --git a/.github/workflows/verify-dunder-init.yml b/.github/workflows/verify-dunder-init.yml
index aefc0f53b6f5..e2a7d58cd97b 100644
--- a/.github/workflows/verify-dunder-init.yml
+++ b/.github/workflows/verify-dunder-init.yml
@@ -15,7 +15,7 @@ jobs:
     steps:
 
     - name: Check out branch
-      uses: actions/checkout@v2
+      uses: actions/checkout@v4
 
     - name: Ensure git is installed
       run: |
diff --git a/.github/workflows/verify-gha-unit-tests-count.yml b/.github/workflows/verify-gha-unit-tests-count.yml
index 1f6f91829c89..16671227bafc 100644
--- a/.github/workflows/verify-gha-unit-tests-count.yml
+++ b/.github/workflows/verify-gha-unit-tests-count.yml
@@ -9,20 +9,21 @@ on:
 jobs:
   collect-and-verify:
     if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false))
-    runs-on: [ edx-platform-runner ]
+    runs-on: [ "ubuntu-20.04" ]
     steps:
-      - name: sync directory owner
-        run: sudo chown runner:runner -R .*
-
+      - uses: actions/checkout@v4
       - name: Setup Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.11'
 
-      - uses: actions/checkout@v2
+      - name: install system requirements
+        run: |
+          make ubuntu-requirements
+
       - name: install requirements
         run: |
-          sudo make test-requirements
+          sudo env "PATH=$PATH" make test-requirements
 
       - name: verify unit tests count
         uses: ./.github/actions/verify-tests-count

From f6f75b428ce808d59c9b257f5a0908bc1d5bf181 Mon Sep 17 00:00:00 2001
From: Irtaza Akram <irtaza.akram@arbisoft.com>
Date: Wed, 22 May 2024 10:47:19 +0500
Subject: [PATCH 02/10] fix: lynx dependency

---
 .github/workflows/unit-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
index 6ccf5ae03c88..44179d6a3b2d 100644
--- a/.github/workflows/unit-tests.yml
+++ b/.github/workflows/unit-tests.yml
@@ -61,7 +61,7 @@ jobs:
 
       - name: install system requirements
         run: |
-          make ubuntu-requirements
+          sudo apt-get update && sudo apt-get install libmysqlclient-dev libxmlsec1-dev lynx
 
       - name: install mongo version
         run: |

From a6755edcbb78607917418ae91bca4d2e2a36e8b9 Mon Sep 17 00:00:00 2001
From: Irtaza Akram <irtaza.akram@arbisoft.com>
Date: Wed, 22 May 2024 13:34:33 +0500
Subject: [PATCH 03/10] fix: remove sudo for pip specific commands

---
 .github/workflows/unit-tests.yml                  | 8 ++++----
 .github/workflows/verify-gha-unit-tests-count.yml | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
index 44179d6a3b2d..33fe0c4ce41b 100644
--- a/.github/workflows/unit-tests.yml
+++ b/.github/workflows/unit-tests.yml
@@ -84,15 +84,15 @@ jobs:
 
       - name: install requirements
         run: |
-          sudo env "PATH=$PATH" make test-requirements
+          make test-requirements
           if [[ "${{ matrix.django-version }}" != "pinned" ]]; then
-            sudo env "PATH=$PATH" pip install "django~=${{ matrix.django-version }}.0"
-            sudo env "PATH=$PATH" pip check # fail if this test-reqs/Django combination is broken
+            pip install "django~=${{ matrix.django-version }}.0"
+            pip check # fail if this test-reqs/Django combination is broken
           fi
 
       - name: list installed package versions
         run: |
-          sudo env "PATH=$PATH" pip freeze
+          pip freeze
 
       - name: Setup and run tests
         uses: ./.github/actions/unit-tests
diff --git a/.github/workflows/verify-gha-unit-tests-count.yml b/.github/workflows/verify-gha-unit-tests-count.yml
index 16671227bafc..7430eebd661d 100644
--- a/.github/workflows/verify-gha-unit-tests-count.yml
+++ b/.github/workflows/verify-gha-unit-tests-count.yml
@@ -23,7 +23,7 @@ jobs:
 
       - name: install requirements
         run: |
-          sudo env "PATH=$PATH" make test-requirements
+          make test-requirements
 
       - name: verify unit tests count
         uses: ./.github/actions/verify-tests-count

From 6b5f11a79bb52d069367b7832c8a6907bfc6d9c7 Mon Sep 17 00:00:00 2001
From: Irtaza Akram <irtaza.akram@arbisoft.com>
Date: Mon, 27 May 2024 11:04:56 +0500
Subject: [PATCH 04/10] fix: merged gh-hosted into gha

---
 .github/workflows/unit-tests-gh-hosted.yml    | 141 ------------------
 .github/workflows/unit-tests.yml              |   3 +-
 .../workflows/verify-gha-unit-tests-count.yml |   1 -
 3 files changed, 2 insertions(+), 143 deletions(-)
 delete mode 100644 .github/workflows/unit-tests-gh-hosted.yml

diff --git a/.github/workflows/unit-tests-gh-hosted.yml b/.github/workflows/unit-tests-gh-hosted.yml
deleted file mode 100644
index 7fe7c8961b61..000000000000
--- a/.github/workflows/unit-tests-gh-hosted.yml
+++ /dev/null
@@ -1,141 +0,0 @@
-name: unit-tests-gh-hosted
-
-on:
-  pull_request:
-  push:
-    branches:
-      - master
-      - open-release/lilac.master
-
-jobs:
-  run-test:
-    name: ${{ matrix.shard_name }}(py=${{ matrix.python-version }},dj=${{ matrix.django-version }},mongo=${{ matrix.mongo-version }})
-    if: (github.repository != 'openedx/edx-platform' && github.repository != 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == true))
-    runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        python-version:
-          - "3.11"
-        django-version:
-          - "pinned"
-        # When updating the shards, remember to make the same changes in
-        # .github/workflows/unit-tests.yml
-        shard_name:
-          - "lms-1"
-          - "lms-2"
-          - "lms-3"
-          - "lms-4"
-          - "lms-5"
-          - "lms-6"
-          - "openedx-1-with-lms"
-          - "openedx-2-with-lms"
-          - "openedx-1-with-cms"
-          - "openedx-2-with-cms"
-          - "cms-1"
-          - "cms-2"
-          - "common-with-lms"
-          - "common-with-cms"
-          - "xmodule-with-lms"
-          - "xmodule-with-cms"
-        mongo-version:
-          - "7.0"
-
-        # We only need to test older versions of Mongo with modules that directly interface with Mongo (that is: xmodule.modulestore)
-        # This code is left here as an example for future refernce in case we need to reduce the number of shards we're
-        # testing but still have good confidence with older versions of mongo.  We use Mongo 4.4 in the example.
-        #
-        # exclude:
-        #   - mongo-version: "4.4"
-        # include:
-        #   - shard_name: "xmodule-with-cms"
-        #     python-version: "3.11"
-        #     django-version: "pinned"
-        #     mongo-version: "4.4"
-        #   - shard_name: "xmodule-with-lms"
-        #     python-version: "3.11"
-        #     django-version: "pinned"
-        #     mongo-version: "4.4"
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Install Required System Packages
-        run: sudo apt-get update && sudo apt-get install libxmlsec1-dev lynx
-
-      - name: Start MongoDB
-        uses: supercharge/mongodb-github-action@1.11.0
-        with:
-          mongodb-version: ${{ matrix.mongo-version}}
-
-      - name: Setup Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Get pip cache dir
-        id: pip-cache-dir
-        run: |
-          echo "::set-output name=dir::$(pip cache dir)"
-
-      - name: Cache pip dependencies
-        id: cache-dependencies
-        uses: actions/cache@v4
-        with:
-          path: ${{ steps.pip-cache-dir.outputs.dir }}
-          key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/testing.txt') }}
-          restore-keys: ${{ runner.os }}-pip-
-
-      - name: Install Required Python Dependencies
-        env:
-          PIP_SRC: ${{ runner.temp }}
-        run: |
-          make test-requirements
-          if [[ "${{ matrix.django-version }}" != "pinned" ]]; then
-            pip install "django~=${{ matrix.django-version }}.0"
-            pip check # fail if this test-reqs/Django combination is broken
-          fi
-
-      - name: Setup and run tests
-        uses: ./.github/actions/unit-tests
-
-  collect-and-verify:
-    if: (github.repository != 'openedx/edx-platform' && github.repository != 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == true))
-    runs-on: ubuntu-20.04
-    strategy:
-      matrix:
-        python-version: [ '3.11' ]
-        django-version:
-          - "pinned"
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Install Required System Packages
-        run: sudo apt-get update && sudo apt-get install libxmlsec1-dev
-
-      - name: Setup Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Get pip cache dir
-        id: pip-cache-dir
-        run: |
-          echo "::set-output name=dir::$(pip cache dir)"
-
-      - name: Cache pip dependencies
-        id: cache-dependencies
-        uses: actions/cache@v4
-        with:
-          path: ${{ steps.pip-cache-dir.outputs.dir }}
-          key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/testing.txt') }}
-          restore-keys: ${{ runner.os }}-pip-
-
-      - name: Install Required Python Dependencies
-        run: |
-          make test-requirements
-          if [[ "${{ matrix.django-version }}" != "pinned" ]]; then
-            pip install "django~=${{ matrix.django-version }}.0"
-            pip check # fail if this test-reqs/Django combination is broken
-          fi
-
-      - name: verify unit tests count
-        uses: ./.github/actions/verify-tests-count
diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
index 33fe0c4ce41b..0b6e29b18678 100644
--- a/.github/workflows/unit-tests.yml
+++ b/.github/workflows/unit-tests.yml
@@ -9,7 +9,6 @@ on:
 jobs:
   run-tests:
     name: ${{ matrix.shard_name }}(py=${{ matrix.python-version }},dj=${{ matrix.django-version }},mongo=${{ matrix.mongo-version }})
-    if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false))
     runs-on: [ "ubuntu-20.04" ]
     strategy:
       matrix:
@@ -98,10 +97,12 @@ jobs:
         uses: ./.github/actions/unit-tests
 
       - name: Renaming coverage data file
+        if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false))
         run: |
           mv reports/.coverage reports/${{ matrix.shard_name }}.coverage
 
       - name: Upload coverage
+        if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false))
         uses: actions/upload-artifact@v4
         with:
           name: coverage
diff --git a/.github/workflows/verify-gha-unit-tests-count.yml b/.github/workflows/verify-gha-unit-tests-count.yml
index 7430eebd661d..bfa38852a0d5 100644
--- a/.github/workflows/verify-gha-unit-tests-count.yml
+++ b/.github/workflows/verify-gha-unit-tests-count.yml
@@ -8,7 +8,6 @@ on:
 
 jobs:
   collect-and-verify:
-    if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false))
     runs-on: [ "ubuntu-20.04" ]
     steps:
       - uses: actions/checkout@v4

From 5a46a3aa0f5759dea07c035daa2106b1d1c0b8c1 Mon Sep 17 00:00:00 2001
From: Irtaza Akram <irtaza.akram@arbisoft.com>
Date: Mon, 27 May 2024 12:46:33 +0500
Subject: [PATCH 05/10] fix: tests on forks

---
 .github/workflows/unit-tests.yml | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
index 0b6e29b18678..f80eab2bff77 100644
--- a/.github/workflows/unit-tests.yml
+++ b/.github/workflows/unit-tests.yml
@@ -115,9 +115,8 @@ jobs:
   success:
     name: Unit tests successful
     if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false))
-    needs:
-      - run-tests
-    runs-on: ubuntu-latest
+    runs-on: [ "ubuntu-20.04" ]
+    needs: [ run-tests ]
     steps:
       - name: Decide whether the needed jobs succeeded or failed
         # uses: re-actors/alls-green@v1.2.1
@@ -126,6 +125,7 @@ jobs:
           jobs: ${{ toJSON(needs) }}
 
   compile-warnings-report:
+    if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false))
     runs-on: [ "ubuntu-20.04" ]
     needs: [ run-tests ]
     steps:
@@ -154,8 +154,9 @@ jobs:
 
   # Combine and upload coverage reports.
   coverage:
-    needs: run-tests
-    runs-on: ubuntu-latest
+    if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false))
+    runs-on: [ "ubuntu-20.04" ]
+    needs: [ run-tests ]
     strategy:
       matrix:
         python-version:

From 99248ae992b0a3c70240066b1641a100ff3dff37 Mon Sep 17 00:00:00 2001
From: Irtaza Akram <irtaza.akram@arbisoft.com>
Date: Wed, 29 May 2024 11:19:02 +0500
Subject: [PATCH 06/10] fix: review changes

---
 .github/workflows/unit-tests.yml | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
index f80eab2bff77..f1ec71a649d0 100644
--- a/.github/workflows/unit-tests.yml
+++ b/.github/workflows/unit-tests.yml
@@ -6,10 +6,13 @@ on:
     branches:
       - master
 
+env:
+  RUNNER: "ubuntu-20.04"
+
 jobs:
   run-tests:
     name: ${{ matrix.shard_name }}(py=${{ matrix.python-version }},dj=${{ matrix.django-version }},mongo=${{ matrix.mongo-version }})
-    runs-on: [ "ubuntu-20.04" ]
+    runs-on: ${{ env.RUNNER }}
     strategy:
       matrix:
         python-version:
@@ -97,12 +100,10 @@ jobs:
         uses: ./.github/actions/unit-tests
 
       - name: Renaming coverage data file
-        if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false))
         run: |
           mv reports/.coverage reports/${{ matrix.shard_name }}.coverage
 
       - name: Upload coverage
-        if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false))
         uses: actions/upload-artifact@v4
         with:
           name: coverage
@@ -114,8 +115,7 @@ jobs:
   # https://github.com/orgs/community/discussions/33579
   success:
     name: Unit tests successful
-    if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false))
-    runs-on: [ "ubuntu-20.04" ]
+    runs-on: ${{ env.RUNNER }}
     needs: [ run-tests ]
     steps:
       - name: Decide whether the needed jobs succeeded or failed
@@ -125,8 +125,7 @@ jobs:
           jobs: ${{ toJSON(needs) }}
 
   compile-warnings-report:
-    if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false))
-    runs-on: [ "ubuntu-20.04" ]
+    runs-on: ${{ env.RUNNER }}
     needs: [ run-tests ]
     steps:
       - uses: actions/checkout@v4
@@ -155,7 +154,7 @@ jobs:
   # Combine and upload coverage reports.
   coverage:
     if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false))
-    runs-on: [ "ubuntu-20.04" ]
+    runs-on: ${{ env.RUNNER }}
     needs: [ run-tests ]
     strategy:
       matrix:

From 2506d9373a4a0504acacee596019029889fae29d Mon Sep 17 00:00:00 2001
From: Irtaza Akram <irtaza.akram@arbisoft.com>
Date: Wed, 29 May 2024 12:16:34 +0500
Subject: [PATCH 07/10] fix: test runner

---
 .github/workflows/unit-tests.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
index f1ec71a649d0..3eeb0da48a11 100644
--- a/.github/workflows/unit-tests.yml
+++ b/.github/workflows/unit-tests.yml
@@ -7,12 +7,12 @@ on:
       - master
 
 env:
-  RUNNER: "ubuntu-20.04"
+  runner: "ubuntu-20.04"
 
 jobs:
   run-tests:
     name: ${{ matrix.shard_name }}(py=${{ matrix.python-version }},dj=${{ matrix.django-version }},mongo=${{ matrix.mongo-version }})
-    runs-on: ${{ env.RUNNER }}
+    runs-on: ${{ github.env.runner }}
     strategy:
       matrix:
         python-version:
@@ -115,7 +115,7 @@ jobs:
   # https://github.com/orgs/community/discussions/33579
   success:
     name: Unit tests successful
-    runs-on: ${{ env.RUNNER }}
+    runs-on: ${{ github.env.runner }}
     needs: [ run-tests ]
     steps:
       - name: Decide whether the needed jobs succeeded or failed
@@ -125,7 +125,7 @@ jobs:
           jobs: ${{ toJSON(needs) }}
 
   compile-warnings-report:
-    runs-on: ${{ env.RUNNER }}
+    runs-on: ${{ github.env.runner }}
     needs: [ run-tests ]
     steps:
       - uses: actions/checkout@v4
@@ -154,7 +154,7 @@ jobs:
   # Combine and upload coverage reports.
   coverage:
     if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false))
-    runs-on: ${{ env.RUNNER }}
+    runs-on: ${{ github.env.runner }}
     needs: [ run-tests ]
     strategy:
       matrix:

From b3185ffe7b3c1c526d4c6eeeb201992fa04464fc Mon Sep 17 00:00:00 2001
From: Irtaza Akram <irtaza.akram@arbisoft.com>
Date: Wed, 29 May 2024 13:18:53 +0500
Subject: [PATCH 08/10] fix: env for runner

---
 .github/workflows/unit-tests.yml | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
index 3eeb0da48a11..f51945d30802 100644
--- a/.github/workflows/unit-tests.yml
+++ b/.github/workflows/unit-tests.yml
@@ -6,13 +6,10 @@ on:
     branches:
       - master
 
-env:
-  runner: "ubuntu-20.04"
-
 jobs:
   run-tests:
     name: ${{ matrix.shard_name }}(py=${{ matrix.python-version }},dj=${{ matrix.django-version }},mongo=${{ matrix.mongo-version }})
-    runs-on: ${{ github.env.runner }}
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
         python-version:
@@ -115,7 +112,7 @@ jobs:
   # https://github.com/orgs/community/discussions/33579
   success:
     name: Unit tests successful
-    runs-on: ${{ github.env.runner }}
+    runs-on: ubuntu-20.04
     needs: [ run-tests ]
     steps:
       - name: Decide whether the needed jobs succeeded or failed
@@ -125,7 +122,7 @@ jobs:
           jobs: ${{ toJSON(needs) }}
 
   compile-warnings-report:
-    runs-on: ${{ github.env.runner }}
+    runs-on: ubuntu-20.04
     needs: [ run-tests ]
     steps:
       - uses: actions/checkout@v4
@@ -154,7 +151,7 @@ jobs:
   # Combine and upload coverage reports.
   coverage:
     if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false))
-    runs-on: ${{ github.env.runner }}
+    runs-on: ubuntu-20.04
     needs: [ run-tests ]
     strategy:
       matrix:

From c22fbec6a90a331047f8adcfeccb13fc4c4016ff Mon Sep 17 00:00:00 2001
From: Irtaza Akram <irtaza.akram@arbisoft.com>
Date: Wed, 29 May 2024 21:15:00 +0500
Subject: [PATCH 09/10] fix: move collect-and-verify to unit-tests.yml

---
 .github/workflows/unit-tests.yml              | 20 +++++++++++++
 .../workflows/verify-gha-unit-tests-count.yml | 28 -------------------
 2 files changed, 20 insertions(+), 28 deletions(-)
 delete mode 100644 .github/workflows/verify-gha-unit-tests-count.yml

diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
index f51945d30802..694e911fe9fc 100644
--- a/.github/workflows/unit-tests.yml
+++ b/.github/workflows/unit-tests.yml
@@ -107,6 +107,26 @@ jobs:
           path: reports/${{matrix.shard_name}}.coverage
           overwrite: true
 
+  collect-and-verify:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+
+      - name: install system requirements
+        run: |
+          make ubuntu-requirements
+
+      - name: install requirements
+        run: |
+          make test-requirements
+
+      - name: verify unit tests count
+        uses: ./.github/actions/verify-tests-count
+
   # This job aggregates test results. It's the required check for branch protection.
   # https://github.com/marketplace/actions/alls-green#why
   # https://github.com/orgs/community/discussions/33579
diff --git a/.github/workflows/verify-gha-unit-tests-count.yml b/.github/workflows/verify-gha-unit-tests-count.yml
deleted file mode 100644
index bfa38852a0d5..000000000000
--- a/.github/workflows/verify-gha-unit-tests-count.yml
+++ /dev/null
@@ -1,28 +0,0 @@
-name: verify unit tests count
-
-on:
-  pull_request:
-  push:
-    branches:
-      - master
-
-jobs:
-  collect-and-verify:
-    runs-on: [ "ubuntu-20.04" ]
-    steps:
-      - uses: actions/checkout@v4
-      - name: Setup Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.11'
-
-      - name: install system requirements
-        run: |
-          make ubuntu-requirements
-
-      - name: install requirements
-        run: |
-          make test-requirements
-
-      - name: verify unit tests count
-        uses: ./.github/actions/verify-tests-count

From 11e8b8ba904ef62fe9f8ed789fb358277d1734a5 Mon Sep 17 00:00:00 2001
From: Irtaza Akram <irtaza.akram@arbisoft.com>
Date: Wed, 29 May 2024 21:19:19 +0500
Subject: [PATCH 10/10] fix: verify-and-collect issues

---
 .github/workflows/unit-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
index 694e911fe9fc..7df346e65810 100644
--- a/.github/workflows/unit-tests.yml
+++ b/.github/workflows/unit-tests.yml
@@ -118,7 +118,7 @@ jobs:
 
       - name: install system requirements
         run: |
-          make ubuntu-requirements
+          sudo apt-get update && sudo apt-get install libxmlsec1-dev
 
       - name: install requirements
         run: |