From a3d228b60a31ad6cfb53fa4a41902cd14da0d060 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Fri, 19 Jan 2024 12:29:58 -0500 Subject: [PATCH 1/7] temp: Try to run everything on Actions runnes. --- .github/workflows/unit-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index c3b1086c20eb..06d740e3b82c 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -10,7 +10,7 @@ jobs: run-tests: name: python-${{ matrix.python-version }},django-${{ matrix.django-version }},${{ matrix.shard_name }} if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false)) - runs-on: [ edx-platform-runner ] + runs-on: ubuntu-latest strategy: matrix: python-version: @@ -95,7 +95,7 @@ jobs: jobs: ${{ toJSON(needs) }} compile-warnings-report: - runs-on: [ edx-platform-runner ] + runs-on: ubuntu-latest needs: [ run-tests ] steps: - name: sync directory owner From 5a7c5329569f80f8a7e6cc1bb922568da93456b6 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Fri, 19 Jan 2024 12:45:44 -0500 Subject: [PATCH 2/7] build: Be more explicit about which services we're using. - Use mongo via a github service. - Setup python using `setup-python` instead of assuming the system is right. --- .github/workflows/migrations-check.yml | 6 +++--- .github/workflows/unit-tests.yml | 25 ++++++++++++++++++------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/.github/workflows/migrations-check.yml b/.github/workflows/migrations-check.yml index 3929e7f10e6f..f3ac2222c577 100644 --- a/.github/workflows/migrations-check.yml +++ b/.github/workflows/migrations-check.yml @@ -17,11 +17,11 @@ jobs: python-version: [ 3.8 ] # 'pinned' is used to install the latest patch version of Django # within the global constraint i.e. Django==3.2.21 in current case - # because we have global constraint of Django<4.2 + # because we have global constraint of Django<4.2 django-version: ["pinned", "4.2"] mongo-version: ["4"] mysql-version: ["5.7", "8"] - # excluding mysql5.7 with Django 4.2 since Django 4.2 has + # excluding mysql5.7 with Django 4.2 since Django 4.2 has # dropped support for MySQL<8 exclude: - django-version: "4.2" @@ -120,7 +120,7 @@ jobs: ./manage.py lms migrate echo "Running the CMS migrations." ./manage.py cms migrate - + # This job aggregates test results. It's the required check for branch protection. # https://github.com/marketplace/actions/alls-green#why # https://github.com/orgs/community/discussions/33579 diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 06d740e3b82c..58412d6459a1 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -18,6 +18,7 @@ jobs: django-version: - "pinned" - "4.2" + mongo-version: ["4"] # When updating the shards, remember to make the same changes in # .github/workflows/unit-tests-gh-hosted.yml shard_name: @@ -39,20 +40,30 @@ jobs: - "xmodule-with-cms" # We expect Django 4.0 to fail, so don't stop when it fails. continue-on-error: ${{ matrix.django-version == '4.0' }} - + services: + mongo: + image: mongo:${{ matrix.mongo-version }} + ports: + - 27017:27017 + # Note: Calling mongo here only works with mongo 4, in newer versions of mongo + # we'll have to use `mongosh` + options: >- + --health-cmd "mongo --quiet --eval 'db.runCommand(\"ping\")'" + --health-interval 10s + --health-timeout 5s + --health-retries 3 steps: - name: sync directory owner run: sudo chown runner:runner -R .* + - name: Setup Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: checkout repo uses: actions/checkout@v3 - - name: start mongod server for tests - run: | - sudo mkdir -p /data/db - sudo chmod -R a+rw /data/db - mongod & - - name: install requirements run: | sudo make test-requirements From 579635af35593ac9848592f2b3b22b8a84900b99 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Fri, 19 Jan 2024 12:57:28 -0500 Subject: [PATCH 3/7] build: setup monge user and drop sudos. --- .github/workflows/unit-tests.yml | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 58412d6459a1..12ad24434ac1 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -53,8 +53,27 @@ jobs: --health-timeout 5s --health-retries 3 steps: - - name: sync directory owner - run: sudo chown runner:runner -R .* +# - name: sync directory owner +# run: sudo chown runner:runner -R .* + - name: Setup mongodb user + run: | + mongosh edxapp --eval ' + db.createUser( + { + user: "edxapp", + pwd: "password", + roles: [ + { role: "readWrite", db: "edxapp" }, + ] + } + ); + ' + + - name: Verify mongo and mysql db credentials + run: | + mysql -h 127.0.0.1 -uedxapp001 -ppassword -e "select 1;" edxapp + mongosh --host 127.0.0.1 --username edxapp --password password --eval 'use edxapp; db.adminCommand("ping");' edxapp + - name: Setup Python ${{ matrix.python-version }} uses: actions/setup-python@v4 @@ -66,15 +85,15 @@ jobs: - name: install requirements run: | - sudo make test-requirements + make test-requirements if [[ "${{ matrix.django-version }}" != "pinned" ]]; then - sudo pip install "django~=${{ matrix.django-version }}.0" - sudo pip check # fail if this test-reqs/Django combination is broken + pip install "django~=${{ matrix.django-version }}.0" + pip check # fail if this test-reqs/Django combination is broken fi - name: list installed package versions run: | - sudo pip freeze + pip freeze - name: Setup and run tests uses: ./.github/actions/unit-tests From b6e074e25f9e54c1c8070163590debfcf2b07ce0 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Fri, 19 Jan 2024 13:03:02 -0500 Subject: [PATCH 4/7] tmp: reduce number of tests and change mongosh to mongo. --- .github/workflows/unit-tests.yml | 48 +++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 12ad24434ac1..aee01a0b757d 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -23,21 +23,21 @@ jobs: # .github/workflows/unit-tests-gh-hosted.yml shard_name: - "lms-1" - - "lms-2" - - "lms-3" - - "lms-4" - - "lms-5" - - "lms-6" - - "openedx-1-with-lms" - - "openedx-2-with-lms" - - "openedx-1-with-cms" - - "openedx-2-with-cms" - - "cms-1" - - "cms-2" - - "common-with-lms" - - "common-with-cms" - - "xmodule-with-lms" - - "xmodule-with-cms" +# - "lms-2" +# - "lms-3" +# - "lms-4" +# - "lms-5" +# - "lms-6" +# - "openedx-1-with-lms" +# - "openedx-2-with-lms" +# - "openedx-1-with-cms" +# - "openedx-2-with-cms" +# - "cms-1" +# - "cms-2" +# - "common-with-lms" +# - "common-with-cms" +# - "xmodule-with-lms" +# - "xmodule-with-cms" # We expect Django 4.0 to fail, so don't stop when it fails. continue-on-error: ${{ matrix.django-version == '4.0' }} services: @@ -52,12 +52,26 @@ jobs: --health-interval 10s --health-timeout 5s --health-retries 3 + mysql: + image: mysql:${{ matrix.mysql-version }} + ports: + - 3306:3306 + env: + MYSQL_DATABASE: "edxapp" + MYSQL_USER: "edxapp001" + MYSQL_PASSWORD: "password" + MYSQL_RANDOM_ROOT_PASSWORD: true + options: >- + --health-cmd "mysqladmin ping" + --health-interval 10s + --health-timeout 5s + --health-retries 3 steps: # - name: sync directory owner # run: sudo chown runner:runner -R .* - name: Setup mongodb user run: | - mongosh edxapp --eval ' + mongo edxapp --eval ' db.createUser( { user: "edxapp", @@ -72,7 +86,7 @@ jobs: - name: Verify mongo and mysql db credentials run: | mysql -h 127.0.0.1 -uedxapp001 -ppassword -e "select 1;" edxapp - mongosh --host 127.0.0.1 --username edxapp --password password --eval 'use edxapp; db.adminCommand("ping");' edxapp + mongo --host 127.0.0.1 --username edxapp --password password --eval 'use edxapp; db.adminCommand("ping");' edxapp - name: Setup Python ${{ matrix.python-version }} From 39fd70473c6c1ecb0f7da2996fe1fa4e981add1a Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Fri, 19 Jan 2024 13:07:49 -0500 Subject: [PATCH 5/7] temp: add mysql version. --- .github/workflows/unit-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index aee01a0b757d..b80c7bd722b4 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -19,6 +19,7 @@ jobs: - "pinned" - "4.2" mongo-version: ["4"] + mysql-version: ["8"] # When updating the shards, remember to make the same changes in # .github/workflows/unit-tests-gh-hosted.yml shard_name: From 9905784955455bff507e3d8edc2d36cbeacf6c42 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Fri, 19 Jan 2024 14:51:47 -0500 Subject: [PATCH 6/7] temp: to see if sudo remova worked.w --- .github/workflows/unit-tests.yml | 38 ++++++++++++++++---------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index b80c7bd722b4..53eb9091db7b 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -46,7 +46,7 @@ jobs: image: mongo:${{ matrix.mongo-version }} ports: - 27017:27017 - # Note: Calling mongo here only works with mongo 4, in newer versions of mongo + # Note: Calling `mongo` here only works with mongo 4, in newer versions of mongo # we'll have to use `mongosh` options: >- --health-cmd "mongo --quiet --eval 'db.runCommand(\"ping\")'" @@ -70,24 +70,24 @@ jobs: steps: # - name: sync directory owner # run: sudo chown runner:runner -R .* - - name: Setup mongodb user - run: | - mongo edxapp --eval ' - db.createUser( - { - user: "edxapp", - pwd: "password", - roles: [ - { role: "readWrite", db: "edxapp" }, - ] - } - ); - ' - - - name: Verify mongo and mysql db credentials - run: | - mysql -h 127.0.0.1 -uedxapp001 -ppassword -e "select 1;" edxapp - mongo --host 127.0.0.1 --username edxapp --password password --eval 'use edxapp; db.adminCommand("ping");' edxapp +# - name: Setup mongodb user +# run: | +# mongo edxapp --eval ' +# db.createUser( +# { +# user: "edxapp", +# pwd: "password", +# roles: [ +# { role: "readWrite", db: "edxapp" }, +# ] +# } +# ); +# ' +# +# - name: Verify mongo and mysql db credentials +# run: | +# mysql -h 127.0.0.1 -uedxapp001 -ppassword -e "select 1;" edxapp +# mongo --host 127.0.0.1 --username edxapp --password password --eval 'use edxapp; db.adminCommand("ping");' edxapp - name: Setup Python ${{ matrix.python-version }} From 0e7f5c68705874dc6e558d226c41de6dcc4c206e Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Fri, 19 Jan 2024 14:57:26 -0500 Subject: [PATCH 7/7] temp: add ubuntu requirements. --- .github/workflows/unit-tests.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 53eb9091db7b..051a4e326af5 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -98,6 +98,11 @@ jobs: - name: checkout repo uses: actions/checkout@v3 + - name: Install system Packages + run: | + sudo apt-get update + make ubuntu-requirements + - name: install requirements run: | make test-requirements