From ab60a0800d95ce916961cb27825593c2461e75c2 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 14 Mar 2024 10:48:07 -0400 Subject: [PATCH 1/7] feat!: Unpin scipy. Scipy was constrained to prevent a numpy upgrade however the numpy upgrade happened a while ago and this never got updated. Now the scipy version is preventing us from going to an even newer numpy version that would be compatible with both Python 3.8 and Python 3.12 BREAKING CHANGE: This will update scipy in the codejail sandbox from 1.7.3 to a newer version. This contains many backwards incompatible changes that are all documented in the SciPy changelogs: https://docs.scipy.org/doc/scipy/release/1.8.0-notes.html --- requirements/constraints.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index ed7c0ee9b075..dd0b48e09727 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -48,9 +48,6 @@ django-webpack-loader==0.7.0 # version of py2neo will work with Neo4j 3.5. py2neo<2022 -# scipy version 1.8 requires numpy>=1.17.3, we've pinned numpy to <1.17.0 in requirements/edx-sandbox/py38.in -scipy<1.8.0 - # edx-enterprise, snowflake-connector-python require charset-normalizer==2.0.0 # Can be removed once snowflake-connector-python>2.7.9 is released with the fix. charset-normalizer<2.1.0 From 05ec09fcdc28df190af10d9d9699cde8a43ae74f Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 14 Mar 2024 11:59:42 -0400 Subject: [PATCH 2/7] chore: Upgrade scipy Ran `package=scipy make upgrade-package` to upgrade just this one dependency. --- requirements/edx-sandbox/base.txt | 3 +-- requirements/edx/base.txt | 4 ++-- requirements/edx/development.txt | 4 ++-- requirements/edx/doc.txt | 4 ++-- requirements/edx/testing.txt | 4 ++-- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/requirements/edx-sandbox/base.txt b/requirements/edx-sandbox/base.txt index 53c796077848..b8651a0340b6 100644 --- a/requirements/edx-sandbox/base.txt +++ b/requirements/edx-sandbox/base.txt @@ -76,9 +76,8 @@ random2==1.0.2 # via -r requirements/edx-sandbox/base.in regex==2024.4.16 # via nltk -scipy==1.7.3 +scipy==1.10.1 # via - # -c requirements/edx-sandbox/../constraints.txt # -r requirements/edx-sandbox/base.in # chem # openedx-calc diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 12e24dd01972..23e6ccca456d 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -57,6 +57,7 @@ backoff==1.10.0 # via analytics-python backports-zoneinfo[tzdata]==0.2.1 # via + # -c requirements/edx/../constraints.txt # celery # django # edx-milestones @@ -1045,9 +1046,8 @@ s3transfer==0.10.0 # via boto3 sailthru-client==2.2.3 # via edx-ace -scipy==1.7.3 +scipy==1.10.1 # via - # -c requirements/edx/../constraints.txt # chem # openedx-calc semantic-version==2.10.0 diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 4c2d23507355..b533ffb745cb 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -113,6 +113,7 @@ backoff==1.10.0 # analytics-python backports-zoneinfo[tzdata]==0.2.1 # via + # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # celery @@ -1797,9 +1798,8 @@ sailthru-client==2.2.3 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edx-ace -scipy==1.7.3 +scipy==1.10.1 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # chem diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 8be896b0b22c..015694f99dbf 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -80,6 +80,7 @@ backoff==1.10.0 # analytics-python backports-zoneinfo[tzdata]==0.2.1 # via + # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # celery # django @@ -1239,9 +1240,8 @@ sailthru-client==2.2.3 # via # -r requirements/edx/base.txt # edx-ace -scipy==1.7.3 +scipy==1.10.1 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # chem # openedx-calc diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 4a0d69073588..062a86b6a301 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -80,6 +80,7 @@ backoff==1.10.0 # analytics-python backports-zoneinfo[tzdata]==0.2.1 # via + # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # celery # django @@ -1360,9 +1361,8 @@ sailthru-client==2.2.3 # via # -r requirements/edx/base.txt # edx-ace -scipy==1.7.3 +scipy==1.10.1 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # chem # openedx-calc From fb727c101e049510aa9f7d755b1def8d249c2550 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 14 Mar 2024 12:08:41 -0400 Subject: [PATCH 3/7] feat!: Upgrade numpy to be python 3.8 and 3.12 compatible. BREAKING CHANGE: Because numpy is available in the codejail sandbox, the update from 1.22.4 to 1.24.4 does have completed deprecations, and more details can be found in the Numpy Release Notes: https://numpy.org/doc/stable/release/1.23.0-notes.html#expired-deprecations --- requirements/edx-sandbox/base.txt | 2 +- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/doc.txt | 2 +- requirements/edx/testing.txt | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements/edx-sandbox/base.txt b/requirements/edx-sandbox/base.txt index b8651a0340b6..fbc05dac94a8 100644 --- a/requirements/edx-sandbox/base.txt +++ b/requirements/edx-sandbox/base.txt @@ -49,7 +49,7 @@ nltk==3.8.1 # via # -r requirements/edx-sandbox/base.in # chem -numpy==1.22.4 +numpy==1.24.4 # via # chem # contourpy diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 23e6ccca456d..97dbdb1090be 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -748,7 +748,7 @@ nltk==3.8.1 # via chem nodeenv==1.8.0 # via -r requirements/edx/kernel.in -numpy==1.22.4 +numpy==1.24.4 # via # chem # openedx-calc diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index b533ffb745cb..2c9482f60685 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -1247,7 +1247,7 @@ nodeenv==1.8.0 # -r requirements/edx/assets.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -numpy==1.22.4 +numpy==1.24.4 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 015694f99dbf..d27ccebc4bfa 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -881,7 +881,7 @@ nltk==3.8.1 # chem nodeenv==1.8.0 # via -r requirements/edx/base.txt -numpy==1.22.4 +numpy==1.24.4 # via # -r requirements/edx/base.txt # chem diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 062a86b6a301..077ac742cb28 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -936,7 +936,7 @@ nltk==3.8.1 # chem nodeenv==1.8.0 # via -r requirements/edx/base.txt -numpy==1.22.4 +numpy==1.24.4 # via # -r requirements/edx/base.txt # chem From 1d485ca2b84dca7006786679a754b018c53d34fa Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 14 Mar 2024 12:41:22 -0400 Subject: [PATCH 4/7] build: Don't require the backports.zoneinfo package for newer python. We do this so that we can build requirements files that could work to `pip install` on both the old (3.8) and new (3.11) versions of python. --- requirements/constraints.txt | 3 +++ requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/doc.txt | 2 +- requirements/edx/testing.txt | 2 +- scripts/user_retirement/requirements/base.txt | 3 ++- scripts/user_retirement/requirements/testing.txt | 2 +- 7 files changed, 10 insertions(+), 6 deletions(-) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index dd0b48e09727..ad17ea9f3c3d 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -130,3 +130,6 @@ moto<5.0 # path==16.12.0 breaks the unit test collections check # needs to be investigated and fixed separately path<16.12.0 + +# Temporary to Support the python 3.11 Upgrade +backports.zoneinfo;python_version<"3.9" # Newer versions have zoneinfo available in the standard library diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 97dbdb1090be..4e027596a0c3 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -55,7 +55,7 @@ babel==2.14.0 # enmerkar-underscore backoff==1.10.0 # via analytics-python -backports-zoneinfo[tzdata]==0.2.1 +backports-zoneinfo[tzdata]==0.2.1 ; python_version < "3.9" # via # -c requirements/edx/../constraints.txt # celery diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 2c9482f60685..10b88b4afd9f 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -111,7 +111,7 @@ backoff==1.10.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # analytics-python -backports-zoneinfo[tzdata]==0.2.1 +backports-zoneinfo[tzdata]==0.2.1 ; python_version < "3.9" # via # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index d27ccebc4bfa..1a63a42e57de 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -78,7 +78,7 @@ backoff==1.10.0 # via # -r requirements/edx/base.txt # analytics-python -backports-zoneinfo[tzdata]==0.2.1 +backports-zoneinfo[tzdata]==0.2.1 ; python_version < "3.9" # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 077ac742cb28..7dd6f913e9c0 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -78,7 +78,7 @@ backoff==1.10.0 # via # -r requirements/edx/base.txt # analytics-python -backports-zoneinfo[tzdata]==0.2.1 +backports-zoneinfo[tzdata]==0.2.1 ; python_version < "3.9" # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt diff --git a/scripts/user_retirement/requirements/base.txt b/scripts/user_retirement/requirements/base.txt index f66759b556fa..bd3fc4848f4f 100644 --- a/scripts/user_retirement/requirements/base.txt +++ b/scripts/user_retirement/requirements/base.txt @@ -10,8 +10,9 @@ attrs==23.2.0 # via zeep backoff==2.2.1 # via -r scripts/user_retirement/requirements/base.in -backports-zoneinfo==0.2.1 +backports-zoneinfo==0.2.1 ; python_version < "3.9" # via + # -c scripts/user_retirement/requirements/../../../requirements/constraints.txt # django # pendulum boto3==1.34.26 diff --git a/scripts/user_retirement/requirements/testing.txt b/scripts/user_retirement/requirements/testing.txt index b61e49596e9a..37444f469465 100644 --- a/scripts/user_retirement/requirements/testing.txt +++ b/scripts/user_retirement/requirements/testing.txt @@ -14,7 +14,7 @@ attrs==23.2.0 # zeep backoff==2.2.1 # via -r scripts/user_retirement/requirements/base.txt -backports-zoneinfo==0.2.1 +backports-zoneinfo==0.2.1 ; python_version < "3.9" # via # -r scripts/user_retirement/requirements/base.txt # django From 85db730cef3f8ef819a6c96997f9a2ba70e6510d Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 14 Mar 2024 13:24:13 -0400 Subject: [PATCH 5/7] build: Run tests on python 3.11 as well as 3.8 --- .github/workflows/unit-tests-gh-hosted.yml | 2 +- .github/workflows/unit-tests.yml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests-gh-hosted.yml b/.github/workflows/unit-tests-gh-hosted.yml index b9681db67f46..5f4e3c5a68cd 100644 --- a/.github/workflows/unit-tests-gh-hosted.yml +++ b/.github/workflows/unit-tests-gh-hosted.yml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [ '3.8' ] + python-version: [ '3.8', '3.11' ] django-version: - "pinned" # When updating the shards, remember to make the same changes in diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index d08dc1aadc81..8069ee8ef676 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -15,6 +15,7 @@ jobs: matrix: python-version: - "3.8" + - "3.11" django-version: - "pinned" # When updating the shards, remember to make the same changes in From e9d716375fafecf598fcc034ed6795195cc2d7d4 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Thu, 14 Mar 2024 15:35:53 -0400 Subject: [PATCH 6/7] build: Update to allow tests to run on python 3.11 * Update so tox will allow running on the new version. * Update so the hosted runner actually respects the python version in the matrix --- .github/workflows/unit-tests.yml | 11 ++++++++--- tox.ini | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 8069ee8ef676..9a503420248f 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -57,15 +57,20 @@ jobs: sudo apt-get update && sudo apt-get install -y mongodb-org="${{ matrix.mongo-version }}.*" fi - - name: checkout repo - uses: actions/checkout@v3 - - name: start mongod server for tests run: | sudo mkdir -p /data/db sudo chmod -R a+rw /data/db mongod & + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: checkout repo + uses: actions/checkout@v3 + - name: install requirements run: | sudo make test-requirements diff --git a/tox.ini b/tox.ini index 9b1937f3640a..1b4252fd1905 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py38, quality +envlist = py{38,311} quality # This is needed to prevent the lms, cms, and openedx packages inside the "Open # edX" package (defined in setup.py) from getting installed into site-packages From 325a7b91b20e1a68a383bad250a8f32b8435f7b6 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Tue, 9 Apr 2024 11:32:43 -0400 Subject: [PATCH 7/7] build: Update more workflows to test on Python 3.11 --- .github/workflows/ci-static-analysis.yml | 2 +- .github/workflows/js-tests.yml | 2 +- .github/workflows/lint-imports.yml | 2 +- .github/workflows/migrations-check.yml | 6 +++--- .github/workflows/pylint-checks.yml | 2 +- .github/workflows/quality-checks.yml | 2 +- .github/workflows/semgrep.yml | 2 +- .github/workflows/static-assets-check.yml | 2 +- .github/workflows/unit-tests-gh-hosted.yml | 2 +- .github/workflows/units-test-scripts-structures-pruning.yml | 2 +- .github/workflows/units-test-scripts-user-retirement.yml | 2 +- 11 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci-static-analysis.yml b/.github/workflows/ci-static-analysis.yml index dd21eb3cac53..33415601fed4 100644 --- a/.github/workflows/ci-static-analysis.yml +++ b/.github/workflows/ci-static-analysis.yml @@ -8,7 +8,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: ['3.8'] + python-version: ['3.8', '3.11'] os: ['ubuntu-20.04'] steps: diff --git a/.github/workflows/js-tests.yml b/.github/workflows/js-tests.yml index 650ce9efb422..179b376b6855 100644 --- a/.github/workflows/js-tests.yml +++ b/.github/workflows/js-tests.yml @@ -14,7 +14,7 @@ jobs: matrix: os: [ ubuntu-20.04 ] node-version: [ 18 ] - python-version: [ 3.8 ] + python-version: [ '3.8', '3.11' ] steps: diff --git a/.github/workflows/lint-imports.yml b/.github/workflows/lint-imports.yml index 3bc7d2bf9d56..243b44d2039d 100644 --- a/.github/workflows/lint-imports.yml +++ b/.github/workflows/lint-imports.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: '3.8' + python-version: '3.11' - name: Install system requirements run: sudo apt update && sudo apt install -y libxmlsec1-dev diff --git a/.github/workflows/migrations-check.yml b/.github/workflows/migrations-check.yml index 68023a6f4b09..417371c64626 100644 --- a/.github/workflows/migrations-check.yml +++ b/.github/workflows/migrations-check.yml @@ -14,10 +14,10 @@ jobs: strategy: matrix: os: [ ubuntu-20.04 ] - python-version: [ 3.8 ] + python-version: [ '3.8', '3.11' ] # 'pinned' is used to install the latest patch version of Django # within the global constraint i.e. Django==4.2.8 in current case - # because we have global constraint of Django<4.2 + # because we have global constraint of Django<4.2 django-version: ["pinned"] mongo-version: ["4", "7"] mysql-version: ["8"] @@ -115,7 +115,7 @@ jobs: ./manage.py lms migrate echo "Running the CMS migrations." ./manage.py cms migrate - + # This job aggregates test results. It's the required check for branch protection. # https://github.com/marketplace/actions/alls-green#why # https://github.com/orgs/community/discussions/33579 diff --git a/.github/workflows/pylint-checks.yml b/.github/workflows/pylint-checks.yml index 840dc985e3c2..c2e04fc191d3 100644 --- a/.github/workflows/pylint-checks.yml +++ b/.github/workflows/pylint-checks.yml @@ -39,7 +39,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: 3.8 + python-version: 3.11 - name: Get pip cache dir id: pip-cache-dir diff --git a/.github/workflows/quality-checks.yml b/.github/workflows/quality-checks.yml index 8c8a248878e1..920659631958 100644 --- a/.github/workflows/quality-checks.yml +++ b/.github/workflows/quality-checks.yml @@ -14,7 +14,7 @@ jobs: strategy: matrix: os: [ ubuntu-20.04 ] - python-version: [ 3.8 ] + python-version: [ '3.8', '3.11' ] node-version: [ 18 ] steps: diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index f5ff2f63e5c8..870de732cf4f 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -18,7 +18,7 @@ jobs: strategy: matrix: os: [ "ubuntu-20.04" ] - python-version: [ "3.8" ] + python-version: [ "3.8", '3.11' ] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/static-assets-check.yml b/.github/workflows/static-assets-check.yml index 177d84af3d29..db5ead905a8d 100644 --- a/.github/workflows/static-assets-check.yml +++ b/.github/workflows/static-assets-check.yml @@ -13,7 +13,7 @@ jobs: strategy: matrix: os: [ ubuntu-20.04 ] - python-version: [ 3.8 ] + python-version: [ '3.8', '3.11' ] node-version: [ 18 ] npm-version: [ 10.5.x ] mongo-version: ["4.4", "7.0"] diff --git a/.github/workflows/unit-tests-gh-hosted.yml b/.github/workflows/unit-tests-gh-hosted.yml index 5f4e3c5a68cd..75b86d1dface 100644 --- a/.github/workflows/unit-tests-gh-hosted.yml +++ b/.github/workflows/unit-tests-gh-hosted.yml @@ -87,7 +87,7 @@ jobs: runs-on: ubuntu-20.04 strategy: matrix: - python-version: [ '3.8' ] + python-version: [ '3.11' ] django-version: - "pinned" steps: diff --git a/.github/workflows/units-test-scripts-structures-pruning.yml b/.github/workflows/units-test-scripts-structures-pruning.yml index 434b617c1736..9fab62643c3b 100644 --- a/.github/workflows/units-test-scripts-structures-pruning.yml +++ b/.github/workflows/units-test-scripts-structures-pruning.yml @@ -12,7 +12,7 @@ jobs: strategy: matrix: - python-version: [ '3.8', '3.12' ] + python-version: [ '3.8', '3.11', '3.12' ] steps: - name: Checkout code diff --git a/.github/workflows/units-test-scripts-user-retirement.yml b/.github/workflows/units-test-scripts-user-retirement.yml index e11d70193f0e..22bbdc3f617e 100644 --- a/.github/workflows/units-test-scripts-user-retirement.yml +++ b/.github/workflows/units-test-scripts-user-retirement.yml @@ -12,7 +12,7 @@ jobs: strategy: matrix: - python-version: [ '3.8' ] + python-version: [ '3.8', '3.11', '3.12'] steps: - name: Checkout code