From 1fc2e8a771a85db9b9a7e0b8c2da925bf81ed5bb Mon Sep 17 00:00:00 2001 From: Muhammad Soban Javed <58461728+iamsobanjaved@users.noreply.github.com> Date: Thu, 1 Feb 2024 16:10:34 +0500 Subject: [PATCH] feat!: upgrade Django version to 4.2 (LTS) (#34162) * feat!: upgrade Django version to 4.2 (LTS) --------- Co-authored-by: iamsobanjaved --- .github/workflows/migrations-check.yml | 11 ++----- .github/workflows/unit-tests.yml | 1 - Makefile | 2 ++ cms/envs/common.py | 4 +-- lms/envs/common.py | 4 +-- .../tests/test_safe_cookie_data.py | 2 +- openedx/core/djangoapps/theming/storage.py | 32 +++++++++++++------ requirements/common_constraints.txt | 2 +- requirements/constraints.txt | 5 +++ requirements/edx/base.txt | 9 +++--- requirements/edx/development.txt | 9 +++--- requirements/edx/doc.txt | 9 +++--- requirements/edx/testing.txt | 9 +++--- 13 files changed, 58 insertions(+), 41 deletions(-) diff --git a/.github/workflows/migrations-check.yml b/.github/workflows/migrations-check.yml index 3929e7f10e6f..fb66cfe775cb 100644 --- a/.github/workflows/migrations-check.yml +++ b/.github/workflows/migrations-check.yml @@ -16,16 +16,11 @@ jobs: os: [ ubuntu-20.04 ] python-version: [ 3.8 ] # 'pinned' is used to install the latest patch version of Django - # within the global constraint i.e. Django==3.2.21 in current case + # within the global constraint i.e. Django==4.2.8 in current case # because we have global constraint of Django<4.2 - django-version: ["pinned", "4.2"] + django-version: ["pinned"] mongo-version: ["4"] - mysql-version: ["5.7", "8"] - # excluding mysql5.7 with Django 4.2 since Django 4.2 has - # dropped support for MySQL<8 - exclude: - - django-version: "4.2" - mysql-version: "5.7" + mysql-version: ["8"] services: mongo: image: mongo:${{ matrix.mongo-version }} diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index c3b1086c20eb..be7e122f2fd8 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -17,7 +17,6 @@ jobs: - "3.8" django-version: - "pinned" - - "4.2" # When updating the shards, remember to make the same changes in # .github/workflows/unit-tests-gh-hosted.yml shard_name: diff --git a/Makefile b/Makefile index cc6fa5591376..92c89d844aa8 100644 --- a/Makefile +++ b/Makefile @@ -150,6 +150,8 @@ compile-requirements: pre-requirements $(COMMON_CONSTRAINTS_TXT) ## Re-compile * @# time someone tries to use the outputs. sed '/^django-simple-history==/d' requirements/common_constraints.txt > requirements/common_constraints.tmp mv requirements/common_constraints.tmp requirements/common_constraints.txt + sed 's/Django<4.0//g' requirements/common_constraints.txt > requirements/common_constraints.tmp + mv requirements/common_constraints.tmp requirements/common_constraints.txt pip-compile -v --allow-unsafe ${COMPILE_OPTS} -o requirements/pip.txt requirements/pip.in pip install -r requirements/pip.txt diff --git a/cms/envs/common.py b/cms/envs/common.py index 4ded7a9cff34..8f974e73e521 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -1105,8 +1105,8 @@ } DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' -# This will be overridden through CMS config -DEFAULT_HASHING_ALGORITHM = 'sha1' +DEFAULT_HASHING_ALGORITHM = 'sha256' + #################### Python sandbox ############################################ CODE_JAIL = { diff --git a/lms/envs/common.py b/lms/envs/common.py index 3da23a4ad84e..f14ec3962783 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -1768,8 +1768,8 @@ def _make_mako_template_dirs(settings): DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' -# This will be overridden through LMS config -DEFAULT_HASHING_ALGORITHM = 'sha1' +DEFAULT_HASHING_ALGORITHM = 'sha256' + #################### Python sandbox ############################################ CODE_JAIL = { diff --git a/openedx/core/djangoapps/safe_sessions/tests/test_safe_cookie_data.py b/openedx/core/djangoapps/safe_sessions/tests/test_safe_cookie_data.py index 47efa626ec99..650a8d157baf 100644 --- a/openedx/core/djangoapps/safe_sessions/tests/test_safe_cookie_data.py +++ b/openedx/core/djangoapps/safe_sessions/tests/test_safe_cookie_data.py @@ -236,7 +236,7 @@ def test_pinned_values(self): "|HvGnjXf1b3jU" "|ImExZWZiNzVlZGFmM2FkZWZmYjM4YjI0ZmZkOWU4MzExODU0MTk4NmVlNGRiYzBlODdhYWUzOGM5MzVlNzk4NjUi" ":1m6Hve" - ":OMhY2FL2pudJjSSXChtI-zR8QVA" + ":Pra4iochviPvKUoIV33gdVZFDgG-cMDlIYfl8iFIMaY" ) @pytest.mark.skipif(django.VERSION[0] < 4, reason="For django42 default algorithm is sha256. No need for django32.") diff --git a/openedx/core/djangoapps/theming/storage.py b/openedx/core/djangoapps/theming/storage.py index e8e3eefcd3d7..4a48cbb6e0a5 100644 --- a/openedx/core/djangoapps/theming/storage.py +++ b/openedx/core/djangoapps/theming/storage.py @@ -192,33 +192,41 @@ def converter(matchobj): This requires figuring out which files the matched URL resolves to and calling the url() method of the storage. """ - matched, url = matchobj.groups() + matches = matchobj.groupdict() + matched = matches["matched"] + url = matches["url"] # Ignore absolute/protocol-relative and data-uri URLs. - if re.match(r'^[a-z]+:', url): + if re.match(r"^[a-z]+:", url): return matched # Ignore absolute URLs that don't point to a static file (dynamic # CSS / JS?). Note that STATIC_URL cannot be empty. - if url.startswith('/') and not url.startswith(settings.STATIC_URL): + if url.startswith("/") and not url.startswith(settings.STATIC_URL): return matched # Strip off the fragment so a path-like fragment won't interfere. url_path, fragment = urldefrag(url) - if url_path.startswith('/'): + # Ignore URLs without a path + if not url_path: + return matched + + if url_path.startswith("/"): # Otherwise the condition above would have returned prematurely. assert url_path.startswith(settings.STATIC_URL) target_name = url_path[len(settings.STATIC_URL):] else: # We're using the posixpath module to mix paths and URLs conveniently. - source_name = name if os.sep == '/' else name.replace(os.sep, '/') + source_name = name if os.sep == "/" else name.replace(os.sep, "/") target_name = posixpath.join(posixpath.dirname(source_name), url_path) # Determine the hashed name of the target file with the storage backend. hashed_url = self._url( - self._stored_name, unquote(target_name), - force=True, hashed_files=hashed_files, + self._stored_name, + unquote(target_name), + force=True, + hashed_files=hashed_files, ) # NOTE: @@ -228,15 +236,19 @@ def converter(matchobj): # The line is commented and not removed to make future django upgrade easier and show exactly what is # changed in this method override # - #transformed_url = '/'.join(url_path.split('/')[:-1] + hashed_url.split('/')[-1:]) + # transformed_url = "/".join( + # url_path.split("/")[:-1] + hashed_url.split("/")[-1:] + # ) + transformed_url = hashed_url # This line was added. # Restore the fragment that was stripped off earlier. if fragment: - transformed_url += ('?#' if '?#' in url else '#') + fragment + transformed_url += ("?#" if "?#" in url else "#") + fragment # Return the hashed version to the file - return template % unquote(transformed_url) + matches["url"] = unquote(transformed_url) + return template % matches return converter diff --git a/requirements/common_constraints.txt b/requirements/common_constraints.txt index 53dff4a22d75..7313473a1675 100644 --- a/requirements/common_constraints.txt +++ b/requirements/common_constraints.txt @@ -17,7 +17,7 @@ # using LTS django version -Django<4.0 + # elasticsearch>=7.14.0 includes breaking changes in it which caused issues in discovery upgrade process. # elastic search changelog: https://www.elastic.co/guide/en/enterprise-search/master/release-notes-7.14.0.html diff --git a/requirements/constraints.txt b/requirements/constraints.txt index ebf7f376defc..55ce227b2160 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -25,10 +25,15 @@ click>=8.0,<9.0 # for them. edx-enterprise==4.11.3 +# Stay on LTS version, remove once this is added to common constraint +Django<5.0 + # django-oauth-toolkit version >=2.0.0 has breaking changes. More details # mentioned on this issue https://github.com/openedx/edx-platform/issues/32884 django-oauth-toolkit==1.7.1 +# incremental upgrade +django-simple-history==3.4.0 # constrained in opaque_keys. migration guide here: https://pymongo.readthedocs.io/en/4.0/migrate-to-pymongo4.html # Major upgrade will be done in separate ticket. diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 8a8980796e20..5ebfd2070345 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -56,6 +56,7 @@ backoff==1.10.0 backports-zoneinfo[tzdata]==0.2.1 # via # celery + # django # icalendar # kombu beautifulsoup4==4.12.3 @@ -173,9 +174,9 @@ defusedxml==0.7.1 # social-auth-core deprecated==1.2.14 # via jwcrypto -django==3.2.23 +django==4.2.9 # via - # -c requirements/edx/../common_constraints.txt + # -c requirements/edx/../constraints.txt # -r requirements/edx/kernel.in # django-appconf # django-celery-results @@ -338,6 +339,7 @@ django-ses==3.5.2 # via -r requirements/edx/bundled.in django-simple-history==3.4.0 # via + # -c requirements/edx/../constraints.txt # -r requirements/edx/kernel.in # edx-enterprise # edx-name-affirmation @@ -431,7 +433,7 @@ edx-ccx-keys==1.2.1 # via # -r requirements/edx/kernel.in # lti-consumer-xblock -edx-celeryutils==1.2.3 +edx-celeryutils==1.2.5 # via # -r requirements/edx/kernel.in # edx-name-affirmation @@ -940,7 +942,6 @@ pytz==2023.4 # via # -r requirements/edx/kernel.in # babel - # django # django-ses # djangorestframework # drf-yasg diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 7f56649bc440..150ef1c2aaf4 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -116,6 +116,7 @@ backports-zoneinfo[tzdata]==0.2.1 # -r requirements/edx/testing.txt # backports-zoneinfo # celery + # django # icalendar # kombu beautifulsoup4==4.12.3 @@ -341,9 +342,9 @@ distlib==0.3.8 # via # -r requirements/edx/testing.txt # virtualenv -django==3.2.23 +django==4.2.9 # via - # -c requirements/edx/../common_constraints.txt + # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # django-appconf @@ -559,6 +560,7 @@ django-ses==3.5.2 # -r requirements/edx/testing.txt django-simple-history==3.4.0 # via + # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edx-enterprise @@ -701,7 +703,7 @@ edx-ccx-keys==1.2.1 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # lti-consumer-xblock -edx-celeryutils==1.2.3 +edx-celeryutils==1.2.5 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1670,7 +1672,6 @@ pytz==2023.4 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # babel - # django # django-ses # djangorestframework # drf-yasg diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 7a3fc1bb9c35..44655affcfdc 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -79,6 +79,7 @@ backports-zoneinfo[tzdata]==0.2.1 # -r requirements/edx/base.txt # backports-zoneinfo # celery + # django # icalendar # kombu beautifulsoup4==4.12.3 @@ -223,9 +224,9 @@ deprecated==1.2.14 # via # -r requirements/edx/base.txt # jwcrypto -django==3.2.23 +django==4.2.9 # via - # -c requirements/edx/../common_constraints.txt + # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # django-appconf # django-celery-results @@ -404,6 +405,7 @@ django-ses==3.5.2 # via -r requirements/edx/base.txt django-simple-history==3.4.0 # via + # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # edx-enterprise # edx-name-affirmation @@ -509,7 +511,7 @@ edx-ccx-keys==1.2.1 # via # -r requirements/edx/base.txt # lti-consumer-xblock -edx-celeryutils==1.2.3 +edx-celeryutils==1.2.5 # via # -r requirements/edx/base.txt # edx-name-affirmation @@ -1123,7 +1125,6 @@ pytz==2023.4 # via # -r requirements/edx/base.txt # babel - # django # django-ses # djangorestframework # drf-yasg diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 0e993cd22058..8f1152d55344 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -81,6 +81,7 @@ backports-zoneinfo[tzdata]==0.2.1 # -r requirements/edx/base.txt # backports-zoneinfo # celery + # django # icalendar # kombu beautifulsoup4==4.12.3 @@ -254,9 +255,9 @@ dill==0.3.8 # via pylint distlib==0.3.8 # via virtualenv -django==3.2.23 +django==4.2.9 # via - # -c requirements/edx/../common_constraints.txt + # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # django-appconf # django-celery-results @@ -435,6 +436,7 @@ django-ses==3.5.2 # via -r requirements/edx/base.txt django-simple-history==3.4.0 # via + # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # edx-enterprise # edx-name-affirmation @@ -535,7 +537,7 @@ edx-ccx-keys==1.2.1 # via # -r requirements/edx/base.txt # lti-consumer-xblock -edx-celeryutils==1.2.3 +edx-celeryutils==1.2.5 # via # -r requirements/edx/base.txt # edx-name-affirmation @@ -1251,7 +1253,6 @@ pytz==2023.4 # via # -r requirements/edx/base.txt # babel - # django # django-ses # djangorestframework # drf-yasg