feat!: Django 4 (#2379)

* feat!: Upgrade to Django 4.2

* Comment out common

* Fix a whole bunch of stuff

* Temporarily disable captcha fields

* Install RR branch

* Clean dependencies

* Fix some 2fa stuff

* Fix cypress test

* Update two factor elements

* Bring back custom views

* Upgrade recaptcha extended templates

* Clean

* Redirect to teacher login if trying to 2FA

* Install actual RR

Pipfile

@@ -9,18 +9,18 @@ codeforlife-portal = {path = ".", editable = true}
 
 [dev-packages]
 black = "*"
-django-import-export = "*"
-django-selenium-clean = "==1.0.0"
-django-test-migrations = "==1.2.0"
+django-selenium-clean = "==1.0.1"
+django-test-migrations = "==1.4.0"
 isort = "*"
 PyPDF2 = "==2.10.6"
 pytest = "==8.*"
 pytest-cov = "*"
-pytest-django = "==4.5.2"
+pytest-django = "==4.8.0"
 pytest-mock = "*"
 pytest-order = "*"
 pytest-xdist = "*"
 pyvirtualdisplay = "*"
+rapid-router = ">=6.3.6"
 responses = "==0.18.0"
 selenium = "==4.9.0"
 snapshottest = "==1.0.0a1"

cfl_common/setup.py

@@ -17,12 +17,17 @@
     version=version,
     include_package_data=True,
     install_requires=[
-        "django==3.2.25",
-        "djangorestframework==3.13.1",
-        "django-two-factor-auth==1.13.2",
-        "django-countries==7.3.1",
-        "pyjwt==2.6.0",
+        "django==4.2.16",
+        "django-countries==7.6.1",
+        "django-csp==3.8",
+        "django-import-export==4.2.0",
+        "django-pipeline==3.1.0",
+        "django-two-factor-auth==1.17.0",
+        "djangorestframework==3.15.1",
+        "libsass==0.23.0",
+        "more-itertools==8.7.0",
         "pgeocode==0.4.0",
+        "pyjwt==2.6.0",
     ],
     tests_require=[],
     test_suite="tests",

deploy/middleware/security.py

@@ -4,19 +4,18 @@
 class CustomSecurityMiddleware(SecurityMiddleware):
     """
     Extends Django's Security Middleware.
-    See https://docs.djangoproject.com/en/3.2/_modules/django/middleware/security/ for
-    the source code, as well as https://docs.djangoproject.com/en/3.2/ref/middleware/#module-django.middleware.security
+    See https://docs.djangoproject.com/en/4.2/_modules/django/middleware/security/ for
+    the source code, as well as https://docs.djangoproject.com/en/4.2/ref/middleware/#module-django.middleware.security
     for docs on security middleware.
     """
 
     def process_response(self, request, response):
         """
         Extends the original security middleware to ensure the X-XSS-Protection header
         is set to 1.
+        https://docs.djangoproject.com/en/5.1/releases/4.0/#securitymiddleware-no-longer-sets-the-x-xss-protection-header
         """
         super().process_response(request, response)
-
-        if self.xss_filter:
-            response["X-XSS-Protection"] = "1"
+        response.headers.setdefault("X-XSS-Protection", "1; mode=block")
 
         return response

example_project/portal_test_settings.py

@@ -86,7 +86,7 @@
     "game",
     "pipeline",
     "portal",
-    "captcha",
+    "django_recaptcha",
     "common",
     "django.contrib.admin",
     "django.contrib.admindocs",

example_project/settings.py

@@ -36,13 +36,13 @@
 
 EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend"
 
-SILENCED_SYSTEM_CHECKS = ["captcha.recaptcha_test_key_error"]
+SILENCED_SYSTEM_CHECKS = ["django_recaptcha.recaptcha_test_key_error"]
 
 INSTALLED_APPS = [
     "game",
     "pipeline",
     "portal",
-    "captcha",
+    "django_recaptcha",
     "common",
     "django.contrib.admin",
     "django.contrib.admindocs",

example_project/urls.py

@@ -1,6 +1,5 @@
-from django.conf.urls import include, url
 from django.contrib import admin
-from django.urls import path
+from django.urls import include, path, re_path
 from game import python_den_urls
 from game import urls as game_urls
 
@@ -9,8 +8,8 @@
 admin.autodiscover()
 
 urlpatterns = [
-    url(r"^", include(portal_urls)),
+    re_path(r"^", include(portal_urls)),
     path("administration/", admin.site.urls),
-    url(r"^rapidrouter/", include(game_urls)),
-    url(r"^pythonden/", include(python_den_urls)),
+    re_path(r"^rapidrouter/", include(game_urls)),
+    re_path(r"^pythonden/", include(python_den_urls)),
 ]

portal/admin.py

@@ -12,7 +12,7 @@
     UserProfile,
 )
 from django.contrib import admin
-from django.contrib.auth.admin import UserAdmin
+from django.contrib.auth.admin import UserAdmin as _UserAdmin
 from django.contrib.auth.models import User
 from django.http import HttpResponse
 from import_export.admin import ExportActionMixin
@@ -172,38 +172,40 @@ def has_delete_permission(self, request, obj=None):
         return False
 
 
-def anonymise_user(user_admin, request, queryset):
-    for user in queryset:
-        anonymise(user)
+_UserAdmin.list_display += ("date_joined", "id")
+_UserAdmin.list_filter += ("date_joined",)
 
 
-def export_as_csv(self, request, queryset):
-    meta = self.model._meta
-    field_names = [
-        field.name for field in meta.fields if field.name != "password"
-    ]
+class UserAdmin(_UserAdmin):
+    actions = ["anonymise_user", "export_as_csv"]
+    add_form = AdminUserCreationForm
+    change_password_form = AdminChangeUserPasswordForm
 
-    response = HttpResponse(content_type="text/csv")
-    response["Content-Disposition"] = "attachment; filename={}.csv".format(meta)
-    writer = csv.writer(response)
+    def anonymise_user(self, request, queryset):
+        for user in queryset:
+            anonymise(user)
 
-    writer.writerow(field_names)
-    for obj in queryset:
-        writer.writerow([getattr(obj, field) for field in field_names])
+    anonymise_user.short_description = "Anonymise selected users"
 
-    return response
+    def export_as_csv(self, request, queryset):
+        meta = self.model._meta
+        field_names = [
+            field.name for field in meta.fields if field.name != "password"
+        ]
 
+        response = HttpResponse(content_type="text/csv")
+        response["Content-Disposition"] = "attachment; filename={}.csv".format(
+            meta
+        )
+        writer = csv.writer(response)
 
-anonymise_user.short_description = "Anonymise selected users"
-export_as_csv.short_description = "Export selected users data as CSV"
+        writer.writerow(field_names)
+        for obj in queryset:
+            writer.writerow([getattr(obj, field) for field in field_names])
 
+        return response
 
-UserAdmin.list_display += ("date_joined", "id")
-UserAdmin.list_filter += ("date_joined",)
-UserAdmin.add_form = AdminUserCreationForm
-UserAdmin.change_password_form = AdminChangeUserPasswordForm
-UserAdmin.actions.append(anonymise_user)
-UserAdmin.actions.append(export_as_csv)
+    export_as_csv.short_description = "Export selected users data as CSV"
 
 
 admin.site.register(Class, ClassAdmin)

portal/forms/play.py

@@ -1,15 +1,15 @@
 import re
 from datetime import timedelta, date
 
-from captcha.fields import ReCaptchaField
-from captcha.widgets import ReCaptchaV2Invisible
 from common.helpers.emails import send_verification_email
 from common.models import Class, Student, stripStudentName
 from common.permissions import logged_in_as_independent_student
 from django import forms
 from django.contrib.auth import authenticate
 from django.contrib.auth.forms import AuthenticationForm
 from django.utils import timezone
+from django_recaptcha.fields import ReCaptchaField
+from django_recaptcha.widgets import ReCaptchaV2Invisible
 
 from portal.forms.error_messages import INVALID_LOGIN_MESSAGE
 from portal.helpers.password import PasswordStrength, form_clean_password

portal/forms/registration.py

@@ -1,5 +1,3 @@
-from captcha.fields import ReCaptchaField
-from captcha.widgets import ReCaptchaV2Invisible
 from common.mail import campaign_ids, send_dotdigital_email
 from common.models import Student, Teacher
 from django import forms
@@ -10,6 +8,8 @@
 from django.urls import reverse_lazy
 from django.utils.encoding import force_bytes
 from django.utils.http import urlsafe_base64_encode
+from django_recaptcha.fields import ReCaptchaField
+from django_recaptcha.widgets import ReCaptchaV2Invisible
 
 from portal.helpers.password import PasswordStrength, form_clean_password
 

portal/forms/teach.py

@@ -2,14 +2,14 @@
 import re
 from builtins import map, range, str
 
-from captcha.fields import ReCaptchaField
-from captcha.widgets import ReCaptchaV2Invisible
 from common.helpers.emails import send_verification_email
 from common.models import Student, stripStudentName, UserSession, Teacher
 from django import forms
 from django.contrib.auth import authenticate
 from django.contrib.auth.forms import AuthenticationForm
 from django.contrib.auth.models import User
+from django_recaptcha.fields import ReCaptchaField
+from django_recaptcha.widgets import ReCaptchaV2Invisible
 from game.models import Episode
 
 from portal.forms.error_messages import INVALID_LOGIN_MESSAGE

portal/helpers/decorators.py

@@ -1,28 +1,24 @@
 from __future__ import absolute_import
 
 import datetime
-import pytz
-import re
-
+from functools import wraps
 
+import pytz
 from common.models import Teacher, Student
 from django.contrib.auth import logout
 from django.shortcuts import render
-from functools import wraps
 from ratelimit import ALL, UNSAFE
 
 from portal.helpers.ratelimit import is_ratelimited
-from portal.templatetags.app_tags import is_logged_in
-from portal.views.registration import blocked_and_not_expired
-from portal.views.login import has_user_lockout_expired
-from portal.helpers.ratelimit import get_ratelimit_count_for_user
-
 from portal.helpers.request_handlers import get_access_code_from_request
+from portal.templatetags.app_tags import is_logged_in
 
 __all__ = ["ratelimit"]
 
 
-def ratelimit(group=None, key=None, rate=None, method=ALL, block=False, is_teacher=True):
+def ratelimit(
+    group=None, key=None, rate=None, method=ALL, block=False, is_teacher=True
+):
     """
     Ratelimit decorator, adding custom functionality to django-ratelimit's default
     decorator. On block, the user is logged out, redirected to the "locked out" page,
@@ -63,25 +59,29 @@ def _wrapped(request, *args, **kw):
 
                     access_code = get_access_code_from_request(request)
                     model_finder = model.objects.get
-                    # look for school student
-                    # if access code not found (AttributeError)
-                    # if student not found (IndexError)
-                    # move on to another try block
-                    # similar logic followed afterwards
+                    # look for school student if access code not found
+                    # (AttributeError) if student not found (IndexError) move
+                    # on to another try block similar logic followed afterwards
                     if access_code:
                         user_to_lockout = model_finder(
                             new_user__first_name=username,
                             class_field__access_code=access_code,  # extract the found text from regex
                         )
-                        lockout_template = "portal/locked_out_school_student.html"
+                        lockout_template = (
+                            "portal/locked_out_school_student.html"
+                        )
                     # look for indy student or teacher
                     else:
-                        user_to_lockout = model_finder(new_user__username=username)
+                        user_to_lockout = model_finder(
+                            new_user__username=username
+                        )
                 else:
                     user_to_lockout = model.objects.get(new_user=request.user)
 
                 if user_to_lockout:
-                    user_to_lockout.blocked_time = datetime.datetime.now(tz=pytz.utc)
+                    user_to_lockout.blocked_time = datetime.datetime.now(
+                        tz=pytz.utc
+                    )
                     user_to_lockout.save()
 
                     if is_logged_in(request.user):

portal/templates/captcha/includes/js_v2_invisible.html → portal/templates/django_recaptcha/includes/js_v2_invisible.html

@@ -1,8 +1,8 @@
-{# This is adapted from django-recaptcha 2.0.5 to work on multiple reCAPTCHA's on the same page #}
-{# See verifyCaptcha_{{ widget_uuid}} for the edit #}
+<!-- This is adapted from django-recaptcha 4.0.0 to work on multiple
+reCAPTCHA's on the same page
+See verifyCaptcha_{{ widget_uuid}} for the edit #} -->
 <script src="https://{{ recaptcha_domain }}/recaptcha/api.js{% if api_params %}?{{ api_params }}{% endif %}"></script>
 <script type="text/javascript">
-
     // Submit function to be called, after reCAPTCHA was successful.
     var onSubmit_{{ widget_uuid }} = function(token) {
         console.log("reCAPTCHA validated for 'data-widget-uuid=\"{{ widget_uuid }}\"'. Submitting form...")

portal/templates/captcha/widget_v2_invisible.html → portal/templates/django_recaptcha/widget_v2_invisible.html

@@ -1,5 +1,5 @@
-{# copied from django-recaptcha 2.0.5 #}
-{% include "captcha/includes/js_v2_invisible.html" %}
+<!-- copied from django-recaptcha 4.0.0 -->
+{% include "django_recaptcha/includes/js_v2_invisible.html" %}
 <div class="g-recaptcha"
     {% for name, value in widget.attrs.items %}{% if value is not False %} {{ name }}{% if value is not True %}="{{ value|stringformat:'s' }}"{% endif %}{% endif %}{% endfor %}
 >