Merge branch 'master' into unpinpylint

.annotation_safe_list.yml

@@ -142,8 +142,6 @@ workflow.AssessmentWorkflowStep:
 # Via edx-celeryutils
 celery_utils.ChordData:
   ".. no_pii:": "No PII"
-celery_utils.FailedTask:
-  ".. no_pii:": "No PII"
 
 # Via completion XBlock
 completion.BlockCompletion:

.eslintignore

@@ -20,9 +20,21 @@ test_root/staticfiles
 common/static/xmodule
 
 
-# Symlinks into xmodule/js
+# Various intra-repo symlinks that we've added over the years to duct-tape the JS build together.
+# Ignore them so that we're not double-counting these violations.
+cms/static/edx-ui-toolkit
 cms/static/xmodule_js
+lms/static/common
+lms/static/course_bookmarks
+lms/static/course_experience
+lms/static/course_search
+lms/static/discussion
+lms/static/edx-ui-toolkit
+lms/static/learner_profile
+lms/static/support
+lms/static/teams
 lms/static/xmodule_js
+xmodule/js/common_static
 
 
 # Mako templates that generate .js files

.github/CODEOWNERS

@@ -22,7 +22,7 @@ openedx/core/djangoapps/enrollments/                                @openedx/2U-
 openedx/core/djangoapps/heartbeat/
 openedx/core/djangoapps/oauth_dispatch
 openedx/core/djangoapps/user_api/                                   @openedx/2U-aperture
-openedx/core/djangoapps/user_authn/                                 @openedx/2U-vanguards                                 
+openedx/core/djangoapps/user_authn/                                 @openedx/2U-infinity
 openedx/core/djangoapps/verified_track_content/                     @openedx/2u-infinity
 openedx/features/course_experience/
 xmodule/

.github/renovate.json5

@@ -32,7 +32,11 @@
   // When adding an ignoreDep, please include a reason and a public link that we can use to follow up and ensure
   // that the ignoreDep is removed.
   // This can be done as a comment within the ignoreDeps list.
-  "ignoreDeps": [],
+  "ignoreDeps": [
+    // karma-spec-reporter>0.20.0 does not seem compatible with our super-old 2016 Karma version (0.13.22).
+    // Ticket link: None, as upgrading Karma does not strike as worth the benefit.
+    "karma-spec-reporter"
+  ],
   "timezone": "America/New_York",
   "prConcurrentLimit": 3,
   "enabledManagers": ["npm"]

.github/workflows/js-tests.yml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        node-version: [18, 20]
+        node-version: [20]
         python-version:
           - "3.11"
 
@@ -28,7 +28,7 @@ jobs:
           node-version: ${{ matrix.node-version }}
 
       - name: Setup npm
-        run: npm i -g npm@10.5.x
+        run: npm i -g npm@10.7.x
 
       - name: Install Firefox 123.0
         run: |
@@ -64,13 +64,13 @@ jobs:
           make base-requirements
 
       - uses: c-hive/gha-npm-cache@v1
+
+      - name: Install npm
+        run: npm ci  
+
       - name: Run JS Tests
-        env:
-          TEST_SUITE: js-unit
-          SCRIPT_TO_RUN: ./scripts/generic-ci-tests.sh
         run: |
-          npm install -g jest
-          xvfb-run --auto-servernum ./scripts/all-tests.sh
+          npm run test
 
       - name: Save Job Artifacts
         uses: actions/upload-artifact@v4

.github/workflows/quality-checks.yml

@@ -60,16 +60,29 @@ jobs:
           PIP_SRC: ${{ runner.temp }}
         run: |
           make test-requirements
-
+      
+      - name: Install npm
+        env:
+          PIP_SRC: ${{ runner.temp }}
+        run: npm ci  
+
+      - name: Install python packages
+        env:
+          PIP_SRC: ${{ runner.temp }}
+        run: |
+            pip install -e .
+      
       - name: Run Quality Tests
         env:
-          TEST_SUITE: quality
-          SCRIPT_TO_RUN: ./scripts/generic-ci-tests.sh
           PIP_SRC: ${{ runner.temp }}
           TARGET_BRANCH: ${{ github.base_ref }}
         run: |
-          ./scripts/all-tests.sh
-
+            make pycodestyle
+            npm run lint
+            make xsslint
+            make pii_check
+            make check_keywords
+          
       - name: Save Job Artifacts
         if: always()
         uses: actions/upload-artifact@v4

.github/workflows/static-assets-check.yml

@@ -15,8 +15,8 @@ jobs:
         os: [ubuntu-24.04]
         python-version:
           - "3.11"
-        node-version: [18, 20]
-        npm-version: [10.5.x]
+        node-version: [20]
+        npm-version: [10.7.x]
         mongo-version:
           - "7.0"
 

.github/workflows/unit-test-shards.json

@@ -239,7 +239,6 @@
       "cms/djangoapps/cms_user_tasks/",
       "cms/djangoapps/course_creators/",
       "cms/djangoapps/export_course_metadata/",
-      "cms/djangoapps/maintenance/",
       "cms/djangoapps/models/",
       "cms/djangoapps/pipeline_js/",
       "cms/djangoapps/xblock_config/",

.nvmrc

@@ -1 +1 @@
-18
+20

.pii_annotations.yml

@@ -1,7 +1,7 @@
 source_path: ./
 report_path: pii_report
 safelist_path: .annotation_safe_list.yml
-coverage_target: 94.5
+coverage_target: 85.3
 # See OEP-30 for more information on these values and what they mean:
 # https://open-edx-proposals.readthedocs.io/en/latest/oep-0030-arch-pii-markup-and-auditing.html#docstring-annotations
 annotations:

Dockerfile

@@ -48,36 +48,36 @@ RUN echo "locales locales/locales_to_be_generated multiselect en_US.UTF-8 UTF-8"
 
 # Setting up ppa deadsnakes to get python 3.11
 RUN apt-get update && \
-  apt-get install -y software-properties-common && \
-  apt-add-repository -y ppa:deadsnakes/ppa
+    apt-get install -y software-properties-common && \
+    apt-add-repository -y ppa:deadsnakes/ppa
 
 # Install requirements that are absolutely necessary
 RUN apt-get update && \
     apt-get -y dist-upgrade && \
     apt-get -y install --no-install-recommends \
-        python3-pip \
-        python3.11 \
-        # python3-dev: required for building mysqlclient python package
-        python3.11-dev \
-        python3.11-venv \
-        libpython3.11 \
-        libpython3.11-stdlib \
-        libmysqlclient21 \
-        # libmysqlclient-dev: required for building mysqlclient python package
-        libmysqlclient-dev \
-        pkg-config \
-        libssl1.1 \
-        libxmlsec1-openssl \
-        # lynx: Required by https://github.com/openedx/edx-platform/blob/b489a4ecb122/openedx/core/lib/html_to_text.py#L16
-        lynx \
-        ntp \
-        git \
-        build-essential \
-        gettext \
-        gfortran \
-        graphviz \
-        locales \
-        swig \
+    python3-pip \
+    python3.11 \
+    # python3-dev: required for building mysqlclient python package
+    python3.11-dev \
+    python3.11-venv \
+    libpython3.11 \
+    libpython3.11-stdlib \
+    libmysqlclient21 \
+    # libmysqlclient-dev: required for building mysqlclient python package
+    libmysqlclient-dev \
+    pkg-config \
+    libssl1.1 \
+    libxmlsec1-openssl \
+    # lynx: Required by https://github.com/openedx/edx-platform/blob/b489a4ecb122/openedx/core/lib/html_to_text.py#L16
+    lynx \
+    ntp \
+    git \
+    build-essential \
+    gettext \
+    gfortran \
+    graphviz \
+    locales \
+    swig \
     && \
     apt-get clean all && \
     rm -rf /var/lib/apt/*
@@ -93,19 +93,19 @@ FROM minimal-system as builder-production
 
 RUN apt-get update && \
     apt-get -y install --no-install-recommends \
-        curl \
-        libssl-dev \
-        libffi-dev \
-        libfreetype6-dev \
-        libgeos-dev \
-        libgraphviz-dev \
-        libjpeg8-dev \
-        liblapack-dev \
-        libpng-dev \
-        libsqlite3-dev \
-        libxml2-dev \
-        libxmlsec1-dev \
-        libxslt1-dev
+    curl \
+    libssl-dev \
+    libffi-dev \
+    libfreetype6-dev \
+    libgeos-dev \
+    libgraphviz-dev \
+    libjpeg8-dev \
+    liblapack-dev \
+    libpng-dev \
+    libsqlite3-dev \
+    libxml2-dev \
+    libxmlsec1-dev \
+    libxslt1-dev
 
 # Setup python virtual environment
 # It is already 'activated' because $VIRTUAL_ENV/bin was put on $PATH
@@ -118,8 +118,8 @@ RUN pip install -r requirements/pip.txt
 RUN pip install -r requirements/edx/base.txt
 
 # Install node and npm
-RUN nodeenv /edx/app/edxapp/nodeenv --node=18.19.0 --prebuilt
-RUN npm install -g npm@10.5.x
+RUN nodeenv /edx/app/edxapp/nodeenv --node=20.15.1 --prebuilt
+RUN npm install -g npm@10.7.x
 
 # This script is used by an npm post-install hook.
 # We copy it into the image now so that it will be available when we run `npm install` in the next step.
@@ -178,8 +178,8 @@ FROM base as development
 
 RUN apt-get update && \
     apt-get -y install --no-install-recommends \
-        # wget is used in Makefile for common_constraints.txt
-        wget \
+    # wget is used in Makefile for common_constraints.txt
+    wget \
     && \
     apt-get clean all && \
     rm -rf /var/lib/apt/*

Makefile

@@ -204,3 +204,37 @@ migrate: migrate-lms migrate-cms
 # Part of https://github.com/openedx/wg-developer-experience/issues/136
 ubuntu-requirements: ## Install ubuntu 22.04 system packages needed for `pip install` to work on ubuntu.
 	sudo apt install libmysqlclient-dev libxmlsec1-dev
+
+xsslint: ## check xss for quality issuest
+	python scripts/xsslint/xss_linter.py \
+	--rule-totals \
+	--config=scripts.xsslint_config \
+	--thresholds=scripts/xsslint_thresholds.json
+
+pycodestyle: ## check python files for quality issues 
+	pycodestyle .
+
+## Re-enable --lint flag when this issue https://github.com/openedx/edx-platform/issues/35775 is resolved
+pii_check: ## check django models for pii annotations
+	DJANGO_SETTINGS_MODULE=cms.envs.test \
+	code_annotations django_find_annotations \
+		--config_file .pii_annotations.yml \
+		--app_name cms \
+		--coverage \
+		--lint
+
+	DJANGO_SETTINGS_MODULE=lms.envs.test \
+	code_annotations django_find_annotations \
+		--config_file .pii_annotations.yml \
+		--app_name lms \
+		--coverage \
+		--lint	
+
+check_keywords: ## check django models for reserve keywords
+	DJANGO_SETTINGS_MODULE=cms.envs.test \
+	python manage.py cms check_reserved_keywords \
+	--override_file db_keyword_overrides.yml
+
+	DJANGO_SETTINGS_MODULE=lms.envs.test \
+	python manage.py lms check_reserved_keywords \
+	--override_file db_keyword_overrides.yml

README.rst

@@ -71,10 +71,10 @@ System Dependencies
 -------------------
 
 OS:
-* Ubuntu 20.04
-
 * Ubuntu 22.04
 
+* Ubuntu 24.04
+
 Interperters/Tools:
 
 * Python 3.11
@@ -107,6 +107,15 @@ Language Packages:
 
   Some Python packages have system dependencies. For example, installing these packages on Debian or Ubuntu will require first running ``sudo apt install python3-dev default-libmysqlclient-dev build-essential pkg-config`` to satisfy the requirements of the ``mysqlclient`` Python package.
 
+Codejail Setup
+--------------
+
+As a part of the baremetal setup, you will need to configure your system to
+work properly with codejail.  See the `codejail installation steps`_ for more
+details.
+
+.. _codejail installation steps: https://github.com/openedx/codejail?tab=readme-ov-file#installation
+
 Build Steps
 -----------
 

cms/djangoapps/contentstore/helpers.py

@@ -7,6 +7,7 @@
 import urllib
 from lxml import etree
 from mimetypes import guess_type
+import re
 
 from attrs import frozen, Factory
 from django.conf import settings
@@ -447,6 +448,10 @@ def _import_xml_node_to_parent(
         temp_xblock = xblock_class.parse_xml(node_without_children, runtime, keys)
         child_nodes = list(node)
 
+    if issubclass(xblock_class, XmlMixin) and "x-is-pointer-node" in getattr(temp_xblock, "data", ""):
+        # Undo the "pointer node" hack if needed (e.g. for capa problems)
+        temp_xblock.data = re.sub(r'([^>]+) x-is-pointer-node="no"', r'\1', temp_xblock.data, count=1)
+
     # Restore the original id_generator
     runtime.id_generator = original_id_generator
 

cms/djangoapps/contentstore/rest_api/v1/serializers/course_waffle_flags.py

@@ -27,6 +27,7 @@ class CourseWaffleFlagsSerializer(serializers.Serializer):
     use_new_certificates_page = serializers.SerializerMethodField()
     use_new_textbooks_page = serializers.SerializerMethodField()
     use_new_group_configurations_page = serializers.SerializerMethodField()
+    enable_course_optimizer = serializers.SerializerMethodField()
 
     def get_course_key(self):
         """
@@ -144,3 +145,10 @@ def get_use_new_group_configurations_page(self, obj):
         """
         course_key = self.get_course_key()
         return toggles.use_new_group_configurations_page(course_key)
+
+    def get_enable_course_optimizer(self, obj):
+        """
+        Method to get the enable_course_optimizer waffle flag
+        """
+        course_key = self.get_course_key()
+        return toggles.enable_course_optimizer(course_key)

cms/djangoapps/contentstore/rest_api/v1/serializers/home.py

@@ -66,6 +66,7 @@ class StudioHomeSerializer(serializers.Serializer):
     request_course_creator_url = serializers.CharField()
     rerun_creator_status = serializers.BooleanField()
     show_new_library_button = serializers.BooleanField()
+    show_new_library_v2_button = serializers.BooleanField()
     split_studio_home = serializers.BooleanField()
     studio_name = serializers.CharField()
     studio_short_name = serializers.CharField()