% for config in all_group_configurations: -
+

${config['name']}

${_("Loading")}

diff --git a/lms/djangoapps/teams/team_partition_scheme.py b/lms/djangoapps/teams/team_partition_scheme.py index d573bf51a2d2..8b58997caa57 100644 --- a/lms/djangoapps/teams/team_partition_scheme.py +++ b/lms/djangoapps/teams/team_partition_scheme.py @@ -64,6 +64,8 @@ class TeamPartitionScheme: - A user is assigned to a group if they are a member of the team. """ + read_only = True + @classmethod def get_group_for_user(cls, course_key, user, user_partition): """Get the (Content) Group from the specified user partition for the user. From 4c0284b87dbca6363040a18fd5fa67297df85afa Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Mon, 6 May 2024 12:57:51 -0400 Subject: [PATCH 031/113] Revert "build: finish replacing paver assets (#34554)" (#34700) Reverts #34554, which causes compilation of edX.org's legacy comprehensive theme to be skipped in their deployment pipeline. We have not determined the precise cause yet, but it seems like the compile_sass management command is not correctly getting the list of comprehensive theme directories from Django settings. --- cms/envs/common.py | 37 +- .../0017-reimplement-asset-processing.rst | 52 ++- lms/envs/common.py | 19 +- .../management/commands/compile_sass.py | 124 +++--- .../djangoapps/theming/tests/test_commands.py | 56 ++- pavelib/assets.py | 321 ++++++++------- pavelib/paver_tests/test_assets.py | 387 +++++++++++++----- scripts/compile_sass.py | 8 +- scripts/watch_sass.sh | 4 +- 9 files changed, 633 insertions(+), 375 deletions(-) diff --git a/cms/envs/common.py b/cms/envs/common.py index 6aa2dfb835b1..39d5c7d13251 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -39,7 +39,6 @@ # pylint: disable=unused-import, useless-suppression, wrong-import-order, wrong-import-position import importlib.util -import json import os import sys @@ -1276,11 +1275,14 @@ # Static content STATIC_URL = '/static/studio/' -STATIC_ROOT = os.environ.get('STATIC_ROOT_CMS', ENV_ROOT / 'staticfiles' / 'studio') +STATIC_ROOT = ENV_ROOT / "staticfiles" / 'studio' STATICFILES_DIRS = [ COMMON_ROOT / "static", PROJECT_ROOT / "static", + + # This is how you would use the textbook images locally + # ("book", ENV_ROOT / "book_images"), ] # Locale/Internationalization @@ -1320,16 +1322,11 @@ EMBARGO_SITE_REDIRECT_URL = None ##### custom vendor plugin variables ##### - -# .. setting_name: JS_ENV_EXTRA_CONFIG -# .. setting_default: {} -# .. setting_description: JavaScript code can access this dictionary using `process.env.JS_ENV_EXTRA_CONFIG` -# One of the current use cases for this is enabling custom TinyMCE plugins -# (TINYMCE_ADDITIONAL_PLUGINS) and overriding the TinyMCE configuration (TINYMCE_CONFIG_OVERRIDES). -# .. setting_warning: This Django setting is DEPRECATED! Starting in Sumac, Webpack will no longer -# use Django settings. Please set the JS_ENV_EXTRA_CONFIG environment variable to an equivalent JSON -# string instead. For details, see: https://github.com/openedx/edx-platform/issues/31895 -JS_ENV_EXTRA_CONFIG = json.loads(os.environ.get('JS_ENV_EXTRA_CONFIG', '{}')) +# JavaScript code can access this data using `process.env.JS_ENV_EXTRA_CONFIG` +# One of the current use cases for this is enabling custom TinyMCE plugins +# (TINYMCE_ADDITIONAL_PLUGINS) and overriding the TinyMCE configuration +# (TINYMCE_CONFIG_OVERRIDES). +JS_ENV_EXTRA_CONFIG = {} ############################### PIPELINE ####################################### @@ -1506,14 +1503,7 @@ 'STATS_FILE': os.path.join(STATIC_ROOT, 'webpack-worker-stats.json') } } - -# .. setting_name: WEBPACK_CONFIG_PATH -# .. setting_default: "webpack.prod.config.js" -# .. setting_description: Path to the Webpack configuration file. Used by Paver scripts. -# .. setting_warning: This Django setting is DEPRECATED! Starting in Sumac, Webpack will no longer -# use Django settings. Please set the WEBPACK_CONFIG_PATH environment variable instead. For details, -# see: https://github.com/openedx/edx-platform/issues/31895 -WEBPACK_CONFIG_PATH = os.environ.get('WEBPACK_CONFIG_PATH', 'webpack.prod.config.js') +WEBPACK_CONFIG_PATH = 'webpack.prod.config.js' ############################ SERVICE_VARIANT ################################## @@ -2190,11 +2180,8 @@ # .. setting_name: COMPREHENSIVE_THEME_DIRS # .. setting_default: [] -# .. setting_description: A list of paths to directories, each of which will -# be searched for comprehensive themes. Do not override this Django setting directly. -# Instead, set the COMPREHENSIVE_THEME_DIRS environment variable, using colons (:) to -# separate paths. -COMPREHENSIVE_THEME_DIRS = os.environ.get("COMPREHENSIVE_THEME_DIRS", "").split(":") +# .. setting_description: See LMS annotation. +COMPREHENSIVE_THEME_DIRS = [] # .. setting_name: COMPREHENSIVE_THEME_LOCALE_PATHS # .. setting_default: [] diff --git a/docs/decisions/0017-reimplement-asset-processing.rst b/docs/decisions/0017-reimplement-asset-processing.rst index e689c34eb9cf..1048edf4c075 100644 --- a/docs/decisions/0017-reimplement-asset-processing.rst +++ b/docs/decisions/0017-reimplement-asset-processing.rst @@ -11,14 +11,16 @@ Overview Status ****** -**Accepted** +**Provisional** -This was `originally authored `_ in March 2023. We `modified it in July 2023 `_ based on learnings from the implementation process, and then `modified and it again in May 2024 `_ to make the migration easier for operators to understand. +This was `originally authored `_ in March 2023. We `modified it in July 2023 `_ based on learnings from the implementation process. -Related deprecation tickets: +The status will be moved to *Accepted* upon completion of reimplementation. Related work: * `[DEPR]: Asset processing in Paver `_ -* `[DEPR]: Paver `_ +* `Process edx-platform assets without Paver `_ +* `Process edx-platform assets without Python `_ + Context ******* @@ -90,6 +92,7 @@ Three particular issues have surfaced in Developer Experience Working Group disc All of these potential solutions would involve refactoring or entirely replacing parts of the current asset processing system. + Decision ******** @@ -111,9 +114,6 @@ Reimplementation Specification Commands and stages ------------------- -**May 2024 update:** See the `static assets reference <../references/static-assets.rst>`_ for -the latest commands. - The three top-level edx-platform asset processing actions are *build*, *collect*, and *watch*. The build action can be further broken down into five stages. Here is how those actions and stages will be reimplemented: @@ -226,9 +226,6 @@ The three top-level edx-platform asset processing actions are *build*, *collect* Build Configuration ------------------- -**May 2024 update:** See the `static assets reference <../references/static-assets.rst>`_ for -the latest configuration settings. - To facilitate a generally Python-free build reimplementation, we will require that certain Django settings now be specified as environment variables, which can be passed to the build like so:: MY_ENV_VAR="my value" npm run build # Set for the whole build. @@ -269,7 +266,7 @@ Some of these options will remain as Django settings because they are used in ed * - ``COMPREHENSIVE_THEME_DIRS`` - Directories that will be searched when compiling themes. - ``COMPREHENSIVE_THEME_DIRS`` - - ``COMPREHENSIVE_THEME_DIRS`` + - ``EDX_PLATFORM_THEME_DIRS`` Migration ========= @@ -288,16 +285,37 @@ As a consequence of this ADR, Tutor will either need to: * reimplement the script as a thin wrapper around the new asset processing commands, or * deprecate and remove the script. -**May 2024 update:** The ``openedx-assets`` script will be removed from Tutor, -with migration instructions documented in -`Tutor's changelog `_. +Either way, the migration path is straightforward: + +.. list-table:: + :header-rows: 1 + + * - Existing Tutor-provided command + - New upstream command + * - ``openedx-assets build`` + - ``npm run build`` + * - ``openedx-assets npm`` + - ``scripts/copy-node-modules.sh # (automatically invoked by 'npm install'!)`` + * - ``openedx-assets xmodule`` + - (no longer needed) + * - ``openedx-assets common`` + - ``npm run compile-sass -- --skip-themes`` + * - ``openedx-assets themes`` + - ``npm run compile-sass -- --skip-default`` + * - ``openedx-assets webpack`` + - ``npm run webpack`` + * - ``openedx-assets collect`` + - ``./manage.py [lms|cms] collectstatic --noinput`` + * - ``openedx-assets watch-themes`` + - ``npm run watch`` + +The options accepted by ``openedx-assets`` will all be valid inputs to ``scripts/build-assets.sh``. non-Tutor migration guide ------------------------- -A migration guide for site operators who are directly referencing Paver will be -included in the -`Paver deprecation ticket `_. +Operators using distributions other than Tutor should refer to the upstream edx-platform changes described above in **Reimplementation Specification**, and adapt them accordingly to their distribution. + See also ******** diff --git a/lms/envs/common.py b/lms/envs/common.py index 02e52ea5a317..f6266fdaa3cb 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -1928,7 +1928,7 @@ def _make_mako_template_dirs(settings): # Static content STATIC_URL = '/static/' -STATIC_ROOT = os.environ.get('STATIC_ROOT_LMS', ENV_ROOT / "staticfiles") +STATIC_ROOT = ENV_ROOT / "staticfiles" STATIC_URL_BASE = '/static/' STATICFILES_DIRS = [ @@ -2822,14 +2822,7 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring 'STATS_FILE': os.path.join(STATIC_ROOT, 'webpack-worker-stats.json') } } - -# .. setting_name: WEBPACK_CONFIG_PATH -# .. setting_default: "webpack.prod.config.js" -# .. setting_description: Path to the Webpack configuration file. Used by Paver scripts. -# .. setting_warning: This Django setting is DEPRECATED! Starting in Sumac, Webpack will no longer -# use Django settings. Please set the WEBPACK_CONFIG_PATH environment variable instead. For details, -# see: https://github.com/openedx/edx-platform/issues/31895 -WEBPACK_CONFIG_PATH = os.environ.get('WEBPACK_CONFIG_PATH', 'webpack.prod.config.js') +WEBPACK_CONFIG_PATH = 'webpack.prod.config.js' ########################## DJANGO DEBUG TOOLBAR ############################### @@ -4556,11 +4549,9 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring # .. setting_name: COMPREHENSIVE_THEME_DIRS # .. setting_default: [] -# .. setting_description: A list of paths to directories, each of which will -# be searched for comprehensive themes. Do not override this Django setting directly. -# Instead, set the COMPREHENSIVE_THEME_DIRS environment variable, using colons (:) to -# separate paths. -COMPREHENSIVE_THEME_DIRS = os.environ.get("COMPREHENSIVE_THEME_DIRS", "").split(":") +# .. setting_description: A list of directories containing themes folders, +# each entry should be a full path to the directory containing the theme folder. +COMPREHENSIVE_THEME_DIRS = [] # .. setting_name: COMPREHENSIVE_THEME_LOCALE_PATHS # .. setting_default: [] diff --git a/openedx/core/djangoapps/theming/management/commands/compile_sass.py b/openedx/core/djangoapps/theming/management/commands/compile_sass.py index eecbe684b528..765ef98aeac3 100644 --- a/openedx/core/djangoapps/theming/management/commands/compile_sass.py +++ b/openedx/core/djangoapps/theming/management/commands/compile_sass.py @@ -1,14 +1,13 @@ """ Management command for compiling sass. - -DEPRECATED in favor of `npm run compile-sass`. """ -import shlex -from django.core.management import BaseCommand -from django.conf import settings -from pavelib.assets import run_deprecated_command_wrapper +from django.core.management import BaseCommand, CommandError +from paver.easy import call_task + +from openedx.core.djangoapps.theming.helpers import get_theme_base_dirs, get_themes, is_comprehensive_theming_enabled +from pavelib.assets import ALL_SYSTEMS class Command(BaseCommand): @@ -16,7 +15,7 @@ class Command(BaseCommand): Compile theme sass and collect theme assets. """ - help = "DEPRECATED. Use 'npm run compile-sass' instead." + help = 'Compile and collect themed assets...' # NOTE (CCB): This allows us to compile static assets in Docker containers without database access. requires_system_checks = [] @@ -29,7 +28,7 @@ def add_arguments(self, parser): parser (django.core.management.base.CommandParser): parsed for parsing command line arguments. """ parser.add_argument( - 'system', type=str, nargs='*', default=["lms", "cms"], + 'system', type=str, nargs='*', default=ALL_SYSTEMS, help="lms or studio", ) @@ -56,7 +55,7 @@ def add_arguments(self, parser): '--force', action='store_true', default=False, - help="DEPRECATED. Full recompilation is now always forced.", + help="Force full compilation", ) parser.add_argument( '--debug', @@ -65,48 +64,77 @@ def add_arguments(self, parser): help="Disable Sass compression", ) - def handle(self, *args, **options): + @staticmethod + def parse_arguments(*args, **options): # pylint: disable=unused-argument """ - Handle compile_sass command. + Parse and validate arguments for compile_sass command. + + Args: + *args: Positional arguments passed to the update_assets command + **options: optional arguments passed to the update_assets command + Returns: + A tuple containing parsed values for themes, system, source comments and output style. + 1. system (list): list of system names for whom to compile theme sass e.g. 'lms', 'cms' + 2. theme_dirs (list): list of Theme objects + 3. themes (list): list of Theme objects + 4. force (bool): Force full compilation + 5. debug (bool): Disable Sass compression """ - systems = set( - {"lms": "lms", "cms": "cms", "studio": "cms"}[sys] - for sys in options.get("system", ["lms", "cms"]) - ) - theme_dirs = options.get("theme_dirs", settings.COMPREHENSIVE_THEME_DIRS) or [] - themes_option = options.get("themes", []) # '[]' means 'all' - if not settings.ENABLE_COMPREHENSIVE_THEMING: - compile_themes = False - themes = [] - elif "no" in themes_option: - compile_themes = False - themes = [] - elif "all" in themes_option: - compile_themes = True - themes = [] + system = options.get("system", ALL_SYSTEMS) + given_themes = options.get("themes", ["all"]) + theme_dirs = options.get("theme_dirs", None) + + force = options.get("force", True) + debug = options.get("debug", True) + + if theme_dirs: + available_themes = {} + for theme_dir in theme_dirs: + available_themes.update({t.theme_dir_name: t for t in get_themes(theme_dir)}) else: - compile_themes = True - themes = themes_option - run_deprecated_command_wrapper( - old_command="./manage.py [lms|cms] compile_sass", - ignored_old_flags=list(set(["force"]) & set(options)), - new_command=shlex.join([ - "npm", - "run", - ("compile-sass-dev" if options.get("debug") else "compile-sass"), - "--", - *(["--skip-lms"] if "lms" not in systems else []), - *(["--skip-cms"] if "cms" not in systems else []), - *(["--skip-themes"] if not compile_themes else []), - *( - arg - for theme_dir in theme_dirs - for arg in ["--theme-dir", str(theme_dir)] + theme_dirs = get_theme_base_dirs() + available_themes = {t.theme_dir_name: t for t in get_themes()} + + if 'no' in given_themes or 'all' in given_themes: + # Raise error if 'all' or 'no' is present and theme names are also given. + if len(given_themes) > 1: + raise CommandError("Invalid themes value, It must either be 'all' or 'no' or list of themes.") + # Raise error if any of the given theme name is invalid + # (theme name would be invalid if it does not exist in themes directory) + elif (not set(given_themes).issubset(list(available_themes.keys()))) and is_comprehensive_theming_enabled(): + raise CommandError( + "Given themes '{themes}' do not exist inside any of the theme directories '{theme_dirs}'".format( + themes=", ".join(set(given_themes) - set(available_themes.keys())), + theme_dirs=theme_dirs, ), - *( - arg - for theme in themes - for arg in ["--theme", theme] + ) + + if "all" in given_themes: + themes = list(available_themes.values()) + elif "no" in given_themes: + themes = [] + else: + # convert theme names to Theme objects, this will remove all themes if theming is disabled + themes = [available_themes.get(theme) for theme in given_themes if theme in available_themes] + + return system, theme_dirs, themes, force, debug + + def handle(self, *args, **options): + """ + Handle compile_sass command. + """ + system, theme_dirs, themes, force, debug = self.parse_arguments(*args, **options) + themes = [theme.theme_dir_name for theme in themes] + + if options.get("themes", None) and not is_comprehensive_theming_enabled(): + # log a warning message to let the user know that asset compilation for themes is skipped + self.stdout.write( + self.style.WARNING( + "Skipping theme asset compilation: enable theming to process themed assets" ), - ]), + ) + + call_task( + 'pavelib.assets.compile_sass', + options={'system': system, 'theme_dirs': theme_dirs, 'themes': themes, 'force': force, 'debug': debug}, ) diff --git a/openedx/core/djangoapps/theming/tests/test_commands.py b/openedx/core/djangoapps/theming/tests/test_commands.py index 7ca56f604a41..b4abee9bed0e 100644 --- a/openedx/core/djangoapps/theming/tests/test_commands.py +++ b/openedx/core/djangoapps/theming/tests/test_commands.py @@ -2,23 +2,57 @@ Tests for Management commands of comprehensive theming. """ -from django.core.management import call_command -from django.test import TestCase, override_settings -from unittest.mock import patch +import pytest +from django.core.management import CommandError, call_command +from django.test import TestCase -import pavelib.assets +from openedx.core.djangoapps.theming.helpers import get_themes +from openedx.core.djangoapps.theming.management.commands.compile_sass import Command class TestUpdateAssets(TestCase): """ Test comprehensive theming helper functions. """ + def setUp(self): + super().setUp() + self.themes = get_themes() - @patch.object(pavelib.assets, 'sh') - @override_settings(COMPREHENSIVE_THEME_DIRS='common/test') - def test_deprecated_wrapper(self, mock_sh): - call_command('compile_sass', '--themes', 'fake-theme1', 'fake-theme2') - assert mock_sh.called_once_with( - "npm run compile-sass -- " + - "--theme-dir common/test --theme fake-theme-1 --theme fake-theme-2" + def test_errors_for_invalid_arguments(self): + """ + Test update_asset command. + """ + # make sure error is raised for invalid theme list + with pytest.raises(CommandError): + call_command("compile_sass", themes=["all", "test-theme"]) + + # make sure error is raised for invalid theme list + with pytest.raises(CommandError): + call_command("compile_sass", themes=["no", "test-theme"]) + + # make sure error is raised for invalid theme list + with pytest.raises(CommandError): + call_command("compile_sass", themes=["all", "no"]) + + # make sure error is raised for invalid theme list + with pytest.raises(CommandError): + call_command("compile_sass", themes=["test-theme", "non-existing-theme"]) + + def test_parse_arguments(self): + """ + Test parse arguments method for update_asset command. + """ + # make sure compile_sass picks all themes when called with 'themes=all' option + parsed_args = Command.parse_arguments(themes=["all"]) + self.assertCountEqual(parsed_args[2], get_themes()) + + # make sure compile_sass picks no themes when called with 'themes=no' option + parsed_args = Command.parse_arguments(themes=["no"]) + self.assertCountEqual(parsed_args[2], []) + + # make sure compile_sass picks only specified themes + parsed_args = Command.parse_arguments(themes=["test-theme"]) + self.assertCountEqual( + parsed_args[2], + [theme for theme in get_themes() if theme.theme_dir_name == "test-theme"] ) diff --git a/pavelib/assets.py b/pavelib/assets.py index f437b6427f93..466ffc9ed919 100644 --- a/pavelib/assets.py +++ b/pavelib/assets.py @@ -1,9 +1,5 @@ """ Asset compilation and collection. - -This entire module is DEPRECATED. In Redwood, it exists just as a collection of temporary compatibility wrappers. -In Sumac, this module will be deleted. To migrate, follow the advice in the printed warnings and/or read the -instructions on the DEPR ticket: https://github.com/openedx/edx-platform/issues/31895 """ import argparse @@ -17,48 +13,31 @@ from paver import tasks from paver.easy import call_task, cmdopts, consume_args, needs, no_help, sh, task from watchdog.events import PatternMatchingEventHandler -from watchdog.observers import Observer # pylint: disable=unused-import # Used by Tutor. Remove after Sumac cut. +from watchdog.observers import Observer # pylint disable=unused-import # Used by Tutor. Remove after Sumac cut. -from .utils.cmd import django_cmd +from .utils.cmd import cmd, django_cmd from .utils.envs import Env +from .utils.process import run_background_process from .utils.timer import timed +# setup baseline paths + +ALL_SYSTEMS = ['lms', 'studio'] + +LMS = 'lms' +CMS = 'cms' SYSTEMS = { - 'lms': 'lms', - 'cms': 'cms', - 'studio': 'cms', + 'lms': LMS, + 'cms': CMS, + 'studio': CMS } -WARNING_SYMBOLS = "⚠️ " * 50 # A row of 'warning' emoji to catch CLI users' attention - +# Collectstatic log directory setting +COLLECTSTATIC_LOG_DIR_ARG = 'collect_log_dir' -def run_deprecated_command_wrapper(*, old_command, ignored_old_flags, new_command): - """ - Run the new version of shell command, plus a warning that the old version is deprecated. - """ - depr_warning = ( - "\n" + - f"{WARNING_SYMBOLS}\n" + - "\n" + - f"WARNING: '{old_command}' is DEPRECATED! It will be removed before Sumac.\n" + - "The command you ran is now just a temporary wrapper around a new,\n" + - "supported command, which you should use instead:\n" + - "\n" + - f"\t{new_command}\n" + - "\n" + - "Details: https://github.com/openedx/edx-platform/issues/31895\n" + - "".join( - f" WARNING: ignored deprecated paver flag '{flag}'\n" - for flag in ignored_old_flags - ) + - f"{WARNING_SYMBOLS}\n" + - "\n" - ) - # Print deprecation warning twice so that it's more likely to be seen in the logs. - print(depr_warning) - sh(new_command) - print(depr_warning) +# Webpack command +WEBPACK_COMMAND = 'STATIC_ROOT_LMS={static_root_lms} STATIC_ROOT_CMS={static_root_cms} webpack {options}' def debounce(seconds=1): @@ -66,8 +45,6 @@ def debounce(seconds=1): Prevents the decorated function from being called more than every `seconds` seconds. Waits until calls stop coming in before calling the decorated function. - - This is DEPRECATED. It exists in Redwood just to ease the transition for Tutor. """ def decorator(func): func.timer = None @@ -89,8 +66,6 @@ def call(): class SassWatcher(PatternMatchingEventHandler): """ Watches for sass file changes - - This is DEPRECATED. It exists in Redwood just to ease the transition for Tutor. """ ignore_directories = True patterns = ['*.scss'] @@ -127,7 +102,7 @@ def on_any_event(self, event): ('system=', 's', 'The system to compile sass for (defaults to all)'), ('theme-dirs=', '-td', 'Theme dirs containing all themes (defaults to None)'), ('themes=', '-t', 'The theme to compile sass for (defaults to None)'), - ('debug', 'd', 'Whether to use development settings'), + ('debug', 'd', 'DEPRECATED. Debug mode is now determined by NODE_ENV.'), ('force', '', 'DEPRECATED. Full recompilation is now always forced.'), ]) @timed @@ -168,18 +143,16 @@ def compile_sass(options): This is a DEPRECATED COMPATIBILITY WRAPPER. Use `npm run compile-sass` instead. """ - systems = [SYSTEMS[sys] for sys in get_parsed_option(options, 'system', ['lms', 'cms'])] # normalize studio->cms - run_deprecated_command_wrapper( - old_command="paver compile_sass", - ignored_old_flags=(set(["--force"]) & set(options)), - new_command=shlex.join([ + systems = set(get_parsed_option(options, 'system', ALL_SYSTEMS)) + command = shlex.join( + [ "npm", "run", - ("compile-sass-dev" if options.get("debug") else "compile-sass"), + "compile-sass", "--", *(["--dry"] if tasks.environment.dry_run else []), - *(["--skip-lms"] if "lms" not in systems else []), - *(["--skip-cms"] if "cms" not in systems else []), + *(["--skip-lms"] if not systems & {"lms"} else []), + *(["--skip-cms"] if not systems & {"cms", "studio"} else []), *( arg for theme_dir in get_parsed_option(options, 'theme_dirs', []) @@ -187,50 +160,77 @@ def compile_sass(options): ), *( arg - for theme in get_parsed_option(options, "themes", []) + for theme in get_parsed_option(options, "theme", []) for arg in ["--theme", theme] ), - ]), + ] + ) + depr_warning = ( + "\n" + + "⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ \n" + + "\n" + + "WARNING: 'paver compile_sass' is DEPRECATED! It will be removed before Sumac.\n" + + "The command you ran is now just a temporary wrapper around a new,\n" + + "supported command, which you should use instead:\n" + + "\n" + + f"\t{command}\n" + + "\n" + + "Details: https://github.com/openedx/edx-platform/issues/31895\n" + + "\n" + + ("WARNING: ignoring deprecated flag '--debug'\n" if options.get("debug") else "") + + ("WARNING: ignoring deprecated flag '--force'\n" if options.get("force") else "") + + "⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ \n" + + "\n" ) + # Print deprecation warning twice so that it's more likely to be seen in the logs. + print(depr_warning) + sh(command) + print(depr_warning) -def _compile_sass(system, theme, debug, force, _timing_info): +def _compile_sass(system, theme, _debug, _force, _timing_info): """ This is a DEPRECATED COMPATIBILITY WRAPPER It exists to ease the transition for Tutor in Redwood, which directly imported and used this function. """ - run_deprecated_command_wrapper( - old_command="pavelib.assets:_compile_sass", - ignored_old_flags=(set(["--force"]) if force else set()), - new_command=[ + command = shlex.join( + [ "npm", "run", - ("compile-sass-dev" if debug else "compile-sass"), + "compile-sass", "--", *(["--dry"] if tasks.environment.dry_run else []), - *( - ["--skip-default", "--theme-dir", str(theme.parent), "--theme", str(theme.name)] - if theme - else [] - ), + *(["--skip-default", "--theme-dir", str(theme.parent), "--theme", str(theme.name)] if theme else []), ("--skip-cms" if system == "lms" else "--skip-lms"), ] ) + depr_warning = ( + "\n" + + "⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ \n" + + "\n" + + "WARNING: 'pavelib/assets.py' is DEPRECATED! It will be removed before Sumac.\n" + + "The function you called is just a temporary wrapper around a new, supported command,\n" + + "which you should use instead:\n" + + "\n" + + f"\t{command}\n" + + "\n" + + "Details: https://github.com/openedx/edx-platform/issues/31895\n" + + "\n" + + "⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ \n" + + "\n" + ) + # Print deprecation warning twice so that it's more likely to be seen in the logs. + print(depr_warning) + sh(command) + print(depr_warning) def process_npm_assets(): """ Process vendor libraries installed via NPM. - - This is a DEPRECATED COMPATIBILITY WRAPPER. It is now handled as part of `npm clean-install`. - If you need to invoke it explicitly, you can run `npm run postinstall`. """ - run_deprecated_command_wrapper( - old_command="pavelib.assets:process_npm_assets", - ignored_old_flags=[], - new_command=shlex.join(["npm", "run", "postinstall"]), - ) + sh('scripts/copy-node-modules.sh') @task @@ -238,21 +238,9 @@ def process_npm_assets(): def process_xmodule_assets(): """ Process XModule static assets. - - This is a DEPRECATED COMPATIBILITY STUB. Refrences to it should be deleted. """ - print( - "\n" + - f"{WARNING_SYMBOLS}", - "\n" + - "WARNING: 'paver process_xmodule_assets' is DEPRECATED! It will be removed before Sumac.\n" + - "\n" + - "Starting with Quince, it is no longer necessary to post-process XModule assets, so \n" + - "'paver process_xmodule_assets' is a no-op. Please simply remove it from your build scripts.\n" + - "\n" + - "Details: https://github.com/openedx/edx-platform/issues/31895\n" + - f"{WARNING_SYMBOLS}", - ) + print("\t\tProcessing xmodule assets is no longer needed. This task is now a no-op.") + print("\t\tWhen paver is removed from edx-platform, this step will not replaced.") def collect_assets(systems, settings, **kwargs): @@ -261,29 +249,33 @@ def collect_assets(systems, settings, **kwargs): `systems` is a list of systems (e.g. 'lms' or 'studio' or both) `settings` is the Django settings module to use. `**kwargs` include arguments for using a log directory for collectstatic output. Defaults to /dev/null. + """ + for sys in systems: + collectstatic_stdout_str = _collect_assets_cmd(sys, **kwargs) + sh(django_cmd(sys, settings, "collectstatic --noinput {logfile_str}".format( + logfile_str=collectstatic_stdout_str + ))) + print(f"\t\tFinished collecting {sys} assets.") - This is a DEPRECATED COMPATIBILITY WRAPPER - It exists to ease the transition for Tutor in Redwood, which directly imported and used this function. +def _collect_assets_cmd(system, **kwargs): """ - run_deprecated_command_wrapper( - old_command="pavelib.asset:collect_assets", - ignored_old_flags=[], - new_command=" && ".join( - "( " + - shlex.join( - ["./manage.py", SYSTEMS[sys], f"--settings={settings}", "collectstatic", "--noinput"] - ) + ( - "" - if "collect_log_dir" not in kwargs else - " > /dev/null" - if kwargs["collect_log_dir"] is None else - f"> {kwargs['collect_log_dir']}/{SYSTEMS[sys]}-collectstatic.out" - ) + - " )" - for sys in systems - ), - ) + Returns the collecstatic command to be used for the given system + + Unless specified, collectstatic (which can be verbose) pipes to /dev/null + """ + try: + if kwargs[COLLECTSTATIC_LOG_DIR_ARG] is None: + collectstatic_stdout_str = "" + else: + collectstatic_stdout_str = "> {output_dir}/{sys}-collectstatic.log".format( + output_dir=kwargs[COLLECTSTATIC_LOG_DIR_ARG], + sys=system + ) + except KeyError: + collectstatic_stdout_str = "> /dev/null" + + return collectstatic_stdout_str def execute_compile_sass(args): @@ -291,8 +283,6 @@ def execute_compile_sass(args): Construct django management command compile_sass (defined in theming app) and execute it. Args: args: command line argument passed via update_assets command - - This is a DEPRECATED COMPATIBILITY WRAPPER. Use `npm run compile-sass` instead. """ for sys in args.system: options = "" @@ -315,14 +305,12 @@ def execute_compile_sass(args): @task @cmdopts([ ('settings=', 's', "Django settings (defaults to devstack)"), - ('watch', 'w', "DEPRECATED. This flag never did anything anyway."), + ('watch', 'w', "Watch file system and rebuild on change (defaults to off)"), ]) @timed def webpack(options): """ Run a Webpack build. - - This is a DEPRECATED COMPATIBILITY WRAPPER. Use `npm run webpack` instead. """ settings = getattr(options, 'settings', Env.DEVSTACK_SETTINGS) result = Env.get_django_settings(['STATIC_ROOT', 'WEBPACK_CONFIG_PATH'], "lms", settings=settings) @@ -330,20 +318,44 @@ def webpack(options): static_root_cms, = Env.get_django_settings(["STATIC_ROOT"], "cms", settings=settings) js_env_extra_config_setting, = Env.get_django_json_settings(["JS_ENV_EXTRA_CONFIG"], "cms", settings=settings) js_env_extra_config = json.dumps(js_env_extra_config_setting or "{}") - node_env = "development" if config_path == 'webpack.dev.config.js' else "production" - run_deprecated_command_wrapper( - old_command="paver webpack", - ignored_old_flags=(set(["watch"]) & set(options)), - new_command=' '.join([ - f"WEBPACK_CONFIG_PATH={config_path}", - f"NODE_ENV={node_env}", - f"STATIC_ROOT_LMS={static_root_lms}", - f"STATIC_ROOT_CMS={static_root_cms}", - f"JS_ENV_EXTRA_CONFIG={js_env_extra_config}", - "npm", - "run", - "webpack", - ]), + environment = ( + "NODE_ENV={node_env} STATIC_ROOT_LMS={static_root_lms} STATIC_ROOT_CMS={static_root_cms} " + "JS_ENV_EXTRA_CONFIG={js_env_extra_config}" + ).format( + node_env="development" if config_path == 'webpack.dev.config.js' else "production", + static_root_lms=static_root_lms, + static_root_cms=static_root_cms, + js_env_extra_config=js_env_extra_config, + ) + sh( + cmd( + '{environment} webpack --config={config_path}'.format( + environment=environment, + config_path=config_path + ) + ) + ) + + +def execute_webpack_watch(settings=None): + """ + Run the Webpack file system watcher. + """ + # We only want Webpack to re-run on changes to its own entry points, + # not all JS files, so we use its own watcher instead of subclassing + # from Watchdog like the other watchers do. + + result = Env.get_django_settings(["STATIC_ROOT", "WEBPACK_CONFIG_PATH"], "lms", settings=settings) + static_root_lms, config_path = result + static_root_cms, = Env.get_django_settings(["STATIC_ROOT"], "cms", settings=settings) + run_background_process( + 'STATIC_ROOT_LMS={static_root_lms} STATIC_ROOT_CMS={static_root_cms} webpack {options}'.format( + options='--watch --config={config_path}'.format( + config_path=config_path + ), + static_root_lms=static_root_lms, + static_root_cms=static_root_cms, + ) ) @@ -399,19 +411,39 @@ def watch_assets(options): return theme_dirs = ':'.join(get_parsed_option(options, 'theme_dirs', [])) - run_deprecated_command_wrapper( - old_command="paver watch_assets", - ignored_old_flags=(set(["debug", "themes", "settings", "background"]) & set(options)), - new_command=shlex.join([ + command = shlex.join( + [ *( - ["env", f"COMPREHENSIVE_THEME_DIRS={theme_dirs}"] - if theme_dirs else [] + ["env", f"EDX_PLATFORM_THEME_DIRS={theme_dirs}"] if theme_dirs else [] ), "npm", "run", "watch", - ]), + ] ) + depr_warning = ( + "\n" + + "⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ \n" + + "\n" + + "WARNING: 'paver watch_assets' is DEPRECATED! It will be removed before Sumac.\n" + + "The command you ran is now just a temporary wrapper around a new,\n" + + "supported command, which you should use instead:\n" + + "\n" + + f"\t{command}\n" + + "\n" + + "Details: https://github.com/openedx/edx-platform/issues/31895\n" + + "\n" + + ("WARNING: ignoring deprecated flag '--debug'\n" if options.get("debug") else "") + + ("WARNING: ignoring deprecated flag '--themes'\n" if options.get("themes") else "") + + ("WARNING: ignoring deprecated flag '--settings'\n" if options.get("settings") else "") + + ("WARNING: ignoring deprecated flag '--background'\n" if options.get("background") else "") + + "⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ \n" + + "\n" + ) + # Print deprecation warning twice so that it's more likely to be seen in the logs. + print(depr_warning) + sh(command) + print(depr_warning) @task @@ -424,19 +456,10 @@ def watch_assets(options): def update_assets(args): """ Compile Sass, then collect static assets. - - This is a DEPRECATED COMPATIBILITY WRAPPER around other DEPRECATED COMPATIBILITY WRAPPERS. - The aggregate affect of this command can be achieved with this sequence of commands instead: - - * pip install -r requirements/edx/assets.txt # replaces install_python_prereqs - * npm clean-install # replaces install_node_prereqs - * npm run build # replaces execute_compile_sass and webpack - * ./manage.py lms collectstatic --noinput # replaces collect_assets (for LMS) - * ./manage.py cms collectstatic --noinput # replaces collect_assets (for CMS) """ parser = argparse.ArgumentParser(prog='paver update_assets') parser.add_argument( - 'system', type=str, nargs='*', default=["lms", "studio"], + 'system', type=str, nargs='*', default=ALL_SYSTEMS, help="lms or studio", ) parser.add_argument( @@ -465,17 +488,18 @@ def update_assets(args): ) parser.add_argument( '--themes', type=str, nargs='+', default=None, - help="list of themes to compile sass for. ignored when --watch is used; all themes are watched.", + help="list of themes to compile sass for", ) parser.add_argument( - '--collect-log', dest="collect_log_dir", default=None, + '--collect-log', dest=COLLECTSTATIC_LOG_DIR_ARG, default=None, help="When running collectstatic, direct output to specified log directory", ) parser.add_argument( '--wait', type=float, default=0.0, - help="DEPRECATED. Watchdog's default wait time is now used.", + help="How long to pause between filesystem scans" ) args = parser.parse_args(args) + collect_log_args = {} # Build Webpack call_task('pavelib.assets.webpack', options={'settings': args.settings}) @@ -484,12 +508,11 @@ def update_assets(args): execute_compile_sass(args) if args.collect: + if args.debug or args.debug_collect: + collect_log_args.update({COLLECTSTATIC_LOG_DIR_ARG: None}) + if args.collect_log_dir: - collect_log_args = {"collect_log_dir": args.collect_log_dir} - elif args.debug or args.debug_collect: - collect_log_args = {"collect_log_dir": None} - else: - collect_log_args = {} + collect_log_args.update({COLLECTSTATIC_LOG_DIR_ARG: args.collect_log_dir}) collect_assets(args.system, args.settings, **collect_log_args) diff --git a/pavelib/paver_tests/test_assets.py b/pavelib/paver_tests/test_assets.py index 3578a5043f7e..029a8db67c4a 100644 --- a/pavelib/paver_tests/test_assets.py +++ b/pavelib/paver_tests/test_assets.py @@ -1,130 +1,305 @@ """Unit tests for the Paver asset tasks.""" -import json + import os -from pathlib import Path from unittest import TestCase from unittest.mock import patch import ddt -import paver.easy -from paver import tasks +import paver.tasks +from paver.easy import call_task, path -import pavelib.assets -from pavelib.assets import Env +from pavelib.assets import COLLECTSTATIC_LOG_DIR_ARG, collect_assets +from ..utils.envs import Env +from .utils import PaverTestCase -REPO_ROOT = Path(__file__).parent.parent.parent +ROOT_PATH = path(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) +TEST_THEME_DIR = ROOT_PATH / "common/test/test-theme" -LMS_SETTINGS = { - "WEBPACK_CONFIG_PATH": "webpack.fake.config.js", - "STATIC_ROOT": "/fake/lms/staticfiles", -} -CMS_SETTINGS = { - "WEBPACK_CONFIG_PATH": "webpack.fake.config", - "STATIC_ROOT": "/fake/cms/staticfiles", - "JS_ENV_EXTRA_CONFIG": json.dumps({"key1": [True, False], "key2": {"key2.1": 1369, "key2.2": "1369"}}), -} +@ddt.ddt +class TestPaverAssetTasks(PaverTestCase): + """ + Test the Paver asset tasks. + """ + @ddt.data( + [""], + ["--force"], + ["--debug"], + ["--system=lms"], + ["--system=lms --force"], + ["--system=studio"], + ["--system=studio --force"], + ["--system=lms,studio"], + ["--system=lms,studio --force"], + ) + @ddt.unpack + def test_compile_sass(self, options): + """ + Test the "compile_sass" task. + """ + parameters = options.split(" ") + system = [] + if '--system=studio' not in parameters: + system += ['lms'] + if '--system=lms' not in parameters: + system += ['studio'] + debug = '--debug' in parameters + force = '--force' in parameters + self.reset_task_messages() + call_task('pavelib.assets.compile_sass', options={'system': system, 'debug': debug, 'force': force}) + expected_messages = [] + if force: + expected_messages.append('rm -rf common/static/css/*.css') + expected_messages.append('libsass common/static/sass') + if "lms" in system: + if force: + expected_messages.append('rm -rf lms/static/css/*.css') + expected_messages.append('libsass lms/static/sass') + expected_messages.append( + 'rtlcss lms/static/css/bootstrap/lms-main.css lms/static/css/bootstrap/lms-main-rtl.css' + ) + expected_messages.append( + 'rtlcss lms/static/css/discussion/lms-discussion-bootstrap.css' + ' lms/static/css/discussion/lms-discussion-bootstrap-rtl.css' + ) + if force: + expected_messages.append('rm -rf lms/static/certificates/css/*.css') + expected_messages.append('libsass lms/static/certificates/sass') + if "studio" in system: + if force: + expected_messages.append('rm -rf cms/static/css/*.css') + expected_messages.append('libsass cms/static/sass') + expected_messages.append( + 'rtlcss cms/static/css/bootstrap/studio-main.css cms/static/css/bootstrap/studio-main-rtl.css' + ) -def _mock_get_django_settings(django_settings, system, settings=None): # pylint: disable=unused-argument - return [(LMS_SETTINGS if system == "lms" else CMS_SETTINGS)[s] for s in django_settings] + assert len(self.task_messages) == len(expected_messages) @ddt.ddt -@patch.object(Env, 'get_django_settings', _mock_get_django_settings) -@patch.object(Env, 'get_django_json_settings', _mock_get_django_settings) -class TestDeprecatedPaverAssets(TestCase): +class TestPaverThemeAssetTasks(PaverTestCase): """ - Simple test to ensure that the soon-to-be-removed Paver commands are correctly translated into the new npm-run - commands. + Test the Paver asset tasks. """ - def setUp(self): - super().setUp() - self.maxDiff = None - os.environ['NO_PREREQ_INSTALL'] = 'true' - tasks.environment = tasks.Environment() + @ddt.data( + [""], + ["--force"], + ["--debug"], + ["--system=lms"], + ["--system=lms --force"], + ["--system=studio"], + ["--system=studio --force"], + ["--system=lms,studio"], + ["--system=lms,studio --force"], + ) + @ddt.unpack + def test_compile_theme_sass(self, options): + """ + Test the "compile_sass" task. + """ + parameters = options.split(" ") + system = [] + + if '--system=studio' not in parameters: + system += ['lms'] + if "--system=lms" not in parameters: + system += ['studio'] + debug = '--debug' in parameters + force = '--force' in parameters + + self.reset_task_messages() + call_task( + 'pavelib.assets.compile_sass', + options=dict( + system=system, + debug=debug, + force=force, + theme_dirs=[TEST_THEME_DIR.dirname()], + themes=[TEST_THEME_DIR.basename()] + ), + ) + expected_messages = [] + if force: + expected_messages.append('rm -rf common/static/css/*.css') + expected_messages.append('libsass common/static/sass') + + if 'lms' in system: + expected_messages.append('mkdir_p ' + repr(TEST_THEME_DIR / 'lms/static/css')) + if force: + expected_messages.append( + f'rm -rf {str(TEST_THEME_DIR)}/lms/static/css/*.css' + ) + expected_messages.append("libsass lms/static/sass") + expected_messages.append( + 'rtlcss {test_theme_dir}/lms/static/css/bootstrap/lms-main.css' + ' {test_theme_dir}/lms/static/css/bootstrap/lms-main-rtl.css'.format( + test_theme_dir=str(TEST_THEME_DIR), + ) + ) + expected_messages.append( + 'rtlcss {test_theme_dir}/lms/static/css/discussion/lms-discussion-bootstrap.css' + ' {test_theme_dir}/lms/static/css/discussion/lms-discussion-bootstrap-rtl.css'.format( + test_theme_dir=str(TEST_THEME_DIR), + ) + ) + if force: + expected_messages.append( + f'rm -rf {str(TEST_THEME_DIR)}/lms/static/css/*.css' + ) + expected_messages.append( + f'libsass {str(TEST_THEME_DIR)}/lms/static/sass' + ) + if force: + expected_messages.append('rm -rf lms/static/css/*.css') + expected_messages.append('libsass lms/static/sass') + expected_messages.append( + 'rtlcss lms/static/css/bootstrap/lms-main.css lms/static/css/bootstrap/lms-main-rtl.css' + ) + expected_messages.append( + 'rtlcss lms/static/css/discussion/lms-discussion-bootstrap.css' + ' lms/static/css/discussion/lms-discussion-bootstrap-rtl.css' + ) + if force: + expected_messages.append('rm -rf lms/static/certificates/css/*.css') + expected_messages.append('libsass lms/static/certificates/sass') - def tearDown(self): - super().tearDown() - del os.environ['NO_PREREQ_INSTALL'] + if "studio" in system: + expected_messages.append('mkdir_p ' + repr(TEST_THEME_DIR / 'cms/static/css')) + if force: + expected_messages.append( + f'rm -rf {str(TEST_THEME_DIR)}/cms/static/css/*.css' + ) + expected_messages.append('libsass cms/static/sass') + expected_messages.append( + 'rtlcss {test_theme_dir}/cms/static/css/bootstrap/studio-main.css' + ' {test_theme_dir}/cms/static/css/bootstrap/studio-main-rtl.css'.format( + test_theme_dir=str(TEST_THEME_DIR), + ) + ) + if force: + expected_messages.append( + f'rm -rf {str(TEST_THEME_DIR)}/cms/static/css/*.css' + ) + expected_messages.append( + f'libsass {str(TEST_THEME_DIR)}/cms/static/sass' + ) + if force: + expected_messages.append('rm -rf cms/static/css/*.css') + expected_messages.append('libsass cms/static/sass') + expected_messages.append( + 'rtlcss cms/static/css/bootstrap/studio-main.css cms/static/css/bootstrap/studio-main-rtl.css' + ) + + assert len(self.task_messages) == len(expected_messages) + + +@ddt.ddt +class TestCollectAssets(PaverTestCase): + """ + Test the collectstatic process call. + + ddt data is organized thusly: + * debug: whether or not collect_assets is called with the debug flag + * specified_log_location: used when collect_assets is called with a specific + log location for collectstatic output + * expected_log_location: the expected string to be used for piping collectstatic logs + """ + + @ddt.data( + [{ + "collect_log_args": {}, # Test for default behavior + "expected_log_location": "> /dev/null" + }], + [{ + "collect_log_args": {COLLECTSTATIC_LOG_DIR_ARG: "/foo/bar"}, + "expected_log_location": "> /foo/bar/lms-collectstatic.log" + }], # can use specified log location + [{ + "systems": ["lms", "cms"], + "collect_log_args": {}, + "expected_log_location": "> /dev/null" + }], # multiple systems can be called + ) + @ddt.unpack + def test_collect_assets(self, options): + """ + Ensure commands sent to the environment for collect_assets are as expected + """ + specified_log_loc = options.get("collect_log_args", {}) + specified_log_dict = specified_log_loc + log_loc = options.get("expected_log_location", "> /dev/null") + systems = options.get("systems", ["lms"]) + if specified_log_loc is None: + collect_assets( + systems, + Env.DEVSTACK_SETTINGS + ) + else: + collect_assets( + systems, + Env.DEVSTACK_SETTINGS, + **specified_log_dict + ) + self._assert_correct_messages(log_location=log_loc, systems=systems) + + def test_collect_assets_debug(self): + """ + When the method is called specifically with None for the collectstatic log dir, then + it should run in debug mode and pipe to console. + """ + expected_log_loc = "" + systems = ["lms"] + kwargs = {COLLECTSTATIC_LOG_DIR_ARG: None} + collect_assets(systems, Env.DEVSTACK_SETTINGS, **kwargs) + self._assert_correct_messages(log_location=expected_log_loc, systems=systems) + + def _assert_correct_messages(self, log_location, systems): + """ + Asserts that the expected commands were run. + + We just extract the pieces we care about here instead of specifying an + exact command, so that small arg changes don't break this test. + """ + for i, sys in enumerate(systems): + msg = self.task_messages[i] + assert msg.startswith(f'python manage.py {sys}') + assert ' collectstatic ' in msg + assert f'--settings={Env.DEVSTACK_SETTINGS}' in msg + assert msg.endswith(f' {log_location}') + + +@ddt.ddt +class TestUpdateAssetsTask(PaverTestCase): + """ + These are nearly end-to-end tests, because they observe output from the commandline request, + but do not actually execute the commandline on the terminal/process + """ @ddt.data( - dict( - task_name='pavelib.assets.compile_sass', - args=[], - kwargs={}, - expected=["npm run compile-sass --"], - ), - dict( - task_name='pavelib.assets.compile_sass', - args=[], - kwargs={"system": "lms,studio"}, - expected=["npm run compile-sass --"], - ), - dict( - task_name='pavelib.assets.compile_sass', - args=[], - kwargs={"debug": True}, - expected=["npm run compile-sass-dev --"], - ), - dict( - task_name='pavelib.assets.compile_sass', - args=[], - kwargs={"system": "lms"}, - expected=["npm run compile-sass -- --skip-cms"], - ), - dict( - task_name='pavelib.assets.compile_sass', - args=[], - kwargs={"system": "studio"}, - expected=["npm run compile-sass -- --skip-lms"], - ), - dict( - task_name='pavelib.assets.compile_sass', - args=[], - kwargs={"system": "cms", "theme_dirs": f"{REPO_ROOT}/common/test,{REPO_ROOT}/themes"}, - expected=[ - "npm run compile-sass -- --skip-lms " + - f"--theme-dir {REPO_ROOT}/common/test --theme-dir {REPO_ROOT}/themes" - ], - ), - dict( - task_name='pavelib.assets.compile_sass', - args=[], - kwargs={"theme_dirs": f"{REPO_ROOT}/common/test,{REPO_ROOT}/themes", "themes": "red-theme,test-theme"}, - expected=[ - "npm run compile-sass -- " + - f"--theme-dir {REPO_ROOT}/common/test --theme-dir {REPO_ROOT}/themes " + - "--theme red-theme --theme test-theme" - ], - ), - dict( - task_name='pavelib.assets.update_assets', - args=["lms", "studio", "--settings=fake.settings"], - kwargs={}, - expected=[ - ( - "WEBPACK_CONFIG_PATH=webpack.fake.config.js " + - "NODE_ENV=production " + - "STATIC_ROOT_LMS=/fake/lms/staticfiles " + - "STATIC_ROOT_CMS=/fake/cms/staticfiles " + - 'JS_ENV_EXTRA_CONFIG=' + + - '"{\\"key1\\": [true, false], \\"key2\\": {\\"key2.1\\": 1369, \\"key2.2\\": \\"1369\\"}}" ' + - "npm run webpack" - ), - "python manage.py lms --settings=fake.settings compile_sass lms ", - "python manage.py cms --settings=fake.settings compile_sass cms ", - ( - "( ./manage.py lms --settings=fake.settings collectstatic --noinput ) && " + - "( ./manage.py cms --settings=fake.settings collectstatic --noinput )" - ), - ], - ), + [{"expected_substring": "> /dev/null"}], # go to /dev/null by default + [{"cmd_args": ["--debug"], "expected_substring": "collectstatic"}] # TODO: make this regex ) @ddt.unpack - @patch.object(pavelib.assets, 'sh') - def test_paver_assets_wrapper_invokes_new_commands(self, mock_sh, task_name, args, kwargs, expected): - paver.easy.call_task(task_name, args=args, options=kwargs) - assert [call_args[0] for (call_args, call_kwargs) in mock_sh.call_args_list] == expected + def test_update_assets_task_collectstatic_log_arg(self, options): + """ + Scoped test that only looks at what is passed to the collecstatic options + """ + cmd_args = options.get("cmd_args", [""]) + expected_substring = options.get("expected_substring", None) + call_task('pavelib.assets.update_assets', args=cmd_args) + self.assertTrue( + self._is_substring_in_list(self.task_messages, expected_substring), + msg=f"{expected_substring} not found in messages" + ) + + def _is_substring_in_list(self, messages_list, expected_substring): + """ + Return true a given string is somewhere in a list of strings + """ + for message in messages_list: + if expected_substring in message: + return True + return False diff --git a/scripts/compile_sass.py b/scripts/compile_sass.py index ec1efee24d2b..41a2d56b3bda 100755 --- a/scripts/compile_sass.py +++ b/scripts/compile_sass.py @@ -67,12 +67,14 @@ "theme_dirs", metavar="PATH", multiple=True, - envvar="COMPREHENSIVE_THEME_DIRS", - type=click.Path(path_type=Path), + envvar="EDX_PLATFORM_THEME_DIRS", + type=click.Path( + exists=True, file_okay=False, readable=True, writable=True, path_type=Path + ), help=( "Consider sub-dirs of PATH as themes. " "Multiple theme dirs are accepted. " - "If none are provided, we look at colon-separated paths on the COMPREHENSIVE_THEME_DIRS env var." + "If none are provided, we look at colon-separated paths on the EDX_PLATFORM_THEME_DIRS env var." ), ) @click.option( diff --git a/scripts/watch_sass.sh b/scripts/watch_sass.sh index e88cb9e3cd6f..68d4b1f471a5 100755 --- a/scripts/watch_sass.sh +++ b/scripts/watch_sass.sh @@ -4,11 +4,11 @@ # Invoke from repo root as `npm run watch-sass`. # By default, only watches default Sass. -# To watch themes too, provide colon-separated paths in the COMPREHENSIVE_THEME_DIRS environment variable. +# To watch themes too, provide colon-separated paths in the EDX_PLATFORM_THEME_DIRS environment variable. # Each path will be treated as a "theme dir", which means that every immediate child directory is watchable as a theme. # For example: # -# COMPREHENSIVE_THEME_DIRS=/openedx/themes:./themes npm run watch-sass +# EDX_PLATFORM_THEME_DIRS=/openedx/themes:./themes npm run watch-sass # # would watch default Sass as well as /openedx/themes/indigo, /openedx/themes/mytheme, ./themes/red-theme, etc. From e6610f5ece39aa6ef7a45202798a4596797eb8e2 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Mon, 6 May 2024 14:37:38 -0400 Subject: [PATCH 032/113] test: only run Mongo 4.4 tests on xmodule directory (#34683) Cuts the number of unit test checks from 64 down to 34 Also, we rename test contexts so that they're easier to read in GHA. --- .github/workflows/unit-tests-gh-hosted.yml | 20 +++++++++++++++++--- .github/workflows/unit-tests.yml | 17 ++++++++++++++--- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/.github/workflows/unit-tests-gh-hosted.yml b/.github/workflows/unit-tests-gh-hosted.yml index 75b86d1dface..3f3e60ae8ff5 100644 --- a/.github/workflows/unit-tests-gh-hosted.yml +++ b/.github/workflows/unit-tests-gh-hosted.yml @@ -9,12 +9,14 @@ on: jobs: run-test: + name: ${{ matrix.shard_name }}(py=${{ matrix.python-version }},dj=${{ matrix.django-version }},mongo=${{ matrix.mongo-version }}) if: (github.repository != 'openedx/edx-platform' && github.repository != 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == true)) runs-on: ubuntu-20.04 strategy: - fail-fast: false matrix: - python-version: [ '3.8', '3.11' ] + python-version: + - "3.8" + - "3.11" django-version: - "pinned" # When updating the shards, remember to make the same changes in @@ -39,7 +41,19 @@ jobs: mongo-version: - "4.4" - "7.0" - name: gh-hosted-python-${{ matrix.python-version }},django-${{ matrix.django-version }},mongo-${{ matrix.mongo-version }}${{ matrix.shard_name }} + + # We only need to test Mongo 4.4 for modules that directly interface with Mongo (that is: xmodule.modulestore) + exclude: + - mongo-version: "4.4" + include: + - shard_name: "xmodule-with-cms" + python-version: "3.11" + django-version: "pinned" + mongo-version: "4.4" + - shard_name: "xmodule-with-lms" + python-version: "3.11" + django-version: "pinned" + mongo-version: "4.4" steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 9a503420248f..bf7fe86ab765 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -8,7 +8,7 @@ on: jobs: run-tests: - name: python-${{ matrix.python-version }},django-${{ matrix.django-version }},mongo-${{ matrix.mongo-version }},${{ matrix.shard_name }} + name: ${{ matrix.shard_name }}(py=${{ matrix.python-version }},dj=${{ matrix.django-version }},mongo=${{ matrix.mongo-version }}) if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false)) runs-on: [ edx-platform-runner ] strategy: @@ -40,8 +40,19 @@ jobs: mongo-version: - "4.4" - "7.0" - # We expect Django 4.0 to fail, so don't stop when it fails. - continue-on-error: ${{ matrix.django-version == '4.0' }} + + # We only need to test Mongo 4.4 for modules that directly interface with Mongo (that is: xmodule.modulestore) + exclude: + - mongo-version: "4.4" + include: + - shard_name: "xmodule-with-cms" + python-version: "3.11" + django-version: "pinned" + mongo-version: "4.4" + - shard_name: "xmodule-with-lms" + python-version: "3.11" + django-version: "pinned" + mongo-version: "4.4" steps: - name: sync directory owner From 1d23fc1089eeb1fd68bde722a89ef5d578d6d544 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Mon, 6 May 2024 09:39:09 -0400 Subject: [PATCH 033/113] fix: Include script requirements in upgrade. The `upgrade-one-python-dependency` workflow would fail if there were changes in the scripts directory because they wouldn't get added to the PR. Update the list of `add-paths` for the workflow so that it doesn't fail like this in the future. --- .github/workflows/upgrade-one-python-dependency.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/upgrade-one-python-dependency.yml b/.github/workflows/upgrade-one-python-dependency.yml index dbc7eb94487a..40ede7758138 100644 --- a/.github/workflows/upgrade-one-python-dependency.yml +++ b/.github/workflows/upgrade-one-python-dependency.yml @@ -86,7 +86,9 @@ jobs: with: branch: "${{ github.triggering_actor }}/upgrade-${{ inputs.package }}" branch-suffix: short-commit-hash - add-paths: requirements + add-paths: | + requirements + scripts/**/requirements commit-message: | feat: Upgrade Python dependency ${{ inputs.package }} From b9623833c8756c9c07536193647f73ac38c4ba84 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Mon, 6 May 2024 09:50:12 -0400 Subject: [PATCH 034/113] fix: Ensure depedency checks run on script requirements as well. --- .github/workflows/check-consistent-dependencies.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-consistent-dependencies.yml b/.github/workflows/check-consistent-dependencies.yml index b2466b883070..51a4d5f24fe2 100644 --- a/.github/workflows/check-consistent-dependencies.yml +++ b/.github/workflows/check-consistent-dependencies.yml @@ -28,7 +28,7 @@ jobs: echo $'Paths touched in PR:\n'"$paths" # The ^"? is because git may quote weird file paths - matched="$(echo "$paths" | grep -P '^"?requirements/' || true)" + matched="$(echo "$paths" | grep -P '^"?((requirements/)|(scripts/.*?/requirements/))' || true)" echo $'Relevant paths:\n'"$matched" if [[ -n "$matched" ]]; then echo "RELEVANT=true" >> "$GITHUB_ENV" From 321d40bf920708456db3e0a4c1bf247227a597ff Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Fri, 3 May 2024 15:01:05 -0400 Subject: [PATCH 035/113] build: Remove sqlparse from kernel dependencies --- requirements/edx/kernel.in | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements/edx/kernel.in b/requirements/edx/kernel.in index 253926e2b385..47a2435939af 100644 --- a/requirements/edx/kernel.in +++ b/requirements/edx/kernel.in @@ -152,7 +152,6 @@ slumber # The following dependency is unsupported an social-auth-app-django sorl-thumbnail sortedcontainers # Provides SortedKeyList, used for lists of XBlock assets -sqlparse # Required by Django to run migrations.RunSQL stevedore # Support for runtime plugins, used for XBlocks and edx-platform Django app plugins unicodecsv # Easier support for CSV files with unicode text user-util # Functionality for retiring users (GDPR compliance) From 324a2ae88a6f54ffede5c5d1c33c9e16cb1f6d14 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Fri, 3 May 2024 15:10:31 -0400 Subject: [PATCH 036/113] chore: Run `make compile-requirements` --- requirements/edx/base.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index c364b075220b..a51c04e18de0 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -1120,7 +1120,6 @@ soupsieve==2.5 # via beautifulsoup4 sqlparse==0.4.4 # via - # -r requirements/edx/kernel.in # django # openedx-blockstore staff-graded-xblock==2.3.0 From 25ecaa8f76e872e9f607ec8a2f2647ff7d9152ec Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Fri, 3 May 2024 15:22:33 -0400 Subject: [PATCH 037/113] chore: Run `package=sqlparse make upgrade-package` --- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/doc.txt | 2 +- requirements/edx/testing.txt | 2 +- scripts/user_retirement/requirements/base.txt | 2 +- scripts/user_retirement/requirements/testing.txt | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index a51c04e18de0..faae22c957c6 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -1118,7 +1118,7 @@ sortedcontainers==2.4.0 # snowflake-connector-python soupsieve==2.5 # via beautifulsoup4 -sqlparse==0.4.4 +sqlparse==0.5.0 # via # django # openedx-blockstore diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 796d806bbd1e..2be48ab94263 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -1959,7 +1959,7 @@ sphinxcontrib-serializinghtml==1.1.5 # sphinx sphinxext-rediraffe==0.2.7 # via -r requirements/edx/doc.txt -sqlparse==0.4.4 +sqlparse==0.5.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 3276a559f892..aaac2b369436 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -1359,7 +1359,7 @@ sphinxcontrib-serializinghtml==1.1.5 # via sphinx sphinxext-rediraffe==0.2.7 # via -r requirements/edx/doc.in -sqlparse==0.4.4 +sqlparse==0.5.0 # via # -r requirements/edx/base.txt # django diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index bfc44d2fc44a..3470e1dfac40 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -1445,7 +1445,7 @@ soupsieve==2.5 # via # -r requirements/edx/base.txt # beautifulsoup4 -sqlparse==0.4.4 +sqlparse==0.5.0 # via # -r requirements/edx/base.txt # django diff --git a/scripts/user_retirement/requirements/base.txt b/scripts/user_retirement/requirements/base.txt index bd3fc4848f4f..b272793aa4c4 100644 --- a/scripts/user_retirement/requirements/base.txt +++ b/scripts/user_retirement/requirements/base.txt @@ -164,7 +164,7 @@ six==1.16.0 # requests-file slumber==0.7.1 # via edx-rest-api-client -sqlparse==0.4.4 +sqlparse==0.5.0 # via django stevedore==5.1.0 # via edx-django-utils diff --git a/scripts/user_retirement/requirements/testing.txt b/scripts/user_retirement/requirements/testing.txt index 37444f469465..f30e373cbce5 100644 --- a/scripts/user_retirement/requirements/testing.txt +++ b/scripts/user_retirement/requirements/testing.txt @@ -268,7 +268,7 @@ slumber==0.7.1 # via # -r scripts/user_retirement/requirements/base.txt # edx-rest-api-client -sqlparse==0.4.4 +sqlparse==0.5.0 # via # -r scripts/user_retirement/requirements/base.txt # django From 5ce580e2f6484ce07e20d5c113742e6ed625fd9d Mon Sep 17 00:00:00 2001 From: feanil Date: Mon, 6 May 2024 13:36:01 +0000 Subject: [PATCH 038/113] feat: Upgrade Python dependency faker Routine requirement upgrade. Doing it individually because there are too many changes in `make upgrade`. Commit generated by workflow `openedx/edx-platform/.github/workflows/upgrade-one-python-dependency.yml@refs/heads/master` --- requirements/edx/development.txt | 3 +-- requirements/edx/testing.txt | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 796d806bbd1e..3ff1b887a88b 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -895,7 +895,7 @@ execnet==2.0.2 # pytest-xdist factory-boy==3.3.0 # via -r requirements/edx/testing.txt -faker==24.14.0 +faker==25.0.1 # via # -r requirements/edx/testing.txt # factory-boy @@ -2062,7 +2062,6 @@ typing-extensions==4.9.0 # djangorestframework-stubs # drf-spectacular # edx-opaque-keys - # faker # fastapi # grimp # import-linter diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index bfc44d2fc44a..32b15dd73541 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -687,7 +687,7 @@ execnet==2.0.2 # via pytest-xdist factory-boy==3.3.0 # via -r requirements/edx/testing.in -faker==24.14.0 +faker==25.0.1 # via factory-boy fastapi==0.109.2 # via pact-python @@ -1513,7 +1513,6 @@ typing-extensions==4.9.0 # django-countries # drf-spectacular # edx-opaque-keys - # faker # fastapi # grimp # import-linter From a454da9ca65e3c567d34ec4380efcb8972dd909a Mon Sep 17 00:00:00 2001 From: Muhammad Adeel Tajamul <77053848+muhammadadeeltajamul@users.noreply.github.com> Date: Tue, 7 May 2024 12:15:03 +0500 Subject: [PATCH 039/113] feat: added daily and weekly email digest (#34539) * feat: added daily and weekly email digest --- .../djangoapps/notifications/config/waffle.py | 14 +- .../djangoapps/notifications/email/tasks.py | 116 +++++++++++ .../notifications/email/tests/__init__.py | 0 .../notifications/email/tests/test_tasks.py | 195 +++++++++++++++++ .../notifications/email/tests/test_utils.py | 196 +++++++++++++++++ .../notifications/email/tests/utils.py | 40 ++++ .../djangoapps/notifications/email/utils.py | 197 +++++++++++++++++- .../management/commands/send_email_digest.py | 32 +++ .../notifications/digest_content.html | 29 +-- .../notifications/digest_header.html | 4 +- .../notifications/tests/test_tasks.py | 64 ++++++ 11 files changed, 862 insertions(+), 25 deletions(-) create mode 100644 openedx/core/djangoapps/notifications/email/tasks.py create mode 100644 openedx/core/djangoapps/notifications/email/tests/__init__.py create mode 100644 openedx/core/djangoapps/notifications/email/tests/test_tasks.py create mode 100644 openedx/core/djangoapps/notifications/email/tests/test_utils.py create mode 100644 openedx/core/djangoapps/notifications/email/tests/utils.py create mode 100644 openedx/core/djangoapps/notifications/management/commands/send_email_digest.py diff --git a/openedx/core/djangoapps/notifications/config/waffle.py b/openedx/core/djangoapps/notifications/config/waffle.py index 9d54a0abb8fd..fa1d02adf1d3 100644 --- a/openedx/core/djangoapps/notifications/config/waffle.py +++ b/openedx/core/djangoapps/notifications/config/waffle.py @@ -3,7 +3,7 @@ waffle switches for the notifications app. """ -from openedx.core.djangoapps.waffle_utils import CourseWaffleFlag +from openedx.core.djangoapps.waffle_utils import CourseWaffleFlag, WaffleFlag WAFFLE_NAMESPACE = 'notifications' @@ -48,7 +48,6 @@ # .. toggle_tickets: INF-1145 ENABLE_COURSEWIDE_NOTIFICATIONS = CourseWaffleFlag(f"{WAFFLE_NAMESPACE}.enable_coursewide_notifications", __name__) - # .. toggle_name: notifications.enable_ora_staff_notifications # .. toggle_implementation: CourseWaffleFlag # .. toggle_default: False @@ -58,3 +57,14 @@ # .. toggle_target_removal_date: 2024-06-04 # .. toggle_tickets: INF-1304 ENABLE_ORA_STAFF_NOTIFICATION = CourseWaffleFlag(f"{WAFFLE_NAMESPACE}.enable_ora_staff_notifications", __name__) + +# .. toggle_name: notifications.enable_email_notifications +# .. toggle_implementation: WaffleFlag +# .. toggle_default: False +# .. toggle_description: Waffle flag to enable the Email Notifications feature +# .. toggle_use_cases: temporary, open_edx +# .. toggle_creation_date: 2024-03-25 +# .. toggle_target_removal_date: 2025-06-01 +# .. toggle_warning: When the flag is ON, Email Notifications feature is enabled. +# .. toggle_tickets: INF-1259 +ENABLE_EMAIL_NOTIFICATIONS = WaffleFlag(f'{WAFFLE_NAMESPACE}.enable_email_notifications', __name__) diff --git a/openedx/core/djangoapps/notifications/email/tasks.py b/openedx/core/djangoapps/notifications/email/tasks.py new file mode 100644 index 000000000000..3b1783f42b3a --- /dev/null +++ b/openedx/core/djangoapps/notifications/email/tasks.py @@ -0,0 +1,116 @@ +""" +Celery tasks for sending email notifications +""" +from celery import shared_task +from celery.utils.log import get_task_logger +from django.contrib.auth import get_user_model +from edx_ace import ace +from edx_ace.recipient import Recipient +from edx_django_utils.monitoring import set_code_owner_attribute + +from openedx.core.djangoapps.notifications.email_notifications import EmailCadence +from openedx.core.djangoapps.notifications.models import ( + CourseNotificationPreference, + Notification, + get_course_notification_preference_config_version +) +from .message_type import EmailNotificationMessageType +from .utils import ( + create_app_notifications_dict, + create_email_digest_context, + filter_notification_with_email_enabled_preferences, + get_start_end_date, + get_unique_course_ids, + is_email_notification_flag_enabled +) + + +User = get_user_model() +logger = get_task_logger(__name__) + + +def get_audience_for_cadence_email(cadence_type): + """ + Returns users that are eligible to receive cadence email + """ + if cadence_type not in [EmailCadence.DAILY, EmailCadence.WEEKLY]: + raise ValueError("Invalid value for parameter cadence_type") + start_date, end_date = get_start_end_date(cadence_type) + user_ids = Notification.objects.filter( + email=True, + created__gte=start_date, + created__lte=end_date + ).values_list('user__id', flat=True).distinct() + users = User.objects.filter(id__in=user_ids) + return users + + +def get_user_preferences_for_courses(course_ids, user): + """ + Returns updated user preference for course_ids + """ + # Create new preferences + new_preferences = [] + preferences = CourseNotificationPreference.objects.filter(user=user, course_id__in=course_ids) + preferences = list(preferences) + for course_id in course_ids: + if not any(preference.course_id == course_id for preference in preferences): + pref = CourseNotificationPreference(user=user, course_id=course_id) + new_preferences.append(pref) + if new_preferences: + CourseNotificationPreference.objects.bulk_create(new_preferences, ignore_conflicts=True) + # Update preferences to latest config version + current_version = get_course_notification_preference_config_version() + for preference in preferences: + if preference.config_version != current_version: + preference = preference.get_user_course_preference(user.id, preference.course_id) + new_preferences.append(preference) + return new_preferences + + +def send_digest_email_to_user(user, cadence_type, course_language='en', courses_data=None): + """ + Send [cadence_type] email to user. + Cadence Type can be EmailCadence.DAILY or EmailCadence.WEEKLY + """ + if cadence_type not in [EmailCadence.DAILY, EmailCadence.WEEKLY]: + raise ValueError('Invalid cadence_type') + logger.info(f' Sending email to user {user.username} ==Temp Log==') + if not is_email_notification_flag_enabled(user): + logger.info(f' Flag disabled for {user.username} ==Temp Log==') + return + start_date, end_date = get_start_end_date(cadence_type) + notifications = Notification.objects.filter(user=user, email=True, + created__gte=start_date, created__lte=end_date) + if not notifications: + logger.info(f' No notification for {user.username} ==Temp Log==') + return + course_ids = get_unique_course_ids(notifications) + preferences = get_user_preferences_for_courses(course_ids, user) + notifications = filter_notification_with_email_enabled_preferences(notifications, preferences, cadence_type) + if not notifications: + logger.info(f' No filtered notification for {user.username} ==Temp Log==') + return + apps_dict = create_app_notifications_dict(notifications) + message_context = create_email_digest_context(apps_dict, start_date, end_date, cadence_type, + courses_data=courses_data) + recipient = Recipient(user.id, user.email) + message = EmailNotificationMessageType( + app_label="notifications", name="email_digest" + ).personalize(recipient, course_language, message_context) + ace.send(message) + logger.info(f' Email sent to {user.username} ==Temp Log==') + + +@shared_task(ignore_result=True) +@set_code_owner_attribute +def send_digest_email_to_all_users(cadence_type): + """ + Send email digest to all eligible users + """ + logger.info(f' Sending cadence email of type {cadence_type}') + users = get_audience_for_cadence_email(cadence_type) + courses_data = {} + logger.info(f' Email Cadence Audience {len(users)}') + for user in users: + send_digest_email_to_user(user, cadence_type, courses_data=courses_data) diff --git a/openedx/core/djangoapps/notifications/email/tests/__init__.py b/openedx/core/djangoapps/notifications/email/tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/openedx/core/djangoapps/notifications/email/tests/test_tasks.py b/openedx/core/djangoapps/notifications/email/tests/test_tasks.py new file mode 100644 index 000000000000..78dbc957833b --- /dev/null +++ b/openedx/core/djangoapps/notifications/email/tests/test_tasks.py @@ -0,0 +1,195 @@ +""" +Test cases for notifications/email/tasks.py +""" +import datetime +import ddt + +from unittest.mock import patch + +from edx_toggles.toggles.testutils import override_waffle_flag + +from common.djangoapps.student.tests.factories import UserFactory +from openedx.core.djangoapps.notifications.config.waffle import ENABLE_EMAIL_NOTIFICATIONS +from openedx.core.djangoapps.notifications.email_notifications import EmailCadence +from openedx.core.djangoapps.notifications.email.tasks import ( + get_audience_for_cadence_email, + send_digest_email_to_all_users, + send_digest_email_to_user +) +from openedx.core.djangoapps.notifications.models import CourseNotificationPreference +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory + +from .utils import create_notification + + +@ddt.ddt +class TestEmailDigestForUser(ModuleStoreTestCase): + """ + Tests email notification for a specific user + """ + + def setUp(self): + """ + Setup + """ + super().setUp() + self.user = UserFactory() + self.course = CourseFactory.create(display_name='test course', run="Testing_course") + + @patch('edx_ace.ace.send') + def test_email_is_not_sent_if_no_notifications(self, mock_func): + """ + Tests email is sent iff waffle flag is enabled + """ + with override_waffle_flag(ENABLE_EMAIL_NOTIFICATIONS, True): + send_digest_email_to_user(self.user, EmailCadence.DAILY) + assert not mock_func.called + + @ddt.data(True, False) + @patch('edx_ace.ace.send') + def test_email_is_sent_iff_flag_enabled(self, flag_value, mock_func): + """ + Tests email is sent iff waffle flag is enabled + """ + created_date = datetime.datetime.now() - datetime.timedelta(days=1) + create_notification(self.user, self.course.id, created=created_date) + with override_waffle_flag(ENABLE_EMAIL_NOTIFICATIONS, flag_value): + send_digest_email_to_user(self.user, EmailCadence.DAILY) + assert mock_func.called is flag_value + + @patch('edx_ace.ace.send') + def test_notification_not_send_if_created_on_next_day(self, mock_func): + """ + Tests email is not sent if notification is created on next day + """ + create_notification(self.user, self.course.id) + with override_waffle_flag(ENABLE_EMAIL_NOTIFICATIONS, True): + send_digest_email_to_user(self.user, EmailCadence.DAILY) + assert not mock_func.called + + @patch('edx_ace.ace.send') + def test_notification_not_send_if_created_day_before_yesterday(self, mock_func): + """ + Tests email is not sent if notification is created day before yesterday + """ + created_date = datetime.datetime.now() - datetime.timedelta(days=2) + create_notification(self.user, self.course.id, created=created_date) + with override_waffle_flag(ENABLE_EMAIL_NOTIFICATIONS, True): + send_digest_email_to_user(self.user, EmailCadence.DAILY) + assert not mock_func.called + + +@ddt.ddt +class TestEmailDigestAudience(ModuleStoreTestCase): + """ + Tests audience for notification digest email + """ + + def setUp(self): + """ + Setup + """ + super().setUp() + self.user = UserFactory() + self.course = CourseFactory.create(display_name='test course', run="Testing_course") + + @patch('openedx.core.djangoapps.notifications.email.tasks.send_digest_email_to_user') + def test_email_func_not_called_if_no_notification(self, mock_func): + """ + Tests email sending function is not called if user has no notifications + """ + with override_waffle_flag(ENABLE_EMAIL_NOTIFICATIONS, True): + send_digest_email_to_all_users(EmailCadence.DAILY) + assert not mock_func.called + + @patch('openedx.core.djangoapps.notifications.email.tasks.send_digest_email_to_user') + def test_email_func_called_if_user_has_notification(self, mock_func): + """ + Tests email sending function is called if user has notification + """ + created_date = datetime.datetime.now() - datetime.timedelta(days=1) + create_notification(self.user, self.course.id, created=created_date) + with override_waffle_flag(ENABLE_EMAIL_NOTIFICATIONS, True): + send_digest_email_to_all_users(EmailCadence.DAILY) + assert mock_func.called + + @patch('openedx.core.djangoapps.notifications.email.tasks.send_digest_email_to_user') + def test_email_func_not_called_if_user_notification_is_not_duration(self, mock_func): + """ + Tests email sending function is not called if user has notification + which is not in duration + """ + created_date = datetime.datetime.now() - datetime.timedelta(days=10) + create_notification(self.user, self.course.id, created=created_date) + with override_waffle_flag(ENABLE_EMAIL_NOTIFICATIONS, True): + send_digest_email_to_all_users(EmailCadence.DAILY) + assert not mock_func.called + + @patch('edx_ace.ace.send') + def test_email_is_sent_to_user_when_task_is_called(self, mock_func): + created_date = datetime.datetime.now() - datetime.timedelta(days=1) + create_notification(self.user, self.course.id, created=created_date) + with override_waffle_flag(ENABLE_EMAIL_NOTIFICATIONS, True): + send_digest_email_to_all_users(EmailCadence.DAILY) + assert mock_func.called + assert mock_func.call_count == 1 + + def test_audience_query_count(self): + with self.assertNumQueries(1): + audience = get_audience_for_cadence_email(EmailCadence.DAILY) + list(audience) # evaluating queryset + + @ddt.data(True, False) + @patch('edx_ace.ace.send') + def test_digest_should_contain_email_enabled_notifications(self, email_value, mock_func): + """ + Tests email is sent only when notifications with email=True exists + """ + created_date = datetime.datetime.now() - datetime.timedelta(days=1) + create_notification(self.user, self.course.id, created=created_date, email=email_value) + with override_waffle_flag(ENABLE_EMAIL_NOTIFICATIONS, True): + send_digest_email_to_user(self.user, EmailCadence.DAILY) + assert mock_func.called is email_value + + +class TestPreferences(ModuleStoreTestCase): + """ + Tests preferences + """ + def setUp(self): + """ + Setup + """ + super().setUp() + self.user = UserFactory() + self.course = CourseFactory.create(display_name='test course', run="Testing_course") + self.preference = CourseNotificationPreference.objects.create(user=self.user, course_id=self.course.id) + created_date = datetime.datetime.now() - datetime.timedelta(days=1) + create_notification(self.user, self.course.id, notification_type='new_discussion_post', created=created_date) + + @patch('edx_ace.ace.send') + def test_email_send_for_digest_preference(self, mock_func): + """ + Tests email is send for digest notification preference + """ + config = self.preference.notification_preference_config + types = config['discussion']['notification_types'] + types['new_discussion_post']['email_cadence'] = EmailCadence.DAILY + self.preference.save() + with override_waffle_flag(ENABLE_EMAIL_NOTIFICATIONS, True): + send_digest_email_to_user(self.user, EmailCadence.DAILY) + assert mock_func.called + + @patch('edx_ace.ace.send') + def test_email_not_send_if_different_digest_preference(self, mock_func): + """ + Tests email is not send if digest notification preference doesnot match + """ + config = self.preference.notification_preference_config + types = config['discussion']['notification_types'] + types['new_discussion_post']['email_cadence'] = EmailCadence.WEEKLY + self.preference.save() + with override_waffle_flag(ENABLE_EMAIL_NOTIFICATIONS, True): + send_digest_email_to_user(self.user, EmailCadence.DAILY) + assert not mock_func.called diff --git a/openedx/core/djangoapps/notifications/email/tests/test_utils.py b/openedx/core/djangoapps/notifications/email/tests/test_utils.py new file mode 100644 index 000000000000..d7c9f6c98133 --- /dev/null +++ b/openedx/core/djangoapps/notifications/email/tests/test_utils.py @@ -0,0 +1,196 @@ +""" +Test utils.py +""" +import datetime +import ddt + +from pytz import utc +from waffle import get_waffle_flag_model # pylint: disable=invalid-django-waffle-import + +from common.djangoapps.student.tests.factories import UserFactory +from openedx.core.djangoapps.notifications.config.waffle import ENABLE_EMAIL_NOTIFICATIONS +from openedx.core.djangoapps.notifications.models import Notification +from openedx.core.djangoapps.notifications.email.utils import ( + add_additional_attributes_to_notifications, + create_app_notifications_dict, + create_datetime_string, + create_email_digest_context, + create_email_template_context, + get_course_info, + get_time_ago, + is_email_notification_flag_enabled, +) +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory + +from .utils import assert_list_equal, create_notification + + +class TestUtilFunctions(ModuleStoreTestCase): + """ + Test utils functions + """ + def setUp(self): + """ + Setup + """ + super().setUp() + self.user = UserFactory() + self.course = CourseFactory.create(display_name='test course', run="Testing_course") + + def test_additional_attributes(self): + """ + Tests additional attributes are added when notifications list is passed to + add_additional_attributes_to_notifications function + """ + notification = create_notification(self.user, self.course.id) + additional_params = ['course_name', 'icon', 'time_ago'] + for param in additional_params: + assert not hasattr(notification, param) + add_additional_attributes_to_notifications([notification]) + for param in additional_params: + assert hasattr(notification, param) + + def test_create_app_notifications_dict(self): + """ + Tests notifications are divided based on their app_name + """ + Notification.objects.all().delete() + create_notification(self.user, self.course.id, app_name='discussion', notification_type='new_comment') + create_notification(self.user, self.course.id, app_name='updates', notification_type='course_update') + app_dict = create_app_notifications_dict(Notification.objects.all()) + assert len(app_dict.keys()) == 2 + for key in ['discussion', 'updates']: + assert key in app_dict.keys() + assert app_dict[key]['count'] == 1 + assert len(app_dict[key]['notifications']) == 1 + + def test_get_course_info(self): + """ + Tests get_course_info function + """ + assert get_course_info(self.course.id) == {'name': 'test course'} + + def test_get_time_ago(self): + """ + Tests time_ago string + """ + current_datetime = utc.localize(datetime.datetime.now()) + assert "Today" == get_time_ago(current_datetime) + assert "1d" == get_time_ago(current_datetime - datetime.timedelta(days=1)) + assert "1w" == get_time_ago(current_datetime - datetime.timedelta(days=7)) + + def test_datetime_string(self): + dt = datetime.datetime(2024, 3, 25) + assert create_datetime_string(dt) == "Monday, Mar 25" + + +@ddt.ddt +class TestContextFunctions(ModuleStoreTestCase): + """ + Test template context functions in utils.py + """ + def setUp(self): + """ + Setup + """ + super().setUp() + self.user = UserFactory() + self.course = CourseFactory.create(display_name='test course', run="Testing_course") + + def test_email_template_context(self): + """ + Tests common header and footer context + """ + context = create_email_template_context() + keys = ['platform_name', 'mailing_address', 'logo_url', 'social_media', 'notification_settings_url'] + for key in keys: + assert key in context + + @ddt.data('Daily', 'Weekly') + def test_email_digest_context(self, digest_frequency): + """ + Tests context for email digest + """ + Notification.objects.all().delete() + discussion_notification = create_notification(self.user, self.course.id, app_name='discussion', + notification_type='new_comment') + update_notification = create_notification(self.user, self.course.id, app_name='updates', + notification_type='course_update') + app_dict = create_app_notifications_dict(Notification.objects.all()) + end_date = datetime.datetime(2024, 3, 24, 12, 0) + params = { + "app_notifications_dict": app_dict, + "start_date": end_date - datetime.timedelta(days=0 if digest_frequency == "Daily" else 6), + "end_date": end_date, + "digest_frequency": digest_frequency, + "courses_data": None + } + context = create_email_digest_context(**params) + expected_start_date = 'Sunday, Mar 24' if digest_frequency == 'Daily' else 'Monday, Mar 18' + expected_digest_updates = [ + {'title': 'Total Notifications', 'count': 2}, + {'title': 'Discussion', 'count': 1}, + {'title': 'Updates', 'count': 1}, + ] + expected_email_content = [ + {'title': 'Discussion', 'help_text': '', 'help_text_url': '', 'notifications': [discussion_notification]}, + {'title': 'Updates', 'help_text': '', 'help_text_url': '', 'notifications': [update_notification]} + ] + assert context['start_date'] == expected_start_date + assert context['end_date'] == 'Sunday, Mar 24' + assert context['digest_frequency'] == digest_frequency + assert_list_equal(context['email_digest_updates'], expected_digest_updates) + assert_list_equal(context['email_content'], expected_email_content) + + +class TestWaffleFlag(ModuleStoreTestCase): + """ + Test user level email notifications waffle flag + """ + def setUp(self): + """ + Setup + """ + super().setUp() + self.user_1 = UserFactory() + self.user_2 = UserFactory() + self.course_1 = CourseFactory.create(display_name='test course 1', run="Testing_course_1") + self.course_1 = CourseFactory.create(display_name='test course 2', run="Testing_course_2") + + def test_waffle_flag_for_everyone(self): + """ + Tests if waffle flag is enabled for everyone + """ + assert is_email_notification_flag_enabled() is False + waffle_model = get_waffle_flag_model() + flag, _ = waffle_model.objects.get_or_create(name=ENABLE_EMAIL_NOTIFICATIONS.name) + flag.everyone = True + flag.save() + assert is_email_notification_flag_enabled() is True + + def test_waffle_flag_for_user(self): + """ + Tests user level waffle flag + """ + assert is_email_notification_flag_enabled() is False + waffle_model = get_waffle_flag_model() + flag, _ = waffle_model.objects.get_or_create(name=ENABLE_EMAIL_NOTIFICATIONS.name) + flag.users.add(self.user_1) + flag.save() + assert is_email_notification_flag_enabled(self.user_1) is True + assert is_email_notification_flag_enabled(self.user_2) is False + + def test_waffle_flag_everyone_priority(self): + """ + Tests if everyone field has more priority over user field + """ + assert is_email_notification_flag_enabled() is False + waffle_model = get_waffle_flag_model() + flag, _ = waffle_model.objects.get_or_create(name=ENABLE_EMAIL_NOTIFICATIONS.name) + flag.everyone = False + flag.users.add(self.user_1) + flag.save() + assert is_email_notification_flag_enabled() is False + assert is_email_notification_flag_enabled(self.user_1) is False + assert is_email_notification_flag_enabled(self.user_2) is False diff --git a/openedx/core/djangoapps/notifications/email/tests/utils.py b/openedx/core/djangoapps/notifications/email/tests/utils.py new file mode 100644 index 000000000000..c27f60443b31 --- /dev/null +++ b/openedx/core/djangoapps/notifications/email/tests/utils.py @@ -0,0 +1,40 @@ +""" +Utils for tests +""" +from openedx.core.djangoapps.notifications.models import Notification + + +def create_notification(user, course_key, **kwargs): + """ + Create a test notification + """ + notification_params = { + 'user': user, + 'course_id': course_key, + 'app_name': "discussion", + 'notification_type': "new_comment", + 'content_url': '', + 'content_context': { + "replier_name": "replier", + "username": "username", + "author_name": "author_name", + "post_title": "post_title", + "course_update_content": "Course update content", + "content_type": 'post', + "content": "post_title" + }, + 'email': True, + 'web': True + } + notification_params.update(kwargs) + notification = Notification.objects.create(**notification_params) + return notification + + +def assert_list_equal(list_1, list_2): + """ + Asserts if list is equal + """ + assert len(list_1) == len(list_2) + for element in list_1: + assert element in list_2 diff --git a/openedx/core/djangoapps/notifications/email/utils.py b/openedx/core/djangoapps/notifications/email/utils.py index 2993b68fc2dd..9538620b06c4 100644 --- a/openedx/core/djangoapps/notifications/email/utils.py +++ b/openedx/core/djangoapps/notifications/email/utils.py @@ -1,12 +1,46 @@ """ Email Notifications Utils """ +import datetime + from django.conf import settings +from pytz import utc +from waffle import get_waffle_flag_model # pylint: disable=invalid-django-waffle-import + from lms.djangoapps.branding.api import get_logo_url_for_email +from openedx.core.djangoapps.notifications.config.waffle import ENABLE_EMAIL_NOTIFICATIONS +from openedx.core.djangoapps.notifications.email_notifications import EmailCadence +from xmodule.modulestore.django import modulestore + from .notification_icons import NotificationTypeIcons +def is_email_notification_flag_enabled(user=None): + """ + Returns if waffle flag is enabled for user or not + """ + flag_model = get_waffle_flag_model() + try: + flag = flag_model.objects.get(name=ENABLE_EMAIL_NOTIFICATIONS.name) + except flag_model.DoesNotExist: + return False + if flag.everyone is not None: + return flag.everyone + if user: + role_value = flag.is_active_for_user(user) + if role_value is not None: + return role_value + try: + return flag.users.contains(user) + except ValueError: + pass + return False + + def create_datetime_string(datetime_instance): + """ + Returns string for datetime object + """ return datetime_instance.strftime('%A, %b %d') @@ -40,25 +74,172 @@ def create_email_template_context(): } -def create_email_digest_content(start_date, end_date=None, digest_frequency="Daily", - notifications_count=0, updates_count=0, email_content=None): +def create_email_digest_context(app_notifications_dict, start_date, end_date=None, digest_frequency="Daily", + courses_data=None): """ Creates email context based on content + app_notifications_dict: Mapping of notification app and its count, title and notifications start_date: datetime instance end_date: datetime instance + digest_frequency: EmailCadence.DAILY or EmailCadence.WEEKLY + courses_data: Dictionary to cache course info (avoid additional db calls) """ context = create_email_template_context() start_date_str = create_datetime_string(start_date) end_date_str = create_datetime_string(end_date if end_date else start_date) + email_digest_updates = [{ + 'title': 'Total Notifications', + 'count': sum(value['count'] for value in app_notifications_dict.values()) + }] + email_digest_updates.extend([ + { + 'title': value['title'], + 'count': value['count'], + } + for key, value in app_notifications_dict.items() + ]) + email_content = [ + { + 'title': value['title'], + 'help_text': value.get('help_text', ''), + 'help_text_url': value.get('help_text_url', ''), + 'notifications': add_additional_attributes_to_notifications( + value.get('notifications', []), courses_data=courses_data + ) + } + for key, value in app_notifications_dict.items() + ] context.update({ "start_date": start_date_str, "end_date": end_date_str, "digest_frequency": digest_frequency, - "updates": [ - {"count": updates_count, "type": "Updates"}, - {"count": notifications_count, "type": "Notifications"} - ], - "email_content": email_content if email_content else [], - "get_icon_url_for_notification_type": get_icon_url_for_notification_type, + "email_digest_updates": email_digest_updates, + "email_content": email_content, }) return context + + +def get_start_end_date(cadence_type): + """ + Returns start_date and end_date for email digest + """ + if cadence_type not in [EmailCadence.DAILY, EmailCadence.WEEKLY]: + raise ValueError('Invalid cadence_type') + date_today = datetime.datetime.now() + yesterday = date_today - datetime.timedelta(days=1) + end_date = datetime.datetime.combine(yesterday, datetime.time.max) + start_date = end_date + if cadence_type == EmailCadence.WEEKLY: + start_date = end_date - datetime.timedelta(days=6) + start_date = datetime.datetime.combine(start_date, datetime.time.min) + return utc.localize(start_date), utc.localize(end_date) + + +def get_course_info(course_key): + """ + Returns course info for course_key + """ + store = modulestore() + course = store.get_course(course_key) + return {'name': course.display_name} + + +def get_time_ago(datetime_obj): + """ + Returns time_ago for datetime instance + """ + current_date = utc.localize(datetime.datetime.today()) + days_diff = (current_date - datetime_obj).days + if days_diff == 0: + return "Today" + if days_diff >= 7: + return f"{int(days_diff / 7)}w" + return f"{days_diff}d" + + +def add_additional_attributes_to_notifications(notifications, courses_data=None): + """ + Add attributes required for email content to notification instance + notifications: list[Notification] + course_data: Cache course info + """ + if courses_data is None: + courses_data = {} + + for notification in notifications: + notification_type = notification.notification_type + course_key = notification.course_id + course_key_str = str(course_key) + if course_key_str not in courses_data.keys(): + courses_data[course_key_str] = get_course_info(course_key) + course_info = courses_data[course_key_str] + notification.course_name = course_info.get('name', '') + notification.icon = get_icon_url_for_notification_type(notification_type) + notification.time_ago = get_time_ago(notification.created) + return notifications + + +def create_app_notifications_dict(notifications): + """ + Return a dictionary with notification app as key and + title, count and notifications as its value + """ + app_names = list({notification.app_name for notification in notifications}) + app_notifications = { + name: { + 'count': 0, + 'title': name.title(), + 'notifications': [] + } + for name in app_names + } + for notification in notifications: + app_data = app_notifications[notification.app_name] + app_data['count'] += 1 + app_data['notifications'].append(notification) + return app_notifications + + +def get_unique_course_ids(notifications): + """ + Returns unique course_ids from notifications + """ + course_ids = [] + for notification in notifications: + if notification.course_id not in course_ids: + course_ids.append(notification.course_id) + return course_ids + + +def get_enabled_notification_types_for_cadence(preferences, cadence_type=EmailCadence.DAILY): + """ + Returns a dictionary that returns notification_types with cadence_types for course_ids + """ + if cadence_type not in [EmailCadence.DAILY, EmailCadence.WEEKLY]: + raise ValueError('Invalid cadence_type') + course_types = {} + for preference in preferences: + key = preference.course_id + value = [] + config = preference.notification_preference_config + for app_data in config.values(): + for notification_type, type_dict in app_data['notification_types'].items(): + if type_dict['email_cadence'] == cadence_type: + value.append(notification_type) + if 'core' in value: + value.remove('core') + value.extend(app_data['core_notification_types']) + course_types[key] = value + return course_types + + +def filter_notification_with_email_enabled_preferences(notifications, preferences, cadence_type=EmailCadence.DAILY): + """ + Filter notifications for types with email cadence preference enabled + """ + enabled_course_prefs = get_enabled_notification_types_for_cadence(preferences, cadence_type) + filtered_notifications = [] + for notification in notifications: + if notification.notification_type in enabled_course_prefs[notification.course_id]: + filtered_notifications.append(notification) + return filtered_notifications diff --git a/openedx/core/djangoapps/notifications/management/commands/send_email_digest.py b/openedx/core/djangoapps/notifications/management/commands/send_email_digest.py new file mode 100644 index 000000000000..cfaef2d15f67 --- /dev/null +++ b/openedx/core/djangoapps/notifications/management/commands/send_email_digest.py @@ -0,0 +1,32 @@ +""" +Management command for sending email digest +""" +from django.core.management.base import BaseCommand + +from openedx.core.djangoapps.notifications.email_notifications import EmailCadence +from openedx.core.djangoapps.notifications.email.tasks import send_digest_email_to_all_users + + +class Command(BaseCommand): + """ + Invoke with: + + python manage.py lms send_email_digest [cadence_type] + cadence_type: Daily or Weekly + """ + help = ( + "Send email digest to user." + ) + + def add_arguments(self, parser): + """ + Adds management commands parser arguments + """ + cadence_type_choices = [EmailCadence.DAILY, EmailCadence.WEEKLY] + parser.add_argument('cadence_type', choices=cadence_type_choices, required=True) + + def handle(self, *args, **kwargs): + """ + Start task to send email digest to users + """ + send_digest_email_to_all_users.delay(args=(kwargs['cadence_type'],)) diff --git a/openedx/core/djangoapps/notifications/templates/notifications/digest_content.html b/openedx/core/djangoapps/notifications/templates/notifications/digest_content.html index aa1ee903ad7e..f7cb37fcfe79 100644 --- a/openedx/core/djangoapps/notifications/templates/notifications/digest_content.html +++ b/openedx/core/djangoapps/notifications/templates/notifications/digest_content.html @@ -1,15 +1,18 @@ -{% for update in email_content %} + +{% for notification_app in email_content %}

- {{ update.title }} + {{ notification_app.title }}

- {% if update.help_text %} + {% if notification_app.help_text %}

- {{ update.help_text }} + {{ notification_app.help_text }} - {% if update.help_text_url %} + {% if notification_app.help_text_url %} - + View all @@ -20,27 +23,27 @@

- {% for content in update.content %} + {% for notification in notification_app.notifications %} - From f976266b21a01cb8d16eb6f194a48efc252bdef1 Mon Sep 17 00:00:00 2001 From: sundasnoreen12 Date: Thu, 9 May 2024 13:46:46 +0500 Subject: [PATCH 083/113] refactor: paragraph tag is not holding children in email template --- .../notifications/digest_header.html | 48 +++++++------------ 1 file changed, 17 insertions(+), 31 deletions(-) diff --git a/openedx/core/djangoapps/notifications/templates/notifications/digest_header.html b/openedx/core/djangoapps/notifications/templates/notifications/digest_header.html index 1f0a533aea54..9791201b5dc3 100644 --- a/openedx/core/djangoapps/notifications/templates/notifications/digest_header.html +++ b/openedx/core/djangoapps/notifications/templates/notifications/digest_header.html @@ -38,39 +38,25 @@ {% for update in email_digest_updates %} - {% if forloop.last or forloop.revcounter0 == 1 %} - {% if email_digest_updates|length|add:'1'|divisibleby:3 and not forloop.last%} - - {% elif email_digest_updates|length|add:'2'|divisibleby:3 and forloop.last%} - - {% endif %} - {% endif %} - {% if forloop.counter|divisibleby:3 %} - - {% endif %} + + {% endif %} {% endfor %}
+

- {{ content.title }} + {{ notification.content | safe }}

- {{ content.course_name }} + {{ notification.course_name }} · - {{ content.time_ago }} + {{ notification.time_ago }} - + View diff --git a/openedx/core/djangoapps/notifications/templates/notifications/digest_header.html b/openedx/core/djangoapps/notifications/templates/notifications/digest_header.html index fc91d77fa0bf..82077e37c0fd 100644 --- a/openedx/core/djangoapps/notifications/templates/notifications/digest_header.html +++ b/openedx/core/djangoapps/notifications/templates/notifications/digest_header.html @@ -37,7 +37,7 @@ - {% for update in updates %} + {% for update in email_digest_updates %} diff --git a/openedx/core/djangoapps/notifications/tests/test_tasks.py b/openedx/core/djangoapps/notifications/tests/test_tasks.py index 83ac9d17633e..036d4d326198 100644 --- a/openedx/core/djangoapps/notifications/tests/test_tasks.py +++ b/openedx/core/djangoapps/notifications/tests/test_tasks.py @@ -433,3 +433,67 @@ def test_course_id_param(self): delete_notifications({'course_id': self.course_1.id}) assert not Notification.objects.filter(course_id=self.course_1.id) assert Notification.objects.filter(course_id=self.course_2.id) + + +@ddt.ddt +class NotificationCreationOnChannelsTests(ModuleStoreTestCase): + """ + Tests for notification creation and channels value. + """ + + def setUp(self): + """ + Create a course and users for tests. + """ + + super().setUp() + self.user = UserFactory() + self.course = CourseFactory.create( + org='testorg', + number='testcourse', + run='testrun' + ) + + self.preference = CourseNotificationPreference.objects.create( + user_id=self.user.id, + course_id=self.course.id, + config_version=0, + ) + + @override_waffle_flag(ENABLE_NOTIFICATIONS, active=True) + @ddt.data( + (False, False, 0), + (False, True, 1), + (True, False, 1), + (True, True, 1), + ) + @ddt.unpack + def test_notification_is_created_when_any_channel_is_enabled(self, web_value, email_value, generated_count): + """ + Tests if notification is created if any preference is enabled + """ + app_name = 'discussion' + notification_type = 'new_discussion_post' + app_prefs = self.preference.notification_preference_config[app_name] + app_prefs['notification_types'][notification_type]['web'] = web_value + app_prefs['notification_types'][notification_type]['email'] = email_value + kwargs = { + 'user_ids': [self.user.id], + 'course_key': str(self.course.id), + 'app_name': app_name, + 'notification_type': notification_type, + 'content_url': 'https://example.com/', + 'context': { + 'post_title': 'Post title', + 'username': 'user name', + }, + } + self.preference.save() + with patch('openedx.core.djangoapps.notifications.tasks.notification_generated_event') as event_mock: + send_notifications(**kwargs) + notifications = Notification.objects.all() + assert len(notifications) == generated_count + if notifications: + notification = Notification.objects.all()[0] + assert notification.web == web_value + assert notification.email == email_value From ba56109498f81729f57c4fe71016de978564acd8 Mon Sep 17 00:00:00 2001 From: muhammad-ammar Date: Tue, 7 May 2024 14:27:49 +0500 Subject: [PATCH 040/113] chore: upgrade edx-enterprise to 4.17.1 --- requirements/constraints.txt | 4 ++-- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/doc.txt | 2 +- requirements/edx/testing.txt | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index ad17ea9f3c3d..42cef16089d3 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -23,7 +23,7 @@ click>=8.0,<9.0 # The team that owns this package will manually bump this package rather than having it pulled in automatically. # This is to allow them to better control its deployment and to do it in a process that works better # for them. -edx-enterprise==4.17.0 +edx-enterprise==4.17.1 # Stay on LTS version, remove once this is added to common constraint Django<5.0 @@ -117,7 +117,7 @@ openai<=0.28.1 optimizely-sdk<5.0 # lxml>=5.0 introduced breaking changes related to system dependencies -# lxml==5.2.1 introduced new extra so we'll nee to rename lxml --> lxml[html-clean] +# lxml==5.2.1 introduced new extra so we'll nee to rename lxml --> lxml[html-clean] # This constraint can be removed once we upgrade to Python 3.11 lxml<5.0 diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index c364b075220b..0308daf1a491 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -480,7 +480,7 @@ edx-drf-extensions==10.3.0 # edx-when # edxval # openedx-learning -edx-enterprise==4.17.0 +edx-enterprise==4.17.1 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/kernel.in diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 3ff1b887a88b..3641c96b0597 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -754,7 +754,7 @@ edx-drf-extensions==10.3.0 # edx-when # edxval # openedx-learning -edx-enterprise==4.17.0 +edx-enterprise==4.17.1 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 3276a559f892..cce6317ef84d 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -558,7 +558,7 @@ edx-drf-extensions==10.3.0 # edx-when # edxval # openedx-learning -edx-enterprise==4.17.0 +edx-enterprise==4.17.1 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 32b15dd73541..566ca73ae084 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -582,7 +582,7 @@ edx-drf-extensions==10.3.0 # edx-when # edxval # openedx-learning -edx-enterprise==4.17.0 +edx-enterprise==4.17.1 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt From 21a1235a284d9aa4891dc75b31777df31255fc62 Mon Sep 17 00:00:00 2001 From: "Kyle D. McCormick" Date: Mon, 6 May 2024 14:50:17 -0400 Subject: [PATCH 041/113] revert: revert: build: finish replacing paver assets This reverts commit 4c0284b87dbca6363040a18fd5fa67297df85afa. --- cms/envs/common.py | 37 +- .../0017-reimplement-asset-processing.rst | 52 +-- lms/envs/common.py | 19 +- .../management/commands/compile_sass.py | 124 +++--- .../djangoapps/theming/tests/test_commands.py | 56 +-- pavelib/assets.py | 321 +++++++-------- pavelib/paver_tests/test_assets.py | 387 +++++------------- scripts/compile_sass.py | 8 +- scripts/watch_sass.sh | 4 +- 9 files changed, 375 insertions(+), 633 deletions(-) diff --git a/cms/envs/common.py b/cms/envs/common.py index 39d5c7d13251..6aa2dfb835b1 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -39,6 +39,7 @@ # pylint: disable=unused-import, useless-suppression, wrong-import-order, wrong-import-position import importlib.util +import json import os import sys @@ -1275,14 +1276,11 @@ # Static content STATIC_URL = '/static/studio/' -STATIC_ROOT = ENV_ROOT / "staticfiles" / 'studio' +STATIC_ROOT = os.environ.get('STATIC_ROOT_CMS', ENV_ROOT / 'staticfiles' / 'studio') STATICFILES_DIRS = [ COMMON_ROOT / "static", PROJECT_ROOT / "static", - - # This is how you would use the textbook images locally - # ("book", ENV_ROOT / "book_images"), ] # Locale/Internationalization @@ -1322,11 +1320,16 @@ EMBARGO_SITE_REDIRECT_URL = None ##### custom vendor plugin variables ##### -# JavaScript code can access this data using `process.env.JS_ENV_EXTRA_CONFIG` -# One of the current use cases for this is enabling custom TinyMCE plugins -# (TINYMCE_ADDITIONAL_PLUGINS) and overriding the TinyMCE configuration -# (TINYMCE_CONFIG_OVERRIDES). -JS_ENV_EXTRA_CONFIG = {} + +# .. setting_name: JS_ENV_EXTRA_CONFIG +# .. setting_default: {} +# .. setting_description: JavaScript code can access this dictionary using `process.env.JS_ENV_EXTRA_CONFIG` +# One of the current use cases for this is enabling custom TinyMCE plugins +# (TINYMCE_ADDITIONAL_PLUGINS) and overriding the TinyMCE configuration (TINYMCE_CONFIG_OVERRIDES). +# .. setting_warning: This Django setting is DEPRECATED! Starting in Sumac, Webpack will no longer +# use Django settings. Please set the JS_ENV_EXTRA_CONFIG environment variable to an equivalent JSON +# string instead. For details, see: https://github.com/openedx/edx-platform/issues/31895 +JS_ENV_EXTRA_CONFIG = json.loads(os.environ.get('JS_ENV_EXTRA_CONFIG', '{}')) ############################### PIPELINE ####################################### @@ -1503,7 +1506,14 @@ 'STATS_FILE': os.path.join(STATIC_ROOT, 'webpack-worker-stats.json') } } -WEBPACK_CONFIG_PATH = 'webpack.prod.config.js' + +# .. setting_name: WEBPACK_CONFIG_PATH +# .. setting_default: "webpack.prod.config.js" +# .. setting_description: Path to the Webpack configuration file. Used by Paver scripts. +# .. setting_warning: This Django setting is DEPRECATED! Starting in Sumac, Webpack will no longer +# use Django settings. Please set the WEBPACK_CONFIG_PATH environment variable instead. For details, +# see: https://github.com/openedx/edx-platform/issues/31895 +WEBPACK_CONFIG_PATH = os.environ.get('WEBPACK_CONFIG_PATH', 'webpack.prod.config.js') ############################ SERVICE_VARIANT ################################## @@ -2180,8 +2190,11 @@ # .. setting_name: COMPREHENSIVE_THEME_DIRS # .. setting_default: [] -# .. setting_description: See LMS annotation. -COMPREHENSIVE_THEME_DIRS = [] +# .. setting_description: A list of paths to directories, each of which will +# be searched for comprehensive themes. Do not override this Django setting directly. +# Instead, set the COMPREHENSIVE_THEME_DIRS environment variable, using colons (:) to +# separate paths. +COMPREHENSIVE_THEME_DIRS = os.environ.get("COMPREHENSIVE_THEME_DIRS", "").split(":") # .. setting_name: COMPREHENSIVE_THEME_LOCALE_PATHS # .. setting_default: [] diff --git a/docs/decisions/0017-reimplement-asset-processing.rst b/docs/decisions/0017-reimplement-asset-processing.rst index 1048edf4c075..e689c34eb9cf 100644 --- a/docs/decisions/0017-reimplement-asset-processing.rst +++ b/docs/decisions/0017-reimplement-asset-processing.rst @@ -11,16 +11,14 @@ Overview Status ****** -**Provisional** +**Accepted** -This was `originally authored `_ in March 2023. We `modified it in July 2023 `_ based on learnings from the implementation process. +This was `originally authored `_ in March 2023. We `modified it in July 2023 `_ based on learnings from the implementation process, and then `modified and it again in May 2024 `_ to make the migration easier for operators to understand. -The status will be moved to *Accepted* upon completion of reimplementation. Related work: +Related deprecation tickets: * `[DEPR]: Asset processing in Paver `_ -* `Process edx-platform assets without Paver `_ -* `Process edx-platform assets without Python `_ - +* `[DEPR]: Paver `_ Context ******* @@ -92,7 +90,6 @@ Three particular issues have surfaced in Developer Experience Working Group disc All of these potential solutions would involve refactoring or entirely replacing parts of the current asset processing system. - Decision ******** @@ -114,6 +111,9 @@ Reimplementation Specification Commands and stages ------------------- +**May 2024 update:** See the `static assets reference <../references/static-assets.rst>`_ for +the latest commands. + The three top-level edx-platform asset processing actions are *build*, *collect*, and *watch*. The build action can be further broken down into five stages. Here is how those actions and stages will be reimplemented: @@ -226,6 +226,9 @@ The three top-level edx-platform asset processing actions are *build*, *collect* Build Configuration ------------------- +**May 2024 update:** See the `static assets reference <../references/static-assets.rst>`_ for +the latest configuration settings. + To facilitate a generally Python-free build reimplementation, we will require that certain Django settings now be specified as environment variables, which can be passed to the build like so:: MY_ENV_VAR="my value" npm run build # Set for the whole build. @@ -266,7 +269,7 @@ Some of these options will remain as Django settings because they are used in ed * - ``COMPREHENSIVE_THEME_DIRS`` - Directories that will be searched when compiling themes. - ``COMPREHENSIVE_THEME_DIRS`` - - ``EDX_PLATFORM_THEME_DIRS`` + - ``COMPREHENSIVE_THEME_DIRS`` Migration ========= @@ -285,37 +288,16 @@ As a consequence of this ADR, Tutor will either need to: * reimplement the script as a thin wrapper around the new asset processing commands, or * deprecate and remove the script. -Either way, the migration path is straightforward: - -.. list-table:: - :header-rows: 1 - - * - Existing Tutor-provided command - - New upstream command - * - ``openedx-assets build`` - - ``npm run build`` - * - ``openedx-assets npm`` - - ``scripts/copy-node-modules.sh # (automatically invoked by 'npm install'!)`` - * - ``openedx-assets xmodule`` - - (no longer needed) - * - ``openedx-assets common`` - - ``npm run compile-sass -- --skip-themes`` - * - ``openedx-assets themes`` - - ``npm run compile-sass -- --skip-default`` - * - ``openedx-assets webpack`` - - ``npm run webpack`` - * - ``openedx-assets collect`` - - ``./manage.py [lms|cms] collectstatic --noinput`` - * - ``openedx-assets watch-themes`` - - ``npm run watch`` - -The options accepted by ``openedx-assets`` will all be valid inputs to ``scripts/build-assets.sh``. +**May 2024 update:** The ``openedx-assets`` script will be removed from Tutor, +with migration instructions documented in +`Tutor's changelog `_. non-Tutor migration guide ------------------------- -Operators using distributions other than Tutor should refer to the upstream edx-platform changes described above in **Reimplementation Specification**, and adapt them accordingly to their distribution. - +A migration guide for site operators who are directly referencing Paver will be +included in the +`Paver deprecation ticket `_. See also ******** diff --git a/lms/envs/common.py b/lms/envs/common.py index f6266fdaa3cb..02e52ea5a317 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -1928,7 +1928,7 @@ def _make_mako_template_dirs(settings): # Static content STATIC_URL = '/static/' -STATIC_ROOT = ENV_ROOT / "staticfiles" +STATIC_ROOT = os.environ.get('STATIC_ROOT_LMS', ENV_ROOT / "staticfiles") STATIC_URL_BASE = '/static/' STATICFILES_DIRS = [ @@ -2822,7 +2822,14 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring 'STATS_FILE': os.path.join(STATIC_ROOT, 'webpack-worker-stats.json') } } -WEBPACK_CONFIG_PATH = 'webpack.prod.config.js' + +# .. setting_name: WEBPACK_CONFIG_PATH +# .. setting_default: "webpack.prod.config.js" +# .. setting_description: Path to the Webpack configuration file. Used by Paver scripts. +# .. setting_warning: This Django setting is DEPRECATED! Starting in Sumac, Webpack will no longer +# use Django settings. Please set the WEBPACK_CONFIG_PATH environment variable instead. For details, +# see: https://github.com/openedx/edx-platform/issues/31895 +WEBPACK_CONFIG_PATH = os.environ.get('WEBPACK_CONFIG_PATH', 'webpack.prod.config.js') ########################## DJANGO DEBUG TOOLBAR ############################### @@ -4549,9 +4556,11 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring # .. setting_name: COMPREHENSIVE_THEME_DIRS # .. setting_default: [] -# .. setting_description: A list of directories containing themes folders, -# each entry should be a full path to the directory containing the theme folder. -COMPREHENSIVE_THEME_DIRS = [] +# .. setting_description: A list of paths to directories, each of which will +# be searched for comprehensive themes. Do not override this Django setting directly. +# Instead, set the COMPREHENSIVE_THEME_DIRS environment variable, using colons (:) to +# separate paths. +COMPREHENSIVE_THEME_DIRS = os.environ.get("COMPREHENSIVE_THEME_DIRS", "").split(":") # .. setting_name: COMPREHENSIVE_THEME_LOCALE_PATHS # .. setting_default: [] diff --git a/openedx/core/djangoapps/theming/management/commands/compile_sass.py b/openedx/core/djangoapps/theming/management/commands/compile_sass.py index 765ef98aeac3..eecbe684b528 100644 --- a/openedx/core/djangoapps/theming/management/commands/compile_sass.py +++ b/openedx/core/djangoapps/theming/management/commands/compile_sass.py @@ -1,13 +1,14 @@ """ Management command for compiling sass. -""" +DEPRECATED in favor of `npm run compile-sass`. +""" +import shlex -from django.core.management import BaseCommand, CommandError -from paver.easy import call_task +from django.core.management import BaseCommand +from django.conf import settings -from openedx.core.djangoapps.theming.helpers import get_theme_base_dirs, get_themes, is_comprehensive_theming_enabled -from pavelib.assets import ALL_SYSTEMS +from pavelib.assets import run_deprecated_command_wrapper class Command(BaseCommand): @@ -15,7 +16,7 @@ class Command(BaseCommand): Compile theme sass and collect theme assets. """ - help = 'Compile and collect themed assets...' + help = "DEPRECATED. Use 'npm run compile-sass' instead." # NOTE (CCB): This allows us to compile static assets in Docker containers without database access. requires_system_checks = [] @@ -28,7 +29,7 @@ def add_arguments(self, parser): parser (django.core.management.base.CommandParser): parsed for parsing command line arguments. """ parser.add_argument( - 'system', type=str, nargs='*', default=ALL_SYSTEMS, + 'system', type=str, nargs='*', default=["lms", "cms"], help="lms or studio", ) @@ -55,7 +56,7 @@ def add_arguments(self, parser): '--force', action='store_true', default=False, - help="Force full compilation", + help="DEPRECATED. Full recompilation is now always forced.", ) parser.add_argument( '--debug', @@ -64,77 +65,48 @@ def add_arguments(self, parser): help="Disable Sass compression", ) - @staticmethod - def parse_arguments(*args, **options): # pylint: disable=unused-argument - """ - Parse and validate arguments for compile_sass command. - - Args: - *args: Positional arguments passed to the update_assets command - **options: optional arguments passed to the update_assets command - Returns: - A tuple containing parsed values for themes, system, source comments and output style. - 1. system (list): list of system names for whom to compile theme sass e.g. 'lms', 'cms' - 2. theme_dirs (list): list of Theme objects - 3. themes (list): list of Theme objects - 4. force (bool): Force full compilation - 5. debug (bool): Disable Sass compression - """ - system = options.get("system", ALL_SYSTEMS) - given_themes = options.get("themes", ["all"]) - theme_dirs = options.get("theme_dirs", None) - - force = options.get("force", True) - debug = options.get("debug", True) - - if theme_dirs: - available_themes = {} - for theme_dir in theme_dirs: - available_themes.update({t.theme_dir_name: t for t in get_themes(theme_dir)}) - else: - theme_dirs = get_theme_base_dirs() - available_themes = {t.theme_dir_name: t for t in get_themes()} - - if 'no' in given_themes or 'all' in given_themes: - # Raise error if 'all' or 'no' is present and theme names are also given. - if len(given_themes) > 1: - raise CommandError("Invalid themes value, It must either be 'all' or 'no' or list of themes.") - # Raise error if any of the given theme name is invalid - # (theme name would be invalid if it does not exist in themes directory) - elif (not set(given_themes).issubset(list(available_themes.keys()))) and is_comprehensive_theming_enabled(): - raise CommandError( - "Given themes '{themes}' do not exist inside any of the theme directories '{theme_dirs}'".format( - themes=", ".join(set(given_themes) - set(available_themes.keys())), - theme_dirs=theme_dirs, - ), - ) - - if "all" in given_themes: - themes = list(available_themes.values()) - elif "no" in given_themes: - themes = [] - else: - # convert theme names to Theme objects, this will remove all themes if theming is disabled - themes = [available_themes.get(theme) for theme in given_themes if theme in available_themes] - - return system, theme_dirs, themes, force, debug - def handle(self, *args, **options): """ Handle compile_sass command. """ - system, theme_dirs, themes, force, debug = self.parse_arguments(*args, **options) - themes = [theme.theme_dir_name for theme in themes] - - if options.get("themes", None) and not is_comprehensive_theming_enabled(): - # log a warning message to let the user know that asset compilation for themes is skipped - self.stdout.write( - self.style.WARNING( - "Skipping theme asset compilation: enable theming to process themed assets" + systems = set( + {"lms": "lms", "cms": "cms", "studio": "cms"}[sys] + for sys in options.get("system", ["lms", "cms"]) + ) + theme_dirs = options.get("theme_dirs", settings.COMPREHENSIVE_THEME_DIRS) or [] + themes_option = options.get("themes", []) # '[]' means 'all' + if not settings.ENABLE_COMPREHENSIVE_THEMING: + compile_themes = False + themes = [] + elif "no" in themes_option: + compile_themes = False + themes = [] + elif "all" in themes_option: + compile_themes = True + themes = [] + else: + compile_themes = True + themes = themes_option + run_deprecated_command_wrapper( + old_command="./manage.py [lms|cms] compile_sass", + ignored_old_flags=list(set(["force"]) & set(options)), + new_command=shlex.join([ + "npm", + "run", + ("compile-sass-dev" if options.get("debug") else "compile-sass"), + "--", + *(["--skip-lms"] if "lms" not in systems else []), + *(["--skip-cms"] if "cms" not in systems else []), + *(["--skip-themes"] if not compile_themes else []), + *( + arg + for theme_dir in theme_dirs + for arg in ["--theme-dir", str(theme_dir)] ), - ) - - call_task( - 'pavelib.assets.compile_sass', - options={'system': system, 'theme_dirs': theme_dirs, 'themes': themes, 'force': force, 'debug': debug}, + *( + arg + for theme in themes + for arg in ["--theme", theme] + ), + ]), ) diff --git a/openedx/core/djangoapps/theming/tests/test_commands.py b/openedx/core/djangoapps/theming/tests/test_commands.py index b4abee9bed0e..7ca56f604a41 100644 --- a/openedx/core/djangoapps/theming/tests/test_commands.py +++ b/openedx/core/djangoapps/theming/tests/test_commands.py @@ -2,57 +2,23 @@ Tests for Management commands of comprehensive theming. """ -import pytest -from django.core.management import CommandError, call_command -from django.test import TestCase +from django.core.management import call_command +from django.test import TestCase, override_settings +from unittest.mock import patch -from openedx.core.djangoapps.theming.helpers import get_themes -from openedx.core.djangoapps.theming.management.commands.compile_sass import Command +import pavelib.assets class TestUpdateAssets(TestCase): """ Test comprehensive theming helper functions. """ - def setUp(self): - super().setUp() - self.themes = get_themes() - def test_errors_for_invalid_arguments(self): - """ - Test update_asset command. - """ - # make sure error is raised for invalid theme list - with pytest.raises(CommandError): - call_command("compile_sass", themes=["all", "test-theme"]) - - # make sure error is raised for invalid theme list - with pytest.raises(CommandError): - call_command("compile_sass", themes=["no", "test-theme"]) - - # make sure error is raised for invalid theme list - with pytest.raises(CommandError): - call_command("compile_sass", themes=["all", "no"]) - - # make sure error is raised for invalid theme list - with pytest.raises(CommandError): - call_command("compile_sass", themes=["test-theme", "non-existing-theme"]) - - def test_parse_arguments(self): - """ - Test parse arguments method for update_asset command. - """ - # make sure compile_sass picks all themes when called with 'themes=all' option - parsed_args = Command.parse_arguments(themes=["all"]) - self.assertCountEqual(parsed_args[2], get_themes()) - - # make sure compile_sass picks no themes when called with 'themes=no' option - parsed_args = Command.parse_arguments(themes=["no"]) - self.assertCountEqual(parsed_args[2], []) - - # make sure compile_sass picks only specified themes - parsed_args = Command.parse_arguments(themes=["test-theme"]) - self.assertCountEqual( - parsed_args[2], - [theme for theme in get_themes() if theme.theme_dir_name == "test-theme"] + @patch.object(pavelib.assets, 'sh') + @override_settings(COMPREHENSIVE_THEME_DIRS='common/test') + def test_deprecated_wrapper(self, mock_sh): + call_command('compile_sass', '--themes', 'fake-theme1', 'fake-theme2') + assert mock_sh.called_once_with( + "npm run compile-sass -- " + + "--theme-dir common/test --theme fake-theme-1 --theme fake-theme-2" ) diff --git a/pavelib/assets.py b/pavelib/assets.py index 466ffc9ed919..f437b6427f93 100644 --- a/pavelib/assets.py +++ b/pavelib/assets.py @@ -1,5 +1,9 @@ """ Asset compilation and collection. + +This entire module is DEPRECATED. In Redwood, it exists just as a collection of temporary compatibility wrappers. +In Sumac, this module will be deleted. To migrate, follow the advice in the printed warnings and/or read the +instructions on the DEPR ticket: https://github.com/openedx/edx-platform/issues/31895 """ import argparse @@ -13,31 +17,48 @@ from paver import tasks from paver.easy import call_task, cmdopts, consume_args, needs, no_help, sh, task from watchdog.events import PatternMatchingEventHandler -from watchdog.observers import Observer # pylint disable=unused-import # Used by Tutor. Remove after Sumac cut. +from watchdog.observers import Observer # pylint: disable=unused-import # Used by Tutor. Remove after Sumac cut. -from .utils.cmd import cmd, django_cmd +from .utils.cmd import django_cmd from .utils.envs import Env -from .utils.process import run_background_process from .utils.timer import timed -# setup baseline paths - -ALL_SYSTEMS = ['lms', 'studio'] - -LMS = 'lms' -CMS = 'cms' SYSTEMS = { - 'lms': LMS, - 'cms': CMS, - 'studio': CMS + 'lms': 'lms', + 'cms': 'cms', + 'studio': 'cms', } -# Collectstatic log directory setting -COLLECTSTATIC_LOG_DIR_ARG = 'collect_log_dir' +WARNING_SYMBOLS = "⚠️ " * 50 # A row of 'warning' emoji to catch CLI users' attention + -# Webpack command -WEBPACK_COMMAND = 'STATIC_ROOT_LMS={static_root_lms} STATIC_ROOT_CMS={static_root_cms} webpack {options}' +def run_deprecated_command_wrapper(*, old_command, ignored_old_flags, new_command): + """ + Run the new version of shell command, plus a warning that the old version is deprecated. + """ + depr_warning = ( + "\n" + + f"{WARNING_SYMBOLS}\n" + + "\n" + + f"WARNING: '{old_command}' is DEPRECATED! It will be removed before Sumac.\n" + + "The command you ran is now just a temporary wrapper around a new,\n" + + "supported command, which you should use instead:\n" + + "\n" + + f"\t{new_command}\n" + + "\n" + + "Details: https://github.com/openedx/edx-platform/issues/31895\n" + + "".join( + f" WARNING: ignored deprecated paver flag '{flag}'\n" + for flag in ignored_old_flags + ) + + f"{WARNING_SYMBOLS}\n" + + "\n" + ) + # Print deprecation warning twice so that it's more likely to be seen in the logs. + print(depr_warning) + sh(new_command) + print(depr_warning) def debounce(seconds=1): @@ -45,6 +66,8 @@ def debounce(seconds=1): Prevents the decorated function from being called more than every `seconds` seconds. Waits until calls stop coming in before calling the decorated function. + + This is DEPRECATED. It exists in Redwood just to ease the transition for Tutor. """ def decorator(func): func.timer = None @@ -66,6 +89,8 @@ def call(): class SassWatcher(PatternMatchingEventHandler): """ Watches for sass file changes + + This is DEPRECATED. It exists in Redwood just to ease the transition for Tutor. """ ignore_directories = True patterns = ['*.scss'] @@ -102,7 +127,7 @@ def on_any_event(self, event): ('system=', 's', 'The system to compile sass for (defaults to all)'), ('theme-dirs=', '-td', 'Theme dirs containing all themes (defaults to None)'), ('themes=', '-t', 'The theme to compile sass for (defaults to None)'), - ('debug', 'd', 'DEPRECATED. Debug mode is now determined by NODE_ENV.'), + ('debug', 'd', 'Whether to use development settings'), ('force', '', 'DEPRECATED. Full recompilation is now always forced.'), ]) @timed @@ -143,16 +168,18 @@ def compile_sass(options): This is a DEPRECATED COMPATIBILITY WRAPPER. Use `npm run compile-sass` instead. """ - systems = set(get_parsed_option(options, 'system', ALL_SYSTEMS)) - command = shlex.join( - [ + systems = [SYSTEMS[sys] for sys in get_parsed_option(options, 'system', ['lms', 'cms'])] # normalize studio->cms + run_deprecated_command_wrapper( + old_command="paver compile_sass", + ignored_old_flags=(set(["--force"]) & set(options)), + new_command=shlex.join([ "npm", "run", - "compile-sass", + ("compile-sass-dev" if options.get("debug") else "compile-sass"), "--", *(["--dry"] if tasks.environment.dry_run else []), - *(["--skip-lms"] if not systems & {"lms"} else []), - *(["--skip-cms"] if not systems & {"cms", "studio"} else []), + *(["--skip-lms"] if "lms" not in systems else []), + *(["--skip-cms"] if "cms" not in systems else []), *( arg for theme_dir in get_parsed_option(options, 'theme_dirs', []) @@ -160,77 +187,50 @@ def compile_sass(options): ), *( arg - for theme in get_parsed_option(options, "theme", []) + for theme in get_parsed_option(options, "themes", []) for arg in ["--theme", theme] ), - ] - ) - depr_warning = ( - "\n" + - "⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ \n" + - "\n" + - "WARNING: 'paver compile_sass' is DEPRECATED! It will be removed before Sumac.\n" + - "The command you ran is now just a temporary wrapper around a new,\n" + - "supported command, which you should use instead:\n" + - "\n" + - f"\t{command}\n" + - "\n" + - "Details: https://github.com/openedx/edx-platform/issues/31895\n" + - "\n" + - ("WARNING: ignoring deprecated flag '--debug'\n" if options.get("debug") else "") + - ("WARNING: ignoring deprecated flag '--force'\n" if options.get("force") else "") + - "⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ \n" + - "\n" + ]), ) - # Print deprecation warning twice so that it's more likely to be seen in the logs. - print(depr_warning) - sh(command) - print(depr_warning) -def _compile_sass(system, theme, _debug, _force, _timing_info): +def _compile_sass(system, theme, debug, force, _timing_info): """ This is a DEPRECATED COMPATIBILITY WRAPPER It exists to ease the transition for Tutor in Redwood, which directly imported and used this function. """ - command = shlex.join( - [ + run_deprecated_command_wrapper( + old_command="pavelib.assets:_compile_sass", + ignored_old_flags=(set(["--force"]) if force else set()), + new_command=[ "npm", "run", - "compile-sass", + ("compile-sass-dev" if debug else "compile-sass"), "--", *(["--dry"] if tasks.environment.dry_run else []), - *(["--skip-default", "--theme-dir", str(theme.parent), "--theme", str(theme.name)] if theme else []), + *( + ["--skip-default", "--theme-dir", str(theme.parent), "--theme", str(theme.name)] + if theme + else [] + ), ("--skip-cms" if system == "lms" else "--skip-lms"), ] ) - depr_warning = ( - "\n" + - "⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ \n" + - "\n" + - "WARNING: 'pavelib/assets.py' is DEPRECATED! It will be removed before Sumac.\n" + - "The function you called is just a temporary wrapper around a new, supported command,\n" + - "which you should use instead:\n" + - "\n" + - f"\t{command}\n" + - "\n" + - "Details: https://github.com/openedx/edx-platform/issues/31895\n" + - "\n" + - "⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ \n" + - "\n" - ) - # Print deprecation warning twice so that it's more likely to be seen in the logs. - print(depr_warning) - sh(command) - print(depr_warning) def process_npm_assets(): """ Process vendor libraries installed via NPM. + + This is a DEPRECATED COMPATIBILITY WRAPPER. It is now handled as part of `npm clean-install`. + If you need to invoke it explicitly, you can run `npm run postinstall`. """ - sh('scripts/copy-node-modules.sh') + run_deprecated_command_wrapper( + old_command="pavelib.assets:process_npm_assets", + ignored_old_flags=[], + new_command=shlex.join(["npm", "run", "postinstall"]), + ) @task @@ -238,9 +238,21 @@ def process_npm_assets(): def process_xmodule_assets(): """ Process XModule static assets. + + This is a DEPRECATED COMPATIBILITY STUB. Refrences to it should be deleted. """ - print("\t\tProcessing xmodule assets is no longer needed. This task is now a no-op.") - print("\t\tWhen paver is removed from edx-platform, this step will not replaced.") + print( + "\n" + + f"{WARNING_SYMBOLS}", + "\n" + + "WARNING: 'paver process_xmodule_assets' is DEPRECATED! It will be removed before Sumac.\n" + + "\n" + + "Starting with Quince, it is no longer necessary to post-process XModule assets, so \n" + + "'paver process_xmodule_assets' is a no-op. Please simply remove it from your build scripts.\n" + + "\n" + + "Details: https://github.com/openedx/edx-platform/issues/31895\n" + + f"{WARNING_SYMBOLS}", + ) def collect_assets(systems, settings, **kwargs): @@ -249,33 +261,29 @@ def collect_assets(systems, settings, **kwargs): `systems` is a list of systems (e.g. 'lms' or 'studio' or both) `settings` is the Django settings module to use. `**kwargs` include arguments for using a log directory for collectstatic output. Defaults to /dev/null. - """ - for sys in systems: - collectstatic_stdout_str = _collect_assets_cmd(sys, **kwargs) - sh(django_cmd(sys, settings, "collectstatic --noinput {logfile_str}".format( - logfile_str=collectstatic_stdout_str - ))) - print(f"\t\tFinished collecting {sys} assets.") - -def _collect_assets_cmd(system, **kwargs): - """ - Returns the collecstatic command to be used for the given system + This is a DEPRECATED COMPATIBILITY WRAPPER - Unless specified, collectstatic (which can be verbose) pipes to /dev/null + It exists to ease the transition for Tutor in Redwood, which directly imported and used this function. """ - try: - if kwargs[COLLECTSTATIC_LOG_DIR_ARG] is None: - collectstatic_stdout_str = "" - else: - collectstatic_stdout_str = "> {output_dir}/{sys}-collectstatic.log".format( - output_dir=kwargs[COLLECTSTATIC_LOG_DIR_ARG], - sys=system - ) - except KeyError: - collectstatic_stdout_str = "> /dev/null" - - return collectstatic_stdout_str + run_deprecated_command_wrapper( + old_command="pavelib.asset:collect_assets", + ignored_old_flags=[], + new_command=" && ".join( + "( " + + shlex.join( + ["./manage.py", SYSTEMS[sys], f"--settings={settings}", "collectstatic", "--noinput"] + ) + ( + "" + if "collect_log_dir" not in kwargs else + " > /dev/null" + if kwargs["collect_log_dir"] is None else + f"> {kwargs['collect_log_dir']}/{SYSTEMS[sys]}-collectstatic.out" + ) + + " )" + for sys in systems + ), + ) def execute_compile_sass(args): @@ -283,6 +291,8 @@ def execute_compile_sass(args): Construct django management command compile_sass (defined in theming app) and execute it. Args: args: command line argument passed via update_assets command + + This is a DEPRECATED COMPATIBILITY WRAPPER. Use `npm run compile-sass` instead. """ for sys in args.system: options = "" @@ -305,12 +315,14 @@ def execute_compile_sass(args): @task @cmdopts([ ('settings=', 's', "Django settings (defaults to devstack)"), - ('watch', 'w', "Watch file system and rebuild on change (defaults to off)"), + ('watch', 'w', "DEPRECATED. This flag never did anything anyway."), ]) @timed def webpack(options): """ Run a Webpack build. + + This is a DEPRECATED COMPATIBILITY WRAPPER. Use `npm run webpack` instead. """ settings = getattr(options, 'settings', Env.DEVSTACK_SETTINGS) result = Env.get_django_settings(['STATIC_ROOT', 'WEBPACK_CONFIG_PATH'], "lms", settings=settings) @@ -318,44 +330,20 @@ def webpack(options): static_root_cms, = Env.get_django_settings(["STATIC_ROOT"], "cms", settings=settings) js_env_extra_config_setting, = Env.get_django_json_settings(["JS_ENV_EXTRA_CONFIG"], "cms", settings=settings) js_env_extra_config = json.dumps(js_env_extra_config_setting or "{}") - environment = ( - "NODE_ENV={node_env} STATIC_ROOT_LMS={static_root_lms} STATIC_ROOT_CMS={static_root_cms} " - "JS_ENV_EXTRA_CONFIG={js_env_extra_config}" - ).format( - node_env="development" if config_path == 'webpack.dev.config.js' else "production", - static_root_lms=static_root_lms, - static_root_cms=static_root_cms, - js_env_extra_config=js_env_extra_config, - ) - sh( - cmd( - '{environment} webpack --config={config_path}'.format( - environment=environment, - config_path=config_path - ) - ) - ) - - -def execute_webpack_watch(settings=None): - """ - Run the Webpack file system watcher. - """ - # We only want Webpack to re-run on changes to its own entry points, - # not all JS files, so we use its own watcher instead of subclassing - # from Watchdog like the other watchers do. - - result = Env.get_django_settings(["STATIC_ROOT", "WEBPACK_CONFIG_PATH"], "lms", settings=settings) - static_root_lms, config_path = result - static_root_cms, = Env.get_django_settings(["STATIC_ROOT"], "cms", settings=settings) - run_background_process( - 'STATIC_ROOT_LMS={static_root_lms} STATIC_ROOT_CMS={static_root_cms} webpack {options}'.format( - options='--watch --config={config_path}'.format( - config_path=config_path - ), - static_root_lms=static_root_lms, - static_root_cms=static_root_cms, - ) + node_env = "development" if config_path == 'webpack.dev.config.js' else "production" + run_deprecated_command_wrapper( + old_command="paver webpack", + ignored_old_flags=(set(["watch"]) & set(options)), + new_command=' '.join([ + f"WEBPACK_CONFIG_PATH={config_path}", + f"NODE_ENV={node_env}", + f"STATIC_ROOT_LMS={static_root_lms}", + f"STATIC_ROOT_CMS={static_root_cms}", + f"JS_ENV_EXTRA_CONFIG={js_env_extra_config}", + "npm", + "run", + "webpack", + ]), ) @@ -411,39 +399,19 @@ def watch_assets(options): return theme_dirs = ':'.join(get_parsed_option(options, 'theme_dirs', [])) - command = shlex.join( - [ + run_deprecated_command_wrapper( + old_command="paver watch_assets", + ignored_old_flags=(set(["debug", "themes", "settings", "background"]) & set(options)), + new_command=shlex.join([ *( - ["env", f"EDX_PLATFORM_THEME_DIRS={theme_dirs}"] if theme_dirs else [] + ["env", f"COMPREHENSIVE_THEME_DIRS={theme_dirs}"] + if theme_dirs else [] ), "npm", "run", "watch", - ] + ]), ) - depr_warning = ( - "\n" + - "⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ \n" + - "\n" + - "WARNING: 'paver watch_assets' is DEPRECATED! It will be removed before Sumac.\n" + - "The command you ran is now just a temporary wrapper around a new,\n" + - "supported command, which you should use instead:\n" + - "\n" + - f"\t{command}\n" + - "\n" + - "Details: https://github.com/openedx/edx-platform/issues/31895\n" + - "\n" + - ("WARNING: ignoring deprecated flag '--debug'\n" if options.get("debug") else "") + - ("WARNING: ignoring deprecated flag '--themes'\n" if options.get("themes") else "") + - ("WARNING: ignoring deprecated flag '--settings'\n" if options.get("settings") else "") + - ("WARNING: ignoring deprecated flag '--background'\n" if options.get("background") else "") + - "⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ ⚠️ \n" + - "\n" - ) - # Print deprecation warning twice so that it's more likely to be seen in the logs. - print(depr_warning) - sh(command) - print(depr_warning) @task @@ -456,10 +424,19 @@ def watch_assets(options): def update_assets(args): """ Compile Sass, then collect static assets. + + This is a DEPRECATED COMPATIBILITY WRAPPER around other DEPRECATED COMPATIBILITY WRAPPERS. + The aggregate affect of this command can be achieved with this sequence of commands instead: + + * pip install -r requirements/edx/assets.txt # replaces install_python_prereqs + * npm clean-install # replaces install_node_prereqs + * npm run build # replaces execute_compile_sass and webpack + * ./manage.py lms collectstatic --noinput # replaces collect_assets (for LMS) + * ./manage.py cms collectstatic --noinput # replaces collect_assets (for CMS) """ parser = argparse.ArgumentParser(prog='paver update_assets') parser.add_argument( - 'system', type=str, nargs='*', default=ALL_SYSTEMS, + 'system', type=str, nargs='*', default=["lms", "studio"], help="lms or studio", ) parser.add_argument( @@ -488,18 +465,17 @@ def update_assets(args): ) parser.add_argument( '--themes', type=str, nargs='+', default=None, - help="list of themes to compile sass for", + help="list of themes to compile sass for. ignored when --watch is used; all themes are watched.", ) parser.add_argument( - '--collect-log', dest=COLLECTSTATIC_LOG_DIR_ARG, default=None, + '--collect-log', dest="collect_log_dir", default=None, help="When running collectstatic, direct output to specified log directory", ) parser.add_argument( '--wait', type=float, default=0.0, - help="How long to pause between filesystem scans" + help="DEPRECATED. Watchdog's default wait time is now used.", ) args = parser.parse_args(args) - collect_log_args = {} # Build Webpack call_task('pavelib.assets.webpack', options={'settings': args.settings}) @@ -508,11 +484,12 @@ def update_assets(args): execute_compile_sass(args) if args.collect: - if args.debug or args.debug_collect: - collect_log_args.update({COLLECTSTATIC_LOG_DIR_ARG: None}) - if args.collect_log_dir: - collect_log_args.update({COLLECTSTATIC_LOG_DIR_ARG: args.collect_log_dir}) + collect_log_args = {"collect_log_dir": args.collect_log_dir} + elif args.debug or args.debug_collect: + collect_log_args = {"collect_log_dir": None} + else: + collect_log_args = {} collect_assets(args.system, args.settings, **collect_log_args) diff --git a/pavelib/paver_tests/test_assets.py b/pavelib/paver_tests/test_assets.py index 029a8db67c4a..3578a5043f7e 100644 --- a/pavelib/paver_tests/test_assets.py +++ b/pavelib/paver_tests/test_assets.py @@ -1,305 +1,130 @@ """Unit tests for the Paver asset tasks.""" - +import json import os +from pathlib import Path from unittest import TestCase from unittest.mock import patch import ddt -import paver.tasks -from paver.easy import call_task, path +import paver.easy +from paver import tasks -from pavelib.assets import COLLECTSTATIC_LOG_DIR_ARG, collect_assets +import pavelib.assets +from pavelib.assets import Env -from ..utils.envs import Env -from .utils import PaverTestCase -ROOT_PATH = path(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) -TEST_THEME_DIR = ROOT_PATH / "common/test/test-theme" +REPO_ROOT = Path(__file__).parent.parent.parent +LMS_SETTINGS = { + "WEBPACK_CONFIG_PATH": "webpack.fake.config.js", + "STATIC_ROOT": "/fake/lms/staticfiles", -@ddt.ddt -class TestPaverAssetTasks(PaverTestCase): - """ - Test the Paver asset tasks. - """ - @ddt.data( - [""], - ["--force"], - ["--debug"], - ["--system=lms"], - ["--system=lms --force"], - ["--system=studio"], - ["--system=studio --force"], - ["--system=lms,studio"], - ["--system=lms,studio --force"], - ) - @ddt.unpack - def test_compile_sass(self, options): - """ - Test the "compile_sass" task. - """ - parameters = options.split(" ") - system = [] - if '--system=studio' not in parameters: - system += ['lms'] - if '--system=lms' not in parameters: - system += ['studio'] - debug = '--debug' in parameters - force = '--force' in parameters - self.reset_task_messages() - call_task('pavelib.assets.compile_sass', options={'system': system, 'debug': debug, 'force': force}) - expected_messages = [] - if force: - expected_messages.append('rm -rf common/static/css/*.css') - expected_messages.append('libsass common/static/sass') +} +CMS_SETTINGS = { + "WEBPACK_CONFIG_PATH": "webpack.fake.config", + "STATIC_ROOT": "/fake/cms/staticfiles", + "JS_ENV_EXTRA_CONFIG": json.dumps({"key1": [True, False], "key2": {"key2.1": 1369, "key2.2": "1369"}}), +} - if "lms" in system: - if force: - expected_messages.append('rm -rf lms/static/css/*.css') - expected_messages.append('libsass lms/static/sass') - expected_messages.append( - 'rtlcss lms/static/css/bootstrap/lms-main.css lms/static/css/bootstrap/lms-main-rtl.css' - ) - expected_messages.append( - 'rtlcss lms/static/css/discussion/lms-discussion-bootstrap.css' - ' lms/static/css/discussion/lms-discussion-bootstrap-rtl.css' - ) - if force: - expected_messages.append('rm -rf lms/static/certificates/css/*.css') - expected_messages.append('libsass lms/static/certificates/sass') - if "studio" in system: - if force: - expected_messages.append('rm -rf cms/static/css/*.css') - expected_messages.append('libsass cms/static/sass') - expected_messages.append( - 'rtlcss cms/static/css/bootstrap/studio-main.css cms/static/css/bootstrap/studio-main-rtl.css' - ) - assert len(self.task_messages) == len(expected_messages) +def _mock_get_django_settings(django_settings, system, settings=None): # pylint: disable=unused-argument + return [(LMS_SETTINGS if system == "lms" else CMS_SETTINGS)[s] for s in django_settings] @ddt.ddt -class TestPaverThemeAssetTasks(PaverTestCase): +@patch.object(Env, 'get_django_settings', _mock_get_django_settings) +@patch.object(Env, 'get_django_json_settings', _mock_get_django_settings) +class TestDeprecatedPaverAssets(TestCase): """ - Test the Paver asset tasks. + Simple test to ensure that the soon-to-be-removed Paver commands are correctly translated into the new npm-run + commands. """ - @ddt.data( - [""], - ["--force"], - ["--debug"], - ["--system=lms"], - ["--system=lms --force"], - ["--system=studio"], - ["--system=studio --force"], - ["--system=lms,studio"], - ["--system=lms,studio --force"], - ) - @ddt.unpack - def test_compile_theme_sass(self, options): - """ - Test the "compile_sass" task. - """ - parameters = options.split(" ") - system = [] - - if '--system=studio' not in parameters: - system += ['lms'] - if "--system=lms" not in parameters: - system += ['studio'] - debug = '--debug' in parameters - force = '--force' in parameters - - self.reset_task_messages() - call_task( - 'pavelib.assets.compile_sass', - options=dict( - system=system, - debug=debug, - force=force, - theme_dirs=[TEST_THEME_DIR.dirname()], - themes=[TEST_THEME_DIR.basename()] - ), - ) - expected_messages = [] - if force: - expected_messages.append('rm -rf common/static/css/*.css') - expected_messages.append('libsass common/static/sass') - - if 'lms' in system: - expected_messages.append('mkdir_p ' + repr(TEST_THEME_DIR / 'lms/static/css')) - if force: - expected_messages.append( - f'rm -rf {str(TEST_THEME_DIR)}/lms/static/css/*.css' - ) - expected_messages.append("libsass lms/static/sass") - expected_messages.append( - 'rtlcss {test_theme_dir}/lms/static/css/bootstrap/lms-main.css' - ' {test_theme_dir}/lms/static/css/bootstrap/lms-main-rtl.css'.format( - test_theme_dir=str(TEST_THEME_DIR), - ) - ) - expected_messages.append( - 'rtlcss {test_theme_dir}/lms/static/css/discussion/lms-discussion-bootstrap.css' - ' {test_theme_dir}/lms/static/css/discussion/lms-discussion-bootstrap-rtl.css'.format( - test_theme_dir=str(TEST_THEME_DIR), - ) - ) - if force: - expected_messages.append( - f'rm -rf {str(TEST_THEME_DIR)}/lms/static/css/*.css' - ) - expected_messages.append( - f'libsass {str(TEST_THEME_DIR)}/lms/static/sass' - ) - if force: - expected_messages.append('rm -rf lms/static/css/*.css') - expected_messages.append('libsass lms/static/sass') - expected_messages.append( - 'rtlcss lms/static/css/bootstrap/lms-main.css lms/static/css/bootstrap/lms-main-rtl.css' - ) - expected_messages.append( - 'rtlcss lms/static/css/discussion/lms-discussion-bootstrap.css' - ' lms/static/css/discussion/lms-discussion-bootstrap-rtl.css' - ) - if force: - expected_messages.append('rm -rf lms/static/certificates/css/*.css') - expected_messages.append('libsass lms/static/certificates/sass') + def setUp(self): + super().setUp() + self.maxDiff = None + os.environ['NO_PREREQ_INSTALL'] = 'true' + tasks.environment = tasks.Environment() - if "studio" in system: - expected_messages.append('mkdir_p ' + repr(TEST_THEME_DIR / 'cms/static/css')) - if force: - expected_messages.append( - f'rm -rf {str(TEST_THEME_DIR)}/cms/static/css/*.css' - ) - expected_messages.append('libsass cms/static/sass') - expected_messages.append( - 'rtlcss {test_theme_dir}/cms/static/css/bootstrap/studio-main.css' - ' {test_theme_dir}/cms/static/css/bootstrap/studio-main-rtl.css'.format( - test_theme_dir=str(TEST_THEME_DIR), - ) - ) - if force: - expected_messages.append( - f'rm -rf {str(TEST_THEME_DIR)}/cms/static/css/*.css' - ) - expected_messages.append( - f'libsass {str(TEST_THEME_DIR)}/cms/static/sass' - ) - if force: - expected_messages.append('rm -rf cms/static/css/*.css') - expected_messages.append('libsass cms/static/sass') - expected_messages.append( - 'rtlcss cms/static/css/bootstrap/studio-main.css cms/static/css/bootstrap/studio-main-rtl.css' - ) - - assert len(self.task_messages) == len(expected_messages) - - -@ddt.ddt -class TestCollectAssets(PaverTestCase): - """ - Test the collectstatic process call. - - ddt data is organized thusly: - * debug: whether or not collect_assets is called with the debug flag - * specified_log_location: used when collect_assets is called with a specific - log location for collectstatic output - * expected_log_location: the expected string to be used for piping collectstatic logs - """ - - @ddt.data( - [{ - "collect_log_args": {}, # Test for default behavior - "expected_log_location": "> /dev/null" - }], - [{ - "collect_log_args": {COLLECTSTATIC_LOG_DIR_ARG: "/foo/bar"}, - "expected_log_location": "> /foo/bar/lms-collectstatic.log" - }], # can use specified log location - [{ - "systems": ["lms", "cms"], - "collect_log_args": {}, - "expected_log_location": "> /dev/null" - }], # multiple systems can be called - ) - @ddt.unpack - def test_collect_assets(self, options): - """ - Ensure commands sent to the environment for collect_assets are as expected - """ - specified_log_loc = options.get("collect_log_args", {}) - specified_log_dict = specified_log_loc - log_loc = options.get("expected_log_location", "> /dev/null") - systems = options.get("systems", ["lms"]) - if specified_log_loc is None: - collect_assets( - systems, - Env.DEVSTACK_SETTINGS - ) - else: - collect_assets( - systems, - Env.DEVSTACK_SETTINGS, - **specified_log_dict - ) - self._assert_correct_messages(log_location=log_loc, systems=systems) - - def test_collect_assets_debug(self): - """ - When the method is called specifically with None for the collectstatic log dir, then - it should run in debug mode and pipe to console. - """ - expected_log_loc = "" - systems = ["lms"] - kwargs = {COLLECTSTATIC_LOG_DIR_ARG: None} - collect_assets(systems, Env.DEVSTACK_SETTINGS, **kwargs) - self._assert_correct_messages(log_location=expected_log_loc, systems=systems) - - def _assert_correct_messages(self, log_location, systems): - """ - Asserts that the expected commands were run. - - We just extract the pieces we care about here instead of specifying an - exact command, so that small arg changes don't break this test. - """ - for i, sys in enumerate(systems): - msg = self.task_messages[i] - assert msg.startswith(f'python manage.py {sys}') - assert ' collectstatic ' in msg - assert f'--settings={Env.DEVSTACK_SETTINGS}' in msg - assert msg.endswith(f' {log_location}') - - -@ddt.ddt -class TestUpdateAssetsTask(PaverTestCase): - """ - These are nearly end-to-end tests, because they observe output from the commandline request, - but do not actually execute the commandline on the terminal/process - """ + def tearDown(self): + super().tearDown() + del os.environ['NO_PREREQ_INSTALL'] @ddt.data( - [{"expected_substring": "> /dev/null"}], # go to /dev/null by default - [{"cmd_args": ["--debug"], "expected_substring": "collectstatic"}] # TODO: make this regex + dict( + task_name='pavelib.assets.compile_sass', + args=[], + kwargs={}, + expected=["npm run compile-sass --"], + ), + dict( + task_name='pavelib.assets.compile_sass', + args=[], + kwargs={"system": "lms,studio"}, + expected=["npm run compile-sass --"], + ), + dict( + task_name='pavelib.assets.compile_sass', + args=[], + kwargs={"debug": True}, + expected=["npm run compile-sass-dev --"], + ), + dict( + task_name='pavelib.assets.compile_sass', + args=[], + kwargs={"system": "lms"}, + expected=["npm run compile-sass -- --skip-cms"], + ), + dict( + task_name='pavelib.assets.compile_sass', + args=[], + kwargs={"system": "studio"}, + expected=["npm run compile-sass -- --skip-lms"], + ), + dict( + task_name='pavelib.assets.compile_sass', + args=[], + kwargs={"system": "cms", "theme_dirs": f"{REPO_ROOT}/common/test,{REPO_ROOT}/themes"}, + expected=[ + "npm run compile-sass -- --skip-lms " + + f"--theme-dir {REPO_ROOT}/common/test --theme-dir {REPO_ROOT}/themes" + ], + ), + dict( + task_name='pavelib.assets.compile_sass', + args=[], + kwargs={"theme_dirs": f"{REPO_ROOT}/common/test,{REPO_ROOT}/themes", "themes": "red-theme,test-theme"}, + expected=[ + "npm run compile-sass -- " + + f"--theme-dir {REPO_ROOT}/common/test --theme-dir {REPO_ROOT}/themes " + + "--theme red-theme --theme test-theme" + ], + ), + dict( + task_name='pavelib.assets.update_assets', + args=["lms", "studio", "--settings=fake.settings"], + kwargs={}, + expected=[ + ( + "WEBPACK_CONFIG_PATH=webpack.fake.config.js " + + "NODE_ENV=production " + + "STATIC_ROOT_LMS=/fake/lms/staticfiles " + + "STATIC_ROOT_CMS=/fake/cms/staticfiles " + + 'JS_ENV_EXTRA_CONFIG=' + + + '"{\\"key1\\": [true, false], \\"key2\\": {\\"key2.1\\": 1369, \\"key2.2\\": \\"1369\\"}}" ' + + "npm run webpack" + ), + "python manage.py lms --settings=fake.settings compile_sass lms ", + "python manage.py cms --settings=fake.settings compile_sass cms ", + ( + "( ./manage.py lms --settings=fake.settings collectstatic --noinput ) && " + + "( ./manage.py cms --settings=fake.settings collectstatic --noinput )" + ), + ], + ), ) @ddt.unpack - def test_update_assets_task_collectstatic_log_arg(self, options): - """ - Scoped test that only looks at what is passed to the collecstatic options - """ - cmd_args = options.get("cmd_args", [""]) - expected_substring = options.get("expected_substring", None) - call_task('pavelib.assets.update_assets', args=cmd_args) - self.assertTrue( - self._is_substring_in_list(self.task_messages, expected_substring), - msg=f"{expected_substring} not found in messages" - ) - - def _is_substring_in_list(self, messages_list, expected_substring): - """ - Return true a given string is somewhere in a list of strings - """ - for message in messages_list: - if expected_substring in message: - return True - return False + @patch.object(pavelib.assets, 'sh') + def test_paver_assets_wrapper_invokes_new_commands(self, mock_sh, task_name, args, kwargs, expected): + paver.easy.call_task(task_name, args=args, options=kwargs) + assert [call_args[0] for (call_args, call_kwargs) in mock_sh.call_args_list] == expected diff --git a/scripts/compile_sass.py b/scripts/compile_sass.py index 41a2d56b3bda..ec1efee24d2b 100755 --- a/scripts/compile_sass.py +++ b/scripts/compile_sass.py @@ -67,14 +67,12 @@ "theme_dirs", metavar="PATH", multiple=True, - envvar="EDX_PLATFORM_THEME_DIRS", - type=click.Path( - exists=True, file_okay=False, readable=True, writable=True, path_type=Path - ), + envvar="COMPREHENSIVE_THEME_DIRS", + type=click.Path(path_type=Path), help=( "Consider sub-dirs of PATH as themes. " "Multiple theme dirs are accepted. " - "If none are provided, we look at colon-separated paths on the EDX_PLATFORM_THEME_DIRS env var." + "If none are provided, we look at colon-separated paths on the COMPREHENSIVE_THEME_DIRS env var." ), ) @click.option( diff --git a/scripts/watch_sass.sh b/scripts/watch_sass.sh index 68d4b1f471a5..e88cb9e3cd6f 100755 --- a/scripts/watch_sass.sh +++ b/scripts/watch_sass.sh @@ -4,11 +4,11 @@ # Invoke from repo root as `npm run watch-sass`. # By default, only watches default Sass. -# To watch themes too, provide colon-separated paths in the EDX_PLATFORM_THEME_DIRS environment variable. +# To watch themes too, provide colon-separated paths in the COMPREHENSIVE_THEME_DIRS environment variable. # Each path will be treated as a "theme dir", which means that every immediate child directory is watchable as a theme. # For example: # -# EDX_PLATFORM_THEME_DIRS=/openedx/themes:./themes npm run watch-sass +# COMPREHENSIVE_THEME_DIRS=/openedx/themes:./themes npm run watch-sass # # would watch default Sass as well as /openedx/themes/indigo, /openedx/themes/mytheme, ./themes/red-theme, etc. From b1393ac3eb2ca799b9f1d847113ba3bfc81462e6 Mon Sep 17 00:00:00 2001 From: "Kyle D. McCormick" Date: Mon, 6 May 2024 15:26:07 -0400 Subject: [PATCH 042/113] fix: fall back to settings.COMPREHENSIVE_THEME_DIRS correctly There was a logical error in the compile_sass management command: instead of falling back to settings.COMPREHENSIVE_THEME_DIRS when --theme-dirs was *None or missing*, we only fell back to it when --theme-dirs was *missing*. This caused theme compilation to be skipped when COMREHENSIVE_THEME_DIRS *is not set* in the environment, even though settings.COMPREHENSIVE_THEME_DIRS *is set* in Django settings, which is currently the case for edx.org. --- .../djangoapps/theming/management/commands/compile_sass.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openedx/core/djangoapps/theming/management/commands/compile_sass.py b/openedx/core/djangoapps/theming/management/commands/compile_sass.py index eecbe684b528..fbfdd2f222a4 100644 --- a/openedx/core/djangoapps/theming/management/commands/compile_sass.py +++ b/openedx/core/djangoapps/theming/management/commands/compile_sass.py @@ -73,8 +73,8 @@ def handle(self, *args, **options): {"lms": "lms", "cms": "cms", "studio": "cms"}[sys] for sys in options.get("system", ["lms", "cms"]) ) - theme_dirs = options.get("theme_dirs", settings.COMPREHENSIVE_THEME_DIRS) or [] - themes_option = options.get("themes", []) # '[]' means 'all' + theme_dirs = options.get("theme_dirs") or settings.COMPREHENSIVE_THEME_DIRS or [] + themes_option = options.get("themes") or [] # '[]' means 'all' if not settings.ENABLE_COMPREHENSIVE_THEMING: compile_themes = False themes = [] From d8aab3f72c9a269ae90567dc91574f7d7671fdfc Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Mon, 29 Apr 2024 15:46:14 -0400 Subject: [PATCH 043/113] fix: remove modulestore init to avoid pymongo deadlocks Prior to this commit, the LMS would log the following error in tutor production: pymongo/topology.py:175: UserWarning: MongoClient opened before fork. Create MongoClient only after forking. See PyMongo's documentation for details: https://pymongo.readthedocs.io/en/stable/faq.html#is-pymongo-fork-safe Quoting from that page: > PyMongo is not fork-safe. Care must be taken when using instances of > MongoClient with fork(). Specifically, instances of MongoClient must > not be copied from a parent process to a child process. Instead, the > parent process and each child process must create their own instances > of MongoClient. Instances of MongoClient copied from the parent > process have a high probability of deadlock in the child process due > to the inherent incompatibilities between fork(), threads, and locks > described below. PyMongo will attempt to issue a warning if there is a > chance of this deadlock occurring. For edx-platform, the MongoClient connection is initalized with the modulestore() invocation. That call creates and caches a global variable that Studio or the LMS will reuse across the life of the worker process. That initialization was put into lms/wsgi.py in 7c758ec9, but originated in lms/startup.py with 51d0dd1. The original reason for it is because at that time (2013), we still supported the XML Modulestore, which stored courses on disk as directories of OLX files and static assets. The XML Modulestore would then read the entirety of those courses into memory at startup. This meant that the startup process was *extremely* expensive, so we needed to have it happen before the workers started serving requests to users, instead of having the system lazily read them in when the first user request arrived. Loading course content in this form hasn't been supported since 2016, meaning that modulestore initialization is no longer the performance time bomb that it once was. The fact that this code remained here is likely an oversight, which was considered harmless until @ztraboo reported these pymongo log messages during the course of investigating performance issues: https://discuss.openedx.org/t/atlas-mongodb-performance-issues-un-indexed-queries/12803/16 Two potential followups that should be explored after this: 1. Tutor should probably be forking earlier than this, before we load Django settings and initialize database and cache connections. 2. It's possible that the caching mechanism for modulestore should be revisited to operate at the request cache level. The performance benefit of keeping it around may not be worth the potential memory leaks. Anything we do here would have to be very carefully monitored though, since connection costs may add up. --- lms/wsgi.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/lms/wsgi.py b/lms/wsgi.py index bb0d0f7ede8c..169b6929fbfa 100644 --- a/lms/wsgi.py +++ b/lms/wsgi.py @@ -18,13 +18,6 @@ import lms.startup as startup # lint-amnesty, pylint: disable=wrong-import-position startup.run() -from xmodule.modulestore.django import modulestore # lint-amnesty, pylint: disable=wrong-import-position - -# Trigger a forced initialization of our modulestores since this can take a -# while to complete and we want this done before HTTP requests are accepted. -modulestore() - - # This application object is used by the development server # as well as any WSGI server configured to use this file. from django.core.wsgi import get_wsgi_application # lint-amnesty, pylint: disable=wrong-import-order, wrong-import-position From 1dea3e78d4a4569a42e68163bf1121183c10b30a Mon Sep 17 00:00:00 2001 From: feanil <781561+feanil@users.noreply.github.com> Date: Tue, 7 May 2024 15:50:54 +0000 Subject: [PATCH 044/113] feat: Upgrade Python dependency acid-xblock Routine requirement upgrade. Doing it individually because there are too many changes in `make upgrade`. Commit generated by workflow `openedx/edx-platform/.github/workflows/upgrade-one-python-dependency.yml@refs/heads/feanil/fix_one_upgrade` --- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/doc.txt | 2 +- requirements/edx/testing.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index d444b4674641..a94e4f85187a 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -6,7 +6,7 @@ # -e git+https://github.com/anupdhabarde/edx-proctoring-proctortrack.git@31c6c9923a51c903ae83760ecbbac191363aa2a2#egg=edx_proctoring_proctortrack # via -r requirements/edx/github.in -acid-xblock==0.3.0 +acid-xblock==0.3.1 # via -r requirements/edx/kernel.in aiohttp==3.9.3 # via diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 28f89175a88e..a70bcbda854a 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -12,7 +12,7 @@ accessible-pygments==0.0.4 # via # -r requirements/edx/doc.txt # pydata-sphinx-theme -acid-xblock==0.3.0 +acid-xblock==0.3.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index b8537bb33e67..f5e15aa81c17 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -8,7 +8,7 @@ # via -r requirements/edx/base.txt accessible-pygments==0.0.4 # via pydata-sphinx-theme -acid-xblock==0.3.0 +acid-xblock==0.3.1 # via -r requirements/edx/base.txt aiohttp==3.9.3 # via diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 79057ddaee30..f39741c19b55 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -6,7 +6,7 @@ # -e git+https://github.com/anupdhabarde/edx-proctoring-proctortrack.git@31c6c9923a51c903ae83760ecbbac191363aa2a2#egg=edx_proctoring_proctortrack # via -r requirements/edx/base.txt -acid-xblock==0.3.0 +acid-xblock==0.3.1 # via -r requirements/edx/base.txt aiohttp==3.9.3 # via From c66f28dceef0f30003b15cf3f0fa2163a0f99b6a Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Tue, 7 May 2024 09:15:27 -0400 Subject: [PATCH 045/113] fix: Add a trailing wildcard because we don't use an exact path. We were seeing the following error: ``` /usr/bin/git add -- requirements scripts/**/requirements fatal: pathspec 'scripts/**/requirements' did not match any files ``` Once we introduce wildcards, the whole path needs to be valid so adding a trailing wildcard to catch all the relevant directories and files. --- .github/workflows/upgrade-one-python-dependency.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/upgrade-one-python-dependency.yml b/.github/workflows/upgrade-one-python-dependency.yml index 40ede7758138..91e042a112f4 100644 --- a/.github/workflows/upgrade-one-python-dependency.yml +++ b/.github/workflows/upgrade-one-python-dependency.yml @@ -82,13 +82,13 @@ jobs: - name: Make a PR id: make-pr - uses: peter-evans/create-pull-request@v5 + uses: peter-evans/create-pull-request@v6 with: branch: "${{ github.triggering_actor }}/upgrade-${{ inputs.package }}" branch-suffix: short-commit-hash add-paths: | requirements - scripts/**/requirements + scripts/**/requirements* commit-message: | feat: Upgrade Python dependency ${{ inputs.package }} From 6b88d2db35aab5541cb02f6053696edcc8203a2f Mon Sep 17 00:00:00 2001 From: feanil <781561+feanil@users.noreply.github.com> Date: Tue, 7 May 2024 16:55:37 +0000 Subject: [PATCH 046/113] feat: Upgrade Python dependency aiohttp Routine requirement upgrade. Doing it individually because there are too many changes in `make upgrade`. Commit generated by workflow `openedx/edx-platform/.github/workflows/upgrade-one-python-dependency.yml@refs/heads/feanil/fix_one_upgrade` --- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/doc.txt | 2 +- requirements/edx/testing.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index d444b4674641..bd2fb37dc614 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -8,7 +8,7 @@ # via -r requirements/edx/github.in acid-xblock==0.3.0 # via -r requirements/edx/kernel.in -aiohttp==3.9.3 +aiohttp==3.9.5 # via # geoip2 # openai diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 28f89175a88e..a251b46d57df 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -16,7 +16,7 @@ acid-xblock==0.3.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -aiohttp==3.9.3 +aiohttp==3.9.5 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index b8537bb33e67..0554765bdf29 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -10,7 +10,7 @@ accessible-pygments==0.0.4 # via pydata-sphinx-theme acid-xblock==0.3.0 # via -r requirements/edx/base.txt -aiohttp==3.9.3 +aiohttp==3.9.5 # via # -r requirements/edx/base.txt # geoip2 diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 79057ddaee30..ffc2cffce0ad 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -8,7 +8,7 @@ # via -r requirements/edx/base.txt acid-xblock==0.3.0 # via -r requirements/edx/base.txt -aiohttp==3.9.3 +aiohttp==3.9.5 # via # -r requirements/edx/base.txt # geoip2 From 1d17f2fba8b4988fd7d7095bf5fc4399620f0fd1 Mon Sep 17 00:00:00 2001 From: feanil <781561+feanil@users.noreply.github.com> Date: Tue, 7 May 2024 16:58:44 +0000 Subject: [PATCH 047/113] feat: Upgrade Python dependency babel Routine requirement upgrade. Doing it individually because there are too many changes in `make upgrade`. Commit generated by workflow `openedx/edx-platform/.github/workflows/upgrade-one-python-dependency.yml@refs/heads/feanil/fix_one_upgrade` --- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/doc.txt | 2 +- requirements/edx/testing.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index d444b4674641..9307aca3fdd5 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -48,7 +48,7 @@ attrs==23.2.0 # openedx-events # openedx-learning # referencing -babel==2.14.0 +babel==2.15.0 # via # -r requirements/edx/kernel.in # enmerkar diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 28f89175a88e..06f8c8c2dd57 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -98,7 +98,7 @@ attrs==23.2.0 # openedx-events # openedx-learning # referencing -babel==2.14.0 +babel==2.15.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index b8537bb33e67..313c6efe9244 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -67,7 +67,7 @@ attrs==23.2.0 # openedx-events # openedx-learning # referencing -babel==2.14.0 +babel==2.15.0 # via # -r requirements/edx/base.txt # enmerkar diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 79057ddaee30..f17130073d47 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -69,7 +69,7 @@ attrs==23.2.0 # openedx-events # openedx-learning # referencing -babel==2.14.0 +babel==2.15.0 # via # -r requirements/edx/base.txt # enmerkar From a3924f687b6c8e64aeefa4807ac5fcb411581181 Mon Sep 17 00:00:00 2001 From: Yusuf Musleh Date: Tue, 7 May 2024 20:29:35 +0300 Subject: [PATCH 048/113] feat: Tag sections, subsections, and the whole course (#34690) (In the legacy UI, if the 'new_studio_mfe.use_tagging_taxonomy_list_page' waffle flag is enabled) --- .../xblock_storage_handlers/view_handlers.py | 28 +++++++--- cms/static/js/views/course_manage_tags.js | 54 +++++++++++++++++++ cms/static/js/views/course_outline.js | 13 +++-- cms/static/js/views/pages/course_outline.js | 15 +++++- cms/static/sass/views/_outline.scss | 7 +++ cms/templates/course_outline.html | 3 +- .../js/course-manage-tags.underscore | 8 +++ cms/templates/js/course-outline.underscore | 14 ++--- 8 files changed, 121 insertions(+), 21 deletions(-) create mode 100644 cms/static/js/views/course_manage_tags.js create mode 100644 cms/templates/js/course-manage-tags.underscore diff --git a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py index aac29de20aa9..de3dbc55e54f 100644 --- a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py +++ b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py @@ -1243,7 +1243,8 @@ def create_xblock_info( # lint-amnesty, pylint: disable=too-many-statements # If the ENABLE_TAGGING_TAXONOMY_LIST_PAGE feature flag is enabled, we show the "Manage Tags" options if use_tagging_taxonomy_list_page(): xblock_info["use_tagging_taxonomy_list_page"] = True - xblock_info["tag_counts_by_unit"] = _get_course_unit_tags(xblock.location.context_key) + xblock_info["course_tags_count"] = _get_course_tags_count(course.id) + xblock_info["tag_counts_by_block"] = _get_course_block_tags(xblock.location.context_key) xblock_info[ "has_partition_group_components" @@ -1263,16 +1264,29 @@ def create_xblock_info( # lint-amnesty, pylint: disable=too-many-statements @request_cached() -def _get_course_unit_tags(course_key) -> dict: +def _get_course_tags_count(course_key) -> dict: """ - Get the count of tags that are applied to each unit (vertical) in this course, as a dict. + Get the count of tags that are applied to the course as a dict: {course_key: tags_count} + """ + if not course_key.is_course: + return {} # Unsupported key type + + return get_object_tag_counts(str(course_key), count_implicit=True) + + +@request_cached() +def _get_course_block_tags(course_key) -> dict: + """ + Get the count of tags that are applied to each block in this course, as a dict. """ if not course_key.is_course: return {} # Unsupported key type, e.g. a library - # Create a pattern to match the IDs of the units, e.g. "block-v1:org+course+run+type@vertical+block@*" - vertical_key = course_key.make_usage_key('vertical', 'x') - unit_key_pattern = str(vertical_key).rsplit("@", 1)[0] + "@*" - return get_object_tag_counts(unit_key_pattern, count_implicit=True) + + # Create a pattern to match the IDs of all blocks, e.g. "block-v1:org+course+run+type@*" + catch_all_key = course_key.make_usage_key("*", "x") + catch_all_key_pattern = str(catch_all_key).rsplit("@*", 1)[0] + "@*" + + return get_object_tag_counts(catch_all_key_pattern, count_implicit=True) def _was_xblock_ever_exam_linked_with_external(course, xblock): diff --git a/cms/static/js/views/course_manage_tags.js b/cms/static/js/views/course_manage_tags.js new file mode 100644 index 000000000000..b276069b345d --- /dev/null +++ b/cms/static/js/views/course_manage_tags.js @@ -0,0 +1,54 @@ +define([ + 'jquery', 'underscore', 'backbone', 'js/utils/templates', + 'edx-ui-toolkit/js/utils/html-utils', 'js/views/utils/tagging_drawer_utils', + 'js/views/tag_count', 'js/models/tag_count'], +function( + $, _, Backbone, TemplateUtils, HtmlUtils, TaggingDrawerUtils, TagCountView, TagCountModel +) { + 'use strict'; + + var CourseManageTagsView = Backbone.View.extend({ + events: { + 'click .manage-tags-button': 'openManageTagsDrawer', + }, + + initialize: function() { + this.template = TemplateUtils.loadTemplate('course-manage-tags'); + this.courseId = course.id; + }, + + openManageTagsDrawer: function(event) { + const taxonomyTagsWidgetUrl = this.model.get('taxonomy_tags_widget_url'); + const contentId = this.courseId; + TaggingDrawerUtils.openDrawer(taxonomyTagsWidgetUrl, contentId); + }, + + renderTagCount: function() { + const contentId = this.courseId; + const tagCountsForCourse = this.model.get('course_tags_count'); + const tagsCount = tagCountsForCourse !== undefined ? tagCountsForCourse[contentId] : 0; + var countModel = new TagCountModel({ + content_id: contentId, + tags_count: tagsCount, + course_authoring_url: this.model.get('course_authoring_url'), + }, {parse: true}); + var tagCountView = new TagCountView({el: this.$('.tag-count'), model: countModel}); + tagCountView.setupMessageListener(); + tagCountView.render(); + this.$('.tag-count').click((event) => { + event.preventDefault(); + this.openManageTagsDrawer(); + }); + }, + + render: function() { + var html = this.template(this.model.attributes); + HtmlUtils.setHtml(this.$el, HtmlUtils.HTML(html)); + this.renderTagCount(); + return this; + } + }); + + return CourseManageTagsView; +} +); diff --git a/cms/static/js/views/course_outline.js b/cms/static/js/views/course_outline.js index 8319e42eb3ea..b0e89127aa28 100644 --- a/cms/static/js/views/course_outline.js +++ b/cms/static/js/views/course_outline.js @@ -34,17 +34,22 @@ function( renderTagCount: function() { const contentId = this.model.get('id'); - const tagCountsByUnit = this.model.get('tag_counts_by_unit') - const tagsCount = tagCountsByUnit !== undefined ? tagCountsByUnit[contentId] : 0 + const tagCountsByBlock = this.model.get('tag_counts_by_block') + // Skip the course block since that is handled elsewhere in course_manage_tags + if (contentId.includes('@course')) { + return + } + const tagsCount = tagCountsByBlock !== undefined ? tagCountsByBlock[contentId] : 0 + const tagCountElem = this.$(`.tag-count[data-locator="${contentId}"]`); var countModel = new TagCountModel({ content_id: contentId, tags_count: tagsCount, course_authoring_url: this.model.get('course_authoring_url'), }, {parse: true}); - var tagCountView = new TagCountView({el: this.$('.tag-count'), model: countModel}); + var tagCountView = new TagCountView({el: tagCountElem, model: countModel}); tagCountView.setupMessageListener(); tagCountView.render(); - this.$('.tag-count').click((event) => { + tagCountElem.click((event) => { event.preventDefault(); this.openManageTagsDrawer(); }); diff --git a/cms/static/js/views/pages/course_outline.js b/cms/static/js/views/pages/course_outline.js index dd1c13eda9fc..627a21e563e5 100644 --- a/cms/static/js/views/pages/course_outline.js +++ b/cms/static/js/views/pages/course_outline.js @@ -4,10 +4,12 @@ define([ 'jquery', 'underscore', 'gettext', 'js/views/pages/base_page', 'js/views/utils/xblock_utils', 'js/views/course_outline', 'common/js/components/utils/view_utils', 'common/js/components/views/feedback_alert', - 'common/js/components/views/feedback_notification', 'js/views/course_highlights_enable', 'js/views/course_video_sharing_enable'], + 'common/js/components/views/feedback_notification', 'js/views/course_highlights_enable', 'js/views/course_video_sharing_enable', + 'js/views/course_manage_tags'], function($, _, gettext, BasePage, XBlockViewUtils, CourseOutlineView, ViewUtils, AlertView, NoteView, CourseHighlightsEnableView, - CourseVideoSharingEnableView + CourseVideoSharingEnableView, + CourseManageTagsView ) { 'use strict'; @@ -135,6 +137,15 @@ function($, _, gettext, BasePage, XBlockViewUtils, CourseOutlineView, ViewUtils, this.highlightsEnableView.render(); } + // if tagging enabled + if (this.model.get('use_tagging_taxonomy_list_page')) { + this.courseManageTagsView = new CourseManageTagsView({ + el: this.$('.status-manage-tags'), + model: this.model + }); + this.courseManageTagsView.render(); + } + // if video sharing enable if (this.model.get('video_sharing_enabled')) { this.videoSharingEnableView = new CourseVideoSharingEnableView({ diff --git a/cms/static/sass/views/_outline.scss b/cms/static/sass/views/_outline.scss index 480335211392..469939cc8d97 100644 --- a/cms/static/sass/views/_outline.scss +++ b/cms/static/sass/views/_outline.scss @@ -185,6 +185,7 @@ .status-release, .status-highlights-enabled, + .status-manage-tags, .status-video-sharing-enabled, .status-studio-frontend { @extend %t-copy-base; @@ -202,6 +203,7 @@ } .status-highlights-enabled, + .status-manage-tags, .status-video-sharing-enabled { vertical-align: top; } @@ -209,9 +211,12 @@ .status-release-label, .status-release-value, .status-highlights-enabled-label, + .status-course-manage-tags-label, .status-video-sharing-enabled-label, .status-highlights-enabled-value, + .status-course-manage-tags-value, .status-highlights-enabled-info, + .status-course-manage-tags-info, .status-video-sharing-enabled-info, .status-actions { display: inline-block; @@ -221,6 +226,7 @@ .status-release-value, .status-highlights-enabled-value, + .status-course-manage-tags-value, .status-video-sharing-enabled-value { @extend %t-strong; @@ -228,6 +234,7 @@ } .status-highlights-enabled-info, + .status-course-manage-tags-info, .status-video-sharing-enabled-info { font-size: smaller; margin-left: $baseline / 2; diff --git a/cms/templates/course_outline.html b/cms/templates/course_outline.html index f1e426212bce..16d9ccbd4ca5 100644 --- a/cms/templates/course_outline.html +++ b/cms/templates/course_outline.html @@ -29,7 +29,7 @@ <%block name="header_extras"> -% for template_name in ['course-outline', 'xblock-string-field-editor', 'basic-modal', 'modal-button', 'course-outline-modal', 'due-date-editor', 'self-paced-due-date-editor', 'release-date-editor', 'grading-editor', 'publish-editor', 'staff-lock-editor', 'unit-access-editor', 'discussion-editor', 'content-visibility-editor', 'verification-access-editor', 'timed-examination-preference-editor', 'access-editor', 'settings-modal-tabs', 'show-correctness-editor', 'highlights-editor', 'highlights-enable-editor', 'course-highlights-enable', 'course-video-sharing-enable', 'summary-configuration-editor', 'tag-count', 'subsection-share-link-modal-tabs', 'full-page-share-link-editor', 'embed-link-share-link-editor']: +% for template_name in ['course-outline', 'xblock-string-field-editor', 'basic-modal', 'modal-button', 'course-outline-modal', 'due-date-editor', 'self-paced-due-date-editor', 'release-date-editor', 'grading-editor', 'publish-editor', 'staff-lock-editor', 'unit-access-editor', 'discussion-editor', 'content-visibility-editor', 'verification-access-editor', 'timed-examination-preference-editor', 'access-editor', 'settings-modal-tabs', 'show-correctness-editor', 'highlights-editor', 'highlights-enable-editor', 'course-highlights-enable', 'course-manage-tags', 'course-video-sharing-enable', 'summary-configuration-editor', 'tag-count', 'subsection-share-link-modal-tabs', 'full-page-share-link-editor', 'embed-link-share-link-editor']: @@ -269,6 +269,7 @@

${_("Page Actions")}

+
+

+ <%- gettext('Course tags') %> +

+
+ +<%- gettext('Manage tags') %> +
diff --git a/cms/templates/js/course-outline.underscore b/cms/templates/js/course-outline.underscore index 6525c4dc21e2..f8d60d696a3e 100644 --- a/cms/templates/js/course-outline.underscore +++ b/cms/templates/js/course-outline.underscore @@ -8,7 +8,7 @@ var userPartitionInfo = xblockInfo.get('user_partition_info'); var selectedGroupsLabel = userPartitionInfo['selected_groups_label']; var selectedPartitionIndex = userPartitionInfo['selected_partition_index']; var xblockId = xblockInfo.get('id') -var tagsCount = (xblockInfo.get('tag_counts_by_unit') || {})[xblockId] || 0; +var tagsCount = (xblockInfo.get('tag_counts_by_block') || {})[xblockId] || 0; var statusMessages = []; var messageType; @@ -187,7 +187,7 @@ if (is_proctored_exam) { <% } %> - <% if (xblockInfo.isVertical() && typeof useTaggingTaxonomyListPage !== "undefined" && useTaggingTaxonomyListPage) { %> + <% if (typeof useTaggingTaxonomyListPage !== "undefined" && useTaggingTaxonomyListPage) { %>
  • <% } %> @@ -210,12 +210,12 @@ if (is_proctored_exam) { <%- gettext('Configure') %> <% } %> + <% if (typeof useTaggingTaxonomyListPage !== "undefined" && useTaggingTaxonomyListPage) { %> + + <% } %> <% if (xblockInfo.isVertical()) { %> - <% if (typeof useTaggingTaxonomyListPage !== "undefined" && useTaggingTaxonomyListPage) { %> - - <% } %> From b551a32a49ba9b03aa531ad61ba1ceb40c0dd3b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Chris=20Ch=C3=A1vez?= Date: Tue, 7 May 2024 13:14:33 -0500 Subject: [PATCH 049/113] fix: Stop autotagging nor indexing the course root XBlock (#34627) * fix: Multiple auto-tagging when creating a course * Avoid tagging course-block and course-info blocks * Tests * feat: Avoid create index for course blocks --- openedx/core/djangoapps/content/search/api.py | 7 +++++++ .../content/search/tests/test_api.py | 8 ++++++++ .../djangoapps/content_tagging/handlers.py | 5 +++++ .../content_tagging/tests/test_tasks.py | 19 ++++++++++++++++++- 4 files changed, 38 insertions(+), 1 deletion(-) diff --git a/openedx/core/djangoapps/content/search/api.py b/openedx/core/djangoapps/content/search/api.py index 24d69dfd9fcf..bbde4fc98230 100644 --- a/openedx/core/djangoapps/content/search/api.py +++ b/openedx/core/djangoapps/content/search/api.py @@ -55,6 +55,8 @@ MAX_ACCESS_IDS_IN_FILTER = 1_000 MAX_ORGS_IN_FILTER = 1_000 +EXCLUDED_XBLOCK_TYPES = ['course', 'course_info'] + @contextmanager def _index_rebuild_lock() -> Generator[str, None, None]: @@ -372,6 +374,7 @@ def add_with_children(block): docs.append(doc) # pylint: disable=cell-var-from-loop _recurse_children(block, add_with_children) # pylint: disable=cell-var-from-loop + # Index course children _recurse_children(course, add_with_children) if docs: @@ -393,6 +396,10 @@ def upsert_xblock_index_doc(usage_key: UsageKey, recursive: bool = True) -> None recursive (bool): If True, also index all children of the XBlock """ xblock = modulestore().get_item(usage_key) + xblock_type = xblock.scope_ids.block_type + + if xblock_type in EXCLUDED_XBLOCK_TYPES: + return docs = [] diff --git a/openedx/core/djangoapps/content/search/tests/test_api.py b/openedx/core/djangoapps/content/search/tests/test_api.py index f156f9bbb22e..cd1acc31b7d9 100644 --- a/openedx/core/djangoapps/content/search/tests/test_api.py +++ b/openedx/core/djangoapps/content/search/tests/test_api.py @@ -6,6 +6,7 @@ import copy from unittest.mock import MagicMock, call, patch +from opaque_keys.edx.keys import UsageKey import ddt from django.test import override_settings @@ -62,6 +63,7 @@ def setUp(self): fields={"display_name": "Test Course"}, ) course_access, _ = SearchAccess.objects.get_or_create(context_key=self.course.id) + self.course_block_key = "block-v1:org1+test_course+test_run+type@course+block@course" # Create XBlocks self.sequential = self.store.create_child(self.user_id, self.course.location, "sequential", "test_sequential") @@ -184,6 +186,12 @@ def test_index_xblock_metadata(self, recursive, mock_meilisearch): mock_meilisearch.return_value.index.return_value.update_documents.assert_called_once_with(expected_docs) + @override_settings(MEILISEARCH_ENABLED=True) + def test_no_index_excluded_xblocks(self, mock_meilisearch): + api.upsert_xblock_index_doc(UsageKey.from_string(self.course_block_key)) + + mock_meilisearch.return_value.index.return_value.update_document.assert_not_called() + @override_settings(MEILISEARCH_ENABLED=True) def test_index_xblock_tags(self, mock_meilisearch): """ diff --git a/openedx/core/djangoapps/content_tagging/handlers.py b/openedx/core/djangoapps/content_tagging/handlers.py index 74b4dd5b3331..cc86f7e0dcd6 100644 --- a/openedx/core/djangoapps/content_tagging/handlers.py +++ b/openedx/core/djangoapps/content_tagging/handlers.py @@ -67,9 +67,14 @@ def auto_tag_xblock(**kwargs): if not CONTENT_TAGGING_AUTO.is_enabled(xblock_info.usage_key.course_key): return + if xblock_info.block_type == 'course_info': + # We want to add tags only to the course id, not with its XBlock + return + if xblock_info.block_type == "course": # Course update is handled by XBlock of course type update_course_tags.delay(str(xblock_info.usage_key.course_key)) + return update_xblock_tags.delay(str(xblock_info.usage_key)) diff --git a/openedx/core/djangoapps/content_tagging/tests/test_tasks.py b/openedx/core/djangoapps/content_tagging/tests/test_tasks.py index ed0cf2c06025..3e396eb754b3 100644 --- a/openedx/core/djangoapps/content_tagging/tests/test_tasks.py +++ b/openedx/core/djangoapps/content_tagging/tests/test_tasks.py @@ -8,7 +8,7 @@ from django.test import override_settings, LiveServerTestCase from django.http import HttpRequest from edx_toggles.toggles.testutils import override_waffle_flag -from openedx_tagging.core.tagging.models import Tag, Taxonomy +from openedx_tagging.core.tagging.models import Tag, Taxonomy, ObjectTag from organizations.models import Organization from common.djangoapps.student.tests.factories import UserFactory @@ -111,6 +111,23 @@ def test_create_course(self): # Check if the tags are created in the Course assert self._check_tag(course.id, LANGUAGE_TAXONOMY_ID, "Polski") + def test_only_tag_course_id(self): + # Create course + course = self.store.create_course( + self.orgA.short_name, + "test_course", + "test_run", + self.user_id, + fields={"language": "pl"}, + ) + object_id = str(course.id).replace('course-v1:', '') + + # Check that only one object tag is created for the course + tags = ObjectTag.objects.filter(object_id__contains=object_id) + assert len(tags) == 1 + assert tags[0].value == "Polski" + assert tags[0].object_id == str(course.id) + @override_settings(LANGUAGE_CODE='pt-br') def test_create_course_invalid_language(self): # Create course From d3d3225fa6d72dad0b2732b0f3be61e4ecfbd260 Mon Sep 17 00:00:00 2001 From: Muhammad Farhan Khan Date: Tue, 7 May 2024 23:45:04 +0500 Subject: [PATCH 050/113] Move user-retirement scripts docs near code (#34695) * chore: Move user-retirement scripts docs near code --------- Co-authored-by: Feanil Patel --- docs/conf.py | 1 + scripts/user_retirement/README.rst | 6 +- scripts/user_retirement/docs/driver_setup.rst | 134 +++++++++++++ .../docs/implementation_overview.rst | 117 ++++++++++++ scripts/user_retirement/docs/index.rst | 38 ++++ .../user_retirement/docs/service_setup.rst | 179 ++++++++++++++++++ .../user_retirement/docs/special_cases.rst | 86 +++++++++ 7 files changed, 558 insertions(+), 3 deletions(-) create mode 100644 scripts/user_retirement/docs/driver_setup.rst create mode 100644 scripts/user_retirement/docs/implementation_overview.rst create mode 100644 scripts/user_retirement/docs/index.rst create mode 100644 scripts/user_retirement/docs/service_setup.rst create mode 100644 scripts/user_retirement/docs/special_cases.rst diff --git a/docs/conf.py b/docs/conf.py index 027e56808b6a..f37fc32f6160 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -58,6 +58,7 @@ 'sphinx.ext.autodoc', 'sphinx.ext.coverage', 'sphinx.ext.doctest', + 'sphinx.ext.graphviz', 'sphinx.ext.ifconfig', 'sphinx.ext.intersphinx', 'sphinx.ext.mathjax', diff --git a/scripts/user_retirement/README.rst b/scripts/user_retirement/README.rst index 20c99197ed6d..d5417f7f7e75 100644 --- a/scripts/user_retirement/README.rst +++ b/scripts/user_retirement/README.rst @@ -1,7 +1,7 @@ User Retirement Scripts ======================= -`This `_ directory contains python scripts which are migrated from the `tubular `_ respository. +`This `_ directory contains python scripts which are migrated from the `tubular `_ respository. These scripts are intended to drive the user retirement workflow which involves handling the deactivation or removal of user accounts as part of the platform's management process. These scripts could be called from any automation/CD framework. @@ -49,9 +49,9 @@ In-depth Documentation and Configuration Steps For in-depth documentation and essential configurations follow these docs -`Documentation `_ +`Documentation `_ -`Configuration Docs `_ +`Configuration Docs `_ Execute Script diff --git a/scripts/user_retirement/docs/driver_setup.rst b/scripts/user_retirement/docs/driver_setup.rst new file mode 100644 index 000000000000..c6d7b8e6f292 --- /dev/null +++ b/scripts/user_retirement/docs/driver_setup.rst @@ -0,0 +1,134 @@ +.. _driver-setup: + +############################################# +Setting Up the User Retirement Driver Scripts +############################################# + +`scripts/user_retirement `_ +is a directory of Python scripts designed to plug into various automation +tooling. It also contains readme file having details of how to run the scripts. +Included in this directory are two scripts intended to drive the user +retirement workflow. + +``get_learners_to_retire.py`` + Generates a list of users that are ready for immediate retirement. Users + are "ready" after a certain number of days spent in the ``PENDING`` state, + specified by the ``--cool_off_days`` argument. Produces an output intended + for consumption by Jenkins in order to spawn separate downstream builds for + each user. +``retire_one_learner.py`` + Retires the user specified by the ``--username`` argument. + +These two scripts share a required ``--config_file`` argument, which specifies +the driver configuration file for your environment (for example, production). +This configuration file is a YAML file that contains LMS auth secrets, API URLs, +and retirement pipeline stages specific to that environment. Here is an example +of a driver configuration file. + +.. code-block:: yaml + + client_id: + client_secret: + + base_urls: + lms: https://courses.example.com/ + ecommerce: https://ecommerce.example.com/ + credentials: https://credentials.example.com/ + + retirement_pipeline: + - ['RETIRING_EMAIL_LISTS', 'EMAIL_LISTS_COMPLETE', 'LMS', 'retirement_retire_mailings'] + - ['RETIRING_ENROLLMENTS', 'ENROLLMENTS_COMPLETE', 'LMS', 'retirement_unenroll'] + - ['RETIRING_LMS_MISC', 'LMS_MISC_COMPLETE', 'LMS', 'retirement_lms_retire_misc'] + - ['RETIRING_LMS', 'LMS_COMPLETE', 'LMS', 'retirement_lms_retire'] + +The ``client_id`` and ``client_secret`` keys contain the oauth credentials. +These credentials are simply copied from the output of the +``create_dot_application`` management command described in +:ref:`retirement-service-user`. + +The ``base_urls`` section in the configuration file defines the mappings of +IDA to base URLs used by the scripts to construct API URLs. Only the LMS is +mandatory here, but if any of your pipeline states contain API calls to other +services, those services must also be present in the ``base_urls`` section. + +The ``retirement_pipeline`` section defines the steps, state names, and order +of execution for each environment. Each item is a list in the form of: + +#. Start state name +#. End state name +#. IDA to call against (LMS, ECOMMERCE, or CREDENTIALS currently) +#. Method name to call in + `edx_api.py `_ + +For example: ``['RETIRING_CREDENTIALS', 'CREDENTIALS_COMPLETE', 'CREDENTIALS', +'retire_learner']`` will set the user's state to ``RETIRING_CREDENTIALS``, call +a pre-instantiated ``retire_learner`` method in the ``CredentialsApi``, then set +the user's state to ``CREDENTIALS_COMPLETE``. + +******** +Examples +******** + +The following are some examples of how to use the driver scripts. + +================== +Set Up Environment +================== + +Follow this `readme `_ to set up your execution environment. + +========================= +List of Targeted Learners +========================= + +Generate a list of learners that are ready for retirement (those learners who +have selected and confirmed account deletion and have been in the ``PENDING`` +state for the time specified ``cool_off_days``). + +.. code-block:: bash + + mkdir learners_to_retire + get_learners_to_retire.py \ + --config_file=path/to/config.yml \ + --output_dir=learners_to_retire \ + --cool_off_days=5 + +===================== +Run Retirement Script +===================== + +After running these commands, the ``learners_to_retire`` directory contains +several INI files, each containing a single line in the form of ``USERNAME +=``. Iterate over these files while executing the +``retire_one_learner.py`` script on each learner with a command like the following. + +.. code-block:: bash + + retire_one_learner.py \ + --config_file=path/to/config.yml \ + --username= + + +************************************************** +Using the Driver Scripts in an Automated Framework +************************************************** + +At edX, we call the user retirement scripts from +`Jenkins `_ jobs on one of our internal Jenkins +services. The user retirement driver scripts are intended to be agnostic +about which automation framework you use, but they were only fully tested +from Jenkins. + +For more information about how we execute these scripts at edX, see the +following wiki articles: + +* `User Retirement Jenkins Implementation `_ +* `How to: retirement Jenkins jobs development and testing `_ + +And check out the Groovy DSL files we use to seed these jobs: + +* `platform/jobs/RetirementJobs.groovy in edx/jenkins-job-dsl `_ +* `platform/jobs/RetirementJobEdxTriggers.groovy in edx/jenkins-job-dsl `_ + +.. include:: ../../../../links/links.rst + diff --git a/scripts/user_retirement/docs/implementation_overview.rst b/scripts/user_retirement/docs/implementation_overview.rst new file mode 100644 index 000000000000..37a814c1d583 --- /dev/null +++ b/scripts/user_retirement/docs/implementation_overview.rst @@ -0,0 +1,117 @@ +.. _Implmentation: + +####################### +Implementation Overview +####################### + +In the Open edX platform, the user experience is enabled by several +services, such as LMS, Studio, ecommerce, credentials, discovery, and more. +Personally Identifiable Identification (PII) about a user can exist in many of +these services. As a consequence, to remove a user's PII, you must be able +to request each service containing PII to remove, delete, or unlink the +data for that user in that service. + +In the user retirement feature, a centralized process (the *driver* scripts) +orchestrates all of these requests. For information about how to configure the +driver scripts, see :ref:`driver-setup`. + +**************************** +The User Retirement Workflow +**************************** + +The user retirement workflow is a configurable pipeline of building-block +APIs. These APIs are used to: + + * "Forget" a retired user's PII + * Prevent a retired user from logging back in + * Prevent re-use of the username or email address of a retired user + +Depending on which third parties a given Open edX instance integrates with, +the user retirement process may need to call out to external services or to +generate reports for later processing. Any such reports must subsequently be +destroyed. + +Configurability and adaptability were design goals from the beginning, so this +user retirement tooling should be able to accommodate a wide range of Open edX +sites and custom use cases. + +The workflow is designed to be linear and rerunnable, allowing recovery and +continuation in cases where a particular stage fails. Each user who has +requested retirement will be individually processed through this workflow, so +multiple users could be in the same state simultaneously. The LMS is the +authoritative source of information about the state of each user in the +retirement process, and the arbiter of state progressions, using the +``UserRetirementStatus`` model and associated APIs. The LMS also holds a +table of the states themselves (the ``RetirementState`` model), rather than +hard-coding the states. This was done because we cannot predict all the +possible states required by all members of the Open edX community. + +This example state diagram outlines the pathways users follow throughout the +workflow: + +.. digraph:: retirement_states_example + :align: center + + ranksep = "0.3"; + + node[fontname=Courier,fontsize=12,shape=box,group=main] + { rank = same INIT[style=invis] PENDING } + INIT -> PENDING; + "..."[shape=none] + PENDING -> RETIRING_ENROLLMENTS -> ENROLLMENTS_COMPLETE -> RETIRING_FORUMS -> FORUMS_COMPLETE -> "..." -> COMPLETE; + + node[group=""]; + RETIRING_ENROLLMENTS -> ERRORED; + RETIRING_FORUMS -> ERRORED; + PENDING -> ABORTED; + + subgraph cluster_terminal_states { + label = "Terminal States"; + labelloc = b // put label at bottom + {rank = same ERRORED COMPLETE ABORTED} + } + +Unless an error occurs internal to the user retirement tooling, a user's +retirement state should always land in one of the terminal states. At that +point, either their entry should be cleaned up from the +``UserRetirementStatus`` table or, if the state is ``ERRORED``, the +administrator needs to examine the error and resolve it. For more information, +see :ref:`recovering-from-errored`. + +******************* +The User Experience +******************* + +From the learner's perspective, the vast majority of this process is obscured. +The Account page contains a new section titled **Delete My Account**. In this +section, a learner may click the **Delete My Account** button and enter +their password to confirm their request. Subsequently, all of the learner's +browser sessions are logged off, and they become locked out of their account. + +An informational email is immediately sent to the learner to confirm the +deletion of their account. After this email is sent, the learner has a limited +amount of time (defined by the ``--cool_off_days`` argument described in +:ref:`driver-setup`) to contact the site administrators and rescind their +request. + +At this point, the learner's account has been deactivated, but *not* retired. +An entry in the ``UserRetirementStatus`` table is added, and their state set to +``PENDING``. + +By default, the **Delete My Account** section is visible and the button is +enabled, allowing account deletions to queue up. The +``ENABLE_ACCOUNT_DELETION`` feature in django settings toggles the visibility +of this section. See :ref:`django-settings`. + +================ +Third Party Auth +================ + +Learners who registered using social authentication must first unlink their +LMS account from their third-party account. For those learners, the **Delete +My Account** button will be disabled until they do so; meanwhile, they will be +instructed to follow the procedure in this help center article: `How do I link +or unlink my edX account to a social media +account? `_. + +.. include:: ../../../../links/links.rst diff --git a/scripts/user_retirement/docs/index.rst b/scripts/user_retirement/docs/index.rst new file mode 100644 index 000000000000..383c7a6aa8e1 --- /dev/null +++ b/scripts/user_retirement/docs/index.rst @@ -0,0 +1,38 @@ +.. _Enabling User Retirement: + +#################################### +Enabling the User Retirement Feature +#################################### + +There have been many changes to privacy laws (for example, GDPR or the +European Union General Data Protection Regulation) intended to change the way +that businesses think about and handle Personally Identifiable Information +(PII). + +As a step toward enabling Open edX to support some of the key updates in privacy +laws, edX has implemented APIs and tooling that enable Open edX instances to +retire registered users. When you implement this user retirement feature, your +Open edX instance can automatically erase PII for a given user from systems that +are internal to Open edX (for example, the LMS, forums, credentials, and other +independently deployable applications (IDAs)), as well as external systems, such +as third-party marketing services. + +This section is intended not only for instructing Open edX admins to perform +the basic setup, but also to offer some insight into the implementation of the +user retirement feature in order to help the Open edX community build +additional APIs and states that meet their special needs. Custom code, +plugins, packages, or XBlocks in your Open edX instance might store PII, but +this feature will not magically find and clean up that PII. You may need to +create your own custom code to include PII that is not covered by the user +retirement feature. + +.. toctree:: + :maxdepth: 1 + + implementation_overview + service_setup + driver_setup + special_cases + +.. include:: ../../../../links/links.rst + diff --git a/scripts/user_retirement/docs/service_setup.rst b/scripts/user_retirement/docs/service_setup.rst new file mode 100644 index 000000000000..4fd59fcfd3ad --- /dev/null +++ b/scripts/user_retirement/docs/service_setup.rst @@ -0,0 +1,179 @@ +.. _Service Setup: + +##################################### +Setting Up User Retirement in the LMS +##################################### + +This section describes how to set up and configure the user retirement feature +in the Open edX LMS. + +.. _django-settings: + +*************** +Django Settings +*************** + +The following Django settings control the behavior of the user retirement +feature. Note that some of these settings values are lambda functions rather +than standard string literals. This is intentional; it is a pattern for +defining *derived* settings specific to Open edX. Read more about it in +`openedx/core/lib/derived.py +`_. + +.. list-table:: + :header-rows: 1 + + * - Setting Name + - Default + - Description + * - RETIRED_USERNAME_PREFIX + - ``'retired__user_'`` + - The prefix part of hashed usernames. Used in ``RETIRED_USERNAME_FMT``. + * - RETIRED_EMAIL_PREFIX + - ``'retired__user_'`` + - The prefix part of hashed emails. Used in ``RETIRED_EMAIL_FMT``. + * - RETIRED_EMAIL_DOMAIN + - ``'retired.invalid'`` + - The domain part of hashed emails. Used in ``RETIRED_EMAIL_FMT``. + * - RETIRED_USERNAME_FMT + - ``lambda settings: + settings.RETIRED_USERNAME_PREFIX + '{}'`` + - The username field for a retired user gets transformed into this format, + where ``{}`` is replaced with the hash of their username. + * - RETIRED_EMAIL_FMT + - ``lambda settings: + settings.RETIRED_EMAIL_PREFIX + '{}@' + + settings.RETIRED_EMAIL_DOMAIN`` + - The email field for a retired user gets transformed into this format, where + ``{}`` is replaced with the hash of their email. + * - RETIRED_USER_SALTS + - None + - A list of salts used for hashing usernames and emails. Only the last item in this list is used as a salt for all new retirements, but historical salts are preserved in order to guarantee that all hashed usernames and emails can still be checked. The default value **MUST** be overridden! + * - RETIREMENT_SERVICE_WORKER_USERNAME + - ``'RETIREMENT_SERVICE_USER'`` + - The username of the retirement service worker. + * - RETIREMENT_STATES + - See `lms/envs/common.py `_ + in the ``RETIREMENT_STATES`` setting + - A list that defines the name and order of states for the retirement + workflow. See `Retirement States`_ for details. + * - FEATURES['ENABLE_ACCOUNT_DELETION'] + - True + - Whether to display the "Delete My Account" section the account settings page. + + +================= +Retirement States +================= + +The state of each user's retirement is stored in the LMS database, and the +state list itself is also separately stored in the database. We expect the +list of states will be variable over time and across different Open edX +installations, so it is the responsibility of the administrator to populate +the state list. + +The default states are defined in `lms/envs/common.py +`_ +in the ``RETIREMENT_STATES`` setting. There must be, at minimum, a ``PENDING`` +state at the beginning, and ``COMPLETED``, ``ERRORED``, and ``ABORTED`` states +at the end of the list. Also, for every ``RETIRING_foo`` state, there must be +a corresponding ``foo_COMPLETE`` state. + +Override these states if you need to add any states. Typically, these +settings are set in ``lms.yml``. + +After you have defined any custom states, populate the states table with the +following management command: + +.. code-block:: bash + + $ ./manage.py lms --settings= populate_retirement_states + + All states removed and new states added. Differences: + Added: set([u'RETIRING_ENROLLMENTS', u'RETIRING_LMS', u'LMS_MISC_COMPLETE', u'RETIRING_LMS_MISC', u'ENROLLMENTS_COMPLETE', u'LMS_COMPLETE']) + Removed: set([]) + Remaining: set([u'ERRORED', u'PENDING', u'ABORTED', u'COMPLETE']) + States updated successfully. Current states: + PENDING (step 1) + RETIRING_ENROLLMENTS (step 11) + ENROLLMENTS_COMPLETE (step 21) + RETIRING_LMS_MISC (step 31) + LMS_MISC_COMPLETE (step 41) + RETIRING_LMS (step 51) + LMS_COMPLETE (step 61) + ERRORED (step 71) + ABORTED (step 81) + COMPLETE (step 91) + +In this example, some states specified in settings were already present, so +they were listed under ``Remaining`` and were not re-added. The command output +also prints the ``Current states``; this represents all the states in the +states table. The ``populate_retirement_states`` command is idempotent, and +always attempts to make the states table reflect the ``RETIREMENT_STATES`` +list in settings. + +.. _retirement-service-user: + +*********************** +Retirement Service User +*********************** + +The user retirement driver scripts authenticate with the LMS and IDAs as the +retirement service user with oauth client credentials. Therefore, to use the +driver scripts, you must create a retirement service user, and generate a DOT +application and client credentials, as in the following command. + +.. code-block:: bash + + app_name=retirement + user_name=retirement_service_worker + ./manage.py lms --settings= manage_user $user_name $user_name@example.com --staff --superuser + ./manage.py lms --settings= create_dot_application $app_name $user_name + +.. note:: + The client credentials (client ID and client secret) will be printed to the + terminal, so take this opportunity to copy them for future reference. You + will use these credentials to configure the driver scripts. For more + information, see :ref:`driver-setup`. + +The retirement service user needs permission to perform retirement tasks, and +that is done by specifying the ``RETIREMENT_SERVICE_WORKER_USERNAME`` variable +in Django settings: + +.. code-block:: python + + RETIREMENT_SERVICE_WORKER_USERNAME = 'retirement_service_worker' + +************ +Django Admin +************ + +The Django admin interface contains the following models under ``USER_API`` +that relate to user retirement. + +.. list-table:: + :widths: 15 30 55 + :header-rows: 1 + + * - Name + - URI + - Description + * - Retirement States + - ``/admin/user_api/retirementstate/`` + - Represents the table of states defined in ``RETIREMENT_STATES`` and + populated with ``populate_retirement_states``. + * - User Retirement Requests + - ``/admin/user_api/userretirementrequest/`` + - Represents the table that tracks the user IDs of every learner who + has ever requested account deletion. This table is primarily used for + internal bookkeeping, and normally isn't useful for administrators. + * - User Retirement Statuses + - ``/admin/user_api/userretirementstatus/`` + - Model for managing the retirement state for each individual learner. + +In special cases where you may need to manually intervene with the pipeline, +you can use the User Retirement Statuses management page to change the +state for an individual user. For more information about how to handle these +cases, see :ref:`handling-special-cases`. + +.. include:: ../../../../links/links.rst diff --git a/scripts/user_retirement/docs/special_cases.rst b/scripts/user_retirement/docs/special_cases.rst new file mode 100644 index 000000000000..ae544c3208c6 --- /dev/null +++ b/scripts/user_retirement/docs/special_cases.rst @@ -0,0 +1,86 @@ +.. _handling-special-cases: + +###################### +Handling Special Cases +###################### + +.. _recovering-from-errored: + +Recovering from ERRORED +*********************** + +If a retirement API indicates failure (4xx or 5xx status code), the driver +immediately sets the user's state to ``ERRORED``. To debug this error state, +check the ``responses`` field in the user's row in +``user_api_userretirementstatus`` (User Retirement Status) for any relevant +logging. Once the issue is resolved, you need to manually set the user's +``current_state`` to the state immediately prior to the state which should be +re-tried. You can do this using the Django admin. In this example, a user +retirement errored during forums retirement, so we manually reset their state +from ``ERRORED`` to ``ENROLLMENTS_COMPLETE``. + +.. digraph:: retirement_states_example + :align: center + + //rankdir=LR; // Rank Direction Left to Right + ranksep = "0.3"; + + edge[color=grey] + + node[fontname=Courier,fontsize=12,shape=box,group=main] + { rank = same INIT[style=invis] PENDING } + { + edge[style=bold,color=black] + INIT -> PENDING; + "..."[shape=none] + PENDING -> RETIRING_ENROLLMENTS -> ENROLLMENTS_COMPLETE -> RETIRING_FORUMS; + } + RETIRING_FORUMS -> FORUMS_COMPLETE -> "..." -> COMPLETE + + node[group=""]; + RETIRING_ENROLLMENTS -> ERRORED; + RETIRING_FORUMS -> ERRORED[style=bold,color=black]; + PENDING -> ABORTED; + + subgraph cluster_terminal_states { + label = "Terminal States"; + labelloc = b // put label at bottom + {rank = same ERRORED COMPLETE ABORTED} + } + + ERRORED -> ENROLLMENTS_COMPLETE[style="bold,dashed",color=black,label=" via django\nadmin"] + +Now, the user retirement driver scripts will automatically resume this user's +retirement the next time they are executed. + +Rerunning some or all states +***************************** + +If you decide you want to rerun all retirements from the beginning, set +``current_state`` to ``PENDING`` for all retirements with ``current_state`` == +``COMPLETE``. This would be useful in the case where a new stage in the user +retirement workflow is added after running all retirements (but before the +retirement queue is cleaned up), and you want to run all the retirements +through the new stage. Or, perhaps you were developing a stage/API that +didn't work correctly but still indicated success, so the pipeline progressed +all users into ``COMPLETED``. Retirement APIs are designed to be idempotent, +so this should be a no-op for stages already run for a given user. + +Cancelling a retirement +*********************** + +Users who have recently requested account deletion but are still in the +``PENDING`` retirement state may request to rescind their account deletion by +emailing or otherwise contacting the administrators directly. edx-platform +offers a Django management command that administrators can invoke manually to +cancel a retirement, given the user's email address. It restores a given +user's login capabilities and removes them from all retirement queues. The +syntax is as follows: + +.. code-block:: bash + + $ ./manage.py lms --settings= cancel_user_retirement_request + +Keep in mind, this will only work for users which have not had their retirement +states advance beyond ``PENDING``. Additionally, the user will need to reset +their password in order to restore access to their account. From 02f4669870011b2b01f14386f4589efab03e965e Mon Sep 17 00:00:00 2001 From: feanil <781561+feanil@users.noreply.github.com> Date: Tue, 7 May 2024 17:00:26 +0000 Subject: [PATCH 051/113] feat: Upgrade Python dependency build Routine requirement upgrade. Doing it individually because there are too many changes in `make upgrade`. Commit generated by workflow `openedx/edx-platform/.github/workflows/upgrade-one-python-dependency.yml@refs/heads/feanil/fix_one_upgrade` --- requirements/edx/development.txt | 2 +- requirements/pip-tools.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 44694be9c674..f7785dae045c 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -163,7 +163,7 @@ bridgekeeper==0.9 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -build==1.0.3 +build==1.2.1 # via # -r requirements/edx/../pip-tools.txt # pip-tools diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index 06c75582c9b6..389fadc32095 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -4,7 +4,7 @@ # # make upgrade # -build==1.0.3 +build==1.2.1 # via pip-tools click==8.1.6 # via From 7092400da9bc2c65b067ee1c9fd20f8cf5cd0cca Mon Sep 17 00:00:00 2001 From: feanil <781561+feanil@users.noreply.github.com> Date: Tue, 7 May 2024 17:01:23 +0000 Subject: [PATCH 052/113] feat: Upgrade Python dependency celery Routine requirement upgrade. Doing it individually because there are too many changes in `make upgrade`. Commit generated by workflow `openedx/edx-platform/.github/workflows/upgrade-one-python-dependency.yml@refs/heads/feanil/fix_one_upgrade` --- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/doc.txt | 2 +- requirements/edx/testing.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index df095d1526e9..ab41e3399119 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -93,7 +93,7 @@ bridgekeeper==0.9 # via -r requirements/edx/kernel.in camel-converter[pydantic]==3.1.1 # via meilisearch -celery==5.3.6 +celery==5.4.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/kernel.in diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index f7785dae045c..6d9f39c58bbf 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -176,7 +176,7 @@ camel-converter[pydantic]==3.1.1 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # meilisearch -celery==5.3.6 +celery==5.4.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index fec113e3574b..c465e801d547 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -124,7 +124,7 @@ camel-converter[pydantic]==3.1.1 # via # -r requirements/edx/base.txt # meilisearch -celery==5.3.6 +celery==5.4.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index b156b7ed0126..b1a8c8a71b53 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -126,7 +126,7 @@ camel-converter[pydantic]==3.1.1 # via # -r requirements/edx/base.txt # meilisearch -celery==5.3.6 +celery==5.4.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt From e94b942ec99313020062b988662ad5ad5e05e5ac Mon Sep 17 00:00:00 2001 From: feanil <781561+feanil@users.noreply.github.com> Date: Tue, 7 May 2024 17:00:27 +0000 Subject: [PATCH 053/113] feat: Upgrade Python dependency cachetools Routine requirement upgrade. Doing it individually because there are too many changes in `make upgrade`. Commit generated by workflow `openedx/edx-platform/.github/workflows/upgrade-one-python-dependency.yml@refs/heads/feanil/fix_one_upgrade` --- requirements/edx/development.txt | 2 +- requirements/edx/testing.txt | 2 +- scripts/user_retirement/requirements/base.txt | 2 +- scripts/user_retirement/requirements/testing.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 6d9f39c58bbf..31921c05f4e8 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -167,7 +167,7 @@ build==1.2.1 # via # -r requirements/edx/../pip-tools.txt # pip-tools -cachetools==5.3.2 +cachetools==5.3.3 # via # -r requirements/edx/testing.txt # tox diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index b1a8c8a71b53..e38f83662ab2 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -120,7 +120,7 @@ botocore==1.34.45 # s3transfer bridgekeeper==0.9 # via -r requirements/edx/base.txt -cachetools==5.3.2 +cachetools==5.3.3 # via tox camel-converter[pydantic]==3.1.1 # via diff --git a/scripts/user_retirement/requirements/base.txt b/scripts/user_retirement/requirements/base.txt index b272793aa4c4..d7cbcbdc7eef 100644 --- a/scripts/user_retirement/requirements/base.txt +++ b/scripts/user_retirement/requirements/base.txt @@ -21,7 +21,7 @@ botocore==1.34.26 # via # boto3 # s3transfer -cachetools==5.3.2 +cachetools==5.3.3 # via google-auth certifi==2023.11.17 # via requests diff --git a/scripts/user_retirement/requirements/testing.txt b/scripts/user_retirement/requirements/testing.txt index f30e373cbce5..f7a64ece8980 100644 --- a/scripts/user_retirement/requirements/testing.txt +++ b/scripts/user_retirement/requirements/testing.txt @@ -29,7 +29,7 @@ botocore==1.34.26 # boto3 # moto # s3transfer -cachetools==5.3.2 +cachetools==5.3.3 # via # -r scripts/user_retirement/requirements/base.txt # google-auth From d43a2f73eb0d1247fef8f03d1bdb009927033c21 Mon Sep 17 00:00:00 2001 From: Kaustav Banerjee Date: Wed, 8 May 2024 01:14:03 +0530 Subject: [PATCH 054/113] feat: default grade designations configurable from settings (#32406) --- .../contentstore/rest_api/v1/serializers/grading.py | 3 +++ .../contentstore/rest_api/v1/views/grading.py | 3 ++- .../rest_api/v1/views/tests/test_grading.py | 11 +++++++++++ cms/djangoapps/contentstore/utils.py | 3 ++- cms/envs/common.py | 9 +++++++++ cms/static/js/factories/settings_graders.js | 5 +++-- cms/static/js/views/settings/grading.js | 6 +++++- cms/static/sass/views/_settings.scss | 10 +++++----- cms/templates/settings_graders.html | 1 + 9 files changed, 41 insertions(+), 10 deletions(-) diff --git a/cms/djangoapps/contentstore/rest_api/v1/serializers/grading.py b/cms/djangoapps/contentstore/rest_api/v1/serializers/grading.py index 1ff8b794f83d..41d2b72811e0 100644 --- a/cms/djangoapps/contentstore/rest_api/v1/serializers/grading.py +++ b/cms/djangoapps/contentstore/rest_api/v1/serializers/grading.py @@ -40,3 +40,6 @@ class CourseGradingSerializer(serializers.Serializer): course_details = CourseGradingModelSerializer() show_credit_eligibility = serializers.BooleanField() is_credit_course = serializers.BooleanField() + default_grade_designations = serializers.ListSerializer( + child=serializers.CharField() + ) diff --git a/cms/djangoapps/contentstore/rest_api/v1/views/grading.py b/cms/djangoapps/contentstore/rest_api/v1/views/grading.py index ca405446b8f1..9275fecb58ab 100644 --- a/cms/djangoapps/contentstore/rest_api/v1/views/grading.py +++ b/cms/djangoapps/contentstore/rest_api/v1/views/grading.py @@ -85,7 +85,8 @@ def get(self, request: Request, course_id: str): "minimum_grade_credit": 0.7 }, "show_credit_eligibility": false, - "is_credit_course": true + "is_credit_course": true, + "default_grade_designations": ["A","B","C","D"] } ``` """ diff --git a/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_grading.py b/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_grading.py index f12bf13afe54..6123096048df 100644 --- a/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_grading.py +++ b/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_grading.py @@ -42,11 +42,22 @@ def test_course_grading_response(self): "course_details": grading_data.__dict__, "show_credit_eligibility": False, "is_credit_course": False, + "default_grade_designations": ['A', 'B', 'C', 'D'], } self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertDictEqual(expected_response, response.data) + @patch("django.conf.settings.DEFAULT_GRADE_DESIGNATIONS", ['A', 'B']) + def test_default_grade_designations_setting(self): + """ + Check that DEFAULT_GRADE_DESIGNATIONS setting reflects correctly in API. + """ + response = self.client.get(self.url) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(['A', 'B'], response.data["default_grade_designations"]) + @patch.dict("django.conf.settings.FEATURES", {"ENABLE_CREDIT_ELIGIBILITY": True}) def test_credit_eligibility_setting(self): """ diff --git a/cms/djangoapps/contentstore/utils.py b/cms/djangoapps/contentstore/utils.py index 8fec374ae6e6..3fe718dfa149 100644 --- a/cms/djangoapps/contentstore/utils.py +++ b/cms/djangoapps/contentstore/utils.py @@ -1486,7 +1486,8 @@ def get_course_grading(course_key): 'grading_url': reverse_course_url('grading_handler', course_key), 'is_credit_course': is_credit_course(course_key), 'mfe_proctored_exam_settings_url': get_proctored_exam_settings_url(course_key), - 'course_assignment_lists': dict(course_assignment_lists) + 'course_assignment_lists': dict(course_assignment_lists), + 'default_grade_designations': settings.DEFAULT_GRADE_DESIGNATIONS } return grading_context diff --git a/cms/envs/common.py b/cms/envs/common.py index 6aa2dfb835b1..ddd79f6d169c 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -2439,6 +2439,15 @@ # Rate limit for regrading tasks that a grading policy change can kick off POLICY_CHANGE_TASK_RATE_LIMIT = '900/h' +# .. setting_name: DEFAULT_GRADE_DESIGNATIONS +# .. setting_default: ['A', 'B', 'C', 'D'] +# .. setting_description: The default 'pass' grade cutoff designations to be used. The failure grade +# is always 'F' and should not be included in this list. +# .. setting_warning: The DEFAULT_GRADE_DESIGNATIONS list must have more than one designation, +# or else ['A', 'B', 'C', 'D'] will be used as the default grade designations. Also, only the first +# 11 grade designations are used by the UI, so it's advisable to restrict the list to 11 items. +DEFAULT_GRADE_DESIGNATIONS = ['A', 'B', 'C', 'D'] + ############## Settings for CourseGraph ############################ # .. setting_name: COURSEGRAPH_JOB_QUEUE diff --git a/cms/static/js/factories/settings_graders.js b/cms/static/js/factories/settings_graders.js index dc75029e0f26..61b8fcbcad03 100644 --- a/cms/static/js/factories/settings_graders.js +++ b/cms/static/js/factories/settings_graders.js @@ -3,7 +3,7 @@ define([ ], function($, GradingView, CourseGradingPolicyModel) { 'use strict'; - return function(courseDetails, gradingUrl, courseAssignmentLists) { + return function(courseDetails, gradingUrl, courseAssignmentLists, gradeDesignations) { var model, editor; $('form :input') @@ -19,7 +19,8 @@ define([ editor = new GradingView({ el: $('.settings-grading'), model: model, - courseAssignmentLists: courseAssignmentLists + courseAssignmentLists: courseAssignmentLists, + gradeDesignations: gradeDesignations }); editor.render(); }; diff --git a/cms/static/js/views/settings/grading.js b/cms/static/js/views/settings/grading.js index dcb2ce9b3e96..71fdfe3f9a8c 100644 --- a/cms/static/js/views/settings/grading.js +++ b/cms/static/js/views/settings/grading.js @@ -34,6 +34,7 @@ function(ValidatingView, _, $, ui, GraderView, StringUtils, HtmlUtils) { $('#course_grade_cutoff-tpl').text() ); this.setupCutoffs(); + this.setupGradeDesignations(options.gradeDesignations); this.listenTo(this.model, 'invalid', this.handleValidationError); this.listenTo(this.model, 'change', this.showNotificationBar); @@ -318,7 +319,7 @@ function(ValidatingView, _, $, ui, GraderView, StringUtils, HtmlUtils) { addNewGrade: function(e) { e.preventDefault(); var gradeLength = this.descendingCutoffs.length; // cutoffs doesn't include fail/f so this is only the passing grades - if (gradeLength > 3) { + if (gradeLength > this.GRADES.length - 1) { // TODO shouldn't we disable the button return; } @@ -399,6 +400,9 @@ function(ValidatingView, _, $, ui, GraderView, StringUtils, HtmlUtils) { this.descendingCutoffs = _.sortBy(this.descendingCutoffs, function(gradeEle) { return -gradeEle.cutoff; }); }, + setupGradeDesignations: function(gradeDesignations) { + if (Array.isArray(gradeDesignations) && gradeDesignations.length > 1) { this.GRADES = gradeDesignations.slice(0, 11); } + }, revertView: function() { var self = this; this.model.fetch({ diff --git a/cms/static/sass/views/_settings.scss b/cms/static/sass/views/_settings.scss index cc62fd436f0d..c571af4b8cb8 100644 --- a/cms/static/sass/views/_settings.scss +++ b/cms/static/sass/views/_settings.scss @@ -777,23 +777,23 @@ height: 17px; } - &:nth-child(1) { + &:nth-child(5n+1) { background: #4fe696; } - &:nth-child(2) { + &:nth-child(5n+2) { background: #ffdf7e; } - &:nth-child(3) { + &:nth-child(5n+3) { background: #ffb657; } - &:nth-child(4) { + &:nth-child(5n+4) { background: #ef54a1; } - &:nth-child(5), + &:nth-child(5n+5), &.bar-fail { background: #fb336c; } diff --git a/cms/templates/settings_graders.html b/cms/templates/settings_graders.html index c3b6f8f73a2a..eb0a057b046e 100644 --- a/cms/templates/settings_graders.html +++ b/cms/templates/settings_graders.html @@ -37,6 +37,7 @@ ), "${grading_url | n, js_escaped_string}", ${course_assignment_lists | n, dump_js_escaped_json}, + ${default_grade_designations | n, dump_js_escaped_json}, ); }); From f209ce3d91c1d24f58863b09de876f2903381a36 Mon Sep 17 00:00:00 2001 From: feanil <781561+feanil@users.noreply.github.com> Date: Tue, 7 May 2024 17:59:34 +0000 Subject: [PATCH 055/113] feat: Upgrade Python dependency jwcrypto Routine requirement upgrade. Doing it individually because there are too many changes in `make upgrade`. Commit generated by workflow `openedx/edx-platform/.github/workflows/upgrade-one-python-dependency.yml@refs/heads/master` --- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/doc.txt | 2 +- requirements/edx/testing.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index ab41e3399119..a311f1b1e191 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -658,7 +658,7 @@ jsonschema==4.21.1 # optimizely-sdk jsonschema-specifications==2023.12.1 # via jsonschema -jwcrypto==1.5.4 +jwcrypto==1.5.6 # via # django-oauth-toolkit # pylti1p3 diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 31921c05f4e8..ca3a58073b89 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -1092,7 +1092,7 @@ jsonschema-specifications==2023.12.1 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # jsonschema -jwcrypto==1.5.4 +jwcrypto==1.5.6 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index c465e801d547..26aa0ca3d4e3 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -776,7 +776,7 @@ jsonschema-specifications==2023.12.1 # via # -r requirements/edx/base.txt # jsonschema -jwcrypto==1.5.4 +jwcrypto==1.5.6 # via # -r requirements/edx/base.txt # django-oauth-toolkit diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index e38f83662ab2..0778594d9211 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -827,7 +827,7 @@ jsonschema-specifications==2023.12.1 # via # -r requirements/edx/base.txt # jsonschema -jwcrypto==1.5.4 +jwcrypto==1.5.6 # via # -r requirements/edx/base.txt # django-oauth-toolkit From 5efda044162f9e479600c8062cc1f8a804afc86b Mon Sep 17 00:00:00 2001 From: feanil <781561+feanil@users.noreply.github.com> Date: Tue, 7 May 2024 18:00:12 +0000 Subject: [PATCH 056/113] feat: Upgrade Python dependency tqdm Routine requirement upgrade. Doing it individually because there are too many changes in `make upgrade`. Commit generated by workflow `openedx/edx-platform/.github/workflows/upgrade-one-python-dependency.yml@refs/heads/master` --- requirements/edx-sandbox/base.txt | 2 +- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/doc.txt | 2 +- requirements/edx/testing.txt | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements/edx-sandbox/base.txt b/requirements/edx-sandbox/base.txt index fbc05dac94a8..ea32acd9098b 100644 --- a/requirements/edx-sandbox/base.txt +++ b/requirements/edx-sandbox/base.txt @@ -89,7 +89,7 @@ sympy==1.12 # via # -r requirements/edx-sandbox/base.in # openedx-calc -tqdm==4.66.2 +tqdm==4.66.4 # via nltk zipp==3.17.0 # via importlib-resources diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index ab41e3399119..266f23b3c212 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -1145,7 +1145,7 @@ tinycss2==1.2.1 # via bleach tomlkit==0.12.3 # via snowflake-connector-python -tqdm==4.66.2 +tqdm==4.66.4 # via # nltk # openai diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 31921c05f4e8..61831c988389 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -2032,7 +2032,7 @@ tomlkit==0.12.3 # snowflake-connector-python tox==4.11.4 # via -r requirements/edx/testing.txt -tqdm==4.66.2 +tqdm==4.66.4 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index c465e801d547..80ffb9267d5c 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -1398,7 +1398,7 @@ tomlkit==0.12.3 # via # -r requirements/edx/base.txt # snowflake-connector-python -tqdm==4.66.2 +tqdm==4.66.4 # via # -r requirements/edx/base.txt # nltk diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index e38f83662ab2..84d3277af165 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -1498,7 +1498,7 @@ tomlkit==0.12.3 # snowflake-connector-python tox==4.11.4 # via -r requirements/edx/testing.in -tqdm==4.66.2 +tqdm==4.66.4 # via # -r requirements/edx/base.txt # nltk From a92c215d70928846e9e52b72ee17f1f1ff1a6a7c Mon Sep 17 00:00:00 2001 From: feanil <781561+feanil@users.noreply.github.com> Date: Tue, 7 May 2024 17:57:37 +0000 Subject: [PATCH 057/113] feat: Upgrade Python dependency Jinja2 Routine requirement upgrade. Doing it individually because there are too many changes in `make upgrade`. Commit generated by workflow `openedx/edx-platform/.github/workflows/upgrade-one-python-dependency.yml@refs/heads/master` --- requirements/edx/base.txt | 2 +- requirements/edx/coverage.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/doc.txt | 2 +- requirements/edx/testing.txt | 2 +- scripts/user_retirement/requirements/testing.txt | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index ab41e3399119..111ced9db1e6 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -631,7 +631,7 @@ isodate==0.6.1 # via python3-saml itypes==1.2.0 # via coreapi -jinja2==3.1.3 +jinja2==3.1.4 # via # code-annotations # coreschema diff --git a/requirements/edx/coverage.txt b/requirements/edx/coverage.txt index bd2cbf10d012..ec0e1bfea514 100644 --- a/requirements/edx/coverage.txt +++ b/requirements/edx/coverage.txt @@ -10,7 +10,7 @@ coverage==7.4.1 # via -r requirements/edx/coverage.in diff-cover==9.0.0 # via -r requirements/edx/coverage.in -jinja2==3.1.3 +jinja2==3.1.4 # via diff-cover markupsafe==2.1.5 # via jinja2 diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 31921c05f4e8..f59615f9c922 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -1046,7 +1046,7 @@ itypes==1.2.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # coreapi -jinja2==3.1.3 +jinja2==3.1.4 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index c465e801d547..571157dc4459 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -738,7 +738,7 @@ itypes==1.2.0 # via # -r requirements/edx/base.txt # coreapi -jinja2==3.1.3 +jinja2==3.1.4 # via # -r requirements/edx/base.txt # code-annotations diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index e38f83662ab2..88204b379159 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -789,7 +789,7 @@ itypes==1.2.0 # via # -r requirements/edx/base.txt # coreapi -jinja2==3.1.3 +jinja2==3.1.4 # via # -r requirements/edx/base.txt # -r requirements/edx/coverage.txt diff --git a/scripts/user_retirement/requirements/testing.txt b/scripts/user_retirement/requirements/testing.txt index f7a64ece8980..1a87444f82a9 100644 --- a/scripts/user_retirement/requirements/testing.txt +++ b/scripts/user_retirement/requirements/testing.txt @@ -120,7 +120,7 @@ isodate==0.6.1 # zeep jenkinsapi==0.3.13 # via -r scripts/user_retirement/requirements/base.txt -jinja2==3.1.3 +jinja2==3.1.4 # via moto jmespath==1.0.1 # via From c72908f71941bc04f49140ef55c1a53181c7a123 Mon Sep 17 00:00:00 2001 From: Muhammad Adeel Tajamul <77053848+muhammadadeeltajamul@users.noreply.github.com> Date: Wed, 8 May 2024 11:58:01 +0500 Subject: [PATCH 058/113] fix: updated email digest management command (#34729) --- .../notifications/management/commands/send_email_digest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openedx/core/djangoapps/notifications/management/commands/send_email_digest.py b/openedx/core/djangoapps/notifications/management/commands/send_email_digest.py index cfaef2d15f67..79857c18cf8a 100644 --- a/openedx/core/djangoapps/notifications/management/commands/send_email_digest.py +++ b/openedx/core/djangoapps/notifications/management/commands/send_email_digest.py @@ -23,10 +23,10 @@ def add_arguments(self, parser): Adds management commands parser arguments """ cadence_type_choices = [EmailCadence.DAILY, EmailCadence.WEEKLY] - parser.add_argument('cadence_type', choices=cadence_type_choices, required=True) + parser.add_argument('cadence_type', choices=cadence_type_choices) def handle(self, *args, **kwargs): """ Start task to send email digest to users """ - send_digest_email_to_all_users.delay(args=(kwargs['cadence_type'],)) + send_digest_email_to_all_users.delay(kwargs['cadence_type']) From 6f21d1383ca961f8ef4813a7e21341bfd0b6cc95 Mon Sep 17 00:00:00 2001 From: MueezKhan246 <93375917+MueezKhan246@users.noreply.github.com> Date: Wed, 8 May 2024 09:03:48 +0000 Subject: [PATCH 059/113] feat: Upgrade Python dependency edx-enterprise replaced non encrypted fields of blackboard config model with encrypted ones Commit generated by workflow `openedx/edx-platform/.github/workflows/upgrade-one-python-dependency.yml@refs/heads/master` --- requirements/constraints.txt | 2 +- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/doc.txt | 2 +- requirements/edx/testing.txt | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 42cef16089d3..c52903b8a5da 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -23,7 +23,7 @@ click>=8.0,<9.0 # The team that owns this package will manually bump this package rather than having it pulled in automatically. # This is to allow them to better control its deployment and to do it in a process that works better # for them. -edx-enterprise==4.17.1 +edx-enterprise==4.17.3 # Stay on LTS version, remove once this is added to common constraint Django<5.0 diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 111ced9db1e6..2b2650cf4e7b 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -480,7 +480,7 @@ edx-drf-extensions==10.3.0 # edx-when # edxval # openedx-learning -edx-enterprise==4.17.1 +edx-enterprise==4.17.3 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/kernel.in diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index f59615f9c922..6f04bf69884c 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -754,7 +754,7 @@ edx-drf-extensions==10.3.0 # edx-when # edxval # openedx-learning -edx-enterprise==4.17.1 +edx-enterprise==4.17.3 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 571157dc4459..43e2cf3ee8a6 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -558,7 +558,7 @@ edx-drf-extensions==10.3.0 # edx-when # edxval # openedx-learning -edx-enterprise==4.17.1 +edx-enterprise==4.17.3 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 88204b379159..3adfd77019ea 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -582,7 +582,7 @@ edx-drf-extensions==10.3.0 # edx-when # edxval # openedx-learning -edx-enterprise==4.17.1 +edx-enterprise==4.17.3 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt From 2c78e86316824bb52d5cfeba814d14f74dc3528a Mon Sep 17 00:00:00 2001 From: muhammad-ammar Date: Wed, 8 May 2024 16:05:39 +0500 Subject: [PATCH 060/113] chore: upgrade edx-enterprise to 4.17.4 --- requirements/constraints.txt | 2 +- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/doc.txt | 2 +- requirements/edx/testing.txt | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index c52903b8a5da..394598da6b6d 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -23,7 +23,7 @@ click>=8.0,<9.0 # The team that owns this package will manually bump this package rather than having it pulled in automatically. # This is to allow them to better control its deployment and to do it in a process that works better # for them. -edx-enterprise==4.17.3 +edx-enterprise==4.17.4 # Stay on LTS version, remove once this is added to common constraint Django<5.0 diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 2b2650cf4e7b..99f0c626c53f 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -480,7 +480,7 @@ edx-drf-extensions==10.3.0 # edx-when # edxval # openedx-learning -edx-enterprise==4.17.3 +edx-enterprise==4.17.4 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/kernel.in diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 6f04bf69884c..80e0eb28f888 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -754,7 +754,7 @@ edx-drf-extensions==10.3.0 # edx-when # edxval # openedx-learning -edx-enterprise==4.17.3 +edx-enterprise==4.17.4 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 43e2cf3ee8a6..54050edb5ffd 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -558,7 +558,7 @@ edx-drf-extensions==10.3.0 # edx-when # edxval # openedx-learning -edx-enterprise==4.17.3 +edx-enterprise==4.17.4 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 3adfd77019ea..35e2994737e9 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -582,7 +582,7 @@ edx-drf-extensions==10.3.0 # edx-when # edxval # openedx-learning -edx-enterprise==4.17.3 +edx-enterprise==4.17.4 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt From 6f45093af2950dc32874c251e8a7af60cd43a351 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Tue, 7 May 2024 14:14:02 -0400 Subject: [PATCH 061/113] chore: Upgrade cryptography and pyopenssl Ran 'make upgrade-package package="cryptography --upgrade-package pyopenssl"` --- requirements/constraints.txt | 8 -------- requirements/edx-sandbox/base.txt | 6 ++---- requirements/edx/base.txt | 6 ++---- requirements/edx/development.txt | 6 ++---- requirements/edx/doc.txt | 6 ++---- requirements/edx/testing.txt | 6 ++---- scripts/user_retirement/requirements/base.txt | 6 ++---- scripts/user_retirement/requirements/testing.txt | 2 +- 8 files changed, 13 insertions(+), 33 deletions(-) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 394598da6b6d..dc52f4317ba2 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -60,14 +60,6 @@ markdown<3.4.0 # Constraint can be removed once the issue https://github.com/PyCQA/pycodestyle/issues/1090 is fixed. pycodestyle<2.9.0 -# pyopenssl>22.0.0 requires cryptography>=38.0 && conflicts with snowflak-connector-python requires cryptography<37 -# which causes the requirements upgrade job to fail due to constraint conflict -# This constraint can be removed once https://github.com/snowflakedb/snowflake-connector-python/issues/1259 is resolved -# and snowflake-connector-python>2.8.0 is released. -pyopenssl==22.0.0 - -cryptography==38.0.4 # greater version has some issues with openssl. - pylint<2.16.0 # greater version failing quality test. Fix them in seperate ticket. # adding these constraints to minimize boto3 and botocore changeset diff --git a/requirements/edx-sandbox/base.txt b/requirements/edx-sandbox/base.txt index fbc05dac94a8..6cab4255e77d 100644 --- a/requirements/edx-sandbox/base.txt +++ b/requirements/edx-sandbox/base.txt @@ -16,10 +16,8 @@ codejail-includes==1.0.0 # via -r requirements/edx-sandbox/base.in contourpy==1.1.1 # via matplotlib -cryptography==38.0.4 - # via - # -c requirements/edx-sandbox/../constraints.txt - # -r requirements/edx-sandbox/base.in +cryptography==42.0.7 + # via -r requirements/edx-sandbox/base.in cycler==0.12.1 # via matplotlib fonttools==4.49.0 diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 99f0c626c53f..f5dd55871573 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -156,9 +156,8 @@ coreschema==0.0.4 # drf-yasg crowdsourcehinter-xblock==0.7 # via -r requirements/edx/bundled.in -cryptography==38.0.4 +cryptography==41.0.7 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/kernel.in # django-fernet-fields-v2 # edx-enterprise @@ -912,9 +911,8 @@ pynacl==1.5.0 # via edx-django-utils pynliner==0.8.0 # via -r requirements/edx/kernel.in -pyopenssl==22.0.0 +pyopenssl==23.3.0 # via - # -c requirements/edx/../constraints.txt # optimizely-sdk # snowflake-connector-python pyparsing==3.1.1 diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 80e0eb28f888..096ec5809359 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -295,9 +295,8 @@ crowdsourcehinter-xblock==0.7 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -cryptography==38.0.4 +cryptography==41.0.7 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # django-fernet-fields-v2 @@ -1564,9 +1563,8 @@ pynliner==0.8.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -pyopenssl==22.0.0 +pyopenssl==23.3.0 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # optimizely-sdk diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 54050edb5ffd..cf6fdf5576a7 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -202,9 +202,8 @@ coreschema==0.0.4 # drf-yasg crowdsourcehinter-xblock==0.7 # via -r requirements/edx/base.txt -cryptography==38.0.4 +cryptography==41.0.7 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # django-fernet-fields-v2 # edx-enterprise @@ -1082,9 +1081,8 @@ pynacl==1.5.0 # edx-django-utils pynliner==0.8.0 # via -r requirements/edx/base.txt -pyopenssl==22.0.0 +pyopenssl==23.3.0 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # optimizely-sdk # snowflake-connector-python diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 35e2994737e9..e30e8919c4a3 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -221,9 +221,8 @@ coverage[toml]==7.4.1 # pytest-cov crowdsourcehinter-xblock==0.7 # via -r requirements/edx/base.txt -cryptography==38.0.4 +cryptography==41.0.7 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # django-fernet-fields-v2 # edx-enterprise @@ -1171,9 +1170,8 @@ pynacl==1.5.0 # edx-django-utils pynliner==0.8.0 # via -r requirements/edx/base.txt -pyopenssl==22.0.0 +pyopenssl==23.3.0 # via - # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # optimizely-sdk # snowflake-connector-python diff --git a/scripts/user_retirement/requirements/base.txt b/scripts/user_retirement/requirements/base.txt index d7cbcbdc7eef..409c99113b01 100644 --- a/scripts/user_retirement/requirements/base.txt +++ b/scripts/user_retirement/requirements/base.txt @@ -38,10 +38,8 @@ click==8.1.6 # -c scripts/user_retirement/requirements/../../../requirements/constraints.txt # -r scripts/user_retirement/requirements/base.in # edx-django-utils -cryptography==38.0.4 - # via - # -c scripts/user_retirement/requirements/../../../requirements/constraints.txt - # simple-salesforce +cryptography==42.0.7 + # via simple-salesforce django==4.2.9 # via # -c scripts/user_retirement/requirements/../../../requirements/common_constraints.txt diff --git a/scripts/user_retirement/requirements/testing.txt b/scripts/user_retirement/requirements/testing.txt index 1a87444f82a9..e6a69871fabc 100644 --- a/scripts/user_retirement/requirements/testing.txt +++ b/scripts/user_retirement/requirements/testing.txt @@ -50,7 +50,7 @@ click==8.1.6 # via # -r scripts/user_retirement/requirements/base.txt # edx-django-utils -cryptography==38.0.4 +cryptography==42.0.7 # via # -r scripts/user_retirement/requirements/base.txt # moto From fd396685928b5338df96e474bb51f16865499ec8 Mon Sep 17 00:00:00 2001 From: feanil <781561+feanil@users.noreply.github.com> Date: Tue, 7 May 2024 17:59:04 +0000 Subject: [PATCH 062/113] feat: Upgrade Python dependency pillow Routine requirement upgrade. Doing it individually because there are too many changes in `make upgrade`. Commit generated by workflow `openedx/edx-platform/.github/workflows/upgrade-one-python-dependency.yml@refs/heads/master` --- requirements/edx-sandbox/base.txt | 2 +- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/doc.txt | 2 +- requirements/edx/testing.txt | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements/edx-sandbox/base.txt b/requirements/edx-sandbox/base.txt index 6cab4255e77d..74fb5b2d87c5 100644 --- a/requirements/edx-sandbox/base.txt +++ b/requirements/edx-sandbox/base.txt @@ -58,7 +58,7 @@ openedx-calc==3.1.0 # via -r requirements/edx-sandbox/base.in packaging==24.0 # via matplotlib -pillow==10.2.0 +pillow==10.3.0 # via matplotlib pycparser==2.21 # via cffi diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index f5dd55871573..47b7d874f2de 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -834,7 +834,7 @@ pgpy==0.6.0 # via edx-enterprise piexif==1.1.3 # via -r requirements/edx/kernel.in -pillow==10.2.0 +pillow==10.3.0 # via # -r requirements/edx/kernel.in # edx-enterprise diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 096ec5809359..e36e1bd448f8 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -1388,7 +1388,7 @@ piexif==1.1.3 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -pillow==10.2.0 +pillow==10.3.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index cf6fdf5576a7..6e57411966f1 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -979,7 +979,7 @@ picobox==4.0.0 # via sphinxcontrib-openapi piexif==1.1.3 # via -r requirements/edx/base.txt -pillow==10.2.0 +pillow==10.3.0 # via # -r requirements/edx/base.txt # edx-enterprise diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index e30e8919c4a3..c7f7876b77cc 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -1035,7 +1035,7 @@ pgpy==0.6.0 # edx-enterprise piexif==1.1.3 # via -r requirements/edx/base.txt -pillow==10.2.0 +pillow==10.3.0 # via # -r requirements/edx/base.txt # edx-enterprise From 0a61a6b7408dd400fe1e275711cd1779ad9da86e Mon Sep 17 00:00:00 2001 From: feanil <781561+feanil@users.noreply.github.com> Date: Tue, 7 May 2024 19:42:22 +0000 Subject: [PATCH 063/113] feat: Upgrade Python dependency django-stubs-ext Routine requirement upgrade. Doing it individually because there are too many changes in `make upgrade`. Commit generated by workflow `openedx/edx-platform/.github/workflows/upgrade-one-python-dependency.yml@refs/heads/master` --- requirements/edx/development.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index e36e1bd448f8..669359a56004 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -583,7 +583,7 @@ django-stubs==1.16.0 # -c requirements/edx/../constraints.txt # -r requirements/edx/development.in # djangorestframework-stubs -django-stubs-ext==4.2.7 +django-stubs-ext==5.0.0 # via django-stubs django-user-tasks==3.2.0 # via From 9c3fb1ee12f8c48a73351e5eee72b8d3fe96c6d8 Mon Sep 17 00:00:00 2001 From: feanil <781561+feanil@users.noreply.github.com> Date: Tue, 7 May 2024 19:42:10 +0000 Subject: [PATCH 064/113] feat: Upgrade Python dependency certifi Routine requirement upgrade. Doing it individually because there are too many changes in `make upgrade`. Commit generated by workflow `openedx/edx-platform/.github/workflows/upgrade-one-python-dependency.yml@refs/heads/master` --- scripts/user_retirement/requirements/base.txt | 2 +- scripts/user_retirement/requirements/testing.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/user_retirement/requirements/base.txt b/scripts/user_retirement/requirements/base.txt index 409c99113b01..37217268fd06 100644 --- a/scripts/user_retirement/requirements/base.txt +++ b/scripts/user_retirement/requirements/base.txt @@ -23,7 +23,7 @@ botocore==1.34.26 # s3transfer cachetools==5.3.3 # via google-auth -certifi==2023.11.17 +certifi==2024.2.2 # via requests cffi==1.16.0 # via diff --git a/scripts/user_retirement/requirements/testing.txt b/scripts/user_retirement/requirements/testing.txt index e6a69871fabc..730bfd730152 100644 --- a/scripts/user_retirement/requirements/testing.txt +++ b/scripts/user_retirement/requirements/testing.txt @@ -33,7 +33,7 @@ cachetools==5.3.3 # via # -r scripts/user_retirement/requirements/base.txt # google-auth -certifi==2023.11.17 +certifi==2024.2.2 # via # -r scripts/user_retirement/requirements/base.txt # requests From 4d87410890522954febed1f106f82e4f56a759d9 Mon Sep 17 00:00:00 2001 From: magajh Date: Wed, 8 May 2024 09:40:31 -0400 Subject: [PATCH 065/113] chore: upgrade Django to 4.2.13 --- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/doc.txt | 2 +- requirements/edx/testing.txt | 2 +- scripts/user_retirement/requirements/base.txt | 2 +- scripts/user_retirement/requirements/testing.txt | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 47b7d874f2de..bed1c667a79b 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -177,7 +177,7 @@ defusedxml==0.7.1 # ora2 # python3-openid # social-auth-core -django==4.2.10 +django==4.2.13 # via # -c requirements/edx/../common_constraints.txt # -c requirements/edx/../constraints.txt diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index e36e1bd448f8..8845df3cb445 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -341,7 +341,7 @@ distlib==0.3.8 # via # -r requirements/edx/testing.txt # virtualenv -django==4.2.10 +django==4.2.13 # via # -c requirements/edx/../common_constraints.txt # -c requirements/edx/../constraints.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 6e57411966f1..14bfc2ce3ac9 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -227,7 +227,7 @@ defusedxml==0.7.1 # ora2 # python3-openid # social-auth-core -django==4.2.10 +django==4.2.13 # via # -c requirements/edx/../common_constraints.txt # -c requirements/edx/../constraints.txt diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index c7f7876b77cc..0a858e1877d1 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -256,7 +256,7 @@ dill==0.3.8 # via pylint distlib==0.3.8 # via virtualenv -django==4.2.10 +django==4.2.13 # via # -c requirements/edx/../common_constraints.txt # -c requirements/edx/../constraints.txt diff --git a/scripts/user_retirement/requirements/base.txt b/scripts/user_retirement/requirements/base.txt index 37217268fd06..e02cd6109345 100644 --- a/scripts/user_retirement/requirements/base.txt +++ b/scripts/user_retirement/requirements/base.txt @@ -40,7 +40,7 @@ click==8.1.6 # edx-django-utils cryptography==42.0.7 # via simple-salesforce -django==4.2.9 +django==4.2.13 # via # -c scripts/user_retirement/requirements/../../../requirements/common_constraints.txt # -c scripts/user_retirement/requirements/../../../requirements/constraints.txt diff --git a/scripts/user_retirement/requirements/testing.txt b/scripts/user_retirement/requirements/testing.txt index 730bfd730152..fb690d185371 100644 --- a/scripts/user_retirement/requirements/testing.txt +++ b/scripts/user_retirement/requirements/testing.txt @@ -57,7 +57,7 @@ cryptography==42.0.7 # simple-salesforce ddt==1.7.1 # via -r scripts/user_retirement/requirements/testing.in -django==4.2.9 +django==4.2.13 # via # -r scripts/user_retirement/requirements/base.txt # django-crum From 1d3fb01d3e89e8f6a0d74a24e6f84bbecebb786d Mon Sep 17 00:00:00 2001 From: feanil <781561+feanil@users.noreply.github.com> Date: Tue, 7 May 2024 19:42:44 +0000 Subject: [PATCH 066/113] feat: Upgrade Python dependency pytz Routine requirement upgrade. Doing it individually because there are too many changes in `make upgrade`. Commit generated by workflow `openedx/edx-platform/.github/workflows/upgrade-one-python-dependency.yml@refs/heads/master` --- scripts/user_retirement/requirements/base.txt | 2 +- scripts/user_retirement/requirements/testing.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/user_retirement/requirements/base.txt b/scripts/user_retirement/requirements/base.txt index e02cd6109345..0c3520d2c2e1 100644 --- a/scripts/user_retirement/requirements/base.txt +++ b/scripts/user_retirement/requirements/base.txt @@ -125,7 +125,7 @@ python-dateutil==2.8.2 # botocore # pendulum # time-machine -pytz==2023.3.post1 +pytz==2024.1 # via # jenkinsapi # zeep diff --git a/scripts/user_retirement/requirements/testing.txt b/scripts/user_retirement/requirements/testing.txt index fb690d185371..a58cca2f4920 100644 --- a/scripts/user_retirement/requirements/testing.txt +++ b/scripts/user_retirement/requirements/testing.txt @@ -207,7 +207,7 @@ python-dateutil==2.8.2 # moto # pendulum # time-machine -pytz==2023.3.post1 +pytz==2024.1 # via # -r scripts/user_retirement/requirements/base.txt # jenkinsapi From cdf2b3873dd870d516720fd4bc71b523b86396d3 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Wed, 8 May 2024 15:48:36 -0400 Subject: [PATCH 067/113] build: Constrain edx-i18n-tools --- requirements/constraints.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index dc52f4317ba2..c667229c04bf 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -112,6 +112,10 @@ optimizely-sdk<5.0 # lxml==5.2.1 introduced new extra so we'll nee to rename lxml --> lxml[html-clean] # This constraint can be removed once we upgrade to Python 3.11 lxml<5.0 +# This has to be constrained as well because newer versions of edx-i18n-tools need the +# newer version of lxml but that requirement was not made expilict in the 1.6.0 version +# of the package. This can be un-pinned when we're upgrading lxml. +edx-i18n-tools<1.6.0 # xmlsec==1.3.14 breaking tests for all builds, can be removed once a fix is available xmlsec<1.3.14 @@ -125,3 +129,4 @@ path<16.12.0 # Temporary to Support the python 3.11 Upgrade backports.zoneinfo;python_version<"3.9" # Newer versions have zoneinfo available in the standard library + From 7734f023c8a78dd22a6d54542d1132f86b105aeb Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Wed, 8 May 2024 13:22:43 -0400 Subject: [PATCH 068/113] chore: Run `make upgrade` --- requirements/constraints.txt | 1 - requirements/edx-sandbox/base.txt | 16 +- requirements/edx/base.txt | 108 ++++---- requirements/edx/coverage.txt | 6 +- requirements/edx/development.txt | 251 ++++++++++++------ requirements/edx/doc.txt | 114 ++++---- requirements/edx/paver.txt | 6 +- requirements/edx/semgrep.txt | 22 +- requirements/edx/testing.txt | 209 ++++++++++----- requirements/pip-tools.txt | 9 +- requirements/pip.txt | 4 +- .../structures_pruning/requirements/base.txt | 4 +- .../requirements/testing.txt | 10 +- scripts/user_retirement/requirements/base.txt | 74 +++--- .../user_retirement/requirements/testing.txt | 101 +++---- scripts/xblock/requirements.txt | 2 +- 16 files changed, 533 insertions(+), 404 deletions(-) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index c667229c04bf..51729b725233 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -129,4 +129,3 @@ path<16.12.0 # Temporary to Support the python 3.11 Upgrade backports.zoneinfo;python_version<"3.9" # Newer versions have zoneinfo available in the standard library - diff --git a/requirements/edx-sandbox/base.txt b/requirements/edx-sandbox/base.txt index e098a33e0d2e..d12c994fd206 100644 --- a/requirements/edx-sandbox/base.txt +++ b/requirements/edx-sandbox/base.txt @@ -20,11 +20,11 @@ cryptography==42.0.7 # via -r requirements/edx-sandbox/base.in cycler==0.12.1 # via matplotlib -fonttools==4.49.0 +fonttools==4.51.0 # via matplotlib -importlib-resources==6.1.1 +importlib-resources==6.4.0 # via matplotlib -joblib==1.3.2 +joblib==1.4.2 # via nltk kiwisolver==1.4.5 # via matplotlib @@ -60,19 +60,19 @@ packaging==24.0 # via matplotlib pillow==10.3.0 # via matplotlib -pycparser==2.21 +pycparser==2.22 # via cffi -pyparsing==3.1.1 +pyparsing==3.1.2 # via # -r requirements/edx-sandbox/base.in # chem # matplotlib # openedx-calc -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via matplotlib random2==1.0.2 # via -r requirements/edx-sandbox/base.in -regex==2024.4.16 +regex==2024.4.28 # via nltk scipy==1.10.1 # via @@ -89,5 +89,5 @@ sympy==1.12 # openedx-calc tqdm==4.66.4 # via nltk -zipp==3.17.0 +zipp==3.18.1 # via importlib-resources diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 11b5bab8ab57..6afc341275e9 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -26,7 +26,7 @@ annotated-types==0.6.0 # via pydantic appdirs==1.4.4 # via fs -asgiref==3.7.2 +asgiref==3.8.1 # via # django # django-cors-headers @@ -78,20 +78,20 @@ bleach[css]==6.1.0 # xblock-poll boto==2.49.0 # via -r requirements/edx/kernel.in -boto3==1.34.45 +boto3==1.34.101 # via # -r requirements/edx/kernel.in # django-ses # fs-s3fs # ora2 -botocore==1.34.45 +botocore==1.34.101 # via # -r requirements/edx/kernel.in # boto3 # s3transfer bridgekeeper==0.9 # via -r requirements/edx/kernel.in -camel-converter[pydantic]==3.1.1 +camel-converter[pydantic]==3.1.2 # via meilisearch celery==5.4.0 # via @@ -136,7 +136,7 @@ click==8.1.6 # edx-django-utils # nltk # user-util -click-didyoumean==0.3.0 +click-didyoumean==0.3.1 # via celery click-plugins==1.1.1 # via celery @@ -156,7 +156,7 @@ coreschema==0.0.4 # drf-yasg crowdsourcehinter-xblock==0.7 # via -r requirements/edx/bundled.in -cryptography==41.0.7 +cryptography==42.0.7 # via # -r requirements/edx/kernel.in # django-fernet-fields-v2 @@ -168,7 +168,7 @@ cryptography==41.0.7 # pyopenssl # snowflake-connector-python # social-auth-core -cssutils==2.9.0 +cssutils==2.10.2 # via pynliner defusedxml==0.7.1 # via @@ -288,7 +288,7 @@ django-filter==24.2 # edx-enterprise # lti-consumer-xblock # openedx-blockstore -django-ipware==6.0.4 +django-ipware==7.0.1 # via # -r requirements/edx/kernel.in # edx-enterprise @@ -297,7 +297,7 @@ django-js-asset==2.2.0 # via django-mptt django-method-override==1.0.4 # via -r requirements/edx/kernel.in -django-model-utils==4.5.0 +django-model-utils==4.5.1 # via # -r requirements/edx/kernel.in # django-user-tasks @@ -321,7 +321,7 @@ django-mptt==0.14.0 # openedx-django-wiki django-multi-email-field==0.7.0 # via edx-enterprise -django-mysql==4.12.0 +django-mysql==4.13.0 # via -r requirements/edx/kernel.in django-oauth-toolkit==1.7.1 # via @@ -349,12 +349,12 @@ django-simple-history==3.4.0 # edx-organizations # edx-proctoring # ora2 -django-statici18n==2.4.0 +django-statici18n==2.5.0 # via # -r requirements/edx/kernel.in # lti-consumer-xblock # xblock-drag-and-drop-v2 -django-storages==1.14.2 +django-storages==1.14.3 # via # -r requirements/edx/kernel.in # edxval @@ -404,7 +404,7 @@ drf-jwt==1.19.2 # via edx-drf-extensions drf-nested-routers==0.93.5 # via openedx-blockstore -drf-spectacular==0.27.1 +drf-spectacular==0.27.2 # via -r requirements/edx/kernel.in drf-yasg==1.21.5 # via @@ -422,7 +422,7 @@ edx-auth-backends==4.3.0 # via # -r requirements/edx/kernel.in # openedx-blockstore -edx-braze-client==0.2.2 +edx-braze-client==0.2.5 # via # -r requirements/edx/bundled.in # edx-enterprise @@ -434,12 +434,13 @@ edx-ccx-keys==1.3.0 # via # -r requirements/edx/kernel.in # lti-consumer-xblock + # openedx-events edx-celeryutils==1.3.0 # via # -r requirements/edx/kernel.in # edx-name-affirmation # super-csv -edx-codejail==3.4.0 +edx-codejail==3.4.1 # via -r requirements/edx/kernel.in edx-completion==4.6.0 # via -r requirements/edx/kernel.in @@ -489,6 +490,7 @@ edx-event-bus-redis==0.5.0 # via -r requirements/edx/kernel.in edx-i18n-tools==1.5.0 # via + # -c requirements/edx/../constraints.txt # -r requirements/edx/bundled.in # ora2 edx-milestones==0.6.0 @@ -526,7 +528,7 @@ edx-rest-api-client==5.7.0 # edx-proctoring edx-search==3.9.1 # via -r requirements/edx/kernel.in -edx-sga==0.24.1 +edx-sga==0.25.0 # via -r requirements/edx/bundled.in edx-submissions==3.7.0 # via @@ -570,7 +572,7 @@ event-tracking==2.4.0 # edx-search fastavro==1.9.4 # via openedx-events -filelock==3.13.1 +filelock==3.14.0 # via snowflake-connector-python frozenlist==1.4.1 # via @@ -600,9 +602,9 @@ html5lib==1.1 # via # -r requirements/edx/kernel.in # ora2 -icalendar==5.0.11 +icalendar==5.0.12 # via -r requirements/edx/kernel.in -idna==3.6 +idna==3.7 # via # -r requirements/edx/paver.txt # optimizely-sdk @@ -638,7 +640,7 @@ jmespath==1.0.1 # via # boto3 # botocore -joblib==1.3.2 +joblib==1.4.2 # via nltk jsondiff==2.0.0 # via edx-enterprise @@ -651,7 +653,7 @@ jsonfield==3.1.0 # edx-submissions # lti-consumer-xblock # ora2 -jsonschema==4.21.1 +jsonschema==4.22.0 # via # drf-spectacular # optimizely-sdk @@ -661,7 +663,7 @@ jwcrypto==1.5.6 # via # django-oauth-toolkit # pylti1p3 -kombu==5.3.5 +kombu==5.3.7 # via celery laboratory==1.0.2 # via -r requirements/edx/kernel.in @@ -695,7 +697,7 @@ lxml==4.9.4 # xmlsec mailsnake==1.6.4 # via -r requirements/edx/bundled.in -mako==1.3.2 +mako==1.3.3 # via # -r requirements/edx/kernel.in # acid-xblock @@ -717,13 +719,13 @@ markupsafe==2.1.5 # mako # openedx-calc # xblock -maxminddb==2.5.2 +maxminddb==2.6.1 # via geoip2 -meilisearch==0.30.0 +meilisearch==0.31.1 # via -r requirements/edx/kernel.in mock==5.1.0 # via -r requirements/edx/paver.txt -mongoengine==0.27.0 +mongoengine==0.28.2 # via -r requirements/edx/kernel.in monotonic==1.6 # via @@ -739,7 +741,7 @@ mysqlclient==2.2.4 # via # -r requirements/edx/kernel.in # openedx-blockstore -newrelic==9.6.0 +newrelic==9.9.0 # via # -r requirements/edx/bundled.in # edx-django-utils @@ -780,7 +782,7 @@ openedx-django-require==2.1.0 # via -r requirements/edx/kernel.in openedx-django-wiki==2.1.0 # via -r requirements/edx/kernel.in -openedx-events==9.9.2 +openedx-events==9.10.0 # via # -r requirements/edx/kernel.in # edx-event-bus-kafka @@ -842,7 +844,7 @@ pillow==10.3.0 # edxval pkgutil-resolve-name==1.3.10 # via jsonschema -platformdirs==3.11.0 +platformdirs==4.2.1 # via snowflake-connector-python polib==1.2.0 # via edx-i18n-tools @@ -856,11 +858,11 @@ py2neo @ https://github.com/overhangio/py2neo/releases/download/2021.2.3/py2neo- # via # -c requirements/edx/../constraints.txt # -r requirements/edx/bundled.in -pyasn1==0.5.1 +pyasn1==0.6.0 # via pgpy pycountry==23.12.11 # via -r requirements/edx/kernel.in -pycparser==2.21 +pycparser==2.22 # via cffi pycryptodomex==3.20.0 # via @@ -868,11 +870,11 @@ pycryptodomex==3.20.0 # edx-proctoring # lti-consumer-xblock # pyjwkest -pydantic==2.6.3 +pydantic==2.7.1 # via camel-converter -pydantic-core==2.16.3 +pydantic-core==2.18.2 # via pydantic -pygments==2.17.2 +pygments==2.18.0 # via # -r requirements/edx/bundled.in # py2neo @@ -911,11 +913,11 @@ pynacl==1.5.0 # via edx-django-utils pynliner==0.8.0 # via -r requirements/edx/kernel.in -pyopenssl==23.3.0 +pyopenssl==24.1.0 # via # optimizely-sdk # snowflake-connector-python -pyparsing==3.1.1 +pyparsing==3.1.2 # via # chem # openedx-calc @@ -925,7 +927,7 @@ pysrt==1.1.2 # via # -r requirements/edx/kernel.in # edxval -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via # -r requirements/edx/kernel.in # analytics-python @@ -938,13 +940,13 @@ python-dateutil==2.8.2 # olxcleaner # ora2 # xblock -python-ipware==2.0.1 +python-ipware==3.0.0 # via django-ipware python-memcached==1.62 # via -r requirements/edx/paver.txt python-slugify==8.0.4 # via code-annotations -python-swiftclient==4.4.0 +python-swiftclient==4.5.0 # via ora2 python3-openid==3.2.0 ; python_version >= "3" # via @@ -987,15 +989,15 @@ random2==1.0.2 # via -r requirements/edx/kernel.in recommender-xblock==2.2.0 # via -r requirements/edx/bundled.in -redis==5.0.1 +redis==5.0.4 # via # -r requirements/edx/kernel.in # walrus -referencing==0.33.0 +referencing==0.35.1 # via # jsonschema # jsonschema-specifications -regex==2024.4.16 +regex==2024.4.28 # via nltk requests==2.31.0 # via @@ -1026,7 +1028,7 @@ requests-oauthlib==2.0.0 # via # -r requirements/edx/kernel.in # social-auth-core -rpds-py==0.18.0 +rpds-py==0.18.1 # via # jsonschema # referencing @@ -1040,7 +1042,7 @@ rules==3.3 # edx-enterprise # edx-proctoring # openedx-learning -s3transfer==0.10.0 +s3transfer==0.10.1 # via boto3 sailthru-client==2.2.3 # via edx-ace @@ -1050,7 +1052,7 @@ scipy==1.10.1 # openedx-calc semantic-version==2.10.0 # via edx-drf-extensions -shapely==2.0.3 +shapely==2.0.4 # via -r requirements/edx/kernel.in simplejson==3.19.2 # via @@ -1093,7 +1095,7 @@ slumber==0.7.1 # edx-bulk-grades # edx-enterprise # edx-rest-api-client -snowflake-connector-python==3.7.0 +snowflake-connector-python==3.10.0 # via edx-enterprise social-auth-app-django==5.0.0 # via @@ -1122,7 +1124,7 @@ sqlparse==0.5.0 # openedx-blockstore staff-graded-xblock==2.3.0 # via -r requirements/edx/bundled.in -stevedore==5.1.0 +stevedore==5.2.0 # via # -r requirements/edx/kernel.in # -r requirements/edx/paver.txt @@ -1135,19 +1137,19 @@ super-csv==3.2.0 # via edx-bulk-grades sympy==1.12 # via openedx-calc -testfixtures==8.0.0 +testfixtures==8.2.0 # via edx-enterprise text-unidecode==1.3 # via python-slugify tinycss2==1.2.1 # via bleach -tomlkit==0.12.3 +tomlkit==0.12.5 # via snowflake-connector-python tqdm==4.66.4 # via # nltk # openai -typing-extensions==4.9.0 +typing-extensions==4.11.0 # via # -r requirements/edx/paver.txt # annotated-types @@ -1217,7 +1219,7 @@ webob==1.8.7 # xblock wrapt==1.16.0 # via -r requirements/edx/paver.txt -xblock[django]==4.0.0 +xblock[django]==4.0.1 # via # -r requirements/edx/kernel.in # acid-xblock @@ -1235,7 +1237,7 @@ xblock[django]==4.0.0 # xblock-utils xblock-drag-and-drop-v2==4.0.2 # via -r requirements/edx/bundled.in -xblock-google-drive==0.6.1 +xblock-google-drive==0.7.0 # via -r requirements/edx/bundled.in xblock-poll==1.13.0 # via -r requirements/edx/bundled.in @@ -1249,7 +1251,7 @@ xss-utils==0.6.0 # via -r requirements/edx/kernel.in yarl==1.9.4 # via aiohttp -zipp==3.17.0 +zipp==3.18.1 # via # importlib-metadata # importlib-resources diff --git a/requirements/edx/coverage.txt b/requirements/edx/coverage.txt index ec0e1bfea514..e150dc3fe238 100644 --- a/requirements/edx/coverage.txt +++ b/requirements/edx/coverage.txt @@ -6,7 +6,7 @@ # chardet==5.2.0 # via diff-cover -coverage==7.4.1 +coverage==7.5.1 # via -r requirements/edx/coverage.in diff-cover==9.0.0 # via -r requirements/edx/coverage.in @@ -14,7 +14,7 @@ jinja2==3.1.4 # via diff-cover markupsafe==2.1.5 # via jinja2 -pluggy==1.4.0 +pluggy==1.5.0 # via diff-cover -pygments==2.17.2 +pygments==2.18.0 # via diff-cover diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 857555cd84a3..e6dd237f2e19 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -57,13 +57,15 @@ annotated-types==0.6.0 anyio==4.3.0 # via # -r requirements/edx/testing.txt + # httpx # starlette + # watchfiles appdirs==1.4.4 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # fs -asgiref==3.7.2 +asgiref==3.8.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -146,14 +148,14 @@ boto==2.49.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -boto3==1.34.45 +boto3==1.34.101 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # django-ses # fs-s3fs # ora2 -botocore==1.34.45 +botocore==1.34.101 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -171,7 +173,7 @@ cachetools==5.3.3 # via # -r requirements/edx/testing.txt # tox -camel-converter[pydantic]==3.1.1 +camel-converter[pydantic]==3.1.2 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -192,6 +194,8 @@ certifi==2024.2.2 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # elasticsearch + # httpcore + # httpx # py2neo # requests # snowflake-connector-python @@ -200,6 +204,7 @@ cffi==1.16.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # cryptography + # pact-python # pynacl # snowflake-connector-python chardet==5.2.0 @@ -240,9 +245,10 @@ click==8.1.6 # nltk # pact-python # pip-tools + # typer # user-util # uvicorn -click-didyoumean==0.3.0 +click-didyoumean==0.3.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -287,7 +293,7 @@ coreschema==0.0.4 # -r requirements/edx/testing.txt # coreapi # drf-yasg -coverage[toml]==7.4.1 +coverage[toml]==7.5.1 # via # -r requirements/edx/testing.txt # pytest-cov @@ -295,7 +301,7 @@ crowdsourcehinter-xblock==0.7 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -cryptography==41.0.7 +cryptography==42.0.7 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -312,12 +318,12 @@ cssselect==1.2.0 # via # -r requirements/edx/testing.txt # pyquery -cssutils==2.9.0 +cssutils==2.10.2 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # pynliner -ddt==1.7.1 +ddt==1.7.2 # via -r requirements/edx/testing.txt deepmerge==1.1.1 # via @@ -481,7 +487,7 @@ django-filter==24.2 # edx-enterprise # lti-consumer-xblock # openedx-blockstore -django-ipware==6.0.4 +django-ipware==7.0.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -496,7 +502,7 @@ django-method-override==1.0.4 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -django-model-utils==4.5.0 +django-model-utils==4.5.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -525,7 +531,7 @@ django-multi-email-field==0.7.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edx-enterprise -django-mysql==4.12.0 +django-mysql==4.13.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -567,13 +573,13 @@ django-simple-history==3.4.0 # edx-organizations # edx-proctoring # ora2 -django-statici18n==2.4.0 +django-statici18n==2.5.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # lti-consumer-xblock # xblock-drag-and-drop-v2 -django-storages==1.14.2 +django-storages==1.14.3 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -637,6 +643,10 @@ djangorestframework-xml==2.0.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edx-enterprise +dnspython==2.6.1 + # via + # -r requirements/edx/testing.txt + # email-validator docutils==0.19 # via # -r requirements/edx/doc.txt @@ -657,7 +667,7 @@ drf-nested-routers==0.93.5 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # openedx-blockstore -drf-spectacular==0.27.1 +drf-spectacular==0.27.2 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -683,7 +693,7 @@ edx-auth-backends==4.3.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # openedx-blockstore -edx-braze-client==0.2.2 +edx-braze-client==0.2.5 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -698,13 +708,14 @@ edx-ccx-keys==1.3.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # lti-consumer-xblock + # openedx-events edx-celeryutils==1.3.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edx-name-affirmation # super-csv -edx-codejail==3.4.0 +edx-codejail==3.4.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -768,6 +779,7 @@ edx-event-bus-redis==0.5.0 # -r requirements/edx/testing.txt edx-i18n-tools==1.5.0 # via + # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # ora2 @@ -821,7 +833,7 @@ edx-search==3.9.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -edx-sga==0.24.1 +edx-sga==0.25.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -867,6 +879,10 @@ elasticsearch==7.13.4 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edx-search +email-validator==2.1.1 + # via + # -r requirements/edx/testing.txt + # fastapi enmerkar==0.7.1 # via # -r requirements/edx/doc.txt @@ -883,12 +899,12 @@ event-tracking==2.4.0 # edx-completion # edx-proctoring # edx-search -exceptiongroup==1.2.0 +exceptiongroup==1.2.1 # via # -r requirements/edx/testing.txt # anyio # pytest -execnet==2.0.2 +execnet==2.1.1 # via # -r requirements/edx/testing.txt # pytest-xdist @@ -898,23 +914,28 @@ faker==25.0.1 # via # -r requirements/edx/testing.txt # factory-boy -fastapi==0.109.2 +fastapi==0.111.0 # via # -r requirements/edx/testing.txt + # fastapi-cli # pact-python +fastapi-cli==0.0.3 + # via + # -r requirements/edx/testing.txt + # fastapi fastavro==1.9.4 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # openedx-events -filelock==3.13.1 +filelock==3.14.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # snowflake-connector-python # tox # virtualenv -freezegun==1.4.0 +freezegun==1.5.0 # via -r requirements/edx/testing.txt frozenlist==1.4.1 # via @@ -947,7 +968,7 @@ gitdb==4.0.11 # via # -r requirements/edx/doc.txt # gitpython -gitpython==3.1.42 +gitpython==3.1.43 # via -r requirements/edx/doc.txt glob2==0.7 # via @@ -964,6 +985,7 @@ gunicorn==22.0.0 h11==0.14.0 # via # -r requirements/edx/testing.txt + # httpcore # uvicorn help-tokens==2.4.0 # via @@ -974,17 +996,31 @@ html5lib==1.1 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # ora2 +httpcore==1.0.5 + # via + # -r requirements/edx/testing.txt + # httpx httpretty==1.1.4 # via -r requirements/edx/testing.txt -icalendar==5.0.11 +httptools==0.6.1 + # via + # -r requirements/edx/testing.txt + # uvicorn +httpx==0.27.0 + # via + # -r requirements/edx/testing.txt + # fastapi +icalendar==5.0.12 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -idna==3.6 +idna==3.7 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # anyio + # email-validator + # httpx # optimizely-sdk # requests # snowflake-connector-python @@ -1052,6 +1088,7 @@ jinja2==3.1.4 # code-annotations # coreschema # diff-cover + # fastapi # sphinx jmespath==1.0.1 # via @@ -1059,7 +1096,7 @@ jmespath==1.0.1 # -r requirements/edx/testing.txt # boto3 # botocore -joblib==1.3.2 +joblib==1.4.2 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1079,7 +1116,7 @@ jsonfield==3.1.0 # edx-submissions # lti-consumer-xblock # ora2 -jsonschema==4.21.1 +jsonschema==4.22.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1097,7 +1134,7 @@ jwcrypto==1.5.6 # -r requirements/edx/testing.txt # django-oauth-toolkit # pylti1p3 -kombu==5.3.5 +kombu==5.3.7 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1152,7 +1189,7 @@ mailsnake==1.6.4 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -mako==1.3.2 +mako==1.3.3 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1168,6 +1205,10 @@ markdown==3.3.7 # openedx-django-wiki # staff-graded-xblock # xblock-poll +markdown-it-py==3.0.0 + # via + # -r requirements/edx/testing.txt + # rich markupsafe==2.1.5 # via # -r requirements/edx/doc.txt @@ -1177,7 +1218,7 @@ markupsafe==2.1.5 # mako # openedx-calc # xblock -maxminddb==2.5.2 +maxminddb==2.6.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1186,11 +1227,15 @@ mccabe==0.7.0 # via # -r requirements/edx/testing.txt # pylint -meilisearch==0.30.0 +mdurl==0.1.2 + # via + # -r requirements/edx/testing.txt + # markdown-it-py +meilisearch==0.31.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -mistune==2.0.5 +mistune==3.0.2 # via # -r requirements/edx/doc.txt # sphinx-mdinclude @@ -1198,7 +1243,7 @@ mock==5.1.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -mongoengine==0.27.0 +mongoengine==0.28.2 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1219,7 +1264,7 @@ multidict==6.0.5 # -r requirements/edx/testing.txt # aiohttp # yarl -mypy==1.8.0 +mypy==1.10.0 # via # -r requirements/edx/development.in # django-stubs @@ -1231,7 +1276,7 @@ mysqlclient==2.2.4 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # openedx-blockstore -newrelic==9.6.0 +newrelic==9.9.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1298,7 +1343,7 @@ openedx-django-wiki==2.1.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -openedx-events==9.9.2 +openedx-events==9.10.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1330,6 +1375,10 @@ ora2==6.9.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt +orjson==3.10.3 + # via + # -r requirements/edx/testing.txt + # fastapi packaging==24.0 # via # -r requirements/edx/../pip-tools.txt @@ -1345,7 +1394,7 @@ packaging==24.0 # snowflake-connector-python # sphinx # tox -pact-python==2.1.1 +pact-python==2.2.0 # via -r requirements/edx/testing.txt pansi==2020.7.3 # via @@ -1395,14 +1444,14 @@ pillow==10.3.0 # edx-enterprise # edx-organizations # edxval -pip-tools==7.4.0 +pip-tools==7.4.1 # via -r requirements/edx/../pip-tools.txt pkgutil-resolve-name==1.3.10 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # jsonschema -platformdirs==3.11.0 +platformdirs==4.2.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1410,7 +1459,7 @@ platformdirs==3.11.0 # snowflake-connector-python # tox # virtualenv -pluggy==1.4.0 +pluggy==1.5.0 # via # -r requirements/edx/testing.txt # diff-cover @@ -1440,7 +1489,7 @@ py2neo @ https://github.com/overhangio/py2neo/releases/download/2021.2.3/py2neo- # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -pyasn1==0.5.1 +pyasn1==0.6.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1453,7 +1502,7 @@ pycountry==23.12.11 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -pycparser==2.21 +pycparser==2.22 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1465,13 +1514,13 @@ pycryptodomex==3.20.0 # edx-proctoring # lti-consumer-xblock # pyjwkest -pydantic==2.6.3 +pydantic==2.7.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # camel-converter # fastapi -pydantic-core==2.16.3 +pydantic-core==2.18.2 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1480,7 +1529,7 @@ pydata-sphinx-theme==0.14.4 # via # -r requirements/edx/doc.txt # sphinx-book-theme -pygments==2.17.2 +pygments==2.18.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1488,6 +1537,7 @@ pygments==2.17.2 # diff-cover # py2neo # pydata-sphinx-theme + # rich # sphinx # sphinx-mdinclude pyjwkest==1.4.2 @@ -1563,13 +1613,13 @@ pynliner==0.8.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -pyopenssl==23.3.0 +pyopenssl==24.1.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # optimizely-sdk # snowflake-connector-python -pyparsing==3.1.1 +pyparsing==3.1.2 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1579,7 +1629,7 @@ pyproject-api==1.6.1 # via # -r requirements/edx/testing.txt # tox -pyproject-hooks==1.0.0 +pyproject-hooks==1.1.0 # via # -r requirements/edx/../pip-tools.txt # build @@ -1596,7 +1646,7 @@ pysrt==1.1.2 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edxval -pytest==8.1.2 +pytest==8.2.0 # via # -r requirements/edx/testing.txt # pylint-pytest @@ -1621,9 +1671,9 @@ pytest-metadata==1.8.0 # pytest-json-report pytest-randomly==3.15.0 # via -r requirements/edx/testing.txt -pytest-xdist[psutil]==3.5.0 +pytest-xdist[psutil]==3.6.1 # via -r requirements/edx/testing.txt -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1639,7 +1689,11 @@ python-dateutil==2.8.2 # olxcleaner # ora2 # xblock -python-ipware==2.0.1 +python-dotenv==1.0.1 + # via + # -r requirements/edx/testing.txt + # uvicorn +python-ipware==3.0.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1648,12 +1702,16 @@ python-memcached==1.62 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt +python-multipart==0.0.9 + # via + # -r requirements/edx/testing.txt + # fastapi python-slugify==8.0.4 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # code-annotations -python-swiftclient==4.4.0 +python-swiftclient==4.5.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1693,7 +1751,7 @@ pyuca==1.2 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -pywatchman==1.4.1 +pywatchman==2.0.0 # via -r requirements/edx/development.in pyyaml==6.0.1 # via @@ -1704,6 +1762,7 @@ pyyaml==6.0.1 # edx-django-release-util # edx-i18n-tools # sphinxcontrib-openapi + # uvicorn # xblock random2==1.0.2 # via @@ -1713,18 +1772,18 @@ recommender-xblock==2.2.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -redis==5.0.1 +redis==5.0.4 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # walrus -referencing==0.33.0 +referencing==0.35.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # jsonschema # jsonschema-specifications -regex==2024.4.16 +regex==2024.4.28 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1763,7 +1822,11 @@ requests-oauthlib==2.0.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # social-auth-core -rpds-py==0.18.0 +rich==13.7.1 + # via + # -r requirements/edx/testing.txt + # typer +rpds-py==0.18.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1786,7 +1849,7 @@ rules==3.3 # edx-enterprise # edx-proctoring # openedx-learning -s3transfer==0.10.0 +s3transfer==0.10.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1807,10 +1870,14 @@ semantic-version==2.10.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edx-drf-extensions -shapely==2.0.3 +shapely==2.0.4 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt +shellingham==1.5.4 + # via + # -r requirements/edx/testing.txt + # typer simplejson==3.19.2 # via # -r requirements/edx/doc.txt @@ -1864,15 +1931,16 @@ smmap==5.0.1 # via # -r requirements/edx/doc.txt # gitdb -sniffio==1.3.0 +sniffio==1.3.1 # via # -r requirements/edx/testing.txt # anyio + # httpx snowballstemmer==2.2.0 # via # -r requirements/edx/doc.txt # sphinx -snowflake-connector-python==3.7.0 +snowflake-connector-python==3.10.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1919,7 +1987,7 @@ sphinx-book-theme==1.0.1 # via -r requirements/edx/doc.txt sphinx-design==0.5.0 # via -r requirements/edx/doc.txt -sphinx-mdinclude==0.5.3 +sphinx-mdinclude==0.6.0 # via # -r requirements/edx/doc.txt # sphinxcontrib-openapi @@ -1968,11 +2036,11 @@ staff-graded-xblock==2.3.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -starlette==0.36.3 +starlette==0.37.2 # via # -r requirements/edx/testing.txt # fastapi -stevedore==5.1.0 +stevedore==5.2.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1991,7 +2059,7 @@ sympy==1.12 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # openedx-calc -testfixtures==8.0.0 +testfixtures==8.2.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -2018,17 +2086,16 @@ tomli==2.0.1 # pip-tools # pylint # pyproject-api - # pyproject-hooks # pytest # tox # vulture -tomlkit==0.12.3 +tomlkit==0.12.5 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # pylint # snowflake-connector-python -tox==4.11.4 +tox==4.15.0 # via -r requirements/edx/testing.txt tqdm==4.66.4 # via @@ -2036,9 +2103,13 @@ tqdm==4.66.4 # -r requirements/edx/testing.txt # nltk # openai -types-pytz==2024.1.0.20240203 +typer==0.12.3 + # via + # -r requirements/edx/testing.txt + # fastapi-cli +types-pytz==2024.1.0.20240417 # via django-stubs -types-pyyaml==6.0.12.12 +types-pyyaml==6.0.12.20240311 # via # django-stubs # djangorestframework-stubs @@ -2046,7 +2117,7 @@ types-requests==2.31.0.6 # via djangorestframework-stubs types-urllib3==1.26.25.14 # via types-requests -typing-extensions==4.9.0 +typing-extensions==4.11.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -2066,13 +2137,16 @@ typing-extensions==4.9.0 # jwcrypto # kombu # mypy + # pact-python # pydantic # pydantic-core # pydata-sphinx-theme # pylint # pylti1p3 + # rich # snowflake-connector-python # starlette + # typer # uvicorn tzdata==2024.1 # via @@ -2080,6 +2154,10 @@ tzdata==2024.1 # -r requirements/edx/testing.txt # backports-zoneinfo # celery +ujson==5.9.0 + # via + # -r requirements/edx/testing.txt + # fastapi unicodecsv==0.14.1 # via # -r requirements/edx/doc.txt @@ -2108,10 +2186,16 @@ user-util==1.1.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -uvicorn==0.27.1 +uvicorn[standard]==0.29.0 # via # -r requirements/edx/testing.txt + # fastapi + # fastapi-cli # pact-python +uvloop==0.19.0 + # via + # -r requirements/edx/testing.txt + # uvicorn vine==5.1.0 # via # -r requirements/edx/doc.txt @@ -2119,7 +2203,7 @@ vine==5.1.0 # amqp # celery # kombu -virtualenv==20.25.0 +virtualenv==20.26.1 # via # -r requirements/edx/testing.txt # tox @@ -2140,6 +2224,10 @@ watchdog==4.0.0 # -r requirements/edx/development.in # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt +watchfiles==0.21.0 + # via + # -r requirements/edx/testing.txt + # uvicorn wcwidth==0.2.13 # via # -r requirements/edx/doc.txt @@ -2166,7 +2254,11 @@ webob==1.8.7 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # xblock -wheel==0.42.0 +websockets==12.0 + # via + # -r requirements/edx/testing.txt + # uvicorn +wheel==0.43.0 # via # -r requirements/edx/../pip-tools.txt # pip-tools @@ -2175,7 +2267,7 @@ wrapt==1.16.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # astroid -xblock[django]==4.0.0 +xblock[django]==4.0.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -2196,7 +2288,7 @@ xblock-drag-and-drop-v2==4.0.2 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -xblock-google-drive==0.6.1 +xblock-google-drive==0.7.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -2224,7 +2316,8 @@ yarl==1.9.4 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # aiohttp -zipp==3.17.0 + # pact-python +zipp==3.18.1 # via # -r requirements/edx/../pip-tools.txt # -r requirements/edx/doc.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index abebb10842eb..854db75da070 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -41,7 +41,7 @@ appdirs==1.4.4 # via # -r requirements/edx/base.txt # fs -asgiref==3.7.2 +asgiref==3.8.1 # via # -r requirements/edx/base.txt # django @@ -107,20 +107,20 @@ bleach[css]==6.1.0 # xblock-poll boto==2.49.0 # via -r requirements/edx/base.txt -boto3==1.34.45 +boto3==1.34.101 # via # -r requirements/edx/base.txt # django-ses # fs-s3fs # ora2 -botocore==1.34.45 +botocore==1.34.101 # via # -r requirements/edx/base.txt # boto3 # s3transfer bridgekeeper==0.9 # via -r requirements/edx/base.txt -camel-converter[pydantic]==3.1.1 +camel-converter[pydantic]==3.1.2 # via # -r requirements/edx/base.txt # meilisearch @@ -171,7 +171,7 @@ click==8.1.6 # edx-django-utils # nltk # user-util -click-didyoumean==0.3.0 +click-didyoumean==0.3.1 # via # -r requirements/edx/base.txt # celery @@ -202,7 +202,7 @@ coreschema==0.0.4 # drf-yasg crowdsourcehinter-xblock==0.7 # via -r requirements/edx/base.txt -cryptography==41.0.7 +cryptography==42.0.7 # via # -r requirements/edx/base.txt # django-fernet-fields-v2 @@ -214,7 +214,7 @@ cryptography==41.0.7 # pyopenssl # snowflake-connector-python # social-auth-core -cssutils==2.9.0 +cssutils==2.10.2 # via # -r requirements/edx/base.txt # pynliner @@ -348,7 +348,7 @@ django-filter==24.2 # edx-enterprise # lti-consumer-xblock # openedx-blockstore -django-ipware==6.0.4 +django-ipware==7.0.1 # via # -r requirements/edx/base.txt # edx-enterprise @@ -359,7 +359,7 @@ django-js-asset==2.2.0 # django-mptt django-method-override==1.0.4 # via -r requirements/edx/base.txt -django-model-utils==4.5.0 +django-model-utils==4.5.1 # via # -r requirements/edx/base.txt # django-user-tasks @@ -385,7 +385,7 @@ django-multi-email-field==0.7.0 # via # -r requirements/edx/base.txt # edx-enterprise -django-mysql==4.12.0 +django-mysql==4.13.0 # via -r requirements/edx/base.txt django-oauth-toolkit==1.7.1 # via @@ -415,12 +415,12 @@ django-simple-history==3.4.0 # edx-organizations # edx-proctoring # ora2 -django-statici18n==2.4.0 +django-statici18n==2.5.0 # via # -r requirements/edx/base.txt # lti-consumer-xblock # xblock-drag-and-drop-v2 -django-storages==1.14.2 +django-storages==1.14.3 # via # -r requirements/edx/base.txt # edxval @@ -481,7 +481,7 @@ drf-nested-routers==0.93.5 # via # -r requirements/edx/base.txt # openedx-blockstore -drf-spectacular==0.27.1 +drf-spectacular==0.27.2 # via -r requirements/edx/base.txt drf-yasg==1.21.5 # via @@ -500,7 +500,7 @@ edx-auth-backends==4.3.0 # via # -r requirements/edx/base.txt # openedx-blockstore -edx-braze-client==0.2.2 +edx-braze-client==0.2.5 # via # -r requirements/edx/base.txt # edx-enterprise @@ -512,12 +512,13 @@ edx-ccx-keys==1.3.0 # via # -r requirements/edx/base.txt # lti-consumer-xblock + # openedx-events edx-celeryutils==1.3.0 # via # -r requirements/edx/base.txt # edx-name-affirmation # super-csv -edx-codejail==3.4.0 +edx-codejail==3.4.1 # via -r requirements/edx/base.txt edx-completion==4.6.0 # via -r requirements/edx/base.txt @@ -567,6 +568,7 @@ edx-event-bus-redis==0.5.0 # via -r requirements/edx/base.txt edx-i18n-tools==1.5.0 # via + # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # ora2 edx-milestones==0.6.0 @@ -605,7 +607,7 @@ edx-rest-api-client==5.7.0 # edx-proctoring edx-search==3.9.1 # via -r requirements/edx/base.txt -edx-sga==0.24.1 +edx-sga==0.25.0 # via -r requirements/edx/base.txt edx-submissions==3.7.0 # via @@ -656,7 +658,7 @@ fastavro==1.9.4 # via # -r requirements/edx/base.txt # openedx-events -filelock==3.13.1 +filelock==3.14.0 # via # -r requirements/edx/base.txt # snowflake-connector-python @@ -683,7 +685,7 @@ geoip2==4.8.0 # via -r requirements/edx/base.txt gitdb==4.0.11 # via gitpython -gitpython==3.1.42 +gitpython==3.1.43 # via -r requirements/edx/doc.in glob2==0.7 # via -r requirements/edx/base.txt @@ -695,9 +697,9 @@ html5lib==1.1 # via # -r requirements/edx/base.txt # ora2 -icalendar==5.0.11 +icalendar==5.0.12 # via -r requirements/edx/base.txt -idna==3.6 +idna==3.7 # via # -r requirements/edx/base.txt # optimizely-sdk @@ -748,7 +750,7 @@ jmespath==1.0.1 # -r requirements/edx/base.txt # boto3 # botocore -joblib==1.3.2 +joblib==1.4.2 # via # -r requirements/edx/base.txt # nltk @@ -765,7 +767,7 @@ jsonfield==3.1.0 # edx-submissions # lti-consumer-xblock # ora2 -jsonschema==4.21.1 +jsonschema==4.22.0 # via # -r requirements/edx/base.txt # drf-spectacular @@ -780,7 +782,7 @@ jwcrypto==1.5.6 # -r requirements/edx/base.txt # django-oauth-toolkit # pylti1p3 -kombu==5.3.5 +kombu==5.3.7 # via # -r requirements/edx/base.txt # celery @@ -818,7 +820,7 @@ lxml==4.9.4 # xmlsec mailsnake==1.6.4 # via -r requirements/edx/base.txt -mako==1.3.2 +mako==1.3.3 # via # -r requirements/edx/base.txt # acid-xblock @@ -840,17 +842,17 @@ markupsafe==2.1.5 # mako # openedx-calc # xblock -maxminddb==2.5.2 +maxminddb==2.6.1 # via # -r requirements/edx/base.txt # geoip2 -meilisearch==0.30.0 +meilisearch==0.31.1 # via -r requirements/edx/base.txt -mistune==2.0.5 +mistune==3.0.2 # via sphinx-mdinclude mock==5.1.0 # via -r requirements/edx/base.txt -mongoengine==0.27.0 +mongoengine==0.28.2 # via -r requirements/edx/base.txt monotonic==1.6 # via @@ -870,7 +872,7 @@ mysqlclient==2.2.4 # via # -r requirements/edx/base.txt # openedx-blockstore -newrelic==9.6.0 +newrelic==9.9.0 # via # -r requirements/edx/base.txt # edx-django-utils @@ -916,7 +918,7 @@ openedx-django-require==2.1.0 # via -r requirements/edx/base.txt openedx-django-wiki==2.1.0 # via -r requirements/edx/base.txt -openedx-events==9.9.2 +openedx-events==9.10.0 # via # -r requirements/edx/base.txt # edx-event-bus-kafka @@ -989,7 +991,7 @@ pkgutil-resolve-name==1.3.10 # via # -r requirements/edx/base.txt # jsonschema -platformdirs==3.11.0 +platformdirs==4.2.1 # via # -r requirements/edx/base.txt # snowflake-connector-python @@ -1009,13 +1011,13 @@ py2neo @ https://github.com/overhangio/py2neo/releases/download/2021.2.3/py2neo- # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt -pyasn1==0.5.1 +pyasn1==0.6.0 # via # -r requirements/edx/base.txt # pgpy pycountry==23.12.11 # via -r requirements/edx/base.txt -pycparser==2.21 +pycparser==2.22 # via # -r requirements/edx/base.txt # cffi @@ -1025,17 +1027,17 @@ pycryptodomex==3.20.0 # edx-proctoring # lti-consumer-xblock # pyjwkest -pydantic==2.6.3 +pydantic==2.7.1 # via # -r requirements/edx/base.txt # camel-converter -pydantic-core==2.16.3 +pydantic-core==2.18.2 # via # -r requirements/edx/base.txt # pydantic pydata-sphinx-theme==0.14.4 # via sphinx-book-theme -pygments==2.17.2 +pygments==2.18.0 # via # -r requirements/edx/base.txt # accessible-pygments @@ -1081,12 +1083,12 @@ pynacl==1.5.0 # edx-django-utils pynliner==0.8.0 # via -r requirements/edx/base.txt -pyopenssl==23.3.0 +pyopenssl==24.1.0 # via # -r requirements/edx/base.txt # optimizely-sdk # snowflake-connector-python -pyparsing==3.1.1 +pyparsing==3.1.2 # via # -r requirements/edx/base.txt # chem @@ -1099,7 +1101,7 @@ pysrt==1.1.2 # via # -r requirements/edx/base.txt # edxval -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via # -r requirements/edx/base.txt # analytics-python @@ -1112,7 +1114,7 @@ python-dateutil==2.8.2 # olxcleaner # ora2 # xblock -python-ipware==2.0.1 +python-ipware==3.0.0 # via # -r requirements/edx/base.txt # django-ipware @@ -1122,7 +1124,7 @@ python-slugify==8.0.4 # via # -r requirements/edx/base.txt # code-annotations -python-swiftclient==4.4.0 +python-swiftclient==4.5.0 # via # -r requirements/edx/base.txt # ora2 @@ -1168,16 +1170,16 @@ random2==1.0.2 # via -r requirements/edx/base.txt recommender-xblock==2.2.0 # via -r requirements/edx/base.txt -redis==5.0.1 +redis==5.0.4 # via # -r requirements/edx/base.txt # walrus -referencing==0.33.0 +referencing==0.35.1 # via # -r requirements/edx/base.txt # jsonschema # jsonschema-specifications -regex==2024.4.16 +regex==2024.4.28 # via # -r requirements/edx/base.txt # nltk @@ -1211,7 +1213,7 @@ requests-oauthlib==2.0.0 # via # -r requirements/edx/base.txt # social-auth-core -rpds-py==0.18.0 +rpds-py==0.18.1 # via # -r requirements/edx/base.txt # jsonschema @@ -1230,7 +1232,7 @@ rules==3.3 # edx-enterprise # edx-proctoring # openedx-learning -s3transfer==0.10.0 +s3transfer==0.10.1 # via # -r requirements/edx/base.txt # boto3 @@ -1247,7 +1249,7 @@ semantic-version==2.10.0 # via # -r requirements/edx/base.txt # edx-drf-extensions -shapely==2.0.3 +shapely==2.0.4 # via -r requirements/edx/base.txt simplejson==3.19.2 # via @@ -1294,7 +1296,7 @@ smmap==5.0.1 # via gitdb snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python==3.7.0 +snowflake-connector-python==3.10.0 # via # -r requirements/edx/base.txt # edx-enterprise @@ -1335,7 +1337,7 @@ sphinx-book-theme==1.0.1 # via -r requirements/edx/doc.in sphinx-design==0.5.0 # via -r requirements/edx/doc.in -sphinx-mdinclude==0.5.3 +sphinx-mdinclude==0.6.0 # via sphinxcontrib-openapi sphinx-reredirects==0.1.3 # via -r requirements/edx/doc.in @@ -1364,7 +1366,7 @@ sqlparse==0.5.0 # openedx-blockstore staff-graded-xblock==2.3.0 # via -r requirements/edx/base.txt -stevedore==5.1.0 +stevedore==5.2.0 # via # -r requirements/edx/base.txt # code-annotations @@ -1380,7 +1382,7 @@ sympy==1.12 # via # -r requirements/edx/base.txt # openedx-calc -testfixtures==8.0.0 +testfixtures==8.2.0 # via # -r requirements/edx/base.txt # edx-enterprise @@ -1392,7 +1394,7 @@ tinycss2==1.2.1 # via # -r requirements/edx/base.txt # bleach -tomlkit==0.12.3 +tomlkit==0.12.5 # via # -r requirements/edx/base.txt # snowflake-connector-python @@ -1401,7 +1403,7 @@ tqdm==4.66.4 # -r requirements/edx/base.txt # nltk # openai -typing-extensions==4.9.0 +typing-extensions==4.11.0 # via # -r requirements/edx/base.txt # annotated-types @@ -1482,7 +1484,7 @@ webob==1.8.7 # xblock wrapt==1.16.0 # via -r requirements/edx/base.txt -xblock[django]==4.0.0 +xblock[django]==4.0.1 # via # -r requirements/edx/base.txt # acid-xblock @@ -1500,7 +1502,7 @@ xblock[django]==4.0.0 # xblock-utils xblock-drag-and-drop-v2==4.0.2 # via -r requirements/edx/base.txt -xblock-google-drive==0.6.1 +xblock-google-drive==0.7.0 # via -r requirements/edx/base.txt xblock-poll==1.13.0 # via -r requirements/edx/base.txt @@ -1519,7 +1521,7 @@ yarl==1.9.4 # via # -r requirements/edx/base.txt # aiohttp -zipp==3.17.0 +zipp==3.18.1 # via # -r requirements/edx/base.txt # importlib-metadata diff --git a/requirements/edx/paver.txt b/requirements/edx/paver.txt index a9f602252d08..e099e333ea96 100644 --- a/requirements/edx/paver.txt +++ b/requirements/edx/paver.txt @@ -12,7 +12,7 @@ charset-normalizer==2.0.12 # requests edx-opaque-keys==2.9.0 # via -r requirements/edx/paver.in -idna==3.6 +idna==3.7 # via requests lazy==1.6 # via -r requirements/edx/paver.in @@ -49,11 +49,11 @@ six==1.16.0 # via # libsass # paver -stevedore==5.1.0 +stevedore==5.2.0 # via # -r requirements/edx/paver.in # edx-opaque-keys -typing-extensions==4.9.0 +typing-extensions==4.11.0 # via edx-opaque-keys urllib3==1.26.18 # via diff --git a/requirements/edx/semgrep.txt b/requirements/edx/semgrep.txt index 7dfa710cf95b..ec1ee016e564 100644 --- a/requirements/edx/semgrep.txt +++ b/requirements/edx/semgrep.txt @@ -38,13 +38,13 @@ face==22.0.0 # via glom glom==22.1.0 # via semgrep -idna==3.6 +idna==3.7 # via requests -importlib-resources==6.1.1 +importlib-resources==6.4.0 # via # jsonschema # jsonschema-specifications -jsonschema==4.21.1 +jsonschema==4.22.0 # via semgrep jsonschema-specifications==2023.12.1 # via jsonschema @@ -54,21 +54,21 @@ mdurl==0.1.2 # via markdown-it-py packaging==24.0 # via semgrep -peewee==3.17.1 +peewee==3.17.3 # via semgrep pkgutil-resolve-name==1.3.10 # via jsonschema -pygments==2.17.2 +pygments==2.18.0 # via rich -referencing==0.33.0 +referencing==0.35.1 # via # jsonschema # jsonschema-specifications requests==2.31.0 # via semgrep -rich==13.7.0 +rich==13.7.1 # via semgrep -rpds-py==0.18.0 +rpds-py==0.18.1 # via # jsonschema # referencing @@ -80,7 +80,7 @@ semgrep==1.52.0 # via -r requirements/edx/semgrep.in tomli==2.0.1 # via semgrep -typing-extensions==4.9.0 +typing-extensions==4.11.0 # via # rich # semgrep @@ -89,7 +89,7 @@ urllib3==1.26.18 # -c requirements/edx/../constraints.txt # requests # semgrep -wcmatch==8.5 +wcmatch==8.5.1 # via semgrep -zipp==3.17.0 +zipp==3.18.1 # via importlib-resources diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 05fd3b815a1c..1fb80f02a1e9 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -34,12 +34,15 @@ annotated-types==0.6.0 # -r requirements/edx/base.txt # pydantic anyio==4.3.0 - # via starlette + # via + # httpx + # starlette + # watchfiles appdirs==1.4.4 # via # -r requirements/edx/base.txt # fs -asgiref==3.7.2 +asgiref==3.8.1 # via # -r requirements/edx/base.txt # django @@ -107,13 +110,13 @@ bleach[css]==6.1.0 # xblock-poll boto==2.49.0 # via -r requirements/edx/base.txt -boto3==1.34.45 +boto3==1.34.101 # via # -r requirements/edx/base.txt # django-ses # fs-s3fs # ora2 -botocore==1.34.45 +botocore==1.34.101 # via # -r requirements/edx/base.txt # boto3 @@ -122,7 +125,7 @@ bridgekeeper==0.9 # via -r requirements/edx/base.txt cachetools==5.3.3 # via tox -camel-converter[pydantic]==3.1.1 +camel-converter[pydantic]==3.1.2 # via # -r requirements/edx/base.txt # meilisearch @@ -140,6 +143,8 @@ certifi==2024.2.2 # via # -r requirements/edx/base.txt # elasticsearch + # httpcore + # httpx # py2neo # requests # snowflake-connector-python @@ -147,6 +152,7 @@ cffi==1.16.0 # via # -r requirements/edx/base.txt # cryptography + # pact-python # pynacl # snowflake-connector-python chardet==5.2.0 @@ -179,9 +185,10 @@ click==8.1.6 # import-linter # nltk # pact-python + # typer # user-util # uvicorn -click-didyoumean==0.3.0 +click-didyoumean==0.3.1 # via # -r requirements/edx/base.txt # celery @@ -215,13 +222,13 @@ coreschema==0.0.4 # -r requirements/edx/base.txt # coreapi # drf-yasg -coverage[toml]==7.4.1 +coverage[toml]==7.5.1 # via # -r requirements/edx/coverage.txt # pytest-cov crowdsourcehinter-xblock==0.7 # via -r requirements/edx/base.txt -cryptography==41.0.7 +cryptography==42.0.7 # via # -r requirements/edx/base.txt # django-fernet-fields-v2 @@ -237,11 +244,11 @@ cssselect==1.2.0 # via # -r requirements/edx/testing.in # pyquery -cssutils==2.9.0 +cssutils==2.10.2 # via # -r requirements/edx/base.txt # pynliner -ddt==1.7.1 +ddt==1.7.2 # via -r requirements/edx/testing.in defusedxml==0.7.1 # via @@ -377,7 +384,7 @@ django-filter==24.2 # edx-enterprise # lti-consumer-xblock # openedx-blockstore -django-ipware==6.0.4 +django-ipware==7.0.1 # via # -r requirements/edx/base.txt # edx-enterprise @@ -388,7 +395,7 @@ django-js-asset==2.2.0 # django-mptt django-method-override==1.0.4 # via -r requirements/edx/base.txt -django-model-utils==4.5.0 +django-model-utils==4.5.1 # via # -r requirements/edx/base.txt # django-user-tasks @@ -414,7 +421,7 @@ django-multi-email-field==0.7.0 # via # -r requirements/edx/base.txt # edx-enterprise -django-mysql==4.12.0 +django-mysql==4.13.0 # via -r requirements/edx/base.txt django-oauth-toolkit==1.7.1 # via @@ -444,12 +451,12 @@ django-simple-history==3.4.0 # edx-organizations # edx-proctoring # ora2 -django-statici18n==2.4.0 +django-statici18n==2.5.0 # via # -r requirements/edx/base.txt # lti-consumer-xblock # xblock-drag-and-drop-v2 -django-storages==1.14.2 +django-storages==1.14.3 # via # -r requirements/edx/base.txt # edxval @@ -495,6 +502,8 @@ djangorestframework-xml==2.0.0 # via # -r requirements/edx/base.txt # edx-enterprise +dnspython==2.6.1 + # via email-validator done-xblock==2.3.0 # via -r requirements/edx/base.txt drf-jwt==1.19.2 @@ -505,7 +514,7 @@ drf-nested-routers==0.93.5 # via # -r requirements/edx/base.txt # openedx-blockstore -drf-spectacular==0.27.1 +drf-spectacular==0.27.2 # via -r requirements/edx/base.txt drf-yasg==1.21.5 # via @@ -524,7 +533,7 @@ edx-auth-backends==4.3.0 # via # -r requirements/edx/base.txt # openedx-blockstore -edx-braze-client==0.2.2 +edx-braze-client==0.2.5 # via # -r requirements/edx/base.txt # edx-enterprise @@ -536,12 +545,13 @@ edx-ccx-keys==1.3.0 # via # -r requirements/edx/base.txt # lti-consumer-xblock + # openedx-events edx-celeryutils==1.3.0 # via # -r requirements/edx/base.txt # edx-name-affirmation # super-csv -edx-codejail==3.4.0 +edx-codejail==3.4.1 # via -r requirements/edx/base.txt edx-completion==4.6.0 # via -r requirements/edx/base.txt @@ -591,6 +601,7 @@ edx-event-bus-redis==0.5.0 # via -r requirements/edx/base.txt edx-i18n-tools==1.5.0 # via + # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt # ora2 edx-lint==5.3.6 @@ -631,7 +642,7 @@ edx-rest-api-client==5.7.0 # edx-proctoring edx-search==3.9.1 # via -r requirements/edx/base.txt -edx-sga==0.24.1 +edx-sga==0.25.0 # via -r requirements/edx/base.txt edx-submissions==3.7.0 # via @@ -666,6 +677,8 @@ elasticsearch==7.13.4 # -c requirements/edx/../common_constraints.txt # -r requirements/edx/base.txt # edx-search +email-validator==2.1.1 + # via fastapi enmerkar==0.7.1 # via # -r requirements/edx/base.txt @@ -678,29 +691,33 @@ event-tracking==2.4.0 # edx-completion # edx-proctoring # edx-search -exceptiongroup==1.2.0 +exceptiongroup==1.2.1 # via # anyio # pytest -execnet==2.0.2 +execnet==2.1.1 # via pytest-xdist factory-boy==3.3.0 # via -r requirements/edx/testing.in faker==25.0.1 # via factory-boy -fastapi==0.109.2 - # via pact-python +fastapi==0.111.0 + # via + # fastapi-cli + # pact-python +fastapi-cli==0.0.3 + # via fastapi fastavro==1.9.4 # via # -r requirements/edx/base.txt # openedx-events -filelock==3.13.1 +filelock==3.14.0 # via # -r requirements/edx/base.txt # snowflake-connector-python # tox # virtualenv -freezegun==1.4.0 +freezegun==1.5.0 # via -r requirements/edx/testing.in frozenlist==1.4.1 # via @@ -730,21 +747,31 @@ grimp==3.2 gunicorn==22.0.0 # via -r requirements/edx/base.txt h11==0.14.0 - # via uvicorn + # via + # httpcore + # uvicorn help-tokens==2.4.0 # via -r requirements/edx/base.txt html5lib==1.1 # via # -r requirements/edx/base.txt # ora2 +httpcore==1.0.5 + # via httpx httpretty==1.1.4 # via -r requirements/edx/testing.in -icalendar==5.0.11 +httptools==0.6.1 + # via uvicorn +httpx==0.27.0 + # via fastapi +icalendar==5.0.12 # via -r requirements/edx/base.txt -idna==3.6 +idna==3.7 # via # -r requirements/edx/base.txt # anyio + # email-validator + # httpx # optimizely-sdk # requests # snowflake-connector-python @@ -795,12 +822,13 @@ jinja2==3.1.4 # code-annotations # coreschema # diff-cover + # fastapi jmespath==1.0.1 # via # -r requirements/edx/base.txt # boto3 # botocore -joblib==1.3.2 +joblib==1.4.2 # via # -r requirements/edx/base.txt # nltk @@ -817,7 +845,7 @@ jsonfield==3.1.0 # edx-submissions # lti-consumer-xblock # ora2 -jsonschema==4.21.1 +jsonschema==4.22.0 # via # -r requirements/edx/base.txt # drf-spectacular @@ -831,7 +859,7 @@ jwcrypto==1.5.6 # -r requirements/edx/base.txt # django-oauth-toolkit # pylti1p3 -kombu==5.3.5 +kombu==5.3.7 # via # -r requirements/edx/base.txt # celery @@ -872,7 +900,7 @@ lxml==4.9.4 # xmlsec mailsnake==1.6.4 # via -r requirements/edx/base.txt -mako==1.3.2 +mako==1.3.3 # via # -r requirements/edx/base.txt # acid-xblock @@ -886,6 +914,8 @@ markdown==3.3.7 # openedx-django-wiki # staff-graded-xblock # xblock-poll +markdown-it-py==3.0.0 + # via rich markupsafe==2.1.5 # via # -r requirements/edx/base.txt @@ -895,17 +925,19 @@ markupsafe==2.1.5 # mako # openedx-calc # xblock -maxminddb==2.5.2 +maxminddb==2.6.1 # via # -r requirements/edx/base.txt # geoip2 mccabe==0.7.0 # via pylint -meilisearch==0.30.0 +mdurl==0.1.2 + # via markdown-it-py +meilisearch==0.31.1 # via -r requirements/edx/base.txt mock==5.1.0 # via -r requirements/edx/base.txt -mongoengine==0.27.0 +mongoengine==0.28.2 # via -r requirements/edx/base.txt monotonic==1.6 # via @@ -925,7 +957,7 @@ mysqlclient==2.2.4 # via # -r requirements/edx/base.txt # openedx-blockstore -newrelic==9.6.0 +newrelic==9.9.0 # via # -r requirements/edx/base.txt # edx-django-utils @@ -971,7 +1003,7 @@ openedx-django-require==2.1.0 # via -r requirements/edx/base.txt openedx-django-wiki==2.1.0 # via -r requirements/edx/base.txt -openedx-events==9.9.2 +openedx-events==9.10.0 # via # -r requirements/edx/base.txt # edx-event-bus-kafka @@ -995,6 +1027,8 @@ optimizely-sdk==4.1.1 # -r requirements/edx/base.txt ora2==6.9.0 # via -r requirements/edx/base.txt +orjson==3.10.3 + # via fastapi packaging==24.0 # via # -r requirements/edx/base.txt @@ -1005,7 +1039,7 @@ packaging==24.0 # pytest # snowflake-connector-python # tox -pact-python==2.1.1 +pact-python==2.2.0 # via -r requirements/edx/testing.in pansi==2020.7.3 # via @@ -1045,14 +1079,14 @@ pkgutil-resolve-name==1.3.10 # via # -r requirements/edx/base.txt # jsonschema -platformdirs==3.11.0 +platformdirs==4.2.1 # via # -r requirements/edx/base.txt # pylint # snowflake-connector-python # tox # virtualenv -pluggy==1.4.0 +pluggy==1.5.0 # via # -r requirements/edx/coverage.txt # diff-cover @@ -1079,7 +1113,7 @@ py2neo @ https://github.com/overhangio/py2neo/releases/download/2021.2.3/py2neo- # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt -pyasn1==0.5.1 +pyasn1==0.6.0 # via # -r requirements/edx/base.txt # pgpy @@ -1089,7 +1123,7 @@ pycodestyle==2.8.0 # -r requirements/edx/testing.in pycountry==23.12.11 # via -r requirements/edx/base.txt -pycparser==2.21 +pycparser==2.22 # via # -r requirements/edx/base.txt # cffi @@ -1099,21 +1133,22 @@ pycryptodomex==3.20.0 # edx-proctoring # lti-consumer-xblock # pyjwkest -pydantic==2.6.3 +pydantic==2.7.1 # via # -r requirements/edx/base.txt # camel-converter # fastapi -pydantic-core==2.16.3 +pydantic-core==2.18.2 # via # -r requirements/edx/base.txt # pydantic -pygments==2.17.2 +pygments==2.18.0 # via # -r requirements/edx/base.txt # -r requirements/edx/coverage.txt # diff-cover # py2neo + # rich pyjwkest==1.4.2 # via # -r requirements/edx/base.txt @@ -1170,12 +1205,12 @@ pynacl==1.5.0 # edx-django-utils pynliner==0.8.0 # via -r requirements/edx/base.txt -pyopenssl==23.3.0 +pyopenssl==24.1.0 # via # -r requirements/edx/base.txt # optimizely-sdk # snowflake-connector-python -pyparsing==3.1.1 +pyparsing==3.1.2 # via # -r requirements/edx/base.txt # chem @@ -1192,7 +1227,7 @@ pysrt==1.1.2 # via # -r requirements/edx/base.txt # edxval -pytest==8.1.2 +pytest==8.2.0 # via # -r requirements/edx/testing.in # pylint-pytest @@ -1217,9 +1252,9 @@ pytest-metadata==1.8.0 # pytest-json-report pytest-randomly==3.15.0 # via -r requirements/edx/testing.in -pytest-xdist[psutil]==3.5.0 +pytest-xdist[psutil]==3.6.1 # via -r requirements/edx/testing.in -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via # -r requirements/edx/base.txt # analytics-python @@ -1234,17 +1269,21 @@ python-dateutil==2.8.2 # olxcleaner # ora2 # xblock -python-ipware==2.0.1 +python-dotenv==1.0.1 + # via uvicorn +python-ipware==3.0.0 # via # -r requirements/edx/base.txt # django-ipware python-memcached==1.62 # via -r requirements/edx/base.txt +python-multipart==0.0.9 + # via fastapi python-slugify==8.0.4 # via # -r requirements/edx/base.txt # code-annotations -python-swiftclient==4.4.0 +python-swiftclient==4.5.0 # via # -r requirements/edx/base.txt # ora2 @@ -1284,21 +1323,22 @@ pyyaml==6.0.1 # drf-spectacular # edx-django-release-util # edx-i18n-tools + # uvicorn # xblock random2==1.0.2 # via -r requirements/edx/base.txt recommender-xblock==2.2.0 # via -r requirements/edx/base.txt -redis==5.0.1 +redis==5.0.4 # via # -r requirements/edx/base.txt # walrus -referencing==0.33.0 +referencing==0.35.1 # via # -r requirements/edx/base.txt # jsonschema # jsonschema-specifications -regex==2024.4.16 +regex==2024.4.28 # via # -r requirements/edx/base.txt # nltk @@ -1332,7 +1372,9 @@ requests-oauthlib==2.0.0 # via # -r requirements/edx/base.txt # social-auth-core -rpds-py==0.18.0 +rich==13.7.1 + # via typer +rpds-py==0.18.1 # via # -r requirements/edx/base.txt # jsonschema @@ -1351,7 +1393,7 @@ rules==3.3 # edx-enterprise # edx-proctoring # openedx-learning -s3transfer==0.10.0 +s3transfer==0.10.1 # via # -r requirements/edx/base.txt # boto3 @@ -1368,8 +1410,10 @@ semantic-version==2.10.0 # via # -r requirements/edx/base.txt # edx-drf-extensions -shapely==2.0.3 +shapely==2.0.4 # via -r requirements/edx/base.txt +shellingham==1.5.4 + # via typer simplejson==3.19.2 # via # -r requirements/edx/base.txt @@ -1414,9 +1458,11 @@ slumber==0.7.1 # edx-bulk-grades # edx-enterprise # edx-rest-api-client -sniffio==1.3.0 - # via anyio -snowflake-connector-python==3.7.0 +sniffio==1.3.1 + # via + # anyio + # httpx +snowflake-connector-python==3.10.0 # via # -r requirements/edx/base.txt # edx-enterprise @@ -1450,9 +1496,9 @@ sqlparse==0.5.0 # openedx-blockstore staff-graded-xblock==2.3.0 # via -r requirements/edx/base.txt -starlette==0.36.3 +starlette==0.37.2 # via fastapi -stevedore==5.1.0 +stevedore==5.2.0 # via # -r requirements/edx/base.txt # code-annotations @@ -1468,7 +1514,7 @@ sympy==1.12 # via # -r requirements/edx/base.txt # openedx-calc -testfixtures==8.0.0 +testfixtures==8.2.0 # via # -r requirements/edx/base.txt # -r requirements/edx/testing.in @@ -1489,19 +1535,21 @@ tomli==2.0.1 # pyproject-api # pytest # tox -tomlkit==0.12.3 +tomlkit==0.12.5 # via # -r requirements/edx/base.txt # pylint # snowflake-connector-python -tox==4.11.4 +tox==4.15.0 # via -r requirements/edx/testing.in tqdm==4.66.4 # via # -r requirements/edx/base.txt # nltk # openai -typing-extensions==4.9.0 +typer==0.12.3 + # via fastapi-cli +typing-extensions==4.11.0 # via # -r requirements/edx/base.txt # annotated-types @@ -1516,18 +1564,23 @@ typing-extensions==4.9.0 # import-linter # jwcrypto # kombu + # pact-python # pydantic # pydantic-core # pylint # pylti1p3 + # rich # snowflake-connector-python # starlette + # typer # uvicorn tzdata==2024.1 # via # -r requirements/edx/base.txt # backports-zoneinfo # celery +ujson==5.9.0 + # via fastapi unicodecsv==0.14.1 # via # -r requirements/edx/base.txt @@ -1551,15 +1604,20 @@ urllib3==1.26.18 # snowflake-connector-python user-util==1.1.0 # via -r requirements/edx/base.txt -uvicorn==0.27.1 - # via pact-python +uvicorn[standard]==0.29.0 + # via + # fastapi + # fastapi-cli + # pact-python +uvloop==0.19.0 + # via uvicorn vine==5.1.0 # via # -r requirements/edx/base.txt # amqp # celery # kombu -virtualenv==20.25.0 +virtualenv==20.26.1 # via tox voluptuous==0.14.2 # via @@ -1571,6 +1629,8 @@ walrus==0.9.3 # edx-event-bus-redis watchdog==4.0.0 # via -r requirements/edx/base.txt +watchfiles==0.21.0 + # via uvicorn wcwidth==0.2.13 # via # -r requirements/edx/base.txt @@ -1593,11 +1653,13 @@ webob==1.8.7 # via # -r requirements/edx/base.txt # xblock +websockets==12.0 + # via uvicorn wrapt==1.16.0 # via # -r requirements/edx/base.txt # astroid -xblock[django]==4.0.0 +xblock[django]==4.0.1 # via # -r requirements/edx/base.txt # acid-xblock @@ -1615,7 +1677,7 @@ xblock[django]==4.0.0 # xblock-utils xblock-drag-and-drop-v2==4.0.2 # via -r requirements/edx/base.txt -xblock-google-drive==0.6.1 +xblock-google-drive==0.7.0 # via -r requirements/edx/base.txt xblock-poll==1.13.0 # via -r requirements/edx/base.txt @@ -1634,7 +1696,8 @@ yarl==1.9.4 # via # -r requirements/edx/base.txt # aiohttp -zipp==3.17.0 + # pact-python +zipp==3.18.1 # via # -r requirements/edx/base.txt # importlib-metadata diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index 389fadc32095..4b631a73d780 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -16,9 +16,9 @@ importlib-metadata==6.11.0 # build packaging==24.0 # via build -pip-tools==7.4.0 +pip-tools==7.4.1 # via -r requirements/pip-tools.in -pyproject-hooks==1.0.0 +pyproject-hooks==1.1.0 # via # build # pip-tools @@ -26,10 +26,9 @@ tomli==2.0.1 # via # build # pip-tools - # pyproject-hooks -wheel==0.42.0 +wheel==0.43.0 # via pip-tools -zipp==3.17.0 +zipp==3.18.1 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/pip.txt b/requirements/pip.txt index 71954cc66a6f..e3ffcc7b6daf 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -4,11 +4,11 @@ # # make upgrade # -wheel==0.42.0 +wheel==0.43.0 # via -r requirements/pip.in # The following packages are considered to be unsafe in a requirements file: pip==24.0 # via -r requirements/pip.in -setuptools==69.1.0 +setuptools==69.5.1 # via -r requirements/pip.in diff --git a/scripts/structures_pruning/requirements/base.txt b/scripts/structures_pruning/requirements/base.txt index 1f6d52e8c309..dcf9fe1eb0ae 100644 --- a/scripts/structures_pruning/requirements/base.txt +++ b/scripts/structures_pruning/requirements/base.txt @@ -11,7 +11,7 @@ click==8.1.6 # click-log click-log==0.4.0 # via -r scripts/structures_pruning/requirements/base.in -edx-opaque-keys==2.5.1 +edx-opaque-keys==2.9.0 # via -r scripts/structures_pruning/requirements/base.in pbr==6.0.0 # via stevedore @@ -22,5 +22,5 @@ pymongo==3.13.0 # edx-opaque-keys stevedore==5.2.0 # via edx-opaque-keys -typing-extensions==4.10.0 +typing-extensions==4.11.0 # via edx-opaque-keys diff --git a/scripts/structures_pruning/requirements/testing.txt b/scripts/structures_pruning/requirements/testing.txt index 0fc423b1b3cc..12c5d2dbdae0 100644 --- a/scripts/structures_pruning/requirements/testing.txt +++ b/scripts/structures_pruning/requirements/testing.txt @@ -12,9 +12,9 @@ click-log==0.4.0 # via -r scripts/structures_pruning/requirements/base.txt ddt==1.7.2 # via -r scripts/structures_pruning/requirements/testing.in -edx-opaque-keys==2.5.1 +edx-opaque-keys==2.9.0 # via -r scripts/structures_pruning/requirements/base.txt -exceptiongroup==1.2.0 +exceptiongroup==1.2.1 # via pytest iniconfig==2.0.0 # via pytest @@ -24,13 +24,13 @@ pbr==6.0.0 # via # -r scripts/structures_pruning/requirements/base.txt # stevedore -pluggy==1.4.0 +pluggy==1.5.0 # via pytest pymongo==3.13.0 # via # -r scripts/structures_pruning/requirements/base.txt # edx-opaque-keys -pytest==8.1.1 +pytest==8.2.0 # via -r scripts/structures_pruning/requirements/testing.in stevedore==5.2.0 # via @@ -38,7 +38,7 @@ stevedore==5.2.0 # edx-opaque-keys tomli==2.0.1 # via pytest -typing-extensions==4.10.0 +typing-extensions==4.11.0 # via # -r scripts/structures_pruning/requirements/base.txt # edx-opaque-keys diff --git a/scripts/user_retirement/requirements/base.txt b/scripts/user_retirement/requirements/base.txt index 0c3520d2c2e1..934f4293d699 100644 --- a/scripts/user_retirement/requirements/base.txt +++ b/scripts/user_retirement/requirements/base.txt @@ -4,7 +4,7 @@ # # make upgrade # -asgiref==3.7.2 +asgiref==3.8.1 # via django attrs==23.2.0 # via zeep @@ -14,10 +14,9 @@ backports-zoneinfo==0.2.1 ; python_version < "3.9" # via # -c scripts/user_retirement/requirements/../../../requirements/constraints.txt # django - # pendulum -boto3==1.34.26 +boto3==1.34.101 # via -r scripts/user_retirement/requirements/base.in -botocore==1.34.26 +botocore==1.34.101 # via # boto3 # s3transfer @@ -39,7 +38,7 @@ click==8.1.6 # -r scripts/user_retirement/requirements/base.in # edx-django-utils cryptography==42.0.7 - # via simple-salesforce + # via pyjwt django==4.2.13 # via # -c scripts/user_retirement/requirements/../../../requirements/common_constraints.txt @@ -51,31 +50,29 @@ django-crum==0.7.9 # via edx-django-utils django-waffle==4.1.0 # via edx-django-utils -edx-django-utils==5.12.0 +edx-django-utils==5.13.0 # via edx-rest-api-client edx-rest-api-client==5.7.0 # via -r scripts/user_retirement/requirements/base.in -google-api-core==2.15.0 +google-api-core==2.19.0 # via google-api-python-client -google-api-python-client==2.115.0 +google-api-python-client==2.128.0 # via -r scripts/user_retirement/requirements/base.in -google-auth==2.26.2 +google-auth==2.29.0 # via # google-api-core # google-api-python-client # google-auth-httplib2 google-auth-httplib2==0.2.0 # via google-api-python-client -googleapis-common-protos==1.62.0 +googleapis-common-protos==1.63.0 # via google-api-core httplib2==0.22.0 # via # google-api-python-client # google-auth-httplib2 -idna==3.6 +idna==3.7 # via requests -importlib-resources==6.1.1 - # via pendulum isodate==0.6.1 # via zeep jenkinsapi==0.3.13 @@ -84,47 +81,45 @@ jmespath==1.0.1 # via # boto3 # botocore -lxml==4.9.3 +lxml==4.9.4 # via # -c scripts/user_retirement/requirements/../../../requirements/constraints.txt # zeep more-itertools==10.2.0 # via simple-salesforce -newrelic==9.5.0 +newrelic==9.9.0 # via edx-django-utils pbr==6.0.0 # via stevedore -pendulum==3.0.0 - # via simple-salesforce -platformdirs==4.1.0 +platformdirs==4.2.1 # via zeep -protobuf==4.25.2 +proto-plus==1.23.0 + # via google-api-core +protobuf==4.25.3 # via # google-api-core # googleapis-common-protos + # proto-plus psutil==5.9.8 # via edx-django-utils -pyasn1==0.5.1 +pyasn1==0.6.0 # via # pyasn1-modules # rsa -pyasn1-modules==0.3.0 +pyasn1-modules==0.4.0 # via google-auth -pycparser==2.21 +pycparser==2.22 # via cffi -pyjwt==2.8.0 +pyjwt[crypto]==2.8.0 # via # edx-rest-api-client # simple-salesforce pynacl==1.5.0 # via edx-django-utils -pyparsing==3.1.1 +pyparsing==3.1.2 # via httplib2 -python-dateutil==2.8.2 - # via - # botocore - # pendulum - # time-machine +python-dateutil==2.9.0.post0 + # via botocore pytz==2024.1 # via # jenkinsapi @@ -142,15 +137,15 @@ requests==2.31.0 # simple-salesforce # slumber # zeep -requests-file==1.5.1 +requests-file==2.0.0 # via zeep requests-toolbelt==1.0.0 # via zeep rsa==4.9 # via google-auth -s3transfer==0.10.0 +s3transfer==0.10.1 # via boto3 -simple-salesforce==1.12.5 +simple-salesforce==1.12.6 # via -r scripts/user_retirement/requirements/base.in simplejson==3.19.2 # via -r scripts/user_retirement/requirements/base.in @@ -159,19 +154,16 @@ six==1.16.0 # isodate # jenkinsapi # python-dateutil - # requests-file slumber==0.7.1 # via edx-rest-api-client sqlparse==0.5.0 # via django -stevedore==5.1.0 +stevedore==5.2.0 # via edx-django-utils -time-machine==2.13.0 - # via pendulum -typing-extensions==4.9.0 - # via asgiref -tzdata==2023.4 - # via pendulum +typing-extensions==4.11.0 + # via + # asgiref + # simple-salesforce unicodecsv==0.14.1 # via -r scripts/user_retirement/requirements/base.in uritemplate==4.1.1 @@ -183,5 +175,3 @@ urllib3==1.26.18 # requests zeep==4.2.1 # via simple-salesforce -zipp==3.17.0 - # via importlib-resources diff --git a/scripts/user_retirement/requirements/testing.txt b/scripts/user_retirement/requirements/testing.txt index a58cca2f4920..2bae9bb1a272 100644 --- a/scripts/user_retirement/requirements/testing.txt +++ b/scripts/user_retirement/requirements/testing.txt @@ -4,7 +4,7 @@ # # make upgrade # -asgiref==3.7.2 +asgiref==3.8.1 # via # -r scripts/user_retirement/requirements/base.txt # django @@ -18,12 +18,11 @@ backports-zoneinfo==0.2.1 ; python_version < "3.9" # via # -r scripts/user_retirement/requirements/base.txt # django - # pendulum -boto3==1.34.26 +boto3==1.34.101 # via # -r scripts/user_retirement/requirements/base.txt # moto -botocore==1.34.26 +botocore==1.34.101 # via # -r scripts/user_retirement/requirements/base.txt # boto3 @@ -54,8 +53,8 @@ cryptography==42.0.7 # via # -r scripts/user_retirement/requirements/base.txt # moto - # simple-salesforce -ddt==1.7.1 + # pyjwt +ddt==1.7.2 # via -r scripts/user_retirement/requirements/testing.in django==4.2.13 # via @@ -71,21 +70,21 @@ django-waffle==4.1.0 # via # -r scripts/user_retirement/requirements/base.txt # edx-django-utils -edx-django-utils==5.12.0 +edx-django-utils==5.13.0 # via # -r scripts/user_retirement/requirements/base.txt # edx-rest-api-client edx-rest-api-client==5.7.0 # via -r scripts/user_retirement/requirements/base.txt -exceptiongroup==1.2.0 +exceptiongroup==1.2.1 # via pytest -google-api-core==2.15.0 +google-api-core==2.19.0 # via # -r scripts/user_retirement/requirements/base.txt # google-api-python-client -google-api-python-client==2.115.0 +google-api-python-client==2.128.0 # via -r scripts/user_retirement/requirements/base.txt -google-auth==2.26.2 +google-auth==2.29.0 # via # -r scripts/user_retirement/requirements/base.txt # google-api-core @@ -95,7 +94,7 @@ google-auth-httplib2==0.2.0 # via # -r scripts/user_retirement/requirements/base.txt # google-api-python-client -googleapis-common-protos==1.62.0 +googleapis-common-protos==1.63.0 # via # -r scripts/user_retirement/requirements/base.txt # google-api-core @@ -104,14 +103,10 @@ httplib2==0.22.0 # -r scripts/user_retirement/requirements/base.txt # google-api-python-client # google-auth-httplib2 -idna==3.6 +idna==3.7 # via # -r scripts/user_retirement/requirements/base.txt # requests -importlib-resources==6.1.1 - # via - # -r scripts/user_retirement/requirements/base.txt - # pendulum iniconfig==2.0.0 # via pytest isodate==0.6.1 @@ -127,11 +122,11 @@ jmespath==1.0.1 # -r scripts/user_retirement/requirements/base.txt # boto3 # botocore -lxml==4.9.3 +lxml==4.9.4 # via # -r scripts/user_retirement/requirements/base.txt # zeep -markupsafe==2.1.4 +markupsafe==2.1.5 # via # jinja2 # werkzeug @@ -141,51 +136,52 @@ more-itertools==10.2.0 # via # -r scripts/user_retirement/requirements/base.txt # simple-salesforce -moto==4.2.13 +moto==4.2.14 # via -r scripts/user_retirement/requirements/testing.in -newrelic==9.5.0 +newrelic==9.9.0 # via # -r scripts/user_retirement/requirements/base.txt # edx-django-utils -packaging==23.2 +packaging==24.0 # via pytest pbr==6.0.0 # via # -r scripts/user_retirement/requirements/base.txt # stevedore -pendulum==3.0.0 - # via - # -r scripts/user_retirement/requirements/base.txt - # simple-salesforce -platformdirs==4.1.0 +platformdirs==4.2.1 # via # -r scripts/user_retirement/requirements/base.txt # zeep -pluggy==1.3.0 +pluggy==1.5.0 # via pytest -protobuf==4.25.2 +proto-plus==1.23.0 + # via + # -r scripts/user_retirement/requirements/base.txt + # google-api-core +protobuf==4.25.3 # via # -r scripts/user_retirement/requirements/base.txt # google-api-core # googleapis-common-protos + # proto-plus psutil==5.9.8 # via # -r scripts/user_retirement/requirements/base.txt # edx-django-utils -pyasn1==0.5.1 +pyasn1==0.6.0 # via # -r scripts/user_retirement/requirements/base.txt # pyasn1-modules # rsa -pyasn1-modules==0.3.0 +pyasn1-modules==0.4.0 # via # -r scripts/user_retirement/requirements/base.txt # google-auth -pycparser==2.21 +pycparser==2.22 # via # -r scripts/user_retirement/requirements/base.txt # cffi -pyjwt==2.8.0 +pyjwt[crypto]==2.8.0 # via # -r scripts/user_retirement/requirements/base.txt # edx-rest-api-client @@ -194,19 +190,17 @@ pynacl==1.5.0 # via # -r scripts/user_retirement/requirements/base.txt # edx-django-utils -pyparsing==3.1.1 +pyparsing==3.1.2 # via # -r scripts/user_retirement/requirements/base.txt # httplib2 -pytest==7.4.4 +pytest==8.2.0 # via -r scripts/user_retirement/requirements/testing.in -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via # -r scripts/user_retirement/requirements/base.txt # botocore # moto - # pendulum - # time-machine pytz==2024.1 # via # -r scripts/user_retirement/requirements/base.txt @@ -230,17 +224,17 @@ requests==2.31.0 # simple-salesforce # slumber # zeep -requests-file==1.5.1 +requests-file==2.0.0 # via # -r scripts/user_retirement/requirements/base.txt # zeep -requests-mock==1.11.0 +requests-mock==1.12.1 # via -r scripts/user_retirement/requirements/testing.in requests-toolbelt==1.0.0 # via # -r scripts/user_retirement/requirements/base.txt # zeep -responses==0.24.1 +responses==0.25.0 # via # -r scripts/user_retirement/requirements/testing.in # moto @@ -248,11 +242,11 @@ rsa==4.9 # via # -r scripts/user_retirement/requirements/base.txt # google-auth -s3transfer==0.10.0 +s3transfer==0.10.1 # via # -r scripts/user_retirement/requirements/base.txt # boto3 -simple-salesforce==1.12.5 +simple-salesforce==1.12.6 # via -r scripts/user_retirement/requirements/base.txt simplejson==3.19.2 # via -r scripts/user_retirement/requirements/base.txt @@ -262,8 +256,6 @@ six==1.16.0 # isodate # jenkinsapi # python-dateutil - # requests-file - # requests-mock slumber==0.7.1 # via # -r scripts/user_retirement/requirements/base.txt @@ -272,24 +264,17 @@ sqlparse==0.5.0 # via # -r scripts/user_retirement/requirements/base.txt # django -stevedore==5.1.0 +stevedore==5.2.0 # via # -r scripts/user_retirement/requirements/base.txt # edx-django-utils -time-machine==2.13.0 - # via - # -r scripts/user_retirement/requirements/base.txt - # pendulum tomli==2.0.1 # via pytest -typing-extensions==4.9.0 +typing-extensions==4.11.0 # via # -r scripts/user_retirement/requirements/base.txt # asgiref -tzdata==2023.4 - # via - # -r scripts/user_retirement/requirements/base.txt - # pendulum + # simple-salesforce unicodecsv==0.14.1 # via -r scripts/user_retirement/requirements/base.txt uritemplate==4.1.1 @@ -302,7 +287,7 @@ urllib3==1.26.18 # botocore # requests # responses -werkzeug==3.0.1 +werkzeug==3.0.3 # via moto xmltodict==0.13.0 # via moto @@ -310,7 +295,3 @@ zeep==4.2.1 # via # -r scripts/user_retirement/requirements/base.txt # simple-salesforce -zipp==3.17.0 - # via - # -r scripts/user_retirement/requirements/base.txt - # importlib-resources diff --git a/scripts/xblock/requirements.txt b/scripts/xblock/requirements.txt index 7a5f64224cb7..241676ac91fd 100644 --- a/scripts/xblock/requirements.txt +++ b/scripts/xblock/requirements.txt @@ -10,7 +10,7 @@ charset-normalizer==2.0.12 # via # -c scripts/xblock/../../requirements/constraints.txt # requests -idna==3.6 +idna==3.7 # via requests requests==2.31.0 # via -r scripts/xblock/requirements.in From 6308c968db639aabeb6e140a6d093d00744cc073 Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Wed, 8 May 2024 15:05:19 -0400 Subject: [PATCH 069/113] refactor: always define CORS_ALLOW_HEADERS centrally The LMS and Studio need to set values for CORS_ALLOW_HEADERS so that the MFEs can work properly, since preflight requests will need to send over extra headers. Prior to this commit, CORS_ALLOW_HEADERS was being redefined in multiple places in edx-platform and again in Tutor's config because it was only being conditionally set if ENABLE_CORS_HEADERS was True (which was a policy setting). But CORS_ALLOW_HEADERS is application logic in that the value is determined by what the view needs, and won't vary by deployment. By consolidating this to always be defined in the common.py files, we make sure that deployment environments don't have to define it. An example of where this bit us was when course import in the course authoring MFE did not work because Tutor was using an outdated value for this setting. A followup to this would be to just rip out the ENABLE_CORS_HEADERS setting entirely, and just always have it on. But that would benefit from a little more discovery to make sure there's no weird use case that still requires it to be False (maybe something in the test suite?). --- cms/envs/common.py | 14 +++++++++----- cms/envs/devstack.py | 6 ------ cms/envs/production.py | 6 ------ lms/envs/common.py | 10 +++++++--- lms/envs/devstack.py | 5 ----- lms/envs/production.py | 4 ---- 6 files changed, 16 insertions(+), 29 deletions(-) diff --git a/cms/envs/common.py b/cms/envs/common.py index ddd79f6d169c..e684946d74af 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -2575,11 +2575,15 @@ CORS_ORIGIN_WHITELIST = () CORS_ORIGIN_ALLOW_ALL = False CORS_ALLOW_INSECURE = False - CORS_ALLOW_HEADERS = corsheaders_default_headers + ( - 'use-jwt-cookie', - 'content-range', - 'content-disposition', - ) + +# Set CORS_ALLOW_HEADERS regardless of whether we've enabled ENABLE_CORS_HEADERS +# because that decision might happen in a later config file. (The headers to +# allow is an application logic, and not site policy.) +CORS_ALLOW_HEADERS = corsheaders_default_headers + ( + 'use-jwt-cookie', + 'content-range', + 'content-disposition', +) LOGIN_REDIRECT_WHITELIST = [] diff --git a/cms/envs/devstack.py b/cms/envs/devstack.py index 39c28a37d2a2..3fe10377d79c 100644 --- a/cms/envs/devstack.py +++ b/cms/envs/devstack.py @@ -5,7 +5,6 @@ import logging from os.path import abspath, dirname, join -from corsheaders.defaults import default_headers as corsheaders_default_headers from .production import * # pylint: disable=wildcard-import, unused-wildcard-import @@ -256,11 +255,6 @@ def should_show_debug_toolbar(request): # lint-amnesty, pylint: disable=missing FEATURES['ENABLE_CORS_HEADERS'] = True CORS_ALLOW_CREDENTIALS = True CORS_ORIGIN_ALLOW_ALL = True -CORS_ALLOW_HEADERS = corsheaders_default_headers + ( - 'use-jwt-cookie', - 'content-range', - 'content-disposition', -) ################### Special Exams (Proctoring) and Prereqs ################### FEATURES['ENABLE_SPECIAL_EXAMS'] = True diff --git a/cms/envs/production.py b/cms/envs/production.py index 2ac80b94c39f..67524e760816 100644 --- a/cms/envs/production.py +++ b/cms/envs/production.py @@ -13,7 +13,6 @@ import warnings import yaml -from corsheaders.defaults import default_headers as corsheaders_default_headers import django from django.core.exceptions import ImproperlyConfigured from django.urls import reverse_lazy @@ -598,11 +597,6 @@ def get_env_setting(setting): CORS_ORIGIN_ALLOW_ALL = ENV_TOKENS.get('CORS_ORIGIN_ALLOW_ALL', False) CORS_ALLOW_INSECURE = ENV_TOKENS.get('CORS_ALLOW_INSECURE', False) - CORS_ALLOW_HEADERS = corsheaders_default_headers + ( - 'use-jwt-cookie', - 'content-range', - 'content-disposition', - ) ################# Settings for brand logos. ################# LOGO_URL = ENV_TOKENS.get('LOGO_URL', LOGO_URL) diff --git a/lms/envs/common.py b/lms/envs/common.py index 02e52ea5a317..6acf880d4f4a 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -3675,9 +3675,13 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring CORS_ORIGIN_WHITELIST = () CORS_ORIGIN_ALLOW_ALL = False CORS_ALLOW_INSECURE = False - CORS_ALLOW_HEADERS = corsheaders_default_headers + ( - 'use-jwt-cookie', - ) + +# Set CORS_ALLOW_HEADERS regardless of whether we've enabled ENABLE_CORS_HEADERS +# because that decision might happen in a later config file. (The headers to +# allow is an application logic, and not site policy.) +CORS_ALLOW_HEADERS = corsheaders_default_headers + ( + 'use-jwt-cookie', +) # Default cache expiration for the cross-domain proxy HTML page. # This is a static page that can be iframed into an external page diff --git a/lms/envs/devstack.py b/lms/envs/devstack.py index 7f69641055af..b783a276fe44 100644 --- a/lms/envs/devstack.py +++ b/lms/envs/devstack.py @@ -7,8 +7,6 @@ import logging from os.path import abspath, dirname, join -from corsheaders.defaults import default_headers as corsheaders_default_headers - # pylint: enable=unicode-format-string # lint-amnesty, pylint: disable=bad-option-value ##################################################################### from edx_django_utils.plugins import add_plugins @@ -295,9 +293,6 @@ def should_show_debug_toolbar(request): # lint-amnesty, pylint: disable=missing CORS_ALLOW_CREDENTIALS = True CORS_ORIGIN_WHITELIST = () CORS_ORIGIN_ALLOW_ALL = True -CORS_ALLOW_HEADERS = corsheaders_default_headers + ( - 'use-jwt-cookie', -) LOGIN_REDIRECT_WHITELIST.extend([ CMS_BASE, diff --git a/lms/envs/production.py b/lms/envs/production.py index d56a5631bb10..948e81977cb4 100644 --- a/lms/envs/production.py +++ b/lms/envs/production.py @@ -22,7 +22,6 @@ import os import yaml -from corsheaders.defaults import default_headers as corsheaders_default_headers import django from django.core.exceptions import ImproperlyConfigured from edx_django_utils.plugins import add_plugins @@ -382,9 +381,6 @@ def get_env_setting(setting): CORS_ORIGIN_ALLOW_ALL = ENV_TOKENS.get('CORS_ORIGIN_ALLOW_ALL', False) CORS_ALLOW_INSECURE = ENV_TOKENS.get('CORS_ALLOW_INSECURE', False) - CORS_ALLOW_HEADERS = corsheaders_default_headers + ( - 'use-jwt-cookie', - ) # If setting a cross-domain cookie, it's really important to choose # a name for the cookie that is DIFFERENT than the cookies used From ff6edf0791d69f67b6a17bc524220759ad45eed6 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Wed, 8 May 2024 23:00:48 -0400 Subject: [PATCH 070/113] fix: Correct shard count verification. The test count was off because without warnings disabled, it was also counting warning lines as tests. The `head -n -2` grabs everything but the last two lines which contain a count (not sure why this isn't used). If you run without `--disable-warnings` this will include any warnings that occur during test collection which we don't want in this case. --- .github/actions/verify-tests-count/action.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/verify-tests-count/action.yml b/.github/actions/verify-tests-count/action.yml index 8260827436a9..576ffc751a65 100644 --- a/.github/actions/verify-tests-count/action.yml +++ b/.github/actions/verify-tests-count/action.yml @@ -6,8 +6,8 @@ runs: - name: collect tests from all modules shell: bash run: | - echo "root_cms_unit_tests_count=$(pytest --collect-only --ds=cms.envs.test cms/ -q | head -n -2 | wc -l)" >> $GITHUB_ENV - echo "root_lms_unit_tests_count=$(pytest --collect-only --ds=lms.envs.test lms/ openedx/ common/djangoapps/ xmodule/ -q | head -n -2 | wc -l)" >> $GITHUB_ENV + echo "root_cms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=cms.envs.test cms/ -q | head -n -2 | wc -l)" >> $GITHUB_ENV + echo "root_lms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=lms.envs.test lms/ openedx/ common/djangoapps/ xmodule/ -q | head -n -2 | wc -l)" >> $GITHUB_ENV - name: get GHA unit test paths shell: bash @@ -19,8 +19,8 @@ runs: - name: collect tests from GHA unit test shards shell: bash run: | - echo "cms_unit_tests_count=$(pytest --collect-only --ds=cms.envs.test ${{ env.cms_unit_test_paths }} -q | head -n -2 | wc -l)" >> $GITHUB_ENV - echo "lms_unit_tests_count=$(pytest --collect-only --ds=lms.envs.test ${{ env.lms_unit_test_paths }} -q | head -n -2 | wc -l)" >> $GITHUB_ENV + echo "cms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=cms.envs.test ${{ env.cms_unit_test_paths }} -q | head -n -2 | wc -l)" >> $GITHUB_ENV + echo "lms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=lms.envs.test ${{ env.lms_unit_test_paths }} -q | head -n -2 | wc -l)" >> $GITHUB_ENV - name: add unit tests count From 67ccd702f513688ab561e4a1ff2117190cd8c17c Mon Sep 17 00:00:00 2001 From: jawad khan Date: Thu, 9 May 2024 12:19:50 +0500 Subject: [PATCH 071/113] feat: Added upgrade deadline in blocks api (#34750) --- lms/djangoapps/mobile_api/course_info/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/djangoapps/mobile_api/course_info/serializers.py b/lms/djangoapps/mobile_api/course_info/serializers.py index b2bb0ce24701..d7a9471088aa 100644 --- a/lms/djangoapps/mobile_api/course_info/serializers.py +++ b/lms/djangoapps/mobile_api/course_info/serializers.py @@ -82,7 +82,7 @@ class MobileCourseEnrollmentSerializer(serializers.ModelSerializer): """ class Meta: - fields = ('created', 'mode', 'is_active') + fields = ('created', 'mode', 'is_active', 'upgrade_deadline') model = CourseEnrollment lookup_field = 'username' From 6e48229616b22fd586d7dd1d61c21305dcc71846 Mon Sep 17 00:00:00 2001 From: Muhammad Adeel Tajamul <77053848+muhammadadeeltajamul@users.noreply.github.com> Date: Thu, 9 May 2024 15:16:59 +0500 Subject: [PATCH 072/113] fix: unregistered task of type for email notification (#34751) --- openedx/core/djangoapps/notifications/apps.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openedx/core/djangoapps/notifications/apps.py b/openedx/core/djangoapps/notifications/apps.py index 63d110191189..ffad68eccde6 100644 --- a/openedx/core/djangoapps/notifications/apps.py +++ b/openedx/core/djangoapps/notifications/apps.py @@ -18,3 +18,4 @@ def ready(self): """ # pylint: disable=unused-import from . import handlers + from .email import tasks From 427f436e256ebc19664fe1d7b71520b8c76fbc3b Mon Sep 17 00:00:00 2001 From: Irtaza Akram <51848298+irtazaakram@users.noreply.github.com> Date: Thu, 9 May 2024 16:48:58 +0500 Subject: [PATCH 073/113] chore: update actions runner (#34731) * fix: update actions runner --- scripts/ci-runner.Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/ci-runner.Dockerfile b/scripts/ci-runner.Dockerfile index 291f1d33d32f..bbe7ef16dbab 100644 --- a/scripts/ci-runner.Dockerfile +++ b/scripts/ci-runner.Dockerfile @@ -1,4 +1,4 @@ -FROM summerwind/actions-runner:v2.288.1-ubuntu-20.04-c221b6e as base +FROM summerwind/actions-runner:v2.316.0-ubuntu-20.04-49490c4 as base USER root @@ -45,6 +45,8 @@ COPY setup.py setup.py COPY openedx/core/lib openedx/core/lib COPY lms lms COPY cms cms +COPY common common +COPY xmodule xmodule COPY requirements/pip.txt requirements/pip.txt COPY requirements/pip-tools.txt requirements/pip-tools.txt COPY requirements/edx/testing.txt requirements/edx/testing.txt From 4c582e64bb832cce1c50b356841466bd0fd4ea91 Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Thu, 9 May 2024 04:50:56 -0700 Subject: [PATCH 074/113] feat: Always enable the "copy-paste units" functionality (#34742) --- cms/djangoapps/contentstore/toggles.py | 16 ---------------- .../xblock_storage_handlers/view_handlers.py | 6 +++--- 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/cms/djangoapps/contentstore/toggles.py b/cms/djangoapps/contentstore/toggles.py index 0d5b0b294fa7..e15aed26b0a4 100644 --- a/cms/djangoapps/contentstore/toggles.py +++ b/cms/djangoapps/contentstore/toggles.py @@ -198,22 +198,6 @@ def individualize_anonymous_user_id(course_id): return INDIVIDUALIZE_ANONYMOUS_USER_ID.is_enabled(course_id) -# .. toggle_name: contentstore.enable_copy_paste_units -# .. toggle_implementation: WaffleFlag -# .. toggle_default: False -# .. toggle_description: Moves most unit-level actions into a submenu and adds new "Copy Unit" and "Paste -# Unit" actions which can be used to copy units within or among courses. -# .. toggle_use_cases: temporary -# .. toggle_creation_date: 2023-08-01 -# .. toggle_target_removal_date: 2023-10-01 -# .. toggle_tickets: https://github.com/openedx/modular-learning/issues/11 https://github.com/openedx/modular-learning/issues/50 -ENABLE_COPY_PASTE_UNITS = WaffleFlag( - f'{CONTENTSTORE_NAMESPACE}.enable_copy_paste_units', - __name__, - CONTENTSTORE_LOG_PREFIX, -) - - # .. toggle_name: contentstore.enable_studio_content_api # .. toggle_implementation: WaffleFlag # .. toggle_default: False diff --git a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py index de3dbc55e54f..0d3f46a2bd0d 100644 --- a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py +++ b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py @@ -33,7 +33,7 @@ from xblock.fields import Scope from cms.djangoapps.contentstore.config.waffle import SHOW_REVIEW_RULES_FLAG -from cms.djangoapps.contentstore.toggles import ENABLE_COPY_PASTE_UNITS, use_tagging_taxonomy_list_page +from cms.djangoapps.contentstore.toggles import use_tagging_taxonomy_list_page from cms.djangoapps.models.settings.course_grading import CourseGradingModel from cms.lib.ai_aside_summary_config import AiAsideSummaryConfig from common.djangoapps.static_replace import replace_static_urls @@ -1254,8 +1254,8 @@ def create_xblock_info( # lint-amnesty, pylint: disable=too-many-statements ) if course_outline or is_xblock_unit: - # If the ENABLE_COPY_PASTE_UNITS feature flag is enabled, we show the newer menu that allows copying/pasting - xblock_info["enable_copy_paste_units"] = ENABLE_COPY_PASTE_UNITS.is_enabled() + # Previously, ENABLE_COPY_PASTE_UNITS was a feature flag; now it's always on. + xblock_info["enable_copy_paste_units"] = True if is_xblock_unit and summary_configuration.is_enabled(): xblock_info["summary_configuration_enabled"] = summary_configuration.is_summary_enabled(xblock_info['id']) From c5d94d21bf28332cc8e4f9b4f4d8f20cc646d55e Mon Sep 17 00:00:00 2001 From: Glib Glugovskiy Date: Thu, 9 May 2024 14:57:43 +0300 Subject: [PATCH 075/113] docs: change default value for a toggle to correct doc (#34747) --- lms/djangoapps/courseware/toggles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/djangoapps/courseware/toggles.py b/lms/djangoapps/courseware/toggles.py index 5a6fa6e62c24..43fb40436a5e 100644 --- a/lms/djangoapps/courseware/toggles.py +++ b/lms/djangoapps/courseware/toggles.py @@ -85,7 +85,7 @@ # .. toggle_name: courseware.enable_navigation_sidebar # .. toggle_implementation: WaffleFlag -# .. toggle_default: True +# .. toggle_default: False # .. toggle_description: Enable navigation sidebar on Learning MFE # .. toggle_use_cases: opt_out, open_edx # .. toggle_creation_date: 2024-03-07 From 6738faa8f7effd6cec8e648271f7dabeca0db389 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Chris=20Ch=C3=A1vez?= Date: Thu, 9 May 2024 06:59:21 -0500 Subject: [PATCH 076/113] feat: Avoid to close tag drawer when click outside (#34740) --- cms/static/js/views/utils/tagging_drawer_utils.js | 5 ----- 1 file changed, 5 deletions(-) diff --git a/cms/static/js/views/utils/tagging_drawer_utils.js b/cms/static/js/views/utils/tagging_drawer_utils.js index 227c19e9cd2a..66a90d7d5707 100644 --- a/cms/static/js/views/utils/tagging_drawer_utils.js +++ b/cms/static/js/views/utils/tagging_drawer_utils.js @@ -25,11 +25,6 @@ function($) { const drawer = document.querySelector("#manage-tags-drawer"); const drawerCover = document.querySelector(".drawer-cover"); - // Add handler to close drawer when dark background is clicked - $(drawerCover).click(function() { - closeDrawer(drawer, drawerCover); - }.bind(this)); - // Add event listen to close drawer when close button is clicked from within the Iframe window.addEventListener("message", function (event) { if (event.data === 'closeManageTagsDrawer') { From 7f6133c9405a8f09d39583aea69e5330c3baac0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Chris=20Ch=C3=A1vez?= Date: Thu, 9 May 2024 07:08:25 -0500 Subject: [PATCH 077/113] [FC-0049] feat: Update remove object-tag permission (#34728) * feat: Update remove object-tag permission * chore: Bump openedx-learning version --- .../core/djangoapps/content_tagging/rules.py | 25 +++++++++++++++++++ requirements/constraints.txt | 2 +- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/doc.txt | 2 +- requirements/edx/testing.txt | 2 +- 6 files changed, 30 insertions(+), 5 deletions(-) diff --git a/openedx/core/djangoapps/content_tagging/rules.py b/openedx/core/djangoapps/content_tagging/rules.py index 265194860159..72b744f5bf8e 100644 --- a/openedx/core/djangoapps/content_tagging/rules.py +++ b/openedx/core/djangoapps/content_tagging/rules.py @@ -273,6 +273,30 @@ def can_view_object_tag_objectid(user: UserType, object_id: str) -> bool: return bool(object_org) and (is_org_admin(user, object_org) or is_org_user(user, object_org)) +@rules.predicate +def can_remove_object_tag_objectid(user: UserType, object_id: str) -> bool: + """ + Everyone that has permission to edit the object should be able remove tags from it. + """ + if not object_id: + raise ValueError("object_id must be provided") + + if not user.is_authenticated: + return False + + try: + context_key = get_context_key_from_key_string(object_id) + assert context_key.org + except (ValueError, AssertionError): + return False + + if has_studio_write_access(user, context_key): + return True + + object_org = rules_cache.get_orgs([context_key.org]) + return bool(object_org) and is_org_admin(user, object_org) + + @rules.predicate def can_change_object_tag( user: UserType, perm_obj: oel_tagging.ObjectTagPermissionItem | None = None @@ -336,3 +360,4 @@ def can_change_taxonomy_tag(user: UserType, tag: oel_tagging.Tag | None = None) rules.set_perm("oel_tagging.view_objecttag_objectid", can_view_object_tag_objectid) rules.set_perm("oel_tagging.change_objecttag_taxonomy", can_view_object_tag_taxonomy) rules.set_perm("oel_tagging.change_objecttag_objectid", can_change_object_tag_objectid) +rules.set_perm("oel_tagging.remove_objecttag_objectid", can_remove_object_tag_objectid) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index dc52f4317ba2..76f2b6057a77 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -97,7 +97,7 @@ libsass==0.10.0 click==8.1.6 # pinning this version to avoid updates while the library is being developed -openedx-learning==0.9.2 +openedx-learning==0.9.3 # Open AI version 1.0.0 dropped support for openai.ChatCompletion which is currently in use in enterprise. openai<=0.28.1 diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 11b5bab8ab57..517f4a722e01 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -792,7 +792,7 @@ openedx-filters==1.8.1 # -r requirements/edx/kernel.in # lti-consumer-xblock # ora2 -openedx-learning==0.9.2 +openedx-learning==0.9.3 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/kernel.in diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 857555cd84a3..22da086ac200 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -1312,7 +1312,7 @@ openedx-filters==1.8.1 # -r requirements/edx/testing.txt # lti-consumer-xblock # ora2 -openedx-learning==0.9.2 +openedx-learning==0.9.3 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index abebb10842eb..8d53f3852cb7 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -928,7 +928,7 @@ openedx-filters==1.8.1 # -r requirements/edx/base.txt # lti-consumer-xblock # ora2 -openedx-learning==0.9.2 +openedx-learning==0.9.3 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 05fd3b815a1c..d640dc7fc5cd 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -983,7 +983,7 @@ openedx-filters==1.8.1 # -r requirements/edx/base.txt # lti-consumer-xblock # ora2 -openedx-learning==0.9.2 +openedx-learning==0.9.3 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt From b42da7429ff69d04bae2634b1388c2a6546dff1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B4mulo=20Penido?= Date: Thu, 9 May 2024 16:27:05 +0300 Subject: [PATCH 078/113] feat: Enable taxonomy/tagging feature in MFE by default (#34633) * feat: make tagging feature enabled by default * fix: use the correct flag for tagging enabled * fix: make compatible with other changes from master * fix: more compatibility fixes * fix: show tag counts at all levels of the outline, not just units * chore: typo * test: fix counts in test suite now that tagging is on by default --------- Co-authored-by: Braden MacDonald Co-authored-by: Yusuf Musleh --- .../rest_api/v1/serializers/vertical_block.py | 4 +- .../v1/views/tests/test_course_index.py | 2 +- .../rest_api/v1/views/tests/test_home.py | 7 +--- .../v1/views/tests/test_vertical_block.py | 5 +-- cms/djangoapps/contentstore/toggles.py | 18 -------- cms/djangoapps/contentstore/utils.py | 42 +++++++++++-------- cms/djangoapps/contentstore/views/block.py | 4 +- .../contentstore/views/tests/test_block.py | 3 -- .../views/tests/test_course_index.py | 2 +- .../xblock_storage_handlers/view_handlers.py | 11 ++--- cms/static/js/views/course_outline.js | 9 ++-- cms/static/js/views/pages/container.js | 14 ++++--- cms/static/js/views/pages/course_outline.js | 2 +- cms/static/js/views/xblock_outline.js | 4 +- cms/templates/js/course-outline.underscore | 4 +- cms/templates/studio_xblock_wrapper.html | 5 ++- .../djangoapps/content_tagging/toggles.py | 24 ++++++++++- 17 files changed, 84 insertions(+), 76 deletions(-) diff --git a/cms/djangoapps/contentstore/rest_api/v1/serializers/vertical_block.py b/cms/djangoapps/contentstore/rest_api/v1/serializers/vertical_block.py index e2b158f1fbba..d72707ed7836 100644 --- a/cms/djangoapps/contentstore/rest_api/v1/serializers/vertical_block.py +++ b/cms/djangoapps/contentstore/rest_api/v1/serializers/vertical_block.py @@ -5,11 +5,11 @@ from django.urls import reverse from rest_framework import serializers -from cms.djangoapps.contentstore.toggles import use_tagging_taxonomy_list_page from cms.djangoapps.contentstore.helpers import ( xblock_studio_url, xblock_type_display_name, ) +from openedx.core.djangoapps.content_tagging.toggles import is_tagging_feature_disabled class MessageValidation(serializers.Serializer): @@ -122,7 +122,7 @@ def get_actions(self, obj): # pylint: disable=unused-argument Method to get actions for each child xlock of the unit. """ - can_manage_tags = use_tagging_taxonomy_list_page() + can_manage_tags = not is_tagging_feature_disabled() xblock = obj["xblock"] is_course = xblock.scope_ids.usage_id.context_key.is_course xblock_url = xblock_studio_url(xblock) diff --git a/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_course_index.py b/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_course_index.py index eebb6cd90ebb..eafc2b37aa0c 100644 --- a/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_course_index.py +++ b/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_course_index.py @@ -150,6 +150,6 @@ def test_number_of_calls_to_db(self): """ Test to check number of queries made to mysql and mongo """ - with self.assertNumQueries(29, table_ignorelist=WAFFLE_TABLES): + with self.assertNumQueries(32, table_ignorelist=WAFFLE_TABLES): with check_mongo_calls(3): self.client.get(self.url) diff --git a/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_home.py b/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_home.py index a7e2e8f03aa8..1b8bfaa84728 100644 --- a/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_home.py +++ b/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_home.py @@ -8,14 +8,12 @@ from django.urls import reverse from edx_toggles.toggles.testutils import ( override_waffle_switch, - override_waffle_flag, ) from rest_framework import status from cms.djangoapps.contentstore.tests.utils import CourseTestCase from cms.djangoapps.contentstore.tests.test_libraries import LibraryTestCase from cms.djangoapps.contentstore.views.course import ENABLE_GLOBAL_STAFF_OPTIMIZATION -from cms.djangoapps.contentstore.toggles import ENABLE_TAGGING_TAXONOMY_LIST_PAGE from openedx.core.djangoapps.content.course_overviews.tests.factories import CourseOverviewFactory from xmodule.modulestore.tests.factories import CourseFactory @@ -52,9 +50,9 @@ def test_home_page_courses_response(self): "in_process_course_actions": [], "libraries": [], "libraries_enabled": True, - "taxonomies_enabled": False, + "taxonomies_enabled": True, "library_authoring_mfe_url": settings.LIBRARY_AUTHORING_MICROFRONTEND_URL, - "taxonomy_list_mfe_url": None, + "taxonomy_list_mfe_url": 'http://course-authoring-mfe/taxonomies', "optimization_enabled": False, "redirect_to_library_authoring_mfe": False, "request_course_creator_url": "/request_course_creator", @@ -72,7 +70,6 @@ def test_home_page_courses_response(self): self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertDictEqual(expected_response, response.data) - @override_waffle_flag(ENABLE_TAGGING_TAXONOMY_LIST_PAGE, True) def test_taxonomy_list_link(self): response = self.client.get(self.url) self.assertTrue(response.data['taxonomies_enabled']) diff --git a/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_vertical_block.py b/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_vertical_block.py index fc7671aac057..d3fc37198213 100644 --- a/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_vertical_block.py +++ b/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_vertical_block.py @@ -8,7 +8,7 @@ from xblock.validation import ValidationMessage from cms.djangoapps.contentstore.tests.utils import CourseTestCase -from cms.djangoapps.contentstore.toggles import ENABLE_TAGGING_TAXONOMY_LIST_PAGE +from openedx.core.djangoapps.content_tagging.toggles import DISABLE_TAGGING_FEATURE from xmodule.partitions.partitions import ( ENROLLMENT_TRACK_PARTITION_ID, Group, @@ -164,7 +164,6 @@ def test_xblock_is_published(self): response = self.client.get(url) self.assertTrue(response.data["is_published"]) - @override_waffle_flag(ENABLE_TAGGING_TAXONOMY_LIST_PAGE, True) def test_children_content(self): """ Check that returns valid response with children of vertical container. @@ -238,7 +237,7 @@ def test_not_valid_usage_key_string(self): response = self.client.get(url) self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) - @override_waffle_flag(ENABLE_TAGGING_TAXONOMY_LIST_PAGE, False) + @override_waffle_flag(DISABLE_TAGGING_FEATURE, True) def test_actions_with_turned_off_taxonomy_flag(self): """ Check that action manage_tags for each child item has the same value as taxonomy flag. diff --git a/cms/djangoapps/contentstore/toggles.py b/cms/djangoapps/contentstore/toggles.py index e15aed26b0a4..c55a0a8a2238 100644 --- a/cms/djangoapps/contentstore/toggles.py +++ b/cms/djangoapps/contentstore/toggles.py @@ -581,21 +581,3 @@ def default_enable_flexible_peer_openassessments(course_key): level to opt in/out of rolling forward this feature. """ return DEFAULT_ENABLE_FLEXIBLE_PEER_OPENASSESSMENTS.is_enabled(course_key) - - -# .. toggle_name: new_studio_mfe.use_tagging_taxonomy_list_page -# .. toggle_implementation: WaffleFlag -# .. toggle_default: False -# .. toggle_description: This flag enables the use of the taxonomy list page. -# .. toggle_use_cases: temporary -# .. toggle_creation_date: 2023-10-06 -# .. toggle_target_removal_date: TBA -# .. toggle_warning: -ENABLE_TAGGING_TAXONOMY_LIST_PAGE = WaffleFlag('new_studio_mfe.use_tagging_taxonomy_list_page', __name__) - - -def use_tagging_taxonomy_list_page(): - """ - Returns a boolean if taxonomy list page is enabled - """ - return ENABLE_TAGGING_TAXONOMY_LIST_PAGE.is_enabled() diff --git a/cms/djangoapps/contentstore/utils.py b/cms/djangoapps/contentstore/utils.py index 3fe718dfa149..9bcd77c3eea9 100644 --- a/cms/djangoapps/contentstore/utils.py +++ b/cms/djangoapps/contentstore/utils.py @@ -58,6 +58,7 @@ from common.djangoapps.xblock_django.api import deprecated_xblocks from common.djangoapps.xblock_django.user_service import DjangoXBlockUserService from openedx.core import toggles as core_toggles +from openedx.core.djangoapps.content_tagging.toggles import is_tagging_feature_disabled from openedx.core.djangoapps.credit.api import get_credit_requirements, is_credit_course from openedx.core.djangoapps.discussions.config.waffle import ENABLE_PAGES_AND_RESOURCES_MICROFRONTEND from openedx.core.djangoapps.discussions.models import DiscussionsConfiguration @@ -91,7 +92,6 @@ use_new_video_editor, use_new_video_uploads_page, use_new_custom_pages, - use_tagging_taxonomy_list_page, ) from cms.djangoapps.models.settings.course_grading import CourseGradingModel from cms.djangoapps.models.settings.course_metadata import CourseMetadata @@ -489,16 +489,19 @@ def get_custom_pages_url(course_locator) -> str: return custom_pages_url -def get_taxonomy_list_url(): +def get_taxonomy_list_url() -> str | None: """ Gets course authoring microfrontend URL for taxonomy list page view. """ - taxonomy_list_url = None - if use_tagging_taxonomy_list_page(): - mfe_base_url = settings.COURSE_AUTHORING_MICROFRONTEND_URL - if mfe_base_url: - taxonomy_list_url = f'{mfe_base_url}/taxonomies' - return taxonomy_list_url + if is_tagging_feature_disabled(): + return None + + mfe_base_url = settings.COURSE_AUTHORING_MICROFRONTEND_URL + + if not mfe_base_url: + return None + + return f'{mfe_base_url}/taxonomies' def get_taxonomy_tags_widget_url(course_locator=None) -> str | None: @@ -507,15 +510,18 @@ def get_taxonomy_tags_widget_url(course_locator=None) -> str | None: The `content_id` needs to be appended to the end of the URL when using it. """ - taxonomy_tags_widget_url = None - # Uses the same waffle flag as taxonomy list page - if use_tagging_taxonomy_list_page(): + if is_tagging_feature_disabled(): + return None + + if course_locator: + mfe_base_url = get_course_authoring_url(course_locator) + else: mfe_base_url = settings.COURSE_AUTHORING_MICROFRONTEND_URL - if course_locator: - mfe_base_url = get_course_authoring_url(course_locator) - if mfe_base_url: - taxonomy_tags_widget_url = f'{mfe_base_url}/tagging/components/widget/' - return taxonomy_tags_widget_url + + if not mfe_base_url: + return None + + return f'{mfe_base_url}/tagging/components/widget/' def course_import_olx_validation_is_enabled(): @@ -1688,7 +1694,7 @@ def get_home_context(request, no_course=False): 'archived_courses': archived_courses, 'in_process_course_actions': in_process_course_actions, 'libraries_enabled': LIBRARIES_ENABLED, - 'taxonomies_enabled': use_tagging_taxonomy_list_page(), + 'taxonomies_enabled': not is_tagging_feature_disabled(), 'redirect_to_library_authoring_mfe': should_redirect_to_library_authoring_mfe(), 'library_authoring_mfe_url': LIBRARY_AUTHORING_MICROFRONTEND_URL, 'taxonomy_list_mfe_url': get_taxonomy_list_url(), @@ -1984,7 +1990,7 @@ def get_container_handler_context(request, usage_key, course, xblock): # pylint prev_url = quote_plus(prev_url) if prev_url else None next_url = quote_plus(next_url) if next_url else None - show_unit_tags = use_tagging_taxonomy_list_page() + show_unit_tags = not is_tagging_feature_disabled() unit_tags = None if show_unit_tags and is_unit_page: unit_tags = get_unit_tags(usage_key) diff --git a/cms/djangoapps/contentstore/views/block.py b/cms/djangoapps/contentstore/views/block.py index e2d1572bc25b..d52cc5eecfd4 100644 --- a/cms/djangoapps/contentstore/views/block.py +++ b/cms/djangoapps/contentstore/views/block.py @@ -29,7 +29,7 @@ from xmodule.modulestore.django import ( modulestore, ) # lint-amnesty, pylint: disable=wrong-import-order -from cms.djangoapps.contentstore.toggles import use_tagging_taxonomy_list_page +from openedx.core.djangoapps.content_tagging.toggles import is_tagging_feature_disabled from xmodule.x_module import ( AUTHOR_VIEW, @@ -242,7 +242,7 @@ def xblock_view_handler(request, usage_key_string, view_name): # Fetch tags of children components tags_count_map = {} - if use_tagging_taxonomy_list_page(): + if not is_tagging_feature_disabled(): tags_count_map = get_children_tags_count(xblock) # Set up the context to be passed to each XBlock's render method. diff --git a/cms/djangoapps/contentstore/views/tests/test_block.py b/cms/djangoapps/contentstore/views/tests/test_block.py index 0a12287ac199..b37ef49f6bda 100644 --- a/cms/djangoapps/contentstore/views/tests/test_block.py +++ b/cms/djangoapps/contentstore/views/tests/test_block.py @@ -17,7 +17,6 @@ from openedx_events.content_authoring.signals import XBLOCK_DUPLICATED from openedx_events.tests.utils import OpenEdxEventsTestMixin from edx_proctoring.exceptions import ProctoredExamNotFoundException -from edx_toggles.toggles.testutils import override_waffle_flag from opaque_keys import InvalidKeyError from opaque_keys.edx.asides import AsideUsageKeyV2 from opaque_keys.edx.keys import CourseKey, UsageKey @@ -85,7 +84,6 @@ add_container_page_publishing_info, create_xblock_info, ) -from cms.djangoapps.contentstore.toggles import ENABLE_TAGGING_TAXONOMY_LIST_PAGE class AsideTest(XBlockAside): @@ -272,7 +270,6 @@ def test_get_container_nested_container_fragment(self): ), ) - @override_waffle_flag(ENABLE_TAGGING_TAXONOMY_LIST_PAGE, True) @patch("cms.djangoapps.contentstore.xblock_storage_handlers.xblock_helpers.get_object_tag_counts") def test_tag_count_in_container_fragment(self, mock_get_object_tag_counts): root_usage_key = self._create_vertical() diff --git a/cms/djangoapps/contentstore/views/tests/test_course_index.py b/cms/djangoapps/contentstore/views/tests/test_course_index.py index 952483893a74..c3dcfe5305b7 100644 --- a/cms/djangoapps/contentstore/views/tests/test_course_index.py +++ b/cms/djangoapps/contentstore/views/tests/test_course_index.py @@ -717,7 +717,7 @@ def test_number_of_calls_to_db(self): """ Test to check number of queries made to mysql and mongo """ - with self.assertNumQueries(26, table_ignorelist=WAFFLE_TABLES): + with self.assertNumQueries(29, table_ignorelist=WAFFLE_TABLES): with check_mongo_calls(3): self.client.get_html(reverse_course_url('course_handler', self.course.id)) diff --git a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py index 0d3f46a2bd0d..2be43c7911de 100644 --- a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py +++ b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py @@ -33,7 +33,6 @@ from xblock.fields import Scope from cms.djangoapps.contentstore.config.waffle import SHOW_REVIEW_RULES_FLAG -from cms.djangoapps.contentstore.toggles import use_tagging_taxonomy_list_page from cms.djangoapps.models.settings.course_grading import CourseGradingModel from cms.lib.ai_aside_summary_config import AiAsideSummaryConfig from common.djangoapps.static_replace import replace_static_urls @@ -44,6 +43,7 @@ from common.djangoapps.util.date_utils import get_default_time_display from common.djangoapps.util.json_request import JsonResponse, expect_json from openedx.core.djangoapps.bookmarks import api as bookmarks_api +from openedx.core.djangoapps.content_tagging.toggles import is_tagging_feature_disabled from openedx.core.djangoapps.discussions.models import DiscussionsConfiguration from openedx.core.djangoapps.video_config.toggles import PUBLIC_VIDEO_SHARE from openedx.core.lib.gating import api as gating_api @@ -1217,7 +1217,10 @@ def create_xblock_info( # lint-amnesty, pylint: disable=too-many-statements xblock_info["ancestor_has_staff_lock"] = False if tags is not None: xblock_info["tags"] = tags - if use_tagging_taxonomy_list_page(): + + # Don't show the "Manage Tags" option and tag counts if the DISABLE_TAGGING_FEATURE waffle is true + xblock_info["is_tagging_feature_disabled"] = is_tagging_feature_disabled() + if not is_tagging_feature_disabled(): xblock_info["taxonomy_tags_widget_url"] = get_taxonomy_tags_widget_url() xblock_info["course_authoring_url"] = settings.COURSE_AUTHORING_MICROFRONTEND_URL @@ -1240,9 +1243,7 @@ def create_xblock_info( # lint-amnesty, pylint: disable=too-many-statements else: xblock_info["hide_from_toc_message"] = False - # If the ENABLE_TAGGING_TAXONOMY_LIST_PAGE feature flag is enabled, we show the "Manage Tags" options - if use_tagging_taxonomy_list_page(): - xblock_info["use_tagging_taxonomy_list_page"] = True + if not is_tagging_feature_disabled(): xblock_info["course_tags_count"] = _get_course_tags_count(course.id) xblock_info["tag_counts_by_block"] = _get_course_block_tags(xblock.location.context_key) diff --git a/cms/static/js/views/course_outline.js b/cms/static/js/views/course_outline.js index b0e89127aa28..ff1d07696342 100644 --- a/cms/static/js/views/course_outline.js +++ b/cms/static/js/views/course_outline.js @@ -33,13 +33,16 @@ function( }, renderTagCount: function() { + if (this.model.get('is_tagging_feature_disabled')) { + return; // Tagging feature is disabled; don't initialize the tag count view. + } const contentId = this.model.get('id'); - const tagCountsByBlock = this.model.get('tag_counts_by_block') // Skip the course block since that is handled elsewhere in course_manage_tags if (contentId.includes('@course')) { - return + return; } - const tagsCount = tagCountsByBlock !== undefined ? tagCountsByBlock[contentId] : 0 + const tagCountsByBlock = this.model.get('tag_counts_by_block'); + const tagsCount = tagCountsByBlock !== undefined ? tagCountsByBlock[contentId] : 0; const tagCountElem = this.$(`.tag-count[data-locator="${contentId}"]`); var countModel = new TagCountModel({ content_id: contentId, diff --git a/cms/static/js/views/pages/container.js b/cms/static/js/views/pages/container.js index 3268b60e416a..1e8e5e8b0cc4 100644 --- a/cms/static/js/views/pages/container.js +++ b/cms/static/js/views/pages/container.js @@ -107,12 +107,14 @@ function($, _, Backbone, gettext, BasePage, }); this.viewLiveActions.render(); - this.tagListView = new ContainerSubviews.TagList({ - el: this.$('.unit-tags'), - model: this.model - }); - this.tagListView.setupMessageListener(); - this.tagListView.render(); + if (!this.model.get('is_tagging_feature_disabled')) { + this.tagListView = new ContainerSubviews.TagList({ + el: this.$('.unit-tags'), + model: this.model + }); + this.tagListView.setupMessageListener(); + this.tagListView.render(); + } this.unitOutlineView = new UnitOutlineView({ el: this.$('.wrapper-unit-overview'), diff --git a/cms/static/js/views/pages/course_outline.js b/cms/static/js/views/pages/course_outline.js index 627a21e563e5..fd24abc3ae78 100644 --- a/cms/static/js/views/pages/course_outline.js +++ b/cms/static/js/views/pages/course_outline.js @@ -138,7 +138,7 @@ function($, _, gettext, BasePage, XBlockViewUtils, CourseOutlineView, ViewUtils, } // if tagging enabled - if (this.model.get('use_tagging_taxonomy_list_page')) { + if (!this.model.get('is_tagging_feature_disabled')) { this.courseManageTagsView = new CourseManageTagsView({ el: this.$('.status-manage-tags'), model: this.model diff --git a/cms/static/js/views/xblock_outline.js b/cms/static/js/views/xblock_outline.js index 79dc80334b87..3e1dba00dfb5 100644 --- a/cms/static/js/views/xblock_outline.js +++ b/cms/static/js/views/xblock_outline.js @@ -115,8 +115,8 @@ function($, _, gettext, BaseView, ViewUtils, XBlockViewUtils, XBlockStringFieldE hideFromTOCMessage: this.model.get('hide_from_toc_message'), enableHideFromTOC: this.model.get('hide_from_toc'), course: course, - enableCopyPasteUnits: this.model.get("enable_copy_paste_units"), // ENABLE_COPY_PASTE_UNITS waffle flag - useTaggingTaxonomyListPage: this.model.get("use_tagging_taxonomy_list_page"), // ENABLE_TAGGING_TAXONOMY_LIST_PAGE waffle flag + enableCopyPasteUnits: this.model.get('enable_copy_paste_units'), // ENABLE_COPY_PASTE_UNITS waffle flag + isTaggingFeatureDisabled: this.model.get('is_tagging_feature_disabled'), // DISABLE_TAGGING_FEATURE waffle flag }; }, diff --git a/cms/templates/js/course-outline.underscore b/cms/templates/js/course-outline.underscore index f8d60d696a3e..eec4be4cb5cf 100644 --- a/cms/templates/js/course-outline.underscore +++ b/cms/templates/js/course-outline.underscore @@ -187,7 +187,7 @@ if (is_proctored_exam) { <% } %> - <% if (typeof useTaggingTaxonomyListPage !== "undefined" && useTaggingTaxonomyListPage) { %> + <% if (!isTaggingFeatureDisabled) { %>
  • <% } %> @@ -210,7 +210,7 @@ if (is_proctored_exam) { <%- gettext('Configure') %> <% } %> - <% if (typeof useTaggingTaxonomyListPage !== "undefined" && useTaggingTaxonomyListPage) { %> + <% if (!isTaggingFeatureDisabled) { %> diff --git a/cms/templates/studio_xblock_wrapper.html b/cms/templates/studio_xblock_wrapper.html index b2fc928f4274..282629456633 100644 --- a/cms/templates/studio_xblock_wrapper.html +++ b/cms/templates/studio_xblock_wrapper.html @@ -7,14 +7,15 @@ from openedx.core.djangolib.js_utils import ( dump_js_escaped_json, js_escaped_string ) -from cms.djangoapps.contentstore.toggles import use_new_text_editor, use_new_problem_editor, use_new_video_editor, use_video_gallery_flow, use_tagging_taxonomy_list_page +from cms.djangoapps.contentstore.toggles import use_new_text_editor, use_new_problem_editor, use_new_video_editor, use_video_gallery_flow +from openedx.core.djangoapps.content_tagging.toggles import is_tagging_feature_disabled %> <% use_new_editor_text = use_new_text_editor() use_new_editor_video = use_new_video_editor() use_new_editor_problem = use_new_problem_editor() use_new_video_gallery_flow = use_video_gallery_flow() -use_tagging = use_tagging_taxonomy_list_page() +use_tagging = not is_tagging_feature_disabled() xblock_url = xblock_studio_url(xblock) show_inline = xblock.has_children and not xblock_url section_class = "level-nesting" if show_inline else "level-element" diff --git a/openedx/core/djangoapps/content_tagging/toggles.py b/openedx/core/djangoapps/content_tagging/toggles.py index 30a21cf77e51..3162a529833c 100644 --- a/openedx/core/djangoapps/content_tagging/toggles.py +++ b/openedx/core/djangoapps/content_tagging/toggles.py @@ -1,12 +1,14 @@ - """ Toggles for content tagging """ +from edx_toggles.toggles import WaffleFlag + from openedx.core.djangoapps.waffle_utils import CourseWaffleFlag + # .. toggle_name: content_tagging.auto -# .. toggle_implementation: WaffleSwitch +# .. toggle_implementation: CourseWaffleFlag # .. toggle_default: False # .. toggle_description: Setting this enables automatic tagging of content # .. toggle_type: feature_flag @@ -15,3 +17,21 @@ # .. toggle_creation_date: 2023-08-30 # .. toggle_tickets: https://github.com/openedx/modular-learning/issues/79 CONTENT_TAGGING_AUTO = CourseWaffleFlag('content_tagging.auto', __name__) + + +# .. toggle_name: content_tagging.disabled +# .. toggle_implementation: WaffleFlag +# .. toggle_default: False +# .. toggle_description: Setting this disables the tagging feature +# .. toggle_type: feature_flag +# .. toggle_category: admin +# .. toggle_use_cases: open_edx +# .. toggle_creation_date: 2024-04-30 +DISABLE_TAGGING_FEATURE = WaffleFlag('content_tagging.disabled', __name__) + + +def is_tagging_feature_disabled(): + """ + Returns a boolean if tagging feature list page is disabled + """ + return DISABLE_TAGGING_FEATURE.is_enabled() From 8ff52ce19789065fd28a6fd0d7238fb8ad90be21 Mon Sep 17 00:00:00 2001 From: Max Sokolski Date: Thu, 9 May 2024 12:52:54 +0300 Subject: [PATCH 079/113] feat: freeze edx-sandbox requirements for redwood We ran: cp requirements/edx-sandbox/base.txt \ requirements/edx-sandbox/releases/redwood.txt --- requirements/edx-sandbox/README.rst | 2 +- requirements/edx-sandbox/releases/redwood.txt | 93 +++++++++++++++++++ 2 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 requirements/edx-sandbox/releases/redwood.txt diff --git a/requirements/edx-sandbox/README.rst b/requirements/edx-sandbox/README.rst index d2118761769c..51dd7488444e 100644 --- a/requirements/edx-sandbox/README.rst +++ b/requirements/edx-sandbox/README.rst @@ -54,7 +54,7 @@ releases/quince.txt * Frozen between the Quince and Redwood releases * Supports only Python 3.8 -releases/redwood.txt (FUTURE PLAN) +releases/redwood.txt ---------------------------------- * Frozen at the time of the Redwood release diff --git a/requirements/edx-sandbox/releases/redwood.txt b/requirements/edx-sandbox/releases/redwood.txt new file mode 100644 index 000000000000..d12c994fd206 --- /dev/null +++ b/requirements/edx-sandbox/releases/redwood.txt @@ -0,0 +1,93 @@ +# +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: +# +# make upgrade +# +cffi==1.16.0 + # via cryptography +chem==1.3.0 + # via -r requirements/edx-sandbox/base.in +click==8.1.6 + # via + # -c requirements/edx-sandbox/../constraints.txt + # nltk +codejail-includes==1.0.0 + # via -r requirements/edx-sandbox/base.in +contourpy==1.1.1 + # via matplotlib +cryptography==42.0.7 + # via -r requirements/edx-sandbox/base.in +cycler==0.12.1 + # via matplotlib +fonttools==4.51.0 + # via matplotlib +importlib-resources==6.4.0 + # via matplotlib +joblib==1.4.2 + # via nltk +kiwisolver==1.4.5 + # via matplotlib +lxml==4.9.4 + # via + # -c requirements/edx-sandbox/../constraints.txt + # -r requirements/edx-sandbox/base.in + # openedx-calc +markupsafe==2.1.5 + # via + # chem + # openedx-calc +matplotlib==3.7.5 + # via -r requirements/edx-sandbox/base.in +mpmath==1.3.0 + # via sympy +networkx==3.1 + # via -r requirements/edx-sandbox/base.in +nltk==3.8.1 + # via + # -r requirements/edx-sandbox/base.in + # chem +numpy==1.24.4 + # via + # chem + # contourpy + # matplotlib + # openedx-calc + # scipy +openedx-calc==3.1.0 + # via -r requirements/edx-sandbox/base.in +packaging==24.0 + # via matplotlib +pillow==10.3.0 + # via matplotlib +pycparser==2.22 + # via cffi +pyparsing==3.1.2 + # via + # -r requirements/edx-sandbox/base.in + # chem + # matplotlib + # openedx-calc +python-dateutil==2.9.0.post0 + # via matplotlib +random2==1.0.2 + # via -r requirements/edx-sandbox/base.in +regex==2024.4.28 + # via nltk +scipy==1.10.1 + # via + # -r requirements/edx-sandbox/base.in + # chem + # openedx-calc +six==1.16.0 + # via + # codejail-includes + # python-dateutil +sympy==1.12 + # via + # -r requirements/edx-sandbox/base.in + # openedx-calc +tqdm==4.66.4 + # via nltk +zipp==3.18.1 + # via importlib-resources From 4599e45b2e9c2c05d39ed1ba9fe38f44ddb1f7ce Mon Sep 17 00:00:00 2001 From: Glib Glugovskiy Date: Thu, 9 May 2024 18:52:36 +0300 Subject: [PATCH 080/113] feat: emit passing status updated events for badging (#34749) Introduce emission of the COURSE_PASSING_STATUS_UPDATED as well as CCX_COURSE_PASSING_STATUS_UPDATED events, that are groundwork for the new Credly integration and the future badging initiative. Product GH ticket for tracking - openedx/platform-roadmap#280 --- cms/envs/common.py | 36 +++++ lms/djangoapps/grades/events.py | 65 +++++++- lms/djangoapps/grades/tests/test_events.py | 164 ++++++++++++++++++++- lms/envs/common.py | 40 +++++ 4 files changed, 297 insertions(+), 8 deletions(-) diff --git a/cms/envs/common.py b/cms/envs/common.py index e684946d74af..e505f8245726 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -583,6 +583,14 @@ # See annotations in lms/envs/common.py for details. 'ENABLE_BLAKE2B_HASHING': False, + + # .. toggle_name: FEATURES['BADGES_ENABLED'] + # .. toggle_implementation: DjangoSetting + # .. toggle_default: False + # .. toggle_description: Set to True to enable the Badges feature. + # .. toggle_use_cases: open_edx + # .. toggle_creation_date: 2024-04-10 + 'BADGES_ENABLED': False, } # .. toggle_name: ENABLE_COPPA_COMPLIANCE @@ -2881,6 +2889,10 @@ def _should_send_xblock_events(settings): return settings.FEATURES['ENABLE_SEND_XBLOCK_LIFECYCLE_EVENTS_OVER_BUS'] +def _should_send_learning_badge_events(settings): + return settings.FEATURES['BADGES_ENABLED'] + + # .. setting_name: EVENT_BUS_PRODUCER_CONFIG # .. setting_default: all events disabled # .. setting_description: Dictionary of event_types mapped to dictionaries of topic to topic-related configuration. @@ -2930,6 +2942,18 @@ def _should_send_xblock_events(settings): 'learning-certificate-lifecycle': {'event_key_field': 'certificate.course.course_key', 'enabled': False}, }, + "org.openedx.learning.course.passing.status.updated.v1": { + "learning-badges-lifecycle": { + "event_key_field": "course_passing_status.course.course_key", + "enabled": _should_send_learning_badge_events, + }, + }, + "org.openedx.learning.ccx.course.passing.status.updated.v1": { + "learning-badges-lifecycle": { + "event_key_field": "course_passing_status.course.ccx_course_key", + "enabled": _should_send_learning_badge_events, + }, + }, } @@ -2940,6 +2964,18 @@ def _should_send_xblock_events(settings): derived_collection_entry('EVENT_BUS_PRODUCER_CONFIG', 'org.openedx.content_authoring.xblock.deleted.v1', 'course-authoring-xblock-lifecycle', 'enabled') +derived_collection_entry( + "EVENT_BUS_PRODUCER_CONFIG", + "org.openedx.learning.course.passing.status.updated.v1", + "learning-badges-lifecycle", + "enabled", +) +derived_collection_entry( + "EVENT_BUS_PRODUCER_CONFIG", + "org.openedx.learning.ccx.course.passing.status.updated.v1", + "learning-badges-lifecycle", + "enabled", +) ################### Authoring API ###################### diff --git a/lms/djangoapps/grades/events.py b/lms/djangoapps/grades/events.py index 90279a3e69fe..51d1b13702f0 100644 --- a/lms/djangoapps/grades/events.py +++ b/lms/djangoapps/grades/events.py @@ -6,6 +6,15 @@ from crum import get_current_user from django.conf import settings from eventtracking import tracker +from openedx_events.learning.data import ( + CcxCourseData, + CcxCoursePassingStatusData, + CourseData, + CoursePassingStatusData, + UserData, + UserPersonalData +) +from openedx_events.learning.signals import CCX_COURSE_PASSING_STATUS_UPDATED, COURSE_PASSING_STATUS_UPDATED from common.djangoapps.course_modes.models import CourseMode from common.djangoapps.student.models import CourseEnrollment @@ -174,8 +183,8 @@ def course_grade_passed_first_time(user_id, course_id): def course_grade_now_passed(user, course_id): """ - Emits an edx.course.grade.now_passed event - with data from the course and user passed now . + Emits an edx.course.grade.now_passed and passing status updated events + with data from the course and user passed now. """ event_name = COURSE_GRADE_NOW_PASSED_EVENT_TYPE context = contexts.course_context_from_course_id(course_id) @@ -190,11 +199,13 @@ def course_grade_now_passed(user, course_id): } ) + _emit_course_passing_status_update(user, course_id, is_passing=True) + def course_grade_now_failed(user, course_id): """ - Emits an edx.course.grade.now_failed event - with data from the course and user failed now . + Emits an edx.course.grade.now_failed and passing status updated events + with data from the course and user failed now. """ event_name = COURSE_GRADE_NOW_FAILED_EVENT_TYPE context = contexts.course_context_from_course_id(course_id) @@ -209,6 +220,8 @@ def course_grade_now_failed(user, course_id): } ) + _emit_course_passing_status_update(user, course_id, is_passing=False) + def fire_segment_event_on_course_grade_passed_first_time(user_id, course_locator): """ @@ -258,3 +271,47 @@ def fire_segment_event_on_course_grade_passed_first_time(user_id, course_locator ) log.info("Segment event fired for passed learners. Event: [{}], Data: [{}]".format(event_name, event_properties)) + + +def _emit_course_passing_status_update(user, course_id, is_passing): + """ + Emit course passing status event according to the course type. + The status of event is determined by is_passing parameter. + """ + if hasattr(course_id, 'ccx'): + CCX_COURSE_PASSING_STATUS_UPDATED.send_event( + course_passing_status=CcxCoursePassingStatusData( + is_passing=is_passing, + user=UserData( + pii=UserPersonalData( + username=user.username, + email=user.email, + name=user.get_full_name(), + ), + id=user.id, + is_active=user.is_active, + ), + course=CcxCourseData( + ccx_course_key=course_id, + master_course_key=course_id.to_course_locator(), + ), + ) + ) + else: + COURSE_PASSING_STATUS_UPDATED.send_event( + course_passing_status=CoursePassingStatusData( + is_passing=is_passing, + user=UserData( + pii=UserPersonalData( + username=user.username, + email=user.email, + name=user.get_full_name(), + ), + id=user.id, + is_active=user.is_active, + ), + course=CourseData( + course_key=course_id, + ), + ) + ) diff --git a/lms/djangoapps/grades/tests/test_events.py b/lms/djangoapps/grades/tests/test_events.py index b7843dfc1c1c..eac8cc9a4a70 100644 --- a/lms/djangoapps/grades/tests/test_events.py +++ b/lms/djangoapps/grades/tests/test_events.py @@ -4,16 +4,29 @@ from unittest import mock +from ccx_keys.locator import CCXLocator from django.utils.timezone import now from openedx_events.learning.data import ( + CcxCourseData, + CcxCoursePassingStatusData, CourseData, - PersistentCourseGradeData + CoursePassingStatusData, + PersistentCourseGradeData, + UserData, + UserPersonalData +) +from openedx_events.learning.signals import ( + CCX_COURSE_PASSING_STATUS_UPDATED, + COURSE_PASSING_STATUS_UPDATED, + PERSISTENT_GRADE_SUMMARY_CHANGED ) -from openedx_events.learning.signals import PERSISTENT_GRADE_SUMMARY_CHANGED from openedx_events.tests.utils import OpenEdxEventsTestMixin -from common.djangoapps.student.tests.factories import UserFactory +from common.djangoapps.student.tests.factories import AdminFactory, UserFactory +from lms.djangoapps.ccx.models import CustomCourseForEdX +from lms.djangoapps.grades.course_grade_factory import CourseGradeFactory from lms.djangoapps.grades.models import PersistentCourseGrade +from lms.djangoapps.grades.tests.utils import mock_passing_grade from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase from xmodule.modulestore.tests.factories import CourseFactory @@ -94,5 +107,148 @@ def test_persistent_grade_event_emitted(self): passed_timestamp=grade.passed_timestamp ) }, - event_receiver.call_args.kwargs + event_receiver.call_args.kwargs, + ) + + +class CoursePassingStatusEventsTest(SharedModuleStoreTestCase, OpenEdxEventsTestMixin): + """ + Tests for Open edX passing status update event. + """ + ENABLED_OPENEDX_EVENTS = [ + "org.openedx.learning.course.passing.status.updated.v1", + ] + + @classmethod + def setUpClass(cls): + """ + Set up class method for the Test class. + """ + super().setUpClass() + cls.start_events_isolation() + + def setUp(self): + super().setUp() + self.course = CourseFactory.create() + self.user = UserFactory.create() + self.receiver_called = False + + def _event_receiver_side_effect(self, **kwargs): + """ + Used show that the Open edX Event was called by the Django signal handler. + """ + self.receiver_called = True + + def test_course_passing_status_updated_emitted(self): + """ + Test whether passing status updated event is sent after the grade is being updated for a user. + """ + event_receiver = mock.Mock(side_effect=self._event_receiver_side_effect) + COURSE_PASSING_STATUS_UPDATED.connect(event_receiver) + grade_factory = CourseGradeFactory() + + with mock_passing_grade(): + grade_factory.update(self.user, self.course) + + self.assertTrue(self.receiver_called) + self.assertDictContainsSubset( + { + "signal": COURSE_PASSING_STATUS_UPDATED, + "sender": None, + "course_passing_status": CoursePassingStatusData( + is_passing=True, + user=UserData( + pii=UserPersonalData( + username=self.user.username, + email=self.user.email, + name=self.user.get_full_name(), + ), + id=self.user.id, + is_active=self.user.is_active, + ), + course=CourseData( + course_key=self.course.id, + ), + ), + }, + event_receiver.call_args.kwargs, + ) + + +class CCXCoursePassingStatusEventsTest( + SharedModuleStoreTestCase, OpenEdxEventsTestMixin +): + """ + Tests for Open edX passing status update event in a CCX course. + """ + ENABLED_OPENEDX_EVENTS = [ + "org.openedx.learning.ccx.course.passing.status.updated.v1", + ] + + @classmethod + def setUpClass(cls): + """ + Set up class method for the Test class. + """ + super().setUpClass() + cls.start_events_isolation() + + def setUp(self): + super().setUp() + self.course = CourseFactory.create() + self.user = UserFactory.create() + self.coach = AdminFactory.create() + self.ccx = ccx = CustomCourseForEdX( + course_id=self.course.id, display_name="Test CCX", coach=self.coach + ) + ccx.save() + self.ccx_locator = CCXLocator.from_course_locator(self.course.id, ccx.id) + + self.receiver_called = False + + def _event_receiver_side_effect(self, **kwargs): + """ + Used show that the Open edX Event was called by the Django signal handler. + """ + self.receiver_called = True + + def test_ccx_course_passing_status_updated_emitted(self): + """ + Test whether passing status updated event is sent after the grade is being updated in CCX course. + """ + event_receiver = mock.Mock(side_effect=self._event_receiver_side_effect) + CCX_COURSE_PASSING_STATUS_UPDATED.connect(event_receiver) + grade_factory = CourseGradeFactory() + + with mock_passing_grade(): + grade_factory.update(self.user, self.store.get_course(self.ccx_locator)) + + self.assertTrue(self.receiver_called) + self.assertDictContainsSubset( + { + "signal": CCX_COURSE_PASSING_STATUS_UPDATED, + "sender": None, + "course_passing_status": CcxCoursePassingStatusData( + is_passing=True, + user=UserData( + pii=UserPersonalData( + username=self.user.username, + email=self.user.email, + name=self.user.get_full_name(), + ), + id=self.user.id, + is_active=self.user.is_active, + ), + course=CcxCourseData( + ccx_course_key=self.ccx_locator, + master_course_key=self.course.id, + display_name="", + coach_email="", + start=None, + end=None, + max_students_allowed=self.ccx.max_student_enrollments_allowed, + ), + ), + }, + event_receiver.call_args.kwargs, ) diff --git a/lms/envs/common.py b/lms/envs/common.py index 6acf880d4f4a..5f8714ca9cf7 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -1076,6 +1076,15 @@ # .. toggle_warning: For consistency, keep the value in sync with the setting of the same name in the LMS and CMS. # .. toggle_tickets: https://github.com/openedx/edx-platform/pull/34442 'ENABLE_BLAKE2B_HASHING': False, + + # .. toggle_name: FEATURES['BADGES_ENABLED'] + # .. toggle_implementation: DjangoSetting + # .. toggle_default: False + # .. toggle_description: Set to True to enable badges functionality. + # .. toggle_use_cases: open_edx + # .. toggle_creation_date: 2024-04-02 + # .. toggle_target_removal_date: None + 'BADGES_ENABLED': False, } # Specifies extra XBlock fields that should available when requested via the Course Blocks API @@ -5447,7 +5456,12 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring def _should_send_certificate_events(settings): return settings.FEATURES['SEND_LEARNING_CERTIFICATE_LIFECYCLE_EVENTS_TO_BUS'] + #### Event bus producing #### + +def _should_send_learning_badge_events(settings): + return settings.FEATURES['BADGES_ENABLED'] + # .. setting_name: EVENT_BUS_PRODUCER_CONFIG # .. setting_default: all events disabled # .. setting_description: Dictionary of event_types mapped to dictionaries of topic to topic-related configuration. @@ -5520,11 +5534,37 @@ def _should_send_certificate_events(settings): 'course-authoring-xblock-lifecycle': {'event_key_field': 'xblock_info.usage_key', 'enabled': False}, }, + "org.openedx.learning.course.passing.status.updated.v1": { + "learning-badges-lifecycle": { + "event_key_field": "course_passing_status.course.course_key", + "enabled": _should_send_learning_badge_events, + }, + }, + "org.openedx.learning.ccx.course.passing.status.updated.v1": { + "learning-badges-lifecycle": { + "event_key_field": "course_passing_status.course.ccx_course_key", + "enabled": _should_send_learning_badge_events, + }, + }, } derived_collection_entry('EVENT_BUS_PRODUCER_CONFIG', 'org.openedx.learning.certificate.created.v1', 'learning-certificate-lifecycle', 'enabled') derived_collection_entry('EVENT_BUS_PRODUCER_CONFIG', 'org.openedx.learning.certificate.revoked.v1', 'learning-certificate-lifecycle', 'enabled') + +derived_collection_entry( + "EVENT_BUS_PRODUCER_CONFIG", + "org.openedx.learning.course.passing.status.updated.v1", + "learning-badges-lifecycle", + "enabled", +) +derived_collection_entry( + "EVENT_BUS_PRODUCER_CONFIG", + "org.openedx.learning.ccx.course.passing.status.updated.v1", + "learning-badges-lifecycle", + "enabled", +) + BEAMER_PRODUCT_ID = "" #### Survey Report #### From 75ac5e3f049081971e8386229bce6f35287effd4 Mon Sep 17 00:00:00 2001 From: alex-sheehan-edx <67655836+alex-sheehan-edx@users.noreply.github.com> Date: Wed, 8 May 2024 20:48:23 +0000 Subject: [PATCH 081/113] feat: Upgrade Python dependency edx-enterprise enterprise version bump 4.17.6 Commit generated by workflow `openedx/edx-platform/.github/workflows/upgrade-one-python-dependency.yml@refs/heads/master` --- requirements/constraints.txt | 2 +- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/doc.txt | 2 +- requirements/edx/testing.txt | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index f0c563113acc..b9c135b16e55 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -23,7 +23,7 @@ click>=8.0,<9.0 # The team that owns this package will manually bump this package rather than having it pulled in automatically. # This is to allow them to better control its deployment and to do it in a process that works better # for them. -edx-enterprise==4.17.4 +edx-enterprise==4.17.8 # Stay on LTS version, remove once this is added to common constraint Django<5.0 diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index e466bb8d786e..0c63a94998ed 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -480,7 +480,7 @@ edx-drf-extensions==10.3.0 # edx-when # edxval # openedx-learning -edx-enterprise==4.17.4 +edx-enterprise==4.17.8 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/kernel.in diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index c23a0bca6299..53e3f7a1c7a7 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -764,7 +764,7 @@ edx-drf-extensions==10.3.0 # edx-when # edxval # openedx-learning -edx-enterprise==4.17.4 +edx-enterprise==4.17.8 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index c687ec7720ad..d553ab6ae24d 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -558,7 +558,7 @@ edx-drf-extensions==10.3.0 # edx-when # edxval # openedx-learning -edx-enterprise==4.17.4 +edx-enterprise==4.17.8 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 385f339ad55c..314f28d1cb02 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -591,7 +591,7 @@ edx-drf-extensions==10.3.0 # edx-when # edxval # openedx-learning -edx-enterprise==4.17.4 +edx-enterprise==4.17.8 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt From ab66687a1960aedb05cbf117413cf13a76e808bc Mon Sep 17 00:00:00 2001 From: sundasnoreen12 Date: Tue, 7 May 2024 12:36:47 +0500 Subject: [PATCH 082/113] fix: added fix based on recent master changes --- .../notifications/digest_header.html | 63 ++++++++++--------- 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/openedx/core/djangoapps/notifications/templates/notifications/digest_header.html b/openedx/core/djangoapps/notifications/templates/notifications/digest_header.html index 82077e37c0fd..1f0a533aea54 100644 --- a/openedx/core/djangoapps/notifications/templates/notifications/digest_header.html +++ b/openedx/core/djangoapps/notifications/templates/notifications/digest_header.html @@ -38,35 +38,40 @@
    {% for update in email_digest_updates %} - - {% endfor %} - + {% if forloop.last or forloop.revcounter0 == 1 %} + {% if email_digest_updates|length|add:'1'|divisibleby:3 and not forloop.last%} + + {% elif email_digest_updates|length|add:'2'|divisibleby:3 and forloop.last%} + + {% endif %} + {% endif %} + + {% if forloop.counter|divisibleby:3 %} + + {% endif %} + {% endfor %}

    - {{update.type}} + {{update.title}}
    -

    - - - - - - - - - -
    - {{update.count}} -
    - {{update.title}} -
    -

    -
    +

    + + + + + + + + + +
    + {{update.count}} +
    + {{update.title}} +
    +

    +

    -

    - - - - - - - - - -
    - {{update.count}} -
    - {{update.title}} -
    -

    +
    + + + + + + + + + +
    + {{update.count}} +
    + {{update.title}} +
    From 7789923d21836e2b537cac2e80e3cf4c986f3b77 Mon Sep 17 00:00:00 2001 From: sundasnoreen12 Date: Fri, 10 May 2024 12:59:03 +0500 Subject: [PATCH 084/113] refactor: added tr tags --- .../notifications/templates/notifications/digest_header.html | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/openedx/core/djangoapps/notifications/templates/notifications/digest_header.html b/openedx/core/djangoapps/notifications/templates/notifications/digest_header.html index 9791201b5dc3..9d731e35528a 100644 --- a/openedx/core/djangoapps/notifications/templates/notifications/digest_header.html +++ b/openedx/core/djangoapps/notifications/templates/notifications/digest_header.html @@ -55,7 +55,9 @@ {% if forloop.counter|divisibleby:3 %} - + + {% elif forloop.last %} + {% endif %} {% endfor %} From 2c4e039313312e9baad95f85e816509e546b6b2f Mon Sep 17 00:00:00 2001 From: sundasnoreen12 Date: Fri, 10 May 2024 14:50:30 +0500 Subject: [PATCH 085/113] fix: moved tr outside forloop --- .../notifications/templates/notifications/digest_header.html | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/openedx/core/djangoapps/notifications/templates/notifications/digest_header.html b/openedx/core/djangoapps/notifications/templates/notifications/digest_header.html index 9d731e35528a..7957524e8ae2 100644 --- a/openedx/core/djangoapps/notifications/templates/notifications/digest_header.html +++ b/openedx/core/djangoapps/notifications/templates/notifications/digest_header.html @@ -56,10 +56,9 @@ {% if forloop.counter|divisibleby:3 %} - {% elif forloop.last %} - {% endif %} {% endfor %} + From 0d4adaa5d70364fe9ff0857b15cbc58cf952f584 Mon Sep 17 00:00:00 2001 From: Muhammad Adeel Tajamul <77053848+muhammadadeeltajamul@users.noreply.github.com> Date: Fri, 10 May 2024 17:52:19 +0500 Subject: [PATCH 086/113] fix: updated ace template for notifications email digest (#34778) --- .../edx_ace/email_digest/email/body.html | 23 ++++++++++++++++++- .../edx_ace/email_digest/email/body.txt | 23 +------------------ .../edx_ace/email_digest/email/head.html | 4 +++- 3 files changed, 26 insertions(+), 24 deletions(-) diff --git a/openedx/core/djangoapps/notifications/templates/notifications/edx_ace/email_digest/email/body.html b/openedx/core/djangoapps/notifications/templates/notifications/edx_ace/email_digest/email/body.html index c847f75e4f7b..e45fd8029e2d 100644 --- a/openedx/core/djangoapps/notifications/templates/notifications/edx_ace/email_digest/email/body.html +++ b/openedx/core/djangoapps/notifications/templates/notifications/edx_ace/email_digest/email/body.html @@ -1 +1,22 @@ -<%page expression_filter="h"/> +
    + + + + + + + + + + + + +
    + {% include 'notifications/digest_header.html' %} +
    + {% include 'notifications/digest_content.html' %} +
    + {% include 'notifications/digest_footer.html' %} +
    + +
    diff --git a/openedx/core/djangoapps/notifications/templates/notifications/edx_ace/email_digest/email/body.txt b/openedx/core/djangoapps/notifications/templates/notifications/edx_ace/email_digest/email/body.txt index e45fd8029e2d..3bbe26faf772 100644 --- a/openedx/core/djangoapps/notifications/templates/notifications/edx_ace/email_digest/email/body.txt +++ b/openedx/core/djangoapps/notifications/templates/notifications/edx_ace/email_digest/email/body.txt @@ -1,22 +1 @@ -
    - - - - - - - - - - - - -
    - {% include 'notifications/digest_header.html' %} -
    - {% include 'notifications/digest_content.html' %} -
    - {% include 'notifications/digest_footer.html' %} -
    - -
    +{{ digest_frequency }} Notifications Digest for {% if digest_frequency == "Weekly" %}the Week of {% endif %}{{ start_date }} diff --git a/openedx/core/djangoapps/notifications/templates/notifications/edx_ace/email_digest/email/head.html b/openedx/core/djangoapps/notifications/templates/notifications/edx_ace/email_digest/email/head.html index c847f75e4f7b..8d63916b7a49 100644 --- a/openedx/core/djangoapps/notifications/templates/notifications/edx_ace/email_digest/email/head.html +++ b/openedx/core/djangoapps/notifications/templates/notifications/edx_ace/email_digest/email/head.html @@ -1 +1,3 @@ -<%page expression_filter="h"/> + +{{ platform_name }} + From d3ffb3e88270ee6ac8a7b45eeff9d5aec4a5726d Mon Sep 17 00:00:00 2001 From: Ahtisham Shahid Date: Mon, 13 May 2024 16:50:37 +0500 Subject: [PATCH 087/113] fix: get topics for blocks with discussions enabled (#34732) --- lms/djangoapps/discussion/rest_api/api.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/lms/djangoapps/discussion/rest_api/api.py b/lms/djangoapps/discussion/rest_api/api.py index 27a86c3a0192..7244127dc2a3 100644 --- a/lms/djangoapps/discussion/rest_api/api.py +++ b/lms/djangoapps/discussion/rest_api/api.py @@ -13,7 +13,6 @@ from urllib.parse import urlencode, urlunparse from pytz import UTC - from django.conf import settings from django.contrib.auth import get_user_model from django.core.exceptions import ValidationError @@ -34,7 +33,6 @@ ) from lms.djangoapps.course_api.blocks.api import get_blocks -from lms.djangoapps.course_blocks.api import get_course_blocks from lms.djangoapps.courseware.courses import get_course_with_access from lms.djangoapps.courseware.exceptions import CourseAccessRedirect from lms.djangoapps.discussion.toggles import ENABLE_DISCUSSIONS_MFE @@ -82,6 +80,7 @@ from openedx.core.djangoapps.user_api.accounts.api import get_account_settings from openedx.core.lib.exceptions import CourseNotFoundError, DiscussionNotFoundError, PageNotFoundError from xmodule.course_block import CourseBlock +from xmodule.modulestore import ModuleStoreEnum from xmodule.modulestore.django import modulestore from xmodule.tabs import CourseTabList @@ -131,7 +130,6 @@ is_posting_allowed ) - User = get_user_model() ThreadType = Literal["discussion", "question"] @@ -418,6 +416,7 @@ def sort_categories(category_list): Required arguments: category_list -- list of categories. """ + def convert(text): if text.isdigit(): return int(text) @@ -697,11 +696,19 @@ def get_course_topics_v2( FORUM_ROLE_ADMINISTRATOR, ] ).exists() - course_blocks = get_course_blocks(user, store.make_course_usage_key(course_key)) - accessible_vertical_keys = [ - block for block in course_blocks.get_block_keys() - if block.block_type == 'vertical' - ] + [None] + + with store.branch_setting(ModuleStoreEnum.Branch.draft_preferred, course_key): + blocks = store.get_items( + course_key, + qualifiers={'category': 'vertical'}, + fields=['usage_key', 'discussion_enabled', 'display_name'], + ) + accessible_vertical_keys = [] + for block in blocks: + if block.discussion_enabled and (not block.visible_to_staff_only or user_is_privileged): + accessible_vertical_keys.append(block.usage_key) + accessible_vertical_keys.append(None) + topics_query = DiscussionTopicLink.objects.filter( context_key=course_key, provider_id=provider_type, From 15caa9746fe59960409a8daa09f24840f2524952 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Mon, 13 May 2024 09:48:18 -0400 Subject: [PATCH 088/113] refactor: Completely remove Blockstore (#34739) Blockstore and all of its (experimental) functionality has been replaced with openedx-learning, aka "Learning Core". This commit uninstalls the now-unused openedx-blockstore package and removes all dangling references to it. Note: This also removes the `copy_library_from_v1_to_v2` management command, which has been broken ever since we switched from Blockstore to Learning Core. Part of this DEPR: https://github.com/openedx/public-engineering/issues/238 --- cms/djangoapps/contentstore/config/waffle.py | 2 +- .../commands/copy_libraries_from_v1_to_v2.py | 121 ----------- cms/djangoapps/contentstore/tasks.py | 113 +--------- cms/envs/common.py | 45 +--- cms/envs/devstack.py | 3 - cms/envs/production.py | 5 - cms/envs/test.py | 15 -- .../content_libraries/xblock_iframe.html | 6 +- common/djangoapps/track/contexts.py | 2 +- docs/concepts/extension_points.rst | 2 +- lms/djangoapps/edxnotes/tests.py | 4 +- lms/envs/common.py | 75 +------ lms/envs/devstack.py | 3 - lms/envs/production.py | 5 - lms/envs/test.py | 17 -- lms/urls.py | 2 +- .../content/search/tests/test_handlers.py | 7 +- .../core/djangoapps/content_libraries/api.py | 24 +-- .../core/djangoapps/content_libraries/apps.py | 2 +- .../djangoapps/content_libraries/constants.py | 3 - .../content_libraries/library_context.py | 2 +- .../commands/content_libraries_import.py | 2 +- .../migrations/0008_auto_20210818_2148.py | 2 +- .../djangoapps/content_libraries/models.py | 10 +- .../content_libraries/permissions.py | 2 +- .../content_libraries/serializers.py | 11 - .../djangoapps/content_libraries/tasks.py | 26 +-- .../content_libraries/tests/base.py | 9 +- .../tests/test_content_libraries.py | 12 +- .../content_libraries/tests/test_runtime.py | 49 +---- .../tests/test_static_assets.py | 8 +- .../content_libraries/tests/test_views_lti.py | 2 +- .../djangoapps/content_libraries/views.py | 13 +- .../content_tagging/tests/test_tasks.py | 2 - openedx/core/djangoapps/olx_rest_api/views.py | 8 +- openedx/core/djangoapps/xblock/README.rst | 5 + openedx/core/djangoapps/xblock/apps.py | 18 +- .../learning_context/learning_context.py | 11 +- .../xblock/runtime/learning_core_runtime.py | 8 +- .../core/djangoapps/xblock/runtime/runtime.py | 10 +- .../core/djangoapps/xblock/runtime/shims.py | 7 +- openedx/core/lib/blockstore_api/__init__.py | 51 ----- openedx/core/lib/blockstore_api/db_routers.py | 60 ------ .../core/lib/blockstore_api/tests/__init__.py | 0 openedx/core/lib/blockstore_api/tests/base.py | 36 ---- .../tests/test_blockstore_api.py | 201 ------------------ openedx/core/lib/xblock_serializer/api.py | 8 +- .../lib/xblock_serializer/block_serializer.py | 12 +- .../core/lib/xblock_serializer/test_api.py | 22 +- openedx/core/lib/xblock_serializer/utils.py | 6 +- requirements/edx/base.txt | 29 +-- requirements/edx/development.txt | 28 --- requirements/edx/doc.txt | 30 +-- requirements/edx/github.in | 10 +- requirements/edx/kernel.in | 1 - requirements/edx/testing.txt | 30 +-- xmodule/README.rst | 6 +- xmodule/html_block.py | 2 +- xmodule/library_tools.py | 2 +- xmodule/raw_block.py | 2 +- xmodule/tests/test_library_tools.py | 2 +- xmodule/video_block/transcripts_utils.py | 58 ++--- xmodule/video_block/video_handlers.py | 2 +- xmodule/xml_block.py | 2 +- 64 files changed, 186 insertions(+), 1087 deletions(-) delete mode 100644 cms/djangoapps/contentstore/management/commands/copy_libraries_from_v1_to_v2.py delete mode 100644 openedx/core/lib/blockstore_api/__init__.py delete mode 100644 openedx/core/lib/blockstore_api/db_routers.py delete mode 100644 openedx/core/lib/blockstore_api/tests/__init__.py delete mode 100644 openedx/core/lib/blockstore_api/tests/base.py delete mode 100644 openedx/core/lib/blockstore_api/tests/test_blockstore_api.py diff --git a/cms/djangoapps/contentstore/config/waffle.py b/cms/djangoapps/contentstore/config/waffle.py index b10566bb04cb..49dbab571539 100644 --- a/cms/djangoapps/contentstore/config/waffle.py +++ b/cms/djangoapps/contentstore/config/waffle.py @@ -35,7 +35,7 @@ # .. toggle_creation_date: 2020-08-03 # .. toggle_target_removal_date: 2020-12-31 # .. toggle_warning: Also set settings.LIBRARY_AUTHORING_MICROFRONTEND_URL and ENABLE_LIBRARY_AUTHORING_MICROFRONTEND. -# .. toggle_tickets: https://openedx.atlassian.net/wiki/spaces/COMM/pages/1545011241/BD-14+Blockstore+Powered+Content+Libraries+Taxonomies +# .. toggle_tickets: https://openedx.atlassian.net/wiki/spaces/OEPM/pages/4106944527/Libraries+Relaunch+Proposal+For+Product+Review REDIRECT_TO_LIBRARY_AUTHORING_MICROFRONTEND = WaffleFlag( f'{WAFFLE_NAMESPACE}.library_authoring_mfe', __name__, LOG_PREFIX ) diff --git a/cms/djangoapps/contentstore/management/commands/copy_libraries_from_v1_to_v2.py b/cms/djangoapps/contentstore/management/commands/copy_libraries_from_v1_to_v2.py deleted file mode 100644 index 3be5cab11c23..000000000000 --- a/cms/djangoapps/contentstore/management/commands/copy_libraries_from_v1_to_v2.py +++ /dev/null @@ -1,121 +0,0 @@ -"""A Command to Copy or uncopy V1 Content Libraries entires to be stored as v2 content libraries.""" - -import logging -import csv -from textwrap import dedent - -from django.core.management import BaseCommand, CommandError - -from opaque_keys.edx.keys import CourseKey -from opaque_keys.edx.locator import LibraryLocator - -from xmodule.modulestore.django import modulestore - - -from celery import group - -from cms.djangoapps.contentstore.tasks import create_v2_library_from_v1_library, delete_v2_library_from_v1_library - -from .prompt import query_yes_no - -log = logging.getLogger(__name__) - - -class Command(BaseCommand): - """ - Copy or uncopy V1 Content Libraries (default all) entires to be stored as v2 content libraries. - First Specify the uuid for the collection to store the content libraries in. - Specfiy --all for all libraries, library ids for specific libraries, - and -- file followed by the path for a list of libraries from a file. - - Example usage: - $ ./manage.py cms copy_libraries_from_v1_to_v2 'collection_uuid' --all - $ ./manage.py cms copy_libraries_from_v1_to_v2 'collection_uuid' --all --uncopy - $ ./manage.py cms copy_libraries_from_v1_to_v2 'collection_uuid 'library-v1:edX+DemoX+Better_Library' - $ ./manage.py cms copy_libraries_from_v1_to_v2 'collection_uuid 'library-v1:edX+DemoX+Better_Library' --uncopy - $ ./manage.py cms copy_libraries_from_v1_to_v2 - '11111111-2111-4111-8111-111111111111' - './list_of--library-locators.csv --all - - Note: - This Command Also produces an "output file" which contains the mapping of locators and the status of the copy. - """ - - help = dedent(__doc__) - CONFIRMATION_PROMPT = "Reindexing all libraries might be a time consuming operation. Do you want to continue?" - - def add_arguments(self, parser): - """arguements for command""" - - parser.add_argument( - 'collection_uuid', - type=str, - help='the uuid for the collection to create the content library in.' - ) - parser.add_argument( - 'output_csv', - type=str, - nargs='?', - default=None, - help='a file path to write the tasks output to. Without this the result is simply logged.' - ) - - parser.add_argument( - '--all', - action='store_true', - dest='all', - help='Copy all libraries' - ) - parser.add_argument( - '--uncopy', - action='store_true', - dest='uncopy', - help='Delete libraries specified' - ) - parser.add_argument( - 'library_ids', - nargs='*', - default=[], - help='a space-seperated list of v1 library ids to copy' - ) - - def _parse_library_key(self, raw_value): - """ Parses library key from string """ - result = CourseKey.from_string(raw_value) - - if not isinstance(result, LibraryLocator): - raise CommandError(f"Argument {raw_value} is not a library key") - return result - - def handle(self, *args, **options): # lint-amnesty, pylint: disable=unused-argument - """Parse args and generate tasks for copying content.""" - - if (not options['library_ids'] and not options['all']) or (options['library_ids'] and options['all']): - raise CommandError("copy_libraries_from_v1_to_v2 requires one or more s or the --all flag.") - - if options['all']: - store = modulestore() - if query_yes_no(self.CONFIRMATION_PROMPT, default="no"): - v1_library_keys = [ - library.location.library_key.replace(branch=None) for library in store.get_libraries() - ] - else: - return - else: - v1_library_keys = list(map(self._parse_library_key, options['library_ids'])) - - create_library_task_group = group([ - delete_v2_library_from_v1_library.s(str(v1_library_key), options['collection_uuid']) - if options['uncopy'] - else create_v2_library_from_v1_library.s(str(v1_library_key), options['collection_uuid']) - for v1_library_key in v1_library_keys - ]) - - group_result = create_library_task_group.apply_async().get() - if options['output_csv']: - with open(options['output_csv'], 'w', encoding='utf-8', newline='') as file: - output_writer = csv.writer(file) - output_writer.writerow(["v1_library_id", "v2_library_id", "status", "error_msg"]) - for result in group_result: - output_writer.writerow(result.values()) - log.info(group_result) diff --git a/cms/djangoapps/contentstore/tasks.py b/cms/djangoapps/contentstore/tasks.py index faaf9dc7e1ca..bb220c371711 100644 --- a/cms/djangoapps/contentstore/tasks.py +++ b/cms/djangoapps/contentstore/tasks.py @@ -19,7 +19,6 @@ from django.contrib.auth import get_user_model from django.core.exceptions import SuspiciousOperation from django.core.files import File -from django.db.transaction import atomic from django.test import RequestFactory from django.utils.text import get_valid_filename from edx_django_utils.monitoring import ( @@ -31,7 +30,7 @@ from olxcleaner.exceptions import ErrorLevel from olxcleaner.reporting import report_error_summary, report_errors from opaque_keys.edx.keys import CourseKey -from opaque_keys.edx.locator import LibraryLocator, LibraryLocatorV2 +from opaque_keys.edx.locator import LibraryLocator from organizations.api import add_organization_course, ensure_organization from organizations.exceptions import InvalidOrganizationException from organizations.models import Organization, OrganizationCourse @@ -66,7 +65,6 @@ from openedx.core.djangoapps.discussions.models import DiscussionsConfiguration, Provider from openedx.core.djangoapps.discussions.tasks import update_unit_discussion_state_from_discussion_blocks from openedx.core.djangoapps.embargo.models import CountryAccessRule, RestrictedCourse -from openedx.core.lib.blockstore_api import get_collection from openedx.core.lib.extract_archive import safe_extractall from xmodule.contentstore.django import contentstore from xmodule.course_block import CourseFields @@ -900,115 +898,6 @@ def _create_copy_content_task(v2_library_key, v1_library_key): ) -def _create_metadata(v1_library_key, collection_uuid): - """instansiate an index for the V2 lib in the collection""" - - store = modulestore() - v1_library = store.get_library(v1_library_key) - collection = get_collection(collection_uuid).uuid - # To make it easy, all converted libs are complex, meaning they can contain problems, videos, and text - library_type = 'complex' - org = _parse_organization(v1_library.location.library_key.org) - slug = v1_library.location.library_key.library - title = v1_library.display_name - # V1 libraries do not have descriptions. - description = '' - # permssions & license are most restrictive. - allow_public_learning = False - allow_public_read = False - library_license = '' # '' = ALL_RIGHTS_RESERVED - with atomic(): - return v2contentlib_api.create_library( - org, - slug, - title, - description, - allow_public_learning, - allow_public_read, - library_license, - library_type, - ) - - -@shared_task(time_limit=30) -@set_code_owner_attribute -def delete_v2_library_from_v1_library(v1_library_key_string, collection_uuid): - """ - For a V1 Library, delete the matching v2 library, where the library is the result of the copy operation - This method relys on _create_metadata failling for LibraryAlreadyExists in order to obtain the v2 slug. - """ - v1_library_key = CourseKey.from_string(v1_library_key_string) - v2_library_key = LibraryLocatorV2.from_string('lib:' + v1_library_key.org + ':' + v1_library_key.course) - - try: - v2contentlib_api.delete_library(v2_library_key) - return { - "v1_library_id": v1_library_key_string, - "v2_library_id": v2_library_key, - "status": "SUCCESS", - "msg": None - } - except Exception as error: # lint-amnesty, pylint: disable=broad-except - return { - "v1_library_id": v1_library_key_string, - "v2_library_id": v2_library_key, - "status": "FAILED", - "msg": f"Exception: {v2_library_key} did not delete: {error}" - } - - -@shared_task(time_limit=30) -@set_code_owner_attribute -def create_v2_library_from_v1_library(v1_library_key_string, collection_uuid): - """ - write the metadata, permissions, and content of a v1 library into a v2 library in the given collection. - """ - - v1_library_key = CourseKey.from_string(v1_library_key_string) - - LOGGER.info(f"Copy Library task created for library: {v1_library_key}") - - try: - v2_library_metadata = _create_metadata(v1_library_key, collection_uuid) - - except v2contentlib_api.LibraryAlreadyExists: - return { - "v1_library_id": v1_library_key_string, - "v2_library_id": None, - "status": "FAILED", - "msg": f"Exception: LibraryAlreadyExists {v1_library_key_string} aleady exists" - } - - try: - _create_copy_content_task(v2_library_metadata.key, v1_library_key) - except Exception as error: # lint-amnesty, pylint: disable=broad-except - return { - "v1_library_id": v1_library_key_string, - "v2_library_id": str(v2_library_metadata.key), - "status": "FAILED", - "msg": - f"Could not import content from {v1_library_key_string} into {str(v2_library_metadata.key)}: {str(error)}" - } - - try: - copy_v1_user_roles_into_v2_library(v2_library_metadata.key, v1_library_key) - except Exception as error: # lint-amnesty, pylint: disable=broad-except - return { - "v1_library_id": v1_library_key_string, - "v2_library_id": str(v2_library_metadata.key), - "status": "FAILED", - "msg": - f"Could not copy permissions from {v1_library_key_string} into {str(v2_library_metadata.key)}: {str(error)}" - } - - return { - "v1_library_id": v1_library_key_string, - "v2_library_id": str(v2_library_metadata.key), - "status": "SUCCESS", - "msg": None - } - - @shared_task(time_limit=30) @set_code_owner_attribute def delete_v1_library(v1_library_key_string): diff --git a/cms/envs/common.py b/cms/envs/common.py index e505f8245726..ebc101801fe2 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -115,9 +115,6 @@ ENTERPRISE_BACKEND_SERVICE_EDX_OAUTH2_SECRET, ENTERPRISE_BACKEND_SERVICE_EDX_OAUTH2_PROVIDER_URL, - # Blockstore - BUNDLE_ASSET_STORAGE_SETTINGS, - # Methods to derive settings _make_mako_template_dirs, _make_locale_paths, @@ -444,7 +441,7 @@ # .. toggle_use_cases: temporary # .. toggle_creation_date: 2020-06-20 # .. toggle_target_removal_date: 2020-12-31 - # .. toggle_tickets: https://openedx.atlassian.net/wiki/spaces/COMM/pages/1545011241/BD-14+Blockstore+Powered+Content+Libraries+Taxonomies + # .. toggle_tickets: https://openedx.atlassian.net/wiki/spaces/OEPM/pages/4106944527/Libraries+Relaunch+Proposal+For+Product+Review # .. toggle_warning: Also set settings.LIBRARY_AUTHORING_MICROFRONTEND_URL and see # REDIRECT_TO_LIBRARY_AUTHORING_MICROFRONTEND for rollout. 'ENABLE_LIBRARY_AUTHORING_MICROFRONTEND': False, @@ -1033,6 +1030,11 @@ # Paths to wrapper methods which should be applied to every XBlock's FieldData. XBLOCK_FIELD_DATA_WRAPPERS = () +# .. setting_name: XBLOCK_RUNTIME_V2_EPHEMERAL_DATA_CACHE +# .. setting_default: default +# .. setting_description: The django cache key of the cache to use for storing anonymous user state for XBlocks. +XBLOCK_RUNTIME_V2_EPHEMERAL_DATA_CACHE = 'default' + ############################ ORA 2 ############################################ # By default, don't use a file prefix @@ -1690,7 +1692,7 @@ 'cms.djangoapps.xblock_config.apps.XBlockConfig', 'cms.djangoapps.export_course_metadata.apps.ExportCourseMetadataConfig', - # New (Blockstore-based) XBlock runtime + # New (Learning-Core-based) XBlock runtime 'openedx.core.djangoapps.xblock.apps.StudioXBlockAppConfig', # Maintenance tools @@ -1873,9 +1875,6 @@ # For edx ace template tags 'edx_ace', - # Blockstore - 'blockstore.apps.bundles', - # alternative swagger generator for CMS API 'drf_spectacular', @@ -2243,25 +2242,11 @@ DATABASE_ROUTERS = [ 'openedx.core.lib.django_courseware_routers.StudentModuleHistoryExtendedRouter', - 'openedx.core.lib.blockstore_api.db_routers.BlockstoreRouter', ] ############################ Cache Configuration ############################### CACHES = { - 'blockstore': { - 'KEY_PREFIX': 'blockstore', - 'KEY_FUNCTION': 'common.djangoapps.util.memcache.safe_key', - 'LOCATION': ['localhost:11211'], - 'TIMEOUT': '86400', # This data should be long-lived for performance, BundleCache handles invalidation - 'BACKEND': 'django.core.cache.backends.memcached.PyMemcacheCache', - 'OPTIONS': { - 'no_delay': True, - 'ignore_exc': True, - 'use_pooling': True, - 'connect_timeout': 0.5 - } - }, 'course_structure_cache': { 'KEY_PREFIX': 'course_structure', 'KEY_FUNCTION': 'common.djangoapps.util.memcache.safe_key', @@ -2699,22 +2684,6 @@ PROCTORING_SETTINGS = {} -################## BLOCKSTORE RELATED SETTINGS ######################### - -# Which of django's caches to use for storing anonymous user state for XBlocks -# in the blockstore-based XBlock runtime -XBLOCK_RUNTIME_V2_EPHEMERAL_DATA_CACHE = 'default' - -# .. setting_name: BLOCKSTORE_BUNDLE_CACHE_TIMEOUT -# .. setting_default: 3000 -# .. setting_description: Maximum time-to-live of cached Bundles fetched from -# Blockstore, in seconds. When the values returned from Blockstore have -# TTLs of their own (such as signed S3 URLs), the maximum TTL of this cache -# must be lower than the minimum TTL of those values. -# We use a default of 3000s (50mins) because temporary URLs are often -# configured to expire after one hour. -BLOCKSTORE_BUNDLE_CACHE_TIMEOUT = 3000 - ###################### LEARNER PORTAL ################################ LEARNER_PORTAL_URL_ROOT = 'https://learner-portal-localhost:18000' diff --git a/cms/envs/devstack.py b/cms/envs/devstack.py index 3fe10377d79c..faacbc1548d0 100644 --- a/cms/envs/devstack.py +++ b/cms/envs/devstack.py @@ -218,9 +218,6 @@ def should_show_debug_toolbar(request): # lint-amnesty, pylint: disable=missing ENTERPRISE_BACKEND_SERVICE_EDX_OAUTH2_PROVIDER_URL = "http://edx.devstack.lms/oauth2" -############################### BLOCKSTORE ##################################### -BLOCKSTORE_API_URL = "http://edx.devstack.blockstore:18250/api/v1/" - ##################################################################### # pylint: disable=wrong-import-order, wrong-import-position diff --git a/cms/envs/production.py b/cms/envs/production.py index 67524e760816..f65b2204328b 100644 --- a/cms/envs/production.py +++ b/cms/envs/production.py @@ -405,11 +405,6 @@ def get_env_setting(setting): CONTENTSTORE = AUTH_TOKENS.get('CONTENTSTORE', CONTENTSTORE) DOC_STORE_CONFIG = AUTH_TOKENS.get('DOC_STORE_CONFIG', DOC_STORE_CONFIG) -############################### BLOCKSTORE ##################################### -BLOCKSTORE_API_URL = ENV_TOKENS.get('BLOCKSTORE_API_URL', None) # e.g. "https://blockstore.example.com/api/v1/" -# Configure an API auth token at (blockstore URL)/admin/authtoken/token/ -BLOCKSTORE_API_AUTH_TOKEN = AUTH_TOKENS.get('BLOCKSTORE_API_AUTH_TOKEN', None) - # Celery Broker CELERY_ALWAYS_EAGER = ENV_TOKENS.get("CELERY_ALWAYS_EAGER", False) CELERY_BROKER_TRANSPORT = ENV_TOKENS.get("CELERY_BROKER_TRANSPORT", "") diff --git a/cms/envs/test.py b/cms/envs/test.py index 118d7e27a79c..38b7c7817149 100644 --- a/cms/envs/test.py +++ b/cms/envs/test.py @@ -173,23 +173,8 @@ 'course_structure_cache': { 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', }, - 'blockstore': { - 'KEY_PREFIX': 'blockstore', - 'KEY_FUNCTION': 'common.djangoapps.util.memcache.safe_key', - 'LOCATION': 'edx_loc_mem_cache', - 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - }, } -############################### BLOCKSTORE ##################################### -BUNDLE_ASSET_STORAGE_SETTINGS = dict( - STORAGE_CLASS='django.core.files.storage.FileSystemStorage', - STORAGE_KWARGS=dict( - location=MEDIA_ROOT, - base_url=MEDIA_URL, - ), -) - ################################# CELERY ###################################### CELERY_ALWAYS_EAGER = True diff --git a/cms/templates/content_libraries/xblock_iframe.html b/cms/templates/content_libraries/xblock_iframe.html index 84d83de42ded..e8eb4c96ead0 100644 --- a/cms/templates/content_libraries/xblock_iframe.html +++ b/cms/templates/content_libraries/xblock_iframe.html @@ -228,7 +228,7 @@ // Does the XBlock HTML contain arguments to pass to the InitFunction? let data = {}; [].forEach.call(element.children, (childNode) => { - // The newer/pure/Blockstore runtime uses 'xblock_json_init_args' + // The newer/pure/LearningCore runtime uses 'xblock_json_init_args' // while the LMS runtime uses 'xblock-json-init-args'. if ( childNode.matches('script.xblock_json_init_args') @@ -257,7 +257,7 @@ // Recursively initialize the JavaScript code of each XBlock: function initializeXBlockAndChildren(element, callback) { - // The newer/pure/Blockstore runtime uses the 'data-usage' attribute, while the LMS uses 'data-usage-id' + // The newer/pure/LearningCore runtime uses the 'data-usage' attribute, while the LMS uses 'data-usage-id' const usageId = element.getAttribute('data-usage') || element.getAttribute('data-usage-id'); if (usageId !== null) { element[USAGE_ID_KEY] = usageId; @@ -297,7 +297,7 @@ } // Find the root XBlock node. - // The newer/pure/Blockstore runtime uses '.xblock-v1' while the LMS runtime uses '.xblock'. + // The newer/pure/LearningCore runtime uses '.xblock-v1' while the LMS runtime uses '.xblock'. const rootNode = document.querySelector('.xblock, .xblock-v1'); // will always return the first matching element initializeXBlockAndChildren(rootNode, () => { }); diff --git a/common/djangoapps/track/contexts.py b/common/djangoapps/track/contexts.py index 217a514e4c0f..0ac8292e258d 100644 --- a/common/djangoapps/track/contexts.py +++ b/common/djangoapps/track/contexts.py @@ -48,7 +48,7 @@ def course_context_from_course_id(course_id): """ Creates a course context from a `course_id`. - For newer parts of the system (i.e. Blockstore-based libraries/courses/etc.) + For newer parts of the system (i.e. Learning-Core-based libraries/courses/etc.) use context_dict_for_learning_context instead of this method. Example Returned Context:: diff --git a/docs/concepts/extension_points.rst b/docs/concepts/extension_points.rst index c5ba0d42c050..d4e802baec0e 100644 --- a/docs/concepts/extension_points.rst +++ b/docs/concepts/extension_points.rst @@ -124,7 +124,7 @@ Here are the different integration points that python plugins can use: - By default, the registration page for each instance of Open edX has fields that ask for information such as a user’s name, country, and highest level of education completed. You can add custom fields to the registration page for your own Open edX instance. These fields can be different types, including text entry fields and drop-down lists. See `Adding Custom Fields to the Registration Page`_. * - Learning Context (``openedx.learning_context``) - Trial, Limited - - A "Learning Context" is a course, a library, a program, a blog, an external site, or some other collection of content where learning happens. If you are trying to build a totally new learning experience that's not a type of course, you may need to implement a new learning context. Learning contexts are a new abstraction and are only supported in the nascent Blockstore-based XBlock runtime. Since existing courses use modulestore instead of Blockstore, they are not yet implemented as learning contexts. However, Blockstore-based content libraries are. See |learning_context.py|_ to learn more. + - A "Learning Context" is a course, a library, a program, a blog, an external site, or some other collection of content where learning happens. If you are trying to build a totally new learning experience that's not a type of course, you may need to implement a new learning context. Learning contexts are a new abstraction and are only supported in the nascent Learning-Core-based XBlock runtime. Since existing courses use modulestore instead of Learning Core, they are not yet implemented as learning contexts. However, Learning-Core-based content libraries are. See |learning_context.py|_ to learn more. * - User partition scheme (``openedx.user_partition_scheme`` and ``openedx.dynamic_partition_generator``) - Unknown, Stable - A user partition scheme is a named way for dividing users in a course into groups, usually to show different content to different users or to run experiments. Partitions may be added to a course manually, or automatically added by a "dynamic partition generator." The core platform includes partition scheme plugins like ``random``, ``cohort``, and ``enrollment_track``. See the |UserPartition docstring|_ to learn more. diff --git a/lms/djangoapps/edxnotes/tests.py b/lms/djangoapps/edxnotes/tests.py index 3cd7bafa5b21..688cd543865f 100644 --- a/lms/djangoapps/edxnotes/tests.py +++ b/lms/djangoapps/edxnotes/tests.py @@ -167,9 +167,9 @@ def test_edxnotes_studio(self): self.problem.runtime.is_author_mode = True assert 'original_get_html' == self.problem.get_html() - def test_edxnotes_blockstore_runtime(self): + def test_edxnotes_learning_core_runtime(self): """ - Tests that get_html is not wrapped when problem is rendered by Blockstore runtime. + Tests that get_html is not wrapped when problem is rendered by the learning core runtime. """ del self.problem.block.runtime.modulestore assert 'original_get_html' == self.problem.get_html() diff --git a/lms/envs/common.py b/lms/envs/common.py index 5f8714ca9cf7..da2bfed626e0 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -1157,26 +1157,12 @@ DATABASE_ROUTERS = [ 'openedx.core.lib.django_courseware_routers.StudentModuleHistoryExtendedRouter', - 'openedx.core.lib.blockstore_api.db_routers.BlockstoreRouter', 'edx_django_utils.db.read_replica.ReadReplicaRouter', ] ############################ Cache Configuration ############################### CACHES = { - 'blockstore': { - 'KEY_PREFIX': 'blockstore', - 'KEY_FUNCTION': 'common.djangoapps.util.memcache.safe_key', - 'LOCATION': ['localhost:11211'], - 'TIMEOUT': '86400', # This data should be long-lived for performance, BundleCache handles invalidation - 'BACKEND': 'django.core.cache.backends.memcached.PyMemcacheCache', - 'OPTIONS': { - 'no_delay': True, - 'ignore_exc': True, - 'use_pooling': True, - 'connect_timeout': 0.5 - } - }, 'course_structure_cache': { 'KEY_PREFIX': 'course_structure', 'KEY_FUNCTION': 'common.djangoapps.util.memcache.safe_key', @@ -1704,6 +1690,11 @@ def _make_mako_template_dirs(settings): # for more reference. XBLOCK_SETTINGS = {} +# .. setting_name: XBLOCK_RUNTIME_V2_EPHEMERAL_DATA_CACHE +# .. setting_default: default +# .. setting_description: The django cache key of the cache to use for storing anonymous user state for XBlocks. +XBLOCK_RUNTIME_V2_EPHEMERAL_DATA_CACHE = 'default' + ############# ModuleStore Configuration ########## MODULESTORE_BRANCH = 'published-only' @@ -3149,7 +3140,7 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring # User tours 'lms.djangoapps.user_tours', - # New (Blockstore-based) XBlock runtime + # New (Learning-Core-based) XBlock runtime 'openedx.core.djangoapps.xblock.apps.LmsXBlockAppConfig', # Student support tools @@ -3385,9 +3376,6 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring # For edx ace template tags 'edx_ace', - # Blockstore - 'blockstore.apps.bundles', - # MFE API 'lms.djangoapps.mfe_config_api', @@ -5196,57 +5184,6 @@ def _make_locale_paths(settings): # pylint: disable=missing-function-docstring ########################## MAILCHIMP SETTINGS ################################# MAILCHIMP_NEW_USER_LIST_ID = "" -########################## BLOCKSTORE ##################################### - -# .. setting_name: XBLOCK_RUNTIME_V2_EPHEMERAL_DATA_CACHE -# .. setting_default: default -# .. setting_description: The django cache key of the cache to use for storing anonymous user state for XBlocks. -XBLOCK_RUNTIME_V2_EPHEMERAL_DATA_CACHE = 'default' - -# .. setting_name: BLOCKSTORE_BUNDLE_CACHE_TIMEOUT -# .. setting_default: 3000 -# .. setting_description: Maximum time-to-live of cached Bundles fetched from -# Blockstore, in seconds. When the values returned from Blockstore have -# TTLs of their own (such as signed S3 URLs), the maximum TTL of this cache -# must be lower than the minimum TTL of those values. -# We use a default of 3000s (50mins) because temporary URLs are often -# configured to expire after one hour. -BLOCKSTORE_BUNDLE_CACHE_TIMEOUT = 3000 - -# .. setting_name: BUNDLE_ASSET_URL_STORAGE_KEY -# .. setting_default: None -# .. setting_description: When this is set, `BUNDLE_ASSET_URL_STORAGE_SECRET` is -# set, and `boto3` is installed, this is used as an AWS IAM access key for -# generating signed, read-only URLs for blockstore assets stored in S3. -# Otherwise, URLs are generated based on the default storage configuration. -# See `blockstore.apps.bundles.storage.LongLivedSignedUrlStorage` for details. -BUNDLE_ASSET_URL_STORAGE_KEY = None - -# .. setting_name: BUNDLE_ASSET_URL_STORAGE_SECRET -# .. setting_default: None -# .. setting_description: When this is set, `BUNDLE_ASSET_URL_STORAGE_KEY` is -# set, and `boto3` is installed, this is used as an AWS IAM secret key for -# generating signed, read-only URLs for blockstore assets stored in S3. -# Otherwise, URLs are generated based on the default storage configuration. -# See `blockstore.apps.bundles.storage.LongLivedSignedUrlStorage` for details. -BUNDLE_ASSET_URL_STORAGE_SECRET = None - -# .. setting_name: BUNDLE_ASSET_STORAGE_SETTINGS -# .. setting_default: dict, appropriate for file system storage. -# .. setting_description: When this is set, `BUNDLE_ASSET_URL_STORAGE_KEY` is -# set, and `boto3` is installed, this provides the bucket name and location for blockstore assets stored in S3. -# See `blockstore.apps.bundles.storage.LongLivedSignedUrlStorage` for details. -BUNDLE_ASSET_STORAGE_SETTINGS = dict( - # Backend storage - # STORAGE_CLASS='storages.backends.s3boto3.S3Boto3Storage', - # STORAGE_KWARGS=dict(bucket='bundle-asset-bucket', location='/path-to-bundles/'), - STORAGE_CLASS='django.core.files.storage.FileSystemStorage', - STORAGE_KWARGS=dict( - location=MEDIA_ROOT, - base_url=MEDIA_URL, - ), -) - SYSLOG_SERVER = '' FEEDBACK_SUBMISSION_EMAIL = '' GITHUB_REPO_ROOT = '/edx/var/edxapp/data' diff --git a/lms/envs/devstack.py b/lms/envs/devstack.py index b783a276fe44..82e9134f456f 100644 --- a/lms/envs/devstack.py +++ b/lms/envs/devstack.py @@ -262,9 +262,6 @@ def should_show_debug_toolbar(request): # lint-amnesty, pylint: disable=missing ) }) -############################### BLOCKSTORE ##################################### -BLOCKSTORE_API_URL = "http://edx.devstack.blockstore:18250/api/v1/" - ########################## PROGRAMS LEARNER PORTAL ############################## LEARNER_PORTAL_URL_ROOT = 'http://localhost:8734' diff --git a/lms/envs/production.py b/lms/envs/production.py index 948e81977cb4..014cf59aa36a 100644 --- a/lms/envs/production.py +++ b/lms/envs/production.py @@ -516,11 +516,6 @@ def get_env_setting(setting): EMAIL_HOST_USER = AUTH_TOKENS.get('EMAIL_HOST_USER', '') # django default is '' EMAIL_HOST_PASSWORD = AUTH_TOKENS.get('EMAIL_HOST_PASSWORD', '') # django default is '' -############################### BLOCKSTORE ##################################### -BLOCKSTORE_API_URL = ENV_TOKENS.get('BLOCKSTORE_API_URL', None) # e.g. "https://blockstore.example.com/api/v1/" -# Configure an API auth token at (blockstore URL)/admin/authtoken/token/ -BLOCKSTORE_API_AUTH_TOKEN = AUTH_TOKENS.get('BLOCKSTORE_API_AUTH_TOKEN', None) - # Analytics API ANALYTICS_API_KEY = AUTH_TOKENS.get("ANALYTICS_API_KEY", ANALYTICS_API_KEY) ANALYTICS_API_URL = ENV_TOKENS.get("ANALYTICS_API_URL", ANALYTICS_API_URL) diff --git a/lms/envs/test.py b/lms/envs/test.py index 14c10e52d36d..3c4bb9564927 100644 --- a/lms/envs/test.py +++ b/lms/envs/test.py @@ -215,13 +215,6 @@ 'course_structure_cache': { 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', }, - # Blockstore caching tests require a cache that actually works: - 'blockstore': { - 'KEY_PREFIX': 'blockstore', - 'KEY_FUNCTION': 'common.djangoapps.util.memcache.safe_key', - 'LOCATION': 'edx_loc_mem_cache', - 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - }, } ############################# SECURITY SETTINGS ################################ @@ -546,16 +539,6 @@ derive_settings(__name__) -############################### BLOCKSTORE ##################################### -XBLOCK_RUNTIME_V2_EPHEMERAL_DATA_CACHE = 'blockstore' # This must be set to a working cache for the tests to pass -BUNDLE_ASSET_STORAGE_SETTINGS = dict( - STORAGE_CLASS='django.core.files.storage.FileSystemStorage', - STORAGE_KWARGS=dict( - location=MEDIA_ROOT, - base_url=MEDIA_URL, - ), -) - # Dummy secret key for dev SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd' diff --git a/lms/urls.py b/lms/urls.py index 2d5d9a200306..5ac6283fddc7 100644 --- a/lms/urls.py +++ b/lms/urls.py @@ -336,7 +336,7 @@ name='xblock_resource_url', ), - # New (Blockstore-based) XBlock REST API + # New (Learning-Core-based) XBlock REST API path('', include(('openedx.core.djangoapps.xblock.rest_api.urls', 'openedx.core.djangoapps.xblock'), namespace='xblock_api')), diff --git a/openedx/core/djangoapps/content/search/tests/test_handlers.py b/openedx/core/djangoapps/content/search/tests/test_handlers.py index 7f209b4ca601..1ce9c57a1ab9 100644 --- a/openedx/core/djangoapps/content/search/tests/test_handlers.py +++ b/openedx/core/djangoapps/content/search/tests/test_handlers.py @@ -9,7 +9,6 @@ from common.djangoapps.student.tests.factories import UserFactory from openedx.core.djangoapps.content_libraries import api as library_api from openedx.core.djangolib.testing.utils import skip_unless_cms -from openedx.core.lib.blockstore_api.tests.base import BlockstoreAppTestMixin from xmodule.modulestore.tests.django_utils import TEST_DATA_SPLIT_MODULESTORE, ModuleStoreTestCase @@ -25,11 +24,7 @@ @patch("openedx.core.djangoapps.content.search.api.MeilisearchClient") @override_settings(MEILISEARCH_ENABLED=True) @skip_unless_cms -class TestUpdateIndexHandlers( - ModuleStoreTestCase, - BlockstoreAppTestMixin, - LiveServerTestCase, -): +class TestUpdateIndexHandlers(ModuleStoreTestCase, LiveServerTestCase): """ Test that the search index is updated when XBlocks and Library Blocks are modified """ diff --git a/openedx/core/djangoapps/content_libraries/api.py b/openedx/core/djangoapps/content_libraries/api.py index 3e276a37abf0..3c47e0d2605f 100644 --- a/openedx/core/djangoapps/content_libraries/api.py +++ b/openedx/core/djangoapps/content_libraries/api.py @@ -94,7 +94,7 @@ from xblock.exceptions import XBlockNotFoundError from openedx.core.djangoapps.xblock.api import get_component_from_usage_key, xblock_type_display_name -from openedx.core.lib.xblock_serializer.api import serialize_modulestore_block_for_blockstore +from openedx.core.lib.xblock_serializer.api import serialize_modulestore_block_for_learning_core from xmodule.library_root_xblock import LibraryRoot as LibraryRootV1 from xmodule.modulestore import ModuleStoreEnum from xmodule.modulestore.django import modulestore @@ -336,6 +336,7 @@ def get_library(library_key): # something that we should remove. It exists to accomodate some complexities # with how Blockstore staged changes, but Learning Core works differently, # and has_unpublished_changes should be sufficient. + # Ref: https://github.com/openedx/edx-platform/issues/34283 has_unpublished_deletes = publishing_api.get_entities_with_unpublished_deletes(learning_package.id) \ .exists() @@ -1012,10 +1013,10 @@ def get_v1_or_v2_library( library = get_library(library_key) if v2_version is not None and library.version != v2_version: raise NotImplementedError( - f"Tried to load version {v2_version} of blockstore-based library {library_key}. " + f"Tried to load version {v2_version} of learning_core-based library {library_key}. " f"Currently, only the latest version ({library.version}) may be loaded. " "This is a known issue. " - "It will be fixed before the production release of blockstore-based (V2) content libraries. " + "It will be fixed before the production release of learning_core-based (V2) content libraries. " ) return library except ContentLibrary.DoesNotExist: @@ -1121,35 +1122,34 @@ def import_block(self, modulestore_key): modulestore_key.block_type, block_id, ) - blockstore_key = library_block.usage_key + dest_key = library_block.usage_key except LibraryBlockAlreadyExists: - blockstore_key = LibraryUsageLocatorV2( + dest_key = LibraryUsageLocatorV2( lib_key=self.library.library_key, block_type=modulestore_key.block_type, usage_id=block_id, ) - get_library_block(blockstore_key) + get_library_block(dest_key) log.warning('Library block already exists: Appending static files ' - 'and overwriting OLX: %s', str(blockstore_key)) + 'and overwriting OLX: %s', str(dest_key)) # Handle static files. files = [ f.path for f in - get_library_block_static_asset_files(blockstore_key) + get_library_block_static_asset_files(dest_key) ] for filename, static_file in block_data.get('static_files', {}).items(): if filename in files: # Files already added, move on. continue file_content = self.get_block_static_data(static_file) - add_library_block_static_asset_file( - blockstore_key, filename, file_content) + add_library_block_static_asset_file(dest_key, filename, file_content) files.append(filename) # Import OLX. - set_library_block_olx(blockstore_key, block_data['olx']) + set_library_block_olx(dest_key, block_data['olx']) def import_blocks_from_course(self, course_key, progress_callback): """ @@ -1200,7 +1200,7 @@ def get_block_data(self, block_key): Get block OLX by serializing it from modulestore directly. """ block = self.modulestore.get_item(block_key) - data = serialize_modulestore_block_for_blockstore(block) + data = serialize_modulestore_block_for_learning_core(block) return {'olx': data.olx_str, 'static_files': {s.name: s for s in data.static_files}} diff --git a/openedx/core/djangoapps/content_libraries/apps.py b/openedx/core/djangoapps/content_libraries/apps.py index 685e9259b6e2..52c3e5179721 100644 --- a/openedx/core/djangoapps/content_libraries/apps.py +++ b/openedx/core/djangoapps/content_libraries/apps.py @@ -16,7 +16,7 @@ class ContentLibrariesConfig(AppConfig): """ name = 'openedx.core.djangoapps.content_libraries' - verbose_name = 'Content Libraries (Blockstore-based)' + verbose_name = 'Content Libraries (Learning-Core-based)' # This is designed as a plugin for now so that # the whole thing is self-contained and can easily be enabled/disabled plugin_app = { diff --git a/openedx/core/djangoapps/content_libraries/constants.py b/openedx/core/djangoapps/content_libraries/constants.py index 0a0614e514d9..9505d52d1cca 100644 --- a/openedx/core/djangoapps/content_libraries/constants.py +++ b/openedx/core/djangoapps/content_libraries/constants.py @@ -1,9 +1,6 @@ """ Constants used for the content libraries. """ from django.utils.translation import gettext_lazy as _ -# ./api.py and ./views.py are only used in Studio, so we always work with this draft of any -# content library bundle: -DRAFT_NAME = 'studio_draft' VIDEO = 'video' COMPLEX = 'complex' diff --git a/openedx/core/djangoapps/content_libraries/library_context.py b/openedx/core/djangoapps/content_libraries/library_context.py index 9408f51e511a..93de022474b0 100644 --- a/openedx/core/djangoapps/content_libraries/library_context.py +++ b/openedx/core/djangoapps/content_libraries/library_context.py @@ -19,7 +19,7 @@ class LibraryContextImpl(LearningContext): """ Implements content libraries as a learning context. - This is the *new* content libraries based on Blockstore, not the old content + This is the *new* content libraries based on Learning Core, not the old content libraries based on modulestore. """ diff --git a/openedx/core/djangoapps/content_libraries/management/commands/content_libraries_import.py b/openedx/core/djangoapps/content_libraries/management/commands/content_libraries_import.py index 9b57b42e6aec..cd68112d2b67 100644 --- a/openedx/core/djangoapps/content_libraries/management/commands/content_libraries_import.py +++ b/openedx/core/djangoapps/content_libraries/management/commands/content_libraries_import.py @@ -89,7 +89,7 @@ def add_arguments(self, parser): def handle(self, *args, **options): """ Collect all blocks from a course that are "importable" and write them to the - a blockstore library. + a learning core library. """ # Search for the library. diff --git a/openedx/core/djangoapps/content_libraries/migrations/0008_auto_20210818_2148.py b/openedx/core/djangoapps/content_libraries/migrations/0008_auto_20210818_2148.py index fb15e4a0faf2..6c368dec24c4 100644 --- a/openedx/core/djangoapps/content_libraries/migrations/0008_auto_20210818_2148.py +++ b/openedx/core/djangoapps/content_libraries/migrations/0008_auto_20210818_2148.py @@ -40,7 +40,7 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='ltigradedresource', name='usage_key', - field=opaque_keys.edx.django.models.UsageKeyField(help_text='The usage key string of the blockstore resource serving the content of this launch.', max_length=255), + field=opaque_keys.edx.django.models.UsageKeyField(help_text='The usage key string of the resource serving the content of this launch.', max_length=255), ), migrations.AlterField( model_name='ltiprofile', diff --git a/openedx/core/djangoapps/content_libraries/models.py b/openedx/core/djangoapps/content_libraries/models.py index a4c128c9bcc1..58a9f6b6863f 100644 --- a/openedx/core/djangoapps/content_libraries/models.py +++ b/openedx/core/djangoapps/content_libraries/models.py @@ -8,7 +8,7 @@ LTI 1.3 Models ============== -Content Libraries serves blockstore-based content through LTI 1.3 launches. +Content Libraries serves learning-core-based content through LTI 1.3 launches. The interface supports resource link launches and grading services. Two use cases justify the current data model to support LTI launches. They are: @@ -27,7 +27,7 @@ The data model above is similar to the one provided by the current LTI 1.1 implementation for modulestore and courseware content. But, Content Libraries is orthogonal. Its use-case is to offer standalone, embedded content from a -specific backend (blockstore). As such, it decouples from LTI 1.1. and the +specific backend (learning core). As such, it decouples from LTI 1.1. and the logic assume no relationship or impact across the two applications. The same reasoning applies to steps beyond the data model, such as at the XBlock runtime, authentication, and score handling, etc. @@ -85,9 +85,9 @@ class ContentLibrary(models.Model): """ A Content Library is a collection of content (XBlocks and/or static assets) - All actual content is stored in Blockstore, and any data that we'd want to + All actual content is stored in Learning Core, and any data that we'd want to transfer to another instance if this library were exported and then - re-imported on another Open edX instance should be kept in Blockstore. This + re-imported on another Open edX instance should be kept in Learning Core. This model in Studio should only be used to track settings specific to this Open edX instance, like who has permission to edit this content library. """ @@ -479,7 +479,7 @@ class LtiGradedResource(models.Model): usage_key = UsageKeyField( max_length=255, - help_text=_('The usage key string of the blockstore resource serving the ' + help_text=_('The usage key string of the resource serving the ' 'content of this launch.'), ) diff --git a/openedx/core/djangoapps/content_libraries/permissions.py b/openedx/core/djangoapps/content_libraries/permissions.py index 3c41e0574c36..c7da012c9fb4 100644 --- a/openedx/core/djangoapps/content_libraries/permissions.py +++ b/openedx/core/djangoapps/content_libraries/permissions.py @@ -1,5 +1,5 @@ """ -Permissions for Content Libraries (v2, Blockstore-based) +Permissions for Content Libraries (v2, Learning-Core-based) """ from bridgekeeper import perms, rules from bridgekeeper.rules import Attribute, ManyRelation, Relation, in_current_groups diff --git a/openedx/core/djangoapps/content_libraries/serializers.py b/openedx/core/djangoapps/content_libraries/serializers.py index ee0e48b59c87..13c6a756fd31 100644 --- a/openedx/core/djangoapps/content_libraries/serializers.py +++ b/openedx/core/djangoapps/content_libraries/serializers.py @@ -14,7 +14,6 @@ from openedx.core.djangoapps.content_libraries.models import ( ContentLibraryPermission, ContentLibraryBlockImportTask ) -from openedx.core.lib import blockstore_api from openedx.core.lib.api.serializers import CourseKeyField @@ -175,16 +174,6 @@ class LibraryXBlockStaticFileSerializer(serializers.Serializer): url = serializers.URLField() size = serializers.IntegerField(min_value=0) - def to_representation(self, instance): - """ - Generate the serialized representation of this static asset file. - """ - result = super().to_representation(instance) - # Make sure the URL is one that will work from the user's browser, - # not one that only works from within a docker container: - result['url'] = blockstore_api.force_browser_url(result['url']) - return result - class LibraryXBlockStaticFilesSerializer(serializers.Serializer): """ diff --git a/openedx/core/djangoapps/content_libraries/tasks.py b/openedx/core/djangoapps/content_libraries/tasks.py index 3714dc55f8d6..9f4f7aaaf7dc 100644 --- a/openedx/core/djangoapps/content_libraries/tasks.py +++ b/openedx/core/djangoapps/content_libraries/tasks.py @@ -4,11 +4,11 @@ Architecture note: Several functions in this file manage the copying/updating of blocks in modulestore - and blockstore. These operations should only be performed within the context of CMS. + and learning core. These operations should only be performed within the context of CMS. However, due to existing edx-platform code structure, we've had to define the functions in shared source tree (openedx/) and the tasks are registered in both LMS and CMS. - To ensure that we're not accidentally importing things from blockstore in the LMS context, + To ensure that we're not accidentally importing things from learning core in the LMS context, we use ensure_cms throughout this module. A longer-term solution to this issue would be to move the content_libraries app to cms: @@ -39,7 +39,7 @@ from opaque_keys.edx.keys import CourseKey from openedx.core.djangoapps.content_libraries import api as library_api from openedx.core.djangoapps.xblock.api import load_block -from openedx.core.lib import ensure_cms, blockstore_api +from openedx.core.lib import ensure_cms from xmodule.capa_block import ProblemBlock from xmodule.library_content_block import ANY_CAPA_TYPE_VALUE, LibraryContentBlock from xmodule.library_root_xblock import LibraryRoot as LibraryRootV1 @@ -86,7 +86,7 @@ def on_progress(block_key, block_num, block_count, exception=None): def _import_block(store, user_id, source_block, dest_parent_key): """ - Recursively import a blockstore block and its children.` + Recursively import a learning core block and its children.` """ def generate_block_key(source_key, dest_parent_key): """ @@ -127,7 +127,7 @@ def generate_block_key(source_key, dest_parent_key): # Prepare a list of this block's static assets; any assets that are referenced as /static/{path} (the # recommended way for referencing them) will stop working, and so we rewrite the url when importing. - # Copying assets not advised because modulestore doesn't namespace assets to each block like blockstore, which + # Copying assets not advised because modulestore doesn't namespace assets to each block like learning core, which # might cause conflicts when the same filename is used across imported blocks. if isinstance(source_key, LibraryUsageLocatorV2): all_assets = library_api.get_library_block_static_asset_files(source_key) @@ -139,12 +139,6 @@ def generate_block_key(source_key, dest_parent_key): continue # Only copy authored field data if field.is_set_on(source_block) or field.is_set_on(new_block): field_value = getattr(source_block, field_name) - if isinstance(field_value, str): - # If string field (which may also be JSON/XML data), rewrite /static/... URLs to point to blockstore - for asset in all_assets: - field_value = field_value.replace(f'/static/{asset.path}', asset.url) - # Make sure the URL is one that will work from the user's browser when using the docker devstack - field_value = blockstore_api.force_browser_url(field_value) setattr(new_block, field_name, field_value) new_block.save() store.update_item(new_block, user_id) @@ -178,9 +172,9 @@ def _problem_type_filter(store, library, capa_type): return [key for key in library.children if _filter_child(store, key, capa_type)] -def _import_from_blockstore(user_id, store, dest_block, blockstore_block_ids): +def _import_from_learning_core(user_id, store, dest_block, source_block_ids): """ - Imports a block from a blockstore-based learning context (usually a + Imports a block from a learning-core-based learning context (usually a content library) into modulestore, as a new child of dest_block. Any existing children of dest_block are replaced. """ @@ -190,7 +184,7 @@ def _import_from_blockstore(user_id, store, dest_block, blockstore_block_ids): if user_id is None: raise ValueError("Cannot check user permissions - LibraryTools user_id is None") - if len(set(blockstore_block_ids)) != len(blockstore_block_ids): + if len(set(source_block_ids)) != len(source_block_ids): # We don't support importing the exact same block twice because it would break the way we generate new IDs # for each block and then overwrite existing copies of blocks when re-importing the same blocks. raise ValueError("One or more library component IDs is a duplicate.") @@ -204,7 +198,7 @@ def _import_from_blockstore(user_id, store, dest_block, blockstore_block_ids): # (This could be slow and use lots of memory, except for the fact that LibraryContentBlock which calls this # should be limiting the number of blocks to a reasonable limit. We load them all now instead of one at a # time in order to raise any errors before we start actually copying blocks over.) - orig_blocks = [load_block(UsageKey.from_string(key), user) for key in blockstore_block_ids] + orig_blocks = [load_block(UsageKey.from_string(key), user) for key in source_block_ids] with store.bulk_operations(dest_course_key): child_ids_updated = set() @@ -347,7 +341,7 @@ def _sync_children( str(library_api.LibraryXBlockMetadata.from_component(library_key, component).usage_key) for component in library_api.get_library_components(library_key) ] - _import_from_blockstore(user_id, store, dest_block, source_block_ids) + _import_from_learning_core(user_id, store, dest_block, source_block_ids) dest_block.source_library_version = str(library.version) store.update_item(dest_block, user_id) except Exception as exception: # pylint: disable=broad-except diff --git a/openedx/core/djangoapps/content_libraries/tests/base.py b/openedx/core/djangoapps/content_libraries/tests/base.py index 5f837628c935..2bb94e3d87c7 100644 --- a/openedx/core/djangoapps/content_libraries/tests/base.py +++ b/openedx/core/djangoapps/content_libraries/tests/base.py @@ -1,5 +1,5 @@ """ -Tests for Blockstore-based Content Libraries +Tests for Learning-Core-based Content Libraries """ import uuid from contextlib import contextmanager @@ -12,9 +12,6 @@ from common.djangoapps.student.tests.factories import UserFactory from openedx.core.djangoapps.content_libraries.constants import COMPLEX, ALL_RIGHTS_RESERVED from openedx.core.djangolib.testing.utils import skip_unless_cms -from openedx.core.lib.blockstore_api.tests.base import ( - BlockstoreAppTestMixin, -) # Define the URLs here - don't use reverse() because we want to detect # backwards-incompatible changes like changed URLs. @@ -46,9 +43,9 @@ @skip_unless_cms # Content Libraries REST API is only available in Studio -class ContentLibrariesRestApiTest(BlockstoreAppTestMixin, APITransactionTestCase): +class ContentLibrariesRestApiTest(APITransactionTestCase): """ - Base class for Blockstore-based Content Libraries test that use the REST API + Base class for Learning-Core-based Content Libraries test that use the REST API These tests use the REST API, which in turn relies on the Python API. Some tests may use the python API directly if necessary to provide diff --git a/openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py b/openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py index c23e728c4b4d..67933f296413 100644 --- a/openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py +++ b/openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py @@ -1,5 +1,5 @@ """ -Tests for Blockstore-based Content Libraries +Tests for Learning-Core-based Content Libraries """ from unittest.mock import Mock, patch from unittest import skip @@ -37,7 +37,7 @@ @ddt.ddt class ContentLibrariesTestCase(ContentLibrariesRestApiTest, OpenEdxEventsTestMixin): """ - General tests for Blockstore-based Content Libraries + General tests for Learning-Core-based Content Libraries These tests use the REST API, which in turn relies on the Python API. Some tests may use the python API directly if necessary to provide @@ -278,7 +278,7 @@ def test_library_blocks(self):

    This is a normal capa problem with unicode 🔥. It has "maximum attempts" set to **5**.

    - + XBlock metadata only XBlock data/metadata and associated static asset files @@ -300,7 +300,7 @@ def test_library_blocks(self): # Now view the XBlock's student_view (including draft changes): fragment = self._render_block_view(block_id, "student_view") assert 'resources' in fragment - assert 'Blockstore is designed to store.' in fragment['content'] + assert 'Learning Core is designed to store.' in fragment['content'] # Also call a handler to make sure that's working: handler_url = self._get_block_handler_url(block_id, "xmodule_handler") + "problem_get" @@ -806,7 +806,7 @@ def test_library_block_olx_update_event(self):

    This is a normal capa problem with unicode 🔥. It has "maximum attempts" set to **5**.

    - + XBlock metadata only XBlock data/metadata and associated static asset files @@ -956,7 +956,7 @@ def test_library_block_delete_event(self): @ddt.ddt class ContentLibraryXBlockValidationTest(APITestCase): - """Tests only focused on service validation, no Blockstore needed.""" + """Tests only focused on service validation, no Learning Core interactions here.""" @ddt.data( (URL_BLOCK_METADATA_URL, dict(block_key='totally_invalid_key')), diff --git a/openedx/core/djangoapps/content_libraries/tests/test_runtime.py b/openedx/core/djangoapps/content_libraries/tests/test_runtime.py index f35ba7ea7b90..89b8cdefd86b 100644 --- a/openedx/core/djangoapps/content_libraries/tests/test_runtime.py +++ b/openedx/core/djangoapps/content_libraries/tests/test_runtime.py @@ -1,5 +1,5 @@ """ -Test the Blockstore-based XBlock runtime and content libraries together. +Test the Learning-Core-based XBlock runtime and content libraries together. """ import json from gettext import GNUTranslations @@ -14,7 +14,6 @@ from lms.djangoapps.courseware.model_data import get_score from openedx.core.djangoapps.content_libraries import api as library_api from openedx.core.djangoapps.content_libraries.tests.base import ( - BlockstoreAppTestMixin, URL_BLOCK_RENDER_VIEW, URL_BLOCK_GET_HANDLER_URL, URL_BLOCK_METADATA_URL, @@ -60,9 +59,9 @@ def setUp(self): ) -class ContentLibraryRuntimeTestMixin(ContentLibraryContentTestMixin): +class ContentLibraryRuntimeTests(ContentLibraryContentTestMixin): """ - Basic tests of the Blockstore-based XBlock runtime using XBlocks in a + Basic tests of the Learning-Core-based XBlock runtime using XBlocks in a content library. """ @@ -95,7 +94,7 @@ def test_xblock_metadata(self):

    This is a normal capa problem. It has "maximum attempts" set to **5**.

    - + XBlock metadata only XBlock data/metadata and associated static asset files @@ -174,18 +173,10 @@ def test_xblock_fields(self): assert block_saved.display_name == 'New Display Name' -class ContentLibraryRuntimeTest(ContentLibraryRuntimeTestMixin, BlockstoreAppTestMixin): - """ - Tests XBlock runtime using XBlocks in a content library using the installed Blockstore app. - - We run this test with a live server, so that the blockstore asset files can be served. - """ - - # We can remove the line below to enable this in Studio once we implement a session-backed # field data store which we can use for both studio users and anonymous users @skip_unless_lms -class ContentLibraryXBlockUserStateTestMixin(ContentLibraryContentTestMixin): +class ContentLibraryXBlockUserStateTest(ContentLibraryContentTestMixin): """ Test that the Blockstore-based XBlock runtime can store and retrieve student state for XBlocks when learners access blocks directly in a library context, @@ -389,7 +380,7 @@ def test_scores_persisted(self):

    This is a normal capa problem. It has "maximum attempts" set to **5**.

    - + XBlock metadata only XBlock data/metadata and associated static asset files @@ -453,7 +444,7 @@ def test_i18n(self):

    This is a normal capa problem. It has "maximum attempts" set to **5**.

    - + XBlock metadata only XBlock data/metadata and associated static asset files @@ -487,19 +478,8 @@ def test_i18n(self): assert 'Submit' not in dummy_public_view.data['content'] -class ContentLibraryXBlockUserStateTest( # type: ignore[misc] - ContentLibraryXBlockUserStateTestMixin, - BlockstoreAppTestMixin, -): - """ - Tests XBlock user state for XBlocks in a content library using the installed Blockstore app. - - We run this test with a live server, so that the blockstore asset files can be served. - """ - - @skip_unless_lms # No completion tracking in Studio -class ContentLibraryXBlockCompletionTestMixin(ContentLibraryContentTestMixin, CompletionWaffleTestMixin): +class ContentLibraryXBlockCompletionTest(ContentLibraryContentTestMixin, CompletionWaffleTestMixin): """ Test that the Blockstore-based XBlocks can track their completion status using the completion library. @@ -550,16 +530,3 @@ def get_block_completion_status(): # Now the block is completed assert get_block_completion_status() == 1 - - -class ContentLibraryXBlockCompletionTest( - ContentLibraryXBlockCompletionTestMixin, - CompletionWaffleTestMixin, - BlockstoreAppTestMixin, -): - """ - Test that the Blockstore-based XBlocks can track their completion status - using the installed Blockstore app. - - We run this test with a live server, so that the blockstore asset files can be served. - """ diff --git a/openedx/core/djangoapps/content_libraries/tests/test_static_assets.py b/openedx/core/djangoapps/content_libraries/tests/test_static_assets.py index 6a75d63110b8..92ff4c1767d0 100644 --- a/openedx/core/djangoapps/content_libraries/tests/test_static_assets.py +++ b/openedx/core/djangoapps/content_libraries/tests/test_static_assets.py @@ -1,5 +1,5 @@ """ -Tests for static asset files in Blockstore-based Content Libraries +Tests for static asset files in Learning-Core-based Content Libraries """ from unittest import skip @@ -26,7 +26,7 @@ @skip("Assets are being reimplemented in Learning Core. Disable until that's ready.") class ContentLibrariesStaticAssetsTest(ContentLibrariesRestApiTest): """ - Tests for static asset files in Blockstore-based Content Libraries + Tests for static asset files in Learning-Core-based Content Libraries WARNING: every test should have a unique library slug, because even though the django/mysql database gets reset for each test case, the lookup between @@ -65,7 +65,7 @@ def test_asset_filenames(self): def test_video_transcripts(self): """ - Test that video blocks can read transcript files out of blockstore. + Test that video blocks can read transcript files out of learning core. """ library = self._create_library(slug="transcript-test-lib", title="Transcripts Test Library") block = self._add_block_to_library(library["id"], "video", "video1") @@ -104,7 +104,7 @@ def check_download(): check_sjson() check_download() # Publish the OLX and the transcript file, since published data gets - # served differently by Blockstore and we should test that too. + # served differently by Learning Core and we should test that too. self._commit_library_changes(library["id"]) check_sjson() check_download() diff --git a/openedx/core/djangoapps/content_libraries/tests/test_views_lti.py b/openedx/core/djangoapps/content_libraries/tests/test_views_lti.py index cb306ebbfe48..a25d02761dde 100644 --- a/openedx/core/djangoapps/content_libraries/tests/test_views_lti.py +++ b/openedx/core/djangoapps/content_libraries/tests/test_views_lti.py @@ -87,5 +87,5 @@ class LibraryBlockLtiUrlViewTest( ContentLibrariesRestApiTest, ): """ - Test generating LTI URL for a block in a library, using the installed Blockstore app. + Test generating LTI URL for a block in a library, using the installed Learning Core app. """ diff --git a/openedx/core/djangoapps/content_libraries/views.py b/openedx/core/djangoapps/content_libraries/views.py index 0df670b1aea6..38f3e7efd6a1 100644 --- a/openedx/core/djangoapps/content_libraries/views.py +++ b/openedx/core/djangoapps/content_libraries/views.py @@ -58,7 +58,8 @@ block. Historical note: These views used to be wrapped with @atomic because we - wanted to make all views that operated on Blockstore data atomic: + wanted to make all views that operated on Blockstore (the predecessor + to Learning Core) atomic: https://github.com/openedx/edx-platform/pull/30456 """ @@ -258,6 +259,7 @@ def post(self, request): # Learning Core. TODO: This can be removed once the frontend stops # sending it to us. This whole bit of deserialization is kind of weird # though, with the renames and such. Look into this later for clennup. + # Ref: https://github.com/openedx/edx-platform/issues/34283 data.pop("collection_uuid", None) try: @@ -708,9 +710,12 @@ def put(self, request, usage_key_str, file_path): ) file_wrapper = request.data['content'] if file_wrapper.size > 20 * 1024 * 1024: # > 20 MiB - # In the future, we need a way to use file_wrapper.chunks() to read - # the file in chunks and stream that to Blockstore, but Blockstore - # currently lacks an API for streaming file uploads. + # TODO: This check was written when V2 Libraries were backed by the Blockstore micro-service. + # Now that we're on Learning Core, do we still need it? Here's the original comment: + # In the future, we need a way to use file_wrapper.chunks() to read + # the file in chunks and stream that to Blockstore, but Blockstore + # currently lacks an API for streaming file uploads. + # Ref: https://github.com/openedx/edx-platform/issues/34737 raise ValidationError("File too big") file_content = file_wrapper.read() try: diff --git a/openedx/core/djangoapps/content_tagging/tests/test_tasks.py b/openedx/core/djangoapps/content_tagging/tests/test_tasks.py index 3e396eb754b3..c14adfcce13a 100644 --- a/openedx/core/djangoapps/content_tagging/tests/test_tasks.py +++ b/openedx/core/djangoapps/content_tagging/tests/test_tasks.py @@ -15,7 +15,6 @@ from openedx.core.djangolib.testing.utils import skip_unless_cms from xmodule.modulestore.tests.django_utils import TEST_DATA_SPLIT_MODULESTORE, ModuleStoreTestCase from openedx.core.djangoapps.content_libraries.api import create_library, create_library_block, delete_library_block -from openedx.core.lib.blockstore_api.tests.base import BlockstoreAppTestMixin from .. import api from ..models.base import TaxonomyOrg @@ -59,7 +58,6 @@ def setUp(self): class TestAutoTagging( # type: ignore[misc] LanguageTaxonomyTestMixin, ModuleStoreTestCase, - BlockstoreAppTestMixin, LiveServerTestCase ): """ diff --git a/openedx/core/djangoapps/olx_rest_api/views.py b/openedx/core/djangoapps/olx_rest_api/views.py index 8977b131530b..1083ee06f68d 100644 --- a/openedx/core/djangoapps/olx_rest_api/views.py +++ b/openedx/core/djangoapps/olx_rest_api/views.py @@ -13,7 +13,7 @@ from openedx.core.lib.api.view_utils import view_auth_classes from xmodule.modulestore.django import modulestore -from openedx.core.lib.xblock_serializer.api import serialize_modulestore_block_for_blockstore +from openedx.core.lib.xblock_serializer.api import serialize_modulestore_block_for_learning_core @api_view(['GET']) @@ -22,7 +22,7 @@ def get_block_olx(request, usage_key_str): """ Given a modulestore XBlock usage ID (block-v1:...), get its OLX and a list of any static asset files it uses. - (There are other APIs for getting the OLX of Blockstore XBlocks.) + (There are other APIs for getting the OLX of Learning Core XBlocks.) """ # Parse the usage key: try: @@ -48,7 +48,7 @@ def serialize_block(block_key): return block = modulestore().get_item(block_key) - serialized_blocks[block_key] = serialize_modulestore_block_for_blockstore(block) + serialized_blocks[block_key] = serialize_modulestore_block_for_learning_core(block) if block.has_children: for child_id in block.children: @@ -103,7 +103,7 @@ def get_block_exportfs_file(request, usage_key_str, path): raise PermissionDenied("You must be a member of the course team in Studio to export OLX using this API.") block = modulestore().get_item(usage_key) - serialized = serialize_modulestore_block_for_blockstore(block) + serialized = serialize_modulestore_block_for_learning_core(block) static_file = None for f in serialized.static_files: if f.name == path: diff --git a/openedx/core/djangoapps/xblock/README.rst b/openedx/core/djangoapps/xblock/README.rst index 0afe70b9d136..245af2259932 100644 --- a/openedx/core/djangoapps/xblock/README.rst +++ b/openedx/core/djangoapps/xblock/README.rst @@ -1,3 +1,8 @@ +This README was written back when the new runtime was backed by Blockstore. +Now that the runtime is backed by Learning Core, this README is out of date. +We need to audit and update it as part of +`this task `_. + XBlock App Suite (New) ====================== diff --git a/openedx/core/djangoapps/xblock/apps.py b/openedx/core/djangoapps/xblock/apps.py index afebd2ec71c1..5ba2361322ec 100644 --- a/openedx/core/djangoapps/xblock/apps.py +++ b/openedx/core/djangoapps/xblock/apps.py @@ -33,10 +33,7 @@ def get_site_root_url(self): def get_learning_context_params(self): """ Get additional kwargs that are passed to learning context implementations - (LearningContext subclass constructors). For example, this can be used to - specify that the course learning context should load the course's list of - blocks from the _draft_ version of the course in studio, but from the - published version of the course in the LMS. + (LearningContext subclass constructors). """ return {} @@ -68,8 +65,6 @@ class StudioXBlockAppConfig(XBlockAppConfig): Studio-specific configuration of the XBlock Runtime django app. """ - BLOCKSTORE_DRAFT_NAME = "studio_draft" - def get_runtime_system_params(self): """ Get the XBlockRuntimeSystem parameters appropriate for viewing and/or @@ -91,14 +86,9 @@ def get_site_root_url(self): def get_learning_context_params(self): """ Get additional kwargs that are passed to learning context implementations - (LearningContext subclass constructors). For example, this can be used to - specify that the course learning context should load the course's list of - blocks from the _draft_ version of the course in studio, but from the - published version of the course in the LMS. - """ - return { - "use_draft": self.BLOCKSTORE_DRAFT_NAME, - } + (LearningContext subclass constructors). + """ + return {} def get_xblock_app_config(): diff --git a/openedx/core/djangoapps/xblock/learning_context/learning_context.py b/openedx/core/djangoapps/xblock/learning_context/learning_context.py index 72a6a7645e9f..1ac621ef244f 100644 --- a/openedx/core/djangoapps/xblock/learning_context/learning_context.py +++ b/openedx/core/djangoapps/xblock/learning_context/learning_context.py @@ -53,15 +53,8 @@ def can_view_block(self, user, usage_key): # pylint: disable=unused-argument def definition_for_usage(self, usage_key, **kwargs): """ - Given a usage key for an XBlock in this context, return the - BundleDefinitionLocator which specifies the actual XBlock definition - (as a path to an OLX in a specific blockstore bundle). + Given a usage key in this context, return the key indicating the actual XBlock definition. - usage_key: the UsageKeyV2 subclass used for this learning context - - kwargs: optional additional parameters unique to the learning context - - Must return a BundleDefinitionLocator if the XBlock exists in this - context, or None otherwise. + Retuns None if the usage key doesn't exist in this context. """ raise NotImplementedError diff --git a/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py b/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py index 2875c4b468d5..dc5a85de3a02 100644 --- a/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py +++ b/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py @@ -21,7 +21,7 @@ from xblock.fields import Field, Scope, ScopeIds from xblock.field_data import FieldData -from openedx.core.lib.xblock_serializer.api import serialize_modulestore_block_for_blockstore +from openedx.core.lib.xblock_serializer.api import serialize_modulestore_block_for_learning_core from ..learning_context.manager import get_learning_context_impl from .runtime import XBlockRuntime @@ -234,11 +234,7 @@ def save_block(self, block): log.warning("User %s does not have permission to edit %s", self.user.username, block.scope_ids.usage_id) raise RuntimeError("You do not have permission to edit this XBlock") - # We need Blockstore's serialization so we don't have `url_name` showing - # up in all the OLX. TODO: Rename this later, after we figure out what - # other changes we need to make in the serialization as part of the - # Blockstore -> Learning Core conversion. - serialized = serialize_modulestore_block_for_blockstore(block) + serialized = serialize_modulestore_block_for_learning_core(block) now = datetime.now(tz=timezone.utc) usage_key = block.scope_ids.usage_id with atomic(): diff --git a/openedx/core/djangoapps/xblock/runtime/runtime.py b/openedx/core/djangoapps/xblock/runtime/runtime.py index 15e0f3f0617e..5746af491d10 100644 --- a/openedx/core/djangoapps/xblock/runtime/runtime.py +++ b/openedx/core/djangoapps/xblock/runtime/runtime.py @@ -219,14 +219,14 @@ def applicable_aside_types(self, block: XBlock): def parse_xml_file(self, fileobj): # Deny access to the inherited method - raise NotImplementedError("XML Serialization is only supported with BlockstoreXBlockRuntime") + raise NotImplementedError("XML Serialization is only supported with LearningCoreXBlockRuntime") def add_node_as_child(self, block, node): """ Called by XBlock.parse_xml to treat a child node as a child block. """ # Deny access to the inherited method - raise NotImplementedError("XML Serialization is only supported with BlockstoreXBlockRuntime") + raise NotImplementedError("XML Serialization is only supported with LearningCoreXBlockRuntime") def service(self, block: XBlock, service_name: str): """ @@ -261,8 +261,8 @@ def service(self, block: XBlock, service_name: str): return DjangoXBlockUserService( self.user, - # The value should be updated to whether the user is staff in the context when Blockstore runtime adds - # support for courses. + # The value should be updated to whether the user is staff in the context when Learning Core runtime + # adds support for courses. user_is_staff=self.user.is_staff, # type: ignore anonymous_user_id=self.anonymous_student_id, # See the docstring of `DjangoXBlockUserService`. @@ -437,7 +437,7 @@ def __init__( student_data_mode: Specifies whether student data should be kept in a temporary in-memory store (e.g. Studio) or persisted forever in the database. - runtime_class: What runtime to use, e.g. BlockstoreXBlockRuntime + runtime_class: What runtime to use, e.g. LearningCoreXBlockRuntime """ self.handler_url = handler_url self.id_reader = id_reader or OpaqueKeyReader() diff --git a/openedx/core/djangoapps/xblock/runtime/shims.py b/openedx/core/djangoapps/xblock/runtime/shims.py index 18aa41eb91f3..c306b82bdc8b 100644 --- a/openedx/core/djangoapps/xblock/runtime/shims.py +++ b/openedx/core/djangoapps/xblock/runtime/shims.py @@ -102,7 +102,7 @@ def get_python_lib_zip(self): Only used for capa problems. """ - # TODO: load the python code from Blockstore. Ensure it's not publicly accessible. + # TODO: load the python code from Learning Core. Ensure it's not publicly accessible. return None @property @@ -166,9 +166,8 @@ def resources_fs(self): """ A filesystem that XBlocks can use to read large binary assets. """ - # TODO: implement this to serve any static assets that - # self._active_block has in its blockstore "folder". But this API should - # be deprecated and we should instead get compatible XBlocks to use a + # TODO: implement this to serve any static assets that self._active_block has. + # But this API should be deprecated and we should instead get compatible XBlocks to use a # runtime filesystem service. Some initial exploration of that (as well # as of the 'FileField' concept) has been done and is included in the # XBlock repo at xblock.reference.plugins.FSService and is available in diff --git a/openedx/core/lib/blockstore_api/__init__.py b/openedx/core/lib/blockstore_api/__init__.py deleted file mode 100644 index 855d8a1f96a0..000000000000 --- a/openedx/core/lib/blockstore_api/__init__.py +++ /dev/null @@ -1,51 +0,0 @@ -""" -API Client for Blockstore - -TODO: This should all get ripped out. - -TODO: This wrapper is extraneous now that Blockstore-as-a-service isn't supported. - This whole directory tree should be removed by https://github.com/openedx/blockstore/issues/296. -""" -from blockstore.apps.api.data import ( - BundleFileData, -) -from blockstore.apps.api.exceptions import ( - CollectionNotFound, - BundleNotFound, - DraftNotFound, - BundleVersionNotFound, - BundleFileNotFound, - BundleStorageError, -) -from blockstore.apps.api.methods import ( - # Collections: - get_collection, - create_collection, - update_collection, - delete_collection, - # Bundles: - get_bundles, - get_bundle, - create_bundle, - update_bundle, - delete_bundle, - # Drafts: - get_draft, - get_or_create_bundle_draft, - write_draft_file, - set_draft_link, - commit_draft, - delete_draft, - # Bundles or drafts: - get_bundle_files, - get_bundle_files_dict, - get_bundle_file_metadata, - get_bundle_file_data, - get_bundle_version, - get_bundle_version_files, - # Links: - get_bundle_links, - get_bundle_version_links, - # Misc: - force_browser_url, -) diff --git a/openedx/core/lib/blockstore_api/db_routers.py b/openedx/core/lib/blockstore_api/db_routers.py deleted file mode 100644 index fd0ff50c9510..000000000000 --- a/openedx/core/lib/blockstore_api/db_routers.py +++ /dev/null @@ -1,60 +0,0 @@ -""" -Blockstore database router. - -Blockstore started life as an IDA, but is now a Django app plugin within edx-platform. -This router exists to smooth blockstore's transition into edxapp. -""" -from django.conf import settings - - -class BlockstoreRouter: - """ - A Database Router that uses the ``blockstore`` database, if it's configured in settings. - """ - ROUTE_APP_LABELS = {'bundles'} - DATABASE_NAME = 'blockstore' - - def _use_blockstore(self, model): - """ - Return True if the given model should use the blockstore database. - - Ensures that a ``blockstore`` database is configured, and checks the ``model``'s app label. - """ - return (self.DATABASE_NAME in settings.DATABASES) and (model._meta.app_label in self.ROUTE_APP_LABELS) - - def db_for_read(self, model, **hints): # pylint: disable=unused-argument - """ - Use the BlockstoreRouter.DATABASE_NAME when reading blockstore app tables. - """ - if self._use_blockstore(model): - return self.DATABASE_NAME - return None - - def db_for_write(self, model, **hints): # pylint: disable=unused-argument - """ - Use the BlockstoreRouter.DATABASE_NAME when writing to blockstore app tables. - """ - if self._use_blockstore(model): - return self.DATABASE_NAME - return None - - def allow_relation(self, obj1, obj2, **hints): # pylint: disable=unused-argument - """ - Allow relations if both objects are blockstore app models. - """ - if self._use_blockstore(obj1) and self._use_blockstore(obj2): - return True - return None - - def allow_migrate(self, db, app_label, model_name=None, **hints): # pylint: disable=unused-argument - """ - Ensure the blockstore tables only appear in the blockstore database. - """ - if model_name is not None: - model = hints.get('model') - if model is not None and self._use_blockstore(model): - return db == self.DATABASE_NAME - if db == self.DATABASE_NAME: - return False - - return None diff --git a/openedx/core/lib/blockstore_api/tests/__init__.py b/openedx/core/lib/blockstore_api/tests/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/openedx/core/lib/blockstore_api/tests/base.py b/openedx/core/lib/blockstore_api/tests/base.py deleted file mode 100644 index 1d202d7671b5..000000000000 --- a/openedx/core/lib/blockstore_api/tests/base.py +++ /dev/null @@ -1,36 +0,0 @@ -""" -Common code for tests that work with Blockstore -""" -from unittest import mock -from urllib.parse import urlparse - -from django.test.client import RequestFactory - - -class BlockstoreAppTestMixin: - """ - Sets up the environment for tests to be run using the installed Blockstore app. - """ - def setUp(self): - """ - Ensure there's an active request, so that bundle file URLs can be made absolute. - """ - super().setUp() - - # Patch the blockstore get_current_request to use our live_server_url - mock.patch('blockstore.apps.api.methods.get_current_request', - mock.Mock(return_value=self._get_current_request())).start() - self.addCleanup(mock.patch.stopall) - - def _get_current_request(self): - """ - Returns a request object using the live_server_url, if available. - """ - request_args = {} - if hasattr(self, 'live_server_url'): - live_server_url = urlparse(self.live_server_url) - name, port = live_server_url.netloc.split(':') - request_args['SERVER_NAME'] = name - request_args['SERVER_PORT'] = port or '80' - request_args['wsgi.url_scheme'] = live_server_url.scheme - return RequestFactory().request(**request_args) diff --git a/openedx/core/lib/blockstore_api/tests/test_blockstore_api.py b/openedx/core/lib/blockstore_api/tests/test_blockstore_api.py deleted file mode 100644 index 859b24e01d03..000000000000 --- a/openedx/core/lib/blockstore_api/tests/test_blockstore_api.py +++ /dev/null @@ -1,201 +0,0 @@ -""" -Tests for xblock_utils.py -""" - -from uuid import UUID - -import pytest -from django.test import TestCase - -from openedx.core.lib import blockstore_api as api -from openedx.core.lib.blockstore_api.tests.base import ( - BlockstoreAppTestMixin, -) - -# A fake UUID that won't represent any real bundle/draft/collection: -BAD_UUID = UUID('12345678-0000-0000-0000-000000000000') - - -class BlockstoreApiClientTestMixin: - """ - Tests for the Blockstore API Client. - - The goal of these tests is not to test that Blockstore works correctly, but - that the API client can interact with it and all the API client methods - work. - """ - - # Collections - - def test_nonexistent_collection(self): - """ Request a collection that doesn't exist -> CollectionNotFound """ - with pytest.raises(api.CollectionNotFound): - api.get_collection(BAD_UUID) - - def test_collection_crud(self): - """ Create, Fetch, Update, and Delete a Collection """ - title = "Fire 🔥 Collection" - # Create: - coll = api.create_collection(title) - assert coll.title == title - assert isinstance(coll.uuid, UUID) - # Fetch: - coll2 = api.get_collection(coll.uuid) - assert coll == coll2 - # Update: - new_title = "Air 🌀 Collection" - coll3 = api.update_collection(coll.uuid, title=new_title) - assert coll3.title == new_title - coll4 = api.get_collection(coll.uuid) - assert coll4.title == new_title - # Delete: - api.delete_collection(coll.uuid) - with pytest.raises(api.CollectionNotFound): - api.get_collection(coll.uuid) - - # Bundles - - def test_nonexistent_bundle(self): - """ Request a bundle that doesn't exist -> BundleNotFound """ - with pytest.raises(api.BundleNotFound): - api.get_bundle(BAD_UUID) - - def test_bundle_crud(self): - """ Create, Fetch, Update, and Delete a Bundle """ - coll = api.create_collection("Test Collection") - args = { - "title": "Water 💧 Bundle", - "slug": "h2o", - "description": "Sploosh", - } - # Create: - bundle = api.create_bundle(coll.uuid, **args) - for attr, value in args.items(): - assert getattr(bundle, attr) == value - assert isinstance(bundle.uuid, UUID) - # Fetch: - bundle2 = api.get_bundle(bundle.uuid) - assert bundle == bundle2 - # Update: - new_description = "Water Nation Bending Lessons" - bundle3 = api.update_bundle(bundle.uuid, description=new_description) - assert bundle3.description == new_description - bundle4 = api.get_bundle(bundle.uuid) - assert bundle4.description == new_description - # Delete: - api.delete_bundle(bundle.uuid) - with pytest.raises(api.BundleNotFound): - api.get_bundle(bundle.uuid) - - # Drafts, files, and reading/writing file contents: - - def test_nonexistent_draft(self): - """ Request a draft that doesn't exist -> DraftNotFound """ - with pytest.raises(api.DraftNotFound): - api.get_draft(BAD_UUID) - - def test_drafts_and_files(self): - """ - Test creating, reading, writing, committing, and reverting drafts and - files. - """ - coll = api.create_collection("Test Collection") - bundle = api.create_bundle(coll.uuid, title="Earth 🗿 Bundle", slug="earth", description="another test bundle") - # Create a draft - draft = api.get_or_create_bundle_draft(bundle.uuid, draft_name="test-draft") - assert draft.bundle_uuid == bundle.uuid - assert draft.name == 'test-draft' - assert draft.updated_at.year >= 2019 - # And retrieve it again: - draft2 = api.get_or_create_bundle_draft(bundle.uuid, draft_name="test-draft") - assert draft == draft2 - # Also test retrieving using get_draft - draft3 = api.get_draft(draft.uuid) - assert draft == draft3 - - # Write a file into the bundle: - api.write_draft_file(draft.uuid, "test.txt", b"initial version") - # Now the file should be visible in the draft: - draft_contents = api.get_bundle_file_data(bundle.uuid, "test.txt", use_draft=draft.name) - assert draft_contents == b'initial version' - api.commit_draft(draft.uuid) - - # Write a new version into the draft: - api.write_draft_file(draft.uuid, "test.txt", b"modified version") - published_contents = api.get_bundle_file_data(bundle.uuid, "test.txt") - assert published_contents == b'initial version' - draft_contents2 = api.get_bundle_file_data(bundle.uuid, "test.txt", use_draft=draft.name) - assert draft_contents2 == b'modified version' - # Now delete the draft: - api.delete_draft(draft.uuid) - draft_contents3 = api.get_bundle_file_data(bundle.uuid, "test.txt", use_draft=draft.name) - # Confirm the file is now reset: - assert draft_contents3 == b'initial version' - - # Finaly, test the get_bundle_file* methods: - file_info1 = api.get_bundle_file_metadata(bundle.uuid, "test.txt") - assert file_info1.path == 'test.txt' - assert file_info1.size == len(b'initial version') - assert file_info1.hash_digest == 'a45a5c6716276a66c4005534a51453ab16ea63c4' - - assert list(api.get_bundle_files(bundle.uuid)) == [file_info1] - assert api.get_bundle_files_dict(bundle.uuid) == {'test.txt': file_info1} - - # Links - - def test_links(self): - """ - Test operations involving bundle links. - """ - coll = api.create_collection("Test Collection") - # Create two library bundles and a course bundle: - lib1_bundle = api.create_bundle(coll.uuid, title="Library 1", slug="lib1") - lib1_draft = api.get_or_create_bundle_draft(lib1_bundle.uuid, draft_name="test-draft") - lib2_bundle = api.create_bundle(coll.uuid, title="Library 1", slug="lib2") - lib2_draft = api.get_or_create_bundle_draft(lib2_bundle.uuid, draft_name="other-draft") - course_bundle = api.create_bundle(coll.uuid, title="Library 1", slug="course") - course_draft = api.get_or_create_bundle_draft(course_bundle.uuid, draft_name="test-draft") - - # To create links, we need valid BundleVersions, which requires having committed at least one change: - api.write_draft_file(lib1_draft.uuid, "lib1-data.txt", "hello world") - api.commit_draft(lib1_draft.uuid) # Creates version 1 - api.write_draft_file(lib2_draft.uuid, "lib2-data.txt", "hello world") - api.commit_draft(lib2_draft.uuid) # Creates version 1 - - # Lib2 has no links: - assert not api.get_bundle_links(lib2_bundle.uuid) - - # Create a link from lib2 to lib1 - link1_name = "lib2_to_lib1" - api.set_draft_link(lib2_draft.uuid, link1_name, lib1_bundle.uuid, version=1) - # Now confirm the link exists in the draft: - lib2_draft_links = api.get_bundle_links(lib2_bundle.uuid, use_draft=lib2_draft.name) - assert link1_name in lib2_draft_links - assert lib2_draft_links[link1_name].direct.bundle_uuid == lib1_bundle.uuid - assert lib2_draft_links[link1_name].direct.version == 1 - # Now commit the change to lib2: - api.commit_draft(lib2_draft.uuid) # Creates version 2 - - # Now create a link from course to lib2 - link2_name = "course_to_lib2" - api.set_draft_link(course_draft.uuid, link2_name, lib2_bundle.uuid, version=2) - api.commit_draft(course_draft.uuid) - - # And confirm the link exists in the resulting bundle version: - course_links = api.get_bundle_links(course_bundle.uuid) - assert link2_name in course_links - assert course_links[link2_name].direct.bundle_uuid == lib2_bundle.uuid - assert course_links[link2_name].direct.version == 2 - # And since the links go course->lib2->lib1, course has an indirect link to lib1: - assert course_links[link2_name].indirect[0].bundle_uuid == lib1_bundle.uuid - assert course_links[link2_name].indirect[0].version == 1 - - # Finally, test deleting a link from course's draft: - api.set_draft_link(course_draft.uuid, link2_name, None, None) - assert not api.get_bundle_links(course_bundle.uuid, use_draft=course_draft.name) - - -class BlockstoreAppApiClientTest(BlockstoreApiClientTestMixin, BlockstoreAppTestMixin, TestCase): - """ - Test the Blockstore API Client, using the installed Blockstore app. - """ diff --git a/openedx/core/lib/xblock_serializer/api.py b/openedx/core/lib/xblock_serializer/api.py index 30dbc8321b3c..8ac1cd5717c3 100644 --- a/openedx/core/lib/xblock_serializer/api.py +++ b/openedx/core/lib/xblock_serializer/api.py @@ -2,7 +2,7 @@ Public python API for serializing XBlocks to OLX """ # pylint: disable=unused-import -from .block_serializer import StaticFile, XBlockSerializer, XBlockSerializerForBlockstore +from .block_serializer import StaticFile, XBlockSerializer, XBlockSerializerForLearningCore def serialize_xblock_to_olx(block): @@ -14,10 +14,10 @@ def serialize_xblock_to_olx(block): return XBlockSerializer(block) -def serialize_modulestore_block_for_blockstore(block): +def serialize_modulestore_block_for_learning_core(block): """ This class will serialize an XBlock, producing: - (1) A new definition ID for use in Blockstore + (1) A new definition ID for use in Learning Core (2) an XML string defining the XBlock and referencing the IDs of its children using syntax (which doesn't actually contain the OLX of its children, just refers to them, so you have to @@ -29,4 +29,4 @@ def serialize_modulestore_block_for_blockstore(block): we have around how we should rewrite this (e.g. are we going to remove ?). """ - return XBlockSerializerForBlockstore(block) + return XBlockSerializerForLearningCore(block) diff --git a/openedx/core/lib/xblock_serializer/block_serializer.py b/openedx/core/lib/xblock_serializer/block_serializer.py index af155a3900b4..966380f25061 100644 --- a/openedx/core/lib/xblock_serializer/block_serializer.py +++ b/openedx/core/lib/xblock_serializer/block_serializer.py @@ -137,10 +137,10 @@ def _serialize_html_block(self, block) -> etree.Element: return olx_node -class XBlockSerializerForBlockstore(XBlockSerializer): +class XBlockSerializerForLearningCore(XBlockSerializer): """ This class will serialize an XBlock, producing: - (1) A new definition ID for use in Blockstore + (1) A new definition ID for use in Learning Core (2) an XML string defining the XBlock and referencing the IDs of its children using syntax (which doesn't actually contain the OLX of its children, just refers to them, so you have to @@ -154,7 +154,7 @@ def __init__(self, block): resulting data in this object. """ super().__init__(block) - self.def_id = utils.blockstore_def_key_from_modulestore_usage_key(self.orig_block_key) + self.def_id = utils.learning_core_def_key_from_modulestore_usage_key(self.orig_block_key) def _serialize_block(self, block) -> etree.Element: """ Serialize an XBlock to OLX/XML. """ @@ -174,12 +174,12 @@ def _serialize_children(self, block, parent_olx_node): # the same block to be used in many places (each with a unique # usage key). However, that functionality is not exposed in # Studio (other than via content libraries). So when we import - # into Blockstore, we assume that each usage is unique, don't + # into Learning Core, we assume that each usage is unique, don't # generate a usage key, and create a new "definition key" from # the original usage key. # So modulestore usage key # block-v1:A+B+C+type@html+block@introduction - # will become Blockstore definition key + # will become Learning Core definition key # html+introduction # # If we needed the real definition key, we could get it via @@ -187,7 +187,7 @@ def _serialize_children(self, block, parent_olx_node): # child_def_id = str(child.scope_ids.def_id) # and then use # - def_id = utils.blockstore_def_key_from_modulestore_usage_key(child_id) + def_id = utils.learning_core_def_key_from_modulestore_usage_key(child_id) parent_olx_node.append(parent_olx_node.makeelement("xblock-include", {"definition": def_id})) def _transform_olx(self, olx_node, usage_id): diff --git a/openedx/core/lib/xblock_serializer/test_api.py b/openedx/core/lib/xblock_serializer/test_api.py index 39ca6bd675e4..6b2c0dfeb988 100644 --- a/openedx/core/lib/xblock_serializer/test_api.py +++ b/openedx/core/lib/xblock_serializer/test_api.py @@ -240,17 +240,17 @@ def test_html_with_static_asset(self): ), ]) - def test_html_with_static_asset_blockstore(self): + def test_html_with_static_asset_learning_core(self): """ - Test the blockstore-specific serialization of an HTML block + Test the learning-core-specific serialization of an HTML block """ block_id = self.course.id.make_usage_key('html', 'just_img') # see sample_courses.py html_block = modulestore().get_item(block_id) serialized = api.serialize_xblock_to_olx(html_block) - serialized_blockstore = api.serialize_modulestore_block_for_blockstore(html_block) + serialized_learning_core = api.serialize_modulestore_block_for_learning_core(html_block) self.assertXmlEqual( - serialized_blockstore.olx_str, - # For blockstore, OLX should never contain "url_name" as that ID is specified by the filename: + serialized_learning_core.olx_str, + # For learning core, OLX should never contain "url_name" as that ID is specified by the filename: """ @@ -259,9 +259,9 @@ def test_html_with_static_asset_blockstore(self): ) self.assertIn("CDATA", serialized.olx_str) # Static files should be identical: - self.assertEqual(serialized.static_files, serialized_blockstore.static_files) - # This is the only other difference - an extra field with the blockstore-specific definition ID: - self.assertEqual(serialized_blockstore.def_id, "html/just_img") + self.assertEqual(serialized.static_files, serialized_learning_core.static_files) + # This is the only other difference - an extra field with the learning-core-specific definition ID: + self.assertEqual(serialized_learning_core.def_id, "html/just_img") def test_html_with_fields(self): """ Test an HTML Block with non-default fields like editor='raw' """ @@ -299,13 +299,13 @@ def test_export_sequential(self): self.assertXmlEqual(serialized.olx_str, EXPECTED_SEQUENTIAL_OLX) - def test_export_sequential_blockstore(self): + def test_export_sequential_learning_core(self): """ - Export a sequential from the toy course, formatted for blockstore. + Export a sequential from the toy course, formatted for learning core. """ sequential_id = self.course.id.make_usage_key('sequential', 'Toy_Videos') # see sample_courses.py sequential = modulestore().get_item(sequential_id) - serialized = api.serialize_modulestore_block_for_blockstore(sequential) + serialized = api.serialize_modulestore_block_for_learning_core(sequential) self.assertXmlEqual(serialized.olx_str, """ diff --git a/openedx/core/lib/xblock_serializer/utils.py b/openedx/core/lib/xblock_serializer/utils.py index 2c736ae2998f..e78c900b1887 100644 --- a/openedx/core/lib/xblock_serializer/utils.py +++ b/openedx/core/lib/xblock_serializer/utils.py @@ -225,17 +225,17 @@ def override_export_fs(block): XmlMixin.export_to_file = old_global_export_to_file -def blockstore_def_key_from_modulestore_usage_key(usage_key): +def learning_core_def_key_from_modulestore_usage_key(usage_key): """ In modulestore, the "definition key" is a MongoDB ObjectID kept in split's definitions table, which theoretically allows the same block to be used in many places (each with a unique usage key). However, that functionality is not exposed in Studio (other than via content libraries). So when we import - into Blockstore, we assume that each usage is unique, don't generate a usage + into learning core, we assume that each usage is unique, don't generate a usage key, and create a new "definition key" from the original usage key. So modulestore usage key block-v1:A+B+C+type@html+block@introduction - will become Blockstore definition key + will become learning core definition key html/introduction """ block_type = usage_key.block_type diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 0c63a94998ed..ff4319a4fc7c 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -44,7 +44,6 @@ attrs==23.2.0 # edx-ace # jsonschema # lti-consumer-xblock - # openedx-blockstore # openedx-events # openedx-learning # referencing @@ -205,7 +204,6 @@ django==4.2.13 # djangorestframework # done-xblock # drf-jwt - # drf-nested-routers # drf-spectacular # drf-yasg # edx-ace @@ -239,7 +237,6 @@ django==4.2.13 # help-tokens # jsonfield # lti-consumer-xblock - # openedx-blockstore # openedx-django-pyfs # openedx-django-wiki # openedx-events @@ -278,8 +275,6 @@ django-crum==0.7.9 # edx-rbac # edx-toggles # super-csv -django-environ==0.11.2 - # via openedx-blockstore django-fernet-fields-v2==0.9 # via edx-enterprise django-filter==24.2 @@ -287,7 +282,6 @@ django-filter==24.2 # -r requirements/edx/kernel.in # edx-enterprise # lti-consumer-xblock - # openedx-blockstore django-ipware==7.0.1 # via # -r requirements/edx/kernel.in @@ -368,7 +362,6 @@ django-waffle==4.1.0 # edx-enterprise # edx-proctoring # edx-toggles - # openedx-blockstore django-webpack-loader==0.7.0 # via # -c requirements/edx/../constraints.txt @@ -381,7 +374,6 @@ djangorestframework==3.14.0 # django-config-models # django-user-tasks # drf-jwt - # drf-nested-routers # drf-spectacular # drf-yasg # edx-api-doc-tools @@ -392,7 +384,6 @@ djangorestframework==3.14.0 # edx-organizations # edx-proctoring # edx-submissions - # openedx-blockstore # openedx-learning # ora2 # super-csv @@ -402,8 +393,6 @@ done-xblock==2.3.0 # via -r requirements/edx/bundled.in drf-jwt==1.19.2 # via edx-drf-extensions -drf-nested-routers==0.93.5 - # via openedx-blockstore drf-spectacular==0.27.2 # via -r requirements/edx/kernel.in drf-yasg==1.21.5 @@ -417,11 +406,8 @@ edx-api-doc-tools==1.8.0 # via # -r requirements/edx/kernel.in # edx-name-affirmation - # openedx-blockstore edx-auth-backends==4.3.0 - # via - # -r requirements/edx/kernel.in - # openedx-blockstore + # via -r requirements/edx/kernel.in edx-braze-client==0.2.5 # via # -r requirements/edx/bundled.in @@ -448,7 +434,6 @@ edx-django-release-util==1.4.0 # via # -r requirements/edx/kernel.in # edxval - # openedx-blockstore edx-django-sites-extensions==4.2.0 # via -r requirements/edx/kernel.in edx-django-utils==5.13.0 @@ -464,7 +449,6 @@ edx-django-utils==5.13.0 # edx-toggles # edx-when # event-tracking - # openedx-blockstore # openedx-events # ora2 # super-csv @@ -738,9 +722,7 @@ multidict==6.0.5 # aiohttp # yarl mysqlclient==2.2.4 - # via - # -r requirements/edx/kernel.in - # openedx-blockstore + # via -r requirements/edx/kernel.in newrelic==9.9.0 # via # -r requirements/edx/bundled.in @@ -770,8 +752,6 @@ openai==0.28.1 # edx-enterprise openedx-atlas==0.6.0 # via -r requirements/edx/kernel.in -openedx-blockstore==1.4.0 - # via -r requirements/edx/kernel.in openedx-calc==3.1.0 # via -r requirements/edx/kernel.in openedx-django-pyfs==3.6.0 @@ -971,7 +951,6 @@ pytz==2024.1 # icalendar # interchange # olxcleaner - # openedx-blockstore # ora2 # snowflake-connector-python # xblock @@ -1119,9 +1098,7 @@ sortedcontainers==2.4.0 soupsieve==2.5 # via beautifulsoup4 sqlparse==0.5.0 - # via - # django - # openedx-blockstore + # via django staff-graded-xblock==2.3.0 # via -r requirements/edx/bundled.in stevedore==5.2.0 diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 53e3f7a1c7a7..7cf594a15e99 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -96,7 +96,6 @@ attrs==23.2.0 # edx-ace # jsonschema # lti-consumer-xblock - # openedx-blockstore # openedx-events # openedx-learning # referencing @@ -379,7 +378,6 @@ django==4.2.13 # djangorestframework # done-xblock # drf-jwt - # drf-nested-routers # drf-spectacular # drf-yasg # edx-ace @@ -413,7 +411,6 @@ django==4.2.13 # help-tokens # jsonfield # lti-consumer-xblock - # openedx-blockstore # openedx-django-pyfs # openedx-django-wiki # openedx-events @@ -470,11 +467,6 @@ django-crum==0.7.9 # super-csv django-debug-toolbar==4.3.0 # via -r requirements/edx/development.in -django-environ==0.11.2 - # via - # -r requirements/edx/doc.txt - # -r requirements/edx/testing.txt - # openedx-blockstore django-fernet-fields-v2==0.9 # via # -r requirements/edx/doc.txt @@ -486,7 +478,6 @@ django-filter==24.2 # -r requirements/edx/testing.txt # edx-enterprise # lti-consumer-xblock - # openedx-blockstore django-ipware==7.0.1 # via # -r requirements/edx/doc.txt @@ -604,7 +595,6 @@ django-waffle==4.1.0 # edx-enterprise # edx-proctoring # edx-toggles - # openedx-blockstore django-webpack-loader==0.7.0 # via # -c requirements/edx/../constraints.txt @@ -619,7 +609,6 @@ djangorestframework==3.14.0 # django-config-models # django-user-tasks # drf-jwt - # drf-nested-routers # drf-spectacular # drf-yasg # edx-api-doc-tools @@ -630,7 +619,6 @@ djangorestframework==3.14.0 # edx-organizations # edx-proctoring # edx-submissions - # openedx-blockstore # openedx-learning # ora2 # super-csv @@ -662,11 +650,6 @@ drf-jwt==1.19.2 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edx-drf-extensions -drf-nested-routers==0.93.5 - # via - # -r requirements/edx/doc.txt - # -r requirements/edx/testing.txt - # openedx-blockstore drf-spectacular==0.27.2 # via # -r requirements/edx/doc.txt @@ -687,12 +670,10 @@ edx-api-doc-tools==1.8.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edx-name-affirmation - # openedx-blockstore edx-auth-backends==4.3.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt - # openedx-blockstore edx-braze-client==0.2.5 # via # -r requirements/edx/doc.txt @@ -728,7 +709,6 @@ edx-django-release-util==1.4.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edxval - # openedx-blockstore edx-django-sites-extensions==4.2.0 # via # -r requirements/edx/doc.txt @@ -747,7 +727,6 @@ edx-django-utils==5.13.0 # edx-toggles # edx-when # event-tracking - # openedx-blockstore # openedx-events # ora2 # super-csv @@ -1275,7 +1254,6 @@ mysqlclient==2.2.4 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt - # openedx-blockstore newrelic==9.9.0 # via # -r requirements/edx/doc.txt @@ -1321,10 +1299,6 @@ openedx-atlas==0.6.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -openedx-blockstore==1.4.0 - # via - # -r requirements/edx/doc.txt - # -r requirements/edx/testing.txt openedx-calc==3.1.0 # via # -r requirements/edx/doc.txt @@ -1743,7 +1717,6 @@ pytz==2024.1 # icalendar # interchange # olxcleaner - # openedx-blockstore # ora2 # snowflake-connector-python # xblock @@ -2031,7 +2004,6 @@ sqlparse==0.5.0 # -r requirements/edx/testing.txt # django # django-debug-toolbar - # openedx-blockstore staff-graded-xblock==2.3.0 # via # -r requirements/edx/doc.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index d553ab6ae24d..57c0936a0989 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -63,7 +63,6 @@ attrs==23.2.0 # edx-ace # jsonschema # lti-consumer-xblock - # openedx-blockstore # openedx-events # openedx-learning # referencing @@ -255,7 +254,6 @@ django==4.2.13 # djangorestframework # done-xblock # drf-jwt - # drf-nested-routers # drf-spectacular # drf-yasg # edx-ace @@ -289,7 +287,6 @@ django==4.2.13 # help-tokens # jsonfield # lti-consumer-xblock - # openedx-blockstore # openedx-django-pyfs # openedx-django-wiki # openedx-events @@ -334,10 +331,6 @@ django-crum==0.7.9 # edx-rbac # edx-toggles # super-csv -django-environ==0.11.2 - # via - # -r requirements/edx/base.txt - # openedx-blockstore django-fernet-fields-v2==0.9 # via # -r requirements/edx/base.txt @@ -347,7 +340,6 @@ django-filter==24.2 # -r requirements/edx/base.txt # edx-enterprise # lti-consumer-xblock - # openedx-blockstore django-ipware==7.0.1 # via # -r requirements/edx/base.txt @@ -434,7 +426,6 @@ django-waffle==4.1.0 # edx-enterprise # edx-proctoring # edx-toggles - # openedx-blockstore django-webpack-loader==0.7.0 # via # -c requirements/edx/../constraints.txt @@ -447,7 +438,6 @@ djangorestframework==3.14.0 # django-config-models # django-user-tasks # drf-jwt - # drf-nested-routers # drf-spectacular # drf-yasg # edx-api-doc-tools @@ -458,7 +448,6 @@ djangorestframework==3.14.0 # edx-organizations # edx-proctoring # edx-submissions - # openedx-blockstore # openedx-learning # ora2 # super-csv @@ -477,10 +466,6 @@ drf-jwt==1.19.2 # via # -r requirements/edx/base.txt # edx-drf-extensions -drf-nested-routers==0.93.5 - # via - # -r requirements/edx/base.txt - # openedx-blockstore drf-spectacular==0.27.2 # via -r requirements/edx/base.txt drf-yasg==1.21.5 @@ -495,11 +480,8 @@ edx-api-doc-tools==1.8.0 # via # -r requirements/edx/base.txt # edx-name-affirmation - # openedx-blockstore edx-auth-backends==4.3.0 - # via - # -r requirements/edx/base.txt - # openedx-blockstore + # via -r requirements/edx/base.txt edx-braze-client==0.2.5 # via # -r requirements/edx/base.txt @@ -526,7 +508,6 @@ edx-django-release-util==1.4.0 # via # -r requirements/edx/base.txt # edxval - # openedx-blockstore edx-django-sites-extensions==4.2.0 # via -r requirements/edx/base.txt edx-django-utils==5.13.0 @@ -542,7 +523,6 @@ edx-django-utils==5.13.0 # edx-toggles # edx-when # event-tracking - # openedx-blockstore # openedx-events # ora2 # super-csv @@ -869,9 +849,7 @@ multidict==6.0.5 # aiohttp # yarl mysqlclient==2.2.4 - # via - # -r requirements/edx/base.txt - # openedx-blockstore + # via -r requirements/edx/base.txt newrelic==9.9.0 # via # -r requirements/edx/base.txt @@ -905,8 +883,6 @@ openai==0.28.1 # edx-enterprise openedx-atlas==0.6.0 # via -r requirements/edx/base.txt -openedx-blockstore==1.4.0 - # via -r requirements/edx/base.txt openedx-calc==3.1.0 # via -r requirements/edx/base.txt openedx-django-pyfs==3.6.0 @@ -1151,7 +1127,6 @@ pytz==2024.1 # icalendar # interchange # olxcleaner - # openedx-blockstore # ora2 # snowflake-connector-python # xblock @@ -1363,7 +1338,6 @@ sqlparse==0.5.0 # via # -r requirements/edx/base.txt # django - # openedx-blockstore staff-graded-xblock==2.3.0 # via -r requirements/edx/base.txt stevedore==5.2.0 diff --git a/requirements/edx/github.in b/requirements/edx/github.in index ea6d47eec8a0..6ec36d3a0681 100644 --- a/requirements/edx/github.in +++ b/requirements/edx/github.in @@ -29,16 +29,16 @@ # # For example: # -# # https://github.com/openedx/blockstore/issues/212 -# git+https://github.com/openedx/blockstore.git@v1.3.0#egg=openedx-blockstore==1.3.0 +# # https://github.com/openedx/foobar/issues/212 +# git+https://github.com/openedx/foobar.git@v1.3.0#egg=openedx-foobar==1.3.0 # # where: # -# ISSUE-LINK = https://github.com/openedx/blockstore/issues/212 +# ISSUE-LINK = https://github.com/openedx/foobar/issues/212 # OWNER = openedx -# REPO-NAME = blockstore +# REPO-NAME = foobar # TAG-OR-SHA = v1.3.0 -# DIST-NAME = openedx-blockstore +# DIST-NAME = openedx-foobar # VERSION = 1.3.0 # # Rules to follow: diff --git a/requirements/edx/kernel.in b/requirements/edx/kernel.in index 47a2435939af..1a7abb83641a 100644 --- a/requirements/edx/kernel.in +++ b/requirements/edx/kernel.in @@ -120,7 +120,6 @@ openedx-filters # Open edX Filters from Hooks Extension Fram openedx-learning # Open edX Learning core (experimental) openedx-mongodbproxy openedx-django-wiki -openedx-blockstore path piexif # Exif image metadata manipulation, used in the profile_images app Pillow # Image manipulation library; used for course assets, profile images, invoice PDFs, etc. diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 314f28d1cb02..45e2b76578a6 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -68,7 +68,6 @@ attrs==23.2.0 # edx-ace # jsonschema # lti-consumer-xblock - # openedx-blockstore # openedx-events # openedx-learning # referencing @@ -291,7 +290,6 @@ django==4.2.13 # djangorestframework # done-xblock # drf-jwt - # drf-nested-routers # drf-spectacular # drf-yasg # edx-ace @@ -325,7 +323,6 @@ django==4.2.13 # help-tokens # jsonfield # lti-consumer-xblock - # openedx-blockstore # openedx-django-pyfs # openedx-django-wiki # openedx-events @@ -370,10 +367,6 @@ django-crum==0.7.9 # edx-rbac # edx-toggles # super-csv -django-environ==0.11.2 - # via - # -r requirements/edx/base.txt - # openedx-blockstore django-fernet-fields-v2==0.9 # via # -r requirements/edx/base.txt @@ -383,7 +376,6 @@ django-filter==24.2 # -r requirements/edx/base.txt # edx-enterprise # lti-consumer-xblock - # openedx-blockstore django-ipware==7.0.1 # via # -r requirements/edx/base.txt @@ -470,7 +462,6 @@ django-waffle==4.1.0 # edx-enterprise # edx-proctoring # edx-toggles - # openedx-blockstore django-webpack-loader==0.7.0 # via # -c requirements/edx/../constraints.txt @@ -483,7 +474,6 @@ djangorestframework==3.14.0 # django-config-models # django-user-tasks # drf-jwt - # drf-nested-routers # drf-spectacular # drf-yasg # edx-api-doc-tools @@ -494,7 +484,6 @@ djangorestframework==3.14.0 # edx-organizations # edx-proctoring # edx-submissions - # openedx-blockstore # openedx-learning # ora2 # super-csv @@ -510,10 +499,6 @@ drf-jwt==1.19.2 # via # -r requirements/edx/base.txt # edx-drf-extensions -drf-nested-routers==0.93.5 - # via - # -r requirements/edx/base.txt - # openedx-blockstore drf-spectacular==0.27.2 # via -r requirements/edx/base.txt drf-yasg==1.21.5 @@ -528,11 +513,8 @@ edx-api-doc-tools==1.8.0 # via # -r requirements/edx/base.txt # edx-name-affirmation - # openedx-blockstore edx-auth-backends==4.3.0 - # via - # -r requirements/edx/base.txt - # openedx-blockstore + # via -r requirements/edx/base.txt edx-braze-client==0.2.5 # via # -r requirements/edx/base.txt @@ -559,7 +541,6 @@ edx-django-release-util==1.4.0 # via # -r requirements/edx/base.txt # edxval - # openedx-blockstore edx-django-sites-extensions==4.2.0 # via -r requirements/edx/base.txt edx-django-utils==5.13.0 @@ -575,7 +556,6 @@ edx-django-utils==5.13.0 # edx-toggles # edx-when # event-tracking - # openedx-blockstore # openedx-events # ora2 # super-csv @@ -954,9 +934,7 @@ multidict==6.0.5 # aiohttp # yarl mysqlclient==2.2.4 - # via - # -r requirements/edx/base.txt - # openedx-blockstore + # via -r requirements/edx/base.txt newrelic==9.9.0 # via # -r requirements/edx/base.txt @@ -990,8 +968,6 @@ openai==0.28.1 # edx-enterprise openedx-atlas==0.6.0 # via -r requirements/edx/base.txt -openedx-blockstore==1.4.0 - # via -r requirements/edx/base.txt openedx-calc==3.1.0 # via -r requirements/edx/base.txt openedx-django-pyfs==3.6.0 @@ -1310,7 +1286,6 @@ pytz==2024.1 # icalendar # interchange # olxcleaner - # openedx-blockstore # ora2 # snowflake-connector-python # xblock @@ -1493,7 +1468,6 @@ sqlparse==0.5.0 # via # -r requirements/edx/base.txt # django - # openedx-blockstore staff-graded-xblock==2.3.0 # via -r requirements/edx/base.txt starlette==0.37.2 diff --git a/xmodule/README.rst b/xmodule/README.rst index 5096c78c2abe..8bbae713b7a4 100644 --- a/xmodule/README.rst +++ b/xmodule/README.rst @@ -41,14 +41,14 @@ Direction Currently, this directory contains a lot of mission-critical functionality, so continued maintenance and simplification of it is important. Still, we aim to eventually dissolve the directory in favor of more focused & decoupled subsystems: -* ModuleStore is superseded by the `Blockstore`_ storage backend. -* Blockstore-backend content is rendered by a new, simplified `edx-platform XBlock runtime`_. +* ModuleStore is superseded by the `Learning Core`_ storage backend. +* Learning Core-backend content is rendered by a new, simplified `edx-platform XBlock runtime`_. * Navigation, partitioning, and composition of learning content is being re-architected in the `openedx-learning`_ library. * All new XBlocks are implemented in separate repositories, such as `xblock-drag-and-drop-v2`_. To help with this direction, please **do not add new functionality to this directory**. If you feel that you need to add code to this directory, reach out on `the forums`_; it's likely that someone can help you find a different way to implement your change that will be more robust and architecturally sound! -.. _Blockstore: https://github.com/openedx/blockstore/ +.. _Learning Core: https://github.com/openedx/openedx-learning/ .. _edx-platform XBlock runtime: https://github.com/openedx/edx-platform/tree/master/openedx/core/djangoapps/xblock .. _openedx-learning: https://github.com/openedx/openedx-learning .. _xblock-drag-and-drop-v2: https://github.com/openedx/xblock-drag-and-drop-v2 diff --git a/xmodule/html_block.py b/xmodule/html_block.py index 6c883e1322d2..2db198360107 100644 --- a/xmodule/html_block.py +++ b/xmodule/html_block.py @@ -279,7 +279,7 @@ def load_definition(cls, xml_object, system, location, id_generator): # lint-am @classmethod def parse_xml_new_runtime(cls, node, runtime, keys): """ - Parse XML in the new blockstore-based runtime. Since it doesn't yet + Parse XML in the new learning-core-based runtime. Since it doesn't yet support loading separate .html files, the HTML data is assumed to be in a CDATA child or otherwise just inline in the OLX. """ diff --git a/xmodule/library_tools.py b/xmodule/library_tools.py index 7aa0362ea47c..2c077a888482 100644 --- a/xmodule/library_tools.py +++ b/xmodule/library_tools.py @@ -25,7 +25,7 @@ class LibraryToolsService: """ Service for LibraryContentBlock. - Allows to interact with libraries in the modulestore and blockstore. + Allows to interact with libraries in the modulestore and learning core. Should only be used in the CMS. """ diff --git a/xmodule/raw_block.py b/xmodule/raw_block.py index 58b21795f282..f24dd7712d85 100644 --- a/xmodule/raw_block.py +++ b/xmodule/raw_block.py @@ -66,7 +66,7 @@ def parse_xml_new_runtime(cls, node, runtime, keys): Interpret the parsed XML in `node`, creating a new instance of this module. """ - # In the new/blockstore-based runtime, XModule parsing (from + # In the new/learning-core-based runtime, XModule parsing (from # XmlMixin) is disabled, so definition_from_xml will not be # called, and instead the "normal" XBlock parse_xml will be used. # However, it's not compatible with RawMixin, so we implement diff --git a/xmodule/tests/test_library_tools.py b/xmodule/tests/test_library_tools.py index c6185f8754b1..f93066cd5c63 100644 --- a/xmodule/tests/test_library_tools.py +++ b/xmodule/tests/test_library_tools.py @@ -28,7 +28,7 @@ class ContentLibraryToolsTest(MixedSplitTestCase, ContentLibrariesRestApiTest): """ Tests for LibraryToolsService. - Tests interaction with blockstore-based (V2) and mongo-based (V1) content libraries. + Tests interaction with learning-core-based (V2) and mongo-based (V1) content libraries. """ def setUp(self): super().setUp() diff --git a/xmodule/video_block/transcripts_utils.py b/xmodule/video_block/transcripts_utils.py index 16de851ea5f2..d82a5d3f4789 100644 --- a/xmodule/video_block/transcripts_utils.py +++ b/xmodule/video_block/transcripts_utils.py @@ -15,7 +15,7 @@ import simplejson as json from django.conf import settings from lxml import etree -from opaque_keys.edx.locator import BundleDefinitionLocator +from opaque_keys.edx.keys import UsageKeyV2 from pysrt import SubRipFile, SubRipItem, SubRipTime from pysrt.srtexc import Error @@ -945,7 +945,7 @@ def get_transcript_for_video(video_location, subs_id, file_name, language): """ Get video transcript from content store. This is a lower level function and is used by `get_transcript_from_contentstore`. Prefer that function instead where possible. If you - need to support getting transcripts from VAL or Blockstore as well, use the `get_transcript` + need to support getting transcripts from VAL or Learning Core as well, use the `get_transcript` function instead. NOTE: Transcripts can be searched from content store by two ways: @@ -1033,29 +1033,31 @@ def get_transcript_from_contentstore(video, language, output_format, transcripts return transcript_content, transcript_name, Transcript.mime_types[output_format] -def get_transcript_from_blockstore(video_block, language, output_format, transcripts_info): +def get_transcript_from_learning_core(video_block, language, output_format, transcripts_info): """ - Get video transcript from Blockstore. - - Blockstore expects video transcripts to be placed into the 'static/' - subfolder of the XBlock's folder in a Blockstore bundle. For example, if the - video XBlock's definition is in the standard location of - video/video1/definition.xml - Then the .srt files should be placed at e.g. - video/video1/static/video1-en.srt - This is the same place where other public static files are placed for other - XBlocks, such as image files used by HTML blocks. - - Video XBlocks in Blockstore must set the 'transcripts' XBlock field to a - JSON dictionary listing the filename of the transcript for each language: -