diff --git a/.github/workflows/js-tests.yml b/.github/workflows/js-tests.yml index c9d2d7ab1191..f1f12d7c24c2 100644 --- a/.github/workflows/js-tests.yml +++ b/.github/workflows/js-tests.yml @@ -61,7 +61,7 @@ jobs: - name: Install Required Python Dependencies run: | - make base-requirements + make test-requirements - uses: c-hive/gha-npm-cache@v1 - name: Run JS Tests diff --git a/openedx/core/djangoapps/theming/management/commands/compile_sass.py b/openedx/core/djangoapps/theming/management/commands/compile_sass.py deleted file mode 100644 index fbfdd2f222a4..000000000000 --- a/openedx/core/djangoapps/theming/management/commands/compile_sass.py +++ /dev/null @@ -1,112 +0,0 @@ -""" -Management command for compiling sass. - -DEPRECATED in favor of `npm run compile-sass`. -""" -import shlex - -from django.core.management import BaseCommand -from django.conf import settings - -from pavelib.assets import run_deprecated_command_wrapper - - -class Command(BaseCommand): - """ - Compile theme sass and collect theme assets. - """ - - help = "DEPRECATED. Use 'npm run compile-sass' instead." - - # NOTE (CCB): This allows us to compile static assets in Docker containers without database access. - requires_system_checks = [] - - def add_arguments(self, parser): - """ - Add arguments for compile_sass command. - - Args: - parser (django.core.management.base.CommandParser): parsed for parsing command line arguments. - """ - parser.add_argument( - 'system', type=str, nargs='*', default=["lms", "cms"], - help="lms or studio", - ) - - # Named (optional) arguments - parser.add_argument( - '--theme-dirs', - dest='theme_dirs', - type=str, - nargs='+', - default=None, - help="List of dirs where given themes would be looked.", - ) - - parser.add_argument( - '--themes', - type=str, - nargs='+', - default=["all"], - help="List of themes whose sass need to compiled. Or 'no'/'all' to compile for no/all themes.", - ) - - # Named (optional) arguments - parser.add_argument( - '--force', - action='store_true', - default=False, - help="DEPRECATED. Full recompilation is now always forced.", - ) - parser.add_argument( - '--debug', - action='store_true', - default=False, - help="Disable Sass compression", - ) - - def handle(self, *args, **options): - """ - Handle compile_sass command. - """ - systems = set( - {"lms": "lms", "cms": "cms", "studio": "cms"}[sys] - for sys in options.get("system", ["lms", "cms"]) - ) - theme_dirs = options.get("theme_dirs") or settings.COMPREHENSIVE_THEME_DIRS or [] - themes_option = options.get("themes") or [] # '[]' means 'all' - if not settings.ENABLE_COMPREHENSIVE_THEMING: - compile_themes = False - themes = [] - elif "no" in themes_option: - compile_themes = False - themes = [] - elif "all" in themes_option: - compile_themes = True - themes = [] - else: - compile_themes = True - themes = themes_option - run_deprecated_command_wrapper( - old_command="./manage.py [lms|cms] compile_sass", - ignored_old_flags=list(set(["force"]) & set(options)), - new_command=shlex.join([ - "npm", - "run", - ("compile-sass-dev" if options.get("debug") else "compile-sass"), - "--", - *(["--skip-lms"] if "lms" not in systems else []), - *(["--skip-cms"] if "cms" not in systems else []), - *(["--skip-themes"] if not compile_themes else []), - *( - arg - for theme_dir in theme_dirs - for arg in ["--theme-dir", str(theme_dir)] - ), - *( - arg - for theme in themes - for arg in ["--theme", theme] - ), - ]), - ) diff --git a/pavelib/__init__.py b/pavelib/__init__.py index 875068166ff5..3307f120714a 100644 --- a/pavelib/__init__.py +++ b/pavelib/__init__.py @@ -3,4 +3,4 @@ """ -from . import assets, js_test, prereqs, quality +from . import js_test, quality diff --git a/pavelib/assets.py b/pavelib/assets.py deleted file mode 100644 index f437b6427f93..000000000000 --- a/pavelib/assets.py +++ /dev/null @@ -1,506 +0,0 @@ -""" -Asset compilation and collection. - -This entire module is DEPRECATED. In Redwood, it exists just as a collection of temporary compatibility wrappers. -In Sumac, this module will be deleted. To migrate, follow the advice in the printed warnings and/or read the -instructions on the DEPR ticket: https://github.com/openedx/edx-platform/issues/31895 -""" - -import argparse -import glob -import json -import shlex -import traceback -from functools import wraps -from threading import Timer - -from paver import tasks -from paver.easy import call_task, cmdopts, consume_args, needs, no_help, sh, task -from watchdog.events import PatternMatchingEventHandler -from watchdog.observers import Observer # pylint: disable=unused-import # Used by Tutor. Remove after Sumac cut. - -from .utils.cmd import django_cmd -from .utils.envs import Env -from .utils.timer import timed - - -SYSTEMS = { - 'lms': 'lms', - 'cms': 'cms', - 'studio': 'cms', -} - -WARNING_SYMBOLS = "⚠️ " * 50 # A row of 'warning' emoji to catch CLI users' attention - - -def run_deprecated_command_wrapper(*, old_command, ignored_old_flags, new_command): - """ - Run the new version of shell command, plus a warning that the old version is deprecated. - """ - depr_warning = ( - "\n" + - f"{WARNING_SYMBOLS}\n" + - "\n" + - f"WARNING: '{old_command}' is DEPRECATED! It will be removed before Sumac.\n" + - "The command you ran is now just a temporary wrapper around a new,\n" + - "supported command, which you should use instead:\n" + - "\n" + - f"\t{new_command}\n" + - "\n" + - "Details: https://github.com/openedx/edx-platform/issues/31895\n" + - "".join( - f" WARNING: ignored deprecated paver flag '{flag}'\n" - for flag in ignored_old_flags - ) + - f"{WARNING_SYMBOLS}\n" + - "\n" - ) - # Print deprecation warning twice so that it's more likely to be seen in the logs. - print(depr_warning) - sh(new_command) - print(depr_warning) - - -def debounce(seconds=1): - """ - Prevents the decorated function from being called more than every `seconds` - seconds. Waits until calls stop coming in before calling the decorated - function. - - This is DEPRECATED. It exists in Redwood just to ease the transition for Tutor. - """ - def decorator(func): - func.timer = None - - @wraps(func) - def wrapper(*args, **kwargs): - def call(): - func(*args, **kwargs) - func.timer = None - if func.timer: - func.timer.cancel() - func.timer = Timer(seconds, call) - func.timer.start() - - return wrapper - return decorator - - -class SassWatcher(PatternMatchingEventHandler): - """ - Watches for sass file changes - - This is DEPRECATED. It exists in Redwood just to ease the transition for Tutor. - """ - ignore_directories = True - patterns = ['*.scss'] - - def register(self, observer, directories): - """ - register files with observer - Arguments: - observer (watchdog.observers.Observer): sass file observer - directories (list): list of directories to be register for sass watcher. - """ - for dirname in directories: - paths = [] - if '*' in dirname: - paths.extend(glob.glob(dirname)) - else: - paths.append(dirname) - - for obs_dirname in paths: - observer.schedule(self, obs_dirname, recursive=True) - - @debounce() - def on_any_event(self, event): - print('\tCHANGED:', event.src_path) - try: - compile_sass() # pylint: disable=no-value-for-parameter - except Exception: # pylint: disable=broad-except - traceback.print_exc() - - -@task -@no_help -@cmdopts([ - ('system=', 's', 'The system to compile sass for (defaults to all)'), - ('theme-dirs=', '-td', 'Theme dirs containing all themes (defaults to None)'), - ('themes=', '-t', 'The theme to compile sass for (defaults to None)'), - ('debug', 'd', 'Whether to use development settings'), - ('force', '', 'DEPRECATED. Full recompilation is now always forced.'), -]) -@timed -def compile_sass(options): - """ - Compile Sass to CSS. If command is called without any arguments, it will - only compile lms, cms sass for the open source theme. And none of the comprehensive theme's sass would be compiled. - - If you want to compile sass for all comprehensive themes you will have to run compile_sass - specifying all the themes that need to be compiled.. - - The following is a list of some possible ways to use this command. - - Command: - paver compile_sass - Description: - compile sass files for both lms and cms. If command is called like above (i.e. without any arguments) it will - only compile lms, cms sass for the open source theme. None of the theme's sass will be compiled. - - Command: - paver compile_sass --theme-dirs /edx/app/edxapp/edx-platform/themes --themes=red-theme - Description: - compile sass files for both lms and cms for 'red-theme' present in '/edx/app/edxapp/edx-platform/themes' - - Command: - paver compile_sass --theme-dirs=/edx/app/edxapp/edx-platform/themes --themes red-theme stanford-style - Description: - compile sass files for both lms and cms for 'red-theme' and 'stanford-style' present in - '/edx/app/edxapp/edx-platform/themes'. - - Command: - paver compile_sass --system=cms - --theme-dirs /edx/app/edxapp/edx-platform/themes /edx/app/edxapp/edx-platform/common/test/ - --themes red-theme stanford-style test-theme - Description: - compile sass files for cms only for 'red-theme', 'stanford-style' and 'test-theme' present in - '/edx/app/edxapp/edx-platform/themes' and '/edx/app/edxapp/edx-platform/common/test/'. - - This is a DEPRECATED COMPATIBILITY WRAPPER. Use `npm run compile-sass` instead. - """ - systems = [SYSTEMS[sys] for sys in get_parsed_option(options, 'system', ['lms', 'cms'])] # normalize studio->cms - run_deprecated_command_wrapper( - old_command="paver compile_sass", - ignored_old_flags=(set(["--force"]) & set(options)), - new_command=shlex.join([ - "npm", - "run", - ("compile-sass-dev" if options.get("debug") else "compile-sass"), - "--", - *(["--dry"] if tasks.environment.dry_run else []), - *(["--skip-lms"] if "lms" not in systems else []), - *(["--skip-cms"] if "cms" not in systems else []), - *( - arg - for theme_dir in get_parsed_option(options, 'theme_dirs', []) - for arg in ["--theme-dir", str(theme_dir)] - ), - *( - arg - for theme in get_parsed_option(options, "themes", []) - for arg in ["--theme", theme] - ), - ]), - ) - - -def _compile_sass(system, theme, debug, force, _timing_info): - """ - This is a DEPRECATED COMPATIBILITY WRAPPER - - It exists to ease the transition for Tutor in Redwood, which directly imported and used this function. - """ - run_deprecated_command_wrapper( - old_command="pavelib.assets:_compile_sass", - ignored_old_flags=(set(["--force"]) if force else set()), - new_command=[ - "npm", - "run", - ("compile-sass-dev" if debug else "compile-sass"), - "--", - *(["--dry"] if tasks.environment.dry_run else []), - *( - ["--skip-default", "--theme-dir", str(theme.parent), "--theme", str(theme.name)] - if theme - else [] - ), - ("--skip-cms" if system == "lms" else "--skip-lms"), - ] - ) - - -def process_npm_assets(): - """ - Process vendor libraries installed via NPM. - - This is a DEPRECATED COMPATIBILITY WRAPPER. It is now handled as part of `npm clean-install`. - If you need to invoke it explicitly, you can run `npm run postinstall`. - """ - run_deprecated_command_wrapper( - old_command="pavelib.assets:process_npm_assets", - ignored_old_flags=[], - new_command=shlex.join(["npm", "run", "postinstall"]), - ) - - -@task -@no_help -def process_xmodule_assets(): - """ - Process XModule static assets. - - This is a DEPRECATED COMPATIBILITY STUB. Refrences to it should be deleted. - """ - print( - "\n" + - f"{WARNING_SYMBOLS}", - "\n" + - "WARNING: 'paver process_xmodule_assets' is DEPRECATED! It will be removed before Sumac.\n" + - "\n" + - "Starting with Quince, it is no longer necessary to post-process XModule assets, so \n" + - "'paver process_xmodule_assets' is a no-op. Please simply remove it from your build scripts.\n" + - "\n" + - "Details: https://github.com/openedx/edx-platform/issues/31895\n" + - f"{WARNING_SYMBOLS}", - ) - - -def collect_assets(systems, settings, **kwargs): - """ - Collect static assets, including Django pipeline processing. - `systems` is a list of systems (e.g. 'lms' or 'studio' or both) - `settings` is the Django settings module to use. - `**kwargs` include arguments for using a log directory for collectstatic output. Defaults to /dev/null. - - This is a DEPRECATED COMPATIBILITY WRAPPER - - It exists to ease the transition for Tutor in Redwood, which directly imported and used this function. - """ - run_deprecated_command_wrapper( - old_command="pavelib.asset:collect_assets", - ignored_old_flags=[], - new_command=" && ".join( - "( " + - shlex.join( - ["./manage.py", SYSTEMS[sys], f"--settings={settings}", "collectstatic", "--noinput"] - ) + ( - "" - if "collect_log_dir" not in kwargs else - " > /dev/null" - if kwargs["collect_log_dir"] is None else - f"> {kwargs['collect_log_dir']}/{SYSTEMS[sys]}-collectstatic.out" - ) + - " )" - for sys in systems - ), - ) - - -def execute_compile_sass(args): - """ - Construct django management command compile_sass (defined in theming app) and execute it. - Args: - args: command line argument passed via update_assets command - - This is a DEPRECATED COMPATIBILITY WRAPPER. Use `npm run compile-sass` instead. - """ - for sys in args.system: - options = "" - options += " --theme-dirs " + " ".join(args.theme_dirs) if args.theme_dirs else "" - options += " --themes " + " ".join(args.themes) if args.themes else "" - options += " --debug" if args.debug else "" - - sh( - django_cmd( - sys, - args.settings, - "compile_sass {system} {options}".format( - system='cms' if sys == 'studio' else sys, - options=options, - ), - ), - ) - - -@task -@cmdopts([ - ('settings=', 's', "Django settings (defaults to devstack)"), - ('watch', 'w', "DEPRECATED. This flag never did anything anyway."), -]) -@timed -def webpack(options): - """ - Run a Webpack build. - - This is a DEPRECATED COMPATIBILITY WRAPPER. Use `npm run webpack` instead. - """ - settings = getattr(options, 'settings', Env.DEVSTACK_SETTINGS) - result = Env.get_django_settings(['STATIC_ROOT', 'WEBPACK_CONFIG_PATH'], "lms", settings=settings) - static_root_lms, config_path = result - static_root_cms, = Env.get_django_settings(["STATIC_ROOT"], "cms", settings=settings) - js_env_extra_config_setting, = Env.get_django_json_settings(["JS_ENV_EXTRA_CONFIG"], "cms", settings=settings) - js_env_extra_config = json.dumps(js_env_extra_config_setting or "{}") - node_env = "development" if config_path == 'webpack.dev.config.js' else "production" - run_deprecated_command_wrapper( - old_command="paver webpack", - ignored_old_flags=(set(["watch"]) & set(options)), - new_command=' '.join([ - f"WEBPACK_CONFIG_PATH={config_path}", - f"NODE_ENV={node_env}", - f"STATIC_ROOT_LMS={static_root_lms}", - f"STATIC_ROOT_CMS={static_root_cms}", - f"JS_ENV_EXTRA_CONFIG={js_env_extra_config}", - "npm", - "run", - "webpack", - ]), - ) - - -def get_parsed_option(command_opts, opt_key, default=None): - """ - Extract user command option and parse it. - Arguments: - command_opts: Command line arguments passed via paver command. - opt_key: name of option to get and parse - default: if `command_opt_value` not in `command_opts`, `command_opt_value` will be set to default. - Returns: - list or None - """ - command_opt_value = getattr(command_opts, opt_key, default) - if command_opt_value: - command_opt_value = listfy(command_opt_value) - - return command_opt_value - - -def listfy(data): - """ - Check and convert data to list. - Arguments: - data: data structure to be converted. - """ - - if isinstance(data, str): - data = data.split(',') - elif not isinstance(data, list): - data = [data] - - return data - - -@task -@cmdopts([ - ('background', 'b', 'DEPRECATED. Use shell tools like & to run in background if needed.'), - ('settings=', 's', "DEPRECATED. Django is not longer invoked to compile JS/Sass."), - ('theme-dirs=', '-td', 'The themes dir containing all themes (defaults to None)'), - ('themes=', '-t', 'DEPRECATED. All themes in --theme-dirs are now watched.'), - ('wait=', '-w', 'DEPRECATED. Watchdog\'s default wait time is now used.'), -]) -@timed -def watch_assets(options): - """ - Watch for changes to asset files, and regenerate js/css - - This is a DEPRECATED COMPATIBILITY WRAPPER. Use `npm run watch` instead. - """ - # Don't watch assets when performing a dry run - if tasks.environment.dry_run: - return - - theme_dirs = ':'.join(get_parsed_option(options, 'theme_dirs', [])) - run_deprecated_command_wrapper( - old_command="paver watch_assets", - ignored_old_flags=(set(["debug", "themes", "settings", "background"]) & set(options)), - new_command=shlex.join([ - *( - ["env", f"COMPREHENSIVE_THEME_DIRS={theme_dirs}"] - if theme_dirs else [] - ), - "npm", - "run", - "watch", - ]), - ) - - -@task -@needs( - 'pavelib.prereqs.install_node_prereqs', - 'pavelib.prereqs.install_python_prereqs', -) -@consume_args -@timed -def update_assets(args): - """ - Compile Sass, then collect static assets. - - This is a DEPRECATED COMPATIBILITY WRAPPER around other DEPRECATED COMPATIBILITY WRAPPERS. - The aggregate affect of this command can be achieved with this sequence of commands instead: - - * pip install -r requirements/edx/assets.txt # replaces install_python_prereqs - * npm clean-install # replaces install_node_prereqs - * npm run build # replaces execute_compile_sass and webpack - * ./manage.py lms collectstatic --noinput # replaces collect_assets (for LMS) - * ./manage.py cms collectstatic --noinput # replaces collect_assets (for CMS) - """ - parser = argparse.ArgumentParser(prog='paver update_assets') - parser.add_argument( - 'system', type=str, nargs='*', default=["lms", "studio"], - help="lms or studio", - ) - parser.add_argument( - '--settings', type=str, default=Env.DEVSTACK_SETTINGS, - help="Django settings module", - ) - parser.add_argument( - '--debug', action='store_true', default=False, - help="Enable all debugging", - ) - parser.add_argument( - '--debug-collect', action='store_true', default=False, - help="Disable collect static", - ) - parser.add_argument( - '--skip-collect', dest='collect', action='store_false', default=True, - help="Skip collection of static assets", - ) - parser.add_argument( - '--watch', action='store_true', default=False, - help="Watch files for changes", - ) - parser.add_argument( - '--theme-dirs', dest='theme_dirs', type=str, nargs='+', default=None, - help="base directories where themes are placed", - ) - parser.add_argument( - '--themes', type=str, nargs='+', default=None, - help="list of themes to compile sass for. ignored when --watch is used; all themes are watched.", - ) - parser.add_argument( - '--collect-log', dest="collect_log_dir", default=None, - help="When running collectstatic, direct output to specified log directory", - ) - parser.add_argument( - '--wait', type=float, default=0.0, - help="DEPRECATED. Watchdog's default wait time is now used.", - ) - args = parser.parse_args(args) - - # Build Webpack - call_task('pavelib.assets.webpack', options={'settings': args.settings}) - - # Compile sass for themes and system - execute_compile_sass(args) - - if args.collect: - if args.collect_log_dir: - collect_log_args = {"collect_log_dir": args.collect_log_dir} - elif args.debug or args.debug_collect: - collect_log_args = {"collect_log_dir": None} - else: - collect_log_args = {} - - collect_assets(args.system, args.settings, **collect_log_args) - - if args.watch: - call_task( - 'pavelib.assets.watch_assets', - options={ - 'background': not args.debug, - 'settings': args.settings, - 'theme_dirs': args.theme_dirs, - 'themes': args.themes, - 'wait': [float(args.wait)] - }, - ) diff --git a/pavelib/js_test.py b/pavelib/js_test.py index fb9c213499ac..1ab19271f473 100644 --- a/pavelib/js_test.py +++ b/pavelib/js_test.py @@ -23,7 +23,6 @@ @task @needs( - 'pavelib.prereqs.install_node_prereqs', 'pavelib.utils.test.utils.clean_reports_dir', ) @cmdopts([ @@ -39,6 +38,7 @@ def test_js(options): """ Run the JavaScript tests """ + sh("npm clean-install") mode = getattr(options, 'mode', 'run') port = None skip_clean = getattr(options, 'skip_clean', False) @@ -100,7 +100,6 @@ def test_js_dev(options): @task -@needs('pavelib.prereqs.install_coverage_prereqs') @cmdopts([ ("compare-branch=", "b", "Branch to compare against, defaults to origin/master"), ], share_with=['coverage']) @@ -109,6 +108,8 @@ def diff_coverage(options): """ Build the diff coverage reports """ + sh("pip install -r requirements/edx/coverage.txt") + compare_branch = options.get('compare_branch', 'origin/master') # Find all coverage XML files (both Python and JavaScript) diff --git a/pavelib/paver_tests/test_assets.py b/pavelib/paver_tests/test_assets.py deleted file mode 100644 index f7100a7f03c3..000000000000 --- a/pavelib/paver_tests/test_assets.py +++ /dev/null @@ -1,130 +0,0 @@ -"""Unit tests for the Paver asset tasks.""" - -import json -import os -from pathlib import Path -from unittest import TestCase -from unittest.mock import patch - -import ddt -import paver.easy -from paver import tasks - -import pavelib.assets -from pavelib.assets import Env - - -REPO_ROOT = Path(__file__).parent.parent.parent - -LMS_SETTINGS = { - "WEBPACK_CONFIG_PATH": "webpack.fake.config.js", - "STATIC_ROOT": "/fake/lms/staticfiles", - -} -CMS_SETTINGS = { - "WEBPACK_CONFIG_PATH": "webpack.fake.config", - "STATIC_ROOT": "/fake/cms/staticfiles", - "JS_ENV_EXTRA_CONFIG": json.dumps({"key1": [True, False], "key2": {"key2.1": 1369, "key2.2": "1369"}}), -} - - -def _mock_get_django_settings(django_settings, system, settings=None): # pylint: disable=unused-argument - return [(LMS_SETTINGS if system == "lms" else CMS_SETTINGS)[s] for s in django_settings] - - -@ddt.ddt -@patch.object(Env, 'get_django_settings', _mock_get_django_settings) -@patch.object(Env, 'get_django_json_settings', _mock_get_django_settings) -class TestDeprecatedPaverAssets(TestCase): - """ - Simple test to ensure that the soon-to-be-removed Paver commands are correctly translated into the new npm-run - commands. - """ - def setUp(self): - super().setUp() - self.maxDiff = None - os.environ['NO_PREREQ_INSTALL'] = 'true' - tasks.environment = tasks.Environment() - - def tearDown(self): - super().tearDown() - del os.environ['NO_PREREQ_INSTALL'] - - @ddt.data( - dict( - task_name='pavelib.assets.compile_sass', - args=[], - kwargs={}, - expected=["npm run compile-sass --"], - ), - dict( - task_name='pavelib.assets.compile_sass', - args=[], - kwargs={"system": "lms,studio"}, - expected=["npm run compile-sass --"], - ), - dict( - task_name='pavelib.assets.compile_sass', - args=[], - kwargs={"debug": True}, - expected=["npm run compile-sass-dev --"], - ), - dict( - task_name='pavelib.assets.compile_sass', - args=[], - kwargs={"system": "lms"}, - expected=["npm run compile-sass -- --skip-cms"], - ), - dict( - task_name='pavelib.assets.compile_sass', - args=[], - kwargs={"system": "studio"}, - expected=["npm run compile-sass -- --skip-lms"], - ), - dict( - task_name='pavelib.assets.compile_sass', - args=[], - kwargs={"system": "cms", "theme_dirs": f"{REPO_ROOT}/common/test,{REPO_ROOT}/themes"}, - expected=[ - "npm run compile-sass -- --skip-lms " + - f"--theme-dir {REPO_ROOT}/common/test --theme-dir {REPO_ROOT}/themes" - ], - ), - dict( - task_name='pavelib.assets.compile_sass', - args=[], - kwargs={"theme_dirs": f"{REPO_ROOT}/common/test,{REPO_ROOT}/themes", "themes": "red-theme,test-theme"}, - expected=[ - "npm run compile-sass -- " + - f"--theme-dir {REPO_ROOT}/common/test --theme-dir {REPO_ROOT}/themes " + - "--theme red-theme --theme test-theme" - ], - ), - dict( - task_name='pavelib.assets.update_assets', - args=["lms", "studio", "--settings=fake.settings"], - kwargs={}, - expected=[ - ( - "WEBPACK_CONFIG_PATH=webpack.fake.config.js " + - "NODE_ENV=production " + - "STATIC_ROOT_LMS=/fake/lms/staticfiles " + - "STATIC_ROOT_CMS=/fake/cms/staticfiles " + - 'JS_ENV_EXTRA_CONFIG=' + - '"{\\"key1\\": [true, false], \\"key2\\": {\\"key2.1\\": 1369, \\"key2.2\\": \\"1369\\"}}" ' + - "npm run webpack" - ), - "python manage.py lms --settings=fake.settings compile_sass lms ", - "python manage.py cms --settings=fake.settings compile_sass cms ", - ( - "( ./manage.py lms --settings=fake.settings collectstatic --noinput ) && " + - "( ./manage.py cms --settings=fake.settings collectstatic --noinput )" - ), - ], - ), - ) - @ddt.unpack - @patch.object(pavelib.assets, 'sh') - def test_paver_assets_wrapper_invokes_new_commands(self, mock_sh, task_name, args, kwargs, expected): - paver.easy.call_task(task_name, args=args, options=kwargs) - assert [call_args[0] for (call_args, call_kwargs) in mock_sh.call_args_list] == expected diff --git a/pavelib/paver_tests/test_js_test.py b/pavelib/paver_tests/test_js_test.py index 4b165a156674..d2cdaaf3fac4 100644 --- a/pavelib/paver_tests/test_js_test.py +++ b/pavelib/paver_tests/test_js_test.py @@ -31,6 +31,7 @@ class TestPaverJavaScriptTestTasks(PaverTestCase): ) EXPECTED_COMMANDS = [ + "npm clean-install", "make report_dir", 'git clean -fqdx test_root/logs test_root/data test_root/staticfiles test_root/uploads', "find . -name '.git' -prune -o -name '*.pyc' -exec rm {} \\;", diff --git a/pavelib/paver_tests/test_pii_check.py b/pavelib/paver_tests/test_pii_check.py index d034360acde0..bd345e68d614 100644 --- a/pavelib/paver_tests/test_pii_check.py +++ b/pavelib/paver_tests/test_pii_check.py @@ -37,7 +37,8 @@ def test_pii_check_report_dir_override(self, mock_paver_sh, mock_needs): mock_needs.return_value = 0 call_task('pavelib.quality.run_pii_check', options={"report_dir": str(self.report_dir)}) - mock_calls = [str(call) for call in mock_paver_sh.mock_calls] + mock_pip_call, *mock_calls = [str(call) for call in mock_paver_sh.mock_calls] + assert 'pip install' in mock_pip_call assert len(mock_calls) == 2 assert any('lms.envs.test' in call for call in mock_calls) assert any('cms.envs.test' in call for call in mock_calls) @@ -70,7 +71,8 @@ def test_pii_check_failed(self, mock_paver_sh, mock_needs): # This is not good, but these tests weren't even running for years, and we're removing this whole test # suite soon anyway. pass - mock_calls = [str(call) for call in mock_paver_sh.mock_calls] + mock_pip_call, *mock_calls = [str(call) for call in mock_paver_sh.mock_calls] + assert 'pip install' in mock_pip_call assert len(mock_calls) == 2 assert any('lms.envs.test' in call for call in mock_calls) assert any('cms.envs.test' in call for call in mock_calls) diff --git a/pavelib/prereqs.py b/pavelib/prereqs.py deleted file mode 100644 index 4453176c94da..000000000000 --- a/pavelib/prereqs.py +++ /dev/null @@ -1,351 +0,0 @@ -""" -Install Python and Node prerequisites. -""" - - -import hashlib -import os -import re -import subprocess -import sys -from distutils import sysconfig # pylint: disable=deprecated-module - -from paver.easy import sh, task # lint-amnesty, pylint: disable=unused-import - -from .utils.envs import Env -from .utils.timer import timed - -PREREQS_STATE_DIR = os.getenv('PREREQ_CACHE_DIR', Env.REPO_ROOT / '.prereqs_cache') -NO_PREREQ_MESSAGE = "NO_PREREQ_INSTALL is set, not installing prereqs" -NO_PYTHON_UNINSTALL_MESSAGE = 'NO_PYTHON_UNINSTALL is set. No attempts will be made to uninstall old Python libs.' -COVERAGE_REQ_FILE = 'requirements/edx/coverage.txt' - -# If you make any changes to this list you also need to make -# a corresponding change to circle.yml, which is how the python -# prerequisites are installed for builds on circleci.com -toxenv = os.environ.get('TOXENV') -if toxenv and toxenv != 'quality': - PYTHON_REQ_FILES = ['requirements/edx/testing.txt'] -else: - PYTHON_REQ_FILES = ['requirements/edx/development.txt'] - -# Developers can have private requirements, for local copies of github repos, -# or favorite debugging tools, etc. -PRIVATE_REQS = 'requirements/edx/private.txt' -if os.path.exists(PRIVATE_REQS): - PYTHON_REQ_FILES.append(PRIVATE_REQS) - - -def str2bool(s): - s = str(s) - return s.lower() in ('yes', 'true', 't', '1') - - -def no_prereq_install(): - """ - Determine if NO_PREREQ_INSTALL should be truthy or falsy. - """ - return str2bool(os.environ.get('NO_PREREQ_INSTALL', 'False')) - - -def no_python_uninstall(): - """ Determine if we should run the uninstall_python_packages task. """ - return str2bool(os.environ.get('NO_PYTHON_UNINSTALL', 'False')) - - -def create_prereqs_cache_dir(): - """Create the directory for storing the hashes, if it doesn't exist already.""" - try: - os.makedirs(PREREQS_STATE_DIR) - except OSError: - if not os.path.isdir(PREREQS_STATE_DIR): - raise - - -def compute_fingerprint(path_list): - """ - Hash the contents of all the files and directories in `path_list`. - Returns the hex digest. - """ - - hasher = hashlib.sha1() - - for path_item in path_list: - - # For directories, create a hash based on the modification times - # of first-level subdirectories - if os.path.isdir(path_item): - for dirname in sorted(os.listdir(path_item)): - path_name = os.path.join(path_item, dirname) - if os.path.isdir(path_name): - hasher.update(str(os.stat(path_name).st_mtime).encode('utf-8')) - - # For files, hash the contents of the file - if os.path.isfile(path_item): - with open(path_item, "rb") as file_handle: - hasher.update(file_handle.read()) - - return hasher.hexdigest() - - -def prereq_cache(cache_name, paths, install_func): - """ - Conditionally execute `install_func()` only if the files/directories - specified by `paths` have changed. - - If the code executes successfully (no exceptions are thrown), the cache - is updated with the new hash. - """ - # Retrieve the old hash - cache_filename = cache_name.replace(" ", "_") - cache_file_path = os.path.join(PREREQS_STATE_DIR, f"{cache_filename}.sha1") - old_hash = None - if os.path.isfile(cache_file_path): - with open(cache_file_path) as cache_file: - old_hash = cache_file.read() - - # Compare the old hash to the new hash - # If they do not match (either the cache hasn't been created, or the files have changed), - # then execute the code within the block. - new_hash = compute_fingerprint(paths) - if new_hash != old_hash: - install_func() - - # Update the cache with the new hash - # If the code executed within the context fails (throws an exception), - # then this step won't get executed. - create_prereqs_cache_dir() - with open(cache_file_path, "wb") as cache_file: - # Since the pip requirement files are modified during the install - # process, we need to store the hash generated AFTER the installation - post_install_hash = compute_fingerprint(paths) - cache_file.write(post_install_hash.encode('utf-8')) - else: - print(f'{cache_name} unchanged, skipping...') - - -def node_prereqs_installation(): - """ - Configures npm and installs Node prerequisites - """ - # Before July 2023, these directories were created and written to - # as root. Afterwards, they are created as being owned by the - # `app` user -- but also need to be deleted by that user (due to - # how npm runs post-install scripts.) Developers with an older - # devstack installation who are reprovisioning will see errors - # here if the files are still owned by root. Deleting the files in - # advance prevents this error. - # - # This hack should probably be left in place for at least a year. - # See ADR 17 for more background on the transition. - sh("rm -rf common/static/common/js/vendor/ common/static/common/css/vendor/") - # At the time of this writing, the js dir has git-versioned files - # but the css dir does not, so the latter would have been created - # as root-owned (in the process of creating the vendor - # subdirectory). Delete it only if empty, just in case - # git-versioned files are added later. - sh("rmdir common/static/common/css || true") - - # NPM installs hang sporadically. Log the installation process so that we - # determine if any packages are chronic offenders. - npm_log_file_path = f'{Env.GEN_LOG_DIR}/npm-install.log' - npm_log_file = open(npm_log_file_path, 'wb') # lint-amnesty, pylint: disable=consider-using-with - npm_command = 'npm ci --verbose'.split() - - # The implementation of Paver's `sh` function returns before the forked - # actually returns. Using a Popen object so that we can ensure that - # the forked process has returned - proc = subprocess.Popen(npm_command, stderr=npm_log_file) # lint-amnesty, pylint: disable=consider-using-with - retcode = proc.wait() - if retcode == 1: - raise Exception(f"npm install failed: See {npm_log_file_path}") - print("Successfully clean-installed NPM packages. Log found at {}".format( - npm_log_file_path - )) - - -def python_prereqs_installation(): - """ - Installs Python prerequisites - """ - # edx-platform installs some Python projects from within the edx-platform repo itself. - sh("pip install -e .") - for req_file in PYTHON_REQ_FILES: - pip_install_req_file(req_file) - - -def pip_install_req_file(req_file): - """Pip install the requirements file.""" - pip_cmd = 'pip install -q --disable-pip-version-check --exists-action w' - sh(f"{pip_cmd} -r {req_file}") - - -@task -@timed -def install_node_prereqs(): - """ - Installs Node prerequisites - """ - if no_prereq_install(): - print(NO_PREREQ_MESSAGE) - return - - prereq_cache("Node prereqs", ["package.json", "package-lock.json"], node_prereqs_installation) - - -# To add a package to the uninstall list, just add it to this list! No need -# to touch any other part of this file. -PACKAGES_TO_UNINSTALL = [ - "MySQL-python", # Because mysqlclient shares the same directory name - "South", # Because it interferes with Django 1.8 migrations. - "edxval", # Because it was bork-installed somehow. - "django-storages", - "django-oauth2-provider", # Because now it's called edx-django-oauth2-provider. - "edx-oauth2-provider", # Because it moved from github to pypi - "enum34", # Because enum34 is not needed in python>3.4 - "i18n-tools", # Because now it's called edx-i18n-tools - "moto", # Because we no longer use it and it conflicts with recent jsondiff versions - "python-saml", # Because python3-saml shares the same directory name - "pytest-faulthandler", # Because it was bundled into pytest - "djangorestframework-jwt", # Because now its called drf-jwt. -] - - -@task -@timed -def uninstall_python_packages(): - """ - Uninstall Python packages that need explicit uninstallation. - - Some Python packages that we no longer want need to be explicitly - uninstalled, notably, South. Some other packages were once installed in - ways that were resistant to being upgraded, like edxval. Also uninstall - them. - """ - - if no_python_uninstall(): - print(NO_PYTHON_UNINSTALL_MESSAGE) - return - - # So that we don't constantly uninstall things, use a hash of the packages - # to be uninstalled. Check it, and skip this if we're up to date. - hasher = hashlib.sha1() - hasher.update(repr(PACKAGES_TO_UNINSTALL).encode('utf-8')) - expected_version = hasher.hexdigest() - state_file_path = os.path.join(PREREQS_STATE_DIR, "Python_uninstall.sha1") - create_prereqs_cache_dir() - - if os.path.isfile(state_file_path): - with open(state_file_path) as state_file: - version = state_file.read() - if version == expected_version: - print('Python uninstalls unchanged, skipping...') - return - - # Run pip to find the packages we need to get rid of. Believe it or not, - # edx-val is installed in a way that it is present twice, so we have a loop - # to really really get rid of it. - for _ in range(3): - uninstalled = False - frozen = sh("pip freeze", capture=True) - - for package_name in PACKAGES_TO_UNINSTALL: - if package_in_frozen(package_name, frozen): - # Uninstall the pacakge - sh(f"pip uninstall --disable-pip-version-check -y {package_name}") - uninstalled = True - if not uninstalled: - break - else: - # We tried three times and didn't manage to get rid of the pests. - print("Couldn't uninstall unwanted Python packages!") - return - - # Write our version. - with open(state_file_path, "wb") as state_file: - state_file.write(expected_version.encode('utf-8')) - - -def package_in_frozen(package_name, frozen_output): - """Is this package in the output of 'pip freeze'?""" - # Look for either: - # - # PACKAGE-NAME== - # - # or: - # - # blah_blah#egg=package_name-version - # - pattern = r"(?mi)^{pkg}==|#egg={pkg_under}-".format( - pkg=re.escape(package_name), - pkg_under=re.escape(package_name.replace("-", "_")), - ) - return bool(re.search(pattern, frozen_output)) - - -@task -@timed -def install_coverage_prereqs(): - """ Install python prereqs for measuring coverage. """ - if no_prereq_install(): - print(NO_PREREQ_MESSAGE) - return - pip_install_req_file(COVERAGE_REQ_FILE) - - -@task -@timed -def install_python_prereqs(): - """ - Installs Python prerequisites. - """ - if no_prereq_install(): - print(NO_PREREQ_MESSAGE) - return - - uninstall_python_packages() - - # Include all of the requirements files in the fingerprint. - files_to_fingerprint = list(PYTHON_REQ_FILES) - - # Also fingerprint the directories where packages get installed: - # ("/edx/app/edxapp/venvs/edxapp/lib/python2.7/site-packages") - files_to_fingerprint.append(sysconfig.get_python_lib()) - - # In a virtualenv, "-e installs" get put in a src directory. - if Env.PIP_SRC: - src_dir = Env.PIP_SRC - else: - src_dir = os.path.join(sys.prefix, "src") - if os.path.isdir(src_dir): - files_to_fingerprint.append(src_dir) - - # Also fingerprint this source file, so that if the logic for installations - # changes, we will redo the installation. - this_file = __file__ - if this_file.endswith(".pyc"): - this_file = this_file[:-1] # use the .py file instead of the .pyc - files_to_fingerprint.append(this_file) - - prereq_cache("Python prereqs", files_to_fingerprint, python_prereqs_installation) - - -@task -@timed -def install_prereqs(): - """ - Installs Node and Python prerequisites - """ - if no_prereq_install(): - print(NO_PREREQ_MESSAGE) - return - - if not str2bool(os.environ.get('SKIP_NPM_INSTALL', 'False')): - install_node_prereqs() - install_python_prereqs() - log_installed_python_prereqs() - - -def log_installed_python_prereqs(): - """ Logs output of pip freeze for debugging. """ - sh("pip freeze > {}".format(Env.GEN_LOG_DIR + "/pip_freeze.log")) diff --git a/pavelib/quality.py b/pavelib/quality.py index 774179f45048..01fe597b40dc 100644 --- a/pavelib/quality.py +++ b/pavelib/quality.py @@ -142,7 +142,6 @@ def run_pep8(options): # pylint: disable=unused-argument @task @needs( - 'pavelib.prereqs.install_node_prereqs', 'pavelib.utils.test.utils.ensure_clean_package_lock', ) @cmdopts([ @@ -154,6 +153,7 @@ def run_eslint(options): Runs eslint on static asset directories. If limit option is passed, fails build if more violations than the limit are found. """ + sh("npm clean-install") eslint_report_dir = (Env.REPORT_DIR / "eslint") eslint_report = eslint_report_dir / "eslint.report" @@ -222,7 +222,6 @@ def _get_stylelint_violations(): @task -@needs('pavelib.prereqs.install_node_prereqs') @cmdopts([ ("limit=", "l", "limit for number of acceptable violations"), ]) @@ -232,6 +231,8 @@ def run_stylelint(options): Runs stylelint on Sass files. If limit option is passed, fails build if more violations than the limit are found. """ + sh("npm clean-install") + violations_limit = 0 num_violations = _get_stylelint_violations() @@ -252,7 +253,6 @@ def run_stylelint(options): @task -@needs('pavelib.prereqs.install_python_prereqs') @cmdopts([ ("thresholds=", "t", "json containing limit for number of acceptable violations per rule"), ]) @@ -261,6 +261,7 @@ def run_xsslint(options): """ Runs xsslint/xss_linter.py on the codebase """ + sh("pip install -r requirements/edx/development.txt -e .") thresholds_option = getattr(options, 'thresholds', '{}') try: @@ -509,7 +510,6 @@ def _extract_missing_pii_annotations(filename): @task -@needs('pavelib.prereqs.install_python_prereqs') @cmdopts([ ("report-dir=", "r", "Directory in which to put PII reports"), ]) @@ -518,6 +518,8 @@ def run_pii_check(options): """ Guarantee that all Django models are PII-annotated. """ + sh("pip install -r requirements/edx/development.txt -e .") + pii_report_name = 'pii' default_report_dir = (Env.REPORT_DIR / pii_report_name) report_dir = getattr(options, 'report_dir', default_report_dir) @@ -566,12 +568,13 @@ def run_pii_check(options): @task -@needs('pavelib.prereqs.install_python_prereqs') @timed def check_keywords(): """ Check Django model fields for names that conflict with a list of reserved keywords """ + sh("pip install -r requirements/edx/development.txt -e .") + report_path = os.path.join(Env.REPORT_DIR, 'reserved_keywords') sh(f"mkdir -p {report_path}") diff --git a/pavelib/utils/cmd.py b/pavelib/utils/cmd.py index a350c90a6a96..ddb591301cc2 100644 --- a/pavelib/utils/cmd.py +++ b/pavelib/utils/cmd.py @@ -8,17 +8,3 @@ def cmd(*args): Concatenate the arguments into a space-separated shell command. """ return " ".join(str(arg) for arg in args if arg) - - -def django_cmd(sys, settings, *args): - """ - Construct a Django management command. - - `sys` is either 'lms' or 'studio'. - `settings` is the Django settings module (such as "dev" or "test") - `args` are concatenated to form the rest of the command. - """ - # Maintain backwards compatibility with manage.py, - # which calls "studio" "cms" - sys = 'cms' if sys == 'studio' else sys - return cmd("python manage.py", sys, f"--settings={settings}", *args) diff --git a/pavelib/utils/envs.py b/pavelib/utils/envs.py index d2cdd4a77d7a..1e08e5dd6fd0 100644 --- a/pavelib/utils/envs.py +++ b/pavelib/utils/envs.py @@ -1,17 +1,11 @@ """ Helper functions for loading environment settings. """ -import configparser -import json import os import sys from time import sleep -from lazy import lazy from path import Path as path -from paver.easy import BuildFailure, sh - -from pavelib.utils.cmd import django_cmd def repo_root(): @@ -137,131 +131,3 @@ class Env: SERVICE_VARIANT = 'cms' else: SERVICE_VARIANT = 'lms' - - @classmethod - def get_django_settings(cls, django_settings, system, settings=None, print_setting_args=None): - """ - Interrogate Django environment for specific settings values - :param django_settings: list of django settings values to get - :param system: the django app to use when asking for the setting (lms | cms) - :param settings: the settings file to use when asking for the value - :param print_setting_args: the additional arguments to send to print_settings - :return: unicode value of the django setting - """ - if not settings: - settings = os.environ.get("EDX_PLATFORM_SETTINGS", "aws") - log_dir = os.path.dirname(cls.PRINT_SETTINGS_LOG_FILE) - if not os.path.exists(log_dir): - os.makedirs(log_dir) - settings_length = len(django_settings) - django_settings = ' '.join(django_settings) # parse_known_args makes a list again - print_setting_args = ' '.join(print_setting_args or []) - try: - value = sh( - django_cmd( - system, - settings, - "print_setting {django_settings} 2>{log_file} {print_setting_args}".format( - django_settings=django_settings, - print_setting_args=print_setting_args, - log_file=cls.PRINT_SETTINGS_LOG_FILE - ).strip() - ), - capture=True - ) - # else for cases where values are not found & sh returns one None value - return tuple(str(value).splitlines()) if value else tuple(None for _ in range(settings_length)) - except BuildFailure: - print(f"Unable to print the value of the {django_settings} setting:") - with open(cls.PRINT_SETTINGS_LOG_FILE) as f: - print(f.read()) - sys.exit(1) - - @classmethod - def get_django_json_settings(cls, django_settings, system, settings=None): - """ - Interrogate Django environment for specific settings value - :param django_settings: list of django settings values to get - :param system: the django app to use when asking for the setting (lms | cms) - :param settings: the settings file to use when asking for the value - :return: json string value of the django setting - """ - return cls.get_django_settings( - django_settings, - system, - settings=settings, - print_setting_args=["--json"], - ) - - @classmethod - def covered_modules(cls): - """ - List the source modules listed in .coveragerc for which coverage - will be measured. - """ - coveragerc = configparser.RawConfigParser() - coveragerc.read(cls.PYTHON_COVERAGERC) - modules = coveragerc.get('run', 'source') - result = [] - for module in modules.split('\n'): - module = module.strip() - if module: - result.append(module) - return result - - @lazy - def env_tokens(self): - """ - Return a dict of environment settings. - If we couldn't find the JSON file, issue a warning and return an empty dict. - """ - - # Find the env JSON file - if self.SERVICE_VARIANT: - env_path = self.REPO_ROOT.parent / f"{self.SERVICE_VARIANT}.env.json" - else: - env_path = path("env.json").abspath() - - # If the file does not exist, here or one level up, - # issue a warning and return an empty dict - if not env_path.isfile(): - env_path = env_path.parent.parent / env_path.basename() - if not env_path.isfile(): - print( - "Warning: could not find environment JSON file " - "at '{path}'".format(path=env_path), - file=sys.stderr, - ) - return {} - - # Otherwise, load the file as JSON and return the resulting dict - try: - with open(env_path) as env_file: - return json.load(env_file) - - except ValueError: - print( - "Error: Could not parse JSON " - "in {path}".format(path=env_path), - file=sys.stderr, - ) - sys.exit(1) - - @lazy - def feature_flags(self): - """ - Return a dictionary of feature flags configured by the environment. - """ - return self.env_tokens.get('FEATURES', {}) - - @classmethod - def rsync_dirs(cls): - """ - List the directories that should be synced during pytest-xdist - execution. Needs to include all modules for which coverage is - measured, not just the tests being run. - """ - result = set() - for module in cls.covered_modules(): - result.add(module.split('/')[0]) - return result diff --git a/requirements/common_constraints.txt b/requirements/common_constraints.txt index 5188f37250ef..b8166ba67540 100644 --- a/requirements/common_constraints.txt +++ b/requirements/common_constraints.txt @@ -15,7 +15,6 @@ # Note: Changes to this file will automatically be used by other repos, referencing # this file from Github directly. It does not require packaging in edx-lint. - # using LTS django version Django<5.0 @@ -26,13 +25,6 @@ elasticsearch<7.14.0 # django-simple-history>3.0.0 adds indexing and causes a lot of migrations to be affected -# Cause: https://github.com/openedx/event-tracking/pull/290 -# event-tracking 2.4.1 upgrades to pymongo 4.4.0 which is not supported on edx-platform. -# We will pin event-tracking to do not break existing installations -# This can be unpinned once https://github.com/openedx/edx-platform/issues/34586 -# has been resolved and edx-platform is running with pymongo>=4.4.0 - - # Cause: https://github.com/openedx/edx-lint/issues/458 # This can be unpinned once https://github.com/openedx/edx-lint/issues/459 has been resolved. pip<24.3 diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 1b81540252bc..22547e335e4c 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -101,7 +101,6 @@ celery==5.4.0 # openedx-learning certifi==2024.8.30 # via - # -r requirements/edx/paver.txt # elasticsearch # py2neo # requests @@ -116,7 +115,6 @@ chardet==5.2.0 charset-normalizer==2.0.12 # via # -c requirements/edx/../constraints.txt - # -r requirements/edx/paver.txt # requests # snowflake-connector-python chem==1.3.0 @@ -388,9 +386,7 @@ djangorestframework==3.14.0 djangorestframework-xml==2.0.0 # via edx-enterprise dnspython==2.7.0 - # via - # -r requirements/edx/paver.txt - # pymongo + # via pymongo done-xblock==2.4.0 # via -r requirements/edx/bundled.in drf-jwt==1.19.2 @@ -487,7 +483,6 @@ edx-name-affirmation==3.0.1 edx-opaque-keys[django]==2.11.0 # via # -r requirements/edx/kernel.in - # -r requirements/edx/paver.txt # edx-bulk-grades # edx-ccx-keys # edx-completion @@ -643,7 +638,6 @@ icalendar==6.0.1 # via -r requirements/edx/kernel.in idna==3.10 # via - # -r requirements/edx/paver.txt # optimizely-sdk # requests # snowflake-connector-python @@ -695,15 +689,10 @@ laboratory==1.0.2 # via -r requirements/edx/kernel.in lazy==1.6 # via - # -r requirements/edx/paver.txt # acid-xblock # lti-consumer-xblock # ora2 # xblock -libsass==0.10.0 - # via - # -c requirements/edx/../constraints.txt - # -r requirements/edx/paver.txt loremipsum==1.0.5 # via ora2 lti-consumer-xblock==9.11.3 @@ -741,7 +730,6 @@ markdown==3.3.7 # xblock-poll markupsafe==3.0.2 # via - # -r requirements/edx/paver.txt # chem # jinja2 # mako @@ -753,8 +741,6 @@ meilisearch==0.31.6 # via # -r requirements/edx/kernel.in # edx-search -mock==5.1.0 - # via -r requirements/edx/paver.txt mongoengine==0.29.1 # via -r requirements/edx/kernel.in monotonic==1.6 @@ -853,7 +839,6 @@ path==16.11.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/kernel.in - # -r requirements/edx/paver.txt # edx-i18n-tools # path-py path-py==12.5.0 @@ -861,12 +846,8 @@ path-py==12.5.0 # edx-enterprise # ora2 # staff-graded-xblock -paver==1.3.4 - # via -r requirements/edx/paver.txt pbr==6.1.0 - # via - # -r requirements/edx/paver.txt - # stevedore + # via stevedore pgpy==0.6.0 # via edx-enterprise piexif==1.1.3 @@ -898,7 +879,7 @@ protobuf==5.28.3 # proto-plus psutil==6.1.0 # via - # -r requirements/edx/paver.txt + # -r requirements/edx/kernel.in # edx-django-utils py2neo @ https://github.com/overhangio/py2neo/releases/download/2021.2.3/py2neo-2021.2.3.tar.gz # via @@ -951,12 +932,11 @@ pylatexenc==2.10 pylti1p3==2.0.0 # via -r requirements/edx/kernel.in pymemcache==4.0.0 - # via -r requirements/edx/paver.txt + # via -r requirements/edx/kernel.in pymongo==4.4.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/kernel.in - # -r requirements/edx/paver.txt # edx-opaque-keys # event-tracking # mongoengine @@ -997,8 +977,6 @@ python-dateutil==2.9.0.post0 # xblock python-ipware==3.0.0 # via django-ipware -python-memcached==1.62 - # via -r requirements/edx/paver.txt python-slugify==8.0.4 # via code-annotations python-swiftclient==4.6.0 @@ -1054,7 +1032,6 @@ regex==2024.9.11 # via nltk requests==2.32.3 # via - # -r requirements/edx/paver.txt # algoliasearch # analytics-python # cachecontrol @@ -1117,7 +1094,6 @@ simplejson==3.19.3 six==1.16.0 # via # -r requirements/edx/kernel.in - # -r requirements/edx/paver.txt # analytics-python # bleach # codejail-includes @@ -1134,10 +1110,8 @@ six==1.16.0 # fs-s3fs # html5lib # interchange - # libsass # optimizely-sdk # pansi - # paver # py2neo # pyjwkest # python-dateutil @@ -1175,7 +1149,6 @@ staff-graded-xblock==2.3.0 stevedore==5.3.0 # via # -r requirements/edx/kernel.in - # -r requirements/edx/paver.txt # code-annotations # edx-ace # edx-django-utils @@ -1199,7 +1172,6 @@ tqdm==4.66.6 # openai typing-extensions==4.12.2 # via - # -r requirements/edx/paver.txt # django-countries # edx-opaque-keys # jwcrypto @@ -1223,7 +1195,6 @@ uritemplate==4.1.1 # google-api-python-client urllib3==2.2.3 # via - # -r requirements/edx/paver.txt # botocore # elasticsearch # py2neo @@ -1239,8 +1210,6 @@ voluptuous==0.15.2 # via ora2 walrus==0.9.4 # via edx-event-bus-redis -watchdog==5.0.3 - # via -r requirements/edx/paver.txt wcwidth==0.2.13 # via prompt-toolkit web-fragments==2.2.0 @@ -1261,7 +1230,7 @@ webob==1.8.9 # -r requirements/edx/kernel.in # xblock wrapt==1.16.0 - # via -r requirements/edx/paver.txt + # via -r requirements/edx/kernel.in xblock[django]==5.1.0 # via # -r requirements/edx/kernel.in diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 0143d81709d1..507d03937d24 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -1172,8 +1172,6 @@ libsass==0.10.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/assets.txt - # -r requirements/edx/doc.txt - # -r requirements/edx/testing.txt loremipsum==1.0.5 # via # -r requirements/edx/doc.txt @@ -1251,9 +1249,7 @@ mistune==3.0.2 # -r requirements/edx/doc.txt # sphinx-mdinclude mock==5.1.0 - # via - # -r requirements/edx/doc.txt - # -r requirements/edx/testing.txt + # via -r requirements/edx/testing.txt mongoengine==0.29.1 # via # -r requirements/edx/doc.txt @@ -1440,9 +1436,7 @@ path-py==12.5.0 # ora2 # staff-graded-xblock paver==1.3.4 - # via - # -r requirements/edx/doc.txt - # -r requirements/edx/testing.txt + # via -r requirements/edx/testing.txt pbr==6.1.0 # via # -r requirements/edx/doc.txt @@ -1743,9 +1737,7 @@ python-ipware==3.0.0 # -r requirements/edx/testing.txt # django-ipware python-memcached==1.62 - # via - # -r requirements/edx/doc.txt - # -r requirements/edx/testing.txt + # via -r requirements/edx/testing.txt python-slugify==8.0.4 # via # -r requirements/edx/doc.txt @@ -2202,10 +2194,7 @@ walrus==0.9.4 # -r requirements/edx/testing.txt # edx-event-bus-redis watchdog==5.0.3 - # via - # -r requirements/edx/development.in - # -r requirements/edx/doc.txt - # -r requirements/edx/testing.txt + # via -r requirements/edx/development.in wcwidth==0.2.13 # via # -r requirements/edx/doc.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 1ed8a41699b7..bcaa0296d996 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -840,10 +840,6 @@ lazy==1.6 # lti-consumer-xblock # ora2 # xblock -libsass==0.10.0 - # via - # -c requirements/edx/../constraints.txt - # -r requirements/edx/base.txt loremipsum==1.0.5 # via # -r requirements/edx/base.txt @@ -901,8 +897,6 @@ meilisearch==0.31.6 # edx-search mistune==3.0.2 # via sphinx-mdinclude -mock==5.1.0 - # via -r requirements/edx/base.txt mongoengine==0.29.1 # via -r requirements/edx/base.txt monotonic==1.6 @@ -1030,8 +1024,6 @@ path-py==12.5.0 # edx-enterprise # ora2 # staff-graded-xblock -paver==1.3.4 - # via -r requirements/edx/base.txt pbr==6.1.0 # via # -r requirements/edx/base.txt @@ -1203,8 +1195,6 @@ python-ipware==3.0.0 # via # -r requirements/edx/base.txt # django-ipware -python-memcached==1.62 - # via -r requirements/edx/base.txt python-slugify==8.0.4 # via # -r requirements/edx/base.txt @@ -1358,10 +1348,8 @@ six==1.16.0 # fs-s3fs # html5lib # interchange - # libsass # optimizely-sdk # pansi - # paver # py2neo # pyjwkest # python-dateutil @@ -1531,8 +1519,6 @@ walrus==0.9.4 # via # -r requirements/edx/base.txt # edx-event-bus-redis -watchdog==5.0.3 - # via -r requirements/edx/base.txt wcwidth==0.2.13 # via # -r requirements/edx/base.txt diff --git a/requirements/edx/kernel.in b/requirements/edx/kernel.in index 7323c243accf..16740f734da5 100644 --- a/requirements/edx/kernel.in +++ b/requirements/edx/kernel.in @@ -3,7 +3,6 @@ -c ../constraints.txt -r github.in # Forks and other dependencies not yet on PyPI --r paver.txt # Requirements for running paver commands # DON'T JUST ADD NEW DEPENDENCIES!!! # Please follow these guidelines whenever you change this file: @@ -125,6 +124,7 @@ openedx-django-wiki path piexif # Exif image metadata manipulation, used in the profile_images app Pillow # Image manipulation library; used for course assets, profile images, invoice PDFs, etc. +psutil # Library for retrieving information on running processes and system utilization pycountry pycryptodomex pyjwkest @@ -132,6 +132,7 @@ pyjwkest # PyJWT 1.6.3 contains PyJWTError, which is required by Apple auth in social-auth-core PyJWT>=1.6.3 pylti1p3 # Required by content_libraries core library to support LTI 1.3 launches +pymemcache # Python interface to the memcached memory cache daemon pymongo # MongoDB driver pynliner # Inlines CSS styles into HTML for email notifications python-dateutil @@ -158,5 +159,6 @@ unicodecsv # Easier support for CSV files with unicode user-util # Functionality for retiring users (GDPR compliance) webob web-fragments # Provides the ability to render fragments of web pages +wrapt # Better functools.wrapped. TODO: functools has since improved, maybe we can switch? XBlock[django] # Courseware component architecture xss-utils # https://github.com/openedx/edx-platform/pull/20633 Fix XSS via Translations diff --git a/requirements/edx/paver.in b/requirements/edx/paver.in index 6987ede82275..a4d779722091 100644 --- a/requirements/edx/paver.in +++ b/requirements/edx/paver.in @@ -10,18 +10,9 @@ -c ../constraints.txt -edx-opaque-keys # Create and introspect course and xblock identities -lazy # Lazily-evaluated attributes for Python objects -libsass # Python bindings for the LibSass CSS compiler -markupsafe # XML/HTML/XHTML Markup safe strings -mock # Stub out code with mock objects and make assertions about how they have been used path # Easier manipulation of filesystem paths paver # Build, distribution and deployment scripting tool psutil # Library for retrieving information on running processes and system utilization -pymongo # via edx-opaque-keys python-memcached # Python interface to the memcached memory cache daemon pymemcache # Python interface to the memcached memory cache daemon -requests # Simple interface for making HTTP requests -stevedore # Support for runtime plugins, used for XBlocks and edx-platform Django app plugins -watchdog # Used in paver watch_assets wrapt # Decorator utilities used in the @timed paver task decorator diff --git a/requirements/edx/paver.txt b/requirements/edx/paver.txt index f3dae3b0efda..09fef1843449 100644 --- a/requirements/edx/paver.txt +++ b/requirements/edx/paver.txt @@ -4,62 +4,19 @@ # # make upgrade # -certifi==2024.8.30 - # via requests -charset-normalizer==2.0.12 - # via - # -c requirements/edx/../constraints.txt - # requests -dnspython==2.7.0 - # via pymongo -edx-opaque-keys==2.11.0 - # via -r requirements/edx/paver.in -idna==3.10 - # via requests -lazy==1.6 - # via -r requirements/edx/paver.in -libsass==0.10.0 - # via - # -c requirements/edx/../constraints.txt - # -r requirements/edx/paver.in -markupsafe==3.0.2 - # via -r requirements/edx/paver.in -mock==5.1.0 - # via -r requirements/edx/paver.in path==16.11.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/paver.in paver==1.3.4 # via -r requirements/edx/paver.in -pbr==6.1.0 - # via stevedore psutil==6.1.0 # via -r requirements/edx/paver.in pymemcache==4.0.0 # via -r requirements/edx/paver.in -pymongo==4.4.0 - # via - # -c requirements/edx/../constraints.txt - # -r requirements/edx/paver.in - # edx-opaque-keys python-memcached==1.62 # via -r requirements/edx/paver.in -requests==2.32.3 - # via -r requirements/edx/paver.in six==1.16.0 - # via - # libsass - # paver -stevedore==5.3.0 - # via - # -r requirements/edx/paver.in - # edx-opaque-keys -typing-extensions==4.12.2 - # via edx-opaque-keys -urllib3==2.2.3 - # via requests -watchdog==5.0.3 - # via -r requirements/edx/paver.in + # via paver wrapt==1.16.0 # via -r requirements/edx/paver.in diff --git a/requirements/edx/testing.in b/requirements/edx/testing.in index b903768f4de6..9077331d4cd4 100644 --- a/requirements/edx/testing.in +++ b/requirements/edx/testing.in @@ -16,6 +16,7 @@ -r base.txt # Core edx-platform production dependencies -r coverage.txt # Utilities for calculating test coverage +-r paver.txt # Quality and JS testing, for now (https://github.com/openedx/edx-platform/issues/34845) beautifulsoup4 # Library for extracting data from HTML and XML files code-annotations # Perform code annotation checking, such as for PII annotations @@ -28,6 +29,7 @@ freezegun # Allows tests to mock the output of assorted datetime httpretty # Library for mocking HTTP requests, used in many tests import-linter # Tool for making assertions about which modules can import which others isort # For checking and fixing the order of imports +mock # Deprecated alias to standard library `unittest.mock` pycodestyle # Checker for compliance with the Python style guide (PEP 8) polib # Library for manipulating gettext translation files, used to test paver i18n commands pyquery # jQuery-like API for retrieving fragments of HTML and XML files in tests diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 28e358a32cf7..3522eea47bae 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -889,10 +889,6 @@ lazy==1.6 # xblock lazy-object-proxy==1.10.0 # via astroid -libsass==0.10.0 - # via - # -c requirements/edx/../constraints.txt - # -r requirements/edx/base.txt loremipsum==1.0.5 # via # -r requirements/edx/base.txt @@ -953,7 +949,7 @@ meilisearch==0.31.6 # -r requirements/edx/base.txt # edx-search mock==5.1.0 - # via -r requirements/edx/base.txt + # via -r requirements/edx/testing.in mongoengine==0.29.1 # via -r requirements/edx/base.txt monotonic==1.6 @@ -1077,6 +1073,7 @@ path==16.11.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt + # -r requirements/edx/paver.txt # edx-i18n-tools # path-py path-py==12.5.0 @@ -1086,7 +1083,7 @@ path-py==12.5.0 # ora2 # staff-graded-xblock paver==1.3.4 - # via -r requirements/edx/base.txt + # via -r requirements/edx/paver.txt pbr==6.1.0 # via # -r requirements/edx/base.txt @@ -1145,6 +1142,7 @@ protobuf==5.28.3 psutil==6.1.0 # via # -r requirements/edx/base.txt + # -r requirements/edx/paver.txt # edx-django-utils # pact-python # pytest-xdist @@ -1237,7 +1235,9 @@ pylint-pytest==0.3.0 pylti1p3==2.0.0 # via -r requirements/edx/base.txt pymemcache==4.0.0 - # via -r requirements/edx/base.txt + # via + # -r requirements/edx/base.txt + # -r requirements/edx/paver.txt pymongo==4.4.0 # via # -c requirements/edx/../constraints.txt @@ -1323,7 +1323,7 @@ python-ipware==3.0.0 # -r requirements/edx/base.txt # django-ipware python-memcached==1.62 - # via -r requirements/edx/base.txt + # via -r requirements/edx/paver.txt python-slugify==8.0.4 # via # -r requirements/edx/base.txt @@ -1462,6 +1462,7 @@ singledispatch==4.1.0 six==1.16.0 # via # -r requirements/edx/base.txt + # -r requirements/edx/paver.txt # analytics-python # bleach # codejail-includes @@ -1479,7 +1480,6 @@ six==1.16.0 # fs-s3fs # html5lib # interchange - # libsass # optimizely-sdk # pact-python # pansi @@ -1628,8 +1628,6 @@ walrus==0.9.4 # via # -r requirements/edx/base.txt # edx-event-bus-redis -watchdog==5.0.3 - # via -r requirements/edx/base.txt wcwidth==0.2.13 # via # -r requirements/edx/base.txt @@ -1655,6 +1653,7 @@ webob==1.8.9 wrapt==1.16.0 # via # -r requirements/edx/base.txt + # -r requirements/edx/paver.txt # astroid xblock[django]==5.1.0 # via