diff --git a/Makefile b/Makefile index 62681f6f3711..717409d4f9e0 100644 --- a/Makefile +++ b/Makefile @@ -104,7 +104,6 @@ shell: ## launch a bash shell in a Docker container with all edx-platform depend # Order is very important in this list: files must appear after everything they include! REQ_FILES = \ requirements/edx/coverage \ - requirements/edx/paver \ requirements/edx-sandbox/base \ requirements/edx/base \ requirements/edx/doc \ @@ -211,7 +210,7 @@ xsslint: ## check xss for quality issuest --config=scripts.xsslint_config \ --thresholds=scripts/xsslint_thresholds.json -pycodestyle: ## check python files for quality issues +pycodestyle: ## check python files for quality issues pycodestyle . ## Re-enable --lint flag when this issue https://github.com/openedx/edx-platform/issues/35775 is resolved @@ -222,13 +221,13 @@ pii_check: ## check django models for pii annotations --app_name cms \ --coverage \ --lint - + DJANGO_SETTINGS_MODULE=lms.envs.test \ code_annotations django_find_annotations \ --config_file .pii_annotations.yml \ --app_name lms \ --coverage \ - --lint + --lint check_keywords: ## check django models for reserve keywords DJANGO_SETTINGS_MODULE=cms.envs.test \ diff --git a/openedx/core/djangoapps/theming/management/commands/compile_sass.py b/openedx/core/djangoapps/theming/management/commands/compile_sass.py deleted file mode 100644 index fbfdd2f222a4..000000000000 --- a/openedx/core/djangoapps/theming/management/commands/compile_sass.py +++ /dev/null @@ -1,112 +0,0 @@ -""" -Management command for compiling sass. - -DEPRECATED in favor of `npm run compile-sass`. -""" -import shlex - -from django.core.management import BaseCommand -from django.conf import settings - -from pavelib.assets import run_deprecated_command_wrapper - - -class Command(BaseCommand): - """ - Compile theme sass and collect theme assets. - """ - - help = "DEPRECATED. Use 'npm run compile-sass' instead." - - # NOTE (CCB): This allows us to compile static assets in Docker containers without database access. - requires_system_checks = [] - - def add_arguments(self, parser): - """ - Add arguments for compile_sass command. - - Args: - parser (django.core.management.base.CommandParser): parsed for parsing command line arguments. - """ - parser.add_argument( - 'system', type=str, nargs='*', default=["lms", "cms"], - help="lms or studio", - ) - - # Named (optional) arguments - parser.add_argument( - '--theme-dirs', - dest='theme_dirs', - type=str, - nargs='+', - default=None, - help="List of dirs where given themes would be looked.", - ) - - parser.add_argument( - '--themes', - type=str, - nargs='+', - default=["all"], - help="List of themes whose sass need to compiled. Or 'no'/'all' to compile for no/all themes.", - ) - - # Named (optional) arguments - parser.add_argument( - '--force', - action='store_true', - default=False, - help="DEPRECATED. Full recompilation is now always forced.", - ) - parser.add_argument( - '--debug', - action='store_true', - default=False, - help="Disable Sass compression", - ) - - def handle(self, *args, **options): - """ - Handle compile_sass command. - """ - systems = set( - {"lms": "lms", "cms": "cms", "studio": "cms"}[sys] - for sys in options.get("system", ["lms", "cms"]) - ) - theme_dirs = options.get("theme_dirs") or settings.COMPREHENSIVE_THEME_DIRS or [] - themes_option = options.get("themes") or [] # '[]' means 'all' - if not settings.ENABLE_COMPREHENSIVE_THEMING: - compile_themes = False - themes = [] - elif "no" in themes_option: - compile_themes = False - themes = [] - elif "all" in themes_option: - compile_themes = True - themes = [] - else: - compile_themes = True - themes = themes_option - run_deprecated_command_wrapper( - old_command="./manage.py [lms|cms] compile_sass", - ignored_old_flags=list(set(["force"]) & set(options)), - new_command=shlex.join([ - "npm", - "run", - ("compile-sass-dev" if options.get("debug") else "compile-sass"), - "--", - *(["--skip-lms"] if "lms" not in systems else []), - *(["--skip-cms"] if "cms" not in systems else []), - *(["--skip-themes"] if not compile_themes else []), - *( - arg - for theme_dir in theme_dirs - for arg in ["--theme-dir", str(theme_dir)] - ), - *( - arg - for theme in themes - for arg in ["--theme", theme] - ), - ]), - ) diff --git a/pavelib/__init__.py b/pavelib/__init__.py deleted file mode 100644 index 24f05618bdd7..000000000000 --- a/pavelib/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -""" # lint-amnesty, pylint: disable=django-not-configured -paver commands -""" - - -from . import assets diff --git a/pavelib/assets.py b/pavelib/assets.py deleted file mode 100644 index f437b6427f93..000000000000 --- a/pavelib/assets.py +++ /dev/null @@ -1,506 +0,0 @@ -""" -Asset compilation and collection. - -This entire module is DEPRECATED. In Redwood, it exists just as a collection of temporary compatibility wrappers. -In Sumac, this module will be deleted. To migrate, follow the advice in the printed warnings and/or read the -instructions on the DEPR ticket: https://github.com/openedx/edx-platform/issues/31895 -""" - -import argparse -import glob -import json -import shlex -import traceback -from functools import wraps -from threading import Timer - -from paver import tasks -from paver.easy import call_task, cmdopts, consume_args, needs, no_help, sh, task -from watchdog.events import PatternMatchingEventHandler -from watchdog.observers import Observer # pylint: disable=unused-import # Used by Tutor. Remove after Sumac cut. - -from .utils.cmd import django_cmd -from .utils.envs import Env -from .utils.timer import timed - - -SYSTEMS = { - 'lms': 'lms', - 'cms': 'cms', - 'studio': 'cms', -} - -WARNING_SYMBOLS = "⚠️ " * 50 # A row of 'warning' emoji to catch CLI users' attention - - -def run_deprecated_command_wrapper(*, old_command, ignored_old_flags, new_command): - """ - Run the new version of shell command, plus a warning that the old version is deprecated. - """ - depr_warning = ( - "\n" + - f"{WARNING_SYMBOLS}\n" + - "\n" + - f"WARNING: '{old_command}' is DEPRECATED! It will be removed before Sumac.\n" + - "The command you ran is now just a temporary wrapper around a new,\n" + - "supported command, which you should use instead:\n" + - "\n" + - f"\t{new_command}\n" + - "\n" + - "Details: https://github.com/openedx/edx-platform/issues/31895\n" + - "".join( - f" WARNING: ignored deprecated paver flag '{flag}'\n" - for flag in ignored_old_flags - ) + - f"{WARNING_SYMBOLS}\n" + - "\n" - ) - # Print deprecation warning twice so that it's more likely to be seen in the logs. - print(depr_warning) - sh(new_command) - print(depr_warning) - - -def debounce(seconds=1): - """ - Prevents the decorated function from being called more than every `seconds` - seconds. Waits until calls stop coming in before calling the decorated - function. - - This is DEPRECATED. It exists in Redwood just to ease the transition for Tutor. - """ - def decorator(func): - func.timer = None - - @wraps(func) - def wrapper(*args, **kwargs): - def call(): - func(*args, **kwargs) - func.timer = None - if func.timer: - func.timer.cancel() - func.timer = Timer(seconds, call) - func.timer.start() - - return wrapper - return decorator - - -class SassWatcher(PatternMatchingEventHandler): - """ - Watches for sass file changes - - This is DEPRECATED. It exists in Redwood just to ease the transition for Tutor. - """ - ignore_directories = True - patterns = ['*.scss'] - - def register(self, observer, directories): - """ - register files with observer - Arguments: - observer (watchdog.observers.Observer): sass file observer - directories (list): list of directories to be register for sass watcher. - """ - for dirname in directories: - paths = [] - if '*' in dirname: - paths.extend(glob.glob(dirname)) - else: - paths.append(dirname) - - for obs_dirname in paths: - observer.schedule(self, obs_dirname, recursive=True) - - @debounce() - def on_any_event(self, event): - print('\tCHANGED:', event.src_path) - try: - compile_sass() # pylint: disable=no-value-for-parameter - except Exception: # pylint: disable=broad-except - traceback.print_exc() - - -@task -@no_help -@cmdopts([ - ('system=', 's', 'The system to compile sass for (defaults to all)'), - ('theme-dirs=', '-td', 'Theme dirs containing all themes (defaults to None)'), - ('themes=', '-t', 'The theme to compile sass for (defaults to None)'), - ('debug', 'd', 'Whether to use development settings'), - ('force', '', 'DEPRECATED. Full recompilation is now always forced.'), -]) -@timed -def compile_sass(options): - """ - Compile Sass to CSS. If command is called without any arguments, it will - only compile lms, cms sass for the open source theme. And none of the comprehensive theme's sass would be compiled. - - If you want to compile sass for all comprehensive themes you will have to run compile_sass - specifying all the themes that need to be compiled.. - - The following is a list of some possible ways to use this command. - - Command: - paver compile_sass - Description: - compile sass files for both lms and cms. If command is called like above (i.e. without any arguments) it will - only compile lms, cms sass for the open source theme. None of the theme's sass will be compiled. - - Command: - paver compile_sass --theme-dirs /edx/app/edxapp/edx-platform/themes --themes=red-theme - Description: - compile sass files for both lms and cms for 'red-theme' present in '/edx/app/edxapp/edx-platform/themes' - - Command: - paver compile_sass --theme-dirs=/edx/app/edxapp/edx-platform/themes --themes red-theme stanford-style - Description: - compile sass files for both lms and cms for 'red-theme' and 'stanford-style' present in - '/edx/app/edxapp/edx-platform/themes'. - - Command: - paver compile_sass --system=cms - --theme-dirs /edx/app/edxapp/edx-platform/themes /edx/app/edxapp/edx-platform/common/test/ - --themes red-theme stanford-style test-theme - Description: - compile sass files for cms only for 'red-theme', 'stanford-style' and 'test-theme' present in - '/edx/app/edxapp/edx-platform/themes' and '/edx/app/edxapp/edx-platform/common/test/'. - - This is a DEPRECATED COMPATIBILITY WRAPPER. Use `npm run compile-sass` instead. - """ - systems = [SYSTEMS[sys] for sys in get_parsed_option(options, 'system', ['lms', 'cms'])] # normalize studio->cms - run_deprecated_command_wrapper( - old_command="paver compile_sass", - ignored_old_flags=(set(["--force"]) & set(options)), - new_command=shlex.join([ - "npm", - "run", - ("compile-sass-dev" if options.get("debug") else "compile-sass"), - "--", - *(["--dry"] if tasks.environment.dry_run else []), - *(["--skip-lms"] if "lms" not in systems else []), - *(["--skip-cms"] if "cms" not in systems else []), - *( - arg - for theme_dir in get_parsed_option(options, 'theme_dirs', []) - for arg in ["--theme-dir", str(theme_dir)] - ), - *( - arg - for theme in get_parsed_option(options, "themes", []) - for arg in ["--theme", theme] - ), - ]), - ) - - -def _compile_sass(system, theme, debug, force, _timing_info): - """ - This is a DEPRECATED COMPATIBILITY WRAPPER - - It exists to ease the transition for Tutor in Redwood, which directly imported and used this function. - """ - run_deprecated_command_wrapper( - old_command="pavelib.assets:_compile_sass", - ignored_old_flags=(set(["--force"]) if force else set()), - new_command=[ - "npm", - "run", - ("compile-sass-dev" if debug else "compile-sass"), - "--", - *(["--dry"] if tasks.environment.dry_run else []), - *( - ["--skip-default", "--theme-dir", str(theme.parent), "--theme", str(theme.name)] - if theme - else [] - ), - ("--skip-cms" if system == "lms" else "--skip-lms"), - ] - ) - - -def process_npm_assets(): - """ - Process vendor libraries installed via NPM. - - This is a DEPRECATED COMPATIBILITY WRAPPER. It is now handled as part of `npm clean-install`. - If you need to invoke it explicitly, you can run `npm run postinstall`. - """ - run_deprecated_command_wrapper( - old_command="pavelib.assets:process_npm_assets", - ignored_old_flags=[], - new_command=shlex.join(["npm", "run", "postinstall"]), - ) - - -@task -@no_help -def process_xmodule_assets(): - """ - Process XModule static assets. - - This is a DEPRECATED COMPATIBILITY STUB. Refrences to it should be deleted. - """ - print( - "\n" + - f"{WARNING_SYMBOLS}", - "\n" + - "WARNING: 'paver process_xmodule_assets' is DEPRECATED! It will be removed before Sumac.\n" + - "\n" + - "Starting with Quince, it is no longer necessary to post-process XModule assets, so \n" + - "'paver process_xmodule_assets' is a no-op. Please simply remove it from your build scripts.\n" + - "\n" + - "Details: https://github.com/openedx/edx-platform/issues/31895\n" + - f"{WARNING_SYMBOLS}", - ) - - -def collect_assets(systems, settings, **kwargs): - """ - Collect static assets, including Django pipeline processing. - `systems` is a list of systems (e.g. 'lms' or 'studio' or both) - `settings` is the Django settings module to use. - `**kwargs` include arguments for using a log directory for collectstatic output. Defaults to /dev/null. - - This is a DEPRECATED COMPATIBILITY WRAPPER - - It exists to ease the transition for Tutor in Redwood, which directly imported and used this function. - """ - run_deprecated_command_wrapper( - old_command="pavelib.asset:collect_assets", - ignored_old_flags=[], - new_command=" && ".join( - "( " + - shlex.join( - ["./manage.py", SYSTEMS[sys], f"--settings={settings}", "collectstatic", "--noinput"] - ) + ( - "" - if "collect_log_dir" not in kwargs else - " > /dev/null" - if kwargs["collect_log_dir"] is None else - f"> {kwargs['collect_log_dir']}/{SYSTEMS[sys]}-collectstatic.out" - ) + - " )" - for sys in systems - ), - ) - - -def execute_compile_sass(args): - """ - Construct django management command compile_sass (defined in theming app) and execute it. - Args: - args: command line argument passed via update_assets command - - This is a DEPRECATED COMPATIBILITY WRAPPER. Use `npm run compile-sass` instead. - """ - for sys in args.system: - options = "" - options += " --theme-dirs " + " ".join(args.theme_dirs) if args.theme_dirs else "" - options += " --themes " + " ".join(args.themes) if args.themes else "" - options += " --debug" if args.debug else "" - - sh( - django_cmd( - sys, - args.settings, - "compile_sass {system} {options}".format( - system='cms' if sys == 'studio' else sys, - options=options, - ), - ), - ) - - -@task -@cmdopts([ - ('settings=', 's', "Django settings (defaults to devstack)"), - ('watch', 'w', "DEPRECATED. This flag never did anything anyway."), -]) -@timed -def webpack(options): - """ - Run a Webpack build. - - This is a DEPRECATED COMPATIBILITY WRAPPER. Use `npm run webpack` instead. - """ - settings = getattr(options, 'settings', Env.DEVSTACK_SETTINGS) - result = Env.get_django_settings(['STATIC_ROOT', 'WEBPACK_CONFIG_PATH'], "lms", settings=settings) - static_root_lms, config_path = result - static_root_cms, = Env.get_django_settings(["STATIC_ROOT"], "cms", settings=settings) - js_env_extra_config_setting, = Env.get_django_json_settings(["JS_ENV_EXTRA_CONFIG"], "cms", settings=settings) - js_env_extra_config = json.dumps(js_env_extra_config_setting or "{}") - node_env = "development" if config_path == 'webpack.dev.config.js' else "production" - run_deprecated_command_wrapper( - old_command="paver webpack", - ignored_old_flags=(set(["watch"]) & set(options)), - new_command=' '.join([ - f"WEBPACK_CONFIG_PATH={config_path}", - f"NODE_ENV={node_env}", - f"STATIC_ROOT_LMS={static_root_lms}", - f"STATIC_ROOT_CMS={static_root_cms}", - f"JS_ENV_EXTRA_CONFIG={js_env_extra_config}", - "npm", - "run", - "webpack", - ]), - ) - - -def get_parsed_option(command_opts, opt_key, default=None): - """ - Extract user command option and parse it. - Arguments: - command_opts: Command line arguments passed via paver command. - opt_key: name of option to get and parse - default: if `command_opt_value` not in `command_opts`, `command_opt_value` will be set to default. - Returns: - list or None - """ - command_opt_value = getattr(command_opts, opt_key, default) - if command_opt_value: - command_opt_value = listfy(command_opt_value) - - return command_opt_value - - -def listfy(data): - """ - Check and convert data to list. - Arguments: - data: data structure to be converted. - """ - - if isinstance(data, str): - data = data.split(',') - elif not isinstance(data, list): - data = [data] - - return data - - -@task -@cmdopts([ - ('background', 'b', 'DEPRECATED. Use shell tools like & to run in background if needed.'), - ('settings=', 's', "DEPRECATED. Django is not longer invoked to compile JS/Sass."), - ('theme-dirs=', '-td', 'The themes dir containing all themes (defaults to None)'), - ('themes=', '-t', 'DEPRECATED. All themes in --theme-dirs are now watched.'), - ('wait=', '-w', 'DEPRECATED. Watchdog\'s default wait time is now used.'), -]) -@timed -def watch_assets(options): - """ - Watch for changes to asset files, and regenerate js/css - - This is a DEPRECATED COMPATIBILITY WRAPPER. Use `npm run watch` instead. - """ - # Don't watch assets when performing a dry run - if tasks.environment.dry_run: - return - - theme_dirs = ':'.join(get_parsed_option(options, 'theme_dirs', [])) - run_deprecated_command_wrapper( - old_command="paver watch_assets", - ignored_old_flags=(set(["debug", "themes", "settings", "background"]) & set(options)), - new_command=shlex.join([ - *( - ["env", f"COMPREHENSIVE_THEME_DIRS={theme_dirs}"] - if theme_dirs else [] - ), - "npm", - "run", - "watch", - ]), - ) - - -@task -@needs( - 'pavelib.prereqs.install_node_prereqs', - 'pavelib.prereqs.install_python_prereqs', -) -@consume_args -@timed -def update_assets(args): - """ - Compile Sass, then collect static assets. - - This is a DEPRECATED COMPATIBILITY WRAPPER around other DEPRECATED COMPATIBILITY WRAPPERS. - The aggregate affect of this command can be achieved with this sequence of commands instead: - - * pip install -r requirements/edx/assets.txt # replaces install_python_prereqs - * npm clean-install # replaces install_node_prereqs - * npm run build # replaces execute_compile_sass and webpack - * ./manage.py lms collectstatic --noinput # replaces collect_assets (for LMS) - * ./manage.py cms collectstatic --noinput # replaces collect_assets (for CMS) - """ - parser = argparse.ArgumentParser(prog='paver update_assets') - parser.add_argument( - 'system', type=str, nargs='*', default=["lms", "studio"], - help="lms or studio", - ) - parser.add_argument( - '--settings', type=str, default=Env.DEVSTACK_SETTINGS, - help="Django settings module", - ) - parser.add_argument( - '--debug', action='store_true', default=False, - help="Enable all debugging", - ) - parser.add_argument( - '--debug-collect', action='store_true', default=False, - help="Disable collect static", - ) - parser.add_argument( - '--skip-collect', dest='collect', action='store_false', default=True, - help="Skip collection of static assets", - ) - parser.add_argument( - '--watch', action='store_true', default=False, - help="Watch files for changes", - ) - parser.add_argument( - '--theme-dirs', dest='theme_dirs', type=str, nargs='+', default=None, - help="base directories where themes are placed", - ) - parser.add_argument( - '--themes', type=str, nargs='+', default=None, - help="list of themes to compile sass for. ignored when --watch is used; all themes are watched.", - ) - parser.add_argument( - '--collect-log', dest="collect_log_dir", default=None, - help="When running collectstatic, direct output to specified log directory", - ) - parser.add_argument( - '--wait', type=float, default=0.0, - help="DEPRECATED. Watchdog's default wait time is now used.", - ) - args = parser.parse_args(args) - - # Build Webpack - call_task('pavelib.assets.webpack', options={'settings': args.settings}) - - # Compile sass for themes and system - execute_compile_sass(args) - - if args.collect: - if args.collect_log_dir: - collect_log_args = {"collect_log_dir": args.collect_log_dir} - elif args.debug or args.debug_collect: - collect_log_args = {"collect_log_dir": None} - else: - collect_log_args = {} - - collect_assets(args.system, args.settings, **collect_log_args) - - if args.watch: - call_task( - 'pavelib.assets.watch_assets', - options={ - 'background': not args.debug, - 'settings': args.settings, - 'theme_dirs': args.theme_dirs, - 'themes': args.themes, - 'wait': [float(args.wait)] - }, - ) diff --git a/pavelib/paver_tests/__init__.py b/pavelib/paver_tests/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/pavelib/paver_tests/pylint_test_list.json b/pavelib/paver_tests/pylint_test_list.json deleted file mode 100644 index d0e8b43aa93d..000000000000 --- a/pavelib/paver_tests/pylint_test_list.json +++ /dev/null @@ -1,6 +0,0 @@ -[ - "foo/bar.py:192: [C0111(missing-docstring), Bliptv] Missing docstring", - "foo/bar/test.py:74: [C0322(no-space-before-operator)] Operator not preceded by a space", - "ugly/string/test.py:16: [C0103(invalid-name)] Invalid name \"whats up\" for type constant (should match (([A-Z_][A-Z0-9_]*)|(__.*__)|log|urlpatterns)$)", - "multiple/lines/test.py:72: [C0322(no-space-before-operator)] Operator not preceded by a space\nFOO_BAR='pipeline.storage.NonPackagingPipelineStorage'\n ^" -] diff --git a/pavelib/paver_tests/test_assets.py b/pavelib/paver_tests/test_assets.py deleted file mode 100644 index f7100a7f03c3..000000000000 --- a/pavelib/paver_tests/test_assets.py +++ /dev/null @@ -1,130 +0,0 @@ -"""Unit tests for the Paver asset tasks.""" - -import json -import os -from pathlib import Path -from unittest import TestCase -from unittest.mock import patch - -import ddt -import paver.easy -from paver import tasks - -import pavelib.assets -from pavelib.assets import Env - - -REPO_ROOT = Path(__file__).parent.parent.parent - -LMS_SETTINGS = { - "WEBPACK_CONFIG_PATH": "webpack.fake.config.js", - "STATIC_ROOT": "/fake/lms/staticfiles", - -} -CMS_SETTINGS = { - "WEBPACK_CONFIG_PATH": "webpack.fake.config", - "STATIC_ROOT": "/fake/cms/staticfiles", - "JS_ENV_EXTRA_CONFIG": json.dumps({"key1": [True, False], "key2": {"key2.1": 1369, "key2.2": "1369"}}), -} - - -def _mock_get_django_settings(django_settings, system, settings=None): # pylint: disable=unused-argument - return [(LMS_SETTINGS if system == "lms" else CMS_SETTINGS)[s] for s in django_settings] - - -@ddt.ddt -@patch.object(Env, 'get_django_settings', _mock_get_django_settings) -@patch.object(Env, 'get_django_json_settings', _mock_get_django_settings) -class TestDeprecatedPaverAssets(TestCase): - """ - Simple test to ensure that the soon-to-be-removed Paver commands are correctly translated into the new npm-run - commands. - """ - def setUp(self): - super().setUp() - self.maxDiff = None - os.environ['NO_PREREQ_INSTALL'] = 'true' - tasks.environment = tasks.Environment() - - def tearDown(self): - super().tearDown() - del os.environ['NO_PREREQ_INSTALL'] - - @ddt.data( - dict( - task_name='pavelib.assets.compile_sass', - args=[], - kwargs={}, - expected=["npm run compile-sass --"], - ), - dict( - task_name='pavelib.assets.compile_sass', - args=[], - kwargs={"system": "lms,studio"}, - expected=["npm run compile-sass --"], - ), - dict( - task_name='pavelib.assets.compile_sass', - args=[], - kwargs={"debug": True}, - expected=["npm run compile-sass-dev --"], - ), - dict( - task_name='pavelib.assets.compile_sass', - args=[], - kwargs={"system": "lms"}, - expected=["npm run compile-sass -- --skip-cms"], - ), - dict( - task_name='pavelib.assets.compile_sass', - args=[], - kwargs={"system": "studio"}, - expected=["npm run compile-sass -- --skip-lms"], - ), - dict( - task_name='pavelib.assets.compile_sass', - args=[], - kwargs={"system": "cms", "theme_dirs": f"{REPO_ROOT}/common/test,{REPO_ROOT}/themes"}, - expected=[ - "npm run compile-sass -- --skip-lms " + - f"--theme-dir {REPO_ROOT}/common/test --theme-dir {REPO_ROOT}/themes" - ], - ), - dict( - task_name='pavelib.assets.compile_sass', - args=[], - kwargs={"theme_dirs": f"{REPO_ROOT}/common/test,{REPO_ROOT}/themes", "themes": "red-theme,test-theme"}, - expected=[ - "npm run compile-sass -- " + - f"--theme-dir {REPO_ROOT}/common/test --theme-dir {REPO_ROOT}/themes " + - "--theme red-theme --theme test-theme" - ], - ), - dict( - task_name='pavelib.assets.update_assets', - args=["lms", "studio", "--settings=fake.settings"], - kwargs={}, - expected=[ - ( - "WEBPACK_CONFIG_PATH=webpack.fake.config.js " + - "NODE_ENV=production " + - "STATIC_ROOT_LMS=/fake/lms/staticfiles " + - "STATIC_ROOT_CMS=/fake/cms/staticfiles " + - 'JS_ENV_EXTRA_CONFIG=' + - '"{\\"key1\\": [true, false], \\"key2\\": {\\"key2.1\\": 1369, \\"key2.2\\": \\"1369\\"}}" ' + - "npm run webpack" - ), - "python manage.py lms --settings=fake.settings compile_sass lms ", - "python manage.py cms --settings=fake.settings compile_sass cms ", - ( - "( ./manage.py lms --settings=fake.settings collectstatic --noinput ) && " + - "( ./manage.py cms --settings=fake.settings collectstatic --noinput )" - ), - ], - ), - ) - @ddt.unpack - @patch.object(pavelib.assets, 'sh') - def test_paver_assets_wrapper_invokes_new_commands(self, mock_sh, task_name, args, kwargs, expected): - paver.easy.call_task(task_name, args=args, options=kwargs) - assert [call_args[0] for (call_args, call_kwargs) in mock_sh.call_args_list] == expected diff --git a/pavelib/paver_tests/utils.py b/pavelib/paver_tests/utils.py deleted file mode 100644 index 1db26cf76a4c..000000000000 --- a/pavelib/paver_tests/utils.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Unit tests for the Paver server tasks.""" - - -import os -from unittest import TestCase -from uuid import uuid4 - -from paver import tasks -from paver.easy import BuildFailure - - -class PaverTestCase(TestCase): - """ - Base class for Paver test cases. - """ - def setUp(self): - super().setUp() - - # Show full length diffs upon test failure - self.maxDiff = None # pylint: disable=invalid-name - - # Create a mock Paver environment - tasks.environment = MockEnvironment() - - # Don't run pre-reqs - os.environ['NO_PREREQ_INSTALL'] = 'true' - - def tearDown(self): - super().tearDown() - tasks.environment = tasks.Environment() - del os.environ['NO_PREREQ_INSTALL'] - - @property - def task_messages(self): - """Returns the messages output by the Paver task.""" - return tasks.environment.messages - - @property - def platform_root(self): - """Returns the current platform's root directory.""" - return os.getcwd() - - def reset_task_messages(self): - """Clear the recorded message""" - tasks.environment.messages = [] - - -class MockEnvironment(tasks.Environment): - """ - Mock environment that collects information about Paver commands. - """ - def __init__(self): - super().__init__() - self.dry_run = True - self.messages = [] - - def info(self, message, *args): - """Capture any messages that have been recorded""" - if args: - output = message % args - else: - output = message - if not output.startswith("--->"): - self.messages.append(str(output)) - - -def fail_on_eslint(*args, **kwargs): - """ - For our tests, we need the call for diff-quality running eslint reports - to fail, since that is what is going to fail when we pass in a - percentage ("p") requirement. - """ - if "eslint" in args[0]: # lint-amnesty, pylint: disable=no-else-raise - raise BuildFailure('Subprocess return code: 1') - else: - if kwargs.get('capture', False): - return uuid4().hex - else: - return - - -def fail_on_npm_install(): - """ - Used to simulate an error when executing "npm install" - """ - return 1 - - -def unexpected_fail_on_npm_install(*args, **kwargs): # pylint: disable=unused-argument - """ - For our tests, we need the call for diff-quality running pycodestyle reports to fail, since that is what - is going to fail when we pass in a percentage ("p") requirement. - """ - if ["npm", "install", "--verbose"] == args[0]: # lint-amnesty, pylint: disable=no-else-raise - raise BuildFailure('Subprocess return code: 50') - else: - return diff --git a/pavelib/prereqs.py b/pavelib/prereqs.py deleted file mode 100644 index 4453176c94da..000000000000 --- a/pavelib/prereqs.py +++ /dev/null @@ -1,351 +0,0 @@ -""" -Install Python and Node prerequisites. -""" - - -import hashlib -import os -import re -import subprocess -import sys -from distutils import sysconfig # pylint: disable=deprecated-module - -from paver.easy import sh, task # lint-amnesty, pylint: disable=unused-import - -from .utils.envs import Env -from .utils.timer import timed - -PREREQS_STATE_DIR = os.getenv('PREREQ_CACHE_DIR', Env.REPO_ROOT / '.prereqs_cache') -NO_PREREQ_MESSAGE = "NO_PREREQ_INSTALL is set, not installing prereqs" -NO_PYTHON_UNINSTALL_MESSAGE = 'NO_PYTHON_UNINSTALL is set. No attempts will be made to uninstall old Python libs.' -COVERAGE_REQ_FILE = 'requirements/edx/coverage.txt' - -# If you make any changes to this list you also need to make -# a corresponding change to circle.yml, which is how the python -# prerequisites are installed for builds on circleci.com -toxenv = os.environ.get('TOXENV') -if toxenv and toxenv != 'quality': - PYTHON_REQ_FILES = ['requirements/edx/testing.txt'] -else: - PYTHON_REQ_FILES = ['requirements/edx/development.txt'] - -# Developers can have private requirements, for local copies of github repos, -# or favorite debugging tools, etc. -PRIVATE_REQS = 'requirements/edx/private.txt' -if os.path.exists(PRIVATE_REQS): - PYTHON_REQ_FILES.append(PRIVATE_REQS) - - -def str2bool(s): - s = str(s) - return s.lower() in ('yes', 'true', 't', '1') - - -def no_prereq_install(): - """ - Determine if NO_PREREQ_INSTALL should be truthy or falsy. - """ - return str2bool(os.environ.get('NO_PREREQ_INSTALL', 'False')) - - -def no_python_uninstall(): - """ Determine if we should run the uninstall_python_packages task. """ - return str2bool(os.environ.get('NO_PYTHON_UNINSTALL', 'False')) - - -def create_prereqs_cache_dir(): - """Create the directory for storing the hashes, if it doesn't exist already.""" - try: - os.makedirs(PREREQS_STATE_DIR) - except OSError: - if not os.path.isdir(PREREQS_STATE_DIR): - raise - - -def compute_fingerprint(path_list): - """ - Hash the contents of all the files and directories in `path_list`. - Returns the hex digest. - """ - - hasher = hashlib.sha1() - - for path_item in path_list: - - # For directories, create a hash based on the modification times - # of first-level subdirectories - if os.path.isdir(path_item): - for dirname in sorted(os.listdir(path_item)): - path_name = os.path.join(path_item, dirname) - if os.path.isdir(path_name): - hasher.update(str(os.stat(path_name).st_mtime).encode('utf-8')) - - # For files, hash the contents of the file - if os.path.isfile(path_item): - with open(path_item, "rb") as file_handle: - hasher.update(file_handle.read()) - - return hasher.hexdigest() - - -def prereq_cache(cache_name, paths, install_func): - """ - Conditionally execute `install_func()` only if the files/directories - specified by `paths` have changed. - - If the code executes successfully (no exceptions are thrown), the cache - is updated with the new hash. - """ - # Retrieve the old hash - cache_filename = cache_name.replace(" ", "_") - cache_file_path = os.path.join(PREREQS_STATE_DIR, f"{cache_filename}.sha1") - old_hash = None - if os.path.isfile(cache_file_path): - with open(cache_file_path) as cache_file: - old_hash = cache_file.read() - - # Compare the old hash to the new hash - # If they do not match (either the cache hasn't been created, or the files have changed), - # then execute the code within the block. - new_hash = compute_fingerprint(paths) - if new_hash != old_hash: - install_func() - - # Update the cache with the new hash - # If the code executed within the context fails (throws an exception), - # then this step won't get executed. - create_prereqs_cache_dir() - with open(cache_file_path, "wb") as cache_file: - # Since the pip requirement files are modified during the install - # process, we need to store the hash generated AFTER the installation - post_install_hash = compute_fingerprint(paths) - cache_file.write(post_install_hash.encode('utf-8')) - else: - print(f'{cache_name} unchanged, skipping...') - - -def node_prereqs_installation(): - """ - Configures npm and installs Node prerequisites - """ - # Before July 2023, these directories were created and written to - # as root. Afterwards, they are created as being owned by the - # `app` user -- but also need to be deleted by that user (due to - # how npm runs post-install scripts.) Developers with an older - # devstack installation who are reprovisioning will see errors - # here if the files are still owned by root. Deleting the files in - # advance prevents this error. - # - # This hack should probably be left in place for at least a year. - # See ADR 17 for more background on the transition. - sh("rm -rf common/static/common/js/vendor/ common/static/common/css/vendor/") - # At the time of this writing, the js dir has git-versioned files - # but the css dir does not, so the latter would have been created - # as root-owned (in the process of creating the vendor - # subdirectory). Delete it only if empty, just in case - # git-versioned files are added later. - sh("rmdir common/static/common/css || true") - - # NPM installs hang sporadically. Log the installation process so that we - # determine if any packages are chronic offenders. - npm_log_file_path = f'{Env.GEN_LOG_DIR}/npm-install.log' - npm_log_file = open(npm_log_file_path, 'wb') # lint-amnesty, pylint: disable=consider-using-with - npm_command = 'npm ci --verbose'.split() - - # The implementation of Paver's `sh` function returns before the forked - # actually returns. Using a Popen object so that we can ensure that - # the forked process has returned - proc = subprocess.Popen(npm_command, stderr=npm_log_file) # lint-amnesty, pylint: disable=consider-using-with - retcode = proc.wait() - if retcode == 1: - raise Exception(f"npm install failed: See {npm_log_file_path}") - print("Successfully clean-installed NPM packages. Log found at {}".format( - npm_log_file_path - )) - - -def python_prereqs_installation(): - """ - Installs Python prerequisites - """ - # edx-platform installs some Python projects from within the edx-platform repo itself. - sh("pip install -e .") - for req_file in PYTHON_REQ_FILES: - pip_install_req_file(req_file) - - -def pip_install_req_file(req_file): - """Pip install the requirements file.""" - pip_cmd = 'pip install -q --disable-pip-version-check --exists-action w' - sh(f"{pip_cmd} -r {req_file}") - - -@task -@timed -def install_node_prereqs(): - """ - Installs Node prerequisites - """ - if no_prereq_install(): - print(NO_PREREQ_MESSAGE) - return - - prereq_cache("Node prereqs", ["package.json", "package-lock.json"], node_prereqs_installation) - - -# To add a package to the uninstall list, just add it to this list! No need -# to touch any other part of this file. -PACKAGES_TO_UNINSTALL = [ - "MySQL-python", # Because mysqlclient shares the same directory name - "South", # Because it interferes with Django 1.8 migrations. - "edxval", # Because it was bork-installed somehow. - "django-storages", - "django-oauth2-provider", # Because now it's called edx-django-oauth2-provider. - "edx-oauth2-provider", # Because it moved from github to pypi - "enum34", # Because enum34 is not needed in python>3.4 - "i18n-tools", # Because now it's called edx-i18n-tools - "moto", # Because we no longer use it and it conflicts with recent jsondiff versions - "python-saml", # Because python3-saml shares the same directory name - "pytest-faulthandler", # Because it was bundled into pytest - "djangorestframework-jwt", # Because now its called drf-jwt. -] - - -@task -@timed -def uninstall_python_packages(): - """ - Uninstall Python packages that need explicit uninstallation. - - Some Python packages that we no longer want need to be explicitly - uninstalled, notably, South. Some other packages were once installed in - ways that were resistant to being upgraded, like edxval. Also uninstall - them. - """ - - if no_python_uninstall(): - print(NO_PYTHON_UNINSTALL_MESSAGE) - return - - # So that we don't constantly uninstall things, use a hash of the packages - # to be uninstalled. Check it, and skip this if we're up to date. - hasher = hashlib.sha1() - hasher.update(repr(PACKAGES_TO_UNINSTALL).encode('utf-8')) - expected_version = hasher.hexdigest() - state_file_path = os.path.join(PREREQS_STATE_DIR, "Python_uninstall.sha1") - create_prereqs_cache_dir() - - if os.path.isfile(state_file_path): - with open(state_file_path) as state_file: - version = state_file.read() - if version == expected_version: - print('Python uninstalls unchanged, skipping...') - return - - # Run pip to find the packages we need to get rid of. Believe it or not, - # edx-val is installed in a way that it is present twice, so we have a loop - # to really really get rid of it. - for _ in range(3): - uninstalled = False - frozen = sh("pip freeze", capture=True) - - for package_name in PACKAGES_TO_UNINSTALL: - if package_in_frozen(package_name, frozen): - # Uninstall the pacakge - sh(f"pip uninstall --disable-pip-version-check -y {package_name}") - uninstalled = True - if not uninstalled: - break - else: - # We tried three times and didn't manage to get rid of the pests. - print("Couldn't uninstall unwanted Python packages!") - return - - # Write our version. - with open(state_file_path, "wb") as state_file: - state_file.write(expected_version.encode('utf-8')) - - -def package_in_frozen(package_name, frozen_output): - """Is this package in the output of 'pip freeze'?""" - # Look for either: - # - # PACKAGE-NAME== - # - # or: - # - # blah_blah#egg=package_name-version - # - pattern = r"(?mi)^{pkg}==|#egg={pkg_under}-".format( - pkg=re.escape(package_name), - pkg_under=re.escape(package_name.replace("-", "_")), - ) - return bool(re.search(pattern, frozen_output)) - - -@task -@timed -def install_coverage_prereqs(): - """ Install python prereqs for measuring coverage. """ - if no_prereq_install(): - print(NO_PREREQ_MESSAGE) - return - pip_install_req_file(COVERAGE_REQ_FILE) - - -@task -@timed -def install_python_prereqs(): - """ - Installs Python prerequisites. - """ - if no_prereq_install(): - print(NO_PREREQ_MESSAGE) - return - - uninstall_python_packages() - - # Include all of the requirements files in the fingerprint. - files_to_fingerprint = list(PYTHON_REQ_FILES) - - # Also fingerprint the directories where packages get installed: - # ("/edx/app/edxapp/venvs/edxapp/lib/python2.7/site-packages") - files_to_fingerprint.append(sysconfig.get_python_lib()) - - # In a virtualenv, "-e installs" get put in a src directory. - if Env.PIP_SRC: - src_dir = Env.PIP_SRC - else: - src_dir = os.path.join(sys.prefix, "src") - if os.path.isdir(src_dir): - files_to_fingerprint.append(src_dir) - - # Also fingerprint this source file, so that if the logic for installations - # changes, we will redo the installation. - this_file = __file__ - if this_file.endswith(".pyc"): - this_file = this_file[:-1] # use the .py file instead of the .pyc - files_to_fingerprint.append(this_file) - - prereq_cache("Python prereqs", files_to_fingerprint, python_prereqs_installation) - - -@task -@timed -def install_prereqs(): - """ - Installs Node and Python prerequisites - """ - if no_prereq_install(): - print(NO_PREREQ_MESSAGE) - return - - if not str2bool(os.environ.get('SKIP_NPM_INSTALL', 'False')): - install_node_prereqs() - install_python_prereqs() - log_installed_python_prereqs() - - -def log_installed_python_prereqs(): - """ Logs output of pip freeze for debugging. """ - sh("pip freeze > {}".format(Env.GEN_LOG_DIR + "/pip_freeze.log")) diff --git a/pavelib/utils/__init__.py b/pavelib/utils/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/pavelib/utils/cmd.py b/pavelib/utils/cmd.py deleted file mode 100644 index a350c90a6a96..000000000000 --- a/pavelib/utils/cmd.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -Helper functions for constructing shell commands. -""" - - -def cmd(*args): - """ - Concatenate the arguments into a space-separated shell command. - """ - return " ".join(str(arg) for arg in args if arg) - - -def django_cmd(sys, settings, *args): - """ - Construct a Django management command. - - `sys` is either 'lms' or 'studio'. - `settings` is the Django settings module (such as "dev" or "test") - `args` are concatenated to form the rest of the command. - """ - # Maintain backwards compatibility with manage.py, - # which calls "studio" "cms" - sys = 'cms' if sys == 'studio' else sys - return cmd("python manage.py", sys, f"--settings={settings}", *args) diff --git a/pavelib/utils/envs.py b/pavelib/utils/envs.py deleted file mode 100644 index 7dc9870fbdea..000000000000 --- a/pavelib/utils/envs.py +++ /dev/null @@ -1,271 +0,0 @@ -""" -Helper functions for loading environment settings. -""" -import configparser -import json -import os -import sys -from time import sleep - -from lazy import lazy -from path import Path as path -from paver.easy import BuildFailure, sh - -from pavelib.utils.cmd import django_cmd - - -def repo_root(): - """ - Get the root of the git repository (edx-platform). - - This sometimes fails on Docker Devstack, so it's been broken - down with some additional error handling. It usually starts - working within 30 seconds or so; for more details, see - https://openedx.atlassian.net/browse/PLAT-1629 and - https://github.com/docker/for-mac/issues/1509 - """ - file_path = path(__file__) - attempt = 1 - while True: - try: - absolute_path = file_path.abspath() - break - except OSError: - print(f'Attempt {attempt}/180 to get an absolute path failed') - if attempt < 180: - attempt += 1 - sleep(1) - else: - print('Unable to determine the absolute path of the edx-platform repo, aborting') - raise - return absolute_path.parent.parent.parent - - -class Env: - """ - Load information about the execution environment. - """ - - # Root of the git repository (edx-platform) - REPO_ROOT = repo_root() - - # Reports Directory - REPORT_DIR = REPO_ROOT / 'reports' - METRICS_DIR = REPORT_DIR / 'metrics' - QUALITY_DIR = REPORT_DIR / 'quality_junitxml' - - # Generic log dir - GEN_LOG_DIR = REPO_ROOT / "test_root" / "log" - - # Python unittest dirs - PYTHON_COVERAGERC = REPO_ROOT / ".coveragerc" - - # Which Python version should be used in xdist workers? - PYTHON_VERSION = os.environ.get("PYTHON_VERSION", "2.7") - - # Directory that videos are served from - VIDEO_SOURCE_DIR = REPO_ROOT / "test_root" / "data" / "video" - - PRINT_SETTINGS_LOG_FILE = GEN_LOG_DIR / "print_settings.log" - - # Detect if in a Docker container, and if so which one - FRONTEND_TEST_SERVER_HOST = os.environ.get('FRONTEND_TEST_SERVER_HOSTNAME', '0.0.0.0') - USING_DOCKER = FRONTEND_TEST_SERVER_HOST != '0.0.0.0' - DEVSTACK_SETTINGS = 'devstack_docker' if USING_DOCKER else 'devstack' - TEST_SETTINGS = 'test' - - # Mongo databases that will be dropped before/after the tests run - MONGO_HOST = 'localhost' - - # Test Ids Directory - TEST_DIR = REPO_ROOT / ".testids" - - # Configured browser to use for the js test suites - SELENIUM_BROWSER = os.environ.get('SELENIUM_BROWSER', 'firefox') - if USING_DOCKER: - KARMA_BROWSER = 'ChromeDocker' if SELENIUM_BROWSER == 'chrome' else 'FirefoxDocker' - else: - KARMA_BROWSER = 'FirefoxNoUpdates' - - # Files used to run each of the js test suites - # TODO: We have [temporarily disabled] the three Webpack-based tests suites. They have been silently - # broken for a long time; after noticing they were broken, we added the DieHardPlugin to - # webpack.common.config.js to prevent future silent breakage, but have not yet been able to - # fix and re-enable the suites. Note that the LMS suite is all Webpack-based even though it's - # not in the name. - # Issue: https://github.com/openedx/edx-platform/issues/35956 - KARMA_CONFIG_FILES = [ - REPO_ROOT / 'cms/static/karma_cms.conf.js', - REPO_ROOT / 'cms/static/karma_cms_squire.conf.js', - ## [temporarily disabled] REPO_ROOT / 'cms/static/karma_cms_webpack.conf.js', - ## [temporarily disabled] REPO_ROOT / 'lms/static/karma_lms.conf.js', - REPO_ROOT / 'xmodule/js/karma_xmodule.conf.js', - ## [temporarily disabled] REPO_ROOT / 'xmodule/js/karma_xmodule_webpack.conf.js', - REPO_ROOT / 'common/static/karma_common.conf.js', - REPO_ROOT / 'common/static/karma_common_requirejs.conf.js', - ] - - JS_TEST_ID_KEYS = [ - 'cms', - 'cms-squire', - ## [temporarily-disabled] 'cms-webpack', - ## [temporarily-disabled] 'lms', - 'xmodule', - ## [temporarily-disabled] 'xmodule-webpack', - 'common', - 'common-requirejs', - 'jest-snapshot' - ] - - JS_REPORT_DIR = REPORT_DIR / 'javascript' - - # Directories used for pavelib/ tests - IGNORED_TEST_DIRS = ('__pycache__', '.cache', '.pytest_cache') - LIB_TEST_DIRS = [path("pavelib/paver_tests"), path("scripts/xsslint/tests")] - - # Directory for i18n test reports - I18N_REPORT_DIR = REPORT_DIR / 'i18n' - - # Directory for keeping src folder that comes with pip installation. - # Setting this is equivalent to passing `--src ` to pip directly. - PIP_SRC = os.environ.get("PIP_SRC") - - # Service variant (lms, cms, etc.) configured with an environment variable - # We use this to determine which envs.json file to load. - SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', None) - - # If service variant not configured in env, then pass the correct - # environment for lms / cms - if not SERVICE_VARIANT: # this will intentionally catch ""; - if any(i in sys.argv[1:] for i in ('cms', 'studio')): - SERVICE_VARIANT = 'cms' - else: - SERVICE_VARIANT = 'lms' - - @classmethod - def get_django_settings(cls, django_settings, system, settings=None, print_setting_args=None): - """ - Interrogate Django environment for specific settings values - :param django_settings: list of django settings values to get - :param system: the django app to use when asking for the setting (lms | cms) - :param settings: the settings file to use when asking for the value - :param print_setting_args: the additional arguments to send to print_settings - :return: unicode value of the django setting - """ - if not settings: - settings = os.environ.get("EDX_PLATFORM_SETTINGS", "aws") - log_dir = os.path.dirname(cls.PRINT_SETTINGS_LOG_FILE) - if not os.path.exists(log_dir): - os.makedirs(log_dir) - settings_length = len(django_settings) - django_settings = ' '.join(django_settings) # parse_known_args makes a list again - print_setting_args = ' '.join(print_setting_args or []) - try: - value = sh( - django_cmd( - system, - settings, - "print_setting {django_settings} 2>{log_file} {print_setting_args}".format( - django_settings=django_settings, - print_setting_args=print_setting_args, - log_file=cls.PRINT_SETTINGS_LOG_FILE - ).strip() - ), - capture=True - ) - # else for cases where values are not found & sh returns one None value - return tuple(str(value).splitlines()) if value else tuple(None for _ in range(settings_length)) - except BuildFailure: - print(f"Unable to print the value of the {django_settings} setting:") - with open(cls.PRINT_SETTINGS_LOG_FILE) as f: - print(f.read()) - sys.exit(1) - - @classmethod - def get_django_json_settings(cls, django_settings, system, settings=None): - """ - Interrogate Django environment for specific settings value - :param django_settings: list of django settings values to get - :param system: the django app to use when asking for the setting (lms | cms) - :param settings: the settings file to use when asking for the value - :return: json string value of the django setting - """ - return cls.get_django_settings( - django_settings, - system, - settings=settings, - print_setting_args=["--json"], - ) - - @classmethod - def covered_modules(cls): - """ - List the source modules listed in .coveragerc for which coverage - will be measured. - """ - coveragerc = configparser.RawConfigParser() - coveragerc.read(cls.PYTHON_COVERAGERC) - modules = coveragerc.get('run', 'source') - result = [] - for module in modules.split('\n'): - module = module.strip() - if module: - result.append(module) - return result - - @lazy - def env_tokens(self): - """ - Return a dict of environment settings. - If we couldn't find the JSON file, issue a warning and return an empty dict. - """ - - # Find the env JSON file - if self.SERVICE_VARIANT: - env_path = self.REPO_ROOT.parent / f"{self.SERVICE_VARIANT}.env.json" - else: - env_path = path("env.json").abspath() - - # If the file does not exist, here or one level up, - # issue a warning and return an empty dict - if not env_path.isfile(): - env_path = env_path.parent.parent / env_path.basename() - if not env_path.isfile(): - print( - "Warning: could not find environment JSON file " - "at '{path}'".format(path=env_path), - file=sys.stderr, - ) - return {} - - # Otherwise, load the file as JSON and return the resulting dict - try: - with open(env_path) as env_file: - return json.load(env_file) - - except ValueError: - print( - "Error: Could not parse JSON " - "in {path}".format(path=env_path), - file=sys.stderr, - ) - sys.exit(1) - - @lazy - def feature_flags(self): - """ - Return a dictionary of feature flags configured by the environment. - """ - return self.env_tokens.get('FEATURES', {}) - - @classmethod - def rsync_dirs(cls): - """ - List the directories that should be synced during pytest-xdist - execution. Needs to include all modules for which coverage is - measured, not just the tests being run. - """ - result = set() - for module in cls.covered_modules(): - result.add(module.split('/')[0]) - return result diff --git a/pavelib/utils/process.py b/pavelib/utils/process.py deleted file mode 100644 index da2dafa8803d..000000000000 --- a/pavelib/utils/process.py +++ /dev/null @@ -1,121 +0,0 @@ -""" -Helper functions for managing processes. -""" - - -import atexit -import os -import signal -import subprocess -import sys - -import psutil -from paver import tasks - - -def kill_process(proc): - """ - Kill the process `proc` created with `subprocess`. - """ - p1_group = psutil.Process(proc.pid) - child_pids = p1_group.children(recursive=True) - - for child_pid in child_pids: - os.kill(child_pid.pid, signal.SIGKILL) - - -def run_multi_processes(cmd_list, out_log=None, err_log=None): - """ - Run each shell command in `cmd_list` in a separate process, - piping stdout to `out_log` (a path) and stderr to `err_log` (also a path). - - Terminates the processes on CTRL-C and ensures the processes are killed - if an error occurs. - """ - kwargs = {'shell': True, 'cwd': None} - pids = [] - - if out_log: - out_log_file = open(out_log, 'w') # lint-amnesty, pylint: disable=consider-using-with - kwargs['stdout'] = out_log_file - - if err_log: - err_log_file = open(err_log, 'w') # lint-amnesty, pylint: disable=consider-using-with - kwargs['stderr'] = err_log_file - - # If the user is performing a dry run of a task, then just log - # the command strings and return so that no destructive operations - # are performed. - if tasks.environment.dry_run: - for cmd in cmd_list: - tasks.environment.info(cmd) - return - - try: - for cmd in cmd_list: - pids.extend([subprocess.Popen(cmd, **kwargs)]) - - # pylint: disable=unused-argument - def _signal_handler(*args): - """ - What to do when process is ended - """ - print("\nEnding...") - - signal.signal(signal.SIGINT, _signal_handler) - print("Enter CTL-C to end") - signal.pause() - print("Processes ending") - - # pylint: disable=broad-except - except Exception as err: - print(f"Error running process {err}", file=sys.stderr) - - finally: - for pid in pids: - kill_process(pid) - - -def run_process(cmd, out_log=None, err_log=None): - """ - Run the shell command `cmd` in a separate process, - piping stdout to `out_log` (a path) and stderr to `err_log` (also a path). - - Terminates the process on CTRL-C or if an error occurs. - """ - return run_multi_processes([cmd], out_log=out_log, err_log=err_log) - - -def run_background_process(cmd, out_log=None, err_log=None, cwd=None): - """ - Runs a command as a background process. Sends SIGINT at exit. - """ - - kwargs = {'shell': True, 'cwd': cwd} - if out_log: - out_log_file = open(out_log, 'w') # lint-amnesty, pylint: disable=consider-using-with - kwargs['stdout'] = out_log_file - - if err_log: - err_log_file = open(err_log, 'w') # lint-amnesty, pylint: disable=consider-using-with - kwargs['stderr'] = err_log_file - - proc = subprocess.Popen(cmd, **kwargs) # lint-amnesty, pylint: disable=consider-using-with - - def exit_handler(): - """ - Send SIGINT to the process's children. This is important - for running commands under coverage, as coverage will not - produce the correct artifacts if the child process isn't - killed properly. - """ - p1_group = psutil.Process(proc.pid) - child_pids = p1_group.children(recursive=True) - - for child_pid in child_pids: - os.kill(child_pid.pid, signal.SIGINT) - - # Wait for process to actually finish - proc.wait() - - atexit.register(exit_handler) diff --git a/pavelib/utils/test/__init__.py b/pavelib/utils/test/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/pavelib/utils/timer.py b/pavelib/utils/timer.py deleted file mode 100644 index fc6f3003736a..000000000000 --- a/pavelib/utils/timer.py +++ /dev/null @@ -1,83 +0,0 @@ -""" -Tools for timing paver tasks -""" - - -import json -import logging -import os -import sys -import traceback -from datetime import datetime -from os.path import dirname, exists - -import wrapt - -LOGGER = logging.getLogger(__file__) -PAVER_TIMER_LOG = os.environ.get('PAVER_TIMER_LOG') - - -@wrapt.decorator -def timed(wrapped, instance, args, kwargs): # pylint: disable=unused-argument - """ - Log execution time for a function to a log file. - - Logging is only actually executed if the PAVER_TIMER_LOG environment variable - is set. That variable is expanded for the current user and current - environment variables. It also can have :meth:`~Datetime.strftime` format - identifiers which are substituted using the time when the task started. - - For example, ``PAVER_TIMER_LOG='~/.paver.logs/%Y-%d-%m.log'`` will create a new - log file every day containing reconds for paver tasks run that day, and - will put those log files in the ``.paver.logs`` directory inside the users - home. - - Must be earlier in the decorator stack than the paver task declaration. - """ - start = datetime.utcnow() - exception_info = {} - try: - return wrapped(*args, **kwargs) - except Exception as exc: - exception_info = { - 'exception': "".join(traceback.format_exception_only(type(exc), exc)).strip() - } - raise - finally: - end = datetime.utcnow() - - # N.B. This is intended to provide a consistent interface and message format - # across all of Open edX tooling, so it deliberately eschews standard - # python logging infrastructure. - if PAVER_TIMER_LOG is not None: - - log_path = start.strftime(PAVER_TIMER_LOG) - - log_message = { - 'python_version': sys.version, - 'task': f"{wrapped.__module__}.{wrapped.__name__}", - 'args': [repr(arg) for arg in args], - 'kwargs': {key: repr(value) for key, value in kwargs.items()}, - 'started_at': start.isoformat(' '), - 'ended_at': end.isoformat(' '), - 'duration': (end - start).total_seconds(), - } - log_message.update(exception_info) - - try: - log_dir = dirname(log_path) - if log_dir and not exists(log_dir): - os.makedirs(log_dir) - - with open(log_path, 'a') as outfile: - json.dump( - log_message, - outfile, - separators=(',', ':'), - sort_keys=True, - ) - outfile.write('\n') - except OSError: - # Squelch OSErrors, because we expect them and they shouldn't - # interrupt the rest of the process. - LOGGER.exception("Unable to write timing logs") diff --git a/requirements/edx/bundled.in b/requirements/edx/bundled.in index a9394b809f55..61f6007aa507 100644 --- a/requirements/edx/bundled.in +++ b/requirements/edx/bundled.in @@ -25,7 +25,6 @@ # Follow up issue to remove this fork: https://github.com/openedx/edx-platform/issues/33456 https://github.com/overhangio/py2neo/releases/download/2021.2.3/py2neo-2021.2.3.tar.gz -pygments # Used to support colors in paver command output # i18n_tool is needed at build time for pulling translations edx-i18n-tools>=0.4.6 # Commands for developers and translators to extract, compile and validate translations diff --git a/requirements/edx/kernel.in b/requirements/edx/kernel.in index 7323c243accf..62e8e43345c6 100644 --- a/requirements/edx/kernel.in +++ b/requirements/edx/kernel.in @@ -3,7 +3,6 @@ -c ../constraints.txt -r github.in # Forks and other dependencies not yet on PyPI --r paver.txt # Requirements for running paver commands # DON'T JUST ADD NEW DEPENDENCIES!!! # Please follow these guidelines whenever you change this file: diff --git a/requirements/edx/paver.in b/requirements/edx/paver.in deleted file mode 100644 index 6987ede82275..000000000000 --- a/requirements/edx/paver.in +++ /dev/null @@ -1,27 +0,0 @@ -# Requirements to run and test Paver -# -# DON'T JUST ADD NEW DEPENDENCIES!!! -# -# If you open a pull request that adds a new dependency, you should: -# * verify that the dependency has a license compatible with AGPLv3 -# * confirm that it has no system requirements beyond what we already install -# * run "make upgrade" to update the detailed requirements files -# - --c ../constraints.txt - -edx-opaque-keys # Create and introspect course and xblock identities -lazy # Lazily-evaluated attributes for Python objects -libsass # Python bindings for the LibSass CSS compiler -markupsafe # XML/HTML/XHTML Markup safe strings -mock # Stub out code with mock objects and make assertions about how they have been used -path # Easier manipulation of filesystem paths -paver # Build, distribution and deployment scripting tool -psutil # Library for retrieving information on running processes and system utilization -pymongo # via edx-opaque-keys -python-memcached # Python interface to the memcached memory cache daemon -pymemcache # Python interface to the memcached memory cache daemon -requests # Simple interface for making HTTP requests -stevedore # Support for runtime plugins, used for XBlocks and edx-platform Django app plugins -watchdog # Used in paver watch_assets -wrapt # Decorator utilities used in the @timed paver task decorator diff --git a/requirements/edx/paver.txt b/requirements/edx/paver.txt deleted file mode 100644 index c9ee8f3aff49..000000000000 --- a/requirements/edx/paver.txt +++ /dev/null @@ -1,65 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# make upgrade -# -certifi==2024.8.30 - # via requests -charset-normalizer==2.0.12 - # via - # -c requirements/edx/../constraints.txt - # requests -dnspython==2.7.0 - # via pymongo -edx-opaque-keys==2.11.0 - # via -r requirements/edx/paver.in -idna==3.10 - # via requests -lazy==1.6 - # via -r requirements/edx/paver.in -libsass==0.10.0 - # via - # -c requirements/edx/../constraints.txt - # -r requirements/edx/paver.in -markupsafe==3.0.2 - # via -r requirements/edx/paver.in -mock==5.1.0 - # via -r requirements/edx/paver.in -path==16.11.0 - # via - # -c requirements/edx/../constraints.txt - # -r requirements/edx/paver.in -paver==1.3.4 - # via -r requirements/edx/paver.in -pbr==6.1.0 - # via stevedore -psutil==6.1.0 - # via -r requirements/edx/paver.in -pymemcache==4.0.0 - # via -r requirements/edx/paver.in -pymongo==4.4.0 - # via - # -c requirements/edx/../constraints.txt - # -r requirements/edx/paver.in - # edx-opaque-keys -python-memcached==1.62 - # via -r requirements/edx/paver.in -requests==2.32.3 - # via -r requirements/edx/paver.in -six==1.17.0 - # via - # libsass - # paver -stevedore==5.4.0 - # via - # -r requirements/edx/paver.in - # edx-opaque-keys -typing-extensions==4.12.2 - # via edx-opaque-keys -urllib3==2.2.3 - # via requests -watchdog==6.0.0 - # via -r requirements/edx/paver.in -wrapt==1.17.0 - # via -r requirements/edx/paver.in