diff --git a/.github/workflows/pylint-checks.yml b/.github/workflows/pylint-checks.yml index 9a654e09e711..1afa032b6ad1 100644 --- a/.github/workflows/pylint-checks.yml +++ b/.github/workflows/pylint-checks.yml @@ -22,7 +22,7 @@ jobs: - module-name: openedx-2 path: "openedx/core/djangoapps/geoinfo/ openedx/core/djangoapps/header_control/ openedx/core/djangoapps/heartbeat/ openedx/core/djangoapps/lang_pref/ openedx/core/djangoapps/models/ openedx/core/djangoapps/monkey_patch/ openedx/core/djangoapps/oauth_dispatch/ openedx/core/djangoapps/olx_rest_api/ openedx/core/djangoapps/password_policy/ openedx/core/djangoapps/plugin_api/ openedx/core/djangoapps/plugins/ openedx/core/djangoapps/profile_images/ openedx/core/djangoapps/programs/ openedx/core/djangoapps/safe_sessions/ openedx/core/djangoapps/schedules/ openedx/core/djangoapps/service_status/ openedx/core/djangoapps/session_inactivity_timeout/ openedx/core/djangoapps/signals/ openedx/core/djangoapps/site_configuration/ openedx/core/djangoapps/system_wide_roles/ openedx/core/djangoapps/theming/ openedx/core/djangoapps/user_api/ openedx/core/djangoapps/user_authn/ openedx/core/djangoapps/util/ openedx/core/djangoapps/verified_track_content/ openedx/core/djangoapps/video_config/ openedx/core/djangoapps/video_pipeline/ openedx/core/djangoapps/waffle_utils/ openedx/core/djangoapps/xblock/ openedx/core/djangoapps/xmodule_django/ openedx/core/tests/ openedx/features/ openedx/testing/ openedx/tests/ openedx/core/djangoapps/notifications/ openedx/core/djangoapps/staticfiles/ openedx/core/djangoapps/content_tagging/" - module-name: common - path: "common pavelib" + path: "common" - module-name: cms path: "cms" - module-name: xmodule diff --git a/.github/workflows/unit-test-shards.json b/.github/workflows/unit-test-shards.json index 1af8af814254..4709930493ce 100644 --- a/.github/workflows/unit-test-shards.json +++ b/.github/workflows/unit-test-shards.json @@ -255,15 +255,13 @@ "common-with-lms": { "settings": "lms.envs.test", "paths": [ - "common/djangoapps/", - "pavelib/" + "common/djangoapps/" ] }, "common-with-cms": { "settings": "cms.envs.test", "paths": [ - "common/djangoapps/", - "pavelib/" + "common/djangoapps/" ] }, "xmodule-with-lms": { diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index e691e16e47f1..063a8779941e 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -180,7 +180,7 @@ jobs: shell: bash run: | echo "root_cms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=cms.envs.test cms/ -q | head -n -2 | wc -l)" >> $GITHUB_ENV - echo "root_lms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=lms.envs.test lms/ openedx/ common/djangoapps/ xmodule/ pavelib/ -q | head -n -2 | wc -l)" >> $GITHUB_ENV + echo "root_lms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=lms.envs.test lms/ openedx/ common/djangoapps/ xmodule/ -q | head -n -2 | wc -l)" >> $GITHUB_ENV - name: get GHA unit test paths shell: bash diff --git a/Makefile b/Makefile index 62681f6f3711..717409d4f9e0 100644 --- a/Makefile +++ b/Makefile @@ -104,7 +104,6 @@ shell: ## launch a bash shell in a Docker container with all edx-platform depend # Order is very important in this list: files must appear after everything they include! REQ_FILES = \ requirements/edx/coverage \ - requirements/edx/paver \ requirements/edx-sandbox/base \ requirements/edx/base \ requirements/edx/doc \ @@ -211,7 +210,7 @@ xsslint: ## check xss for quality issuest --config=scripts.xsslint_config \ --thresholds=scripts/xsslint_thresholds.json -pycodestyle: ## check python files for quality issues +pycodestyle: ## check python files for quality issues pycodestyle . ## Re-enable --lint flag when this issue https://github.com/openedx/edx-platform/issues/35775 is resolved @@ -222,13 +221,13 @@ pii_check: ## check django models for pii annotations --app_name cms \ --coverage \ --lint - + DJANGO_SETTINGS_MODULE=lms.envs.test \ code_annotations django_find_annotations \ --config_file .pii_annotations.yml \ --app_name lms \ --coverage \ - --lint + --lint check_keywords: ## check django models for reserve keywords DJANGO_SETTINGS_MODULE=cms.envs.test \ diff --git a/docs/concepts/testing/testing.rst b/docs/concepts/testing/testing.rst index 9d448afd5bdc..973d56f2602e 100644 --- a/docs/concepts/testing/testing.rst +++ b/docs/concepts/testing/testing.rst @@ -1,4 +1,3 @@ -####### Testing ####### @@ -7,7 +6,7 @@ Testing :depth: 3 Overview -======== +******** We maintain two kinds of tests: unit tests and integration tests. @@ -26,10 +25,10 @@ tests. Most of our tests are unit tests or integration tests. Test Types ----------- +========== Unit Tests -~~~~~~~~~~ +---------- - Each test case should be concise: setup, execute, check, and teardown. If you find yourself writing tests with many steps, @@ -38,18 +37,18 @@ Unit Tests - As a rule of thumb, your unit tests should cover every code branch. -- Mock or patch external dependencies. We use the voidspace `Mock Library`_. +- Mock or patch external dependencies using `unittest.mock`_ functions. - We unit test Python code (using `unittest`_) and Javascript (using `Jasmine`_) -.. _Mock Library: http://www.voidspace.org.uk/python/mock/ +.. _unittest.mock: https://docs.python.org/3/library/unittest.mock.html .. _unittest: http://docs.python.org/2/library/unittest.html .. _Jasmine: http://jasmine.github.io/ Integration Tests -~~~~~~~~~~~~~~~~~ +----------------- - Test several units at the same time. Note that you can still mock or patch dependencies that are not under test! For example, you might test that @@ -67,7 +66,7 @@ Integration Tests .. _Django test client: https://docs.djangoproject.com/en/dev/topics/testing/overview/ Test Locations --------------- +============== - Python unit and integration tests: Located in subpackages called ``tests``. For example, the tests for the ``capa`` package are @@ -80,14 +79,29 @@ Test Locations the test for ``src/views/module.js`` should be written in ``spec/views/module_spec.js``. -Running Tests -============= +Factories +========= -**Unless otherwise mentioned, all the following commands should be run from inside the lms docker container.** +Many tests delegate set-up to a "factory" class. For example, there are +factories for creating courses, problems, and users. This encapsulates +set-up logic from tests. +Factories are often implemented using `FactoryBoy`_. + +In general, factories should be located close to the code they use. For +example, the factory for creating problem XML definitions is located in +``xmodule/capa/tests/response_xml_factory.py`` because the +``capa`` package handles problem XML. + +.. _FactoryBoy: https://readthedocs.org/projects/factoryboy/ Running Python Unit tests -------------------------- +************************* + +The following commands need to be run within a Python environment in +which requirements/edx/testing.txt has been installed. If you are using a +Docker-based Open edX distribution, then you probably will want to run these +commands within the LMS and/or CMS Docker containers. We use `pytest`_ to run Python tests. Pytest is a testing framework for python and should be your goto for local Python unit testing. @@ -97,16 +111,16 @@ Pytest (and all of the plugins we use with it) has a lot of options. Use `pytest Running Python Test Subsets -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=========================== When developing tests, it is often helpful to be able to really just run one single test without the overhead of PIP installs, UX builds, etc. Various ways to run tests using pytest:: - pytest path/test_m­odule.py # Run all tests in a module. - pytest path/test_m­odule.p­y:­:te­st_func # Run a specific test within a module. - pytest path/test_m­odule.p­y:­:Te­stC­las­s # Run all tests in a class - pytest path/test_m­odule.p­y:­:Te­stC­las­s::­tes­t_m­ethod # Run a specific method of a class. + pytest path/test_module.py # Run all tests in a module. + pytest path/test_module.py::test_func # Run a specific test within a module. + pytest path/test_module.py::TestClass # Run all tests in a class + pytest path/test_module.py::TestClass::test_method # Run a specific method of a class. pytest path/testing/ # Run all tests in a directory. For example, this command runs a single python unit test file:: @@ -114,7 +128,7 @@ For example, this command runs a single python unit test file:: pytest xmodule/tests/test_stringify.py Note - -edx-platorm has multiple services (lms, cms) in it. The environment for each service is different enough that we run some tests in both environments in Github Actions. +edx-platorm has multiple services (lms, cms) in it. The environment for each service is different enough that we run some tests in both environments in Github Actions. To test in each of these environments (especially for tests in "common" and "xmodule" directories), you will need to test in each seperately. To specify that the tests are run with the relevant service as root, Add --rootdir flag at end of your pytest call and specify the env to test in:: @@ -139,7 +153,7 @@ Various tools like ddt create tests with very complex names, rather than figurin pytest xmodule/tests/test_stringify.py --collectonly Testing with migrations -*********************** +----------------------- For the sake of speed, by default the python unit test database tables are created directly from apps' models. If you want to run the tests @@ -149,7 +163,7 @@ against a database created by applying the migrations instead, use the pytest test --create-db --migrations Debugging a test -~~~~~~~~~~~~~~~~ +---------------- There are various ways to debug tests in Python and more specifically with pytest: @@ -173,7 +187,7 @@ There are various ways to debug tests in Python and more specifically with pytes How to output coverage locally -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +============================== These are examples of how to run a single test and get coverage:: @@ -221,233 +235,82 @@ run one of these commands:: Debugging Unittest Flakiness -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -As we move over to running our unittests with Jenkins Pipelines and pytest-xdist, -there are new ways for tests to flake, which can sometimes be difficult to debug. -If you run into flakiness, check (and feel free to contribute to) this -`confluence document `__ for help. - -Running Javascript Unit Tests ------------------------------ - -Before running Javascript unit tests, you will need to be running Firefox or Chrome in a place visible to edx-platform. If running this in devstack, you can run ``make dev.up.firefox`` or ``make dev.up.chrome``. Firefox is the default browser for the tests, so if you decide to use Chrome, you will need to prefix the test command with ``SELENIUM_BROWSER=chrome SELENIUM_HOST=edx.devstack.chrome`` (if using devstack). - -We use Jasmine to run JavaScript unit tests. To run all the JavaScript -tests:: - - paver test_js - -To run a specific set of JavaScript tests and print the results to the -console, run these commands:: - - paver test_js_run -s lms - paver test_js_run -s cms - paver test_js_run -s cms-squire - paver test_js_run -s xmodule - paver test_js_run -s xmodule-webpack - paver test_js_run -s common - paver test_js_run -s common-requirejs - -To run JavaScript tests in a browser, run these commands:: - - paver test_js_dev -s lms - paver test_js_dev -s cms - paver test_js_dev -s cms-squire - paver test_js_dev -s xmodule - paver test_js_dev -s xmodule-webpack - paver test_js_dev -s common - paver test_js_dev -s common-requirejs - -Debugging Specific Javascript Tests -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The best way to debug individual tests is to run the test suite in the browser and -use your browser's Javascript debugger. The debug page will allow you to select -an individual test and only view the results of that test. - - -Debugging Tests in a Browser -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -To debug these tests on devstack in a local browser: - -* first run the appropriate test_js_dev command from above -* open http://localhost:19876/debug.html in your host system's browser of choice -* this will run all the tests and show you the results including details of any failures -* you can click on an individually failing test and/or suite to re-run it by itself -* you can now use the browser's developer tools to debug as you would any other JavaScript code - -Note: the port is also output to the console that you ran the tests from if you find that easier. - -These paver commands call through to Karma. For more -info, see `karma-runner.github.io `__. - -Testing internationalization with dummy translations ----------------------------------------------------- - -Any text you add to the platform should be internationalized. To generate translations for your new strings, run the following command:: - - paver i18n_dummy - -This command generates dummy translations for each dummy language in the -platform and puts the dummy strings in the appropriate language files. -You can then preview the dummy languages on your local machine and also in your sandbox, if and when you create one. - -The dummy language files that are generated during this process can be -found in the following locations:: - - conf/locale/{LANG_CODE} - -There are a few JavaScript files that are generated from this process. You can find those in the following locations:: - - lms/static/js/i18n/{LANG_CODE} - cms/static/js/i18n/{LANG_CODE} - -Do not commit the ``.po``, ``.mo``, ``.js`` files that are generated -in the above locations during the dummy translation process! +============================ -Test Coverage and Quality -------------------------- +See this `confluence document `_. -Viewing Test Coverage -~~~~~~~~~~~~~~~~~~~~~ +Running JavaScript Unit Tests +***************************** -We currently collect test coverage information for Python -unit/integration tests. +Before running Javascript unit tests, you will need to be running Firefox or Chrome in a place visible to edx-platform. +If you are using Tutor Dev to run edx-platform, then you can do so by installing and enabling the +``test-legacy-js`` plugin from `openedx-tutor-plugins`_, and then rebuilding +the ``openedx-dev`` image:: -To view test coverage: + tutor plugins install https://github.com/openedx/openedx-tutor-plugins/tree/main/plugins/tutor-contrib-test-legacy-js + tutor plugins enable test-legacy-js + tutor images build openedx-dev -1. Run the test suite with this command:: +.. _openedx-tutor-plugins: https://github.com/openedx/openedx-tutor-plugins/ - paver test +We use Jasmine (via Karma) to run most JavaScript unit tests. We use Jest to +run a small handful of additional JS unit tests. You can use the ``npm run +test*`` commands to run them:: -2. Generate reports with this command:: + npm run test-karma # Run all Jasmine+Karma tests. + npm run test-jest # Run all Jest tests. + npm run test # Run both of the above. - paver coverage +The Karma tests are further broken down into three types depending on how the +JavaScript it is testing is built:: -3. Reports are located in the ``reports`` folder. The command generates - HTML and XML (Cobertura format) reports. + npm run test-karma-vanilla # Our very oldest JS, which doesn't even use RequireJS + npm run test-karma-require # Old JS that uses RequireJS + npm run test-karma-webpack # Slightly "newer" JS which is built with Webpack -Python Code Style Quality -~~~~~~~~~~~~~~~~~~~~~~~~~ +Unfortunately, at the time of writing, the build for the ``test-karma-webpack`` +tests is broken. The tests are excluded from ``npm run test-karma`` as to not +fail CI. We `may fix this one day`_. -To view Python code style quality (including PEP 8 and pylint violations) run this command:: +.. _may fix this one day: https://github.com/openedx/edx-platform/issues/35956 - paver run_quality +To run all Karma+Jasmine tests for a particular top-level edx-platform folder, +you can run:: -More specific options are below. + npm run test-cms + npm run test-lms + npm run test-xmodule + npm run test-common -- These commands run a particular quality report:: - - paver run_pep8 - paver run_pylint - -- This command runs a report, and sets it to fail if it exceeds a given number - of violations:: - - paver run_pep8 --limit=800 - -- The ``run_quality`` uses the underlying diff-quality tool (which is packaged - with `diff-cover`_). With that, the command can be set to fail if a certain - diff threshold is not met. For example, to cause the process to fail if - quality expectations are less than 100% when compared to master (or in other - words, if style quality is worse than what is already on master):: - - paver run_quality --percentage=100 - -- Note that 'fixme' violations are not counted with run\_quality. To - see all 'TODO' lines, use this command:: - - paver find_fixme --system=lms - - ``system`` is an optional argument here. It defaults to - ``cms,lms,common``. - -.. _diff-cover: https://github.com/Bachmann1234/diff-cover - - -JavaScript Code Style Quality -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -To view JavaScript code style quality run this command:: - - paver run_eslint - -- This command also comes with a ``--limit`` switch, this is an example of that switch:: - - paver run_eslint --limit=50000 - - -Code Complexity Tools -===================== - -Tool(s) available for evaluating complexity of edx-platform code: - - -- `plato `__ for JavaScript code - complexity. Several options are available on the command line; see - documentation. Below, the following command will produce an HTML report in a - subdirectory called "jscomplexity":: - - plato -q -x common/static/js/vendor/ -t common -e .eslintrc.json -r -d jscomplexity common/static/js/ - -Other Testing Tips -================== - -Connecting to Browser ---------------------- - -If you want to see the browser being automated for JavaScript, -you can connect to the container running it via VNC. - -+------------------------+----------------------+ -| Browser | VNC connection | -+========================+======================+ -| Firefox (Default) | vnc://0.0.0.0:25900 | -+------------------------+----------------------+ -| Chrome (via Selenium) | vnc://0.0.0.0:15900 | -+------------------------+----------------------+ - -On macOS, enter the VNC connection string in Safari to connect via VNC. The VNC -passwords for both browsers are randomly generated and logged at container -startup, and can be found by running ``make vnc-passwords``. - -Most tests are run in Firefox by default. To use Chrome for tests that normally -use Firefox instead, prefix the test command with -``SELENIUM_BROWSER=chrome SELENIUM_HOST=edx.devstack.chrome`` - -Factories ---------- - -Many tests delegate set-up to a "factory" class. For example, there are -factories for creating courses, problems, and users. This encapsulates -set-up logic from tests. - -Factories are often implemented using `FactoryBoy`_. - -In general, factories should be located close to the code they use. For -example, the factory for creating problem XML definitions is located in -``xmodule/capa/tests/response_xml_factory.py`` because the -``capa`` package handles problem XML. - -.. _FactoryBoy: https://readthedocs.org/projects/factoryboy/ +Finally, if you want to pass any options to the underlying ``node`` invocation +for Karma+Jasmine tests, you can run one of these specific commands, and put +your arguments after the ``--`` separator:: -Running Tests on Paver Scripts ------------------------------- + npm run test-cms-vanilla -- --your --args --here + npm run test-cms-require -- --your --args --here + npm run test-cms-webpack -- --your --args --here + npm run test-lms-webpack -- --your --args --here + npm run test-xmodule-vanilla -- --your --args --here + npm run test-xmodule-webpack -- --your --args --here + npm run test-common-vanilla -- --your --args --here + npm run test-common-require -- --your --args --here -To run tests on the scripts that power the various Paver commands, use the following command:: - pytest pavelib +Code Quality +************ -Testing using queue servers ---------------------------- +We use several tools to analyze code quality. The full set of them is:: -When testing problems that use a queue server on AWS (e.g. -sandbox-xqueue.edx.org), you'll need to run your server on your public IP, like so:: + mypy $PATHS... + pycodestyle $PATHS... + pylint $PATHS... + lint-imports + scripts/verify-dunder-init.sh + make xsslint + make pii_check + make check_keywords + npm run lint - ./manage.py lms runserver 0.0.0.0:8000 +Where ``$PATHS...`` is a list of folders and files to analyze, or nothing if +you would like to analyze the entire codebase (which can take a while). -When you connect to the LMS, you need to use the public ip. Use -``ifconfig`` to figure out the number, and connect e.g. to -``http://18.3.4.5:8000/`` diff --git a/openedx/core/djangoapps/theming/management/commands/compile_sass.py b/openedx/core/djangoapps/theming/management/commands/compile_sass.py deleted file mode 100644 index fbfdd2f222a4..000000000000 --- a/openedx/core/djangoapps/theming/management/commands/compile_sass.py +++ /dev/null @@ -1,112 +0,0 @@ -""" -Management command for compiling sass. - -DEPRECATED in favor of `npm run compile-sass`. -""" -import shlex - -from django.core.management import BaseCommand -from django.conf import settings - -from pavelib.assets import run_deprecated_command_wrapper - - -class Command(BaseCommand): - """ - Compile theme sass and collect theme assets. - """ - - help = "DEPRECATED. Use 'npm run compile-sass' instead." - - # NOTE (CCB): This allows us to compile static assets in Docker containers without database access. - requires_system_checks = [] - - def add_arguments(self, parser): - """ - Add arguments for compile_sass command. - - Args: - parser (django.core.management.base.CommandParser): parsed for parsing command line arguments. - """ - parser.add_argument( - 'system', type=str, nargs='*', default=["lms", "cms"], - help="lms or studio", - ) - - # Named (optional) arguments - parser.add_argument( - '--theme-dirs', - dest='theme_dirs', - type=str, - nargs='+', - default=None, - help="List of dirs where given themes would be looked.", - ) - - parser.add_argument( - '--themes', - type=str, - nargs='+', - default=["all"], - help="List of themes whose sass need to compiled. Or 'no'/'all' to compile for no/all themes.", - ) - - # Named (optional) arguments - parser.add_argument( - '--force', - action='store_true', - default=False, - help="DEPRECATED. Full recompilation is now always forced.", - ) - parser.add_argument( - '--debug', - action='store_true', - default=False, - help="Disable Sass compression", - ) - - def handle(self, *args, **options): - """ - Handle compile_sass command. - """ - systems = set( - {"lms": "lms", "cms": "cms", "studio": "cms"}[sys] - for sys in options.get("system", ["lms", "cms"]) - ) - theme_dirs = options.get("theme_dirs") or settings.COMPREHENSIVE_THEME_DIRS or [] - themes_option = options.get("themes") or [] # '[]' means 'all' - if not settings.ENABLE_COMPREHENSIVE_THEMING: - compile_themes = False - themes = [] - elif "no" in themes_option: - compile_themes = False - themes = [] - elif "all" in themes_option: - compile_themes = True - themes = [] - else: - compile_themes = True - themes = themes_option - run_deprecated_command_wrapper( - old_command="./manage.py [lms|cms] compile_sass", - ignored_old_flags=list(set(["force"]) & set(options)), - new_command=shlex.join([ - "npm", - "run", - ("compile-sass-dev" if options.get("debug") else "compile-sass"), - "--", - *(["--skip-lms"] if "lms" not in systems else []), - *(["--skip-cms"] if "cms" not in systems else []), - *(["--skip-themes"] if not compile_themes else []), - *( - arg - for theme_dir in theme_dirs - for arg in ["--theme-dir", str(theme_dir)] - ), - *( - arg - for theme in themes - for arg in ["--theme", theme] - ), - ]), - ) diff --git a/openedx/core/djangoapps/util/management/commands/print_setting.py b/openedx/core/djangoapps/util/management/commands/print_setting.py index d90a17b9eb42..c53f49d23a19 100644 --- a/openedx/core/djangoapps/util/management/commands/print_setting.py +++ b/openedx/core/djangoapps/util/management/commands/print_setting.py @@ -3,7 +3,11 @@ ============= Django command to output a single Django setting. -Useful when paver or a shell script needs such a value. +Originally used by "paver" scripts before we removed them. +Still useful when a shell script needs such a value. +Keep in mind that the LMS/CMS startup time is slow, so if you invoke this +Django management multiple times in a command that gets run often, you are +going to be sad. This handles the one specific use case of the "print_settings" command from django-extensions that we were actually using. diff --git a/package.json b/package.json index 2f09f8a7df90..8ed93a322633 100644 --- a/package.json +++ b/package.json @@ -14,24 +14,25 @@ "watch-webpack": "npm run webpack-dev -- --watch", "watch-sass": "scripts/watch_sass.sh", "lint": "python scripts/eslint.py", - "test": "npm run test-cms && npm run test-lms && npm run test-xmodule && npm run test-common && npm run test-jest", - "test-kind-vanilla": "npm run test-cms-vanilla && npm run test-xmodule-vanilla && npm run test-common-vanilla", - "test-kind-require": "npm run test-cms-require && npm run test-common-require", - "test-kind-webpack": "npm run test-cms-webpack && npm run test-lms-webpack && npm run test-xmodule-webpack", - "test-cms": "npm run test-cms-vanilla && npm run test-cms-require", - "test-cms-vanilla": "npm run test-suite -- cms/static/karma_cms.conf.js", - "test-cms-require": "npm run test-suite -- cms/static/karma_cms_squire.conf.js", - "test-cms-webpack": "npm run test-suite -- cms/static/karma_cms_webpack.conf.js", - "test-lms": "echo 'WARNING: Webpack JS tests are disabled. No LMS JS tests will be run. See https://github.com/openedx/edx-platform/issues/35956 for details.'", - "test-lms-webpack": "npm run test-suite -- lms/static/karma_lms.conf.js", - "test-xmodule": "npm run test-xmodule-vanilla", - "test-xmodule-vanilla": "npm run test-suite -- xmodule/js/karma_xmodule.conf.js", - "test-xmodule-webpack": "npm run test-suite -- xmodule/js/karma_xmodule_webpack.conf.js", + "test": "npm run test-jest && npm run test-karma", + "test-jest": "jest", + "test-karma": "npm run test-karma-vanilla && npm run test-karma-require && echo 'WARNING: Skipped broken webpack tests. For details, see: https://github.com/openedx/edx-platform/issues/35956'", + "test-karma-vanilla": "npm run test-cms-vanilla && npm run test-xmodule-vanilla && npm run test-common-vanilla", + "test-karma-require": "npm run test-cms-require && npm run test-common-require", + "test-karma-webpack": "npm run test-cms-webpack && npm run test-lms-webpack && npm run test-xmodule-webpack", + "test-karma-conf": "${NODE_WRAPPER:-xvfb-run --auto-servernum} node --max_old_space_size=4096 node_modules/.bin/karma start --single-run=true --capture-timeout=60000 --browsers=FirefoxNoUpdates", + "test-cms": "npm run test-cms-vanilla && npm run test-cms-require && npm run test-cms-webpack", + "test-cms-vanilla": "npm run test-karma-conf -- cms/static/karma_cms.conf.js", + "test-cms-require": "npm run test-karma-conf -- cms/static/karma_cms_squire.conf.js", + "test-cms-webpack": "npm run test-karma-conf -- cms/static/karma_cms_webpack.conf.js", + "test-lms": "npm run test-jest && npm run test-lms-webpack", + "test-lms-webpack": "npm run test-karma-conf -- lms/static/karma_lms.conf.js", + "test-xmodule": "npm run test-xmodule-vanilla && npm run test-xmodule-webpack", + "test-xmodule-vanilla": "npm run test-karma-conf -- xmodule/js/karma_xmodule.conf.js", + "test-xmodule-webpack": "npm run test-karma-conf -- xmodule/js/karma_xmodule_webpack.conf.js", "test-common": "npm run test-common-vanilla && npm run test-common-require", - "test-common-vanilla": "npm run test-suite -- common/static/karma_common.conf.js", - "test-common-require": "npm run test-suite -- common/static/karma_common_requirejs.conf.js", - "test-suite": "${NODE_WRAPPER:-xvfb-run --auto-servernum} node --max_old_space_size=4096 node_modules/.bin/karma start --single-run=true --capture-timeout=60000 --browsers=FirefoxNoUpdates", - "test-jest": "jest" + "test-common-vanilla": "npm run test-karma-conf -- common/static/karma_common.conf.js", + "test-common-require": "npm run test-karma-conf -- common/static/karma_common_requirejs.conf.js" }, "dependencies": { "@babel/core": "7.26.0", diff --git a/pavelib/__init__.py b/pavelib/__init__.py deleted file mode 100644 index 24f05618bdd7..000000000000 --- a/pavelib/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -""" # lint-amnesty, pylint: disable=django-not-configured -paver commands -""" - - -from . import assets diff --git a/pavelib/assets.py b/pavelib/assets.py deleted file mode 100644 index f437b6427f93..000000000000 --- a/pavelib/assets.py +++ /dev/null @@ -1,506 +0,0 @@ -""" -Asset compilation and collection. - -This entire module is DEPRECATED. In Redwood, it exists just as a collection of temporary compatibility wrappers. -In Sumac, this module will be deleted. To migrate, follow the advice in the printed warnings and/or read the -instructions on the DEPR ticket: https://github.com/openedx/edx-platform/issues/31895 -""" - -import argparse -import glob -import json -import shlex -import traceback -from functools import wraps -from threading import Timer - -from paver import tasks -from paver.easy import call_task, cmdopts, consume_args, needs, no_help, sh, task -from watchdog.events import PatternMatchingEventHandler -from watchdog.observers import Observer # pylint: disable=unused-import # Used by Tutor. Remove after Sumac cut. - -from .utils.cmd import django_cmd -from .utils.envs import Env -from .utils.timer import timed - - -SYSTEMS = { - 'lms': 'lms', - 'cms': 'cms', - 'studio': 'cms', -} - -WARNING_SYMBOLS = "⚠️ " * 50 # A row of 'warning' emoji to catch CLI users' attention - - -def run_deprecated_command_wrapper(*, old_command, ignored_old_flags, new_command): - """ - Run the new version of shell command, plus a warning that the old version is deprecated. - """ - depr_warning = ( - "\n" + - f"{WARNING_SYMBOLS}\n" + - "\n" + - f"WARNING: '{old_command}' is DEPRECATED! It will be removed before Sumac.\n" + - "The command you ran is now just a temporary wrapper around a new,\n" + - "supported command, which you should use instead:\n" + - "\n" + - f"\t{new_command}\n" + - "\n" + - "Details: https://github.com/openedx/edx-platform/issues/31895\n" + - "".join( - f" WARNING: ignored deprecated paver flag '{flag}'\n" - for flag in ignored_old_flags - ) + - f"{WARNING_SYMBOLS}\n" + - "\n" - ) - # Print deprecation warning twice so that it's more likely to be seen in the logs. - print(depr_warning) - sh(new_command) - print(depr_warning) - - -def debounce(seconds=1): - """ - Prevents the decorated function from being called more than every `seconds` - seconds. Waits until calls stop coming in before calling the decorated - function. - - This is DEPRECATED. It exists in Redwood just to ease the transition for Tutor. - """ - def decorator(func): - func.timer = None - - @wraps(func) - def wrapper(*args, **kwargs): - def call(): - func(*args, **kwargs) - func.timer = None - if func.timer: - func.timer.cancel() - func.timer = Timer(seconds, call) - func.timer.start() - - return wrapper - return decorator - - -class SassWatcher(PatternMatchingEventHandler): - """ - Watches for sass file changes - - This is DEPRECATED. It exists in Redwood just to ease the transition for Tutor. - """ - ignore_directories = True - patterns = ['*.scss'] - - def register(self, observer, directories): - """ - register files with observer - Arguments: - observer (watchdog.observers.Observer): sass file observer - directories (list): list of directories to be register for sass watcher. - """ - for dirname in directories: - paths = [] - if '*' in dirname: - paths.extend(glob.glob(dirname)) - else: - paths.append(dirname) - - for obs_dirname in paths: - observer.schedule(self, obs_dirname, recursive=True) - - @debounce() - def on_any_event(self, event): - print('\tCHANGED:', event.src_path) - try: - compile_sass() # pylint: disable=no-value-for-parameter - except Exception: # pylint: disable=broad-except - traceback.print_exc() - - -@task -@no_help -@cmdopts([ - ('system=', 's', 'The system to compile sass for (defaults to all)'), - ('theme-dirs=', '-td', 'Theme dirs containing all themes (defaults to None)'), - ('themes=', '-t', 'The theme to compile sass for (defaults to None)'), - ('debug', 'd', 'Whether to use development settings'), - ('force', '', 'DEPRECATED. Full recompilation is now always forced.'), -]) -@timed -def compile_sass(options): - """ - Compile Sass to CSS. If command is called without any arguments, it will - only compile lms, cms sass for the open source theme. And none of the comprehensive theme's sass would be compiled. - - If you want to compile sass for all comprehensive themes you will have to run compile_sass - specifying all the themes that need to be compiled.. - - The following is a list of some possible ways to use this command. - - Command: - paver compile_sass - Description: - compile sass files for both lms and cms. If command is called like above (i.e. without any arguments) it will - only compile lms, cms sass for the open source theme. None of the theme's sass will be compiled. - - Command: - paver compile_sass --theme-dirs /edx/app/edxapp/edx-platform/themes --themes=red-theme - Description: - compile sass files for both lms and cms for 'red-theme' present in '/edx/app/edxapp/edx-platform/themes' - - Command: - paver compile_sass --theme-dirs=/edx/app/edxapp/edx-platform/themes --themes red-theme stanford-style - Description: - compile sass files for both lms and cms for 'red-theme' and 'stanford-style' present in - '/edx/app/edxapp/edx-platform/themes'. - - Command: - paver compile_sass --system=cms - --theme-dirs /edx/app/edxapp/edx-platform/themes /edx/app/edxapp/edx-platform/common/test/ - --themes red-theme stanford-style test-theme - Description: - compile sass files for cms only for 'red-theme', 'stanford-style' and 'test-theme' present in - '/edx/app/edxapp/edx-platform/themes' and '/edx/app/edxapp/edx-platform/common/test/'. - - This is a DEPRECATED COMPATIBILITY WRAPPER. Use `npm run compile-sass` instead. - """ - systems = [SYSTEMS[sys] for sys in get_parsed_option(options, 'system', ['lms', 'cms'])] # normalize studio->cms - run_deprecated_command_wrapper( - old_command="paver compile_sass", - ignored_old_flags=(set(["--force"]) & set(options)), - new_command=shlex.join([ - "npm", - "run", - ("compile-sass-dev" if options.get("debug") else "compile-sass"), - "--", - *(["--dry"] if tasks.environment.dry_run else []), - *(["--skip-lms"] if "lms" not in systems else []), - *(["--skip-cms"] if "cms" not in systems else []), - *( - arg - for theme_dir in get_parsed_option(options, 'theme_dirs', []) - for arg in ["--theme-dir", str(theme_dir)] - ), - *( - arg - for theme in get_parsed_option(options, "themes", []) - for arg in ["--theme", theme] - ), - ]), - ) - - -def _compile_sass(system, theme, debug, force, _timing_info): - """ - This is a DEPRECATED COMPATIBILITY WRAPPER - - It exists to ease the transition for Tutor in Redwood, which directly imported and used this function. - """ - run_deprecated_command_wrapper( - old_command="pavelib.assets:_compile_sass", - ignored_old_flags=(set(["--force"]) if force else set()), - new_command=[ - "npm", - "run", - ("compile-sass-dev" if debug else "compile-sass"), - "--", - *(["--dry"] if tasks.environment.dry_run else []), - *( - ["--skip-default", "--theme-dir", str(theme.parent), "--theme", str(theme.name)] - if theme - else [] - ), - ("--skip-cms" if system == "lms" else "--skip-lms"), - ] - ) - - -def process_npm_assets(): - """ - Process vendor libraries installed via NPM. - - This is a DEPRECATED COMPATIBILITY WRAPPER. It is now handled as part of `npm clean-install`. - If you need to invoke it explicitly, you can run `npm run postinstall`. - """ - run_deprecated_command_wrapper( - old_command="pavelib.assets:process_npm_assets", - ignored_old_flags=[], - new_command=shlex.join(["npm", "run", "postinstall"]), - ) - - -@task -@no_help -def process_xmodule_assets(): - """ - Process XModule static assets. - - This is a DEPRECATED COMPATIBILITY STUB. Refrences to it should be deleted. - """ - print( - "\n" + - f"{WARNING_SYMBOLS}", - "\n" + - "WARNING: 'paver process_xmodule_assets' is DEPRECATED! It will be removed before Sumac.\n" + - "\n" + - "Starting with Quince, it is no longer necessary to post-process XModule assets, so \n" + - "'paver process_xmodule_assets' is a no-op. Please simply remove it from your build scripts.\n" + - "\n" + - "Details: https://github.com/openedx/edx-platform/issues/31895\n" + - f"{WARNING_SYMBOLS}", - ) - - -def collect_assets(systems, settings, **kwargs): - """ - Collect static assets, including Django pipeline processing. - `systems` is a list of systems (e.g. 'lms' or 'studio' or both) - `settings` is the Django settings module to use. - `**kwargs` include arguments for using a log directory for collectstatic output. Defaults to /dev/null. - - This is a DEPRECATED COMPATIBILITY WRAPPER - - It exists to ease the transition for Tutor in Redwood, which directly imported and used this function. - """ - run_deprecated_command_wrapper( - old_command="pavelib.asset:collect_assets", - ignored_old_flags=[], - new_command=" && ".join( - "( " + - shlex.join( - ["./manage.py", SYSTEMS[sys], f"--settings={settings}", "collectstatic", "--noinput"] - ) + ( - "" - if "collect_log_dir" not in kwargs else - " > /dev/null" - if kwargs["collect_log_dir"] is None else - f"> {kwargs['collect_log_dir']}/{SYSTEMS[sys]}-collectstatic.out" - ) + - " )" - for sys in systems - ), - ) - - -def execute_compile_sass(args): - """ - Construct django management command compile_sass (defined in theming app) and execute it. - Args: - args: command line argument passed via update_assets command - - This is a DEPRECATED COMPATIBILITY WRAPPER. Use `npm run compile-sass` instead. - """ - for sys in args.system: - options = "" - options += " --theme-dirs " + " ".join(args.theme_dirs) if args.theme_dirs else "" - options += " --themes " + " ".join(args.themes) if args.themes else "" - options += " --debug" if args.debug else "" - - sh( - django_cmd( - sys, - args.settings, - "compile_sass {system} {options}".format( - system='cms' if sys == 'studio' else sys, - options=options, - ), - ), - ) - - -@task -@cmdopts([ - ('settings=', 's', "Django settings (defaults to devstack)"), - ('watch', 'w', "DEPRECATED. This flag never did anything anyway."), -]) -@timed -def webpack(options): - """ - Run a Webpack build. - - This is a DEPRECATED COMPATIBILITY WRAPPER. Use `npm run webpack` instead. - """ - settings = getattr(options, 'settings', Env.DEVSTACK_SETTINGS) - result = Env.get_django_settings(['STATIC_ROOT', 'WEBPACK_CONFIG_PATH'], "lms", settings=settings) - static_root_lms, config_path = result - static_root_cms, = Env.get_django_settings(["STATIC_ROOT"], "cms", settings=settings) - js_env_extra_config_setting, = Env.get_django_json_settings(["JS_ENV_EXTRA_CONFIG"], "cms", settings=settings) - js_env_extra_config = json.dumps(js_env_extra_config_setting or "{}") - node_env = "development" if config_path == 'webpack.dev.config.js' else "production" - run_deprecated_command_wrapper( - old_command="paver webpack", - ignored_old_flags=(set(["watch"]) & set(options)), - new_command=' '.join([ - f"WEBPACK_CONFIG_PATH={config_path}", - f"NODE_ENV={node_env}", - f"STATIC_ROOT_LMS={static_root_lms}", - f"STATIC_ROOT_CMS={static_root_cms}", - f"JS_ENV_EXTRA_CONFIG={js_env_extra_config}", - "npm", - "run", - "webpack", - ]), - ) - - -def get_parsed_option(command_opts, opt_key, default=None): - """ - Extract user command option and parse it. - Arguments: - command_opts: Command line arguments passed via paver command. - opt_key: name of option to get and parse - default: if `command_opt_value` not in `command_opts`, `command_opt_value` will be set to default. - Returns: - list or None - """ - command_opt_value = getattr(command_opts, opt_key, default) - if command_opt_value: - command_opt_value = listfy(command_opt_value) - - return command_opt_value - - -def listfy(data): - """ - Check and convert data to list. - Arguments: - data: data structure to be converted. - """ - - if isinstance(data, str): - data = data.split(',') - elif not isinstance(data, list): - data = [data] - - return data - - -@task -@cmdopts([ - ('background', 'b', 'DEPRECATED. Use shell tools like & to run in background if needed.'), - ('settings=', 's', "DEPRECATED. Django is not longer invoked to compile JS/Sass."), - ('theme-dirs=', '-td', 'The themes dir containing all themes (defaults to None)'), - ('themes=', '-t', 'DEPRECATED. All themes in --theme-dirs are now watched.'), - ('wait=', '-w', 'DEPRECATED. Watchdog\'s default wait time is now used.'), -]) -@timed -def watch_assets(options): - """ - Watch for changes to asset files, and regenerate js/css - - This is a DEPRECATED COMPATIBILITY WRAPPER. Use `npm run watch` instead. - """ - # Don't watch assets when performing a dry run - if tasks.environment.dry_run: - return - - theme_dirs = ':'.join(get_parsed_option(options, 'theme_dirs', [])) - run_deprecated_command_wrapper( - old_command="paver watch_assets", - ignored_old_flags=(set(["debug", "themes", "settings", "background"]) & set(options)), - new_command=shlex.join([ - *( - ["env", f"COMPREHENSIVE_THEME_DIRS={theme_dirs}"] - if theme_dirs else [] - ), - "npm", - "run", - "watch", - ]), - ) - - -@task -@needs( - 'pavelib.prereqs.install_node_prereqs', - 'pavelib.prereqs.install_python_prereqs', -) -@consume_args -@timed -def update_assets(args): - """ - Compile Sass, then collect static assets. - - This is a DEPRECATED COMPATIBILITY WRAPPER around other DEPRECATED COMPATIBILITY WRAPPERS. - The aggregate affect of this command can be achieved with this sequence of commands instead: - - * pip install -r requirements/edx/assets.txt # replaces install_python_prereqs - * npm clean-install # replaces install_node_prereqs - * npm run build # replaces execute_compile_sass and webpack - * ./manage.py lms collectstatic --noinput # replaces collect_assets (for LMS) - * ./manage.py cms collectstatic --noinput # replaces collect_assets (for CMS) - """ - parser = argparse.ArgumentParser(prog='paver update_assets') - parser.add_argument( - 'system', type=str, nargs='*', default=["lms", "studio"], - help="lms or studio", - ) - parser.add_argument( - '--settings', type=str, default=Env.DEVSTACK_SETTINGS, - help="Django settings module", - ) - parser.add_argument( - '--debug', action='store_true', default=False, - help="Enable all debugging", - ) - parser.add_argument( - '--debug-collect', action='store_true', default=False, - help="Disable collect static", - ) - parser.add_argument( - '--skip-collect', dest='collect', action='store_false', default=True, - help="Skip collection of static assets", - ) - parser.add_argument( - '--watch', action='store_true', default=False, - help="Watch files for changes", - ) - parser.add_argument( - '--theme-dirs', dest='theme_dirs', type=str, nargs='+', default=None, - help="base directories where themes are placed", - ) - parser.add_argument( - '--themes', type=str, nargs='+', default=None, - help="list of themes to compile sass for. ignored when --watch is used; all themes are watched.", - ) - parser.add_argument( - '--collect-log', dest="collect_log_dir", default=None, - help="When running collectstatic, direct output to specified log directory", - ) - parser.add_argument( - '--wait', type=float, default=0.0, - help="DEPRECATED. Watchdog's default wait time is now used.", - ) - args = parser.parse_args(args) - - # Build Webpack - call_task('pavelib.assets.webpack', options={'settings': args.settings}) - - # Compile sass for themes and system - execute_compile_sass(args) - - if args.collect: - if args.collect_log_dir: - collect_log_args = {"collect_log_dir": args.collect_log_dir} - elif args.debug or args.debug_collect: - collect_log_args = {"collect_log_dir": None} - else: - collect_log_args = {} - - collect_assets(args.system, args.settings, **collect_log_args) - - if args.watch: - call_task( - 'pavelib.assets.watch_assets', - options={ - 'background': not args.debug, - 'settings': args.settings, - 'theme_dirs': args.theme_dirs, - 'themes': args.themes, - 'wait': [float(args.wait)] - }, - ) diff --git a/pavelib/paver_tests/__init__.py b/pavelib/paver_tests/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/pavelib/paver_tests/pylint_test_list.json b/pavelib/paver_tests/pylint_test_list.json deleted file mode 100644 index d0e8b43aa93d..000000000000 --- a/pavelib/paver_tests/pylint_test_list.json +++ /dev/null @@ -1,6 +0,0 @@ -[ - "foo/bar.py:192: [C0111(missing-docstring), Bliptv] Missing docstring", - "foo/bar/test.py:74: [C0322(no-space-before-operator)] Operator not preceded by a space", - "ugly/string/test.py:16: [C0103(invalid-name)] Invalid name \"whats up\" for type constant (should match (([A-Z_][A-Z0-9_]*)|(__.*__)|log|urlpatterns)$)", - "multiple/lines/test.py:72: [C0322(no-space-before-operator)] Operator not preceded by a space\nFOO_BAR='pipeline.storage.NonPackagingPipelineStorage'\n ^" -] diff --git a/pavelib/paver_tests/test_assets.py b/pavelib/paver_tests/test_assets.py deleted file mode 100644 index f7100a7f03c3..000000000000 --- a/pavelib/paver_tests/test_assets.py +++ /dev/null @@ -1,130 +0,0 @@ -"""Unit tests for the Paver asset tasks.""" - -import json -import os -from pathlib import Path -from unittest import TestCase -from unittest.mock import patch - -import ddt -import paver.easy -from paver import tasks - -import pavelib.assets -from pavelib.assets import Env - - -REPO_ROOT = Path(__file__).parent.parent.parent - -LMS_SETTINGS = { - "WEBPACK_CONFIG_PATH": "webpack.fake.config.js", - "STATIC_ROOT": "/fake/lms/staticfiles", - -} -CMS_SETTINGS = { - "WEBPACK_CONFIG_PATH": "webpack.fake.config", - "STATIC_ROOT": "/fake/cms/staticfiles", - "JS_ENV_EXTRA_CONFIG": json.dumps({"key1": [True, False], "key2": {"key2.1": 1369, "key2.2": "1369"}}), -} - - -def _mock_get_django_settings(django_settings, system, settings=None): # pylint: disable=unused-argument - return [(LMS_SETTINGS if system == "lms" else CMS_SETTINGS)[s] for s in django_settings] - - -@ddt.ddt -@patch.object(Env, 'get_django_settings', _mock_get_django_settings) -@patch.object(Env, 'get_django_json_settings', _mock_get_django_settings) -class TestDeprecatedPaverAssets(TestCase): - """ - Simple test to ensure that the soon-to-be-removed Paver commands are correctly translated into the new npm-run - commands. - """ - def setUp(self): - super().setUp() - self.maxDiff = None - os.environ['NO_PREREQ_INSTALL'] = 'true' - tasks.environment = tasks.Environment() - - def tearDown(self): - super().tearDown() - del os.environ['NO_PREREQ_INSTALL'] - - @ddt.data( - dict( - task_name='pavelib.assets.compile_sass', - args=[], - kwargs={}, - expected=["npm run compile-sass --"], - ), - dict( - task_name='pavelib.assets.compile_sass', - args=[], - kwargs={"system": "lms,studio"}, - expected=["npm run compile-sass --"], - ), - dict( - task_name='pavelib.assets.compile_sass', - args=[], - kwargs={"debug": True}, - expected=["npm run compile-sass-dev --"], - ), - dict( - task_name='pavelib.assets.compile_sass', - args=[], - kwargs={"system": "lms"}, - expected=["npm run compile-sass -- --skip-cms"], - ), - dict( - task_name='pavelib.assets.compile_sass', - args=[], - kwargs={"system": "studio"}, - expected=["npm run compile-sass -- --skip-lms"], - ), - dict( - task_name='pavelib.assets.compile_sass', - args=[], - kwargs={"system": "cms", "theme_dirs": f"{REPO_ROOT}/common/test,{REPO_ROOT}/themes"}, - expected=[ - "npm run compile-sass -- --skip-lms " + - f"--theme-dir {REPO_ROOT}/common/test --theme-dir {REPO_ROOT}/themes" - ], - ), - dict( - task_name='pavelib.assets.compile_sass', - args=[], - kwargs={"theme_dirs": f"{REPO_ROOT}/common/test,{REPO_ROOT}/themes", "themes": "red-theme,test-theme"}, - expected=[ - "npm run compile-sass -- " + - f"--theme-dir {REPO_ROOT}/common/test --theme-dir {REPO_ROOT}/themes " + - "--theme red-theme --theme test-theme" - ], - ), - dict( - task_name='pavelib.assets.update_assets', - args=["lms", "studio", "--settings=fake.settings"], - kwargs={}, - expected=[ - ( - "WEBPACK_CONFIG_PATH=webpack.fake.config.js " + - "NODE_ENV=production " + - "STATIC_ROOT_LMS=/fake/lms/staticfiles " + - "STATIC_ROOT_CMS=/fake/cms/staticfiles " + - 'JS_ENV_EXTRA_CONFIG=' + - '"{\\"key1\\": [true, false], \\"key2\\": {\\"key2.1\\": 1369, \\"key2.2\\": \\"1369\\"}}" ' + - "npm run webpack" - ), - "python manage.py lms --settings=fake.settings compile_sass lms ", - "python manage.py cms --settings=fake.settings compile_sass cms ", - ( - "( ./manage.py lms --settings=fake.settings collectstatic --noinput ) && " + - "( ./manage.py cms --settings=fake.settings collectstatic --noinput )" - ), - ], - ), - ) - @ddt.unpack - @patch.object(pavelib.assets, 'sh') - def test_paver_assets_wrapper_invokes_new_commands(self, mock_sh, task_name, args, kwargs, expected): - paver.easy.call_task(task_name, args=args, options=kwargs) - assert [call_args[0] for (call_args, call_kwargs) in mock_sh.call_args_list] == expected diff --git a/pavelib/paver_tests/utils.py b/pavelib/paver_tests/utils.py deleted file mode 100644 index 1db26cf76a4c..000000000000 --- a/pavelib/paver_tests/utils.py +++ /dev/null @@ -1,97 +0,0 @@ -"""Unit tests for the Paver server tasks.""" - - -import os -from unittest import TestCase -from uuid import uuid4 - -from paver import tasks -from paver.easy import BuildFailure - - -class PaverTestCase(TestCase): - """ - Base class for Paver test cases. - """ - def setUp(self): - super().setUp() - - # Show full length diffs upon test failure - self.maxDiff = None # pylint: disable=invalid-name - - # Create a mock Paver environment - tasks.environment = MockEnvironment() - - # Don't run pre-reqs - os.environ['NO_PREREQ_INSTALL'] = 'true' - - def tearDown(self): - super().tearDown() - tasks.environment = tasks.Environment() - del os.environ['NO_PREREQ_INSTALL'] - - @property - def task_messages(self): - """Returns the messages output by the Paver task.""" - return tasks.environment.messages - - @property - def platform_root(self): - """Returns the current platform's root directory.""" - return os.getcwd() - - def reset_task_messages(self): - """Clear the recorded message""" - tasks.environment.messages = [] - - -class MockEnvironment(tasks.Environment): - """ - Mock environment that collects information about Paver commands. - """ - def __init__(self): - super().__init__() - self.dry_run = True - self.messages = [] - - def info(self, message, *args): - """Capture any messages that have been recorded""" - if args: - output = message % args - else: - output = message - if not output.startswith("--->"): - self.messages.append(str(output)) - - -def fail_on_eslint(*args, **kwargs): - """ - For our tests, we need the call for diff-quality running eslint reports - to fail, since that is what is going to fail when we pass in a - percentage ("p") requirement. - """ - if "eslint" in args[0]: # lint-amnesty, pylint: disable=no-else-raise - raise BuildFailure('Subprocess return code: 1') - else: - if kwargs.get('capture', False): - return uuid4().hex - else: - return - - -def fail_on_npm_install(): - """ - Used to simulate an error when executing "npm install" - """ - return 1 - - -def unexpected_fail_on_npm_install(*args, **kwargs): # pylint: disable=unused-argument - """ - For our tests, we need the call for diff-quality running pycodestyle reports to fail, since that is what - is going to fail when we pass in a percentage ("p") requirement. - """ - if ["npm", "install", "--verbose"] == args[0]: # lint-amnesty, pylint: disable=no-else-raise - raise BuildFailure('Subprocess return code: 50') - else: - return diff --git a/pavelib/prereqs.py b/pavelib/prereqs.py deleted file mode 100644 index 4453176c94da..000000000000 --- a/pavelib/prereqs.py +++ /dev/null @@ -1,351 +0,0 @@ -""" -Install Python and Node prerequisites. -""" - - -import hashlib -import os -import re -import subprocess -import sys -from distutils import sysconfig # pylint: disable=deprecated-module - -from paver.easy import sh, task # lint-amnesty, pylint: disable=unused-import - -from .utils.envs import Env -from .utils.timer import timed - -PREREQS_STATE_DIR = os.getenv('PREREQ_CACHE_DIR', Env.REPO_ROOT / '.prereqs_cache') -NO_PREREQ_MESSAGE = "NO_PREREQ_INSTALL is set, not installing prereqs" -NO_PYTHON_UNINSTALL_MESSAGE = 'NO_PYTHON_UNINSTALL is set. No attempts will be made to uninstall old Python libs.' -COVERAGE_REQ_FILE = 'requirements/edx/coverage.txt' - -# If you make any changes to this list you also need to make -# a corresponding change to circle.yml, which is how the python -# prerequisites are installed for builds on circleci.com -toxenv = os.environ.get('TOXENV') -if toxenv and toxenv != 'quality': - PYTHON_REQ_FILES = ['requirements/edx/testing.txt'] -else: - PYTHON_REQ_FILES = ['requirements/edx/development.txt'] - -# Developers can have private requirements, for local copies of github repos, -# or favorite debugging tools, etc. -PRIVATE_REQS = 'requirements/edx/private.txt' -if os.path.exists(PRIVATE_REQS): - PYTHON_REQ_FILES.append(PRIVATE_REQS) - - -def str2bool(s): - s = str(s) - return s.lower() in ('yes', 'true', 't', '1') - - -def no_prereq_install(): - """ - Determine if NO_PREREQ_INSTALL should be truthy or falsy. - """ - return str2bool(os.environ.get('NO_PREREQ_INSTALL', 'False')) - - -def no_python_uninstall(): - """ Determine if we should run the uninstall_python_packages task. """ - return str2bool(os.environ.get('NO_PYTHON_UNINSTALL', 'False')) - - -def create_prereqs_cache_dir(): - """Create the directory for storing the hashes, if it doesn't exist already.""" - try: - os.makedirs(PREREQS_STATE_DIR) - except OSError: - if not os.path.isdir(PREREQS_STATE_DIR): - raise - - -def compute_fingerprint(path_list): - """ - Hash the contents of all the files and directories in `path_list`. - Returns the hex digest. - """ - - hasher = hashlib.sha1() - - for path_item in path_list: - - # For directories, create a hash based on the modification times - # of first-level subdirectories - if os.path.isdir(path_item): - for dirname in sorted(os.listdir(path_item)): - path_name = os.path.join(path_item, dirname) - if os.path.isdir(path_name): - hasher.update(str(os.stat(path_name).st_mtime).encode('utf-8')) - - # For files, hash the contents of the file - if os.path.isfile(path_item): - with open(path_item, "rb") as file_handle: - hasher.update(file_handle.read()) - - return hasher.hexdigest() - - -def prereq_cache(cache_name, paths, install_func): - """ - Conditionally execute `install_func()` only if the files/directories - specified by `paths` have changed. - - If the code executes successfully (no exceptions are thrown), the cache - is updated with the new hash. - """ - # Retrieve the old hash - cache_filename = cache_name.replace(" ", "_") - cache_file_path = os.path.join(PREREQS_STATE_DIR, f"{cache_filename}.sha1") - old_hash = None - if os.path.isfile(cache_file_path): - with open(cache_file_path) as cache_file: - old_hash = cache_file.read() - - # Compare the old hash to the new hash - # If they do not match (either the cache hasn't been created, or the files have changed), - # then execute the code within the block. - new_hash = compute_fingerprint(paths) - if new_hash != old_hash: - install_func() - - # Update the cache with the new hash - # If the code executed within the context fails (throws an exception), - # then this step won't get executed. - create_prereqs_cache_dir() - with open(cache_file_path, "wb") as cache_file: - # Since the pip requirement files are modified during the install - # process, we need to store the hash generated AFTER the installation - post_install_hash = compute_fingerprint(paths) - cache_file.write(post_install_hash.encode('utf-8')) - else: - print(f'{cache_name} unchanged, skipping...') - - -def node_prereqs_installation(): - """ - Configures npm and installs Node prerequisites - """ - # Before July 2023, these directories were created and written to - # as root. Afterwards, they are created as being owned by the - # `app` user -- but also need to be deleted by that user (due to - # how npm runs post-install scripts.) Developers with an older - # devstack installation who are reprovisioning will see errors - # here if the files are still owned by root. Deleting the files in - # advance prevents this error. - # - # This hack should probably be left in place for at least a year. - # See ADR 17 for more background on the transition. - sh("rm -rf common/static/common/js/vendor/ common/static/common/css/vendor/") - # At the time of this writing, the js dir has git-versioned files - # but the css dir does not, so the latter would have been created - # as root-owned (in the process of creating the vendor - # subdirectory). Delete it only if empty, just in case - # git-versioned files are added later. - sh("rmdir common/static/common/css || true") - - # NPM installs hang sporadically. Log the installation process so that we - # determine if any packages are chronic offenders. - npm_log_file_path = f'{Env.GEN_LOG_DIR}/npm-install.log' - npm_log_file = open(npm_log_file_path, 'wb') # lint-amnesty, pylint: disable=consider-using-with - npm_command = 'npm ci --verbose'.split() - - # The implementation of Paver's `sh` function returns before the forked - # actually returns. Using a Popen object so that we can ensure that - # the forked process has returned - proc = subprocess.Popen(npm_command, stderr=npm_log_file) # lint-amnesty, pylint: disable=consider-using-with - retcode = proc.wait() - if retcode == 1: - raise Exception(f"npm install failed: See {npm_log_file_path}") - print("Successfully clean-installed NPM packages. Log found at {}".format( - npm_log_file_path - )) - - -def python_prereqs_installation(): - """ - Installs Python prerequisites - """ - # edx-platform installs some Python projects from within the edx-platform repo itself. - sh("pip install -e .") - for req_file in PYTHON_REQ_FILES: - pip_install_req_file(req_file) - - -def pip_install_req_file(req_file): - """Pip install the requirements file.""" - pip_cmd = 'pip install -q --disable-pip-version-check --exists-action w' - sh(f"{pip_cmd} -r {req_file}") - - -@task -@timed -def install_node_prereqs(): - """ - Installs Node prerequisites - """ - if no_prereq_install(): - print(NO_PREREQ_MESSAGE) - return - - prereq_cache("Node prereqs", ["package.json", "package-lock.json"], node_prereqs_installation) - - -# To add a package to the uninstall list, just add it to this list! No need -# to touch any other part of this file. -PACKAGES_TO_UNINSTALL = [ - "MySQL-python", # Because mysqlclient shares the same directory name - "South", # Because it interferes with Django 1.8 migrations. - "edxval", # Because it was bork-installed somehow. - "django-storages", - "django-oauth2-provider", # Because now it's called edx-django-oauth2-provider. - "edx-oauth2-provider", # Because it moved from github to pypi - "enum34", # Because enum34 is not needed in python>3.4 - "i18n-tools", # Because now it's called edx-i18n-tools - "moto", # Because we no longer use it and it conflicts with recent jsondiff versions - "python-saml", # Because python3-saml shares the same directory name - "pytest-faulthandler", # Because it was bundled into pytest - "djangorestframework-jwt", # Because now its called drf-jwt. -] - - -@task -@timed -def uninstall_python_packages(): - """ - Uninstall Python packages that need explicit uninstallation. - - Some Python packages that we no longer want need to be explicitly - uninstalled, notably, South. Some other packages were once installed in - ways that were resistant to being upgraded, like edxval. Also uninstall - them. - """ - - if no_python_uninstall(): - print(NO_PYTHON_UNINSTALL_MESSAGE) - return - - # So that we don't constantly uninstall things, use a hash of the packages - # to be uninstalled. Check it, and skip this if we're up to date. - hasher = hashlib.sha1() - hasher.update(repr(PACKAGES_TO_UNINSTALL).encode('utf-8')) - expected_version = hasher.hexdigest() - state_file_path = os.path.join(PREREQS_STATE_DIR, "Python_uninstall.sha1") - create_prereqs_cache_dir() - - if os.path.isfile(state_file_path): - with open(state_file_path) as state_file: - version = state_file.read() - if version == expected_version: - print('Python uninstalls unchanged, skipping...') - return - - # Run pip to find the packages we need to get rid of. Believe it or not, - # edx-val is installed in a way that it is present twice, so we have a loop - # to really really get rid of it. - for _ in range(3): - uninstalled = False - frozen = sh("pip freeze", capture=True) - - for package_name in PACKAGES_TO_UNINSTALL: - if package_in_frozen(package_name, frozen): - # Uninstall the pacakge - sh(f"pip uninstall --disable-pip-version-check -y {package_name}") - uninstalled = True - if not uninstalled: - break - else: - # We tried three times and didn't manage to get rid of the pests. - print("Couldn't uninstall unwanted Python packages!") - return - - # Write our version. - with open(state_file_path, "wb") as state_file: - state_file.write(expected_version.encode('utf-8')) - - -def package_in_frozen(package_name, frozen_output): - """Is this package in the output of 'pip freeze'?""" - # Look for either: - # - # PACKAGE-NAME== - # - # or: - # - # blah_blah#egg=package_name-version - # - pattern = r"(?mi)^{pkg}==|#egg={pkg_under}-".format( - pkg=re.escape(package_name), - pkg_under=re.escape(package_name.replace("-", "_")), - ) - return bool(re.search(pattern, frozen_output)) - - -@task -@timed -def install_coverage_prereqs(): - """ Install python prereqs for measuring coverage. """ - if no_prereq_install(): - print(NO_PREREQ_MESSAGE) - return - pip_install_req_file(COVERAGE_REQ_FILE) - - -@task -@timed -def install_python_prereqs(): - """ - Installs Python prerequisites. - """ - if no_prereq_install(): - print(NO_PREREQ_MESSAGE) - return - - uninstall_python_packages() - - # Include all of the requirements files in the fingerprint. - files_to_fingerprint = list(PYTHON_REQ_FILES) - - # Also fingerprint the directories where packages get installed: - # ("/edx/app/edxapp/venvs/edxapp/lib/python2.7/site-packages") - files_to_fingerprint.append(sysconfig.get_python_lib()) - - # In a virtualenv, "-e installs" get put in a src directory. - if Env.PIP_SRC: - src_dir = Env.PIP_SRC - else: - src_dir = os.path.join(sys.prefix, "src") - if os.path.isdir(src_dir): - files_to_fingerprint.append(src_dir) - - # Also fingerprint this source file, so that if the logic for installations - # changes, we will redo the installation. - this_file = __file__ - if this_file.endswith(".pyc"): - this_file = this_file[:-1] # use the .py file instead of the .pyc - files_to_fingerprint.append(this_file) - - prereq_cache("Python prereqs", files_to_fingerprint, python_prereqs_installation) - - -@task -@timed -def install_prereqs(): - """ - Installs Node and Python prerequisites - """ - if no_prereq_install(): - print(NO_PREREQ_MESSAGE) - return - - if not str2bool(os.environ.get('SKIP_NPM_INSTALL', 'False')): - install_node_prereqs() - install_python_prereqs() - log_installed_python_prereqs() - - -def log_installed_python_prereqs(): - """ Logs output of pip freeze for debugging. """ - sh("pip freeze > {}".format(Env.GEN_LOG_DIR + "/pip_freeze.log")) diff --git a/pavelib/utils/__init__.py b/pavelib/utils/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/pavelib/utils/cmd.py b/pavelib/utils/cmd.py deleted file mode 100644 index a350c90a6a96..000000000000 --- a/pavelib/utils/cmd.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -Helper functions for constructing shell commands. -""" - - -def cmd(*args): - """ - Concatenate the arguments into a space-separated shell command. - """ - return " ".join(str(arg) for arg in args if arg) - - -def django_cmd(sys, settings, *args): - """ - Construct a Django management command. - - `sys` is either 'lms' or 'studio'. - `settings` is the Django settings module (such as "dev" or "test") - `args` are concatenated to form the rest of the command. - """ - # Maintain backwards compatibility with manage.py, - # which calls "studio" "cms" - sys = 'cms' if sys == 'studio' else sys - return cmd("python manage.py", sys, f"--settings={settings}", *args) diff --git a/pavelib/utils/envs.py b/pavelib/utils/envs.py deleted file mode 100644 index 7dc9870fbdea..000000000000 --- a/pavelib/utils/envs.py +++ /dev/null @@ -1,271 +0,0 @@ -""" -Helper functions for loading environment settings. -""" -import configparser -import json -import os -import sys -from time import sleep - -from lazy import lazy -from path import Path as path -from paver.easy import BuildFailure, sh - -from pavelib.utils.cmd import django_cmd - - -def repo_root(): - """ - Get the root of the git repository (edx-platform). - - This sometimes fails on Docker Devstack, so it's been broken - down with some additional error handling. It usually starts - working within 30 seconds or so; for more details, see - https://openedx.atlassian.net/browse/PLAT-1629 and - https://github.com/docker/for-mac/issues/1509 - """ - file_path = path(__file__) - attempt = 1 - while True: - try: - absolute_path = file_path.abspath() - break - except OSError: - print(f'Attempt {attempt}/180 to get an absolute path failed') - if attempt < 180: - attempt += 1 - sleep(1) - else: - print('Unable to determine the absolute path of the edx-platform repo, aborting') - raise - return absolute_path.parent.parent.parent - - -class Env: - """ - Load information about the execution environment. - """ - - # Root of the git repository (edx-platform) - REPO_ROOT = repo_root() - - # Reports Directory - REPORT_DIR = REPO_ROOT / 'reports' - METRICS_DIR = REPORT_DIR / 'metrics' - QUALITY_DIR = REPORT_DIR / 'quality_junitxml' - - # Generic log dir - GEN_LOG_DIR = REPO_ROOT / "test_root" / "log" - - # Python unittest dirs - PYTHON_COVERAGERC = REPO_ROOT / ".coveragerc" - - # Which Python version should be used in xdist workers? - PYTHON_VERSION = os.environ.get("PYTHON_VERSION", "2.7") - - # Directory that videos are served from - VIDEO_SOURCE_DIR = REPO_ROOT / "test_root" / "data" / "video" - - PRINT_SETTINGS_LOG_FILE = GEN_LOG_DIR / "print_settings.log" - - # Detect if in a Docker container, and if so which one - FRONTEND_TEST_SERVER_HOST = os.environ.get('FRONTEND_TEST_SERVER_HOSTNAME', '0.0.0.0') - USING_DOCKER = FRONTEND_TEST_SERVER_HOST != '0.0.0.0' - DEVSTACK_SETTINGS = 'devstack_docker' if USING_DOCKER else 'devstack' - TEST_SETTINGS = 'test' - - # Mongo databases that will be dropped before/after the tests run - MONGO_HOST = 'localhost' - - # Test Ids Directory - TEST_DIR = REPO_ROOT / ".testids" - - # Configured browser to use for the js test suites - SELENIUM_BROWSER = os.environ.get('SELENIUM_BROWSER', 'firefox') - if USING_DOCKER: - KARMA_BROWSER = 'ChromeDocker' if SELENIUM_BROWSER == 'chrome' else 'FirefoxDocker' - else: - KARMA_BROWSER = 'FirefoxNoUpdates' - - # Files used to run each of the js test suites - # TODO: We have [temporarily disabled] the three Webpack-based tests suites. They have been silently - # broken for a long time; after noticing they were broken, we added the DieHardPlugin to - # webpack.common.config.js to prevent future silent breakage, but have not yet been able to - # fix and re-enable the suites. Note that the LMS suite is all Webpack-based even though it's - # not in the name. - # Issue: https://github.com/openedx/edx-platform/issues/35956 - KARMA_CONFIG_FILES = [ - REPO_ROOT / 'cms/static/karma_cms.conf.js', - REPO_ROOT / 'cms/static/karma_cms_squire.conf.js', - ## [temporarily disabled] REPO_ROOT / 'cms/static/karma_cms_webpack.conf.js', - ## [temporarily disabled] REPO_ROOT / 'lms/static/karma_lms.conf.js', - REPO_ROOT / 'xmodule/js/karma_xmodule.conf.js', - ## [temporarily disabled] REPO_ROOT / 'xmodule/js/karma_xmodule_webpack.conf.js', - REPO_ROOT / 'common/static/karma_common.conf.js', - REPO_ROOT / 'common/static/karma_common_requirejs.conf.js', - ] - - JS_TEST_ID_KEYS = [ - 'cms', - 'cms-squire', - ## [temporarily-disabled] 'cms-webpack', - ## [temporarily-disabled] 'lms', - 'xmodule', - ## [temporarily-disabled] 'xmodule-webpack', - 'common', - 'common-requirejs', - 'jest-snapshot' - ] - - JS_REPORT_DIR = REPORT_DIR / 'javascript' - - # Directories used for pavelib/ tests - IGNORED_TEST_DIRS = ('__pycache__', '.cache', '.pytest_cache') - LIB_TEST_DIRS = [path("pavelib/paver_tests"), path("scripts/xsslint/tests")] - - # Directory for i18n test reports - I18N_REPORT_DIR = REPORT_DIR / 'i18n' - - # Directory for keeping src folder that comes with pip installation. - # Setting this is equivalent to passing `--src ` to pip directly. - PIP_SRC = os.environ.get("PIP_SRC") - - # Service variant (lms, cms, etc.) configured with an environment variable - # We use this to determine which envs.json file to load. - SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', None) - - # If service variant not configured in env, then pass the correct - # environment for lms / cms - if not SERVICE_VARIANT: # this will intentionally catch ""; - if any(i in sys.argv[1:] for i in ('cms', 'studio')): - SERVICE_VARIANT = 'cms' - else: - SERVICE_VARIANT = 'lms' - - @classmethod - def get_django_settings(cls, django_settings, system, settings=None, print_setting_args=None): - """ - Interrogate Django environment for specific settings values - :param django_settings: list of django settings values to get - :param system: the django app to use when asking for the setting (lms | cms) - :param settings: the settings file to use when asking for the value - :param print_setting_args: the additional arguments to send to print_settings - :return: unicode value of the django setting - """ - if not settings: - settings = os.environ.get("EDX_PLATFORM_SETTINGS", "aws") - log_dir = os.path.dirname(cls.PRINT_SETTINGS_LOG_FILE) - if not os.path.exists(log_dir): - os.makedirs(log_dir) - settings_length = len(django_settings) - django_settings = ' '.join(django_settings) # parse_known_args makes a list again - print_setting_args = ' '.join(print_setting_args or []) - try: - value = sh( - django_cmd( - system, - settings, - "print_setting {django_settings} 2>{log_file} {print_setting_args}".format( - django_settings=django_settings, - print_setting_args=print_setting_args, - log_file=cls.PRINT_SETTINGS_LOG_FILE - ).strip() - ), - capture=True - ) - # else for cases where values are not found & sh returns one None value - return tuple(str(value).splitlines()) if value else tuple(None for _ in range(settings_length)) - except BuildFailure: - print(f"Unable to print the value of the {django_settings} setting:") - with open(cls.PRINT_SETTINGS_LOG_FILE) as f: - print(f.read()) - sys.exit(1) - - @classmethod - def get_django_json_settings(cls, django_settings, system, settings=None): - """ - Interrogate Django environment for specific settings value - :param django_settings: list of django settings values to get - :param system: the django app to use when asking for the setting (lms | cms) - :param settings: the settings file to use when asking for the value - :return: json string value of the django setting - """ - return cls.get_django_settings( - django_settings, - system, - settings=settings, - print_setting_args=["--json"], - ) - - @classmethod - def covered_modules(cls): - """ - List the source modules listed in .coveragerc for which coverage - will be measured. - """ - coveragerc = configparser.RawConfigParser() - coveragerc.read(cls.PYTHON_COVERAGERC) - modules = coveragerc.get('run', 'source') - result = [] - for module in modules.split('\n'): - module = module.strip() - if module: - result.append(module) - return result - - @lazy - def env_tokens(self): - """ - Return a dict of environment settings. - If we couldn't find the JSON file, issue a warning and return an empty dict. - """ - - # Find the env JSON file - if self.SERVICE_VARIANT: - env_path = self.REPO_ROOT.parent / f"{self.SERVICE_VARIANT}.env.json" - else: - env_path = path("env.json").abspath() - - # If the file does not exist, here or one level up, - # issue a warning and return an empty dict - if not env_path.isfile(): - env_path = env_path.parent.parent / env_path.basename() - if not env_path.isfile(): - print( - "Warning: could not find environment JSON file " - "at '{path}'".format(path=env_path), - file=sys.stderr, - ) - return {} - - # Otherwise, load the file as JSON and return the resulting dict - try: - with open(env_path) as env_file: - return json.load(env_file) - - except ValueError: - print( - "Error: Could not parse JSON " - "in {path}".format(path=env_path), - file=sys.stderr, - ) - sys.exit(1) - - @lazy - def feature_flags(self): - """ - Return a dictionary of feature flags configured by the environment. - """ - return self.env_tokens.get('FEATURES', {}) - - @classmethod - def rsync_dirs(cls): - """ - List the directories that should be synced during pytest-xdist - execution. Needs to include all modules for which coverage is - measured, not just the tests being run. - """ - result = set() - for module in cls.covered_modules(): - result.add(module.split('/')[0]) - return result diff --git a/pavelib/utils/process.py b/pavelib/utils/process.py deleted file mode 100644 index da2dafa8803d..000000000000 --- a/pavelib/utils/process.py +++ /dev/null @@ -1,121 +0,0 @@ -""" -Helper functions for managing processes. -""" - - -import atexit -import os -import signal -import subprocess -import sys - -import psutil -from paver import tasks - - -def kill_process(proc): - """ - Kill the process `proc` created with `subprocess`. - """ - p1_group = psutil.Process(proc.pid) - child_pids = p1_group.children(recursive=True) - - for child_pid in child_pids: - os.kill(child_pid.pid, signal.SIGKILL) - - -def run_multi_processes(cmd_list, out_log=None, err_log=None): - """ - Run each shell command in `cmd_list` in a separate process, - piping stdout to `out_log` (a path) and stderr to `err_log` (also a path). - - Terminates the processes on CTRL-C and ensures the processes are killed - if an error occurs. - """ - kwargs = {'shell': True, 'cwd': None} - pids = [] - - if out_log: - out_log_file = open(out_log, 'w') # lint-amnesty, pylint: disable=consider-using-with - kwargs['stdout'] = out_log_file - - if err_log: - err_log_file = open(err_log, 'w') # lint-amnesty, pylint: disable=consider-using-with - kwargs['stderr'] = err_log_file - - # If the user is performing a dry run of a task, then just log - # the command strings and return so that no destructive operations - # are performed. - if tasks.environment.dry_run: - for cmd in cmd_list: - tasks.environment.info(cmd) - return - - try: - for cmd in cmd_list: - pids.extend([subprocess.Popen(cmd, **kwargs)]) - - # pylint: disable=unused-argument - def _signal_handler(*args): - """ - What to do when process is ended - """ - print("\nEnding...") - - signal.signal(signal.SIGINT, _signal_handler) - print("Enter CTL-C to end") - signal.pause() - print("Processes ending") - - # pylint: disable=broad-except - except Exception as err: - print(f"Error running process {err}", file=sys.stderr) - - finally: - for pid in pids: - kill_process(pid) - - -def run_process(cmd, out_log=None, err_log=None): - """ - Run the shell command `cmd` in a separate process, - piping stdout to `out_log` (a path) and stderr to `err_log` (also a path). - - Terminates the process on CTRL-C or if an error occurs. - """ - return run_multi_processes([cmd], out_log=out_log, err_log=err_log) - - -def run_background_process(cmd, out_log=None, err_log=None, cwd=None): - """ - Runs a command as a background process. Sends SIGINT at exit. - """ - - kwargs = {'shell': True, 'cwd': cwd} - if out_log: - out_log_file = open(out_log, 'w') # lint-amnesty, pylint: disable=consider-using-with - kwargs['stdout'] = out_log_file - - if err_log: - err_log_file = open(err_log, 'w') # lint-amnesty, pylint: disable=consider-using-with - kwargs['stderr'] = err_log_file - - proc = subprocess.Popen(cmd, **kwargs) # lint-amnesty, pylint: disable=consider-using-with - - def exit_handler(): - """ - Send SIGINT to the process's children. This is important - for running commands under coverage, as coverage will not - produce the correct artifacts if the child process isn't - killed properly. - """ - p1_group = psutil.Process(proc.pid) - child_pids = p1_group.children(recursive=True) - - for child_pid in child_pids: - os.kill(child_pid.pid, signal.SIGINT) - - # Wait for process to actually finish - proc.wait() - - atexit.register(exit_handler) diff --git a/pavelib/utils/test/__init__.py b/pavelib/utils/test/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/pavelib/utils/timer.py b/pavelib/utils/timer.py deleted file mode 100644 index fc6f3003736a..000000000000 --- a/pavelib/utils/timer.py +++ /dev/null @@ -1,83 +0,0 @@ -""" -Tools for timing paver tasks -""" - - -import json -import logging -import os -import sys -import traceback -from datetime import datetime -from os.path import dirname, exists - -import wrapt - -LOGGER = logging.getLogger(__file__) -PAVER_TIMER_LOG = os.environ.get('PAVER_TIMER_LOG') - - -@wrapt.decorator -def timed(wrapped, instance, args, kwargs): # pylint: disable=unused-argument - """ - Log execution time for a function to a log file. - - Logging is only actually executed if the PAVER_TIMER_LOG environment variable - is set. That variable is expanded for the current user and current - environment variables. It also can have :meth:`~Datetime.strftime` format - identifiers which are substituted using the time when the task started. - - For example, ``PAVER_TIMER_LOG='~/.paver.logs/%Y-%d-%m.log'`` will create a new - log file every day containing reconds for paver tasks run that day, and - will put those log files in the ``.paver.logs`` directory inside the users - home. - - Must be earlier in the decorator stack than the paver task declaration. - """ - start = datetime.utcnow() - exception_info = {} - try: - return wrapped(*args, **kwargs) - except Exception as exc: - exception_info = { - 'exception': "".join(traceback.format_exception_only(type(exc), exc)).strip() - } - raise - finally: - end = datetime.utcnow() - - # N.B. This is intended to provide a consistent interface and message format - # across all of Open edX tooling, so it deliberately eschews standard - # python logging infrastructure. - if PAVER_TIMER_LOG is not None: - - log_path = start.strftime(PAVER_TIMER_LOG) - - log_message = { - 'python_version': sys.version, - 'task': f"{wrapped.__module__}.{wrapped.__name__}", - 'args': [repr(arg) for arg in args], - 'kwargs': {key: repr(value) for key, value in kwargs.items()}, - 'started_at': start.isoformat(' '), - 'ended_at': end.isoformat(' '), - 'duration': (end - start).total_seconds(), - } - log_message.update(exception_info) - - try: - log_dir = dirname(log_path) - if log_dir and not exists(log_dir): - os.makedirs(log_dir) - - with open(log_path, 'a') as outfile: - json.dump( - log_message, - outfile, - separators=(',', ':'), - sort_keys=True, - ) - outfile.write('\n') - except OSError: - # Squelch OSErrors, because we expect them and they shouldn't - # interrupt the rest of the process. - LOGGER.exception("Unable to write timing logs") diff --git a/pavement.py b/pavement.py deleted file mode 100644 index 41a6227dbfb5..000000000000 --- a/pavement.py +++ /dev/null @@ -1,12 +0,0 @@ -import sys # lint-amnesty, pylint: disable=django-not-configured, missing-module-docstring -import os - -# Ensure that we can import pavelib, and that our copy of pavelib -# takes precedence over anything else installed in the virtualenv. -# In local dev, we usually don't need to do this, because Python -# automatically puts the current working directory on the system path. -# Until we re-run pip install, the other copies of edx-platform could -# take precedence, leading to some strange results. -sys.path.insert(0, os.path.dirname(__file__)) - -from pavelib import * # lint-amnesty, pylint: disable=wildcard-import, wrong-import-position diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index f95ae8adb2c9..7c3d52403590 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -103,7 +103,6 @@ celery==5.4.0 # openedx-learning certifi==2024.8.30 # via - # -r requirements/edx/paver.txt # elasticsearch # py2neo # requests @@ -118,7 +117,6 @@ chardet==5.2.0 charset-normalizer==2.0.12 # via # -c requirements/edx/../constraints.txt - # -r requirements/edx/paver.txt # requests # snowflake-connector-python chem==1.3.0 @@ -393,9 +391,7 @@ djangorestframework==3.14.0 djangorestframework-xml==2.0.0 # via edx-enterprise dnspython==2.7.0 - # via - # -r requirements/edx/paver.txt - # pymongo + # via pymongo done-xblock==2.4.0 # via -r requirements/edx/bundled.in drf-jwt==1.19.2 @@ -493,7 +489,6 @@ edx-name-affirmation==3.0.1 edx-opaque-keys[django]==2.11.0 # via # -r requirements/edx/kernel.in - # -r requirements/edx/paver.txt # edx-bulk-grades # edx-ccx-keys # edx-completion @@ -652,7 +647,6 @@ icalendar==6.1.0 # via -r requirements/edx/kernel.in idna==3.10 # via - # -r requirements/edx/paver.txt # optimizely-sdk # requests # snowflake-connector-python @@ -704,15 +698,10 @@ laboratory==1.0.2 # via -r requirements/edx/kernel.in lazy==1.6 # via - # -r requirements/edx/paver.txt # acid-xblock # lti-consumer-xblock # ora2 # xblock -libsass==0.10.0 - # via - # -c requirements/edx/../constraints.txt - # -r requirements/edx/paver.txt loremipsum==1.0.5 # via ora2 lti-consumer-xblock==9.12.0 @@ -750,7 +739,6 @@ markdown==3.3.7 # xblock-poll markupsafe==3.0.2 # via - # -r requirements/edx/paver.txt # chem # jinja2 # mako @@ -762,8 +750,6 @@ meilisearch==0.33.0 # via # -r requirements/edx/kernel.in # edx-search -mock==5.1.0 - # via -r requirements/edx/paver.txt mongoengine==0.29.1 # via -r requirements/edx/kernel.in monotonic==1.6 @@ -869,7 +855,6 @@ path==16.11.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/kernel.in - # -r requirements/edx/paver.txt # edx-i18n-tools # path-py path-py==12.5.0 @@ -877,12 +862,8 @@ path-py==12.5.0 # edx-enterprise # ora2 # staff-graded-xblock -paver==1.3.4 - # via -r requirements/edx/paver.txt pbr==6.1.0 - # via - # -r requirements/edx/paver.txt - # stevedore + # via stevedore pgpy==0.6.0 # via edx-enterprise piexif==1.1.3 @@ -917,7 +898,7 @@ protobuf==5.29.1 # proto-plus psutil==6.1.0 # via - # -r requirements/edx/paver.txt + # -r requirements/edx/kernel.in # edx-django-utils py2neo @ https://github.com/overhangio/py2neo/releases/download/2021.2.3/py2neo-2021.2.3.tar.gz # via @@ -945,9 +926,7 @@ pydantic==2.10.3 pydantic-core==2.27.1 # via pydantic pygments==2.18.0 - # via - # -r requirements/edx/bundled.in - # py2neo + # via py2neo pyjwkest==1.4.2 # via # -r requirements/edx/kernel.in @@ -970,12 +949,11 @@ pylatexenc==2.10 pylti1p3==2.0.0 # via -r requirements/edx/kernel.in pymemcache==4.0.0 - # via -r requirements/edx/paver.txt + # via -r requirements/edx/kernel.in pymongo==4.4.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/kernel.in - # -r requirements/edx/paver.txt # edx-opaque-keys # event-tracking # mongoengine @@ -1017,8 +995,6 @@ python-dateutil==2.9.0.post0 # xblock python-ipware==3.0.0 # via django-ipware -python-memcached==1.62 - # via -r requirements/edx/paver.txt python-slugify==8.0.4 # via code-annotations python-swiftclient==4.6.0 @@ -1074,7 +1050,6 @@ regex==2024.11.6 # via nltk requests==2.32.3 # via - # -r requirements/edx/paver.txt # algoliasearch # analytics-python # cachecontrol @@ -1138,7 +1113,6 @@ simplejson==3.19.3 six==1.17.0 # via # -r requirements/edx/kernel.in - # -r requirements/edx/paver.txt # analytics-python # codejail-includes # crowdsourcehinter-xblock @@ -1154,9 +1128,7 @@ six==1.17.0 # fs-s3fs # html5lib # interchange - # libsass # optimizely-sdk - # paver # py2neo # pyjwkest # python-dateutil @@ -1194,7 +1166,6 @@ staff-graded-xblock==2.3.0 stevedore==5.4.0 # via # -r requirements/edx/kernel.in - # -r requirements/edx/paver.txt # code-annotations # edx-ace # edx-django-utils @@ -1218,7 +1189,6 @@ tqdm==4.67.1 # openai typing-extensions==4.12.2 # via - # -r requirements/edx/paver.txt # django-countries # edx-opaque-keys # jwcrypto @@ -1242,7 +1212,6 @@ uritemplate==4.1.1 # google-api-python-client urllib3==2.2.3 # via - # -r requirements/edx/paver.txt # botocore # elasticsearch # py2neo @@ -1258,8 +1227,6 @@ voluptuous==0.15.2 # via ora2 walrus==0.9.4 # via edx-event-bus-redis -watchdog==6.0.0 - # via -r requirements/edx/paver.txt wcwidth==0.2.13 # via prompt-toolkit web-fragments==2.2.0 @@ -1280,7 +1247,7 @@ webob==1.8.9 # -r requirements/edx/kernel.in # xblock wrapt==1.17.0 - # via -r requirements/edx/paver.txt + # via -r requirements/edx/kernel.in xblock[django]==5.1.0 # via # -r requirements/edx/kernel.in diff --git a/requirements/edx/bundled.in b/requirements/edx/bundled.in index a9394b809f55..61f6007aa507 100644 --- a/requirements/edx/bundled.in +++ b/requirements/edx/bundled.in @@ -25,7 +25,6 @@ # Follow up issue to remove this fork: https://github.com/openedx/edx-platform/issues/33456 https://github.com/overhangio/py2neo/releases/download/2021.2.3/py2neo-2021.2.3.tar.gz -pygments # Used to support colors in paver command output # i18n_tool is needed at build time for pulling translations edx-i18n-tools>=0.4.6 # Commands for developers and translators to extract, compile and validate translations diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 75c533947407..eb98844d15f1 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -1184,8 +1184,6 @@ libsass==0.10.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/assets.txt - # -r requirements/edx/doc.txt - # -r requirements/edx/testing.txt loremipsum==1.0.5 # via # -r requirements/edx/doc.txt @@ -1263,9 +1261,7 @@ mistune==3.0.2 # -r requirements/edx/doc.txt # sphinx-mdinclude mock==5.1.0 - # via - # -r requirements/edx/doc.txt - # -r requirements/edx/testing.txt + # via -r requirements/edx/testing.txt mongoengine==0.29.1 # via # -r requirements/edx/doc.txt @@ -1458,10 +1454,6 @@ path-py==12.5.0 # edx-enterprise # ora2 # staff-graded-xblock -paver==1.3.4 - # via - # -r requirements/edx/doc.txt - # -r requirements/edx/testing.txt pbr==6.1.0 # via # -r requirements/edx/doc.txt @@ -1764,10 +1756,6 @@ python-ipware==3.0.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # django-ipware -python-memcached==1.62 - # via - # -r requirements/edx/doc.txt - # -r requirements/edx/testing.txt python-slugify==8.0.4 # via # -r requirements/edx/doc.txt @@ -1962,7 +1950,6 @@ six==1.17.0 # libsass # optimizely-sdk # pact-python - # paver # py2neo # pyjwkest # python-dateutil @@ -2228,10 +2215,7 @@ walrus==0.9.4 # -r requirements/edx/testing.txt # edx-event-bus-redis watchdog==6.0.0 - # via - # -r requirements/edx/development.in - # -r requirements/edx/doc.txt - # -r requirements/edx/testing.txt + # via -r requirements/edx/development.in wcwidth==0.2.13 # via # -r requirements/edx/doc.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index f7031d349784..3e95e6212051 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -855,10 +855,6 @@ lazy==1.6 # xblock lazy-object-proxy==1.10.0 # via astroid -libsass==0.10.0 - # via - # -c requirements/edx/../constraints.txt - # -r requirements/edx/base.txt loremipsum==1.0.5 # via # -r requirements/edx/base.txt @@ -916,8 +912,6 @@ meilisearch==0.33.0 # edx-search mistune==3.0.2 # via sphinx-mdinclude -mock==5.1.0 - # via -r requirements/edx/base.txt mongoengine==0.29.1 # via -r requirements/edx/base.txt monotonic==1.6 @@ -1052,8 +1046,6 @@ path-py==12.5.0 # edx-enterprise # ora2 # staff-graded-xblock -paver==1.3.4 - # via -r requirements/edx/base.txt pbr==6.1.0 # via # -r requirements/edx/base.txt @@ -1228,8 +1220,6 @@ python-ipware==3.0.0 # via # -r requirements/edx/base.txt # django-ipware -python-memcached==1.62 - # via -r requirements/edx/base.txt python-slugify==8.0.4 # via # -r requirements/edx/base.txt @@ -1384,9 +1374,7 @@ six==1.17.0 # fs-s3fs # html5lib # interchange - # libsass # optimizely-sdk - # paver # py2neo # pyjwkest # python-dateutil @@ -1559,8 +1547,6 @@ walrus==0.9.4 # via # -r requirements/edx/base.txt # edx-event-bus-redis -watchdog==6.0.0 - # via -r requirements/edx/base.txt wcwidth==0.2.13 # via # -r requirements/edx/base.txt diff --git a/requirements/edx/kernel.in b/requirements/edx/kernel.in index 60f49c5917e1..d2ec04314801 100644 --- a/requirements/edx/kernel.in +++ b/requirements/edx/kernel.in @@ -3,7 +3,6 @@ -c ../constraints.txt -r github.in # Forks and other dependencies not yet on PyPI --r paver.txt # Requirements for running paver commands # DON'T JUST ADD NEW DEPENDENCIES!!! # Please follow these guidelines whenever you change this file: @@ -126,6 +125,7 @@ openedx-django-wiki path piexif # Exif image metadata manipulation, used in the profile_images app Pillow # Image manipulation library; used for course assets, profile images, invoice PDFs, etc. +psutil # Library for retrieving information on running processes and system utilization pycountry pycryptodomex pyjwkest @@ -133,6 +133,7 @@ pyjwkest # PyJWT 1.6.3 contains PyJWTError, which is required by Apple auth in social-auth-core PyJWT>=1.6.3 pylti1p3 # Required by content_libraries core library to support LTI 1.3 launches +pymemcache # Python interface to the memcached memory cache daemon pymongo # MongoDB driver pynliner # Inlines CSS styles into HTML for email notifications python-dateutil @@ -159,5 +160,6 @@ unicodecsv # Easier support for CSV files with unicode user-util # Functionality for retiring users (GDPR compliance) webob web-fragments # Provides the ability to render fragments of web pages +wrapt # Better functools.wrapped. TODO: functools has since improved, maybe we can switch? XBlock[django] # Courseware component architecture xss-utils # https://github.com/openedx/edx-platform/pull/20633 Fix XSS via Translations diff --git a/requirements/edx/paver.in b/requirements/edx/paver.in deleted file mode 100644 index 6987ede82275..000000000000 --- a/requirements/edx/paver.in +++ /dev/null @@ -1,27 +0,0 @@ -# Requirements to run and test Paver -# -# DON'T JUST ADD NEW DEPENDENCIES!!! -# -# If you open a pull request that adds a new dependency, you should: -# * verify that the dependency has a license compatible with AGPLv3 -# * confirm that it has no system requirements beyond what we already install -# * run "make upgrade" to update the detailed requirements files -# - --c ../constraints.txt - -edx-opaque-keys # Create and introspect course and xblock identities -lazy # Lazily-evaluated attributes for Python objects -libsass # Python bindings for the LibSass CSS compiler -markupsafe # XML/HTML/XHTML Markup safe strings -mock # Stub out code with mock objects and make assertions about how they have been used -path # Easier manipulation of filesystem paths -paver # Build, distribution and deployment scripting tool -psutil # Library for retrieving information on running processes and system utilization -pymongo # via edx-opaque-keys -python-memcached # Python interface to the memcached memory cache daemon -pymemcache # Python interface to the memcached memory cache daemon -requests # Simple interface for making HTTP requests -stevedore # Support for runtime plugins, used for XBlocks and edx-platform Django app plugins -watchdog # Used in paver watch_assets -wrapt # Decorator utilities used in the @timed paver task decorator diff --git a/requirements/edx/paver.txt b/requirements/edx/paver.txt deleted file mode 100644 index c9ee8f3aff49..000000000000 --- a/requirements/edx/paver.txt +++ /dev/null @@ -1,65 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# make upgrade -# -certifi==2024.8.30 - # via requests -charset-normalizer==2.0.12 - # via - # -c requirements/edx/../constraints.txt - # requests -dnspython==2.7.0 - # via pymongo -edx-opaque-keys==2.11.0 - # via -r requirements/edx/paver.in -idna==3.10 - # via requests -lazy==1.6 - # via -r requirements/edx/paver.in -libsass==0.10.0 - # via - # -c requirements/edx/../constraints.txt - # -r requirements/edx/paver.in -markupsafe==3.0.2 - # via -r requirements/edx/paver.in -mock==5.1.0 - # via -r requirements/edx/paver.in -path==16.11.0 - # via - # -c requirements/edx/../constraints.txt - # -r requirements/edx/paver.in -paver==1.3.4 - # via -r requirements/edx/paver.in -pbr==6.1.0 - # via stevedore -psutil==6.1.0 - # via -r requirements/edx/paver.in -pymemcache==4.0.0 - # via -r requirements/edx/paver.in -pymongo==4.4.0 - # via - # -c requirements/edx/../constraints.txt - # -r requirements/edx/paver.in - # edx-opaque-keys -python-memcached==1.62 - # via -r requirements/edx/paver.in -requests==2.32.3 - # via -r requirements/edx/paver.in -six==1.17.0 - # via - # libsass - # paver -stevedore==5.4.0 - # via - # -r requirements/edx/paver.in - # edx-opaque-keys -typing-extensions==4.12.2 - # via edx-opaque-keys -urllib3==2.2.3 - # via requests -watchdog==6.0.0 - # via -r requirements/edx/paver.in -wrapt==1.17.0 - # via -r requirements/edx/paver.in diff --git a/requirements/edx/testing.in b/requirements/edx/testing.in index b903768f4de6..cf57aeb0fc46 100644 --- a/requirements/edx/testing.in +++ b/requirements/edx/testing.in @@ -28,6 +28,7 @@ freezegun # Allows tests to mock the output of assorted datetime httpretty # Library for mocking HTTP requests, used in many tests import-linter # Tool for making assertions about which modules can import which others isort # For checking and fixing the order of imports +mock # Deprecated alias to standard library `unittest.mock` pycodestyle # Checker for compliance with the Python style guide (PEP 8) polib # Library for manipulating gettext translation files, used to test paver i18n commands pyquery # jQuery-like API for retrieving fragments of HTML and XML files in tests diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 57a0dc6341ad..275eb258b156 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -898,10 +898,6 @@ lazy==1.6 # xblock lazy-object-proxy==1.10.0 # via astroid -libsass==0.10.0 - # via - # -c requirements/edx/../constraints.txt - # -r requirements/edx/base.txt loremipsum==1.0.5 # via # -r requirements/edx/base.txt @@ -962,7 +958,7 @@ meilisearch==0.33.0 # -r requirements/edx/base.txt # edx-search mock==5.1.0 - # via -r requirements/edx/base.txt + # via -r requirements/edx/testing.in mongoengine==0.29.1 # via -r requirements/edx/base.txt monotonic==1.6 @@ -1101,8 +1097,6 @@ path-py==12.5.0 # edx-enterprise # ora2 # staff-graded-xblock -paver==1.3.4 - # via -r requirements/edx/base.txt pbr==6.1.0 # via # -r requirements/edx/base.txt @@ -1341,8 +1335,6 @@ python-ipware==3.0.0 # via # -r requirements/edx/base.txt # django-ipware -python-memcached==1.62 - # via -r requirements/edx/base.txt python-slugify==8.0.4 # via # -r requirements/edx/base.txt @@ -1498,10 +1490,8 @@ six==1.17.0 # fs-s3fs # html5lib # interchange - # libsass # optimizely-sdk # pact-python - # paver # py2neo # pyjwkest # python-dateutil @@ -1647,8 +1637,6 @@ walrus==0.9.4 # via # -r requirements/edx/base.txt # edx-event-bus-redis -watchdog==6.0.0 - # via -r requirements/edx/base.txt wcwidth==0.2.13 # via # -r requirements/edx/base.txt diff --git a/scripts/paver_autocomplete.sh b/scripts/paver_autocomplete.sh deleted file mode 100644 index 8b4e8111411c..000000000000 --- a/scripts/paver_autocomplete.sh +++ /dev/null @@ -1,89 +0,0 @@ -# shellcheck disable=all -# ^ Paver in edx-platform is on the way out -# (https://github.com/openedx/edx-platform/issues/31798) -# so we're not going to bother fixing these shellcheck -# violations. - -# Courtesy of Gregory Nicholas - -_subcommand_opts() -{ - local awkfile command cur usage - command=$1 - cur=${COMP_WORDS[COMP_CWORD]} - awkfile=/tmp/paver-option-awkscript-$$.awk - echo ' -BEGIN { - opts = ""; -} - -{ - for (i = 1; i <= NF; i = i + 1) { - # Match short options (-a, -S, -3) - # or long options (--long-option, --another_option) - # in output from paver help [subcommand] - if ($i ~ /^(-[A-Za-z0-9]|--[A-Za-z][A-Za-z0-9_-]*)/) { - opt = $i; - # remove trailing , and = characters. - match(opt, "[,=]"); - if (RSTART > 0) { - opt = substr(opt, 0, RSTART); - } - opts = opts " " opt; - } - } -} - -END { - print opts -}' > $awkfile - - usage=`paver help $command` - options=`echo "$usage"|awk -f $awkfile` - - COMPREPLY=( $(compgen -W "$options" -- "$cur") ) -} - - -_paver() -{ - local cur prev - COMPREPLY=() - # Variable to hold the current word - cur="${COMP_WORDS[COMP_CWORD]}" - prev="${COMP_WORDS[COMP_CWORD - 1]}" - - # Build a list of the available tasks from: `paver --help --quiet` - local cmds=$(paver -hq | awk '/^ ([a-zA-Z][a-zA-Z0-9_]+)/ {print $1}') - - subcmd="${COMP_WORDS[1]}" - # Generate possible matches and store them in the - # array variable COMPREPLY - - if [[ -n $subcmd ]] - then - - if [[ ${#COMP_WORDS[*]} == 3 ]] - then - _subcommand_opts $subcmd - return 0 - else - if [[ "$cur" == -* ]] - then - _subcommand_opts $subcmd - return 0 - else - COMPREPLY=( $(compgen -o nospace -- "$cur") ) - fi - fi - fi - - if [[ ${#COMP_WORDS[*]} == 2 ]] - then - COMPREPLY=( $(compgen -W "${cmds}" -- "$cur") ) - fi -} - -# Assign the auto-completion function for our command. - -complete -F _paver -o default paver