diff --git a/changelog.d/20240318_154804_kyle_assets_build.md b/changelog.d/20240318_154804_kyle_assets_build.md new file mode 100644 index 0000000000..1d38acb66e --- /dev/null +++ b/changelog.d/20240318_154804_kyle_assets_build.md @@ -0,0 +1,16 @@ +- 💥[Feature] The `openedx-assets` command is replaced with `npm run` subcommands. + This will slightly reduce build time for edx-platform assets and themes. + It will also open up the door for more significant build time reductions in the future. + Here is a migration guide, where each command is to be run in the `lms` or `cms` container. + | **Before** | **After** | + |------------------------------------------|-------------------------------------------------------------------------------------| + | `openedx-assets build --env=prod ARGS` | `npm run build -- ARGS` | + | `openedx-assets build --env=dev ARGS` | `npm run build-dev -- ARGS` | + | `openedx-assets common --env=prod ARGS` | `npm run compile-sass -- --skip-themes ARGS` | + | `openedx-assets common --env=dev ARGS` | `npm run compile-sass-dev -- --skip-themes ARGS` | + | `openedx-assets webpack --env=prod ARGS` | `npm run webpack -- ARGS` | + | `openedx-assets webpack --env=dev ARGS` | `npm run webpack-dev -- ARGS` | + | `openedx-assets npm` | `npm run postinstall` (`npm clean-install` runs this automatically) | + | `openedx-assets xmodule` | (no longer necessary) | + | `openedx-assets collect ARGS` | `./manage.py lms collecstatic --noinput ARGS && ./manage.py cms collectstatic ARGS` | + | `openedx-assets watch-themes ARGS` | `npm run watch-themes -- ARGS` | diff --git a/docs/__init__.py b/docs/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/dev.rst b/docs/dev.rst index 6b1f1975bf..216dfb3819 100644 --- a/docs/dev.rst +++ b/docs/dev.rst @@ -99,9 +99,9 @@ To open a python shell in the LMS or CMS, run:: You can then import edx-platform and django modules and execute python code. -To rebuild assets, you can use the ``openedx-assets`` command that ships with Tutor:: +To rebuild assets, you can run the ``build-dev`` NPM script that comes with edx-plaform:: - tutor dev run lms openedx-assets build --env=dev + tutor dev run lms npm run build-dev .. _specialized for developer usage: diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index ebefd7e019..65a841b66a 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -205,27 +205,22 @@ ENV PATH /openedx/bin:${PATH} {{ patch("openedx-dockerfile-pre-assets") }} -# Collect production assets. By default, only assets from the default theme +# Build & collect production assets. By default, only assets from the default theme # will be processed. This makes the docker image lighter and faster to build. -# Only the custom themes added to /openedx/themes will be compiled. -# Here, we don't run "paver update_assets" which is slow, compiles all themes -# and requires a complex settings file. Instead, we decompose the commands -# and run each one individually to collect the production static assets to -# /openedx/staticfiles. -ENV NO_PYTHON_UNINSTALL 1 -ENV NO_PREREQ_INSTALL 1 -# We need to rely on a separate openedx-assets command to accelerate asset processing. -# For instance, we don't want to run all steps of asset collection every time the theme -# is modified. -RUN openedx-assets xmodule \ - && openedx-assets npm \ - && openedx-assets webpack --env=prod \ - && openedx-assets common +RUN npm run postinstall # Postinstall artifacts are stuck in nodejs-requirements layer. Create them here too. +RUN npm run compile-sass -- --skip-themes +RUN npm run webpack + +# Now that the default theme is built, build any custom themes in /openedx/themes. COPY --chown=app:app ./themes/ /openedx/themes/ -RUN openedx-assets themes \ - && openedx-assets collect --settings=tutor.assets \ - # De-duplicate static assets with symlinks - && rdfind -makesymlinks true -followsymlinks true /openedx/staticfiles/ +RUN npm run compile-sass -- --skip-default --theme-dir /openedx/themes + +# and finally, collect assets for the production image, +# de-duping assets with symlinks. +RUN ./manage.py lms collectstatic --noinput --settings=tutor.assets && \ + ./manage.py cms collectstatic --noinput --settings=tutor.assets && \ + # De-duplicate static assets with symlinks \ + rdfind -makesymlinks true -followsymlinks true /openedx/staticfiles/ # Create a data directory, which might be used (or not) RUN mkdir /openedx/data @@ -278,7 +273,7 @@ ENV PYTHONBREAKPOINT=ipdb.set_trace # static assets, then production assets will be served instead. RUN rm -r /openedx/staticfiles && \ mkdir /openedx/staticfiles && \ - openedx-assets webpack --env=dev + npm run build-dev {{ patch("openedx-dev-dockerfile-post-python-requirements") }} diff --git a/tutor/templates/build/openedx/bin/openedx-assets b/tutor/templates/build/openedx/bin/openedx-assets deleted file mode 100755 index 1b89434d67..0000000000 --- a/tutor/templates/build/openedx/bin/openedx-assets +++ /dev/null @@ -1,218 +0,0 @@ -#! /usr/bin/env python -from __future__ import print_function -import argparse -import os -import subprocess -import sys -import traceback - -from path import Path - -from pavelib import assets - - -DEFAULT_STATIC_ROOT = "/openedx/staticfiles" -DEFAULT_THEMES_DIR = "/openedx/themes" - - -def main(): - parser = argparse.ArgumentParser( - description="Various assets processing/building/collection utility for Open edX" - ) - subparsers = parser.add_subparsers() - - npm = subparsers.add_parser("npm", help="Copy static assets from node_modules") - npm.set_defaults(func=run_npm) - - build = subparsers.add_parser("build", help="Build all assets") - build.add_argument("-e", "--env", choices=["prod", "dev"], default="prod") - build.add_argument("--theme-dirs", nargs="+", default=[DEFAULT_THEMES_DIR]) - build.add_argument("--themes", nargs="+", default=["all"]) - build.add_argument("-r", "--static-root", default=DEFAULT_STATIC_ROOT) - build.add_argument("--systems", nargs="+", default=["lms", "cms"]) - build.set_defaults(func=run_build) - - xmodule = subparsers.add_parser("xmodule", help="Process assets from xmodule") - xmodule.set_defaults(func=run_xmodule) - - webpack = subparsers.add_parser("webpack", help="Run webpack") - webpack.add_argument("-r", "--static-root", default=DEFAULT_STATIC_ROOT) - webpack.add_argument("-e", "--env", choices=["prod", "dev"], default="prod") - webpack.set_defaults(func=run_webpack) - - common = subparsers.add_parser( - "common", help="Compile static assets for common theme" - ) - common.add_argument("--systems", nargs="+", default=["lms", "cms"]) - common.set_defaults(func=run_common) - - themes = subparsers.add_parser( - "themes", help="Compile static assets for custom themes" - ) - themes.add_argument("--theme-dirs", nargs="+", default=[DEFAULT_THEMES_DIR]) - themes.add_argument("--themes", nargs="+", default=["all"]) - themes.add_argument("--systems", nargs="+", default=["lms", "cms"]) - themes.set_defaults(func=run_themes) - - collect = subparsers.add_parser( - "collect", help="Collect static assets to be served by webserver" - ) - collect.add_argument( - "-s", - "--settings", - default="tutor.assets", - help="Django settings module", - ) - collect.add_argument( - "--systems", - nargs="+", - choices=["lms", "cms"], - default=["lms", "cms"], - help="Limit collection to lms or cms", - ) - collect.set_defaults(func=run_collect) - - watch_themes = subparsers.add_parser( - "watch-themes", help="Watch theme assets for changes and recompile on-the-fly" - ) - watch_themes.add_argument( - "-e", - "--env", - choices=["prod", "dev"], - default="prod", - help="Webpack target to run", - ) - watch_themes.add_argument("--theme-dirs", default=[DEFAULT_THEMES_DIR]) - watch_themes.set_defaults(func=run_watch_themes) - - args = parser.parse_args() - args.func(args) - - -def run_build(args): - run_xmodule(args) - run_npm(args) - run_webpack(args) - run_common(args) - run_themes(args) - - -def run_xmodule(_args): - # Collecting xmodule assets is incompatible with setting the django path, because - # of an unfortunate call to settings.configure() - django_settings_module = os.environ.get("DJANGO_SETTINGS_MODULE") - if django_settings_module: - os.environ.pop("DJANGO_SETTINGS_MODULE") - - sys.argv[1:] = ["common/static/xmodule"] - import xmodule.static_content - - xmodule.static_content.main() - - if django_settings_module: - os.environ["DJANGO_SETTINGS_MODULE"] = django_settings_module - - -def run_npm(_args): - assets.process_npm_assets() - - -def run_webpack(args): - os.environ["STATIC_ROOT_LMS"] = args.static_root - os.environ["STATIC_ROOT_CMS"] = os.path.join(args.static_root, "studio") - os.environ["NODE_ENV"] = {"prod": "production", "dev": "development"}[args.env] - subprocess.check_call( - [ - "webpack", - "--progress", - "--config=webpack.{env}.config.js".format(env=args.env), - ] - ) - - -def run_common(args): - for system in args.systems: - print("Compiling {} sass assets from common theme...".format(system)) - assets._compile_sass(system, None, False, False, []) - - -def run_themes(args): - for theme_dir in args.theme_dirs: - local_themes = ( - list_subdirectories(theme_dir) if "all" in args.themes else args.themes - ) - for theme in local_themes: - theme_path = os.path.join(theme_dir, theme) - if os.path.exists(theme_path): - for system in args.systems: - print( - "Compiling {} sass assets from theme {}...".format( - system, theme_path - ) - ) - assets._compile_sass(system, Path(theme_path), False, False, []) - - -def run_collect(args): - assets.collect_assets(args.systems, args.settings) - - -def run_watch_themes(args): - """ - Watch static assets for changes and re-compile those changes when - necessary. This piece of code is heavily inspired from the - edx-platform/pavelib/assets.py:watch_assets function, which could not be - used directly because it does not properly read the platform settings - environment variable. - - Note that this function will only work for watching assets in development - mode. In production, watching changes does not make much sense anyway. - """ - observer = assets.Observer() - for theme_dir in args.theme_dirs: - print("Watching changes in {}...".format(theme_dir)) - ThemeWatcher(theme_dir).register(observer) - observer.start() - try: - while True: - observer.join(2) - except KeyboardInterrupt: - observer.stop() - - -def list_subdirectories(path): - return [ - subpath - for subpath in os.listdir(path) - if os.path.isdir(os.path.join(path, subpath)) - ] - - -class ThemeWatcher(assets.SassWatcher): - def __init__(self, theme_dir): - super(ThemeWatcher, self).__init__() - self.theme_dir = theme_dir - - # pylint: disable=arguments-differ - def register(self, observer): - return super(ThemeWatcher, self).register(observer, [self.theme_dir]) - - @assets.debounce() - def on_any_event(self, event): - components = os.path.relpath(event.src_path, self.theme_dir).split("/") - try: - theme = components[0] - system = components[1] - except IndexError: - return - try: - print("Detected change:", event.src_path) - print("\tRecompiling {} theme for {}".format(theme, system)) - assets._compile_sass(system, Path(self.theme_dir) / theme, False, False, []) - print("\tDone recompiling {} theme for {}".format(theme, system)) - except Exception: # pylint: disable=broad-except - traceback.print_exc() - - -if __name__ == "__main__": - main() diff --git a/tutor/templates/dev/docker-compose.yml b/tutor/templates/dev/docker-compose.yml index 4aa01ee56c..1575c9165a 100644 --- a/tutor/templates/dev/docker-compose.yml +++ b/tutor/templates/dev/docker-compose.yml @@ -43,7 +43,7 @@ services: # Additional service for watching theme changes watchthemes: <<: *openedx-service - command: openedx-assets watch-themes --env dev + command: npm run watch-sass restart: unless-stopped {% if RUN_ELASTICSEARCH and is_docker_rootless() %} diff --git a/tutor/templates/jobs/init/mounted-directories.sh b/tutor/templates/jobs/init/mounted-directories.sh index e10dc7b7c9..0f7c615b8b 100644 --- a/tutor/templates/jobs/init/mounted-directories.sh +++ b/tutor/templates/jobs/init/mounted-directories.sh @@ -37,7 +37,7 @@ pip install -e . npm clean-install # Regenerate static assets. -openedx-assets build --env=dev +npm run build-dev set -x echo "Done setting up bind-mounted edx-platform."