build: restructure "vendor" directories to improve build

.eslintignore

@@ -1,5 +1,6 @@
-# Vendor files and generated test artifacts
+# Vendor files, copies of node_modules, and generated test artifacts
 **/vendor
+**/node_copies
 test_root/staticfiles
 
 

common/static/.gitignore

@@ -1 +1,2 @@
 xmodule
+/node_copies

common/static/common/css/vendor

@@ -0,0 +1 @@
+../../node_copies/css

common/static/common/js/vendor

@@ -0,0 +1 @@
+../../node_copies/js

pavelib/prereqs.py

@@ -138,13 +138,7 @@ def node_prereqs_installation():
     #
     # This hack should probably be left in place for at least a year.
     # See ADR 17 for more background on the transition.
-    sh("rm -rf common/static/common/js/vendor/ common/static/common/css/vendor/")
-    # At the time of this writing, the js dir has git-versioned files
-    # but the css dir does not, so the latter would have been created
-    # as root-owned (in the process of creating the vendor
-    # subdirectory). Delete it only if empty, just in case
-    # git-versioned files are added later.
-    sh("rmdir common/static/common/css || true")
+    sh("rm -rf common/static/node_copies")
 
     # NPM installs hang sporadically. Log the installation process so that we
     # determine if any packages are chronic offenders.

scripts/copy-node-modules.sh

@@ -2,10 +2,12 @@
 # Copy certain npm-installed assets from node_modules to other folders in
 # edx-platform. These assets are used by certain especially-old legacy LMS & CMS
 # frontends that are not set up to import from node_modules directly.
-# Many of the destination folders are named "vendor", because they originally
+
+# App code refers to these copied modules via symlinks named "vendor" because they originally
 # held vendored-in (directly-committed) libraries; once we moved most frontends
-# to use NPM, we decided to keep library versions in-sync by copying to the
-# former "vendor" directories.
+# to use NPM, we decided to keep library versions in-sync by copying into the former "vendor" directories.
+# Eventually, we  but established "vendor" symlinks because
+# updaing all referecnes would have been too difficult.
 
 # Enable stricter error handling.
 set -euo pipefail
@@ -18,36 +20,41 @@ COL_OFF="\e[0m"   # Normal color
 # https://github.com/openedx/wg-developer-experience/issues/150
 # https://github.com/openedx/wg-developer-experience/issues/151
 node_modules="node_modules"
-vendor_js="common/static/common/js/vendor"
-vendor_css="common/static/common/css/vendor"
+target_js="common/static/node_copies/js"
+target_css="common/static/node_copies/css"
 
 # Stylized logging.
 log ( ) {
 	echo -e "${COL_LOG}$* $COL_OFF"
 }
 
+# Print a command (prefixed with '+') and then run it, to aid in debugging.
+# This is just like `set -x`, except that `set -x` prints to STDERR, which is hidden
+# by GitHub Actions logs. This functions prints to STDOUT, which is visible.
+log_and_run ( ) {
+    log "+$*"
+    "$@"  # Joins arguments to form a command (quoting as necessary) and runs the command.
+}
+
 log "====================================================================================="
 log "Copying required assets from node_modules..."
 log "-------------------------------------------------------------------------------"
 
-# Start echoing all commands back to user for ease of debugging.
-set -x
+log "Ensuring target directories exist..."
+log_and_run mkdir -p "$target_js"
+log_and_run mkdir -p "$target_css"
 
-log "Ensuring vendor directories exist..."
-mkdir -p "$vendor_js"
-mkdir -p "$vendor_css"
-
-log "Copying studio-frontend JS & CSS from node_modules into vendor directores..."
+log "Copying studio-frontend JS & CSS from node_modules into target directores..."
 while read -r -d $'\0' src_file ; do
     if [[ "$src_file" = *.css ]] || [[ "$src_file" = *.css.map ]] ; then
-        cp --force "$src_file" "$vendor_css"
+        log_and_run cp --force "$src_file" "$target_css"
     else
-        cp --force "$src_file" "$vendor_js"
+        log_and_run cp --force "$src_file" "$target_js"
     fi
 done < <(find "$node_modules/@edx/studio-frontend/dist" -type f -print0)
 
-log "Copying certain JS modules from node_modules into vendor directory..."
-cp --force \
+log "Copying certain JS modules from node_modules into target directory..."
+log_and_run cp --force \
     "$node_modules/backbone.paginator/lib/backbone.paginator.js" \
     "$node_modules/backbone/backbone.js" \
     "$node_modules/bootstrap/dist/js/bootstrap.bundle.js" \
@@ -62,28 +69,25 @@ cp --force \
     "$node_modules/underscore.string/dist/underscore.string.js" \
     "$node_modules/underscore/underscore.js" \
     "$node_modules/which-country/index.js" \
-    "$vendor_js"
+    "$target_js"
 
-log "Copying certain JS developer modules into vendor directory..."
+log "Copying certain JS developer modules into target directory..."
 if [[ "${NODE_ENV:-production}" = development ]] ; then
-    cp --force "$node_modules/sinon/pkg/sinon.js" "$vendor_js"
-    cp --force "$node_modules/squirejs/src/Squire.js" "$vendor_js"
+    log_and_run cp --force "$node_modules/sinon/pkg/sinon.js" "$target_js"
+    log_and_run cp --force "$node_modules/squirejs/src/Squire.js" "$target_js"
 else
     # TODO: https://github.com/openedx/edx-platform/issues/31768
     # In the old implementation of this scipt (pavelib/assets.py), these two
-    # developer libraries were copied into the JS vendor directory whether not
+    # developer libraries were copied into the JS target directory whether not
     # the build was for prod or dev. In order to exactly match the output of
     # the old script, this script will also copy them in for prod builds.
     # However, in the future, it would be good to only copy them for dev
     # builds. Furthermore, these libraries should not be `npm install`ed
     # into prod builds in the first place.
-    cp --force "$node_modules/sinon/pkg/sinon.js" "$vendor_js" || true      # "|| true" means "tolerate errors"; in this case,
-    cp --force "$node_modules/squirejs/src/Squire.js" "$vendor_js" || true  # that's "tolerate if these files don't exist."
+    log_and_run cp --force "$node_modules/sinon/pkg/sinon.js" "$target_js" || true      # "|| true" means "tolerate errors"; in this case,
+    log_and_run cp --force "$node_modules/squirejs/src/Squire.js" "$target_js" || true  # that's "tolerate if these files don't exist."
 fi
 
-# Done echoing.
-set +x
-
 log "-------------------------------------------------------------------------------"
 log " Done copying required assets from node_modules."
 log "====================================================================================="

scripts/xsslint/xsslint/default_config.py

@@ -11,10 +11,10 @@
 SKIP_DIRS = (
     '.git',
     '.pycharm_helpers',
-    'common/static/xmodule/modules',
     'common/static/bundles',
     'perf_tests',
     'node_modules',
+    'node_copies',
     'reports/diff_quality',
     'scripts/xsslint',
     'spec',

scripts/xsslint_config.py

@@ -20,11 +20,11 @@
 SKIP_DIRS = (
     '.git',
     '.pycharm_helpers',
-    'common/static/xmodule/modules',
     'common/static/bundles',
     'docs',
     'perf_tests',
     'node_modules',
+    'node_copies',
     'reports/diff_quality',
     'scripts/xsslint',
     'spec',