Speed up CI by parallelizing more things (#1716)

* speed up CI by parallelizing more things * add node_modules caching for when package-lock hasn't changed * Wasn't quite able to do it with artifacts so instead build a composite action * simplify with matrix * use the new shared setup action where appropriate
yomidevs · Dec 28, 2024 · fc9d5ad · fc9d5ad
1 parent fd577b8
commit fc9d5ad
Show file tree

Hide file tree

Showing 6 changed files with 80 additions and 114 deletions.
diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml
@@ -0,0 +1,25 @@
+name: Setup
+description: Setup the workspace for the CI
+runs:
+  using: "composite"
+  steps:
+    - name: Setup node
+      uses: actions/setup-node@v4
+      with:
+        node-version-file: "package.json"
+
+    - name: Restore dependencies
+      id: restore-dependencies
+      uses: actions/cache@v4
+      with:
+        path: node_modules
+        key: js-depend-${{ runner.os }}-${{ hashFiles('**/package-lock.json') }}
+
+    - name: Install dependencies
+      if: steps.restore-dependencies.outputs.cache-hit != 'true'
+      shell: bash
+      run: npm ci
+
+    - name: Build third-party libraries
+      shell: bash
+      run: npm run build:libs
diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -10,89 +10,48 @@ on:
   merge_group:
 
 jobs:
-  static-analysis:
-    name: Static Analysis
+  tests:
+    name: ${{ matrix.name }}
     runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Setup node
-        uses: actions/setup-node@v4
-        with:
-          node-version-file: "package.json"
-
-      - name: Install dependencies
-        run: npm ci
+    strategy:
+      matrix:
+        include:
+          - name: JavaScript
+            test: test:js
+          - name: TypeScript (main)
+            test: test:ts:main
+          - name: TypeScript (dev)
+            test: test:ts:dev
+          - name: TypeScript (test)
+            test: test:ts:test
+          - name: TypeScript (bench)
+            test: test:ts:bench
+          - name: CSS
+            test: test:css
+          - name: HTML
+            test: test:html
+          - name: Markdown
+            test: test:md
+          - name: JSON
+            test: test:json
+          - name: Unit Tests
+            test: test:unit
+          - name: Unit Tests (options)
+            test: test:unit:options
 
-      - name: Build third-party libraries
-        run: npm run build:libs
-
-      - name: Run static analysis tests
-        run: npm run test:static-analysis
-
-  static-analysis-json:
-    name: Static Analysis (JSON)
-    runs-on: ubuntu-latest
     steps:
-      - name: Checkout
-        uses: actions/checkout@v4
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup
 
-      - name: Setup node
-        uses: actions/setup-node@v4
-        with:
-          node-version-file: "package.json"
-
-      - name: Install dependencies
-        run: npm ci
-
-      - name: Build third-party libraries
-        run: npm run build:libs
-
-      - name: Run static analysis tests
-        run: npm run test:json
-
-  unit-tests:
-    name: Unit Tests
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Setup node
-        uses: actions/setup-node@v4
-        with:
-          node-version-file: "package.json"
-
-      - name: Install dependencies
-        run: npm ci
-
-      - name: Build third-party libraries
-        run: npm run build:libs
-
-      - name: Run unit tests
-        run: npm run test:unit
-
-      - name: Run unit tests (options)
-        run: npm run test:unit:options
+      - name: Run ${{ matrix.name }} tests
+        run: npm run ${{ matrix.test }}
 
   test-build:
     name: Test Build
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Setup node
-        uses: actions/setup-node@v4
-        with:
-          node-version-file: "package.json"
-
-      - name: Install dependencies
-        run: npm ci
-
-      - name: Build Libs
-        run: npm run build:libs
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup
 
       - name: Build Legal
         run: npm run license-report:html
@@ -106,3 +65,16 @@ jobs:
           file: ext/manifest.json
           schema: "https://json.schemastore.org/chrome-manifest.json"
           fixSchemas: true
+
+  bench:
+    name: Benchmarks
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup
+
+      - name: Run Benchmarks
+        uses: CodSpeedHQ/action@513a19673a831f139e8717bf45ead67e47f00044 # v3.2
+        with:
+          token: ${{ secrets.CODSPEED_TOKEN }}
+          run: npm run bench
diff --git a/.github/workflows/create-prerelease-on-tag.yml b/.github/workflows/create-prerelease-on-tag.yml
@@ -22,6 +22,7 @@ jobs:
         with:
           node-version-file: "package.json"
 
+      # intentially do not use cache to keep the build more comprehensible and sandboxed
       - name: Install dependencies
         run: npm ci
 

diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml
@@ -26,13 +26,7 @@ jobs:
           packages: fonts-ipafont-mincho
           execute_install_scripts: true
 
-      - uses: actions/setup-node@v4
-        with:
-          cache: "npm"
-          node-version-file: "package.json"
-
-      - name: Install dependencies
-        run: npm ci
+      - uses: ./.github/actions/setup
 
       - name: Build
         run: npm run build

diff --git a/package.json b/package.json
@@ -16,9 +16,16 @@
         "build:libs": "node ./dev/bin/build-libs.js",
         "build:serve:firefox-android": "node ./dev/bin/build.js --target firefox-android && web-ext run -s ./builds/yomitan-firefox-android",
         "build:serve:kiwi-browser": "node ./dev/bin/build.js --target chrome-dev && adb shell mkdir -p /sdcard/yomitan && adb push ./builds/yomitan-chrome-dev.zip /sdcard/yomitan/yomitan-kiwi-browser.zip",
+
+
+        "//WARNING": "These are convenience functions for running from CLI but are NOT called from CI because they are not parallel. Make sure you update CI scripts if you update these.",
         "test": "npm run test:js && npm run test:ts && npm run test:css && npm run test:html && npm run test:unit && npm run test:unit:options && npm run test:json && npm run test:md && npm run test:build",
         "test:fast": "npm run test:js && npm run test:ts && npm run test:unit && npm run test:json:format",
         "test:static-analysis": "npm run test:js && npm run test:ts && npm run test:css && npm run test:html && npm run test:md",
+        "test:ts": "npm run test:ts:main && npm run test:ts:dev && npm run test:ts:test && npm run test:ts:bench",
+        "//END-WARNING": "",
+
+
         "test:js": "npx eslint . --ignore-pattern '**/*.json'",
         "test:json": "npm run test:json:format && npm run test:json:types",
         "test:json:format": "npx eslint '**/*.json'",
@@ -27,7 +34,6 @@
         "test:html": "npx html-validate \"ext/**/*.html\" \"test/**/*.html\" \"dev/**/*.html\"",
         "test:md": "prettier . --check",
         "test:md:write": "prettier . --write",
-        "test:ts": "npm run test:ts:main && npm run test:ts:dev && npm run test:ts:test && npm run test:ts:bench",
         "test:ts:main": "npx tsc --noEmit --project jsconfig.json",
         "test:ts:dev": "npx tsc --noEmit --project dev/jsconfig.json",
         "test:ts:test": "npx tsc --noEmit --project test/jsconfig.json",