From f342886c13c028dfb1b08f650c181bb05276d03e Mon Sep 17 00:00:00 2001 From: Darius Jahandarie Date: Sun, 29 Dec 2024 00:10:51 +0900 Subject: [PATCH] Speed up CI by parallelizing more things (#1716) * speed up CI by parallelizing more things * add node_modules caching for when package-lock hasn't changed * Wasn't quite able to do it with artifacts so instead build a composite action * simplify with matrix * use the new shared setup action where appropriate --- .github/actions/setup/action.yml | 25 ++++ .github/workflows/bench.yml | 32 ----- .github/workflows/ci.yml | 120 +++++++----------- .../workflows/create-prerelease-on-tag.yml | 1 + .github/workflows/playwright.yml | 8 +- package.json | 8 +- 6 files changed, 80 insertions(+), 114 deletions(-) create mode 100644 .github/actions/setup/action.yml delete mode 100644 .github/workflows/bench.yml diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml new file mode 100644 index 0000000000..a14ac58736 --- /dev/null +++ b/.github/actions/setup/action.yml @@ -0,0 +1,25 @@ +name: Setup +description: Setup the workspace for the CI +runs: + using: "composite" + steps: + - name: Setup node + uses: actions/setup-node@v4 + with: + node-version-file: "package.json" + + - name: Restore dependencies + id: restore-dependencies + uses: actions/cache@v4 + with: + path: node_modules + key: js-depend-${{ runner.os }}-${{ hashFiles('**/package-lock.json') }} + + - name: Install dependencies + if: steps.restore-dependencies.outputs.cache-hit != 'true' + shell: bash + run: npm ci + + - name: Build third-party libraries + shell: bash + run: npm run build:libs diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml deleted file mode 100644 index e8805d5f85..0000000000 --- a/.github/workflows/bench.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: Performance Benchmarks -permissions: - contents: read -on: - push: - branches: [master] - pull_request: - workflow_dispatch: -jobs: - benchmark: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup node - uses: actions/setup-node@v4 - with: - node-version-file: "package.json" - - - name: Install dependencies - run: npm ci - - - name: Build Libs - run: npm run build:libs - - - name: Run Benchmarks - uses: CodSpeedHQ/action@v3 - with: - token: ${{ secrets.CODSPEED_TOKEN }} - run: npm run bench diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 862036dc25..78efc70eef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,89 +10,48 @@ on: merge_group: jobs: - static-analysis: - name: Static Analysis + tests: + name: ${{ matrix.name }} runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup node - uses: actions/setup-node@v4 - with: - node-version-file: "package.json" - - - name: Install dependencies - run: npm ci + strategy: + matrix: + include: + - name: JavaScript + test: test:js + - name: TypeScript (main) + test: test:ts:main + - name: TypeScript (dev) + test: test:ts:dev + - name: TypeScript (test) + test: test:ts:test + - name: TypeScript (bench) + test: test:ts:bench + - name: CSS + test: test:css + - name: HTML + test: test:html + - name: Markdown + test: test:md + - name: JSON + test: test:json + - name: Unit Tests + test: test:unit + - name: Unit Tests (options) + test: test:unit:options - - name: Build third-party libraries - run: npm run build:libs - - - name: Run static analysis tests - run: npm run test:static-analysis - - static-analysis-json: - name: Static Analysis (JSON) - runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup - - name: Setup node - uses: actions/setup-node@v4 - with: - node-version-file: "package.json" - - - name: Install dependencies - run: npm ci - - - name: Build third-party libraries - run: npm run build:libs - - - name: Run static analysis tests - run: npm run test:json - - unit-tests: - name: Unit Tests - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup node - uses: actions/setup-node@v4 - with: - node-version-file: "package.json" - - - name: Install dependencies - run: npm ci - - - name: Build third-party libraries - run: npm run build:libs - - - name: Run unit tests - run: npm run test:unit - - - name: Run unit tests (options) - run: npm run test:unit:options + - name: Run ${{ matrix.name }} tests + run: npm run ${{ matrix.test }} test-build: name: Test Build runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup node - uses: actions/setup-node@v4 - with: - node-version-file: "package.json" - - - name: Install dependencies - run: npm ci - - - name: Build Libs - run: npm run build:libs + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup - name: Build Legal run: npm run license-report:html @@ -106,3 +65,16 @@ jobs: file: ext/manifest.json schema: "https://json.schemastore.org/chrome-manifest.json" fixSchemas: true + + bench: + name: Benchmarks + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup + + - name: Run Benchmarks + uses: CodSpeedHQ/action@513a19673a831f139e8717bf45ead67e47f00044 # v3.2 + with: + token: ${{ secrets.CODSPEED_TOKEN }} + run: npm run bench diff --git a/.github/workflows/create-prerelease-on-tag.yml b/.github/workflows/create-prerelease-on-tag.yml index 19be4e7185..f86e4fd6a0 100644 --- a/.github/workflows/create-prerelease-on-tag.yml +++ b/.github/workflows/create-prerelease-on-tag.yml @@ -22,6 +22,7 @@ jobs: with: node-version-file: "package.json" + # intentially do not use cache to keep the build more comprehensible and sandboxed - name: Install dependencies run: npm ci diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index a4b57fe482..5c2f1db420 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -26,13 +26,7 @@ jobs: packages: fonts-ipafont-mincho execute_install_scripts: true - - uses: actions/setup-node@v4 - with: - cache: "npm" - node-version-file: "package.json" - - - name: Install dependencies - run: npm ci + - uses: ./.github/actions/setup - name: Build run: npm run build diff --git a/package.json b/package.json index 96b4930bbe..8647456eea 100644 --- a/package.json +++ b/package.json @@ -16,9 +16,16 @@ "build:libs": "node ./dev/bin/build-libs.js", "build:serve:firefox-android": "node ./dev/bin/build.js --target firefox-android && web-ext run -s ./builds/yomitan-firefox-android", "build:serve:kiwi-browser": "node ./dev/bin/build.js --target chrome-dev && adb shell mkdir -p /sdcard/yomitan && adb push ./builds/yomitan-chrome-dev.zip /sdcard/yomitan/yomitan-kiwi-browser.zip", + + + "//WARNING": "These are convenience functions for running from CLI but are NOT called from CI because they are not parallel. Make sure you update CI scripts if you update these.", "test": "npm run test:js && npm run test:ts && npm run test:css && npm run test:html && npm run test:unit && npm run test:unit:options && npm run test:json && npm run test:md && npm run test:build", "test:fast": "npm run test:js && npm run test:ts && npm run test:unit && npm run test:json:format", "test:static-analysis": "npm run test:js && npm run test:ts && npm run test:css && npm run test:html && npm run test:md", + "test:ts": "npm run test:ts:main && npm run test:ts:dev && npm run test:ts:test && npm run test:ts:bench", + "//END-WARNING": "", + + "test:js": "npx eslint . --ignore-pattern '**/*.json'", "test:json": "npm run test:json:format && npm run test:json:types", "test:json:format": "npx eslint '**/*.json'", @@ -27,7 +34,6 @@ "test:html": "npx html-validate \"ext/**/*.html\" \"test/**/*.html\" \"dev/**/*.html\"", "test:md": "prettier . --check", "test:md:write": "prettier . --write", - "test:ts": "npm run test:ts:main && npm run test:ts:dev && npm run test:ts:test && npm run test:ts:bench", "test:ts:main": "npx tsc --noEmit --project jsconfig.json", "test:ts:dev": "npx tsc --noEmit --project dev/jsconfig.json", "test:ts:test": "npx tsc --noEmit --project test/jsconfig.json",