test: migrate spec-test-utils and spec tests to vitest (#6222)

* Migrate spec-test-utils and spec tests to vitest * Fix build check * Fix export path * Add config file * Fix spec tests * Fix the duplicate package script * Fix spec tests * Fix lint errors * Update the spec tests runner tasks * Fix the import for a spec test * Fix the import for a spec test * Fix spec tests * Update the task runner * Run spec test in fork not threads * Fix the config file path * Fix validator spec tests * Fix e2e tests * Make the validator spec tests run in parallel * Update the spec test runner options
ChainSafe · Jan 15, 2024 · 4996f4d · 4996f4d · github-actions · Jan 15, 2024
1 parent 88744d8
commit 4996f4d
Show file tree

Hide file tree

Showing 36 changed files with 163 additions and 92 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -321,12 +321,15 @@ jobs:
       # Run them in different steps to quickly identifying which command failed
       # Otherwise just doing `yarn test:spec` you can't tell which specific suite failed
       # many of the suites have identical names for minimal and mainnet
-      - name: Spec tests bls-general
-        run: yarn test:spec-bls-general
+      - name: Spec tests general
+        run: yarn test:spec:general
         working-directory: packages/beacon-node
+      - name: Spec tests bls
+        run: yarn test:spec:bls
+        working-directory: packages/beacon-node        
       - name: Spec tests minimal
-        run: yarn test:spec-minimal
+        run: yarn test:spec:minimal
         working-directory: packages/beacon-node
       - name: Spec tests mainnet
-        run: NODE_OPTIONS='--max-old-space-size=4096' yarn test:spec-mainnet
+        run: NODE_OPTIONS='--max-old-space-size=4096' yarn test:spec:mainnet
         working-directory: packages/beacon-node
diff --git a/packages/beacon-node/.mocharc.spec.cjs b/packages/beacon-node/.mocharc.spec.cjs
diff --git a/packages/beacon-node/.mocharc.yml b/packages/beacon-node/.mocharc.yml
diff --git a/packages/beacon-node/.nycrc.json b/packages/beacon-node/.nycrc.json
diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json
@@ -87,11 +87,11 @@
     "test:sim:withdrawals": "mocha 'test/sim/withdrawal-interop.test.ts'",
     "test:sim:blobs": "mocha 'test/sim/4844-interop.test.ts'",
     "download-spec-tests": "node --loader=ts-node/esm test/spec/downloadTests.ts",
-    "check-spec-tests": "mocha test/spec/checkCoverage.ts",
-    "test:spec-bls-general": "mocha --config .mocharc.spec.cjs 'test/spec/bls/**/*.test.ts' 'test/spec/general/**/*.test.ts'",
-    "test:spec-minimal": "LODESTAR_PRESET=minimal mocha --config .mocharc.spec.cjs 'test/spec/presets/**/*.test.ts'",
-    "test:spec-mainnet": "LODESTAR_PRESET=mainnet mocha --config .mocharc.spec.cjs 'test/spec/presets/**/*.test.ts'",
-    "test:spec": "yarn test:spec-bls-general && yarn test:spec-minimal && yarn test:spec-mainnet",
+    "test:spec:bls": "vitest --run --config vitest.config.spec.ts --dir test/spec/bls/",
+    "test:spec:general": "vitest --run --config vitest.config.spec.ts --dir test/spec/general/",
+    "test:spec:minimal": "LODESTAR_PRESET=minimal vitest --run --config vitest.config.spec.ts --dir test/spec/presets/",
+    "test:spec:mainnet": "LODESTAR_PRESET=mainnet vitest --run --config vitest.config.spec.ts --dir test/spec/presets/",
+    "test:spec": "yarn test:spec:bls && yarn test:spec:general && yarn test:spec:minimal && yarn test:spec:mainnet",
     "check-readme": "typescript-docs-verifier"
   },
   "dependencies": {

diff --git a/packages/beacon-node/test/spec/bls/index.test.ts b/packages/beacon-node/test/spec/bls/index.test.ts
@@ -1,7 +1,7 @@
 import fs from "node:fs";
 import path from "node:path";
 import jsyaml from "js-yaml";
-import {expect} from "chai";
+import {expect, describe, it} from "vitest";
 import {blsSpecTests} from "../specTestVersioning.js";
 import {readdirSyncSpec} from "../utils/specTestIterator.js";
 import {testFnByType} from "./bls.js";
@@ -35,15 +35,21 @@ for (const fnName of readdirSyncSpec(blsSpecTests.outputDir)) {
 
     const fnTestDirpath = path.join(blsSpecTests.outputDir, fnName);
     for (const testName of readdirSyncSpec(fnTestDirpath)) {
-      it(`${fnName}/${testName}`, function () {
+      // TODO: Will be removed when we remove chai/mocha eslint rules
+      // eslint-disable-next-line mocha/handle-done-callback
+      it(`${fnName}/${testName}`, function (context) {
         if (fn === "skip") {
-          this.skip();
+          // TODO: Will be removed when we remove chai/mocha eslint rules
+          // eslint-disable-next-line mocha/no-nested-tests
+          context.skip();
           return;
         }
 
         // Do not manually skip tests here, do it in the top of the file
         if (skippedTestNames.includes(testName)) {
-          this.skip();
+          // TODO: Will be removed when we remove chai/mocha eslint rules
+          // eslint-disable-next-line mocha/no-nested-tests
+          context.skip();
           return;
         }
 

diff --git a/packages/beacon-node/test/spec/downloadTests.ts b/packages/beacon-node/test/spec/downloadTests.ts
@@ -1,4 +1,4 @@
-import {downloadTests} from "@lodestar/spec-test-util";
+import {downloadTests} from "@lodestar/spec-test-util/downloadTests";
 import {ethereumConsensusSpecsTests, blsSpecTests} from "./specTestVersioning.js";
 
 /* eslint-disable no-console */

diff --git a/packages/beacon-node/test/spec/general/ssz_generic.ts b/packages/beacon-node/test/spec/general/ssz_generic.ts
@@ -1,6 +1,6 @@
 import fs from "node:fs";
 import path from "node:path";
-import {expect} from "chai";
+import {expect, it} from "vitest";
 import {TestRunnerCustom} from "../utils/types.js";
 import {parseSszGenericInvalidTestcase, parseSszGenericValidTestcase} from "../utils/sszTestCaseParser.js";
 import {runValidSszTest} from "../utils/runValidSszTest.js";

diff --git a/packages/beacon-node/test/spec/presets/epoch_processing.test.ts b/packages/beacon-node/test/spec/presets/epoch_processing.test.ts
@@ -1,5 +1,5 @@
 import path from "node:path";
-import {expect} from "chai";
+import {expect} from "vitest";
 import {
   CachedBeaconStateAllForks,
   EpochTransitionCache,

diff --git a/packages/beacon-node/test/spec/presets/fork_choice.test.ts b/packages/beacon-node/test/spec/presets/fork_choice.test.ts
@@ -1,5 +1,5 @@
 import path from "node:path";
-import {expect} from "chai";
+import {expect} from "vitest";
 import {toHexString} from "@chainsafe/ssz";
 import {BeaconStateAllForks, isExecutionStateType, signedBlockToSignedHeader} from "@lodestar/state-transition";
 import {InputType} from "@lodestar/spec-test-util";
@@ -8,7 +8,7 @@ import {phase0, allForks, bellatrix, ssz, RootHex, deneb} from "@lodestar/types"
 import {bnToNum, fromHex} from "@lodestar/utils";
 import {createBeaconConfig} from "@lodestar/config";
 import {ACTIVE_PRESET, ForkSeq, isForkBlobs} from "@lodestar/params";
-import {BeaconChain} from "../../../src/chain/index.js";
+import {BeaconChain, ChainEvent} from "../../../src/chain/index.js";
 import {ClockEvent} from "../../../src/util/clock.js";
 import {computeInclusionProof} from "../../../src/util/blobs.js";
 import {createCachedBeaconStateTest} from "../../utils/cachedBeaconState.js";
@@ -112,6 +112,9 @@ const forkChoiceTest =
           }
         );
 
+        // The handler of `ChainEvent.forkChoiceFinalized` access `db.block` and raise error if not found.
+        chain.emitter.removeAllListeners(ChainEvent.forkChoiceFinalized);
+
         const stepsLen = steps.length;
         logger.debug("Fork choice test", {steps: stepsLen});
 

diff --git a/packages/beacon-node/test/spec/presets/genesis.test.ts b/packages/beacon-node/test/spec/presets/genesis.test.ts
@@ -1,5 +1,5 @@
 import path from "node:path";
-import {expect} from "chai";
+import {expect} from "vitest";
 import {phase0, Root, ssz, TimeSeconds, allForks, deneb} from "@lodestar/types";
 import {InputType} from "@lodestar/spec-test-util";
 import {

diff --git a/packages/beacon-node/test/spec/presets/light_client/single_merkle_proof.ts b/packages/beacon-node/test/spec/presets/light_client/single_merkle_proof.ts
@@ -1,4 +1,4 @@
-import {expect} from "chai";
+import {expect} from "vitest";
 import {Tree} from "@chainsafe/persistent-merkle-tree";
 import {TreeViewDU, Type} from "@chainsafe/ssz";
 import {RootHex, ssz} from "@lodestar/types";

diff --git a/packages/beacon-node/test/spec/presets/light_client/sync.ts b/packages/beacon-node/test/spec/presets/light_client/sync.ts
@@ -1,4 +1,4 @@
-import {expect} from "chai";
+import {expect} from "vitest";
 import {init} from "@chainsafe/bls/switchable";
 import {isForkLightClient} from "@lodestar/params";
 import {altair, phase0, RootHex, Slot, ssz} from "@lodestar/types";

diff --git a/packages/beacon-node/test/spec/presets/light_client/update_ranking.ts b/packages/beacon-node/test/spec/presets/light_client/update_ranking.ts
@@ -1,4 +1,4 @@
-import {expect} from "chai";
+import {expect} from "vitest";
 import {altair, ssz, allForks} from "@lodestar/types";
 import {isForkLightClient} from "@lodestar/params";
 import {InputType} from "@lodestar/spec-test-util";

diff --git a/packages/beacon-node/test/spec/presets/merkle.test.ts b/packages/beacon-node/test/spec/presets/merkle.test.ts
@@ -1,5 +1,5 @@
 import path from "node:path";
-import {expect} from "chai";
+import {expect} from "vitest";
 import {ProofType, SingleProof, Tree} from "@chainsafe/persistent-merkle-tree";
 import {fromHexString, toHexString} from "@chainsafe/ssz";
 import {ssz} from "@lodestar/types";

diff --git a/packages/beacon-node/test/spec/presets/rewards.test.ts b/packages/beacon-node/test/spec/presets/rewards.test.ts
@@ -1,5 +1,5 @@
 import path from "node:path";
-import {expect} from "chai";
+import {expect} from "vitest";
 import {VectorCompositeType} from "@chainsafe/ssz";
 import {BeaconStateAllForks, beforeProcessEpoch} from "@lodestar/state-transition";
 import {getRewardsAndPenalties} from "@lodestar/state-transition/epoch";

diff --git a/packages/beacon-node/test/spec/presets/ssz_static.test.ts b/packages/beacon-node/test/spec/presets/ssz_static.test.ts
@@ -1,5 +1,6 @@
 import fs from "node:fs";
 import path from "node:path";
+import {it, vi} from "vitest";
 import {Type} from "@chainsafe/ssz";
 import {ssz} from "@lodestar/types";
 import {ACTIVE_PRESET, ForkName, ForkLightClient} from "@lodestar/params";
@@ -58,7 +59,7 @@ const sszStatic =
       it(testCase, function () {
         // Mainnet must deal with big full states and hash each one multiple times
         if (ACTIVE_PRESET === "mainnet") {
-          this.timeout(30 * 1000);
+          vi.setConfig({testTimeout: 30 * 1000});
         }
 
         const testData = parseSszStaticTestcase(path.join(testSuiteDirpath, testCase));

diff --git a/packages/beacon-node/test/spec/specTestVersioning.ts b/packages/beacon-node/test/spec/specTestVersioning.ts
@@ -1,6 +1,6 @@
 import path from "node:path";
 import {fileURLToPath} from "node:url";
-import {DownloadTestsOptions} from "@lodestar/spec-test-util";
+import {DownloadTestsOptions} from "@lodestar/spec-test-util/downloadTests";
 
 // WARNING! Don't move or rename this file !!!
 //

diff --git a/packages/beacon-node/test/spec/utils/expectEqualBeaconState.ts b/packages/beacon-node/test/spec/utils/expectEqualBeaconState.ts
@@ -1,4 +1,4 @@
-import {expect} from "chai";
+import {expect} from "vitest";
 import {allForks, ssz} from "@lodestar/types";
 import {ForkName} from "@lodestar/params";
 import {InputType} from "@lodestar/spec-test-util";

diff --git a/packages/beacon-node/test/spec/utils/runValidSszTest.ts b/packages/beacon-node/test/spec/utils/runValidSszTest.ts
@@ -1,4 +1,4 @@
-import {expect} from "chai";
+import {expect} from "vitest";
 import {Node} from "@chainsafe/persistent-merkle-tree";
 import {Type, CompositeType, fromHexString, toHexString} from "@chainsafe/ssz";
 

diff --git a/packages/beacon-node/test/spec/utils/specTestIterator.ts b/packages/beacon-node/test/spec/utils/specTestIterator.ts
@@ -1,5 +1,6 @@
 import fs from "node:fs";
 import path from "node:path";
+import {describe, it} from "vitest";
 import {ForkName} from "@lodestar/params";
 import {describeDirectorySpecTest} from "@lodestar/spec-test-util";
 import {RunnerType, TestRunner} from "./types.js";

diff --git a/packages/beacon-node/vitest.config.spec.ts b/packages/beacon-node/vitest.config.spec.ts
@@ -0,0 +1,19 @@
+import {defineConfig, mergeConfig} from "vitest/config";
+import vitestConfig from "../../vitest.base.config";
+
+export default mergeConfig(
+  vitestConfig,
+  defineConfig({
+    test: {
+      globalSetup: ["./test/globalSetup.ts"],
+      testTimeout: 60_000,
+      passWithNoTests: true,
+      pool: "threads",
+      poolOptions: {
+        threads: {
+          isolate: false,
+        },
+      },
+    },
+  })
+);
diff --git a/packages/spec-test-util/.mocharc.yaml b/packages/spec-test-util/.mocharc.yaml
diff --git a/packages/spec-test-util/.nycrc.json b/packages/spec-test-util/.nycrc.json
diff --git a/packages/spec-test-util/package.json b/packages/spec-test-util/package.json
@@ -9,8 +9,24 @@
   },
   "homepage": "https://github.com/ChainSafe/lodestar#readme",
   "type": "module",
-  "exports": "./lib/index.js",
+  "exports": {
+    ".": {
+      "import": "./lib/index.js"
+    },
+    "./downloadTests": {
+      "import": "./lib/downloadTests.js"
+    }
+  },
   "types": "lib/index.d.ts",
+  "typesVersions": {
+    "*": {
+      "*": [
+        "*",
+        "lib/*",
+        "lib/*/index"
+      ]
+    }
+  },
   "files": [
     "lib/**/*.js",
     "lib/**/*.js.map",
@@ -26,12 +42,13 @@
     "build": "tsc -p tsconfig.build.json",
     "build:release": "yarn clean && yarn build",
     "build:watch": "yarn run build --watch",
-    "check-build": "node -e \"(async function() { await import('./lib/index.js') })()\"",
+    "check-build": "node -e \"(async function() { await import('./lib/downloadTests.js') })()\"",
     "check-types": "tsc",
     "lint": "eslint --color --ext .ts src/ test/",
     "lint:fix": "yarn run lint --fix",
     "pretest": "yarn run check-types",
-    "test:e2e": "mocha 'test/e2e/**/*.test.ts'",
+    "test:unit": "vitest --run --passWithNoTests --dir test/unit/ --coverage",
+    "test:e2e": "vitest --run --dir test/e2e/",
     "check-readme": "typescript-docs-verifier"
   },
   "repository": {
@@ -48,8 +65,7 @@
     "@lodestar/utils": "^1.13.0",
     "async-retry": "^1.3.3",
     "axios": "^1.3.4",
-    "chai": "^4.3.7",
-    "mocha": "^10.2.0",
+    "vitest": "^1.0.2",
     "rimraf": "^4.4.1",
     "snappyjs": "^0.7.0",
     "tar": "^6.1.13"
@@ -59,7 +75,6 @@
     "@types/tar": "^6.1.4"
   },
   "peerDependencies": {
-    "chai": "^4.3.7",
-    "mocha": "^10.2.0"
+    "vitest": "^1.0.2"
   }
 }
diff --git a/packages/spec-test-util/src/single.ts b/packages/spec-test-util/src/single.ts
@@ -1,6 +1,6 @@
 import fs from "node:fs";
 import path from "node:path";
-import {expect} from "chai";
+import {describe, it, vi, expect} from "vitest";
 import {uncompress} from "snappyjs";
 import {loadYaml} from "@lodestar/utils";
 
@@ -103,7 +103,7 @@ export function describeDirectorySpecTest<TestCase extends {meta?: any}, Result>
 
   describe(name, function () {
     if (options.timeout !== undefined) {
-      this.timeout(options.timeout || "10 min");
+      vi.setConfig({testTimeout: options.timeout ?? 10 * 60 * 1000});
     }
 
     for (const testSubDirname of fs.readdirSync(testCaseDirectoryPath)) {
@@ -112,9 +112,9 @@ export function describeDirectorySpecTest<TestCase extends {meta?: any}, Result>
         continue;
       }
 
-      // Use full path here, not just `testSubDirname` to allow usage of `mocha --grep`
+      // Use full path here, not just `testSubDirname` to allow usage of `vitest --grep`
       const testName = `${name}/${testSubDirname}`;
-      it(testName, async function () {
+      it(testName, async function (context) {
         // some tests require to load meta.yaml first in order to know respective ssz types.
         const metaFilePath = path.join(testSubDirPath, "meta.yaml");
         const meta: TestCase["meta"] = fs.existsSync(metaFilePath)
@@ -124,7 +124,7 @@ export function describeDirectorySpecTest<TestCase extends {meta?: any}, Result>
         let testCase = loadInputFiles(testSubDirPath, options, meta);
         if (options.mapToTestCase) testCase = options.mapToTestCase(testCase);
         if (options.shouldSkip && options.shouldSkip(testCase, testName, 0)) {
-          this.skip();
+          context.skip();
           return;
         }
 

diff --git a/packages/spec-test-util/test/e2e/single/index.test.ts b/packages/spec-test-util/test/e2e/single/index.test.ts
@@ -1,6 +1,7 @@
 import fs from "node:fs";
 import path from "node:path";
 import {fileURLToPath} from "node:url";
+import {beforeAll, afterAll} from "vitest";
 import {ContainerType, Type} from "@chainsafe/ssz";
 import {ssz} from "@lodestar/types";
 import {describeDirectorySpecTest, InputType, loadYamlFile} from "../../../src/single.js";
@@ -31,14 +32,14 @@ const sampleContainerType = new ContainerType({
   number: ssz.UintNum64,
 });
 
-before(() => {
+beforeAll(() => {
   yamlToSSZ(path.join(__dirname, "../_test_files/single/case0/input.yaml"), sampleContainerType);
   yamlToSSZ(path.join(__dirname, "../_test_files/single/case0/output.yaml"), ssz.UintNum64);
   yamlToSSZ(path.join(__dirname, "../_test_files/single/case1/input.yaml"), sampleContainerType);
   yamlToSSZ(path.join(__dirname, "../_test_files/single/case1/output.yaml"), ssz.UintNum64);
 });
 
-after(() => {
+afterAll(() => {
   fs.unlinkSync(path.join(__dirname, "../_test_files/single/case0/input.ssz"));
   fs.unlinkSync(path.join(__dirname, "../_test_files/single/case0/output.ssz"));
   fs.unlinkSync(path.join(__dirname, "../_test_files/single/case1/input.ssz"));

diff --git a/packages/spec-test-util/test/globalSetup.ts b/packages/spec-test-util/test/globalSetup.ts
@@ -0,0 +1,2 @@
+export async function setup(): Promise<void> {}
+export async function teardown(): Promise<void> {}
diff --git a/packages/spec-test-util/vitest.config.ts b/packages/spec-test-util/vitest.config.ts
@@ -0,0 +1,11 @@
+import {defineConfig, mergeConfig} from "vitest/config";
+import vitestConfig from "../../vitest.base.config";
+
+export default mergeConfig(
+  vitestConfig,
+  defineConfig({
+    test: {
+      globalSetup: ["./test/globalSetup.ts"],
+    },
+  })
+);