fix(bb.js): (breaking change) bundles bb.js properly so that it works…

… in the browser and in node (#1855)

* Reverts prior work and builds upon original webpack.
* Bundles works and wasms into the output bundle. ~12MB but compresses
with e.g. brotli to about 1.5MB (consumers responsbility). This required
a fair bit of refactoring of the bb_wasm ts code into `base`, `main` and
`thread` to avoid what webpack saw as circular dependency.
* Manually testing and working with consumers, zero config webpack,
vite, next.js, hardhat.
* Automatically tests:
  * bb: all acir test vectors through prove_and_verify
  * bb and bb.js: all command line interface commands.
* bb.js: a single test through single and multithreaded on chrome and
webkit.
* bootstrap_docker.sh can be used to build and run these tests as per
CCI.

---------

Co-authored-by: ludamad <[email protected]>

acir_tests/.dockerignore

@@ -1,2 +1,3 @@
-acir_tests
-acir-to-bberg-circuit/target
+acir_tests*
+**/node_modules
+Dockerfile*

acir_tests/Dockerfile.bb

@@ -0,0 +1,11 @@
+FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/barretenberg-x86_64-linux-clang-assert
+
+FROM node:18-alpine
+RUN apk update && apk add git bash curl jq
+COPY --from=0 /usr/src/barretenberg/cpp/build /usr/src/barretenberg/cpp/build
+WORKDIR /usr/src/barretenberg/acir_tests
+COPY . .
+# Run every acir test through native bb build "prove_and_verify".
+RUN ./run_acir_tests.sh
+# Run 1_mul through native bb build, all_cmds flow, to test all cli args.
+RUN VERBOSE=1 FLOW=all_cmds ./run_acir_tests.sh 1_mul

acir_tests/Dockerfile.bb.js

@@ -0,0 +1,22 @@
+FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/bb.js
+
+FROM node:18
+COPY --from=0 /usr/src/barretenberg/ts /usr/src/barretenberg/ts
+RUN apt update && apt install -y lsof jq
+WORKDIR /usr/src/barretenberg/acir_tests
+# Build/install ts apps.
+COPY browser-test-app browser-test-app
+COPY headless-test headless-test
+RUN (cd browser-test-app && yarn && yarn build) && (cd headless-test && yarn && npx playwright install && npx playwright install-deps)
+COPY . .
+ENV VERBOSE=1
+# Run double_verify_proof through bb.js on node to check 512k support.
+RUN BIN=../ts/dest/node/main.js ./run_acir_tests.sh double_verify_proof
+# Run 1_mul through bb.js build, all_cmds flow, to test all cli args.
+RUN BIN=../ts/dest/node/main.js FLOW=all_cmds ./run_acir_tests.sh 1_mul
+# Run double_verify_proof through bb.js on chrome testing multi-threaded browser support.
+# TODO: Currently headless webkit doesn't seem to have shared memory so skipping multi-threaded test.
+RUN BROWSER=chrome THREAD_MODEL=mt ./run_acir_tests_browser.sh double_verify_proof
+# Run 1_mul through bb.js on chrome/webkit testing single threaded browser support.
+RUN BROWSER=chrome THREAD_MODEL=st ./run_acir_tests_browser.sh 1_mul
+RUN BROWSER=webkit THREAD_MODEL=st ./run_acir_tests_browser.sh 1_mul

acir_tests/README.md

@@ -0,0 +1,44 @@
+# Acir Test Vector Runner
+
+The aim is to verify acir tests verify through a given backend binary. "Backend binaries" can include e.g.:
+
+- bb (native CLI)
+- bb.js (typescript CLI)
+- bb.js-dev (symlink in your PATH that runs the typescript CLI via ts-node)
+- bb.js.browser (script in `headless-test` that runs a a test through bb.js in a browser instance via playwright)
+
+To run:
+
+```
+$ ./run_acir_tests.sh
+```
+
+This will clone the acir test vectors from the noir repo, and will iterate over each one, running it through the
+`../cpp/build/bin/bb` binary (by default) `prove_and_verify` command.
+
+You can substitute the backend binary using the `BIN` environment variable.
+You can turn on logging with `VERBOSE` environment variable.
+You can specify a specific test to run.
+
+```
+$ BIN=bb.js VERBOSE=1 ./run_acir_tests.sh 1_mul
+```
+
+You can use a relative path to an executable. e.g. if bb.js-dev is not symlinked into your PATH:
+
+```
+$ BIN=../ts/bb.js-dev VERBOSE=1 ./run_acir_tests.sh 1_mul
+```
+
+```
+$ BIN=./headless-test/bb.js.browser VERBOSE=1 ./run_acir_tests.sh 1_mul
+```
+
+You can specify a different testing "flow" with with `FLOW` environment variable. Flows are in the `flows` dir.
+The default flow is `prove_and_verify`, which is the quickest way to... prove and verify. It's used to test the acir
+test vectors actually all pass in whichever version of the backend is being run.
+The `all_cmds` flow tests all the supported commands on the binary. Slower, but is there to test the cli.
+
+```
+$ FLOW=all_cmds ./run_acir_tests.sh 1_mul
+```

acir_tests/browser-test-app/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "browser-test-app",
+  "version": "1.0.0",
+  "main": "index.js",
+  "license": "MIT",
+  "type": "module",
+  "scripts": {
+    "build": "rm -rf dest && webpack",
+    "serve:dest:st": "serve -n -L -p 8080 dest",
+    "serve:dest:mt": "serve -n -L -p 8080 -c ../serve.mt.json dest"
+  },
+  "devDependencies": {
+    "@aztec/bb.js": "../../ts",
+    "@types/debug": "^4.1.8",
+    "@types/pako": "^2.0.0",
+    "copy-webpack-plugin": "^11.0.0",
+    "debug": "^4.3.4",
+    "html-webpack-plugin": "^5.5.3",
+    "pako": "^2.1.0",
+    "resolve-typescript-plugin": "^2.0.1",
+    "serve": "^14.2.1",
+    "ts-loader": "^9.4.4",
+    "typescript": "^5.1.6",
+    "webpack": "^5.88.2",
+    "webpack-cli": "^5.1.4",
+    "webpack-dev-server": "^4.15.1"
+  }
+}

acir_tests/browser-test-app/serve.mt.json

@@ -0,0 +1,17 @@
+{
+  "headers": [
+    {
+      "source": "**/*",
+      "headers": [
+        {
+          "key": "Cross-Origin-Embedder-Policy",
+          "value": "require-corp"
+        },
+        {
+          "key": "Cross-Origin-Opener-Policy",
+          "value": "same-origin"
+        }
+      ]
+    }
+  ]
+}

acir_tests/browser-test-app/src/index.html

@@ -0,0 +1,9 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>My Test bb.js App</title>
+  </head>
+  <body>
+    <script src="index.js"></script>
+  </body>
+</html>

acir_tests/browser-test-app/src/index.ts

@@ -0,0 +1,76 @@
+import createDebug from "debug";
+import { inflate } from "pako";
+
+createDebug.enable("*");
+const debug = createDebug("simple_test");
+
+async function runTest(
+  bytecode: Uint8Array,
+  witness: Uint8Array,
+  threads?: number
+) {
+  const { Barretenberg, RawBuffer, Crs } = await import("@aztec/bb.js");
+  const CIRCUIT_SIZE = 2 ** 19;
+
+  debug("starting test...");
+  const api = await Barretenberg.new(threads);
+
+  // Important to init slab allocator as first thing, to ensure maximum memory efficiency.
+  await api.commonInitSlabAllocator(CIRCUIT_SIZE);
+
+  // Plus 1 needed!
+  const crs = await Crs.new(CIRCUIT_SIZE + 1);
+  await api.srsInitSrs(
+    new RawBuffer(crs.getG1Data()),
+    crs.numPoints,
+    new RawBuffer(crs.getG2Data())
+  );
+
+  const acirComposer = await api.acirNewAcirComposer(CIRCUIT_SIZE);
+  const proof = await api.acirCreateProof(
+    acirComposer,
+    bytecode,
+    witness,
+    true
+  );
+  debug(`verifying...`);
+  const verified = await api.acirVerifyProof(acirComposer, proof, true);
+  debug(`verified: ${verified}`);
+
+  await api.destroy();
+
+  debug("test complete.");
+  return verified;
+}
+
+(window as any).runTest = runTest;
+
+function base64ToUint8Array(base64: string) {
+  let binaryString = atob(base64);
+  let len = binaryString.length;
+  let bytes = new Uint8Array(len);
+  for (let i = 0; i < len; i++) {
+    bytes[i] = binaryString.charCodeAt(i);
+  }
+  return bytes;
+}
+
+// This is the 1_mul test, for triggering via the button click.
+const acir = inflate(
+  base64ToUint8Array(
+    "H4sIAAAAAAAA/+2W3W6CQBCFB7AqVak/VdM0TXmAXuzyo3BXH6Wm+P6P0G5cZCS2N5whkrCJmWUjh505zPKFRPRB5+H8/lwbQ3bt1q49e82HY+OnjbHaJUmxjwod6y8V5ccsVUl63GU602mWfkdZHBdZku3zY75XuU7iQp/SPD6p8xg014qslvbY/v7bs2o29ACnpfh+H9h8YKPL1jwbhwI5Ue059ToGN9agD5cw6UFAd0i4l18q7yHeI8UkRWuqGg6PqkaR2Gt5OErWF6StBbUvz+C1GNk4Zmu+jeUHxowh86b0yry3B3afw6LDNA7snlv/cf7Q8dlaeX9AMr0icEAr0QO4fKmNgSFVBDCmigCkGglNFC8k05QeZp8XWhkBcx4DfZGqH9pnH+hFW+To47SuyPGRzXtybKjp24KidSd03+Ro8p7gPRIlxwlwn9LkaA7psXB9Qdqtk+PUxhlb68kRo9kKORoDQ6rIcUZy5Fg2EpooXkmmKdHkOAXmPAP6IlU/tM8BdY8cA5Ihxyc278mxoWZgC4rWndN9k6PJe473SJQc59QdcjSH9Ey4viDt1slxYeOSrfXkiNFshRyNgSFV5LgkOXIsGwlNFG8k05RoclwAc14CfZGqH9rnFXWPHFckQ47PbN6TY0PNlS0oWndN902OJu813iNRclxTd8jRHNJL4fqCtFsnx42NW7bWkyNGsxVyNAaGVJHjluTIsWwkNFG8k0xToslxA8x5C/RFqn4u2GcPmDOwfoofTi5df4zq4we8wQQCRCoAAA=="
+  )
+);
+
+const witness = inflate(
+  base64ToUint8Array(
+    "H4sIAAAAAAAC/63UR84DIQyG4b/3mqooinIFG2xsdrkKTOD+R0ibRfb5kEbD6hF6BV7fXdb98duNe7ptxQecJY8Ai3Ph0//pyoqURJqFxpELhVxdSbQmZ2d13QePsbm45ZqNMkts3DWHPpLP1+caMW1myU1tqKmIa6DkWuOgQpJT49KHbl3Z9928t+LhbMoLrhe9Aq03nDW8A9t/ANt/Ant9Aa1vmJXpB9j+F9j+D9jrH2hNQJYep84U2H4GbD8H9loArSVw3q+A82sNfI8b4P3aAnsdAI07wlwMCAAA"
+  )
+);
+
+document.addEventListener("DOMContentLoaded", function () {
+  const button = document.createElement("button");
+  button.innerText = "Run Test";
+  button.addEventListener("click", () => runTest(acir, witness));
+  document.body.appendChild(button);
+});

acir_tests/browser-test-app/tsconfig.json

@@ -0,0 +1,21 @@
+{
+  "compilerOptions": {
+    "target": "es2020",
+    "lib": ["dom", "esnext", "es2017.object"],
+    "module": "NodeNext",
+    "strict": true,
+    "declaration": true,
+    "allowSyntheticDefaultImports": true,
+    "esModuleInterop": true,
+    "downlevelIteration": true,
+    "inlineSourceMap": true,
+    "declarationMap": true,
+    "importHelpers": true,
+    "resolveJsonModule": true,
+    "composite": true,
+    "outDir": "dest",
+    "rootDir": "src",
+    "tsBuildInfoFile": ".tsbuildinfo"
+  },
+  "include": ["src"]
+}

acir_tests/browser-test-app/webpack.config.js

@@ -0,0 +1,45 @@
+import { resolve, dirname } from "path";
+import { fileURLToPath } from "url";
+import ResolveTypeScriptPlugin from "resolve-typescript-plugin";
+import CopyWebpackPlugin from "copy-webpack-plugin";
+import HtmlWebpackPlugin from "html-webpack-plugin";
+import webpack from "webpack";
+
+export default {
+  target: "web",
+  mode: "production",
+  entry: {
+    index: "./src/index.ts",
+  },
+  module: {
+    rules: [
+      {
+        test: /\.tsx?$/,
+        use: [{ loader: "ts-loader" }],
+      },
+    ],
+  },
+  output: {
+    path: resolve(dirname(fileURLToPath(import.meta.url)), "./dest"),
+    filename: "[name].js",
+    chunkFilename: "[name].chunk.js", // This naming pattern is used for chunks produced from code-splitting.
+  },
+  plugins: [
+    new HtmlWebpackPlugin({ inject: false, template: "./src/index.html" }),
+    new webpack.DefinePlugin({ "process.env.NODE_DEBUG": false }),
+  ],
+  resolve: {
+    plugins: [new ResolveTypeScriptPlugin()],
+  },
+  devServer: {
+    hot: false,
+    client: {
+      logging: "none",
+      overlay: false,
+    },
+    headers: {
+      "Cross-Origin-Opener-Policy": "same-origin",
+      "Cross-Origin-Embedder-Policy": "require-corp",
+    },
+  },
+};