feat(hyperchains): Hyperchain command improvements (including docker …

…related) (#273)

# What ❔

- Fixes some flow errors on the hyerpchain initializer
- Changes the hyperchain command name to a "zk stack" umbrella command
- Addes docker steup command - builds server and generates a docker
compose file

## Why ❔

- Improves the experience of starting hyperchains

## Checklist

- [X] PR title corresponds to the body of PR (we generate changelog
entries from PRs).
- [X] Tests for the changes have been added / updated.
- [X] Documentation comments have been added / updated.
- [X] Code has been formatted via `zk fmt` and `zk lint`.

.gitignore

@@ -70,3 +70,5 @@ artifacts-zk/
 cache-zk/
 zksolc
 verified_sources
+
+hyperchain-*.yml

etc/hyperchains/docker-compose-hyperchain-template

@@ -0,0 +1,144 @@
+version: '3.2'
+networks:
+  zkstack:
+    driver: bridge
+volumes:
+  prover_artifacts:
+  server_artifacts:
+services:
+  zkstack_core:
+    networks:
+      - zkstack
+    image: {{orgName}}/server-v2:latest
+    command: ["--components", "tree_new,eth,data_fetcher,state_keeper,housekeeper"]
+    env_file:
+      - {{envFilePath}}
+    environment:
+      ZKSYNC_HOME: /
+    ports: # assumes default ports in .env
+      # - "3312:3312" # prometheus metrics # we need a separate metrics port for each component
+      - "3075:3075" # proof_data_handler api
+    volumes:
+      - prover_artifacts:/etc_prover_artifacts
+      - server_artifacts:/etc_server_artifacts
+  zkstack-apis:
+    networks:
+      - zkstack
+    image: {{orgName}}/server-v2:latest
+    command: ["--components", "http_api,ws_api"]
+    env_file:
+      - {{envFilePath}}
+    environment:
+      ZKSYNC_HOME: /
+    ports: # assumes default ports in .env
+      - "3071:3071" # health
+      - "3312:3312" # prometheus metrics # we need a separate metrics port for each component
+      - "3050:3050" # http_api
+      - "3051:3051" # ws_api
+  {{#if hasProver}}
+  # System requirements for CPU proving:
+  # ~16+ CPU cores
+  # ~384+ GB RAM
+  # TODOS:
+  # - (PRO-47): Figure out how to properly set metrics ports for each service in env
+  zkstack-prover-fri-gateway:
+    networks:
+      - zkstack
+    build:
+      context: .
+      dockerfile: ./docker/prover-fri-gateway/Dockerfile
+    env_file:
+      - ./common.env
+    env:
+      FRI_PROVER_GATEWAY_API_URL: http://zkstack-core:3075
+    ports: # assumes default ports in .env
+      - "3312:3312" # prometheus metrics
+    volumes:
+      - prover_artifacts:/etc_prover_artifacts
+      - server_artifacts:/etc_server_artifacts
+  zkstack-witness-generator-basic-circuits:
+    networks:
+      - zkstack
+    build:
+      context: .
+      dockerfile: ./docker/witness-generator/Dockerfile
+    command: ["--round", "basic_circuits"]
+    env_file:
+      - {{envFilePath}}
+    ports: # assumes default ports in .env
+      - "3312:3312" # prometheus metrics
+    volumes:
+      - prover_artifacts:/etc_prover_artifacts
+      - server_artifacts:/etc_server_artifacts
+  zkstack-witness-generator-leaf-aggregation:
+    networks:
+      - zkstack
+    build:
+      context: .
+      dockerfile: ./docker/witness-generator/Dockerfile
+    command: ["--round", "leaf_aggregation"]
+    env_file:
+      - {{envFilePath}}
+    ports: # assumes default ports in .env
+      - "3312:3312" # prometheus metrics
+    volumes:
+      - prover_artifacts:/etc_prover_artifacts
+      - server_artifacts:/etc_server_artifacts
+  zkstack-witness-generator-node-aggregation:
+    networks:
+      - zkstack
+    build:
+      context: .
+      dockerfile: ./docker/witness-generator/Dockerfile
+    command: ["--round", "node_aggregation"]
+    env_file:
+      - {{envFilePath}}
+    ports: # assumes default ports in .env
+      - "3312:3312" # prometheus metrics
+    volumes:
+      - prover_artifacts:/etc_prover_artifacts
+      - server_artifacts:/etc_server_artifacts
+  zkstack-witness-generator-scheduler:
+    networks:
+      - zkstack
+    build:
+      context: .
+      dockerfile: ./docker/witness-generator/Dockerfile
+    command: ["--round", "scheduler"]
+    env_file:
+      - {{envFilePath}}
+    ports: # assumes default ports in .env
+      - "3312:3312" # prometheus metrics
+    volumes:
+      - prover_artifacts:/etc_prover_artifacts
+      - server_artifacts:/etc_server_artifacts
+  zkstack-prover-fri:
+    networks:
+      - zkstack
+    build:
+      context: .
+      dockerfile: ./docker/prover-fri/Dockerfile
+    env_file:
+      - {{envFilePath}}
+    env:
+      FRI_PROVER_SETUP_DATA_PATH: /etc/prover_setup_data
+    ports: # assumes default ports in .env
+      - "3312:3312" # prometheus metrics
+    volumes:
+      - prover_artifacts:/etc_prover_artifacts
+      - server_artifacts:/etc_server_artifacts
+      - ./prover_setup-data:/etc/prover_setup_data
+  zkstack-proof-fri-compressor:
+    networks:
+      - zkstack
+    build:
+      context: .
+      dockerfile: ./docker/proof-fri-compressor/Dockerfile
+    env_file:
+      - {{envFilePath}}
+    ports: # assumes default ports in .env
+      - "3312:3312" # prometheus metrics
+    volumes:
+      - prover_artifacts:/etc_prover_artifacts
+      - server_artifacts:/etc_server_artifacts
+  {{/if}}

infrastructure/zk/package.json

@@ -16,6 +16,7 @@
         "deep-extend": "^0.6.0",
         "dotenv": "^8.2.0",
         "ethers": "~5.5.0",
+        "handlebars": "^4.7.8",
         "node-fetch": "^2.6.1",
         "tabtab": "^3.0.2",
         "zksync-web3": "link:../../sdk/zksync-web3.js"

infrastructure/zk/src/docker.ts

@@ -25,7 +25,8 @@ async function dockerCommand(
     command: 'push' | 'build',
     image: string,
     customTag?: string,
-    publishPublic: boolean = false
+    publishPublic: boolean = false,
+    dockerOrg: string = 'matterlabs'
 ) {
     // Generating all tags for containers. We need 2 tags here: SHA and SHA+TS
     const { stdout: COMMIT_SHORT_SHA }: { stdout: string } = await utils.exec('git rev-parse --short HEAD');
@@ -52,7 +53,7 @@ async function dockerCommand(
     // COMMIT_SHORT_SHA returns with newline, so we need to trim it
     switch (command) {
         case 'build':
-            await _build(image, tagList);
+            await _build(image, tagList, dockerOrg);
             break;
         case 'push':
             await _push(image, tagList, publishPublic);
@@ -85,13 +86,14 @@ function defaultTagList(image: string, imageTagSha: string, imageTagShaTS: strin
     return tagList;
 }
 
-async function _build(image: string, tagList: string[]) {
+async function _build(image: string, tagList: string[], dockerOrg: string) {
     if (image === 'server-v2' || image === 'external-node' || image === 'prover') {
         await contract.build();
     }
+    let tagsToBuild = '';
 
-    const tagsToBuild = tagList.map((tag) => `-t matterlabs/${image}:${tag}`).join(' ');
     // generate list of tags for image - we want 3 tags (latest, SHA, SHA+TimeStamp) for listed components and only "latest" for everything else
+    tagsToBuild = tagList.map((tag) => `-t ${dockerOrg}/${image}:${tag}`).join(' ');
 
     // Conditionally add build argument if image is prover-v2
     let buildArgs = '';
@@ -138,6 +140,10 @@ export async function build(image: string, cmd: Command) {
     await dockerCommand('build', image, cmd.customTag);
 }
 
+export async function customBuildForHyperchain(image: string, dockerOrg: string) {
+    await dockerCommand('build', image, '', false, dockerOrg);
+}
+
 export async function push(image: string, cmd: Command) {
     await dockerCommand('build', image, cmd.customTag, cmd.public);
     await dockerCommand('push', image, cmd.customTag, cmd.public);