diff --git a/.github/circuits-runner/Dockerfile b/.github/circuits-runner/Dockerfile
new file mode 100644
index 00000000..fe9bcab2
--- /dev/null
+++ b/.github/circuits-runner/Dockerfile
@@ -0,0 +1,40 @@
+# Base image (Ubuntu 22.04)
+FROM ubuntu:22.04
+
+# Install required packages
+RUN apt-get update && apt-get install -y \
+ unzip wget curl sudo git build-essential \
+ && rm -rf /var/lib/apt/lists/*
+
+# Install Node.js 20
+RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \
+ apt-get install -y nodejs
+
+# Install Bun
+RUN curl -fsSL https://bun.sh/install | bash && \
+ echo 'export PATH="$HOME/.bun/bin:$PATH"' >> ~/.bashrc
+
+# Install Rust
+RUN curl https://sh.rustup.rs -sSf | sh -s -- -y
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+# Install Circom
+RUN mkdir -p /root/bin && \
+ wget https://github.com/iden3/circom/releases/download/v2.1.9/circom-linux-amd64 -O /root/bin/circom && \
+ chmod +x /root/bin/circom
+
+# Set PATH
+ENV PATH="/root/bin:${PATH}"
+
+# Install Yarn
+RUN npm install -g yarn
+
+# Install Git
+RUN apt-get update && apt-get install -y git
+
+# Add entrypoint script
+COPY entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+# Entrypoint
+ENTRYPOINT ["/entrypoint.sh"]
\ No newline at end of file
diff --git a/.github/circuits-runner/entrypoint.sh b/.github/circuits-runner/entrypoint.sh
new file mode 100644
index 00000000..c61ea058
--- /dev/null
+++ b/.github/circuits-runner/entrypoint.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+REPO_URL=${REPO_URL:-"https://github.com/your/repo.git"}
+COMMIT_HASH=${COMMIT_HASH:-"your_commit_hash"}
+
+echo "Cloning repository: $REPO_URL"
+git clone $REPO_URL repo
+cd repo
+
+echo "Checking out commit: $COMMIT_HASH"
+git checkout $COMMIT_HASH
+
+echo "Installing dependencies"
+yarn install --frozen-lockfile
+
+echo "Running tests"
+cd packages/circuits
+yarn test-large --no-cache
\ No newline at end of file
diff --git a/.github/workflows/build-fmt.yml b/.github/workflows/build-fmt.yml
index c343169c..f7008038 100644
--- a/.github/workflows/build-fmt.yml
+++ b/.github/workflows/build-fmt.yml
@@ -3,31 +3,8 @@ name: Build and Format
on: [push]
jobs:
- start-spot-vm:
- name: Start Spot VM
- runs-on: ubuntu-latest
-
- steps:
- - name: Authenticate with GCP
- uses: google-github-actions/auth@v2
- with:
- credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
-
- - name: 'Set up Cloud SDK'
- uses: 'google-github-actions/setup-gcloud@v2'
- with:
- version: '>= 363.0.0'
-
- - name: Start Spot VM
- run: gcloud compute instances start email-tx-builder-runner-spot --zone=us-central1-c
-
- - name: Wait for Spot VM to initialize
- run: |
- echo "Waiting for Spot VM to initialize..."
- sleep 60
-
build-and-format:
- runs-on: self-hosted
+ runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
@@ -71,21 +48,3 @@ jobs:
- name: Run clippy
run: cargo clippy -- -D warnings
-
- stop-spot-vm:
- name: Stop Spot VM
- needs: [build-and-format]
- runs-on: ubuntu-latest
- steps:
- - name: Authenticate with GCP
- uses: google-github-actions/auth@v2
- with:
- credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
-
- - name: 'Set up Cloud SDK'
- uses: 'google-github-actions/setup-gcloud@v2'
- with:
- version: '>= 363.0.0'
-
- - name: Stop Spot VM
- run: gcloud compute instances stop email-tx-builder-runner-spot --zone=us-central1-c
\ No newline at end of file
diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml
index 0960ee46..5458c99a 100644
--- a/.github/workflows/unit-tests.yml
+++ b/.github/workflows/unit-tests.yml
@@ -3,11 +3,11 @@ name: Unit Tests
on: [push]
jobs:
- start-spot-vm:
- name: Start Spot VM
+ circuits:
runs-on: ubuntu-latest
-
steps:
+ - uses: actions/checkout@v3
+
- name: Authenticate with GCP
uses: google-github-actions/auth@v2
with:
@@ -18,57 +18,39 @@ jobs:
with:
version: '>= 363.0.0'
- - name: Start Spot VM
- run: gcloud compute instances start email-tx-builder-runner-spot --zone=us-central1-c
-
- - name: Wait for Spot VM to initialize
+ - name: Install gke-gcloud-auth-plugin
run: |
- echo "Waiting for Spot VM to initialize..."
- sleep 60
-
- circuits:
- name: circuits
- runs-on: self-hosted
- steps:
- - uses: actions/checkout@v3
+ gcloud components install gke-gcloud-auth-plugin -q
- - name: Setup Node.js
- uses: actions/setup-node@v3
- with:
- node-version: 18
-
- - name: Setup Rust
- uses: actions-rs/toolchain@v1
- with:
- toolchain: stable
- override: true
- components: rustfmt, clippy
-
- - name: Create bin directory
- run: mkdir -p $HOME/bin
-
- - name: Download circom v2.1.9 (Linux)
+ - name: Configure kubectl
run: |
- wget https://github.com/iden3/circom/releases/download/v2.1.9/circom-linux-amd64 -O $HOME/bin/circom
- chmod +x $HOME/bin/circom
- echo "PATH=$HOME/bin:$PATH" >> $GITHUB_ENV
+ gcloud container clusters get-credentials ${{ secrets.YOUR_CLUSTER_NAME }} --region ${{ secrets.YOUR_REGION }}
- - name: Print circom version
- run: circom --version
-
- - name: Install yarn
- run: npm install -g yarn
-
- - name: Install dependencies
- run: yarn install --frozen-lockfile
+ - name: Prepare and Deploy Circuits Job to GKE
+ env:
+ REPO_URL: "https://github.com/${{ github.repository }}"
+ COMMIT_HASH: "${{ github.sha }}"
+ run: |
+ # Prepare the job file with substituted environment variables
+ envsubst < kubernetes/circuit-test-job.yml > kubernetes/circuit-test-job-processed.yml
+
+ # Output the processed job file for verification
+ cat kubernetes/circuit-test-job-processed.yml
+
+ # Apply the job to GKE
+ kubectl apply -f kubernetes/circuit-test-job-processed.yml
+
+ - name: Wait for Job to Complete
+ run: |
+ kubectl wait --for=condition=complete --timeout=1800s job/circuits-job
- - name: Run tests
- working-directory: packages/circuits
- run: yarn test --maxWorkers=75% --no-cache
+ - name: Stream Logs from GKE
+ run: |
+ kubectl logs job/circuits-job
contracts:
name: contracts
- runs-on: self-hosted
+ runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
@@ -94,7 +76,7 @@ jobs:
relayer:
name: relayer
- runs-on: self-hosted
+ runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
@@ -120,22 +102,4 @@ jobs:
- name: Run tests
working-directory: packages/relayer
- run: cargo test
-
- stop-spot-vm:
- name: Stop Spot VM
- needs: [circuits, contracts, relayer]
- runs-on: ubuntu-latest
- steps:
- - name: Authenticate with GCP
- uses: google-github-actions/auth@v2
- with:
- credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
-
- - name: 'Set up Cloud SDK'
- uses: 'google-github-actions/setup-gcloud@v2'
- with:
- version: '>= 363.0.0'
-
- - name: Stop Spot VM
- run: gcloud compute instances stop email-tx-builder-runner-spot --zone=us-central1-c
\ No newline at end of file
+ run: cargo test
\ No newline at end of file
diff --git a/kubernetes/circuit-test-job.yml b/kubernetes/circuit-test-job.yml
new file mode 100644
index 00000000..ea3214f2
--- /dev/null
+++ b/kubernetes/circuit-test-job.yml
@@ -0,0 +1,27 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: circuits-job
+spec:
+ ttlSecondsAfterFinished: 60 # Delete automatically 1 minute after completion
+ template:
+ spec:
+ nodeSelector:
+ cloud.google.com/machine-family: c3
+ containers:
+ - name: circuits-runner
+ image: wshino/circuits-runner:test5
+ imagePullPolicy: Always # Always pull the latest image
+ env:
+ - name: REPO_URL
+ value: "${REPO_URL}"
+ - name: COMMIT_HASH
+ value: "${COMMIT_HASH}"
+ resources:
+ requests:
+ cpu: "8"
+ memory: "16Gi"
+ limits:
+ cpu: "16"
+ memory: "64Gi"
+ restartPolicy: Never
\ No newline at end of file
diff --git a/packages/circuits/README.md b/packages/circuits/README.md
index c4cf4a96..d02ba534 100644
--- a/packages/circuits/README.md
+++ b/packages/circuits/README.md
@@ -81,4 +81,40 @@ A circuit to verify that a message in the email body, called command, is authori
This is basically the same as the `email_auth.circom` described above except for the following features:
- Instead of `subject_idx`, it additionally takes as a private input a padded email body `padded_cleaned_body` and an index of the command in the email body `command_idx`.
- It extracts a substring `command` between a prefix `(<div id=3D\"[^\"]*zkemail[^\"]*\"[^>]*>)"` and a suffix `</div>` from `padded_cleaned_body`.
-- It outputs `masked_command` instead of `masked_subject`, which removes the invitation code with the prefix and one email address from `command`.
\ No newline at end of file
+- It outputs `masked_command` instead of `masked_subject`, which removes the invitation code with the prefix and one email address from `command`.
+
+## How to Build the Container Image for Unit Tests in GitHub Actions
+
+Follow these steps to build and push the container image for unit tests.
+
+### Steps
+
+1. **Navigate to the Directory**
+
+ ```bash
+ cd .github/circuits-runner
+ ```
+
+2. **Create and Use a Buildx Builder**
+
+ ```bash
+ docker buildx create --use
+ ```
+
+3. **Log in to Docker Hub**
+
+ ```bash
+ docker login
+ ```
+
+4. **Build and Push the Image**
+
+ Build the image for multiple platforms and push it to Docker Hub.
+
+ ```bash
+ docker buildx build --platform linux/amd64,linux/arm64 \
+ -t YOUR_DOCKERHUB_USERNAME/circuits-runner:latest \
+ --push .
+ ```
+
+Replace `YOUR_DOCKERHUB_USERNAME` with your actual Docker Hub username.
diff --git a/packages/circuits/package.json b/packages/circuits/package.json
index 34428f46..5f79a603 100644
--- a/packages/circuits/package.json
+++ b/packages/circuits/package.json
@@ -8,7 +8,8 @@
"build-recipient": "mkdir -p build && circom src/email_auth_with_recipient.circom --r1cs --wasm --sym --c -l ../../node_modules -o ./build",
"dev-setup": "NODE_OPTIONS=--max_old_space_size=16384 npx ts-node scripts/dev-setup.ts --output ./build",
"gen-input": "NODE_OPTIONS=--max_old_space_size=8192 npx ts-node scripts/gen_input.ts",
- "test": "NODE_OPTIONS=--max_old_space_size=8192 jest"
+ "test": "NODE_OPTIONS=--max_old_space_size=8192 jest",
+ "test-large": "NODE_OPTIONS=--max_old_space_size=32768 jest"
},
"dependencies": {
"@zk-email/circuits": "=6.3.2",