From 0e227ef0bf208701cb29b802ff4698fa60ec79dc Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 5 Nov 2024 15:47:43 -0500 Subject: [PATCH 1/2] Add Github CI workflows Update ci.yml disable e2e CI tests for now Update ci.yml Update ci.yml fix docker CI builds fix yaml file Update ci.yml Add foundry to e2e test Update cannon.yml Update cannon.yml Update cannon.yml Update cannon.yml more CI changes more CI changes fixing CI --- .circleci/config.yml | 1674 ----------------------------- .github/workflows/cannon.yml | 253 +++++ .github/workflows/ci.yml | 106 ++ .github/workflows/tag-service.yml | 69 -- .github/workflows/validate.yml | 28 + 5 files changed, 387 insertions(+), 1743 deletions(-) delete mode 100644 .circleci/config.yml create mode 100644 .github/workflows/cannon.yml create mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/tag-service.yml create mode 100644 .github/workflows/validate.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 648217dbcdf8..000000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,1674 +0,0 @@ -version: 2.1 - -parameters: - ci_builder_image: - type: string - default: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder:v0.54.0 - ci_builder_rust_image: - type: string - default: us-docker.pkg.dev/oplabs-tools-artifacts/images/ci-builder-rust:latest - base_image: - type: string - default: default - # The dispatch parameters are used to manually dispatch pipelines that normally only run post-merge on develop - # from the CircleCI UI. Example configuration: - # when: - # or: - # - equal: [ "develop", <> ] - # - equal: [ true, <> ] - # Add a new `*_dispatch` parameter for any pipeline you want manual dispatch for. - main_dispatch: - type: boolean - default: true # default to running main in case the manual run cancelled an automatic run - fault_proofs_dispatch: - type: boolean - default: false - reproducibility_dispatch: - type: boolean - default: false - kontrol_dispatch: - type: boolean - default: false - cannon_full_test_dispatch: - type: boolean - default: false - sdk_dispatch: - type: boolean - default: false - docker_publish_dispatch: - type: boolean - default: false - publish_contract_artifacts_dispatch: - type: boolean - default: false - -orbs: - go: circleci/go@1.8.0 - gcp-cli: circleci/gcp-cli@3.0.1 - slack: circleci/slack@4.10.1 - shellcheck: circleci/shellcheck@3.2.0 -commands: - gcp-oidc-authenticate: - description: "Authenticate with GCP using a CircleCI OIDC token." - parameters: - project_id: - type: env_var_name - default: GCP_PROJECT_ID - workload_identity_pool_id: - type: env_var_name - default: GCP_WIP_ID - workload_identity_pool_provider_id: - type: env_var_name - default: GCP_WIP_PROVIDER_ID - service_account_email: - type: env_var_name - default: GCP_SERVICE_ACCOUNT_EMAIL - gcp_cred_config_file_path: - type: string - default: /home/circleci/gcp_cred_config.json - oidc_token_file_path: - type: string - default: /home/circleci/oidc_token.json - steps: - - run: - name: "Create OIDC credential configuration" - command: | - # Store OIDC token in temp file - echo $CIRCLE_OIDC_TOKEN > << parameters.oidc_token_file_path >> - # Create a credential configuration for the generated OIDC ID Token - gcloud iam workload-identity-pools create-cred-config \ - "projects/${<< parameters.project_id >>}/locations/global/workloadIdentityPools/${<< parameters.workload_identity_pool_id >>}/providers/${<< parameters.workload_identity_pool_provider_id >>}"\ - --output-file="<< parameters.gcp_cred_config_file_path >>" \ - --service-account="${<< parameters.service_account_email >>}" \ - --credential-source-file=<< parameters.oidc_token_file_path >> - - run: - name: "Authenticate with GCP using OIDC" - command: | - # Configure gcloud to leverage the generated credential configuration - gcloud auth login --brief --cred-file "<< parameters.gcp_cred_config_file_path >>" - # Configure ADC - echo "export GOOGLE_APPLICATION_CREDENTIALS='<< parameters.gcp_cred_config_file_path >>'" | tee -a "$BASH_ENV" - - check-changed: - description: "Conditionally halts a step if certain modules change" - parameters: - patterns: - type: string - description: "Comma-separated list of dependencies" - no_go_deps: - type: string - default: "" - description: "If set, does not trigger on `go.mod` / `go.sum` changes." - steps: - - run: - name: "Check for changes" - environment: - CHECK_CHANGED_NO_GO_DEPS: "<>" - command: | - cd ops/check-changed - pip3 install -r requirements.txt - python3 main.py "<>" - - install-contracts-dependencies: - description: "Install the dependencies for the smart contracts" - steps: - - run: - command: just install - working_directory: packages/contracts-bedrock - - notify-failures-on-develop: - description: "Notify Slack" - parameters: - channel: - type: string - default: C03N11M0BBN - mentions: - type: string - default: "" - steps: - - slack/notify: - channel: << parameters.channel >> - event: fail - template: basic_fail_1 - branch_pattern: develop - mentions: "<< parameters.mentions >>" - - run-contracts-check: - parameters: - command: - description: Just command that runs the check - type: string - steps: - - run: - name: <> - command: | - git reset --hard - just <> - git diff --quiet --exit-code - working_directory: packages/contracts-bedrock - when: always - environment: - FOUNDRY_PROFILE: ci - -jobs: - cannon-go-lint-and-test: - machine: true - resource_class: ethereum-optimism/latitude-1 - parameters: - skip_slow_tests: - type: boolean - default: false - notify: - description: Whether to notify on failure - type: boolean - default: false - mips64: - type: boolean - default: false - steps: - - checkout - - check-changed: - patterns: cannon,packages/contracts-bedrock/src/cannon,op-preimage,go.mod - - attach_workspace: - at: "." - - run: - name: prep Cannon results dir - command: | - mkdir -p ./tmp/test-results - mkdir -p ./tmp/testlogs - - run: - name: build Cannon example binaries - command: make elf # only compile ELF binaries with Go, we do not have MIPS GCC for creating the debug-dumps. - working_directory: cannon/testdata/example - - run: - name: Cannon Go lint - command: | - make lint - working_directory: cannon - - when: - condition: - not: <> - steps: - - run: - name: Cannon Go 32-bit tests - command: | - export SKIP_SLOW_TESTS=<> - gotestsum --format=testname --junitfile=../tmp/test-results/cannon.xml --jsonfile=../tmp/testlogs/log.json \ - -- -parallel=$(nproc) -coverpkg=github.com/ethereum-optimism/optimism/cannon/... -coverprofile=coverage.out ./... - working_directory: cannon - - when: - condition: <> - steps: - - run: - name: Cannon Go 64-bit tests - command: | - export SKIP_SLOW_TESTS=<> - gotestsum --format=testname --junitfile=../tmp/test-results/cannon.xml --jsonfile=../tmp/testlogs/log.json \ - -- --tags=cannon64 -parallel=$(nproc) -coverpkg=github.com/ethereum-optimism/optimism/cannon/... -coverprofile=coverage.out ./... - working_directory: cannon - - run: - name: upload Cannon coverage - command: codecov --verbose --clean --flags cannon-go-tests - - store_test_results: - path: ./tmp/test-results - - store_artifacts: - path: ./tmp/testlogs - when: always - - when: - condition: <> - steps: - - notify-failures-on-develop: - mentions: "@proofs-team" - - cannon-build-test-vectors: - docker: - - image: <> - resource_class: medium - steps: - - checkout - - check-changed: - patterns: cannon/mipsevm/tests/open_mips_tests/test - - run: - name: Build MIPS test vectors - command: python3 maketests.py && git diff --exit-code - working_directory: cannon/mipsevm/tests/open_mips_tests - - contracts-bedrock-build: - machine: true - resource_class: ethereum-optimism/latitude-1 - parameters: - skip_pattern: - description: Glob pattern of tests to skip - type: string - default: "" - steps: - - checkout - - install-contracts-dependencies - - run: - name: Print forge version - command: forge --version - - run: - name: Pull artifacts - command: bash scripts/ops/pull-artifacts.sh - working_directory: packages/contracts-bedrock - - run: - name: Build contracts - command: forge build --deny-warnings --skip <> - environment: - FOUNDRY_PROFILE: ci - working_directory: packages/contracts-bedrock - - run: - name: Generate allocs - command: | - make devnet-allocs - - persist_to_workspace: - root: "." - paths: - - "packages/contracts-bedrock/cache" - - "packages/contracts-bedrock/artifacts" - - "packages/contracts-bedrock/forge-artifacts" - - "packages/contracts-bedrock/deploy-config/devnetL1.json" - - "packages/contracts-bedrock/deployments/devnetL1" - - ".devnet" - - ".devnet-standard" - - ".devnet-l2oo" - - ".devnet-alt-da" - - ".devnet-alt-da-generic" - - ".devnet-mt-cannon" - - notify-failures-on-develop - - check-kontrol-build: - docker: - - image: <> - resource_class: xlarge - steps: - - checkout - - attach_workspace: { at: "." } - - install-contracts-dependencies - - check-changed: - patterns: contracts-bedrock - - setup_remote_docker: - docker_layer_caching: true - - run: - name: Run Kontrol build - command: just kontrol-summary-full - working_directory: packages/contracts-bedrock - - run: - name: Build Kontrol summary files - command: forge build ./test/kontrol/proofs - working_directory: packages/contracts-bedrock - - notify-failures-on-develop - - docker-build: - environment: - DOCKER_BUILDKIT: 1 - parameters: - docker_tags: - description: Docker image tags, comma-separated - type: string - docker_name: - description: "Docker buildx bake target" - type: string - default: "" - registry: - description: Docker registry - type: string - default: "us-docker.pkg.dev" - repo: - description: Docker repo - type: string - default: "oplabs-tools-artifacts/images" - save_image_tag: - description: Save docker image with given tag - type: string - default: "" - platforms: - description: Platforms to build for, comma-separated - type: string - default: "linux/amd64" - publish: - description: Publish the docker image (multi-platform, all tags) - type: boolean - default: false - release: - description: Run the release script - type: boolean - default: false - resource_class: - description: Docker resoruce class - type: string - default: medium - machine: - image: <> - resource_class: "<>" - docker_layer_caching: true # we rely on this for faster builds, and actively warm it up for builds with common stages - steps: - - checkout - - attach_workspace: - at: /tmp/docker_images - - run: - command: mkdir -p /tmp/docker_images - - when: - condition: - or: - - "<>" - - "<>" - steps: - - gcp-cli/install - - when: - condition: - or: - - "<>" - - "<>" - steps: - - gcp-oidc-authenticate - - run: - name: Build - command: | - # Check to see if DOCKER_HUB_READ_ONLY_TOKEN is set (i.e. we are in repo) before attempting to use secrets. - # Building should work without this read only login, but may get rate limited. - if [[ -v DOCKER_HUB_READ_ONLY_TOKEN ]]; then - echo "$DOCKER_HUB_READ_ONLY_TOKEN" | docker login -u "$DOCKER_HUB_READ_ONLY_USER" --password-stdin - fi - - export REGISTRY="<>" - export REPOSITORY="<>" - export IMAGE_TAGS="$(echo -ne "<>" | sed "s/[^a-zA-Z0-9\n,]/-/g")" - export GIT_COMMIT="$(git rev-parse HEAD)" - export GIT_DATE="$(git show -s --format='%ct')" - export PLATFORMS="<>" - - echo "Checking git tags pointing at $GIT_COMMIT:" - tags_at_commit=$(git tag --points-at $GIT_COMMIT) - echo "Tags at commit:\n$tags_at_commit" - - filtered_tags=$(echo "$tags_at_commit" | grep "^<>/" || true) - echo "Filtered tags: $filtered_tags" - - if [ -z "$filtered_tags" ]; then - export GIT_VERSION="untagged" - else - sorted_tags=$(echo "$filtered_tags" | sed "s/<>\///" | sort -V) - echo "Sorted tags: $sorted_tags" - - # prefer full release tag over "-rc" release candidate tag if both exist - full_release_tag=$(echo "$sorted_tags" | grep -v -- "-rc" || true) - if [ -z "$full_release_tag" ]; then - export GIT_VERSION=$(echo "$sorted_tags" | tail -n 1) - else - export GIT_VERSION=$(echo "$full_release_tag" | tail -n 1) - fi - fi - - echo "Setting GIT_VERSION=$GIT_VERSION" - - # Create, start (bootstrap) and use a *named* docker builder - # This allows us to cross-build multi-platform, - # and naming allows us to use the DLC (docker-layer-cache) - docker buildx create --driver=docker-container --name=buildx-build --bootstrap --use - - DOCKER_OUTPUT_DESTINATION="" - if [ "<>" == "true" ]; then - gcloud auth configure-docker <> - echo "Building for platforms $PLATFORMS and then publishing to registry" - DOCKER_OUTPUT_DESTINATION="--push" - if [ "<>" != "" ]; then - echo "ERROR: cannot save image to docker when publishing to registry" - exit 1 - fi - else - if [ "<>" == "" ]; then - echo "Running $PLATFORMS build without destination (cache warm-up)" - DOCKER_OUTPUT_DESTINATION="" - elif [[ $PLATFORMS == *,* ]]; then - echo "ERROR: cannot perform multi-arch (platforms: $PLATFORMS) build while also loading the result into regular docker" - exit 1 - else - echo "Running single-platform $PLATFORMS build and loading into docker" - DOCKER_OUTPUT_DESTINATION="--load" - fi - fi - - # Let them cook! - docker buildx bake \ - --progress plain \ - --builder=buildx-build \ - -f docker-bake.hcl \ - $DOCKER_OUTPUT_DESTINATION \ - <> - - no_output_timeout: 45m - - when: - condition: "<>" - steps: - - notify-failures-on-develop - - when: - condition: "<>" - steps: - - run: - name: Save - command: | - IMAGE_NAME="<>/<>/<>:<>" - docker save -o /tmp/docker_images/<>.tar $IMAGE_NAME - - persist_to_workspace: - root: /tmp/docker_images - paths: # only write the one file, to avoid concurrent workspace-file additions - - "<>.tar" - - when: - condition: "<>" - steps: - - run: - name: Tag - command: | - ./ops/scripts/ci-docker-tag-op-stack-release.sh <>/<> $CIRCLE_TAG $CIRCLE_SHA1 - - when: - condition: - or: - - and: - - "<>" - - "<>" - - and: - - "<>" - - equal: [develop, << pipeline.git.branch >>] - steps: - - gcp-oidc-authenticate: - service_account_email: GCP_SERVICE_ATTESTOR_ACCOUNT_EMAIL - - run: - name: Sign - command: | - VER=$(jq -r .binary_signer < versions.json) - wget -O - "https://github.com/ethereum-optimism/binary_signer/archive/refs/tags/v${VER}.tar.gz" | tar xz - cd "binary_signer-${VER}/signer" - - IMAGE_PATH="<>/<>/<>:<>" - echo $IMAGE_PATH - pip3 install -r requirements.txt - - python3 ./sign_image.py --command="sign"\ - --attestor-project-name="$ATTESTOR_PROJECT_NAME"\ - --attestor-name="$ATTESTOR_NAME"\ - --image-path="$IMAGE_PATH"\ - --signer-logging-level="INFO"\ - --attestor-key-id="//cloudkms.googleapis.com/v1/projects/$ATTESTOR_PROJECT_NAME/locations/global/keyRings/$ATTESTOR_NAME-key-ring/cryptoKeys/$ATTESTOR_NAME-key/cryptoKeyVersions/1" - - # Verify newly published images (built on AMD machine) will run on ARM - check-cross-platform: - docker: - - image: cimg/base:current - resource_class: arm.medium - parameters: - registry: - description: Docker registry - type: string - default: "us-docker.pkg.dev" - repo: - description: Docker repo - type: string - default: "oplabs-tools-artifacts/images" - op_component: - description: "Name of op-stack component (e.g. op-node)" - type: string - default: "" - docker_tag: - description: "Tag of docker image" - type: string - default: "<>" - steps: - - setup_remote_docker - - run: - name: "Verify Image Platform" - command: | - image_name="<>/<>/<>:<>" - echo "Retrieving Docker image manifest: $image_name" - MANIFEST=$(docker manifest inspect $image_name) - - echo "Verifying 'linux/arm64' is supported..." - SUPPORTED_PLATFORM=$(echo "$MANIFEST" | jq -r '.manifests[] | select(.platform.architecture == "arm64" and .platform.os == "linux")') - echo $SUPPORT_PLATFORM - if [ -z "$SUPPORTED_PLATFORM" ]; then - echo "Platform 'linux/arm64' not supported by this image" - exit 1 - fi - - run: - name: "Pull and run docker image" - command: | - image_name="<>/<>/<>:<>" - docker pull $image_name || exit 1 - docker run $image_name <> --version || exit 1 - - contracts-bedrock-coverage: - docker: - - image: <> - resource_class: xlarge - steps: - - checkout - - install-contracts-dependencies - - check-changed: - patterns: contracts-bedrock,op-node - - run: - name: print forge version - command: forge --version - working_directory: packages/contracts-bedrock - # We do not use the pre-built contracts becuase forge coverage uses different optimizer settings - - run: - name: test and generate coverage - command: just coverage-lcov - no_output_timeout: 18m - environment: - FOUNDRY_PROFILE: ci - working_directory: packages/contracts-bedrock - - run: - name: upload coverage - command: codecov --verbose --clean --flags contracts-bedrock-tests - environment: - FOUNDRY_PROFILE: ci - - contracts-bedrock-tests: - docker: - - image: <> - resource_class: xlarge - parameters: - test_parallelism: - description: Number of test jobs to run in parallel - type: integer - default: 4 - test_list: - description: List of test files to run - type: string - test_fuzz_runs: - description: Number of fuzz runs to apply - type: integer - default: 512 - test_invariant_runs: - description: Number of invariant runs to apply - type: integer - default: 32 - test_invariant_depth: - description: Depth of invariant runs - type: integer - default: 64 - test_timeout: - description: Timeout for running tests - type: string - default: 15m - test_profile: - description: Profile to use for testing - type: string - default: ci - parallelism: <> - steps: - - checkout - - attach_workspace: { at: "." } - - install-contracts-dependencies - - run: - name: Check if test list is empty - command: | - TEST_FILES=$(<>) - if [ -z "$TEST_FILES" ]; then - echo "No test files to run. Exiting early." - circleci-agent step halt - fi - working_directory: packages/contracts-bedrock - - check-changed: - patterns: contracts-bedrock,op-node - - restore_cache: - name: Restore Go modules cache - key: gomod-{{ checksum "go.sum" }} - - restore_cache: - name: Restore Go build cache - keys: - - golang-build-cache-contracts-bedrock-tests-{{ checksum "go.sum" }} - - golang-build-cache-contracts-bedrock-tests- - - run: - name: Print dependencies - command: just dep-status - working_directory: packages/contracts-bedrock - - run: - name: Print forge version - command: forge --version - working_directory: packages/contracts-bedrock - - run: - name: Pull artifacts - command: bash scripts/ops/pull-artifacts.sh - working_directory: packages/contracts-bedrock - - run: - name: Build go-ffi - command: just build-go-ffi - working_directory: packages/contracts-bedrock - - run: - name: Run tests - command: | - TEST_FILES=$(<>) - TEST_FILES=$(echo "$TEST_FILES" | circleci tests split --split-by=timings) - TEST_FILES=$(echo "$TEST_FILES" | sed 's|^test/||') - MATCH_PATH="./test/{$(echo "$TEST_FILES" | paste -sd "," -)}" - forge test --match-path "$MATCH_PATH" - environment: - FOUNDRY_PROFILE: <> - working_directory: packages/contracts-bedrock - no_output_timeout: <> - - run: - name: Print failed test traces - command: just test-rerun - environment: - FOUNDRY_PROFILE: ci - working_directory: packages/contracts-bedrock - when: on_fail - - save_cache: - name: Save Go build cache - key: golang-build-cache-contracts-bedrock-tests-{{ checksum "go.sum" }} - paths: - - "/root/.cache/go-build" - - notify-failures-on-develop - - contracts-bedrock-checks: - docker: - - image: <> - resource_class: xlarge - steps: - - checkout - - attach_workspace: { at: "." } - - install-contracts-dependencies - - check-changed: - patterns: contracts-bedrock,op-node - - setup_remote_docker: - docker_layer_caching: true - - run: - name: print forge version - command: forge --version - - run-contracts-check: - command: semgrep-test-validity-check - - run-contracts-check: - command: semgrep - - run-contracts-check: - command: semver-lock - - run-contracts-check: - command: semver-diff-check-no-build - - run-contracts-check: - command: semver-natspec-check-no-build - - run-contracts-check: - command: validate-deploy-configs - - run-contracts-check: - command: lint - - run-contracts-check: - command: gas-snapshot-check - - run-contracts-check: - command: snapshots-check-no-build - - run-contracts-check: - command: interfaces-check-no-build - - run-contracts-check: - command: size-check - - run-contracts-check: - command: unused-imports-check-no-build - - contracts-bedrock-validate-spacers: - docker: - - image: <> - resource_class: medium - steps: - - checkout - - attach_workspace: { at: "." } - - install-contracts-dependencies - - check-changed: - patterns: contracts-bedrock - - run: - name: validate spacers - command: just validate-spacers-no-build - working_directory: packages/contracts-bedrock - - todo-issues: - parameters: - check_closed: - type: boolean - default: true - machine: - image: <> - steps: - - checkout - - run: - name: Install ripgrep - command: sudo apt-get install -y ripgrep - - run: - name: Check TODO issues - command: ./ops/scripts/todo-checker.sh --verbose <<#parameters.check_closed>> --check-closed <> - - notify-failures-on-develop - - fuzz-golang: - parameters: - package_name: - description: Go package name - type: string - on_changes: - description: changed pattern to fire fuzzer on - type: string - uses_artifacts: - description: should load in foundry artifacts - type: boolean - default: false - docker: - - image: <> - steps: - - checkout - - check-changed: - patterns: "<>" - - attach_workspace: - at: "." - if: ${{ uses_artifacts }} - - restore_cache: - name: Restore Go modules cache - key: gomod-{{ checksum "go.sum" }} - - restore_cache: - name: Restore Go build cache - keys: - - golang-build-cache-fuzz-golang-{{ checksum "go.sum" }} - - golang-build-cache-fuzz-golang- - - run: - name: Fuzz - command: make fuzz - working_directory: "<>" - - save_cache: - name: Save Go build cache - key: golang-build-cache-fuzz-golang-{{ checksum "go.sum" }} - paths: - - "/root/.cache/go-build" - - go-lint: - machine: true - resource_class: ethereum-optimism/latitude-1 - steps: - - checkout - - run: - name: run Go linter - command: | - make lint-go - working_directory: . - - go-test-kurtosis: - parameters: - module: - description: Go Module Name - type: string - uses_artifacts: - description: Uses contract artifacts - type: boolean - default: false - test_directory: - description: Test directory - type: string - default: "./..." - machine: - image: <> - resource_class: xlarge - steps: - - run: - name: Install components - command: | - go version - go install gotest.tools/gotestsum@v1.11.0 - - run: - name: Install Kurtosis - command: | - echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list - sudo apt update - sudo apt install kurtosis-cli=1.3.0 - kurtosis engine start - - checkout - - when: - condition: <> - steps: - - attach_workspace: { at: "." } - - run: - name: prep results dir - command: | - # Make sure the workspace is properly owned - mkdir -p ./tmp/test-results - mkdir -p ./tmp/testlogs - - run: - name: run tests - command: | - ENABLE_KURTOSIS=true gotestsum \ - --format=testname \ - --junitfile=../tmp/test-results/<>.xml \ - --jsonfile=../tmp/testlogs/log.json \ - -- -parallel=$(nproc) \ - -coverpkg=github.com/ethereum-optimism/optimism/... \ - -coverprofile=coverage.out <> - working_directory: <> - - store_test_results: - path: tmp/test-results - - store_artifacts: - path: tmp/testlogs - when: always - - go-test: - machine: true - resource_class: ethereum-optimism/latitude-1 - steps: - - checkout - - attach_workspace: - at: "." - - run: - name: run tests - command: | - mkdir -p ./tmp/test-results && mkdir -p ./tmp/testlogs - - packages=( - op-batcher - op-chain-ops - op-node - op-proposer - op-challenger - op-dispute-mon - op-conductor - op-program - op-service - op-supervisor - op-deployer - packages/contracts-bedrock/scripts/checks/semver-natspec - ) - formatted_packages="" - for package in "${packages[@]}"; do - formatted_packages="$formatted_packages ./$package/..." - done - - export ENABLE_ANVIL=true - export SEPOLIA_RPC_URL="https://ci-sepolia-l1-archive.optimism.io" - - gotestsum --format=testname \ - --junitfile=./tmp/test-results/results.xml \ - --jsonfile=./tmp/testlogs/log.json \ - -- -coverpkg=github.com/ethereum-optimism/optimism/... \ - -coverprofile=coverage.out $formatted_packages - - store_test_results: - path: ./tmp/test-results - - store_artifacts: - path: ./tmp/testlogs - when: always - - go-e2e-test: - parameters: - module: - description: Go Module Name - type: string - target: - description: The make target to execute - type: string - notify: - description: Whether to notify on failure - type: boolean - default: false - mentions: - description: Slack user or group to mention when notifying of failures - type: string - default: "" - resource_class: - description: Machine resource class - type: string - default: ethereum-optimism/latitude-1 - machine: true - resource_class: <> - steps: - - checkout - - attach_workspace: - at: ./tmp/workspace - - run: - name: Load devnet-allocs and artifacts - command: | - mkdir -p .devnet - cp -r ./tmp/workspace/.devnet* . - cp -r ./tmp/workspace/packages/contracts-bedrock/forge-artifacts packages/contracts-bedrock/forge-artifacts - cp ./tmp/workspace/packages/contracts-bedrock/deploy-config/devnetL1.json packages/contracts-bedrock/deploy-config/devnetL1.json - cp -r ./tmp/workspace/packages/contracts-bedrock/deployments/devnetL1 packages/contracts-bedrock/deployments/devnetL1 - - run: - name: print go's available MIPS targets - command: go tool dist list | grep mips - - run: - name: run tests - no_output_timeout: 20m - command: | - mkdir -p ./tmp/testlogs - mkdir -p ./tmp/test-results - - # The below env var gets overridden when running make test-cannon, but we - # need to explicitly set it here to prevent Cannon from running when we don't - # want it to. - export OP_E2E_CANNON_ENABLED="false" - # Note: We don't use circle CI test splits because we need to split by test name, not by package. There is an additional - # constraint that gotestsum does not currently (nor likely will) accept files from different packages when building. - JUNIT_FILE=../tmp/test-results/<>_<>.xml JSON_LOG_FILE=../tmp/testlogs/test.log make <> - working_directory: <> - - store_artifacts: - path: ./tmp/testlogs - when: always - - store_artifacts: - path: ./tmp/test-results - when: always - - store_test_results: - path: ./tmp/test-results - - when: - condition: "<>" - steps: - - notify-failures-on-develop: - mentions: "<>" - - cannon-prestate: - machine: true - resource_class: ethereum-optimism/latitude-1 - steps: - - checkout - - run: - name: Build cannon - command: make cannon - - run: - name: Build op-program - command: make op-program - - restore_cache: - name: Restore cannon prestate cache - key: cannon-prestate-{{ checksum "./cannon/bin/cannon" }}-{{ checksum "op-program/bin/op-program-client.elf" }} - - run: - name: Sanitize op-program guest - command: make -f cannon/Makefile sanitize-program GUEST_PROGRAM=op-program/bin/op-program-client.elf - - run: - name: generate cannon prestate - command: make cannon-prestate - - save_cache: - key: cannon-prestate-{{ checksum "./cannon/bin/cannon" }}-{{ checksum "op-program/bin/op-program-client.elf" }} - name: Save Cannon prestate to cache - paths: - - "op-program/bin/prestate.bin.gz" - - "op-program/bin/meta.json" - - "op-program/bin/prestate-proof.json" - - run: - name: generate cannon-mt prestate - command: make cannon-prestate-mt - - save_cache: - key: cannon-prestate-mt-{{ checksum "./cannon/bin/cannon" }}-{{ checksum "op-program/bin/op-program-client.elf" }} - name: Save MT-Cannon prestate to cache - paths: - - "op-program/bin/prestate-mt.json" - - "op-program/bin/meta-mt.json" - - "op-program/bin/prestate-proof-mt.json" - - persist_to_workspace: - root: . - paths: - - "op-program/bin/prestate.bin.gz" - - "op-program/bin/meta.json" - - "op-program/bin/prestate-proof.json" - - preimage-reproducibility: - docker: - - image: <> - steps: - - checkout - - setup_remote_docker - - run: - make -C op-program verify-reproducibility - - notify-failures-on-develop: - mentions: "@proofs-team" - - cannon-stf-verify: - docker: - - image: <> - steps: - - checkout - - setup_remote_docker - - restore_cache: - name: Restore Go modules cache - key: gomod-{{ checksum "go.sum" }} - - restore_cache: - name: Restore Go build cache - keys: - - golang-build-cache-cannon-stf-verify-{{ checksum "go.sum" }} - - golang-build-cache-cannon-stf-verify- - - run: - name: Build cannon - command: make cannon - - run: - name: Verify the Cannon STF - command: make -C ./cannon cannon-stf-verify - - save_cache: - name: Save Go build cache - key: golang-build-cache-cannon-stf-verify-{{ checksum "go.sum" }} - paths: - - "/root/.cache/go-build" - - notify-failures-on-develop: - mentions: "@proofs-squad" - - semgrep-scan: - parameters: - diff_branch: - type: string - default: develop - scan_command: - type: string - default: semgrep ci --timeout=100 - environment: - TEMPORARY_BASELINE_REF: << parameters.diff_branch >> - SEMGREP_REPO_URL: << pipeline.project.git_url >> - SEMGREP_BRANCH: << pipeline.git.branch >> - SEMGREP_COMMIT: << pipeline.git.revision >> - docker: - - image: returntocorp/semgrep - resource_class: xlarge - steps: - - checkout - - unless: - condition: - equal: [ "develop", << pipeline.git.branch >> ] - steps: - - run: - # Scan changed files in PRs, block on new issues only (existing issues ignored) - # Do a full scan when scanning develop, otherwise do an incremental scan. - name: "Conditionally set BASELINE env var" - command: | - echo 'export SEMGREP_BASELINE_REF=${TEMPORARY_BASELINE_REF}' >> $BASH_ENV - - run: - name: "Set environment variables" # for PR comments and in-app hyperlinks to findings - command: | - echo 'export SEMGREP_PR_ID=${CIRCLE_PULL_REQUEST##*/}' >> $BASH_ENV - echo 'export SEMGREP_JOB_URL=$CIRCLE_BUILD_URL' >> $BASH_ENV - echo 'export SEMGREP_REPO_NAME=$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME' >> $BASH_ENV - - run: - name: "Semgrep scan" - # --timeout (in seconds) limits the time per rule and file. - # SEMGREP_TIMEOUT is the same, but docs have conflicting defaults (5s in CLI flag, 1800 in some places) - # https://semgrep.dev/docs/troubleshooting/semgrep-app#if-the-job-is-aborted-due-to-taking-too-long - command: << parameters.scan_command >> - # If semgrep hangs, stop the scan after 20m, to prevent a useless 5h job - no_output_timeout: 20m - - notify-failures-on-develop - - go-mod-download: - docker: - - image: <> - parameters: - file: - default: go.sum - description: The file name of checksum for restore_cache and save_cache. - type: string - key: - default: gomod - description: The key of restore_cache and save_cache. - type: string - steps: - - checkout - - restore_cache: - key: << parameters.key >>-{{ checksum "<< parameters.file >>" }} - name: Restore Go modules cache - - run: - name: Sanity check go mod cache path - command: test "$(go env GOMODCACHE)" == "/go/pkg/mod" # yes, it's an odd path - - run: - command: go mod download - name: Download Go module dependencies - - run: - name: "Go mod tidy" - command: make mod-tidy && git diff --exit-code - - save_cache: - key: << parameters.key >>-{{ checksum "<< parameters.file >>" }} - name: Save Go modules cache - paths: - - "/go/pkg/mod" - - bedrock-go-tests: # just a helper, that depends on all the actual test jobs - docker: - # Use a smaller base image to avoid pulling the huge ci-builder - # image which is not needed for this job and sometimes misses - # the cache. - - image: cimg/base:2024.01 - resource_class: medium - steps: - - run: echo Done - - fpp-verify: - docker: - - image: cimg/go:1.21 - steps: - - checkout - - run: - name: verify-sepolia - command: | - make verify-sepolia - working_directory: op-program - - notify-failures-on-develop: - mentions: "@proofs-team" - - op-program-compat: - docker: - - image: <> - steps: - - checkout - - restore_cache: - name: Restore Go modules cache - key: gomod-{{ checksum "go.sum" }} - - restore_cache: - key: golang-build-cache-op-program-compat-{{ checksum "go.sum" }} - - run: - name: compat-sepolia - command: | - make verify-compat - working_directory: op-program - - save_cache: - name: Save Go build cache - key: golang-build-cache-op-program-compat-{{ checksum "go.sum" }} - paths: - - "/root/.cache/go-build" - - check-generated-mocks-op-node: - docker: - - image: <> - steps: - - checkout - - check-changed: - patterns: op-node - - run: - name: check-generated-mocks - command: make generate-mocks-op-node && git diff --exit-code - - check-generated-mocks-op-service: - docker: - - image: <> - steps: - - checkout - - check-changed: - patterns: op-service - - run: - name: check-generated-mocks - command: make generate-mocks-op-service && git diff --exit-code - - kontrol-tests: - docker: - - image: << pipeline.parameters.ci_builder_image >> - resource_class: xlarge - steps: - - checkout - - install-contracts-dependencies - - check-changed: - no_go_deps: "true" - patterns: contracts-bedrock/test/kontrol,contracts-bedrock/src/L1/OptimismPortal\.sol,contracts-bedrock/src/L1/OptimismPortal2\.sol,contracts-bedrock/src/L1/L1CrossDomainMessenger\.sol,contracts-bedrock/src/L1/L1ERC721Bridge\.sol,contracts-bedrock/src/L1/L1StandardBridge\.sol,contracts-bedrock/src/L1/ResourceMetering\.sol,contracts-bedrock/src/universal/StandardBridge\.sol,contracts-bedrock/src/universal/ERC721Bridge\.sol,contracts-bedrock/src/universal/CrossDomainMessenger\.sol - - setup_remote_docker: - docker_layer_caching: true - - run: - name: Run Kontrol Tests - command: | - curl -X POST \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer $RV_COMPUTE_TOKEN" \ - https://api.github.com/repos/runtimeverification/optimism-ci/actions/workflows/optimism-ci.yaml/dispatches \ - -d '{ - "ref": "master", - "inputs": { - "branch_name": "<>", - "extra_args": "script", - "statuses_sha": "<< pipeline.git.revision >>", - "org": "ethereum-optimism", - "repository": "optimism" - } - }' - working_directory: ./packages/contracts-bedrock - - notify-failures-on-develop - - publish-contract-artifacts: - docker: - - image: <> - resource_class: xlarge - steps: - - gcp-cli/install - - gcp-oidc-authenticate: - gcp_cred_config_file_path: /root/gcp_cred_config.json - oidc_token_file_path: /root/oidc_token.json - project_id: GCP_TOOLS_ARTIFACTS_PROJECT_ID - service_account_email: GCP_CONTRACTS_PUBLISHER_SERVICE_ACCOUNT_EMAIL - - checkout - - install-contracts-dependencies - - run: - name: Pull artifacts - command: bash scripts/ops/pull-artifacts.sh - working_directory: packages/contracts-bedrock - - run: - name: Build contracts - environment: - FOUNDRY_PROFILE: ci - command: just forge-build - working_directory: packages/contracts-bedrock - - run: - name: Publish artifacts - command: bash scripts/ops/publish-artifacts.sh - working_directory: packages/contracts-bedrock - - go-release: - parameters: - module: - description: Go Module Name - type: string - filename: - description: Goreleaser config file - default: .goreleaser.yaml - type: string - machine: true - resource_class: ethereum-optimism/latitude-1 - steps: - - checkout - - run: - name: Run goreleaser - command: | - goreleaser release --clean -f ./<>/<> - -workflows: - main: - when: - and: - - or: - # Trigger on new commits - - equal: [ webhook, << pipeline.trigger_source >> ] - # Trigger on manual triggers if explicitly requested - - equal: [ true, << pipeline.parameters.main_dispatch >> ] - - not: - equal: [ scheduled_pipeline, << pipeline.trigger_source >> ] - jobs: - - go-mod-download - - contracts-bedrock-build: - # Build with just core + script contracts. - skip_pattern: test - - check-kontrol-build: - requires: - - contracts-bedrock-build - - contracts-bedrock-tests: - # Test everything except PreimageOracle.t.sol since it's slow. - name: contracts-bedrock-tests - test_parallelism: 4 - test_list: find test -name "*.t.sol" -not -name "PreimageOracle.t.sol" - - contracts-bedrock-tests: - # PreimageOracle test is slow, run it separately to unblock CI. - name: contracts-bedrock-tests-preimage-oracle - test_parallelism: 1 - test_list: find test -name "PreimageOracle.t.sol" - - contracts-bedrock-tests: - # Heavily fuzz any fuzz tests within added or modified test files. - name: contracts-bedrock-tests-heavy-fuzz-modified - test_parallelism: 1 - test_list: git diff origin/develop...HEAD --name-only -- './test/**/*.t.sol' | sed 's|packages/contracts-bedrock/||' - test_timeout: 1h - test_profile: ciheavy - - contracts-bedrock-coverage - - contracts-bedrock-checks: - requires: - - contracts-bedrock-build - - contracts-bedrock-validate-spacers: - requires: - - contracts-bedrock-build - - semgrep-scan - - semgrep-scan: - name: semgrep-scan-local - scan_command: semgrep scan --timeout=100 --config=./semgrep --error . - - semgrep-scan: - name: semgrep-test - scan_command: semgrep scan --test semgrep/ - - go-lint - - fuzz-golang: - name: fuzz-golang-<> - requires: - - go-mod-download - on_changes: <> - matrix: - parameters: - package_name: - - op-challenger - - op-node - - op-service - - op-chain-ops - - fuzz-golang: - name: cannon-fuzz - package_name: cannon - on_changes: cannon,packages/contracts-bedrock/src/cannon - uses_artifacts: true - requires: ["go-mod-download", "contracts-bedrock-build"] - - fuzz-golang: - name: op-e2e-fuzz - package_name: op-e2e - on_changes: op-e2e,packages/contracts-bedrock/src - uses_artifacts: true - requires: ["go-mod-download", "contracts-bedrock-build"] - - go-test: - name: go-test-all - requires: - - contracts-bedrock-build - - go-test-kurtosis: - name: op-deployer-integration - module: op-deployer - test_directory: ./pkg/deployer/integration_test - uses_artifacts: true - requires: ["contracts-bedrock-build"] - - go-e2e-test: - name: op-e2e-HTTP-tests - module: op-e2e - target: test-http - requires: - - contracts-bedrock-build - - go-e2e-test: - name: op-e2e-action-tests - module: op-e2e - target: test-actions - requires: - - contracts-bedrock-build - - go-e2e-test: - name: op-e2e-fault-proof-tests - module: op-e2e - target: test-fault-proofs - requires: - - contracts-bedrock-build - - cannon-prestate - - op-program-compat - - bedrock-go-tests: - requires: - - go-mod-download - - go-lint - - cannon-build-test-vectors - - cannon-go-lint-and-test - - check-generated-mocks-op-node - - check-generated-mocks-op-service - - go-mod-download - - op-deployer-integration - - op-program-compat - - op-e2e-HTTP-tests - - op-e2e-fault-proof-tests - - op-e2e-action-tests - # Not needed for the devnet but we want to make sure they build successfully - - cannon-docker-build - - op-dispute-mon-docker-build - - op-program-docker-build - - op-supervisor-docker-build - - proofs-tools-docker-build - - go-test-all - - docker-build: - name: <>-docker-build - docker_tags: <>,<> - save_image_tag: <> - matrix: - parameters: - docker_name: - - op-node - - op-batcher - - op-program - - op-proposer - - op-challenger - - proofs-tools - - op-dispute-mon - - op-conductor - - da-server - - op-supervisor - - cannon - - cannon-prestate - - check-generated-mocks-op-node - - check-generated-mocks-op-service - - cannon-go-lint-and-test: - requires: - - contracts-bedrock-build - skip_slow_tests: true - notify: true - - cannon-build-test-vectors - - todo-issues: - name: todo-issues-check - check_closed: false - - shellcheck/check: - name: shell-check - # We don't need the `exclude` key as the orb detects the `.shellcheckrc` - dir: . - ignore-dirs: - ./packages/contracts-bedrock/lib - - go-release-deployer: - jobs: - - go-release: - filters: - tags: - only: /^op-deployer.*/ - branches: - ignore: /.*/ - module: op-deployer - context: - - oplabs-gcr-release - - release: - when: - not: - equal: [ scheduled_pipeline, << pipeline.trigger_source >> ] - jobs: - # Wait for approval on the release - - hold: - type: approval - filters: - tags: - only: /^(da-server|ci-builder(-rust)?|proofs-tools|cannon|ufm-[a-z0-9\-]*|op-[a-z0-9\-]*)\/v.*/ - branches: - ignore: /.*/ - # Standard (medium) cross-platform docker images go here - - docker-build: - matrix: - parameters: - docker_name: - - op-node - - op-batcher - - op-proposer - - op-challenger - - op-dispute-mon - - op-conductor - - da-server - - op-ufm - - op-supervisor - - op-deployer - - cannon - name: <>-docker-release - docker_tags: <> - platforms: "linux/amd64,linux/arm64" - publish: true - release: true - filters: - tags: - only: /^<>\/v.*/ - branches: - ignore: /.*/ - context: - - oplabs-gcr-release - requires: - - hold - # Checks for cross-platform images go here - - check-cross-platform: - matrix: - parameters: - op_component: - - op-node - - op-batcher - - op-proposer - - op-challenger - - op-dispute-mon - - op-conductor - - da-server - - op-ufm - - op-supervisor - - op-deployer - - cannon - name: <>-cross-platform - requires: - - op-node-docker-release - - op-batcher-docker-release - - op-proposer-docker-release - - op-challenger-docker-release - - op-dispute-mon-docker-release - - op-conductor-docker-release - - da-server-docker-release - - op-ufm-docker-release - - op-supervisor-docker-release - - cannon-docker-release - # Standard (xlarge) AMD-only docker images go here - - docker-build: - matrix: - parameters: - docker_name: - - ci-builder - - ci-builder-rust - - proofs-tools - name: <>-docker-release - resource_class: xlarge - docker_tags: <> - publish: true - release: true - filters: - tags: - only: /^<>\/v.*/ - branches: - ignore: /.*/ - context: - - oplabs-gcr-release - requires: - - hold - - scheduled-todo-issues: - when: - equal: [ build_four_hours, <> ] - jobs: - - todo-issues: - name: todo-issue-checks - context: - - slack - - scheduled-fpp: - when: - equal: [ build_hourly, <> ] - jobs: - - fpp-verify: - context: - - slack - - oplabs-fpp-nodes - - develop-publish-contract-artifacts: - when: - or: - - equal: [ "develop", <> ] - - equal: [ true, <> ] - jobs: - - publish-contract-artifacts - - develop-fault-proofs: - when: - and: - - or: - - equal: [ "develop", <> ] - - equal: [ true, <> ] - - not: - equal: [ scheduled_pipeline, << pipeline.trigger_source >> ] - jobs: - - go-mod-download - - cannon-prestate - - cannon-stf-verify: - requires: - - go-mod-download - context: - - slack - - contracts-bedrock-build: - skip_pattern: test - context: - - slack - - go-e2e-test: - name: op-e2e-cannon-tests - module: op-e2e - target: test-cannon - notify: true - mentions: "@proofs-team" - requires: - - contracts-bedrock-build - - cannon-prestate - context: - - slack - - develop-kontrol-tests: - when: - and: - - or: - - equal: [ "develop", <> ] - - equal: [ true, <> ] - - not: - equal: [ scheduled_pipeline, << pipeline.trigger_source >> ] - jobs: - - kontrol-tests: - context: - - slack - - runtimeverification - - scheduled-cannon-full-tests: - when: - or: - - equal: [ build_four_hours, <> ] - - equal: [ true, << pipeline.parameters.cannon_full_test_dispatch >> ] - jobs: - - contracts-bedrock-build: - skip_pattern: test - - cannon-go-lint-and-test: - requires: - - contracts-bedrock-build - context: - - slack - - scheduled-docker-publish: - when: - or: - - equal: [ build_hourly, <> ] - # Trigger on manual triggers if explicitly requested - - equal: [ true, << pipeline.parameters.docker_publish_dispatch >> ] - jobs: - - docker-build: - matrix: - parameters: - docker_name: - - op-node - - op-batcher - - op-program - - op-proposer - - op-challenger - - op-dispute-mon - - op-conductor - - op-supervisor - - cannon - name: <>-docker-publish - docker_tags: <>,<> - platforms: "linux/amd64,linux/arm64" - publish: true - context: - - oplabs-gcr - - slack - - check-cross-platform: - matrix: - parameters: - op_component: - - op-node - - op-batcher - - op-program - - op-proposer - - op-challenger - - op-dispute-mon - - op-conductor - - op-supervisor - - cannon - name: <>-cross-platform - requires: - - <>-docker-publish - - docker-build: - name: contracts-bedrock-docker-publish - docker_name: contracts-bedrock - docker_tags: <>,<> - resource_class: xlarge - publish: true - context: - - oplabs-gcr - - slack - - scheduled-preimage-reproducibility: - when: - or: - - equal: [build_daily, <> ] - # Trigger on manual triggers if explicitly requested - - equal: [ true, << pipeline.parameters.reproducibility_dispatch >> ] - jobs: - - preimage-reproducibility: - context: - slack diff --git a/.github/workflows/cannon.yml b/.github/workflows/cannon.yml new file mode 100644 index 000000000000..ce0a8d768810 --- /dev/null +++ b/.github/workflows/cannon.yml @@ -0,0 +1,253 @@ +name: Cannon Tests + +env: + FOUNDRY_PROFILE: ci + OP_E2E_CANNON_ENABLED: "false" + +on: + push: + branches: [celestia-develop] + pull_request: + branches: [celestia-develop] + workflow_dispatch: + +jobs: + contracts-build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + # Debug step to check directory structure + - name: Debug directory structure + run: | + echo "Current directory: $(pwd)" + echo "Directory contents:" + ls -la + echo "Checking for packages directory:" + ls -la packages || echo "No packages directory" + + # Install just command + - name: Install just + run: | + curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to "$HOME/.local/bin" + echo "$HOME/.local/bin" >> "$GITHUB_PATH" + + # Install Node.js without npm cache initially + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: "18" + + - name: Install npm dependencies + run: | + if [ ! -d "packages/contracts-bedrock" ]; then + echo "contracts-bedrock directory not found. Creating it..." + mkdir -p packages/contracts-bedrock + fi + cd packages/contracts-bedrock + + # Initialize package.json if it doesn't exist + if [ ! -f "package.json" ]; then + echo "Creating package.json..." + echo '{ + "name": "contracts-bedrock", + "version": "1.0.0", + "private": true, + "description": "Optimism Bedrock smart contracts", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + } + }' > package.json + fi + + # List contents after setup + echo "Directory contents after setup:" + ls -la + + npm install + + - name: Install Foundry + uses: foundry-rs/foundry-toolchain@v1 + + - name: Install contract dependencies + working-directory: packages/contracts-bedrock + run: just install + + - name: Pull artifacts + working-directory: packages/contracts-bedrock + run: bash scripts/ops/pull-artifacts.sh + + - name: Build contracts + working-directory: packages/contracts-bedrock + run: forge build --deny-warnings + + - name: Upload contract artifacts + uses: actions/upload-artifact@v4 + with: + name: contract-artifacts + path: | + packages/contracts-bedrock/artifacts + packages/contracts-bedrock/forge-artifacts + packages/contracts-bedrock/cache + + cannon-prestate: + needs: [contracts-build] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Create contract artifact directories + run: | + mkdir -p packages/contracts-bedrock/{artifacts,forge-artifacts,cache} + + - name: Download contract artifacts + uses: actions/download-artifact@v4 + with: + name: contract-artifacts + path: packages/contracts-bedrock + + - name: Build cannon + run: make cannon + + - name: Build op-program + run: make op-program + + - name: Create prestate directories + run: mkdir -p op-program/bin + + - name: Cache cannon prestate + id: cache-prestate + uses: actions/cache@v4 + with: + path: | + op-program/bin/prestate.bin.gz + op-program/bin/meta.json + op-program/bin/prestate-proof.json + key: cannon-prestate-${{ hashFiles('cannon/bin/cannon') }}-${{ hashFiles('op-program/bin/op-program-client.elf') }} + + - name: Sanitize op-program guest + run: make -f cannon/Makefile sanitize-program GUEST_PROGRAM=op-program/bin/op-program-client.elf + + - name: Generate cannon prestate + if: steps.cache-prestate.outputs.cache-hit != 'true' + run: make cannon-prestate + + - name: Generate cannon-mt prestate + run: make cannon-prestate-mt + + - name: Upload prestate artifacts + uses: actions/upload-artifact@v4 + with: + name: prestate-artifacts + path: | + op-program/bin/prestate.bin.gz + op-program/bin/meta.json + op-program/bin/prestate-proof.json + + cannon-go-test: + needs: [contracts-build] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Create contract artifact directories + run: | + mkdir -p packages/contracts-bedrock/{artifacts,forge-artifacts,cache} + + - name: Download contract artifacts + uses: actions/download-artifact@v4 + with: + name: contract-artifacts + path: packages/contracts-bedrock + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: "1.21" + + - name: Build example binaries + working-directory: cannon/testdata/example + run: make elf + + - name: Run Go lint + working-directory: cannon + run: make lint + + - name: Create test results directory + run: | + mkdir -p tmp/test-results + mkdir -p tmp/testlogs + + - name: Run tests + working-directory: cannon + run: | + go install gotest.tools/gotestsum@latest + gotestsum --format=testname --junitfile=../tmp/test-results/cannon.xml --jsonfile=../tmp/testlogs/log.json \ + -- -parallel="$(nproc)" -coverpkg=github.com/ethereum-optimism/optimism/cannon/... -coverprofile=coverage.out ./... + + - name: Upload coverage + uses: codecov/codecov-action@v4 + with: + files: ./cannon/coverage.out + flags: cannon-go-tests + + - name: Upload test results + uses: actions/upload-artifact@v4 + with: + name: test-results + path: | + tmp/test-results + tmp/testlogs + + # op-e2e-cannon-tests: + # needs: [contracts-build, cannon-prestate] + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + + # - name: Install Foundry + # uses: foundry-rs/foundry-toolchain@v1 + + # - name: Create directories + # run: | + # mkdir -p packages/contracts-bedrock/{artifacts,forge-artifacts,cache} + # mkdir -p op-program/bin + + # - name: Download contract artifacts + # uses: actions/download-artifact@v4 + # with: + # name: contract-artifacts + # path: packages/contracts-bedrock + + # - name: Download prestate artifacts + # uses: actions/download-artifact@v4 + # with: + # name: prestate-artifacts + # path: op-program/bin + + # # Add Go installation since op-e2e tests are in Go + # - name: Set up Go + # uses: actions/setup-go@v5 + # with: + # go-version: "1.21" + + # # Install any required Go dependencies + # - name: Install Go dependencies + # run: | + # go mod download + + # # Debug step to verify forge installation and environment + # - name: Debug environment + # run: | + # echo "Foundry version:" + # forge --version + # echo "Go version:" + # go version + # echo "Directory structure:" + # ls -R packages/contracts-bedrock/ + # ls -R op-program/bin/ + + # - name: Run cannon tests + # working-directory: op-e2e + # run: | + # make test-cannon diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000000..c57c08f8a38a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,106 @@ +name: CI + +env: + FOUNDRY_PROFILE: ci + DOCKER_BUILDKIT: 1 + REGISTRY: ghcr.io + REPO: ${{ github.repository_owner }} + +on: + push: + branches: + - celestia-develop + - test/github-actions + pull_request: + branches: + - celestia-develop + types: [opened, synchronize, reopened] + workflow_dispatch: + +jobs: + docker-build: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + strategy: + matrix: + docker_name: + - op-node + - op-batcher + - op-program + - op-proposer + - op-challenger + - proofs-tools + - op-dispute-mon + - op-conductor + - da-server + - op-supervisor + - cannon + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Generate clean branch name + id: clean_branch + run: | + # Replace invalid characters with dash + CLEAN_BRANCH="$(echo "${{ github.ref_name }}" | sed 's/[^a-zA-Z0-9.]/-/g')" + echo "name=${CLEAN_BRANCH}" >> "$GITHUB_OUTPUT" + + - name: Set build environment + run: | + { + echo "REGISTRY=${{ env.REGISTRY }}" + echo "REPOSITORY=${{ env.REPO }}" + echo "GIT_COMMIT=$(git rev-parse HEAD)" + echo "GIT_DATE=$(git show -s --format='%ct')" + echo "GIT_VERSION=untagged" + echo "IMAGE_TAGS=${{ github.sha }},${{ steps.clean_branch.outputs.name }}" + echo "PLATFORMS=linux/amd64" + } >> "$GITHUB_ENV" + + - name: Build Docker image + run: | + # Create buildx builder + docker buildx create --driver=docker-container --name=buildx-build --bootstrap --use + + # For PRs, use --load to make the image available locally + # For pushes, use --push to publish to registry + OUTPUT_ARG="" + if [ "${{ github.event_name }}" = "pull_request" ]; then + OUTPUT_ARG="--load" + else + OUTPUT_ARG="--push" + fi + + # Build using docker-bake.hcl + docker buildx bake \ + --progress plain \ + --builder=buildx-build \ + -f docker-bake.hcl \ + "${OUTPUT_ARG}" \ + "${{ matrix.docker_name }}" + + - name: Save Docker image + if: github.event_name == 'pull_request' + run: | + CLEAN_TAG="${{ steps.clean_branch.outputs.name }}" + IMAGE_NAME="${{ env.REGISTRY }}/${{ env.REPO }}/${{ matrix.docker_name }}:${CLEAN_TAG}" + docker save "${IMAGE_NAME}" > "/tmp/${{ matrix.docker_name }}.tar" + + - name: Upload Docker image + if: github.event_name == 'pull_request' + uses: actions/upload-artifact@v4 + with: + name: docker-${{ matrix.docker_name }} + path: /tmp/${{ matrix.docker_name }}.tar diff --git a/.github/workflows/tag-service.yml b/.github/workflows/tag-service.yml deleted file mode 100644 index 439b48f13d4e..000000000000 --- a/.github/workflows/tag-service.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: Tag Service - -on: - workflow_dispatch: - inputs: - bump: - description: 'How much to bump the version by' - required: true - type: choice - options: - - major - - minor - - patch - - prerelease - - finalize-prerelease - service: - description: 'Which service to release' - required: true - type: choice - options: - - ci-builder - - ci-builder-rust - - op-node - - op-batcher - - op-proposer - - op-challenger - - op-program - - op-dispute-mon - - op-ufm - - da-server - - op-contracts - - op-conductor - prerelease: - description: Increment major/minor/patch as prerelease? - required: false - type: boolean - default: false - -permissions: - contents: write - -jobs: - release: - runs-on: ubuntu-latest - environment: op-stack-production - - steps: - - uses: actions/checkout@v4 - - name: Fetch tags - run: git fetch --tags origin --force - - name: Setup Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - name: Install deps - run: pip install -r requirements.txt - working-directory: ops/tag-service - - run: ops/tag-service/tag-service.py --bump="$BUMP" --service="$SERVICE" - env: - INPUT_GITHUB_TOKEN: ${{ github.token }} - BUMP: ${{ github.event.inputs.bump }} - SERVICE: ${{ github.event.inputs.service }} - if: ${{ github.event.inputs.prerelease == 'false' }} - - run: ops/tag-service/tag-service.py --bump="$BUMP" --service="$SERVICE" --pre-release - env: - INPUT_GITHUB_TOKEN: ${{ github.token }} - BUMP: ${{ github.event.inputs.bump }} - SERVICE: ${{ github.event.inputs.service }} - if: ${{ github.event.inputs.prerelease == 'true' }} diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml new file mode 100644 index 000000000000..af25a9533814 --- /dev/null +++ b/.github/workflows/validate.yml @@ -0,0 +1,28 @@ +name: Validate Workflows + +on: + pull_request: + branches: + - celestia-develop + paths: + - ".github/workflows/**" + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Create actionlint config + run: | + echo "self-hosted-runner: + # Add your custom runner labels here if needed + labels: + - ubuntu-latest + - ubuntu-latest-16-cores" > .actionlint.yaml + + - name: Validate workflows + uses: docker://rhysd/actionlint:latest + with: + shellcheck: true + fail-on-error: true From 901ac6584c4a84a49f1cd569e221ccd84678384b Mon Sep 17 00:00:00 2001 From: Diego Date: Tue, 19 Nov 2024 11:16:42 -0500 Subject: [PATCH 2/2] rename CI job and delete validate --- .github/workflows/{ci.yml => docker.yml} | 2 +- .github/workflows/validate.yml | 28 ------------------------ 2 files changed, 1 insertion(+), 29 deletions(-) rename .github/workflows/{ci.yml => docker.yml} (99%) delete mode 100644 .github/workflows/validate.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/docker.yml similarity index 99% rename from .github/workflows/ci.yml rename to .github/workflows/docker.yml index c57c08f8a38a..606bc08f0699 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/docker.yml @@ -1,4 +1,4 @@ -name: CI +name: Docker env: FOUNDRY_PROFILE: ci diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml deleted file mode 100644 index af25a9533814..000000000000 --- a/.github/workflows/validate.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Validate Workflows - -on: - pull_request: - branches: - - celestia-develop - paths: - - ".github/workflows/**" - -jobs: - validate: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Create actionlint config - run: | - echo "self-hosted-runner: - # Add your custom runner labels here if needed - labels: - - ubuntu-latest - - ubuntu-latest-16-cores" > .actionlint.yaml - - - name: Validate workflows - uses: docker://rhysd/actionlint:latest - with: - shellcheck: true - fail-on-error: true