incoming http steal #162
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Hello traveler! | |
| # This is the CI workflow for mirrord. | |
| # It is a bit complicated, but it is also very powerful. | |
| # We try to optimize for speed, but sometimes there are limitations. | |
| # Please try to document it here so people won't try repeating your errors. | |
| # 1. GitHub cache is limited to 10GB, so we try to have less jobs to not exceed it, since if we get over 10GB | |
| # the cache will be evicted then builds will be slower, so it's better to have less jobs. | |
| # 2. I (Aviram) tried to use a container to build the Linux stuff, but couldn't get the e2e to work and (minikube in container) | |
| # and the benefit seemed little. | |
| # 3. Please be mindful, we try to target less than 30 minutes for the CI to run. In a perfect world it'd be less than 5m. | |
| # If you're adding something, please make sure it doesn't impact too much, and if you're reviewing please have it in mind. | |
| # 4. Make sure to specify target for cargo invocations, to re-use cache (cargo build --target X while host is X then cargo build will not use cache | |
| # since it's different targets from it's perspective - https://doc.rust-lang.org/cargo/guide/build-cache.html | |
| # | |
| # - Adding a compiled app to e2e: | |
| # | |
| # If you want to add a rust/go/[other compiled language] to the list of e2e apps, then you have to check 2 other | |
| # places. | |
| # | |
| # 1. The `test-images` repo, if your e2e wants to use a custom image, that's not already there, and; | |
| # 2. The `mirrord-ci` repo, where you should add the compilation call to `e2e-setup-action/action.yaml`. | |
| # | |
| # Forgetting (2) will probably net you a message saying that `mirrord exec` could not find the path of the app, | |
| # while forgetting (1) nets you a message saying that the service could not be reached/found. | |
| # | |
| # Caveats: | |
| # | |
| # You cannot use `workspace` dependencies in the test app, it fails to build, must fully specify the dependency. | |
| # | |
| # Related to (2): If you modify `mirrord-ci/actions.yaml`, you'll be affecting every PR! | |
| name: CI | |
| on: | |
| workflow_dispatch: | |
| push: | |
| pull_request: | |
| branches: [main] | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| # Cancel previous runs on the same PR. | |
| concurrency: | |
| group: ${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| env: | |
| CARGO_NET_GIT_FETCH_WITH_CLI: "true" | |
| MIRRORD_TELEMETRY: false | |
| jobs: | |
| towncrier_check: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: install towncrier | |
| run: pip install towncrier | |
| - name: verify newsfragment exist | |
| run: towncrier check | |
| changed_files: | |
| runs-on: ubuntu-latest | |
| # don't run CI on drafts | |
| if: github.event.pull_request.draft == false | |
| outputs: | |
| rs_changed: ${{ steps.changed-rs.outputs.any_changed }} | |
| markdown_changed: ${{ steps.changed-markdown.outputs.any_changed }} | |
| ci_changed: ${{ steps.changed-ci.outputs.any_changed }} | |
| protocol_changed: ${{ steps.changed-protocol.outputs.any_changed }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: get CI changes | |
| id: changed-ci | |
| uses: tj-actions/changed-files@v39 | |
| with: | |
| files: | | |
| .github/workflows/ci.yaml | |
| - name: get changed rs files | |
| id: changed-rs | |
| uses: tj-actions/changed-files@v39 | |
| with: | |
| files: | | |
| **/*.rs | |
| mirrord/** | |
| tests/** | |
| Cargo.toml | |
| Cargo.lock | |
| .dockerignore | |
| rust-toolchain.toml | |
| rustfmt.toml | |
| .cargo/** | |
| - name: get markdown changes | |
| id: changed-markdown | |
| uses: tj-actions/changed-files@v39 | |
| with: | |
| files: | | |
| README.md | |
| - name: get protocol changes | |
| id: changed-protocol | |
| uses: tj-actions/changed-files@v39 | |
| with: | |
| files: | | |
| mirrord/protocol/** | |
| - name: get protocol toml changes | |
| id: changed-protocol-toml | |
| uses: tj-actions/changed-files@v39 | |
| with: | |
| files: | | |
| mirrord/protocol/Cargo.toml | |
| - name: verify protocol bump | |
| run: | | |
| if [ "${{ steps.changed-protocol.outputs.any_changed }}" == "true" ] && [ "${{ steps.changed-protocol-toml.outputs.any_changed }}" != "true" ]; then | |
| echo "Error: Protocol has changed but Cargo.toml has not. Please update Cargo.toml." | |
| exit 1 | |
| fi | |
| - name: output test | |
| run: | | |
| echo ${{ steps.changed-rs.outputs.any_changed }}; | |
| echo ${{ steps.changed-rs.outputs.all_changed_files }}; | |
| echo ${{ steps.changed-markdown.outputs.any_changed }}; | |
| echo ${{ steps.changed-markdown.outputs.all_changed_files }}; | |
| echo ${{ steps.changed-ci.outputs.any_changed }}; | |
| echo ${{ steps.changed-ci.outputs.all_changed_files }}; | |
| echo ${{ steps.changed-protocol.outputs.any_changed }}; | |
| echo ${{ steps.changed-protocol-toml.outputs.any_changed }}; | |
| lint: | |
| runs-on: ubuntu-latest | |
| needs: changed_files | |
| if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: arduino/setup-protoc@v1 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| # Otherwise the arguments to the setup-rust-toolchain action are ignored. | |
| - run: rm rust-toolchain.toml | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| toolchain: nightly-2023-09-07 | |
| components: rustfmt, clippy | |
| target: aarch64-unknown-linux-gnu,x86_64-unknown-linux-gnu | |
| - run: python3 -m pip install cargo-zigbuild | |
| - run: cargo fmt --all -- --check | |
| # x64 | |
| - run: cargo-zigbuild clippy --lib --bins --all-features --target x86_64-unknown-linux-gnu -- -Wclippy::indexing_slicing -D warnings | |
| # Check that compiles for the supported linux targets (aarch64) | |
| - run: cargo-zigbuild clippy --lib --bins --all-features --target aarch64-unknown-linux-gnu -- -Wclippy::indexing_slicing -D warnings | |
| # if the branch is named is `x.x.x`, x ∈ [0, 9], then it's a release branch | |
| # the output of this test is a boolean indicating if it's a release branch | |
| # which is then used by `build_mirrord_on_release_branch` | |
| check_if_release_branch: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| release_branch: ${{ steps.release-branch.outputs.branch }} | |
| steps: | |
| - id: release-branch | |
| run: | | |
| echo "branch=$([[ "${{ github.head_ref || github.ref_name }}" =~ "^[0-9]+\.[0-9]+\.[0-9]+$" ]] && echo "true" || echo "false" )" >> "$GITHUB_OUTPUT" | |
| - name: output test | |
| run: | | |
| echo ${{ steps.release-branch.outputs.branch }} | |
| check-rust-docs: | |
| runs-on: ubuntu-latest | |
| needs: changed_files | |
| if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}} | |
| env: | |
| # enables the creation of a workspace index.html page. | |
| RUSTDOCFLAGS: "--enable-index-page -Zunstable-options" | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: arduino/setup-protoc@v1 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| # TODO(alex): update this when https://github.com/rust-lang/rust/issues/109631 is fixed | |
| toolchain: nightly-2023-09-07 | |
| - run: cargo doc --document-private-items | |
| test_agent: | |
| runs-on: ubuntu-latest | |
| needs: changed_files | |
| if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}} | |
| container: | |
| image: ghcr.io/metalbear-co/ci-agent-build:f8330d35a2a4b9132138f6fa9a3f3f80768c7c32 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: arduino/setup-protoc@v1 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: test | |
| run: cargo test --target x86_64-unknown-linux-gnu -p mirrord-agent | |
| test_agent_image: | |
| runs-on: ubuntu-latest | |
| needs: changed_files | |
| if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true' }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: docker/setup-buildx-action@v2 | |
| - name: build and export | |
| uses: docker/build-push-action@v3 | |
| with: | |
| context: . | |
| tags: test | |
| file: mirrord/agent/Dockerfile | |
| outputs: type=docker,dest=/tmp/test.tar | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| - name: upload image | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: test | |
| path: /tmp/test.tar | |
| integration_tests: | |
| runs-on: ubuntu-latest | |
| needs: [changed_files] | |
| if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}} | |
| steps: | |
| - uses: actions/checkout@v3 # Checkout the mirrord repo. | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 # Install rust. | |
| with: | |
| target: x86_64-unknown-linux-gnu | |
| - run: | | |
| cd mirrord/layer/tests/apps/issue1123 | |
| rustc issue1123.rs --out-dir target | |
| - run: | | |
| cd mirrord/layer/tests/apps/issue1054 | |
| rustc issue1054.rs --out-dir target | |
| - run: | | |
| cd mirrord/layer/tests/apps/issue1458 | |
| rustc issue1458.rs --out-dir target | |
| - run: | | |
| cd mirrord/layer/tests/apps/issue1458portnot53 | |
| rustc issue1458portnot53.rs --out-dir target | |
| # For the `java_temurin_sip` test. | |
| - uses: sdkman/sdkman-action@b1f9b696c79148b66d3d3a06f7ea801820318d0f | |
| id: sdkman | |
| with: | |
| candidate: java | |
| version: 17.0.6-tem | |
| - run: java -version | |
| - uses: actions/setup-node@v3 # For http mirroring test. | |
| with: | |
| node-version: 14 | |
| - run: npm install express # For http mirroring test with node. | |
| - uses: actions/setup-python@v3 # For http mirroring tests with Flask and FastAPI. | |
| - run: pip3 install flask fastapi uvicorn[standard] # For http mirroring test with Flask. | |
| - uses: actions/setup-go@v4 | |
| with: | |
| go-version: "1.18" | |
| cache-dependency-path: tests/go-e2e/go.sum | |
| - run: | | |
| go version | |
| - run: | # Build Go test apps. | |
| ./scripts/build_go_apps.sh 18 | |
| # don't use "cache" for other Gos since it will try to overwrite and have bad results. | |
| - uses: actions/setup-go@v4 | |
| with: | |
| go-version: "1.19" | |
| cache: false | |
| - run: | | |
| go version | |
| - run: | # Build Go test apps. | |
| ./scripts/build_go_apps.sh 19 | |
| - uses: actions/setup-go@v4 | |
| with: | |
| go-version: "1.20" | |
| cache: false | |
| - run: | | |
| go version | |
| - run: | # Build Go test apps. | |
| ./scripts/build_go_apps.sh 20 | |
| - run: | | |
| cd mirrord/layer/tests/apps/fileops | |
| cargo build | |
| - run: | | |
| cd mirrord/layer/tests/apps/outgoing | |
| cargo build | |
| - run: | | |
| cd mirrord/layer/tests/apps/recv_from | |
| cargo build | |
| - run: | | |
| cd mirrord/layer/tests/apps/dns_resolve | |
| cargo build | |
| - run: | | |
| cd mirrord/layer/tests/apps/listen_ports | |
| cargo build | |
| - run: | | |
| cd mirrord/layer/tests/apps/issue1776 | |
| cargo build | |
| - run: | | |
| cd mirrord/layer/tests/apps/issue1776portnot53 | |
| cargo build | |
| - run: | | |
| cd mirrord/layer/tests/apps/issue1899 | |
| cargo build | |
| - run: ./scripts/build_c_apps.sh | |
| - run: cargo test --target x86_64-unknown-linux-gnu -p mirrord-layer | |
| - name: mirrord protocol UT | |
| run: cargo test --target x86_64-unknown-linux-gnu -p mirrord-protocol | |
| - name: mirrord config UT | |
| run: cargo test --target x86_64-unknown-linux-gnu -p mirrord-config | |
| - name: mirrord kube UT | |
| run: cargo test --target x86_64-unknown-linux-gnu -p mirrord-kube --all-features | |
| macos_tests: | |
| runs-on: macos-13 | |
| needs: changed_files | |
| if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}} | |
| env: | |
| MIRRORD_TEST_USE_EXISTING_LIB: ../../target/x86_64-apple-darwin/debug/libmirrord_layer.dylib | |
| steps: | |
| - uses: actions/checkout@v3 # Checkout the mirrord repo. | |
| # the setup rust toolchain action ignores the input if file exists.. so remove it | |
| - run: rm rust-toolchain.toml | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| components: rustfmt, clippy | |
| target: aarch64-apple-darwin | |
| toolchain: nightly-2023-09-07 | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v2 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: clippy x64 | |
| run: cargo clippy -p mirrord -p mirrord-layer -p mirrord-sip --target=x86_64-apple-darwin -- -Wclippy::indexing_slicing -D warnings | |
| - name: clippy aarch64 | |
| run: cargo clippy -p mirrord -p mirrord-layer -p mirrord-sip --target=aarch64-apple-darwin -- -Wclippy::indexing_slicing -D warnings | |
| - name: mirrord SIP UT | |
| run: cargo test --target=x86_64-apple-darwin -p mirrord-sip | |
| # prepare stuff needed for integration tests | |
| - run: | | |
| cd mirrord/layer/tests/apps/issue1123 | |
| rustc issue1123.rs --out-dir target | |
| - run: | | |
| cd mirrord/layer/tests/apps/issue1054 | |
| rustc issue1054.rs --out-dir target | |
| - run: | | |
| cd mirrord/layer/tests/apps/issue1458 | |
| rustc issue1458.rs --out-dir target | |
| - run: | | |
| cd mirrord/layer/tests/apps/issue1458portnot53 | |
| rustc issue1458portnot53.rs --out-dir target | |
| - uses: actions/setup-go@v4 | |
| with: | |
| go-version: "1.18" | |
| cache-dependency-path: tests/go-e2e/go.sum | |
| - run: | | |
| go version | |
| # don't use "cache" for other Gos since it will try to overwrite and have bad results. | |
| - run: | # Build Go test apps. | |
| ./scripts/build_go_apps.sh 18 | |
| - uses: actions/setup-go@v4 | |
| with: | |
| go-version: "1.19" | |
| cache: false | |
| - run: | | |
| go version | |
| - run: | # Build Go test apps. | |
| ./scripts/build_go_apps.sh 19 | |
| - uses: actions/setup-go@v4 | |
| with: | |
| go-version: "1.20" | |
| cache: false | |
| - run: | | |
| go version | |
| - run: | # Build Go test apps. | |
| ./scripts/build_go_apps.sh 20 | |
| - run: | | |
| cd mirrord/layer/tests/apps/fileops | |
| cargo build | |
| - run: | | |
| cd mirrord/layer/tests/apps/outgoing | |
| cargo build | |
| - run: | | |
| cd mirrord/layer/tests/apps/recv_from | |
| cargo build | |
| - run: | | |
| cd mirrord/layer/tests/apps/dns_resolve | |
| cargo build | |
| - run: | | |
| cd mirrord/layer/tests/apps/issue1776 | |
| cargo build | |
| - run: | | |
| cd mirrord/layer/tests/apps/issue1776portnot53 | |
| cargo build | |
| - run: | | |
| cd mirrord/layer/tests/apps/issue1899 | |
| cargo build | |
| - run: ./scripts/build_c_apps.sh | |
| # For the `java_temurin_sip` test. | |
| - uses: sdkman/sdkman-action@b1f9b696c79148b66d3d3a06f7ea801820318d0f | |
| id: sdkman | |
| with: | |
| candidate: java | |
| version: 17.0.6-tem | |
| - run: java -version | |
| - uses: actions/setup-python@v3 # For http mirroring tests with Flask and FastAPI. | |
| - run: pip3 install flask # For http mirroring test with Flask. | |
| - run: pip3 install fastapi # For http mirroring test with FastAPI. | |
| - run: pip3 install uvicorn[standard] # For http mirroring test with FastAPI. | |
| - uses: actions/setup-node@v3 | |
| with: | |
| node-version: 18 | |
| - run: npm install express # For http mirroring test with node. | |
| - run: cargo build --target=x86_64-apple-darwin -p mirrord-layer # Build layer lib. The tests load it into the apps. | |
| - name: mirrord layer tests | |
| run: cargo test --target=x86_64-apple-darwin -p mirrord-layer | |
| e2e: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| container-runtime: ["docker", "containerd"] | |
| name: e2e | |
| needs: [test_agent_image, changed_files] | |
| if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}} | |
| env: | |
| MIRRORD_AGENT_RUST_LOG: "warn,mirrord=debug" | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 # Install Rust. | |
| - uses: metalbear-co/ci/e2e-setup-action@main | |
| with: | |
| container-runtime: ${{matrix.container-runtime}} | |
| - name: download image | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: test | |
| path: /tmp | |
| - run: minikube image load /tmp/test.tar | |
| # By running the test of the targetless agent first, we prove it works on an empty cluster without any pods. | |
| - name: Run targetless E2E test. | |
| run: cargo test --target=x86_64-unknown-linux-gnu -p tests targetless --no-default-features --features job | |
| - name: Run all E2E test - docker | |
| if: ${{ matrix.container-runtime == 'docker' }} | |
| run: cargo test --target=x86_64-unknown-linux-gnu -p tests --features docker -- --test-threads=6 | |
| - name: Run all E2E test - containerd | |
| if: ${{ matrix.container-runtime == 'containerd' }} | |
| run: cargo test --target=x86_64-unknown-linux-gnu -p tests -- --test-threads=6 | |
| - name: Collect logs | |
| if: ${{ failure() }} | |
| run: | | |
| kubectl describe pods | |
| docker exec minikube find /var/log/pods -print -exec cat {} \; | |
| lint_markdown: | |
| runs-on: ubuntu-latest | |
| needs: changed_files | |
| if: ${{needs.changed_files.outputs.markdown_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: avto-dev/markdown-lint@v1 | |
| with: | |
| config: "markdownlint-config.json" | |
| args: "README.md" | |
| # we build mirrord to run ide tests, while it can take time for e2e/integration to finish, | |
| # building concurrently can run faster the release ide tests | |
| build_mirrord_on_release_branch: | |
| runs-on: ubuntu-latest | |
| name: build mirrord | |
| needs: check_if_release_branch | |
| if: ${{ needs.check_if_release_branch.outputs.release_branch == 'true' }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: arduino/setup-protoc@v1 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| - run: cargo build --manifest-path=./Cargo.toml | |
| - name: upload layer | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: mirrord-artifacts | |
| path: | | |
| target/debug/mirrord | |
| if-no-files-found: error | |
| # depends on `build_mirrord_on_release_branch` which provides the binary on the current tag | |
| # `build_mirrord_on_release_branch` depends on `check_if_release_branch` | |
| # which checks if the branch is a release branch | |
| intellij_e2e_on_release_branch: | |
| needs: build_mirrord_on_release_branch | |
| uses: metalbear-co/mirrord-intellij/.github/workflows/reusable_e2e.yaml@main | |
| with: | |
| mirrord_release_branch: true | |
| vscode_e2e_on_release_branch: | |
| needs: build_mirrord_on_release_branch | |
| uses: metalbear-co/mirrord-vscode/.github/workflows/reusable_e2e.yaml@main | |
| with: | |
| mirrord_release_branch: true | |
| # We need some "accummulation" job here because bors fails (timeouts) to | |
| # listen on matrix builds. | |
| # Hence, we have some kind of dummy here that bors can listen on | |
| ci-success: | |
| name: ci | |
| # We want this to run even if some of the required jobs got skipped | |
| if: always() | |
| needs: | |
| [ | |
| towncrier_check, | |
| changed_files, | |
| intellij_e2e_on_release_branch, | |
| vscode_e2e_on_release_branch, | |
| test_agent_image, | |
| macos_tests, | |
| integration_tests, | |
| e2e, | |
| test_agent, | |
| lint, | |
| lint_markdown, | |
| ] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: CI succeeded | |
| # We have to do it in the shell since if it's in the if condition | |
| # then skipping is considered success by branch protection rules | |
| env: | |
| CI_SUCCESS: ${{ (needs.changed_files.result == 'success') && | |
| (needs.towncrier_check.result == 'success') && | |
| (needs.test_agent_image.result == 'success' || needs.test_agent_image.result == 'skipped') && | |
| (needs.macos_tests.result == 'success' || needs.macos_tests.result == 'skipped') && | |
| (needs.integration_tests.result == 'success' || needs.integration_tests.result == 'skipped') && | |
| (needs.e2e.result == 'success' || needs.e2e.result == 'skipped') && | |
| (needs.test_agent.result == 'success' || needs.test_agent.result == 'skipped') && | |
| (needs.lint.result == 'success' || needs.lint.result == 'skipped') && | |
| (needs.lint_markdown.result == 'success' || needs.lint_markdown.result == 'skipped') }} | |
| run: echo $CI_SUCCESS && if [ "$CI_SUCCESS" == "true" ]; then echo "SUCCESS" && exit 0; else echo "Failure" && exit 1; fi |