Add option to run agent container as privileged (#1808) #131
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches-ignore: [staging-squash-merge.tmp] | |
| pull_request: | |
| branches: [main, staging, trying] | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| # Cancel previous runs on the same PR. | |
| concurrency: | |
| group: ${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| env: | |
| CARGO_NET_GIT_FETCH_WITH_CLI: "true" | |
| MIRRORD_TELEMETRY: false | |
| jobs: | |
| towncrier_check: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: install towncrier | |
| run: pip install towncrier | |
| - name: verify newsfragment exist | |
| run: towncrier check | |
| changed_files: | |
| runs-on: ubuntu-latest | |
| # don't run CI on drafts | |
| if: github.event.pull_request.draft == false | |
| outputs: | |
| rs_changed: ${{ steps.changed-rs.outputs.any_changed }} | |
| markdown_changed: ${{ steps.changed-markdown.outputs.any_changed }} | |
| ci_changed: ${{ steps.changed-ci.outputs.any_changed }} | |
| protocol_changed: ${{ steps.changed-protocol.outputs.any_changed }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: get CI changes | |
| id: changed-ci | |
| uses: tj-actions/changed-files@v36 | |
| with: | |
| files: | | |
| .github/workflows/ci.yaml | |
| - name: get changed rs files | |
| id: changed-rs | |
| uses: tj-actions/changed-files@v36 | |
| with: | |
| files: | | |
| **/*.rs | |
| mirrord/** | |
| tests/** | |
| Cargo.toml | |
| Cargo.lock | |
| .dockerignore | |
| rust-toolchain.toml | |
| rustfmt.toml | |
| .cargo/** | |
| - name: get markdown changes | |
| id: changed-markdown | |
| uses: tj-actions/changed-files@v36 | |
| with: | |
| files: | | |
| README.md | |
| - name: get protocol changes | |
| id: changed-protocol | |
| uses: tj-actions/changed-files@v36 | |
| with: | |
| files: | | |
| mirrord/protocol/** | |
| - name: get protocol toml changes | |
| id: changed-protocol-toml | |
| uses: tj-actions/changed-files@v36 | |
| with: | |
| files: | | |
| mirrord/protocol/Cargo.toml | |
| - name: verify protocol bump | |
| run: | | |
| if [ "${{ steps.changed-protocol.outputs.any_changed }}" == "true" ] && [ "${{ steps.changed-protocol-toml.outputs.any_changed }}" != "true" ]; then | |
| echo "Error: Protocol has changed but Cargo.toml has not. Please update Cargo.toml." | |
| exit 1 | |
| fi | |
| - name: output test | |
| run: | | |
| echo ${{ steps.changed-rs.outputs.any_changed }}; | |
| echo ${{ steps.changed-rs.outputs.all_changed_files }}; | |
| echo ${{ steps.changed-markdown.outputs.any_changed }}; | |
| echo ${{ steps.changed-markdown.outputs.all_changed_files }}; | |
| echo ${{ steps.changed-ci.outputs.any_changed }}; | |
| echo ${{ steps.changed-ci.outputs.all_changed_files }}; | |
| echo ${{ steps.changed-protocol.outputs.any_changed }}; | |
| echo ${{ steps.changed-protocol-toml.outputs.any_changed }}; | |
| lint: | |
| runs-on: ubuntu-latest | |
| needs: changed_files | |
| if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: arduino/setup-protoc@v1 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| # Otherwise the arguments to the setup-rust-toolchain action are ignored. | |
| - run: rm rust-toolchain.toml | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| toolchain: nightly-2023-04-19 | |
| components: rustfmt, clippy | |
| target: aarch64-unknown-linux-gnu,x86_64-unknown-linux-gnu | |
| - run: python3 -m pip install cargo-zigbuild | |
| # Needed for agent | |
| - run: cargo fmt --all -- --check | |
| - run: cargo-zigbuild clippy --lib --bins --all-features -- -Wclippy::indexing_slicing -D warnings | |
| # Check that agent compiles for the supported targets (aarch64) | |
| - run: cargo-zigbuild clippy -p mirrord-agent --target aarch64-unknown-linux-gnu -- -Wclippy::indexing_slicing -D warnings | |
| check-rust-docs: | |
| runs-on: ubuntu-latest | |
| needs: changed_files | |
| if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}} | |
| env: | |
| # enables the creation of a workspace index.html page. | |
| RUSTDOCFLAGS: "--enable-index-page -Zunstable-options" | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: arduino/setup-protoc@v1 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| # TODO(alex): update this when https://github.com/rust-lang/rust/issues/109631 is fixed | |
| toolchain: nightly-2023-04-19 | |
| - run: cargo doc --document-private-items | |
| lint_macos: | |
| runs-on: macos-latest | |
| needs: changed_files | |
| if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| # the setup rust toolchain action ignores the input if file exists.. so remove it | |
| - run: rm rust-toolchain.toml | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| components: rustfmt, clippy | |
| target: aarch64-apple-darwin | |
| toolchain: nightly-2023-04-19 | |
| # only mirrord, mirrord-sip contain macOS specific, so run clippy on that. | |
| - run: cargo clippy -p mirrord -p mirrord-sip --target=x86_64-apple-darwin -- -Wclippy::indexing_slicing -D warnings | |
| - run: cargo clippy -p mirrord -p mirrord-sip --target=aarch64-apple-darwin -- -Wclippy::indexing_slicing -D warnings | |
| test_mirrord_protocol: | |
| runs-on: ubuntu-latest | |
| needs: changed_files | |
| if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| - run: | | |
| cargo test -p mirrord-protocol | |
| test_mirrord_config: | |
| runs-on: ubuntu-latest | |
| needs: changed_files | |
| if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| - run: | | |
| cargo test -p mirrord-config | |
| test_mirrord_kube: | |
| runs-on: ubuntu-latest | |
| needs: changed_files | |
| if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| - run: | | |
| cargo test -p mirrord-kube --all-features | |
| test_mirrord_sip: | |
| runs-on: macos-latest | |
| needs: changed_files | |
| if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| - run: | | |
| cargo test -p mirrord-sip | |
| test_agent: | |
| runs-on: ubuntu-latest | |
| needs: changed_files | |
| if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}} | |
| container: | |
| image: ghcr.io/metalbear-co/ci-agent-build:f8330d35a2a4b9132138f6fa9a3f3f80768c7c32 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: arduino/setup-protoc@v1 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: test | |
| run: cargo test -p mirrord-agent | |
| test_agent_image: | |
| runs-on: ubuntu-latest | |
| needs: changed_files | |
| if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true' }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: docker/setup-buildx-action@v2 | |
| - name: build and export | |
| uses: docker/build-push-action@v3 | |
| with: | |
| context: . | |
| tags: test | |
| file: mirrord/agent/Dockerfile | |
| outputs: type=docker,dest=/tmp/test.tar | |
| cache-from: type=gha | |
| cache-to: type=gha,mode=max | |
| - name: upload image | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: test | |
| path: /tmp/test.tar | |
| test_mirrord_layer_cli: | |
| needs: changed_files | |
| if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest, macos-12, macos-11] | |
| target: | |
| [x86_64-unknown-linux-gnu, x86_64-apple-darwin, aarch64-apple-darwin] | |
| exclude: | |
| - os: ubuntu-latest | |
| target: x86_64-apple-darwin | |
| - os: ubuntu-latest | |
| target: aarch64-apple-darwin | |
| - os: macos-12 | |
| target: x86_64-unknown-linux-gnu | |
| - os: macos-11 | |
| target: aarch64-apple-darwin | |
| - os: macos-11 | |
| target: x86_64-unknown-linux-gnu | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| # the setup rust toolchain action ignores the input if file exists.. so remove it | |
| - if: ${{matrix.target == 'aarch64-apple-darwin'}} | |
| run: rm rust-toolchain.toml | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| target: ${{matrix.target}} | |
| toolchain: nightly-2023-04-19 | |
| # For now, just verify it compiles. | |
| - run: cargo build -p mirrord-layer -p mirrord --target=${{matrix.target}} | |
| build_mirrord: | |
| runs-on: ubuntu-latest | |
| name: build mirrord | |
| needs: changed_files | |
| if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: arduino/setup-protoc@v1 | |
| with: | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| components: rustfmt | |
| - run: cargo build --manifest-path=./Cargo.toml | |
| - name: upload layer | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: mirrord-artifacts | |
| path: | | |
| target/debug/libmirrord_layer.so | |
| target/debug/mirrord | |
| if-no-files-found: error | |
| integration_tests: | |
| runs-on: ubuntu-latest | |
| needs: [build_mirrord, changed_files] | |
| if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}} | |
| env: | |
| MIRRORD_TEST_USE_EXISTING_LIB: /home/runner/work/mirrord/mirrord/target/debug/libmirrord_layer.so | |
| steps: | |
| - uses: actions/checkout@v3 # Checkout the mirrord repo. | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 # Install rust. | |
| with: | |
| target: x86_64-unknown-linux-gnu | |
| - run: | | |
| cd mirrord/layer/tests/apps/issue1123 | |
| rustc issue1123.rs --out-dir target | |
| - run: | | |
| cd mirrord/layer/tests/apps/issue1054 | |
| rustc issue1054.rs --out-dir target | |
| - run: | | |
| cd mirrord/layer/tests/apps/issue1458 | |
| rustc issue1458.rs --out-dir target | |
| - run: | | |
| cd mirrord/layer/tests/apps/issue1458portnot53 | |
| rustc issue1458portnot53.rs --out-dir target | |
| # For the `java_temurin_sip` test. | |
| - uses: sdkman/sdkman-action@b1f9b696c79148b66d3d3a06f7ea801820318d0f | |
| id: sdkman | |
| with: | |
| candidate: java | |
| version: 17.0.6-tem | |
| - run: java -version | |
| - uses: actions/setup-node@v3 # For http mirroring test. | |
| with: | |
| node-version: 14 | |
| - run: npm install express # For http mirroring test with node. | |
| - uses: actions/setup-python@v3 # For http mirroring tests with Flask and FastAPI. | |
| - run: pip3 install flask # For http mirroring test with Flask. | |
| - run: pip3 install fastapi # For http mirroring test with FastAPI. | |
| - run: pip3 install uvicorn[standard] # For http mirroring test with FastAPI. | |
| - uses: actions/setup-go@v3 | |
| with: | |
| go-version: "1.20.0-rc.3" | |
| - run: | | |
| go version | |
| - run: | # Build Go test apps. | |
| ./scripts/build_go_apps.sh 20 | |
| - uses: actions/setup-go@v3 | |
| with: | |
| go-version: "1.19.5" | |
| - run: | | |
| go version | |
| - run: | # Build Go test apps. | |
| ./scripts/build_go_apps.sh 19 | |
| - uses: actions/setup-go@v3 | |
| with: | |
| go-version: "1.18.10" | |
| - run: | | |
| go version | |
| - run: | # Build Go test apps. | |
| ./scripts/build_go_apps.sh 18 | |
| - run: | | |
| cd mirrord/layer/tests/apps/fileops | |
| cargo build | |
| - run: | | |
| cd mirrord/layer/tests/apps/outgoing | |
| cargo build | |
| - run: | | |
| cd mirrord/layer/tests/apps/recv_from | |
| cargo build | |
| - run: | | |
| cd mirrord/layer/tests/apps/dns_resolve | |
| cargo build | |
| - run: | | |
| cd mirrord/layer/tests/apps/listen_ports | |
| cargo build | |
| - run: | | |
| cd mirrord/layer/tests/apps/issue1776 | |
| cargo build | |
| - run: | | |
| cd mirrord/layer/tests/apps/issue1776portnot53 | |
| cargo build | |
| - run: ./scripts/build_c_apps.sh | |
| - name: download layer # Download layer lib built in the `build_mirrord` job. | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: mirrord-artifacts | |
| path: target/debug/ | |
| - run: cargo test -p mirrord-layer | |
| integration_tests_macos: | |
| runs-on: macos-12 | |
| needs: changed_files | |
| if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}} | |
| env: | |
| MIRRORD_TEST_USE_EXISTING_LIB: ../../target/debug/libmirrord_layer.dylib | |
| RUST_LOG: warn,mirrord=trace | |
| steps: | |
| - uses: actions/checkout@v3 # Checkout the mirrord repo. | |
| - uses: actions-rust-lang/setup-rust-toolchain@v1 # Install Rust. | |
| with: | |
| target: x86_64-apple-darwin | |
| - run: | | |
| cd mirrord/layer/tests/apps/issue1123 | |
| rustc issue1123.rs --out-dir target | |
| - run: | | |
| cd mirrord/layer/tests/apps/issue1054 | |
| rustc issue1054.rs --out-dir target | |
| - run: | | |
| cd mirrord/layer/tests/apps/issue1458 | |
| rustc issue1458.rs --out-dir target | |
| - run: | | |
| cd mirrord/layer/tests/apps/issue1458portnot53 | |
| rustc issue1458portnot53.rs --out-dir target | |
| - uses: actions/setup-go@v3 # Install Go for http mirroring tests with a Go webserver. | |
| with: | |
| go-version: "1.18" | |
| - run: | | |
| go version | |
| - run: | # Build Go test apps. | |
| ./scripts/build_go_apps.sh 18 | |
| - uses: actions/setup-go@v3 # Install Go for http mirroring tests with a Go webserver. | |
| with: | |
| go-version: "1.19" | |
| - run: | | |
| go version | |
| - run: | # Build Go test apps. | |
| ./scripts/build_go_apps.sh 19 | |
| - uses: actions/setup-go@v3 # Install Go for http mirroring tests with a Go webserver. | |
| with: | |
| go-version: "1.20.0-rc.3" | |
| - run: | | |
| go version | |
| - run: | # Build Go test apps. | |
| ./scripts/build_go_apps.sh 20 | |
| - run: | | |
| cd mirrord/layer/tests/apps/fileops | |
| cargo build | |
| - run: | | |
| cd mirrord/layer/tests/apps/outgoing | |
| cargo build | |
| - run: | | |
| cd mirrord/layer/tests/apps/recv_from | |
| cargo build | |
| - run: | | |
| cd mirrord/layer/tests/apps/dns_resolve | |
| cargo build | |
| - run: | | |
| cd mirrord/layer/tests/apps/issue1776 | |
| cargo build | |
| - run: | | |
| cd mirrord/layer/tests/apps/issue1776portnot53 | |
| cargo build | |
| - run: ./scripts/build_c_apps.sh | |
| # For the `java_temurin_sip` test. | |
| - uses: sdkman/sdkman-action@b1f9b696c79148b66d3d3a06f7ea801820318d0f | |
| id: sdkman | |
| with: | |
| candidate: java | |
| version: 17.0.6-tem | |
| - run: java -version | |
| - uses: actions/setup-node@v3 | |
| with: | |
| node-version: 14 | |
| - run: npm install express # For http mirroring test with node. | |
| - uses: actions/setup-python@v3 # For http mirroring tests with Flask and FastAPI. | |
| - run: pip3 install flask # For http mirroring test with Flask. | |
| - run: pip3 install fastapi # For http mirroring test with FastAPI. | |
| - run: pip3 install uvicorn[standard] # For http mirroring test with FastAPI. | |
| - run: cargo build -p mirrord-layer # Build layer lib. The tests load it into the apps. | |
| - uses: actions/setup-node@v3 # version 19 spawns processes with `posix_spawn`, so test that also. | |
| with: | |
| node-version: 19 | |
| - run: cargo test -p mirrord-layer | |
| e2e: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| container-runtime: ["docker", "containerd"] | |
| name: e2e | |
| needs: [build_mirrord, test_agent_image, changed_files] | |
| if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}} | |
| env: | |
| MIRRORD_AGENT_RUST_LOG: "warn,mirrord=debug" | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - run: ./scripts/setup_e2e.sh | |
| - name: download image | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: test | |
| path: /tmp | |
| - name: start minikube | |
| uses: medyagh/setup-minikube@master | |
| with: | |
| container-runtime: ${{matrix.container-runtime}} | |
| - run: minikube image load /tmp/test.tar | |
| # By running the test of the targetless agent first, we prove it works on an empty cluster without any pods. | |
| - name: Run targetless E2E test. | |
| run: cargo test -p tests targetless | |
| - name: Run all E2E test | |
| run: cargo test -p tests -- --test-threads=6 | |
| - name: Collect logs | |
| if: ${{ failure() }} | |
| run: | | |
| kubectl describe pods | |
| docker exec minikube find /var/log/pods -print -exec cat {} \; | |
| lint_markdown: | |
| runs-on: ubuntu-latest | |
| needs: changed_files | |
| if: ${{needs.changed_files.outputs.markdown_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: avto-dev/markdown-lint@v1 | |
| with: | |
| config: "markdownlint-config.json" | |
| args: "README.md" | |
| # We need some "accummulation" job here because bors fails (timeouts) to | |
| # listen on matrix builds. | |
| # Hence, we have some kind of dummy here that bors can listen on | |
| ci-success: | |
| name: ci | |
| # We want this to run even if some of the required jobs got skipped | |
| if: always() | |
| needs: | |
| [ | |
| towncrier_check, | |
| changed_files, | |
| build_mirrord, | |
| test_agent_image, | |
| integration_tests_macos, | |
| integration_tests, | |
| e2e, | |
| test_mirrord_layer_cli, | |
| test_agent, | |
| lint, | |
| lint_macos, | |
| test_mirrord_config, | |
| test_mirrord_protocol, | |
| test_mirrord_sip, | |
| lint_markdown, | |
| ] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: CI succeeded | |
| # We have to do it in the shell since if it's in the if condition | |
| # then skipping is considered success by branch protection rules | |
| env: | |
| CI_SUCCESS: ${{ (needs.changed_files.result == 'success') && | |
| (needs.towncrier_check.result == 'success') && | |
| (needs.build_mirrord.result == 'success' || needs.build_mirrord.result == 'skipped') && | |
| (needs.test_agent_image.result == 'success' || needs.test_agent_image.result == 'skipped') && | |
| (needs.integration_tests_macos.result == 'success' || needs.integration_tests_macos.result == 'skipped') && | |
| (needs.integration_tests.result == 'success' || needs.integration_tests.result == 'skipped') && | |
| (needs.e2e.result == 'success' || needs.e2e.result == 'skipped') && | |
| (needs.test_mirrord_layer_cli.result == 'success' || needs.test_mirrord_layer_cli.result == 'skipped') && | |
| (needs.test_agent.result == 'success' || needs.test_agent.result == 'skipped') && | |
| (needs.lint.result == 'success' || needs.lint.result == 'skipped') && | |
| (needs.lint_macos.result == 'success' || needs.lint_macos.result == 'skipped') && | |
| (needs.test_mirrord_config.result == 'success' || needs.test_mirrord_config.result == 'skipped') && | |
| (needs.test_mirrord_protocol.result == 'success' || needs.test_mirrord_protocol.result == 'skipped') && | |
| (needs.test_mirrord_sip.result == 'success' || needs.test_mirrord_sip.result == 'skipped') && | |
| (needs.lint_markdown.result == 'success' || needs.lint_markdown.result == 'skipped') }} | |
| run: echo $CI_SUCCESS && if [ "$CI_SUCCESS" == "true" ]; then echo "SUCCESS" && exit 0; else echo "Failure" && exit 1; fi |