Skip to content

Using json progress tracker as default in extension #129

Using json progress tracker as default in extension

Using json progress tracker as default in extension #129

Workflow file for this run

name: CI
on:
workflow_dispatch:
push:
branches-ignore: [staging-squash-merge.tmp]
pull_request:
branches: [main, staging, trying]
types: [opened, synchronize, reopened, ready_for_review]
# Cancel previous runs on the same PR.
concurrency:
group: ${{ github.head_ref || github.run_id }}
cancel-in-progress: true
env:
CARGO_NET_GIT_FETCH_WITH_CLI: "true"
MIRRORD_TELEMETRY: false
jobs:
towncrier_check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: install towncrier
run: pip install towncrier
- name: verify newsfragment exist
run: towncrier check
changed_files:
runs-on: ubuntu-latest
# don't run CI on drafts
if: github.event.pull_request.draft == false
outputs:
rs_changed: ${{ steps.changed-rs.outputs.any_changed }}
markdown_changed: ${{ steps.changed-markdown.outputs.any_changed }}
ci_changed: ${{ steps.changed-ci.outputs.any_changed }}
protocol_changed: ${{ steps.changed-protocol.outputs.any_changed }}
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: get CI changes
id: changed-ci
uses: tj-actions/changed-files@v36
with:
files: |
.github/workflows/ci.yaml
- name: get changed rs files
id: changed-rs
uses: tj-actions/changed-files@v36
with:
files: |
**/*.rs
mirrord/**
tests/**
Cargo.toml
Cargo.lock
.dockerignore
rust-toolchain.toml
rustfmt.toml
.cargo/**
- name: get markdown changes
id: changed-markdown
uses: tj-actions/changed-files@v36
with:
files: |
README.md
- name: get protocol changes
id: changed-protocol
uses: tj-actions/changed-files@v36
with:
files: |
mirrord/protocol/**
- name: get protocol toml changes
id: changed-protocol-toml
uses: tj-actions/changed-files@v36
with:
files: |
mirrord/protocol/Cargo.toml
- name: verify protocol bump
run: |
if [ "${{ steps.changed-protocol.outputs.any_changed }}" == "true" ] && [ "${{ steps.changed-protocol-toml.outputs.any_changed }}" != "true" ]; then
echo "Error: Protocol has changed but Cargo.toml has not. Please update Cargo.toml."
exit 1
fi
- name: output test
run: |
echo ${{ steps.changed-rs.outputs.any_changed }};
echo ${{ steps.changed-rs.outputs.all_changed_files }};
echo ${{ steps.changed-markdown.outputs.any_changed }};
echo ${{ steps.changed-markdown.outputs.all_changed_files }};
echo ${{ steps.changed-ci.outputs.any_changed }};
echo ${{ steps.changed-ci.outputs.all_changed_files }};
echo ${{ steps.changed-protocol.outputs.any_changed }};
echo ${{ steps.changed-protocol-toml.outputs.any_changed }};
lint:
runs-on: ubuntu-latest
needs: changed_files
if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}}
steps:
- uses: actions/checkout@v3
- uses: arduino/setup-protoc@v1
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
# Otherwise the arguments to the setup-rust-toolchain action are ignored.
- run: rm rust-toolchain.toml
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
toolchain: nightly-2023-04-19
components: rustfmt, clippy
target: aarch64-unknown-linux-gnu,x86_64-unknown-linux-gnu
- run: python3 -m pip install cargo-zigbuild
# Needed for agent
- run: cargo fmt --all -- --check
- run: cargo-zigbuild clippy --lib --bins --all-features -- -Wclippy::indexing_slicing -D warnings
# Check that agent compiles for the supported targets (aarch64)
- run: cargo-zigbuild clippy -p mirrord-agent --target aarch64-unknown-linux-gnu -- -Wclippy::indexing_slicing -D warnings
check-rust-docs:
runs-on: ubuntu-latest
needs: changed_files
if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}}
env:
# enables the creation of a workspace index.html page.
RUSTDOCFLAGS: "--enable-index-page -Zunstable-options"
steps:
- uses: actions/checkout@v3
- uses: arduino/setup-protoc@v1
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
# TODO(alex): update this when https://github.com/rust-lang/rust/issues/109631 is fixed
toolchain: nightly-2023-04-19
- run: cargo doc --document-private-items
lint_macos:
runs-on: macos-latest
needs: changed_files
if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}}
steps:
- uses: actions/checkout@v3
# the setup rust toolchain action ignores the input if file exists.. so remove it
- run: rm rust-toolchain.toml
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
components: rustfmt, clippy
target: aarch64-apple-darwin
toolchain: nightly-2023-04-19
# only mirrord, mirrord-sip contain macOS specific, so run clippy on that.
- run: cargo clippy -p mirrord -p mirrord-sip --target=x86_64-apple-darwin -- -Wclippy::indexing_slicing -D warnings
- run: cargo clippy -p mirrord -p mirrord-sip --target=aarch64-apple-darwin -- -Wclippy::indexing_slicing -D warnings
test_mirrord_protocol:
runs-on: ubuntu-latest
needs: changed_files
if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}}
steps:
- uses: actions/checkout@v3
- uses: actions-rust-lang/setup-rust-toolchain@v1
- run: |
cargo test -p mirrord-protocol
test_mirrord_config:
runs-on: ubuntu-latest
needs: changed_files
if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}}
steps:
- uses: actions/checkout@v3
- uses: actions-rust-lang/setup-rust-toolchain@v1
- run: |
cargo test -p mirrord-config
test_mirrord_kube:
runs-on: ubuntu-latest
needs: changed_files
if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}}
steps:
- uses: actions/checkout@v3
- uses: actions-rust-lang/setup-rust-toolchain@v1
- run: |
cargo test -p mirrord-kube --all-features
test_mirrord_sip:
runs-on: macos-latest
needs: changed_files
if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}}
steps:
- uses: actions/checkout@v3
- uses: actions-rust-lang/setup-rust-toolchain@v1
- run: |
cargo test -p mirrord-sip
test_agent:
runs-on: ubuntu-latest
needs: changed_files
if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}}
container:
image: ghcr.io/metalbear-co/ci-agent-build:f8330d35a2a4b9132138f6fa9a3f3f80768c7c32
steps:
- uses: actions/checkout@v3
- uses: arduino/setup-protoc@v1
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: test
run: cargo test -p mirrord-agent
test_agent_image:
runs-on: ubuntu-latest
needs: changed_files
if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true' }}
steps:
- uses: actions/checkout@v3
- uses: docker/setup-buildx-action@v2
- name: build and export
uses: docker/build-push-action@v3
with:
context: .
tags: test
file: mirrord/agent/Dockerfile
outputs: type=docker,dest=/tmp/test.tar
cache-from: type=gha
cache-to: type=gha,mode=max
- name: upload image
uses: actions/upload-artifact@v3
with:
name: test
path: /tmp/test.tar
test_mirrord_layer_cli:
needs: changed_files
if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}}
strategy:
matrix:
os: [ubuntu-latest, macos-12, macos-11]
target:
[x86_64-unknown-linux-gnu, x86_64-apple-darwin, aarch64-apple-darwin]
exclude:
- os: ubuntu-latest
target: x86_64-apple-darwin
- os: ubuntu-latest
target: aarch64-apple-darwin
- os: macos-12
target: x86_64-unknown-linux-gnu
- os: macos-11
target: aarch64-apple-darwin
- os: macos-11
target: x86_64-unknown-linux-gnu
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
# the setup rust toolchain action ignores the input if file exists.. so remove it
- if: ${{matrix.target == 'aarch64-apple-darwin'}}
run: rm rust-toolchain.toml
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
target: ${{matrix.target}}
toolchain: nightly-2023-04-19
# For now, just verify it compiles.
- run: cargo build -p mirrord-layer -p mirrord --target=${{matrix.target}}
build_mirrord:
runs-on: ubuntu-latest
name: build mirrord
needs: changed_files
if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}}
steps:
- uses: actions/checkout@v3
- uses: arduino/setup-protoc@v1
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
components: rustfmt
- run: cargo build --manifest-path=./Cargo.toml
- name: upload layer
uses: actions/upload-artifact@v3
with:
name: mirrord-artifacts
path: |
target/debug/libmirrord_layer.so
target/debug/mirrord
if-no-files-found: error
integration_tests:
runs-on: ubuntu-latest
needs: [build_mirrord, changed_files]
if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}}
env:
MIRRORD_TEST_USE_EXISTING_LIB: /home/runner/work/mirrord/mirrord/target/debug/libmirrord_layer.so
steps:
- uses: actions/checkout@v3 # Checkout the mirrord repo.
- uses: actions-rust-lang/setup-rust-toolchain@v1 # Install rust.
with:
target: x86_64-unknown-linux-gnu
- run: |
cd mirrord/layer/tests/apps/issue1123
rustc issue1123.rs --out-dir target
- run: |
cd mirrord/layer/tests/apps/issue1054
rustc issue1054.rs --out-dir target
- run: |
cd mirrord/layer/tests/apps/issue1458
rustc issue1458.rs --out-dir target
- run: |
cd mirrord/layer/tests/apps/issue1458portnot53
rustc issue1458portnot53.rs --out-dir target
# For the `java_temurin_sip` test.
- uses: sdkman/sdkman-action@b1f9b696c79148b66d3d3a06f7ea801820318d0f
id: sdkman
with:
candidate: java
version: 17.0.6-tem
- run: java -version
- uses: actions/setup-node@v3 # For http mirroring test.
with:
node-version: 14
- run: npm install express # For http mirroring test with node.
- uses: actions/setup-python@v3 # For http mirroring tests with Flask and FastAPI.
- run: pip3 install flask # For http mirroring test with Flask.
- run: pip3 install fastapi # For http mirroring test with FastAPI.
- run: pip3 install uvicorn[standard] # For http mirroring test with FastAPI.
- uses: actions/setup-go@v3
with:
go-version: "1.20.0-rc.3"
- run: |
go version
- run: | # Build Go test apps.
./scripts/build_go_apps.sh 20
- uses: actions/setup-go@v3
with:
go-version: "1.19.5"
- run: |
go version
- run: | # Build Go test apps.
./scripts/build_go_apps.sh 19
- uses: actions/setup-go@v3
with:
go-version: "1.18.10"
- run: |
go version
- run: | # Build Go test apps.
./scripts/build_go_apps.sh 18
- run: |
cd mirrord/layer/tests/apps/fileops
cargo build
- run: |
cd mirrord/layer/tests/apps/outgoing
cargo build
- run: |
cd mirrord/layer/tests/apps/recv_from
cargo build
- run: |
cd mirrord/layer/tests/apps/dns_resolve
cargo build
- run: |
cd mirrord/layer/tests/apps/listen_ports
cargo build
- run: ./scripts/build_c_apps.sh
- name: download layer # Download layer lib built in the `build_mirrord` job.
uses: actions/download-artifact@v3
with:
name: mirrord-artifacts
path: target/debug/
- run: cargo test -p mirrord-layer
integration_tests_macos:
runs-on: macos-12
needs: changed_files
if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}}
env:
MIRRORD_TEST_USE_EXISTING_LIB: ../../target/debug/libmirrord_layer.dylib
RUST_LOG: warn,mirrord=trace
steps:
- uses: actions/checkout@v3 # Checkout the mirrord repo.
- uses: actions-rust-lang/setup-rust-toolchain@v1 # Install Rust.
with:
target: x86_64-apple-darwin
- run: |
cd mirrord/layer/tests/apps/issue1123
rustc issue1123.rs --out-dir target
- run: |
cd mirrord/layer/tests/apps/issue1054
rustc issue1054.rs --out-dir target
- run: |
cd mirrord/layer/tests/apps/issue1458
rustc issue1458.rs --out-dir target
- run: |
cd mirrord/layer/tests/apps/issue1458portnot53
rustc issue1458portnot53.rs --out-dir target
- uses: actions/setup-go@v3 # Install Go for http mirroring tests with a Go webserver.
with:
go-version: "1.18"
- run: |
go version
- run: | # Build Go test apps.
./scripts/build_go_apps.sh 18
- uses: actions/setup-go@v3 # Install Go for http mirroring tests with a Go webserver.
with:
go-version: "1.19"
- run: |
go version
- run: | # Build Go test apps.
./scripts/build_go_apps.sh 19
- uses: actions/setup-go@v3 # Install Go for http mirroring tests with a Go webserver.
with:
go-version: "1.20.0-rc.3"
- run: |
go version
- run: | # Build Go test apps.
./scripts/build_go_apps.sh 20
- run: |
cd mirrord/layer/tests/apps/fileops
cargo build
- run: |
cd mirrord/layer/tests/apps/outgoing
cargo build
- run: |
cd mirrord/layer/tests/apps/recv_from
cargo build
- run: |
cd mirrord/layer/tests/apps/dns_resolve
cargo build
- run: ./scripts/build_c_apps.sh
# For the `java_temurin_sip` test.
- uses: sdkman/sdkman-action@b1f9b696c79148b66d3d3a06f7ea801820318d0f
id: sdkman
with:
candidate: java
version: 17.0.6-tem
- run: java -version
- uses: actions/setup-node@v3
with:
node-version: 14
- run: npm install express # For http mirroring test with node.
- uses: actions/setup-python@v3 # For http mirroring tests with Flask and FastAPI.
- run: pip3 install flask # For http mirroring test with Flask.
- run: pip3 install fastapi # For http mirroring test with FastAPI.
- run: pip3 install uvicorn[standard] # For http mirroring test with FastAPI.
- run: cargo build -p mirrord-layer # Build layer lib. The tests load it into the apps.
- uses: actions/setup-node@v3 # version 19 spawns processes with `posix_spawn`, so test that also.
with:
node-version: 19
- run: cargo test -p mirrord-layer
e2e:
runs-on: ubuntu-latest
strategy:
matrix:
container-runtime: ["docker", "containerd"]
name: e2e
needs: [build_mirrord, test_agent_image, changed_files]
if: ${{needs.changed_files.outputs.rs_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}}
env:
MIRRORD_AGENT_RUST_LOG: "warn,mirrord=debug"
steps:
- uses: actions/checkout@v3
- uses: actions-rust-lang/setup-rust-toolchain@v1
with:
components: rustfmt
- uses: actions/setup-node@v3
with:
node-version: 14
- run: npm install express
- uses: actions/setup-python@v3
- run: pip3 install flask
- run: pip3 install fastapi
- run: pip3 install uvicorn[standard]
- run: sudo apt-get update && sudo apt-get install -y curl
- uses: actions/setup-go@v3
with:
go-version: "1.18.0"
- run: |
go version
- run: | # Build Go test apps.
./scripts/build_go_apps.sh 18
- uses: actions/setup-go@v3
with:
go-version: "1.19.0"
- run: |
go version
- run: | # Build Go test apps.
./scripts/build_go_apps.sh 19
- uses: actions/setup-go@v3
with:
go-version: "1.20.0-rc.3"
- run: |
go version
- run: | # Build Go test apps.
./scripts/build_go_apps.sh 20
- run: |
cd tests/rust-e2e-fileops
cargo build
- run: |
cd tests/rust-unix-socket-client
cargo build
- run: |
cd tests/rust-bypassed-unix-socket
cargo build
- name: download image
uses: actions/download-artifact@v3
with:
name: test
path: /tmp
- name: start minikube
uses: medyagh/setup-minikube@master
with:
container-runtime: ${{matrix.container-runtime}}
- run: minikube image load /tmp/test.tar
# By running the test of the targetless agent first, we prove it works on an empty cluster without any pods.
- name: Run targetless E2E test.
run: cargo test -p tests targetless
- name: Run all E2E test
run: cargo test -p tests -- --test-threads=6
- name: Collect logs
if: ${{ failure() }}
run: |
kubectl describe pods
docker exec minikube find /var/log/pods -print -exec cat {} \;
lint_markdown:
runs-on: ubuntu-latest
needs: changed_files
if: ${{needs.changed_files.outputs.markdown_changed == 'true' || needs.changed_files.outputs.ci_changed == 'true'}}
steps:
- uses: actions/checkout@v3
- uses: avto-dev/markdown-lint@v1
with:
config: "markdownlint-config.json"
args: "README.md"
# We need some "accummulation" job here because bors fails (timeouts) to
# listen on matrix builds.
# Hence, we have some kind of dummy here that bors can listen on
ci-success:
name: ci
# We want this to run even if some of the required jobs got skipped
if: always()
needs:
[
towncrier_check,
changed_files,
build_mirrord,
test_agent_image,
integration_tests_macos,
integration_tests,
e2e,
test_mirrord_layer_cli,
test_agent,
lint,
lint_macos,
test_mirrord_config,
test_mirrord_protocol,
test_mirrord_sip,
lint_markdown,
]
runs-on: ubuntu-latest
steps:
- name: CI succeeded
# We have to do it in the shell since if it's in the if condition
# then skipping is considered success by branch protection rules
env:
CI_SUCCESS: ${{ (needs.changed_files.result == 'success') &&
(needs.towncrier_check.result == 'success') &&
(needs.build_mirrord.result == 'success' || needs.build_mirrord.result == 'skipped') &&
(needs.test_agent_image.result == 'success' || needs.test_agent_image.result == 'skipped') &&
(needs.integration_tests_macos.result == 'success' || needs.integration_tests_macos.result == 'skipped') &&
(needs.integration_tests.result == 'success' || needs.integration_tests.result == 'skipped') &&
(needs.e2e.result == 'success' || needs.e2e.result == 'skipped') &&
(needs.test_mirrord_layer_cli.result == 'success' || needs.test_mirrord_layer_cli.result == 'skipped') &&
(needs.test_agent.result == 'success' || needs.test_agent.result == 'skipped') &&
(needs.lint.result == 'success' || needs.lint.result == 'skipped') &&
(needs.lint_macos.result == 'success' || needs.lint_macos.result == 'skipped') &&
(needs.test_mirrord_config.result == 'success' || needs.test_mirrord_config.result == 'skipped') &&
(needs.test_mirrord_protocol.result == 'success' || needs.test_mirrord_protocol.result == 'skipped') &&
(needs.test_mirrord_sip.result == 'success' || needs.test_mirrord_sip.result == 'skipped') &&
(needs.lint_markdown.result == 'success' || needs.lint_markdown.result == 'skipped') }}
run: echo $CI_SUCCESS && if [ "$CI_SUCCESS" == "true" ]; then echo "SUCCESS" && exit 0; else echo "Failure" && exit 1; fi