From be2c195b86f6881dd04a36eca143e9a123a61161 Mon Sep 17 00:00:00 2001 From: Michael McCracken Date: Wed, 18 Dec 2024 15:06:34 -0800 Subject: [PATCH] test: extend coverage tracking to bats tests - updates codecov action version - make batstest now builds with coverage and runs all bats tests with coverage, then merges into a file codecov knows about Signed-off-by: Michael McCracken --- .github/workflows/build.yaml | 4 ++-- Makefile | 20 ++++++++++++++------ test/priv-mount-umount-mount.bats | 12 ++++++------ test/priv-mount.bats | 20 ++++++++++---------- test/priv-verify.bats | 2 +- test/unpriv-guestmount.bats | 18 +++++++++--------- 6 files changed, 42 insertions(+), 34 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 7b18cfc..3ed285f 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -67,10 +67,10 @@ jobs: run: | make batstest - name: Upload code coverage - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@v5 with: fail_ci_if_error: true # optional (default = false) - files: ./coverage.txt + files: ./unit-coverage.txt,./integ-coverage.txt token: ${{ secrets.CODECOV_TOKEN }} # required - name: Release uses: softprops/action-gh-release@v1 diff --git a/Makefile b/Makefile index 345a2a5..cc824b6 100644 --- a/Makefile +++ b/Makefile @@ -14,6 +14,7 @@ BATS_VERSION := v1.10.0 STACKER = $(TOOLS_D)/bin/stacker STACKER_VERSION := v1.0.0 TOOLS_D := $(ROOT)/tools +GOCOVERDIR ?= $(ROOT) export PATH := $(TOOLS_D)/bin:$(PATH) @@ -25,11 +26,13 @@ gofmt: .made-gofmt { echo "gofmt made changes: $$o" 1>&2; exit 1; } @touch $@ -atomfs: .made-gofmt $(GO_SRC) - cd $(ROOT)/cmd/atomfs && go build -buildvcs=false -ldflags "$(VERSION_LDFLAGS)" -o $(ROOT)/bin/atomfs ./... +atomfs atomfs-cover: .made-gofmt $(GO_SRC) + cd $(ROOT)/cmd/atomfs && go build $(BUILDCOVERFLAGS) -buildvcs=false -ldflags "$(VERSION_LDFLAGS)" -o $(ROOT)/bin/$@ ./... + +atomfs-cover: BUILDCOVERFLAGS=-cover gotest: $(GO_SRC) - go test -coverprofile=coverage.txt -ldflags "$(VERSION_LDFLAGS)" ./... + go test -coverprofile=unit-coverage.txt -ldflags "$(VERSION_LDFLAGS)" ./... $(STACKER): mkdir -p $(TOOLS_D)/bin @@ -46,9 +49,14 @@ $(BATS): git clone --depth 1 https://github.com/bats-core/bats-assert $(ROOT)/test/test_helper/bats-assert git clone --depth 1 https://github.com/bats-core/bats-file $(ROOT)/test/test_helper/bats-file -batstest: $(BATS) $(STACKER) atomfs test/random.txt testimages - cd $(ROOT)/test; sudo $(BATS) --tap --timing priv-*.bats - cd $(ROOT)/test; $(BATS) --tap --timing unpriv-*.bats +batstest: $(BATS) $(STACKER) atomfs-cover test/random.txt testimages + cd $(ROOT)/test; sudo GOCOVERDIR=$(GOCOVERDIR) $(BATS) --tap --timing priv-*.bats + cd $(ROOT)/test; GOCOVERDIR=$(GOCOVERDIR) $(BATS) --tap --timing unpriv-*.bats + go tool covdata textfmt -i $(GOCOVERDIR) -o integ-coverage.txt + +covreport: gotest batstest + go tool cover -func=unit-coverage.txt + go tool cover -func=integ-coverage.txt testimages: /tmp/atomfs-test-oci/.copydone @echo "busybox image exists at /tmp/atomfs-test-oci already" diff --git a/test/priv-mount-umount-mount.bats b/test/priv-mount-umount-mount.bats index 9c24aaa..305b466 100644 --- a/test/priv-mount-umount-mount.bats +++ b/test/priv-mount-umount-mount.bats @@ -37,18 +37,18 @@ function verity_checkusedloops() { echo MOUNT A mkdir -p $MP/a - run atomfs --debug mount ${BATS_SUITE_TMPDIR}/oci:a-squashfs $MP/a + run atomfs-cover --debug mount ${BATS_SUITE_TMPDIR}/oci:a-squashfs $MP/a assert_success assert_file_exists $MP/a/a echo MOUNT B mkdir -p $MP/b - run atomfs --debug mount ${BATS_SUITE_TMPDIR}/oci:b-squashfs $MP/b + run atomfs-cover --debug mount ${BATS_SUITE_TMPDIR}/oci:b-squashfs $MP/b assert_success assert_file_exists $MP/b/b echo UMOUNT B - atomfs --debug umount $MP/b + atomfs-cover --debug umount $MP/b assert_success # first layer should still exist since a is still mounted @@ -58,12 +58,12 @@ function verity_checkusedloops() { echo MOUNT C mkdir -p $MP/c - atomfs --debug mount ${BATS_SUITE_TMPDIR}/oci:c-squashfs $MP/c + atomfs-cover --debug mount ${BATS_SUITE_TMPDIR}/oci:c-squashfs $MP/c assert_success assert_file_exists $MP/c/c echo UMOUNT A - atomfs --debug umount $MP/a + atomfs-cover --debug umount $MP/a assert_success # first layer should still exist since c is still mounted @@ -75,7 +75,7 @@ function verity_checkusedloops() { assert_file_exists $MP/c/c assert_file_exists $MP/c/bin/sh - atomfs --debug umount $MP/c + atomfs-cover --debug umount $MP/c assert_success # c's last layer shouldn't exist any more, since it is unique diff --git a/test/priv-mount.bats b/test/priv-mount.bats index dc8c4f8..7b6f475 100644 --- a/test/priv-mount.bats +++ b/test/priv-mount.bats @@ -17,7 +17,7 @@ function setup() { } @test "RO mount/umount and verify of good image works" { - run atomfs --debug mount ${BATS_SUITE_TMPDIR}/oci:test-squashfs $MP + run atomfs-cover --debug mount ${BATS_SUITE_TMPDIR}/oci:test-squashfs $MP assert_success assert_file_exists $MP/1.README.md assert_file_exists $MP/random.txt @@ -26,10 +26,10 @@ function setup() { run touch $MP/do-not-let-me assert_failure - run atomfs verify $MP + run atomfs-cover verify $MP assert_success - run atomfs --debug umount $MP + run atomfs-cover --debug umount $MP assert_success # mount point and meta dir should exist but be empty: @@ -41,7 +41,7 @@ function setup() { } @test "mount with missing verity data fails" { - run atomfs --debug mount ${BATS_SUITE_TMPDIR}/oci-no-verity:test-squashfs $MP + run atomfs-cover --debug mount ${BATS_SUITE_TMPDIR}/oci-no-verity:test-squashfs $MP assert_failure assert_line --partial "is missing verity data" @@ -54,10 +54,10 @@ function setup() { } @test "mount with missing verity data passes if you ignore it" { - run atomfs --debug mount --allow-missing-verity ${BATS_SUITE_TMPDIR}/oci-no-verity:test-squashfs $MP + run atomfs-cover --debug mount --allow-missing-verity ${BATS_SUITE_TMPDIR}/oci-no-verity:test-squashfs $MP assert_success - run atomfs --debug umount $MP + run atomfs-cover --debug umount $MP assert_success # mount point and meta dir should exist but be empty: @@ -69,7 +69,7 @@ function setup() { } @test "mount/umount with writeable overlay" { - run atomfs --debug mount --writeable ${BATS_SUITE_TMPDIR}/oci:test-squashfs $MP + run atomfs-cover --debug mount --writeable ${BATS_SUITE_TMPDIR}/oci:test-squashfs $MP assert_success assert_file_exists $MP/1.README.md assert_file_exists $MP/random.txt @@ -81,7 +81,7 @@ function setup() { run cp $MP/1.README.md $MP/3.README.md assert_success - run atomfs --debug umount $MP + run atomfs-cover --debug umount $MP assert_success # mount point and meta dir should exist but be empty: @@ -94,7 +94,7 @@ function setup() { @test "mount with writeable overlay in separate dir" { export PERSIST_DIR=${BATS_TEST_TMPDIR}/persist-dir mkdir -p $PERSIST_DIR - run atomfs --debug mount --persist=${PERSIST_DIR} ${BATS_SUITE_TMPDIR}/oci:test-squashfs $MP + run atomfs-cover --debug mount --persist=${PERSIST_DIR} ${BATS_SUITE_TMPDIR}/oci:test-squashfs $MP assert_success assert_file_exists $MP/1.README.md assert_file_exists $MP/random.txt @@ -108,7 +108,7 @@ function setup() { assert_file_exists $PERSIST_DIR/persist/3.README.md assert_file_not_exists $PERSIST_DIR/persist/1.README.md - run atomfs --debug umount $MP + run atomfs-cover --debug umount $MP assert_success # mount point and meta dir should exist but be empty: assert_dir_exists $MP diff --git a/test/priv-verify.bats b/test/priv-verify.bats index 5ab78e5..2dc3745 100644 --- a/test/priv-verify.bats +++ b/test/priv-verify.bats @@ -26,7 +26,7 @@ function setup_file() { assert_failure mkdir -p mountpoint - run atomfs --debug mount ${BATS_TEST_TMPDIR}/oci:test-squashfs mountpoint + run atomfs-cover --debug mount ${BATS_TEST_TMPDIR}/oci:test-squashfs mountpoint assert_failure } diff --git a/test/unpriv-guestmount.bats b/test/unpriv-guestmount.bats index 515b3e1..c43da23 100644 --- a/test/unpriv-guestmount.bats +++ b/test/unpriv-guestmount.bats @@ -26,7 +26,7 @@ function setup() { export INNER_MNTNSNAME=\$(readlink /proc/self/ns/mnt | cut -c 6-15) set +e - atomfs --debug mount --persist=\$PERSIST_DIR ${BATS_SUITE_TMPDIR}/oci:test-squashfs $MP + atomfs-cover --debug mount --persist=\$PERSIST_DIR ${BATS_SUITE_TMPDIR}/oci:test-squashfs $MP [ \$? -eq 0 ] && { echo guestmount without allow-missing should fail, because we do not have verity exit 1 @@ -34,13 +34,13 @@ function setup() { echo "XFAIL: guestmount without allow-missing did fail" set -e - atomfs --debug mount --allow-missing-verity --persist=\$PERSIST_DIR ${BATS_SUITE_TMPDIR}/oci:test-squashfs $MP + atomfs-cover --debug mount --allow-missing-verity --persist=\$PERSIST_DIR ${BATS_SUITE_TMPDIR}/oci:test-squashfs $MP [ -f $MP/1.README.md ] [ -f $MP/random.txt ] touch $MP/let-me-write set +e - atomfs --debug verify $MP + atomfs-cover --debug verify $MP [ \$? -eq 0 ] && { echo mount with squashfuse ignores verity, so verify should have failed, output should include warning exit 1 @@ -51,7 +51,7 @@ function setup() { find $ATOMFS_TEST_RUN_DIR/meta/\$INNER_MNTNSNAME/ -name config.json|xargs cat find $ATOMFS_TEST_RUN_DIR/meta/\$INNER_MNTNSNAME/ - atomfs --debug umount $MP + atomfs-cover --debug umount $MP [ -f \$PERSIST_DIR/persist/let-me-write ] # mount point and meta dir should be empty @@ -76,20 +76,20 @@ EOF export INNER_MNTNSNAME=\$(readlink /proc/self/ns/mnt | cut -c 6-15) - atomfs --debug mount --allow-missing-verity --persist=\$PERSIST_DIR ${BATS_SUITE_TMPDIR}/oci-no-verity:test-squashfs $MP + atomfs-cover --debug mount --allow-missing-verity --persist=\$PERSIST_DIR ${BATS_SUITE_TMPDIR}/oci-no-verity:test-squashfs $MP [ -f $MP/1.README.md ] [ -f $MP/random.txt ] touch $MP/let-me-write set +e - atomfs --debug verify $MP + atomfs-cover --debug verify $MP [ \$? -eq 0 ] && { echo mount with squashfuse ignores verity, so verify should have failed, output should include warning exit 1 } set -e - atomfs --debug umount $MP + atomfs-cover --debug umount $MP [ -f \$PERSIST_DIR/persist/let-me-write ] [ -d $MP ] @@ -114,11 +114,11 @@ EOF export INNER_MNTNSNAME=\$(readlink /proc/self/ns/mnt | cut -c 6-15) - atomfs --debug mount --allow-missing-verity --metadir=\$META_DIR ${BATS_SUITE_TMPDIR}/oci-no-verity:test-squashfs $MP + atomfs-cover --debug mount --allow-missing-verity --metadir=\$META_DIR ${BATS_SUITE_TMPDIR}/oci-no-verity:test-squashfs $MP [ -f $MP/1.README.md ] [ -f $MP/random.txt ] - atomfs --debug umount --metadir=\$META_DIR $MP + atomfs-cover --debug umount --metadir=\$META_DIR $MP [ -d $MP ] [ -z \$( ls -A $MP) ]