test: extend coverage tracking to bats tests

- updates codecov action version - make batstest now builds with coverage and runs all bats tests with coverage, then merges into a file codecov knows about Signed-off-by: Michael McCracken <[email protected]>
project-machine · Dec 19, 2024 · 1b416ab · 1b416ab
1 parent 8e93665
commit 1b416ab
Show file tree

Hide file tree

Showing 6 changed files with 42 additions and 34 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -67,10 +67,10 @@ jobs:
         run: |
           make batstest
       - name: Upload code coverage
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
         with:
           fail_ci_if_error: true # optional (default = false)
-          files: ./coverage.txt
+          files: ./unit-coverage.txt,./integ-coverage.txt
           token: ${{ secrets.CODECOV_TOKEN }} # required
       - name: Release
         uses: softprops/action-gh-release@v1

diff --git a/Makefile b/Makefile
@@ -14,6 +14,7 @@ BATS_VERSION := v1.10.0
 STACKER = $(TOOLS_D)/bin/stacker
 STACKER_VERSION := v1.0.0
 TOOLS_D := $(ROOT)/tools
+GOCOVERDIR ?= $(ROOT)
 
 export PATH := $(TOOLS_D)/bin:$(PATH)
 
@@ -25,11 +26,13 @@ gofmt: .made-gofmt
 		{ echo "gofmt made changes: $$o" 1>&2; exit 1; }
 	@touch $@
 
-atomfs: .made-gofmt $(GO_SRC)
-	cd $(ROOT)/cmd/atomfs && go build -buildvcs=false -ldflags "$(VERSION_LDFLAGS)" -o $(ROOT)/bin/atomfs ./...
+atomfs atomfs-cover: .made-gofmt $(GO_SRC)
+	cd $(ROOT)/cmd/atomfs && go build $(BUILDCOVERFLAGS) -buildvcs=false -ldflags "$(VERSION_LDFLAGS)" -o $(ROOT)/bin/$@ ./...
+
+atomfs-cover: BUILDCOVERFLAGS=-cover
 
 gotest: $(GO_SRC)
-	go test -coverprofile=coverage.txt -ldflags "$(VERSION_LDFLAGS)"  ./...
+	go test -coverprofile=unit-coverage.txt -ldflags "$(VERSION_LDFLAGS)"  ./...
 
 $(STACKER):
 	mkdir -p $(TOOLS_D)/bin
@@ -46,9 +49,14 @@ $(BATS):
 	git clone --depth 1 https://github.com/bats-core/bats-assert $(ROOT)/test/test_helper/bats-assert
 	git clone --depth 1 https://github.com/bats-core/bats-file $(ROOT)/test/test_helper/bats-file
 
-batstest: $(BATS) $(STACKER) atomfs test/random.txt testimages
-	cd $(ROOT)/test; sudo $(BATS) --tap --timing priv-*.bats
-	cd $(ROOT)/test; $(BATS) --tap --timing unpriv-*.bats
+batstest: $(BATS) $(STACKER) atomfs-cover test/random.txt testimages
+	cd $(ROOT)/test; sudo GOCOVERDIR=$(GOCOVERDIR) $(BATS) --tap --timing priv-*.bats
+	cd $(ROOT)/test; GOCOVERDIR=$(GOCOVERDIR) $(BATS) --tap --timing unpriv-*.bats
+	go tool covdata textfmt -i $(GOCOVERDIR) -o integ-coverage.txt
+
+covreport: gotest batstest
+	go tool cover -func=unit-coverage.txt
+	go tool cover -func=integ-coverage.txt
 
 testimages: /tmp/atomfs-test-oci/.copydone
 	@echo "busybox image exists at /tmp/atomfs-test-oci already"

diff --git a/test/priv-mount-umount-mount.bats b/test/priv-mount-umount-mount.bats
@@ -37,18 +37,18 @@ function verity_checkusedloops() {
 
     echo MOUNT A
     mkdir -p $MP/a
-    run atomfs --debug mount ${BATS_SUITE_TMPDIR}/oci:a-squashfs $MP/a
+    run atomfs-cover --debug mount ${BATS_SUITE_TMPDIR}/oci:a-squashfs $MP/a
     assert_success
     assert_file_exists $MP/a/a
 
     echo MOUNT B
     mkdir -p $MP/b
-    run atomfs --debug mount ${BATS_SUITE_TMPDIR}/oci:b-squashfs $MP/b
+    run atomfs-cover --debug mount ${BATS_SUITE_TMPDIR}/oci:b-squashfs $MP/b
     assert_success
     assert_file_exists $MP/b/b
 
     echo UMOUNT B
-    atomfs --debug umount $MP/b
+    atomfs-cover --debug umount $MP/b
     assert_success
 
     # first layer should still exist since a is still mounted
@@ -58,12 +58,12 @@ function verity_checkusedloops() {
 
     echo MOUNT C
     mkdir -p $MP/c
-    atomfs --debug mount ${BATS_SUITE_TMPDIR}/oci:c-squashfs $MP/c
+    atomfs-cover --debug mount ${BATS_SUITE_TMPDIR}/oci:c-squashfs $MP/c
     assert_success
     assert_file_exists $MP/c/c
 
     echo UMOUNT A
-    atomfs --debug umount $MP/a
+    atomfs-cover --debug umount $MP/a
     assert_success
 
     # first layer should still exist since c is still mounted
@@ -75,7 +75,7 @@ function verity_checkusedloops() {
     assert_file_exists $MP/c/c
     assert_file_exists $MP/c/bin/sh
 
-    atomfs --debug umount $MP/c
+    atomfs-cover --debug umount $MP/c
     assert_success
 
     # c's last layer shouldn't exist any more, since it is unique

diff --git a/test/priv-mount.bats b/test/priv-mount.bats
@@ -17,7 +17,7 @@ function setup() {
 }
 
 @test "RO mount/umount and verify of good image works" {
-    run atomfs --debug mount ${BATS_SUITE_TMPDIR}/oci:test-squashfs $MP
+    run atomfs-cover --debug mount ${BATS_SUITE_TMPDIR}/oci:test-squashfs $MP
     assert_success
     assert_file_exists $MP/1.README.md
     assert_file_exists $MP/random.txt
@@ -26,10 +26,10 @@ function setup() {
     run touch $MP/do-not-let-me
     assert_failure
 
-    run atomfs verify $MP
+    run atomfs-cover verify $MP
     assert_success
 
-    run atomfs --debug umount $MP
+    run atomfs-cover --debug umount $MP
     assert_success
 
     # mount point and meta dir should exist but be empty:
@@ -41,7 +41,7 @@ function setup() {
 }
 
 @test "mount with missing verity data fails" {
-    run atomfs --debug mount ${BATS_SUITE_TMPDIR}/oci-no-verity:test-squashfs $MP
+    run atomfs-cover --debug mount ${BATS_SUITE_TMPDIR}/oci-no-verity:test-squashfs $MP
     assert_failure
     assert_line --partial "is missing verity data"
 
@@ -54,10 +54,10 @@ function setup() {
 }
 
 @test "mount with missing verity data passes if you ignore it" {
-    run atomfs --debug mount --allow-missing-verity ${BATS_SUITE_TMPDIR}/oci-no-verity:test-squashfs $MP
+    run atomfs-cover --debug mount --allow-missing-verity ${BATS_SUITE_TMPDIR}/oci-no-verity:test-squashfs $MP
     assert_success
 
-    run atomfs --debug umount $MP
+    run atomfs-cover --debug umount $MP
     assert_success
 
     # mount point and meta dir should exist but be empty:
@@ -69,7 +69,7 @@ function setup() {
 }
 
 @test "mount/umount with writeable overlay" {
-    run atomfs --debug mount --writeable ${BATS_SUITE_TMPDIR}/oci:test-squashfs $MP
+    run atomfs-cover --debug mount --writeable ${BATS_SUITE_TMPDIR}/oci:test-squashfs $MP
     assert_success
     assert_file_exists $MP/1.README.md
     assert_file_exists $MP/random.txt
@@ -81,7 +81,7 @@ function setup() {
     run cp $MP/1.README.md $MP/3.README.md
     assert_success
 
-    run atomfs --debug umount $MP
+    run atomfs-cover --debug umount $MP
     assert_success
 
     # mount point and meta dir should exist but be empty:
@@ -94,7 +94,7 @@ function setup() {
 @test "mount with writeable overlay in separate dir" {
     export PERSIST_DIR=${BATS_TEST_TMPDIR}/persist-dir
     mkdir -p $PERSIST_DIR
-    run atomfs --debug mount --persist=${PERSIST_DIR} ${BATS_SUITE_TMPDIR}/oci:test-squashfs $MP
+    run atomfs-cover --debug mount --persist=${PERSIST_DIR} ${BATS_SUITE_TMPDIR}/oci:test-squashfs $MP
     assert_success
     assert_file_exists $MP/1.README.md
     assert_file_exists $MP/random.txt
@@ -108,7 +108,7 @@ function setup() {
     assert_file_exists $PERSIST_DIR/persist/3.README.md
     assert_file_not_exists $PERSIST_DIR/persist/1.README.md
 
-    run atomfs --debug umount $MP
+    run atomfs-cover --debug umount $MP
     assert_success
     # mount point and meta dir should exist but be empty:
     assert_dir_exists $MP

diff --git a/test/priv-verify.bats b/test/priv-verify.bats
@@ -26,7 +26,7 @@ function setup_file() {
     assert_failure
 
     mkdir -p mountpoint
-    run atomfs --debug mount ${BATS_TEST_TMPDIR}/oci:test-squashfs mountpoint
+    run atomfs-cover --debug mount ${BATS_TEST_TMPDIR}/oci:test-squashfs mountpoint
     assert_failure
 
 }

diff --git a/test/unpriv-guestmount.bats b/test/unpriv-guestmount.bats
@@ -26,21 +26,21 @@ function setup() {
     export INNER_MNTNSNAME=\$(readlink /proc/self/ns/mnt | cut -c 6-15)
 
     set +e
-    atomfs --debug mount --persist=\$PERSIST_DIR ${BATS_SUITE_TMPDIR}/oci:test-squashfs $MP
+    atomfs-cover --debug mount --persist=\$PERSIST_DIR ${BATS_SUITE_TMPDIR}/oci:test-squashfs $MP
     [ \$? -eq 0 ] && {
       echo guestmount without allow-missing should fail, because we do not have verity
       exit 1
     }
     echo "XFAIL: guestmount without allow-missing did fail"
     set -e
 
-    atomfs --debug mount --allow-missing-verity --persist=\$PERSIST_DIR ${BATS_SUITE_TMPDIR}/oci:test-squashfs $MP
+    atomfs-cover --debug mount --allow-missing-verity --persist=\$PERSIST_DIR ${BATS_SUITE_TMPDIR}/oci:test-squashfs $MP
     [ -f $MP/1.README.md ]
     [ -f $MP/random.txt ]
     touch $MP/let-me-write
 
     set +e
-    atomfs --debug verify $MP
+    atomfs-cover --debug verify $MP
     [ \$? -eq 0 ] && {
        echo mount with squashfuse ignores verity, so verify should have failed, output should include warning
        exit 1
@@ -51,7 +51,7 @@ function setup() {
     find $ATOMFS_TEST_RUN_DIR/meta/\$INNER_MNTNSNAME/ -name config.json|xargs cat
     find $ATOMFS_TEST_RUN_DIR/meta/\$INNER_MNTNSNAME/
 
-    atomfs --debug umount $MP
+    atomfs-cover --debug umount $MP
     [ -f \$PERSIST_DIR/persist/let-me-write ]
 
     # mount point and meta dir should be empty
@@ -76,20 +76,20 @@ EOF
 
     export INNER_MNTNSNAME=\$(readlink /proc/self/ns/mnt | cut -c 6-15)
 
-    atomfs --debug mount --allow-missing-verity --persist=\$PERSIST_DIR ${BATS_SUITE_TMPDIR}/oci-no-verity:test-squashfs $MP
+    atomfs-cover --debug mount --allow-missing-verity --persist=\$PERSIST_DIR ${BATS_SUITE_TMPDIR}/oci-no-verity:test-squashfs $MP
     [ -f $MP/1.README.md ]
     [ -f $MP/random.txt ]
     touch $MP/let-me-write
 
     set +e
-    atomfs --debug verify $MP
+    atomfs-cover --debug verify $MP
     [ \$? -eq 0 ] && {
        echo mount with squashfuse ignores verity, so verify should have failed, output should include warning
        exit 1
     }
     set -e
 
-    atomfs --debug umount $MP
+    atomfs-cover --debug umount $MP
     [ -f \$PERSIST_DIR/persist/let-me-write ]
 
     [ -d $MP ]
@@ -114,11 +114,11 @@ EOF
 
     export INNER_MNTNSNAME=\$(readlink /proc/self/ns/mnt | cut -c 6-15)
 
-    atomfs --debug mount --allow-missing-verity --metadir=\$META_DIR ${BATS_SUITE_TMPDIR}/oci-no-verity:test-squashfs $MP
+    atomfs-cover --debug mount --allow-missing-verity --metadir=\$META_DIR ${BATS_SUITE_TMPDIR}/oci-no-verity:test-squashfs $MP
     [ -f $MP/1.README.md ]
     [ -f $MP/random.txt ]
 
-    atomfs --debug umount --metadir=\$META_DIR $MP
+    atomfs-cover --debug umount --metadir=\$META_DIR $MP
 
     [ -d $MP ]
     [ -z \$( ls -A $MP) ]