CI Profiling precursor changes (#14051)

This PR: 1. Moves the building of the LXD binaries for the system tests into the code test stage and uses GH artifacts system in order to avoid building the LXD binaries multiple times for each concurrent runner. 2. Adds functionality to build LXD binaries with code coverage support and uploads them to the GH artifact system for collection later (this will come in a future PR). 3. Various Makefile improvements and cleanups.
canonical · Sep 9, 2024 · 94110ee · 94110ee
2 parents 8fe6281 + fce81c5
commit 94110ee
Show file tree

Hide file tree

Showing 3 changed files with 123 additions and 66 deletions.
diff --git a/.github/workflows/commits.yml b/.github/workflows/commits.yml
@@ -1,9 +1,5 @@
 name: Commits
 on:
-  push:
-    branches:
-      - main
-      - stable-*
   pull_request:
 
 concurrency:

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -7,9 +7,9 @@ on:
   pull_request:
 
 env:
-  CGO_LDFLAGS_ALLOW: "(-Wl,-wrap,pthread_create)|(-Wl,-z,now)"
   LXD_REQUIRED_TESTS: "storage_buckets"
   LXD_SKIP_TESTS: "clustering_upgrade clustering_upgrade_large"
+  GOCOVERDIR: "/home/runner/work/lxd/lxd/coverage"
 
 permissions:
   contents: read
@@ -27,9 +27,11 @@ defaults:
 jobs:
   code-tests:
     env:
-      CGO_CFLAGS: "-I/home/runner/work/lxd/lxd-test/vendor/dqlite/include/"
-      CGO_LDFLAGS: "-L/home/runner/work/lxd/lxd-test/vendor/dqlite/.libs/"
-      LD_LIBRARY_PATH: "/home/runner/work/lxd/lxd-test/vendor/dqlite/.libs/"
+      CGO_CFLAGS: "-I/home/runner/go/bin/dqlite/include/"
+      CGO_LDFLAGS: "-L/home/runner/go/bin/dqlite/libs/"
+      LD_LIBRARY_PATH: "/home/runner/go/bin/dqlite/libs/"
+      LD_RUN_PATH: "/home/runner/go/bin/dqlite/libs/"
+      CGO_LDFLAGS_ALLOW: "(-Wl,-wrap,pthread_create)|(-Wl,-z,now)"
     name: Code
     runs-on: ubuntu-22.04
     steps:
@@ -93,30 +95,54 @@ jobs:
 
           python3 -m pip install flake8
 
+          # Download minio ready to include in dependencies for system tests.
+          mkdir -p "$(go env GOPATH)/bin"
+          curl -sSfL https://dl.min.io/server/minio/release/linux-amd64/minio --output "$(go env GOPATH)/bin/minio"
+          chmod +x "$(go env GOPATH)/bin/minio"
+
+          # Also grab the latest minio client to maintain compatibility with the server.
+          curl -sSfL https://dl.min.io/client/mc/release/linux-amd64/mc --output "$(go env GOPATH)/bin/mc"
+          chmod +x "$(go env GOPATH)/bin/mc"
+
       - name: Download go dependencies
         run: |
           set -eux
           go mod download
 
+      - name: Check compatibility with min Go version
+        run: |
+          set -eux
+          GOMIN="$(sed -n 's/^GOMIN=\([0-9.]\+\)$/\1/p' Makefile)"
+          go mod tidy -go="${GOMIN}"
+
+          DOC_GOMIN="$(sed -n 's/^LXD requires Go \([0-9.]\+\) .*/\1/p' doc/requirements.md)"
+          [ "${GOMIN}" = "${DOC_GOMIN}" ]
+
       - name: Make LXD tarball and unpack it
         env:
           CUSTOM_VERSION: "test"
         run: |
           set -eux
           make dist
-          tar -xzf lxd-test.tar.gz -C ~/work/lxd/
+          tar -xzf lxd-test.tar.gz -C /home/runner/work/lxd/
           rm lxd-test.tar.gz
 
       - name: Build LXD dependencies
         run: |
           set -eux
-          cd ~/work/lxd/lxd-test
+          cd /home/runner/work/lxd/lxd-test
           make deps
 
-      - name: Run LXD build
+          # Include dqlite libs in dependencies for system tests.
+          mkdir /home/runner/go/bin/dqlite
+          mv /home/runner/work/lxd/lxd-test/vendor/dqlite/include /home/runner/go/bin/dqlite/include
+          mv /home/runner/work/lxd/lxd-test/vendor/dqlite/.libs /home/runner/go/bin/dqlite/libs
+
+      - name: Build binaries
         run: |
           set -eux
-          make lxd
+          make
+          strip --strip-all /home/runner/go/bin/{lxc*,lxd*} -v
 
       - name: Check lxc/lxd-agent binary sizes
         run: |
@@ -162,12 +188,32 @@ jobs:
           sudo chmod o+w ./lxd/metadata/configuration.json
           sudo chmod o+w ./doc/metadata.txt
           sudo chmod o+w ./po/*
+          mkdir -p "${GOCOVERDIR}"
           make static-analysis
 
       - name: Unit tests (all)
         run: |
           set -eux
-          sudo --preserve-env=CGO_CFLAGS,CGO_LDFLAGS,CGO_LDFLAGS_ALLOW,LD_LIBRARY_PATH LD_LIBRARY_PATH=${LD_LIBRARY_PATH} env "PATH=${PATH}" go test -v ./...
+          mkdir -p "${GOCOVERDIR}"
+          sudo --preserve-env=CGO_CFLAGS,CGO_LDFLAGS,CGO_LDFLAGS_ALLOW,GOCOVERDIR,LD_LIBRARY_PATH LD_LIBRARY_PATH=${LD_LIBRARY_PATH} env "PATH=${PATH}" make check-unit
+
+      - name: Upload coverage data
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-unit
+          path: /home/runner/work/lxd/lxd/coverage
+
+      - name: Upload system test dependencies
+        uses: actions/upload-artifact@v4
+        with:
+          name: system-test-deps
+          path: |
+            /home/runner/go/bin/lxc*
+            /home/runner/go/bin/lxd*
+            /home/runner/go/bin/mc
+            /home/runner/go/bin/minio
+            /home/runner/go/bin/dqlite
+          retention-days: 1
 
   system-tests:
     env:
@@ -180,6 +226,7 @@ jobs:
       LXD_TMPFS: "1"
     name: System
     runs-on: ubuntu-22.04
+    needs: code-tests
     strategy:
       fail-fast: false
       matrix:
@@ -242,20 +289,10 @@ jobs:
         with:
           go-version: ${{ matrix.go }}
 
-      - name: Check compatibility with min Go version
-        run: |
-          set -eux
-          GOMIN="$(sed -n 's/^GOMIN=\([0-9.]\+\)$/\1/p' Makefile)"
-          go mod tidy -go="${GOMIN}"
-
-          DOC_GOMIN="$(sed -n 's/^LXD requires Go \([0-9.]\+\) .*/\1/p' doc/requirements.md)"
-          [ "${GOMIN}" = "${DOC_GOMIN}" ]
-
       - name: Install dependencies
         run: |
           set -eux
           sudo add-apt-repository ppa:ubuntu-lxc/daily -y --no-update
-          sudo add-apt-repository ppa:dqlite/dev -y --no-update
           sudo apt-get update
 
           sudo systemctl mask lxc.service lxc-net.service
@@ -266,7 +303,6 @@ jobs:
             libacl1-dev \
             libcap-dev \
             libdbus-1-dev \
-            libdqlite-dev \
             liblxc-dev \
             libseccomp-dev \
             libselinux-dev \
@@ -304,23 +340,17 @@ jobs:
           # reclaim some space
           sudo apt-get clean
 
-          mkdir -p "$(go env GOPATH)/bin"
-          curl -sSfL https://dl.min.io/server/minio/release/linux-amd64/minio --output "$(go env GOPATH)/bin/minio"
-          chmod +x "$(go env GOPATH)/bin/minio"
-
-          # Also grab the latest minio client to maintain compatibility with the server.
-          curl -sSfL https://dl.min.io/client/mc/release/linux-amd64/mc --output "$(go env GOPATH)/bin/mc"
-          chmod +x "$(go env GOPATH)/bin/mc"
-
-      - name: Download go dependencies
-        run: |
-          set -eux
-          go mod download
+      - name: Download system test dependencies
+        uses: actions/download-artifact@v4
+        with:
+          name: system-test-deps
+          merge-multiple: true
+          path: /home/runner/go/bin
 
-      - name: Run LXD build
+      - name: Set exec perms on LXD binaries
         run: |
-          set -eux
-          make lxd
+          ls -lR /home/runner/go/bin/
+          chmod uog+x /home/runner/go/bin/*
 
       - name: Setup MicroCeph
         if: ${{ matrix.backend == 'ceph' }}
@@ -391,8 +421,15 @@ jobs:
           set -eux
           chmod +x ~
           echo "root:1000000:1000000000" | sudo tee /etc/subuid /etc/subgid
+          mkdir -p "${GOCOVERDIR}"
           cd test
-          sudo --preserve-env=PATH,GOPATH,GITHUB_ACTIONS,LXD_VERBOSE,LXD_BACKEND,LXD_CEPH_CLUSTER,LXD_CEPH_CEPHFS,LXD_CEPH_CEPHOBJECT_RADOSGW,LXD_OFFLINE,LXD_SKIP_TESTS,LXD_REQUIRED_TESTS, LXD_BACKEND=${{ matrix.backend }} ./main.sh ${{ matrix.suite }}
+          sudo --preserve-env=PATH,GOPATH,GOCOVERDIR,GITHUB_ACTIONS,LXD_VERBOSE,LXD_BACKEND,LXD_CEPH_CLUSTER,LXD_CEPH_CEPHFS,LXD_CEPH_CEPHOBJECT_RADOSGW,LXD_OFFLINE,LXD_SKIP_TESTS,LXD_REQUIRED_TESTS, LXD_BACKEND=${{ matrix.backend }} ./main.sh ${{ matrix.suite }}
+
+      - name: Upload coverage data
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-${{ matrix.go }}-${{ matrix.suite }}-${{ matrix.backend }}
+          path: /home/runner/work/lxd/lxd/coverage
 
   client:
     name: Client
@@ -471,7 +508,7 @@ jobs:
         uses: actions/upload-artifact@v4
         continue-on-error: true
         with:
-          name: ${{ runner.os }}
+          name: lxd-clients-${{ runner.os }}
           path: bin/
 
   documentation:

diff --git a/Makefile b/Makefile
@@ -12,6 +12,7 @@ CGO_LDFLAGS_ALLOW ?= (-Wl,-wrap,pthread_create)|(-Wl,-z,now)
 SPHINXENV=doc/.sphinx/venv/bin/activate
 SPHINXPIPPATH=doc/.sphinx/venv/bin/pip
 GOMIN=1.22.5
+GOCOVERDIR ?= $(shell go env GOCOVERDIR)
 
 ifneq "$(wildcard vendor)" ""
 	DQLITE_PATH=$(CURDIR)/vendor/dqlite
@@ -23,7 +24,7 @@ endif
 default: all
 
 .PHONY: all
-all: client lxd lxd-agent lxd-benchmark lxd-migrate
+all: client lxd lxd-agent lxd-migrate
 
 .PHONY: build
 build: lxd
@@ -33,32 +34,53 @@ ifeq "$(TAG_SQLITE3)" ""
 	@echo "Missing dqlite, run \"make deps\" to setup."
 	exit 1
 endif
-	CC="$(CC)" CGO_LDFLAGS_ALLOW="$(CGO_LDFLAGS_ALLOW)" go install -v -tags "$(TAG_SQLITE3)" $(DEBUG) ./...
+
+ifeq "$(GOCOVERDIR)" ""
+	CC="$(CC)" CGO_LDFLAGS_ALLOW="$(CGO_LDFLAGS_ALLOW)" go install -v -tags "$(TAG_SQLITE3)" -trimpath $(DEBUG) ./lxd ./lxc-to-lxd ./lxd-user ./lxd-benchmark
+else
+	CC="$(CC)" CGO_LDFLAGS_ALLOW="$(CGO_LDFLAGS_ALLOW)" go install -v -tags "$(TAG_SQLITE3)" -trimpath -cover $(DEBUG) ./lxd ./lxc-to-lxd ./lxd-user ./lxd-benchmark
+endif
+
 	@echo "LXD built successfully"
 
 .PHONY: client
 client:
-	go install -v -tags "$(TAG_SQLITE3)" $(DEBUG) ./lxc
+ifeq "$(GOCOVERDIR)" ""
+	go install -v -trimpath $(DEBUG) ./lxc
+else
+	go install -v -trimpath -cover $(DEBUG) ./lxc
+endif
+
 	@echo "LXD client built successfully"
 
 .PHONY: lxd-agent
 lxd-agent:
-	CGO_ENABLED=0 go install -v -tags agent,netgo ./lxd-agent
-	@echo "LXD agent built successfully"
+ifeq "$(GOCOVERDIR)" ""
+	CGO_ENABLED=0 go install -v -trimpath -tags agent,netgo ./lxd-agent
+else
+	CGO_ENABLED=0 go install -v -trimpath -cover -tags agent,netgo ./lxd-agent
+endif
 
-.PHONY: lxd-benchmark
-lxd-benchmark:
-	CGO_ENABLED=0 go install -v ./lxd-benchmark
-	@echo "LXD benchmark built successfully"
+	@echo "LXD agent built successfully"
 
 .PHONY: lxd-metadata
 lxd-metadata:
-	CGO_ENABLED=0 go install -v -tags lxd-metadata ./lxd/lxd-metadata
+ifeq "$(GOCOVERDIR)" ""
+	CGO_ENABLED=0 go install -v -trimpath -tags lxd-metadata ./lxd/lxd-metadata
+else
+	CGO_ENABLED=0 go install -v -trimpath -cover -tags lxd-metadata ./lxd/lxd-metadata
+endif
+
 	@echo "LXD metadata built successfully"
 
 .PHONY: lxd-migrate
 lxd-migrate:
-	CGO_ENABLED=0 go install -v -tags netgo ./lxd-migrate
+ifeq "$(GOCOVERDIR)" ""
+	CGO_ENABLED=0 go install -v -trimpath -tags netgo ./lxd-migrate
+else
+	CGO_ENABLED=0 go install -v -trimpath -cover -tags netgo ./lxd-migrate
+endif
+
 	@echo "LXD-MIGRATE built successfully"
 
 .PHONY: deps
@@ -102,7 +124,7 @@ update-protobuf:
 
 .PHONY: update-schema
 update-schema:
-	cd lxd/db/generate && go build -o $(GOPATH)/bin/lxd-generate -tags "$(TAG_SQLITE3)" $(DEBUG) && cd -
+	cd lxd/db/generate && go build -v -trimpath -o $(GOPATH)/bin/lxd-generate -tags "$(TAG_SQLITE3)" $(DEBUG) && cd -
 	go generate ./...
 	gofmt -s -w ./lxd/db/
 	goimports -w ./lxd/db/
@@ -139,8 +161,8 @@ ifeq "$(TAG_SQLITE3)" ""
 endif
 
 	CC="$(CC)" CGO_LDFLAGS_ALLOW="$(CGO_LDFLAGS_ALLOW)" go install -v -tags "$(TAG_SQLITE3) logdebug" $(DEBUG) ./...
-	CGO_ENABLED=0 go install -v -tags "netgo,logdebug" ./lxd-migrate
-	CGO_ENABLED=0 go install -v -tags "agent,netgo,logdebug" ./lxd-agent
+	CGO_ENABLED=0 go install -v -trimpath -tags "netgo,logdebug" ./lxd-migrate
+	CGO_ENABLED=0 go install -v -trimpath -tags "agent,netgo,logdebug" ./lxd-agent
 	@echo "LXD built successfully"
 
 .PHONY: nocache
@@ -151,8 +173,8 @@ ifeq "$(TAG_SQLITE3)" ""
 endif
 
 	CC="$(CC)" CGO_LDFLAGS_ALLOW="$(CGO_LDFLAGS_ALLOW)" go install -a -v -tags "$(TAG_SQLITE3)" $(DEBUG) ./...
-	CGO_ENABLED=0 go install -a -v -tags netgo ./lxd-migrate
-	CGO_ENABLED=0 go install -a -v -tags agent,netgo ./lxd-agent
+	CGO_ENABLED=0 go install -a -v -trimpath -tags netgo ./lxd-migrate
+	CGO_ENABLED=0 go install -a -v -trimpath -tags agent,netgo ./lxd-agent
 	@echo "LXD built successfully"
 
 race:
@@ -162,20 +184,22 @@ ifeq "$(TAG_SQLITE3)" ""
 endif
 
 	CC="$(CC)" CGO_LDFLAGS_ALLOW="$(CGO_LDFLAGS_ALLOW)" go install -race -v -tags "$(TAG_SQLITE3)" $(DEBUG) ./...
-	CGO_ENABLED=0 go install -v -tags netgo ./lxd-migrate
-	CGO_ENABLED=0 go install -v -tags agent,netgo ./lxd-agent
+	CGO_ENABLED=0 go install -v -trimpath -tags netgo ./lxd-migrate
+	CGO_ENABLED=0 go install -v -trimpath -tags agent,netgo ./lxd-agent
 	@echo "LXD built successfully"
 
 .PHONY: check
-check: default
-ifeq "$(LXD_OFFLINE)" ""
-	(cd / ; go install github.com/rogpeppe/godeps@latest)
-	(cd / ; go install github.com/tsenart/deadcode@latest)
-	(cd / ; go install golang.org/x/lint/golint@latest)
-endif
-	CGO_LDFLAGS_ALLOW="$(CGO_LDFLAGS_ALLOW)" go test -v -tags "$(TAG_SQLITE3)" $(DEBUG) ./...
+check: default check-unit
 	cd test && ./main.sh
 
+.PHONY: unit
+check-unit:
+ifeq "$(GOCOVERDIR)" ""
+	CGO_LDFLAGS_ALLOW="$(CGO_LDFLAGS_ALLOW)" go test -v -failfast -tags "$(TAG_SQLITE3)" $(DEBUG) ./...
+else
+	CGO_LDFLAGS_ALLOW="$(CGO_LDFLAGS_ALLOW)" go test -v -failfast -tags "$(TAG_SQLITE3)" $(DEBUG) ./... -cover -test.gocoverdir="${GOCOVERDIR}"
+endif
+
 .PHONY: dist
 dist: doc
 	# Cleanup