From 848dc0c22fdfd980de1887c13d4f3d36aaf782a3 Mon Sep 17 00:00:00 2001 From: Thomas Parrott Date: Wed, 11 Sep 2024 16:36:10 +0100 Subject: [PATCH 1/5] github: Add performance tuning to code tests Signed-off-by: Thomas Parrott --- .github/workflows/tests.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 025ec50ce20e..1337ff5e56fc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,6 +36,18 @@ jobs: name: Code runs-on: ubuntu-22.04 steps: + - name: Performance tuning + run: | + set -eux + # optimize ext4 FSes for performance, not reliability + for fs in $(findmnt --noheading --type ext4 --list --uniq | awk '{print $1}'); do + # nombcache and data=writeback cannot be changed on remount + sudo mount -o remount,noatime,barrier=0,commit=6000 "${fs}" || true + done + + # disable dpkg from calling sync() + echo "force-unsafe-io" | sudo tee /etc/dpkg/dpkg.cfg.d/force-unsafe-io + - name: Checkout uses: actions/checkout@v4 with: From 876707e71bfa5c3c79b1929cb7b1032435538594 Mon Sep 17 00:00:00 2001 From: Thomas Parrott Date: Wed, 11 Sep 2024 16:36:20 +0100 Subject: [PATCH 2/5] github: Mask unwanted lxc services in code tests For consistency. Signed-off-by: Thomas Parrott --- .github/workflows/tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1337ff5e56fc..f1c60e3bdbbf 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -86,6 +86,8 @@ jobs: sudo add-apt-repository ppa:ubuntu-lxc/daily -y --no-update sudo apt-get update + sudo systemctl mask lxc.service lxc-net.service + sudo apt-get install --no-install-recommends -y \ build-essential \ curl \ From 84842e383ca063b91aa0f13d280081a4b87e4db0 Mon Sep 17 00:00:00 2001 From: Thomas Parrott Date: Wed, 11 Sep 2024 16:37:14 +0100 Subject: [PATCH 3/5] github: Use set -eux on all test commands Signed-off-by: Thomas Parrott --- .github/workflows/tests.yml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f1c60e3bdbbf..2a1b86857a99 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -368,6 +368,7 @@ jobs: - name: Set exec perms on LXD binaries run: | + set -eux ls -lR /home/runner/go/bin/ chmod uog+x /home/runner/go/bin/* @@ -436,7 +437,9 @@ jobs: sudo rm -f /snap/bin/rbd - name: Make GOCOVERDIR - run: mkdir -p "${GOCOVERDIR}" + run: | + set -eux + mkdir -p "${GOCOVERDIR}" if: env.GOCOVERDIR != '' - name: "Run system tests (${{ matrix.go }}, ${{ matrix.suite }}, ${{ matrix.backend }})" @@ -478,6 +481,7 @@ jobs: - name: Create build directory run: | + set -eux mkdir bin - name: Build static lxc (x86_64) @@ -485,6 +489,7 @@ jobs: CGO_ENABLED: 0 GOARCH: amd64 run: | + set -eux go build -ldflags "-s -w" -o trimpath -o bin/lxc.x86_64 ./lxc - name: Build static lxc (aarch64) @@ -492,6 +497,7 @@ jobs: CGO_ENABLED: 0 GOARCH: arm64 run: | + set -eux go build -ldflags "-s -w" -o trimpath -o bin/lxc.aarch64 ./lxc - name: Build static lxd-benchmark @@ -515,17 +521,23 @@ jobs: - name: Unit tests (client) env: CGO_ENABLED: 0 - run: go test -v ./client/... + run: | + set -eux + go test -v ./client/... - name: Unit tests (lxc) env: CGO_ENABLED: 0 - run: go test -v ./lxc/... + run: | + set -eux + go test -v ./lxc/... - name: Unit tests (shared) env: CGO_ENABLED: 0 - run: go test -v ./shared/... + run: | + set -eux + go test -v ./shared/... - name: Upload lxc client artifacts uses: actions/upload-artifact@v4 From 7807a1c6d64f87ecc14475ab3c5c23f55f880d83 Mon Sep 17 00:00:00 2001 From: Thomas Parrott Date: Wed, 11 Sep 2024 16:47:05 +0100 Subject: [PATCH 4/5] github: Reclaim some space Signed-off-by: Thomas Parrott --- .github/workflows/tests.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2a1b86857a99..2192f7c64e8b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -108,6 +108,9 @@ jobs: pkg-config \ shellcheck + # reclaim some space + sudo apt-get clean + python3 -m pip install flake8 # Download minio ready to include in dependencies for system tests. @@ -285,12 +288,16 @@ jobs: # This was inspired from https://github.com/easimon/maximize-build-space df -h / + # dotnet sudo rm -rf /usr/share/dotnet # android sudo rm -rf /usr/local/lib/android # haskell sudo rm -rf /opt/ghc + # codeql + sudo rm -rf /opt/hostedtoolcache/CodeQL + df -h / - name: Remove docker @@ -436,6 +443,9 @@ jobs: sudo microceph.ceph status sudo rm -f /snap/bin/rbd + # reclaim some space + sudo apt-get clean + - name: Make GOCOVERDIR run: | set -eux From 4d81f6e30a6e0d515a1f17636b6fb8c9f156a459 Mon Sep 17 00:00:00 2001 From: Thomas Parrott Date: Wed, 11 Sep 2024 11:39:33 +0100 Subject: [PATCH 5/5] github: Add initial TICS step Signed-off-by: Thomas Parrott --- .github/workflows/tests.yml | 114 ++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2192f7c64e8b..d5d9f964795d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -467,6 +467,120 @@ jobs: path: ${{env.GOCOVERDIR}} if: env.GOCOVERDIR != '' + tics: + name: Tiobe TICS + runs-on: ubuntu-22.04 + needs: system-tests + env: + CGO_CFLAGS: "-I/home/runner/go/bin/dqlite/include/" + CGO_LDFLAGS: "-L/home/runner/go/bin/dqlite/libs/" + LD_LIBRARY_PATH: "/home/runner/go/bin/dqlite/libs/" + CGO_LDFLAGS_ALLOW: "(-Wl,-wrap,pthread_create)|(-Wl,-z,now)" + TICSAUTHTOKEN: ${{ secrets.TICS_AUTH_TOKEN }} + if: ${{ github.event_name == 'workflow_dispatch' }} + steps: + - name: Performance tuning + run: | + set -eux + # optimize ext4 FSes for performance, not reliability + for fs in $(findmnt --noheading --type ext4 --list --uniq | awk '{print $1}'); do + # nombcache and data=writeback cannot be changed on remount + sudo mount -o remount,noatime,barrier=0,commit=6000 "${fs}" || true + done + + # disable dpkg from calling sync() + echo "force-unsafe-io" | sudo tee /etc/dpkg/dpkg.cfg.d/force-unsafe-io + + - name: Reclaim some space + run: | + set -eux + + # Purge snaps and snapd. + sudo snap remove --purge $(snap list | awk '!/^Name|^core|^snapd/ {print $1}') && sudo apt-get autopurge -y snapd + + # This was inspired from https://github.com/easimon/maximize-build-space + df -h / + + # dotnet + sudo rm -rf /usr/share/dotnet + # android + sudo rm -rf /usr/local/lib/android + # haskell + sudo rm -rf /opt/ghc + # codeql + sudo rm -rf /opt/hostedtoolcache/CodeQL + + df -h / + + - name: Remove docker + run: | + set -eux + sudo apt-get autopurge -y containerd.io moby-containerd docker docker-ce podman uidmap + sudo ip link delete docker0 + sudo nft flush ruleset + + - name: Checkout + uses: actions/checkout@v4 + with: + # A non-shallow clone is needed for the Differential ShellCheck + fetch-depth: 0 + + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: 1.22.x + + - name: Download coverage data + uses: actions/download-artifact@v4 + with: + pattern: coverage-* + path: ${{env.GOCOVERDIR}} + merge-multiple: true + + - name: Download system test dependencies + uses: actions/download-artifact@v4 + with: + name: system-test-deps + merge-multiple: true + path: /home/runner/go/bin + + - name: Install dependencies + run: | + set -eux + sudo add-apt-repository ppa:ubuntu-lxc/daily -y --no-update + sudo apt-get update + + sudo systemctl mask lxc.service lxc-net.service + + sudo apt-get install --no-install-recommends -y \ + libcap-dev \ + libacl1-dev \ + libuv1-dev \ + libudev-dev \ + liblxc-dev \ + pkg-config \ + libdbus-1-dev \ + libseccomp-dev + + # reclaim some space + sudo apt-get clean + + go install github.com/axw/gocov/gocov@latest + go install github.com/AlekSi/gocov-xml@latest + go install honnef.co/go/tools/cmd/staticcheck@latest + + - name: Convert coverage files + run: | + go tool covdata textfmt -i="${GOCOVERDIR}" -o "${GOCOVERDIR}"/coverage.out + gocov convert "${GOCOVERDIR}"/coverage.out > "${GOCOVERDIR}"/coverage.json + gocov-xml < "${GOCOVERDIR}"/coverage.json > "${GOCOVERDIR}"/coverage-go.xml + + - name: Run TICS + run: | + curl --silent --show-error "https://canonical.tiobe.com/tiobeweb/TICS/api/public/v1/fapi/installtics/Script?cfg=default&platform=linux&url=https://canonical.tiobe.com/tiobeweb/TICS/" > install_tics.sh + . ./install_tics.sh + TICSQServer -project LXD -tmpdir /tmp/tics -branchdir . + client: name: Client strategy: