From 93e34028693d96ef004af82ce5d43b592e7d8f77 Mon Sep 17 00:00:00 2001 From: Mark Laing Date: Tue, 27 Aug 2024 17:05:13 +0100 Subject: [PATCH] WIP --- .github/workflows/tests.yml | 114 +++++---------------------------- tests/vm-migration | 122 +++++++++++++++++++++++++++++++----- 2 files changed, 123 insertions(+), 113 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d8cfb5dc8..2e0cb95ca 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -83,103 +83,10 @@ jobs: strategy: fail-fast: false matrix: - os: ${{ fromJSON(inputs.ubuntu-releases || '["20.04", "22.04", "24.04"]') }} - track: ${{ fromJSON(inputs.snap-tracks || '["latest/edge", "5.21/edge", "5.0/edge", "4.0/edge"]') }} + os: ${{ fromJSON(inputs.ubuntu-releases || '["24.04"]') }} + track: ${{ fromJSON(inputs.snap-tracks || '["latest/edge"]') }} test: - - cgroup - - cluster - - container - - container-copy - - conversion - - cpu-vm - - devlxd-container - - devlxd-vm - - docker - - efi-vars-editor-vm - - interception - - network-bridge-firewall - - network-ovn - - network-routed - - pylxd - - snapd - - storage-buckets - - storage-disks-vm - - "storage-vm btrfs" - - "storage-vm ceph" - - "storage-vm dir" - - "storage-vm lvm" - - "storage-vm lvm-thin" - - "storage-vm zfs" - - storage-volumes-vm - - tpm-vm - vm-migration - - vm-nesting - include: - - test: qemu-external-vm - track: "latest/edge" - os: "24.04" - exclude: - # not compatible with 4.0/* - - test: container-copy - track: "4.0/edge" - - test: conversion - track: "4.0/edge" - - test: cpu-vm - track: "4.0/edge" - - test: devlxd-vm - track: "4.0/edge" - - test: efi-vars-editor-vm - track: "4.0/edge" - - test: network-bridge-firewall - os: 20.04 - track: "4.0/edge" - - test: network-ovn - track: "4.0/edge" - # https://github.com/canonical/pylxd/issues/590 - - test: pylxd - track: "4.0/edge" - - test: storage-buckets - track: "4.0/edge" - - test: storage-disks-vm - track: "4.0/edge" - - test: "storage-vm dir" - track: "4.0/edge" - - test: "storage-vm btrfs" - track: "4.0/edge" - - test: "storage-vm ceph" - track: "4.0/edge" - - test: "storage-vm lvm" - track: "4.0/edge" - - test: "storage-vm lvm-thin" - track: "4.0/edge" - - test: "storage-vm zfs" - track: "4.0/edge" - - test: storage-volumes-vm - track: "4.0/edge" - - test: tpm-vm - track: "4.0/edge" - # not compatible with 5.0/* - - test: efi-vars-editor-vm # not compatible with 5.0/* - track: "5.0/edge" - # waiting for integration with microceph - - test: "storage-vm ceph" - # skip track/os combinaisons that are too far appart - - track: "4.0/edge" - os: "24.04" - - track: "5.0/edge" - os: "24.04" - - track: "5.0/edge" - os: "20.04" - - track: "5.21/edge" - os: "20.04" - - track: "latest/edge" - os: "20.04" - - track: "latest/edge" - os: "22.04" - - test: "vm-migration" - track: "4.0/edge" - - test: "vm-migration" - track: "5.0/edge" steps: - name: Performance tuning @@ -195,7 +102,7 @@ jobs: echo "force-unsafe-io" | sudo tee /etc/dpkg/dpkg.cfg.d/force-unsafe-io - name: Reclaim some space (storage tests only) - if: ${{ startsWith(matrix.test, 'storage') || matrix.test == 'vm-nesting' || matrix.test == 'conversion' }} + if: ${{ startsWith(matrix.test, 'storage') || matrix.test == 'vm-nesting' || matrix.test == 'conversion' || matrix.test == 'vm-migration' }} run: | set -eux df -h @@ -225,6 +132,16 @@ jobs: sudo rm -rf /opt/ghc df -h + - name: Reclaim some memory (VM migration tests only) + if: ${{ matrix.test == 'vm-migration' }} + run: | + set -eux + + free -mt + sudo systemctl stop dpkg-db-backup.timer e2scrub_all.timer fstrim.timer logrotate.timer man-db.timer motd-news.timer phpsessionclean.timer update-notifier-download.timer update-notifier-motd.timer + sudo systemctl stop iscsid.socket multipathd.socket + free -mt + - name: Remove docker run: | set -eux @@ -266,7 +183,10 @@ jobs: src_track="$(echo "${dst_track}" | cut -d/ -f1)/stable" EXTRA_ARGS="${EXTRA_ARGS:-3} ${src_track} ${{ matrix.track }}" fi - sudo --preserve-env=PURGE_LXD,TEST_IMG ./bin/local-run "tests/${TEST_SCRIPT}" ${{ matrix.track }} ${EXTRA_ARGS:-} + sudo --preserve-env=PURGE_LXD,TEST_IMG,GITHUB_ACTIONS ./bin/local-run "tests/${TEST_SCRIPT}" ${{ matrix.track }} ${EXTRA_ARGS:-} + + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 # always update cache as we have our own logic of # cache invalidation and updates in addition to a date check diff --git a/tests/vm-migration b/tests/vm-migration index 5b32f568f..9bcea94d1 100644 --- a/tests/vm-migration +++ b/tests/vm-migration @@ -12,16 +12,51 @@ lxc network create lxdbr0 lxc profile device add default eth0 nic network=lxdbr0 poolName="ctpool$$" -poolDriver=dir +poolDriver=zfs echo "==> Create storage pool using driver ${poolDriver}" lxc storage create "${poolName}" "${poolDriver}" lxc profile device add default root disk path="/" pool="${poolName}" +LOCAL_LOOP_DEVICE="" +cleanup() { + if [ -n "${LOCAL_LOOP_DEVICE:-}" ]; then + losetup --detach "${LOCAL_LOOP_DEVICE}" + fi +} + +trap cleanup EXIT HUP INT TERM + # Create ceph node -lxc init "${TEST_IMG:-ubuntu-minimal-daily:24.04}" ceph --vm -c limits.cpu=2 -c limits.memory=4GiB -lxc storage volume create "${poolName}" ceph-disk size=20GiB --type=block -lxc config device add ceph ceph-disk disk pool="${poolName}" source=ceph-disk +lxc init "${TEST_IMG:-ubuntu-minimal-daily:24.04}" ceph --vm -c limits.cpu=4 -c limits.memory=4GiB +if [ -n "${GITHUB_ACTIONS:-}" ]; then + # If the rootfs and the ephemeral part are on the same physical disk, giving the whole + # disk to microceph would wipe our rootfs. Since it is pretty rare for GitHub Action + # runners to have a single disk, we immediately bail rather than trying to gracefully + # handle it. Once snapd releases with https://github.com/snapcore/snapd/pull/13150, + # we will be able to stop worrying about that special case. + if [ "$(stat -c '%d' /)" = "$(stat -c '%d' /mnt)" ]; then + echo "FAIL: rootfs and ephemeral part on the same disk, aborting" + exit 1 + fi + + # Free-up the ephemeral disk to use it as ceph OSD. + # https://github.com/canonical/microceph/issues/288 and https://github.com/canonical/microceph/issues/289 + swapoff /mnt/swapfile + ephemeral_disk="$(findmnt --noheadings --output SOURCE --target /mnt | sed 's/[0-9]\+$//')" + umount /mnt + +# lxc config device add ceph ceph-disk unix-block source="${ephemeral_disk}" path=/dev/sdb + lxc config device add ceph ceph-disk disk source="${ephemeral_disk}" path=/dev/sdb +else + lxc storage volume create "${poolName}" ceph-disk size=20GiB --type=block + lxc config device add ceph ceph-disk disk pool="${poolName}" source=ceph-disk +# dd if=/dev/zero of=blockfile count=20480 bs=1M # 20GiB +# LOCAL_LOOP_DEVICE="$(losetup --find)" +# losetup "${LOCAL_LOOP_DEVICE}" blockfile +# lxc config device add ceph ceph-disk unix-block source="${LOCAL_LOOP_DEVICE}" path=/dev/sdb +fi + lxc start ceph # Wait for snap in ceph instance. @@ -40,7 +75,7 @@ lxc exec ceph -- microceph.ceph osd crush rule create-replicated replicated defa for flag in nosnaptrim noscrub nobackfill norebalance norecover noscrub nodeep-scrub; do lxc exec ceph -- microceph.ceph osd set "${flag}" done -lxc exec ceph -- microceph disk add /dev/sdb +lxc exec ceph -- microceph disk add /dev/sdb --wipe lxc exec ceph -- microceph.ceph osd pool create cephfs_meta 32 lxc exec ceph -- microceph.ceph osd pool create cephfs_data 32 lxc exec ceph -- microceph.ceph fs new cephfs cephfs_meta cephfs_data @@ -53,15 +88,39 @@ for _ in $(seq 60); do fi done -# Launch two instances for our LXD cluster and wait for them to be ready. -lxc launch "${TEST_IMG:-ubuntu-minimal-daily:24.04}" member1 --vm -c limits.memory=2GiB -lxc launch "${TEST_IMG:-ubuntu-minimal-daily:24.04}" member2 --vm -c limits.memory=2GiB +cat << EOF | lxc profile create kvm +config: + limits.cpu: 4 + limits.memory: 4GiB + linux.kernel_modules: kvm,vhost_net,vhost_vsock,rbd + security.devlxd.images: "true" + security.idmap.isolated: "false" + security.nesting: "true" +devices: + kvm: + source: /dev/kvm + type: unix-char + vhost-net: + source: /dev/vhost-net + type: unix-char + vhost-vsock: + source: /dev/vhost-vsock + type: unix-char + vsock: + mode: "0666" + source: /dev/vsock + type: unix-char +EOF + +lxc init "${TEST_IMG:-ubuntu-minimal-daily:24.04}" member1 --profile default --profile kvm +lxc init "${TEST_IMG:-ubuntu-minimal-daily:24.04}" member2 --profile default --profile kvm + +# Start the instances and wait for member1 to be ready. +lxc start member1 +lxc start member2 waitInstanceReady member1 -waitInstanceReady member2 # shellcheck disable=SC3044 # Ignore "declare is undefined" shellcheck error. lxc exec member1 -- sh -c "$(declare -f waitSnapdSeed); waitSnapdSeed" -# shellcheck disable=SC3044 # Ignore "declare is undefined" shellcheck error. -lxc exec member2 -- sh -c "$(declare -f waitSnapdSeed); waitSnapdSeed" # Install LXD in the first member. lxc exec member1 -- snap remove --purge lxd || true @@ -71,14 +130,24 @@ if [ -n "${LXD_SIDELOAD_PATH:-}" ]; then lxc file push "${LXD_SIDELOAD_PATH}" member1/var/snap/lxd/common/lxd.debug lxc exec member1 -- systemctl restart snap.lxd.daemon fi +if [ -n "${LXD_AGENT_SIDELOAD_PATH:-}" ]; then + lxc file push "${LXD_AGENT_SIDELOAD_PATH}" "member1/root/$(basename "${LXD_AGENT_SIDELOAD_PATH}")" + lxc exec member1 -- mount --bind "$(basename "${LXD_AGENT_SIDELOAD_PATH}")" /snap/lxd/current/bin/lxd-agent + lxc exec member1 -- systemctl restart snap.lxd.daemon +fi # Initialise and configure LXD in the first member. lxc exec member1 -- lxd init --auto -member1Address="$(lxc query /1.0/instances/member1?recursion=2 | jq -r ".state.network.enp5s0.addresses[0].address")" +member1Address="$(lxc query /1.0/instances/member1?recursion=2 | jq -r ".state.network.eth0.addresses[0].address")" lxc exec member1 -- lxc config set core.https_address="${member1Address}:8443" lxc exec member1 -- lxc cluster enable member1 joinToken="$(lxc exec member1 -- lxc cluster add member2 --quiet)" +# Ensure member2 is ready. +waitInstanceReady member2 +# shellcheck disable=SC3044 # Ignore "declare is undefined" shellcheck error. +lxc exec member2 -- sh -c "$(declare -f waitSnapdSeed); waitSnapdSeed" + # Install LXD on the second member. lxc exec member2 -- snap remove --purge lxd || true lxc exec member2 -- snap install lxd --channel="${LXD_SNAP_CHANNEL}" @@ -87,9 +156,14 @@ if [ -n "${LXD_SIDELOAD_PATH:-}" ]; then lxc file push "${LXD_SIDELOAD_PATH}" member2/var/snap/lxd/common/lxd.debug lxc exec member2 -- systemctl restart snap.lxd.daemon fi +if [ -n "${LXD_AGENT_SIDELOAD_PATH:-}" ]; then + lxc file push "${LXD_AGENT_SIDELOAD_PATH}" "member2/root/$(basename "${LXD_AGENT_SIDELOAD_PATH}")" + lxc exec member2 -- mount --bind "$(basename "${LXD_AGENT_SIDELOAD_PATH}")" /snap/lxd/current/bin/lxd-agent + lxc exec member2 -- systemctl restart snap.lxd.daemon +fi # Create a preseed file for member2 to join member1. -member2Address="$(lxc query /1.0/instances/member2?recursion=2 | jq -r ".state.network.enp5s0.addresses[0].address")" +member2Address="$(lxc query /1.0/instances/member2?recursion=2 | jq -r ".state.network.eth0.addresses[0].address")" preseed="$( cat <