From e7f3e43cdd6508ecae01046f2783df7f8beb0a13 Mon Sep 17 00:00:00 2001 From: Mark Laing Date: Wed, 7 Aug 2024 09:21:00 +0100 Subject: [PATCH] WIP --- .github/workflows/tests.yml | 109 +++++------------------------------- tests/vm-migration | 36 +++++++----- 2 files changed, 36 insertions(+), 109 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 832c0c5b0..c260c8617 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -79,103 +79,10 @@ jobs: strategy: fail-fast: false matrix: - os: [20.04, 22.04, 24.04] - track: ${{ fromJSON(inputs.snap-tracks || '["latest/edge", "5.21/edge", "5.0/edge", "4.0/edge"]') }} + os: [24.04] + track: ${{ fromJSON(inputs.snap-tracks || '["latest/edge"]') }} test: - - cgroup - - cluster - - container - - container-copy - - conversion - - cpu-vm - - devlxd-vm - - devlxd-container - - docker - - efi-vars-editor-vm - - interception - - pylxd - - network-bridge-firewall - - network-ovn - - network-routed - - snapd - - storage-buckets - - storage-disks-vm - - "storage-vm dir" - - "storage-vm btrfs" - - "storage-vm ceph" - - "storage-vm lvm" - - "storage-vm lvm-thin" - - "storage-vm zfs" - - storage-volumes-vm - - tpm-vm - - vm-nesting - vm-migration - include: - - test: qemu-external-vm - track: "latest/edge" - os: "24.04" - exclude: - # not compatible with 4.0/* - - test: container-copy - track: "4.0/edge" - - test: conversion - track: "4.0/edge" - - test: cpu-vm - track: "4.0/edge" - - test: devlxd-vm - track: "4.0/edge" - - test: efi-vars-editor-vm - track: "4.0/edge" - - test: network-bridge-firewall - os: 20.04 - track: "4.0/edge" - - test: network-ovn - track: "4.0/edge" - # https://github.com/canonical/pylxd/issues/590 - - test: pylxd - track: "4.0/edge" - - test: storage-buckets - track: "4.0/edge" - - test: storage-disks-vm - track: "4.0/edge" - - test: "storage-vm dir" - track: "4.0/edge" - - test: "storage-vm btrfs" - track: "4.0/edge" - - test: "storage-vm ceph" - track: "4.0/edge" - - test: "storage-vm lvm" - track: "4.0/edge" - - test: "storage-vm lvm-thin" - track: "4.0/edge" - - test: "storage-vm zfs" - track: "4.0/edge" - - test: storage-volumes-vm - track: "4.0/edge" - - test: tpm-vm - track: "4.0/edge" - # not compatible with 5.0/* - - test: efi-vars-editor-vm # not compatible with 5.0/* - track: "5.0/edge" - # waiting for integration with microceph - - test: "storage-vm ceph" - # skip track/os combinaisons that are too far appart - - track: "4.0/edge" - os: "24.04" - - track: "5.0/edge" - os: "24.04" - - track: "5.0/edge" - os: "20.04" - - track: "5.21/edge" - os: "20.04" - - track: "latest/edge" - os: "20.04" - - track: "latest/edge" - os: "22.04" - - test: "vm-migration" - track: "4.0/edge" - - test: "vm-migration" - track: "5.0/edge" steps: - name: Performance tuning @@ -191,7 +98,7 @@ jobs: echo "force-unsafe-io" | sudo tee /etc/dpkg/dpkg.cfg.d/force-unsafe-io - name: Reclaim some space (storage tests only) - if: ${{ startsWith(matrix.test, 'storage') || matrix.test == 'vm-nesting' || matrix.test == 'conversion' }} + if: ${{ startsWith(matrix.test, 'storage') || matrix.test == 'vm-nesting' || matrix.test == 'conversion' || matrix.test == 'vm-migration' }} run: | set -eux df -h @@ -212,6 +119,16 @@ jobs: sudo rm -rf /opt/ghc df -h + - name: Reclaim some memory (VM migration tests only) + if: ${{ matrix.test == 'vm-migration' }} + run: | + set -eux + + free -mt + sudo systemctl stop dpkg-db-backup.timer e2scrub_all.timer fstrim.timer logrotate.timer man-db.timer motd-news.timer phpsessionclean.timer update-notifier-download.timer update-notifier-motd.timer + sudo systemctl stop iscsid.socket multipathd.socket + free -mt + - name: Remove docker run: | set -eux diff --git a/tests/vm-migration b/tests/vm-migration index 7bdd40aec..9ca566a05 100644 --- a/tests/vm-migration +++ b/tests/vm-migration @@ -22,6 +22,8 @@ lxc profile device add default root disk path="/" pool="${poolName}" lxc init "${TEST_IMG:-ubuntu-minimal-daily:24.04}" ceph --vm -c limits.cpu=2 -c limits.memory=4GiB lxc storage volume create "${poolName}" ceph-disk size=20GiB --type=block lxc config device add ceph ceph-disk disk pool="${poolName}" source=ceph-disk +# Disable vGPU to save RAM +lxc config set ceph raw.qemu.conf='[device "qemu_gpu"]' lxc start ceph # Wait for snap in ceph instance. @@ -53,22 +55,26 @@ for _ in $(seq 60); do fi done -# Launch two instances for our LXD cluster and wait for them to be ready. If the host supports `devlxd_images_vm` then -# set `security.devlxd.images=true` so that we don't have to download the image again. +# Initialise two instances for our LXD cluster. If the host supports `devlxd_images_vm` then set +# `security.devlxd.images=true` so that we don't have to download the image again. if hasNeededAPIExtension devlxd_images_vm; then - lxc launch "${TEST_IMG:-ubuntu-minimal-daily:24.04}" member1 --vm -c limits.memory=2GiB -c security.devlxd.images=true - lxc launch "${TEST_IMG:-ubuntu-minimal-daily:24.04}" member2 --vm -c limits.memory=2GiB -c security.devlxd.images=true + lxc init "${TEST_IMG:-ubuntu-minimal-daily:24.04}" member1 --vm -c limits.cpu=2 -c limits.memory=4GiB -c security.devlxd.images=true + lxc init "${TEST_IMG:-ubuntu-minimal-daily:24.04}" member2 --vm -c limits.cpu=2 -c limits.memory=4GiB -c security.devlxd.images=true else - lxc launch "${TEST_IMG:-ubuntu-minimal-daily:24.04}" member1 --vm -c limits.memory=2GiB - lxc launch "${TEST_IMG:-ubuntu-minimal-daily:24.04}" member2 --vm -c limits.memory=2GiB + lxc init "${TEST_IMG:-ubuntu-minimal-daily:24.04}" member1 --vm -c limits.cpu=2 -c limits.memory=4GiB + lxc init "${TEST_IMG:-ubuntu-minimal-daily:24.04}" member2 --vm -c limits.cpu=2 -c limits.memory=4GiB fi +# Disable vGPU to save RAM +lxc config set member1 raw.qemu.conf='[device "qemu_gpu"]' +lxc config set member2 raw.qemu.conf='[device "qemu_gpu"]' + +# Start the instances and wait for member1 to be ready. +lxc start member1 +lxc start member2 waitInstanceReady member1 -waitInstanceReady member2 # shellcheck disable=SC3044 # Ignore "declare is undefined" shellcheck error. lxc exec member1 -- sh -c "$(declare -f waitSnapdSeed); waitSnapdSeed" -# shellcheck disable=SC3044 # Ignore "declare is undefined" shellcheck error. -lxc exec member2 -- sh -c "$(declare -f waitSnapdSeed); waitSnapdSeed" # Install LXD in the first member. lxc exec member1 -- snap remove --purge lxd || true @@ -91,6 +97,11 @@ lxc exec member1 -- lxc config set core.https_address="${member1Address}:8443" lxc exec member1 -- lxc cluster enable member1 joinToken="$(lxc exec member1 -- lxc cluster add member2 --quiet)" +# Ensure member2 is ready. +waitInstanceReady member2 +# shellcheck disable=SC3044 # Ignore "declare is undefined" shellcheck error. +lxc exec member2 -- sh -c "$(declare -f waitSnapdSeed); waitSnapdSeed" + # Install LXD on the second member. lxc exec member2 -- snap remove --purge lxd || true lxc exec member2 -- snap install lxd --channel="${LXD_SNAP_CHANNEL}" @@ -136,7 +147,7 @@ lxc exec member1 -- lxc storage create ceph ceph lxc exec member1 -- lxc storage volume create ceph vol1 --type=block size=500MiB # Create a VM in the cluster, on member1. -lxc exec member1 -- lxc init "${TEST_IMG:-ubuntu-minimal-daily:24.04}" v1 --vm --storage ceph --target member1 -c migration.stateful=true -c limits.memory=1GiB +lxc exec member1 -- lxc init "${TEST_IMG:-ubuntu-minimal-daily:24.04}" v1 --vm --storage ceph --target member1 -c migration.stateful=true -c limits.cpu=1 -c limits.memory=500MiB # Add vol1 as a disk device to the VM. lxc exec member1 -- lxc config device add v1 vol1-disk disk pool=ceph source=vol1 @@ -145,9 +156,7 @@ lxc exec member1 -- lxc config device add v1 vol1-disk disk pool=ceph source=vol lxc exec member1 -- lxc start v1 # Wait for a long time for it to boot (doubly nested VM takes a while). -while [ "$(lxc exec member1 -- lxc info v1 | awk '{if ($1 == "Processes:") print $2}')" -le 1 ]; do - sleep 30 -done +lxc exec member1 -- sh -c 'while [ "$(lxc info v1 | awk '"'"'{if ($1 == "Processes:") print $2}'"'"')" -le 1 ]; do echo "Instance v1 still not booted, waiting 60s..." && sleep 60; done' # vol1 should be available as /dev/sdb. Format it as ext4. Then mount it and create a file. lxc exec member1 -- lxc exec v1 -- mkfs -t ext4 /dev/sdb @@ -159,6 +168,7 @@ lxc exec member1 -- lxc exec v1 -- cp /etc/hostname /mnt/vol1/bar lxc exec member1 -- lxc move v1 --target member2 # The VM is slow. So the agent isn't immediately available after the live migration. +lxc exec member1 -- sh -c 'while [ "$(lxc info v1 | awk '"'"'{if ($1 == "Processes:") print $2}'"'"')" -le 1 ]; do echo "Instance v1 still not booted, waiting 60s..." && sleep 60; done' while [ "$(lxc exec member1 -- lxc info v1 | awk '{if ($1 == "Processes:") print $2}')" -le 1 ]; do sleep 5 done