diff --git a/bin/custom-kernel b/bin/custom-kernel index 1d23d0679..f5280f52a 100644 --- a/bin/custom-kernel +++ b/bin/custom-kernel @@ -39,15 +39,20 @@ case "$i" in update-grub ;; - virtual-hwe) - echo "===> Installing the virtual HWE kernel" + hwe) + echo "===> Installing the HWE kernel" echo "MODULES=dep" > /etc/initramfs-tools/conf.d/modules.conf apt-get update apt-get dist-upgrade --yes - . /etc/os-release - apt-get install --no-install-recommends --yes "linux-image-virtual-hwe-${VERSION_ID}" - apt-get autopurge --yes linux-image-virtual "linux-image-$(uname -r)" "linux-modules-$(uname -r)" + . /etc/os-release + + FLAVOR="generic" + if systemd-detect-virt --quiet --vm; then + FLAVOR="virtual" + fi + apt-get install --no-install-recommends --yes "linux-image-${FLAVOR}-hwe-${VERSION_ID}" + apt-get autopurge --yes "linux-image-${FLAVOR}" "linux-image-$(uname -r)" "linux-modules-$(uname -r)" ;; ubuntu) diff --git a/bin/helpers b/bin/helpers new file mode 100644 index 000000000..7085d17a8 --- /dev/null +++ b/bin/helpers @@ -0,0 +1,82 @@ + +# waitSnapdSeed: wait for snapd to be seeded. +waitSnapdSeed() ( + set +x + for i in $(seq 60); do # Wait up to 60s. + if systemctl show snapd.seeded.service --value --property SubState | grep -qx exited; then + return 0 # Success. + fi + + sleep 1 + done + + echo "snapd not seeded after ${i}s" + return 1 # Failed. +) + +# waitVMAgent: waits for the VM agent to be running. +waitVMAgent() ( + set +x + vmName="${1}" + for i in $(seq 90); do + if lxc info "${vmName}" | grep -qF 127.0.0.1; then + return 0 # Success. + fi + + sleep 1 + done + + echo "VM ${vmName} agent not running after ${i}s" + return 1 # Failed. +) + + +# install_lxd: install LXD from a specific channel or `latest/edge` if none is provided. +install_lxd() ( + # Wait for snapd seeding + waitSnapdSeed + + snap remove lxd || true + snap install lxd --channel="${LXD_SNAP_CHANNEL:-"latest/edge"}" + snap list lxd + lxd waitready --timeout=300 +) + +# hasNeededAPIExtension: check if LXD supports the needed extension. +hasNeededAPIExtension() ( + needed_extension="${1}" + + lxc info | sed -ne '/^api_extensions:/,/^[^-]/ s/^- //p' | grep -qxF "${needed_extension}" +) + +# runsMinimumKernel: check if the running kernel is at least the minimum version. +runsMinimumKernel() ( + min_version="${1}" + min_major="$(echo "${min_version}" | cut -d. -f1)" + min_minor="$(echo "${min_version}" | cut -d. -f2)" + running_version="$(uname -r | cut -d. -f 1,2)" + running_major="$(echo "${running_version}" | cut -d. -f1)" + running_minor="$(echo "${running_version}" | cut -d. -f2)" + + if [ "${running_major}" -lt "${min_major}" ]; then + return 1 + elif [ "${running_major}" -eq "${min_major}" ] && [ "${running_minor}" -lt "${min_minor}" ]; then + return 1 + fi + return 0 +) + +# cleanup: report if the test passed or not and return the appropriate return code. +cleanup() { + echo "" + if [ "${FAIL}" = "1" ]; then + echo "Test failed" + exit 1 + fi + + echo "Test passed" + exit 0 +} + +FAIL=1 +trap cleanup EXIT HUP INT TERM diff --git a/bin/openstack-run b/bin/openstack-run index 41498d9e1..1a1ce5878 100755 --- a/bin/openstack-run +++ b/bin/openstack-run @@ -15,17 +15,19 @@ fi serie="${1}" kernel="${2}" script="${3}" +lxd_snap_channel="${4}" +shift 4 +_script="$(mktemp)" test_name="$(basename "${script}")" -shift 3 KEY_NAME="ssh-key" FLAVOR="$(openstack flavor list -f value -c Name | grep -m1 'cpu8-ram32-disk20\b')" NETWORK="$(openstack network list -f value -c Name | grep -Fm1 "net_stg-lxd-cloud-testing")" IMAGE="$(openstack image list -f value -c Name --sort-column Name --sort-descending | grep -m1 "auto-sync/ubuntu-${serie}-.*-amd64-")" -NAME="lxd-ci-${test_name}-${serie}-$$" +NAME="lxd-ci-${test_name}-${serie}-$(echo "${lxd_snap_channel}" | sed 's/[./]/-/g')" if ! [ -e ~/.ssh/id_ed25519 ]; then - mkdir -pm 0700 ~/.ssh + [ -d ~/.ssh ] || mkdir -m 0700 ~/.ssh ssh-keygen -t ed25519 -C "" -f ~/.ssh/id_ed25519 -N "" openstack keypair create --public-key ~/.ssh/id_ed25519.pub ssh-key fi @@ -42,7 +44,7 @@ wait_machine() { # https://bugs.launchpad.net/ubuntu/+source/cloud-init/+bug/2039441 for _ in $(seq 30); do ssh -o ConnectTimeout=1 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no "ubuntu@${IP}" true && break - sleep 1 + sleep 1 done } @@ -57,9 +59,10 @@ create() { RET=1 cleanup() { - # Release the macine + # Release the machine set +e openstack server delete "${NAME}" + rm -f "${_script}" if [ "${RET}" = "0" ]; then echo "" >&2 @@ -92,10 +95,11 @@ fi # Connect and run something echo "==> Running the job (${test_name})" >&2 +sed -e "1 a LXD_SNAP_CHANNEL=${lxd_snap_channel}" -e "1 r bin/helpers" "${script}" > "${_script}" if echo "${IP}" | grep -q ":"; then - scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no "${script}" "ubuntu@[${IP}]:test-script" + scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no "${_script}" "ubuntu@[${IP}]:test-script" else - scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no "${script}" "ubuntu@${IP}:test-script" + scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no "${_script}" "ubuntu@${IP}:test-script" fi ssh -n -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no "ubuntu@${IP}" sudo "https_proxy=http://squid.internal:3128" sh test-script "$@" diff --git a/tests/cgroup b/tests/cgroup index b229fa51f..067c51b40 100755 --- a/tests/cgroup +++ b/tests/cgroup @@ -1,53 +1,24 @@ #!/bin/sh set -eu -waitSnapdSeed() ( - set +x - for i in $(seq 60); do # Wait up to 60s. - if systemctl show snapd.seeded.service --value --property SubState | grep -qx exited; then - return 0 # Success. - fi - - sleep 1 - done - - echo "snapd not seeded after ${i}s" - return 1 # Failed. -) - -cleanup() { - echo "" - if [ "${FAIL}" = "1" ]; then - echo "Test failed" - exit 1 - fi - - echo "Test passed" - exit 0 -} - -FAIL=1 -trap cleanup EXIT HUP INT TERM - # Refresh apt apt-get update -# Wait for snapd seeding -waitSnapdSeed +# Install dependencies +apt-get install --no-install-recommends --yes jq iperf3 # Install LXD -snap remove lxd || true -snap install lxd --channel=latest/edge -apt-get install --no-install-recommends --yes jq iperf3 -lxd waitready --timeout=300 +install_lxd # Configure LXD lxd init --auto + +# Test set -x # Start a container with no limits echo "=> Start a container with no limits" -lxc launch ubuntu:20.04 c1 +lxc launch ubuntu-daily:22.04 c1 echo "==> Validate default values" [ "$(lxc exec c1 -- nproc)" = "$(nproc)" ] @@ -258,5 +229,5 @@ lxc pause c1 ! lxc exec c1 bash || false lxc start c1 -set +x +# shellcheck disable=SC2034 FAIL=0 diff --git a/tests/gpu-container b/tests/gpu-container index 835c76469..3a31538ac 100755 --- a/tests/gpu-container +++ b/tests/gpu-container @@ -67,7 +67,7 @@ lxc profile device add default eth0 nic network=lxdbr0 name=eth0 # Launch a test container echo "==> Launching a test container" -lxc launch ubuntu:22.04 c1 +lxc launch ubuntu-daily:22.04 c1 sleep 10 # Confirm no GPU diff --git a/tests/interception b/tests/interception index c08e7be74..6f8af567b 100755 --- a/tests/interception +++ b/tests/interception @@ -1,54 +1,23 @@ #!/bin/sh set -eu -waitSnapdSeed() ( - set +x - for i in $(seq 60); do # Wait up to 60s. - if systemctl show snapd.seeded.service --value --property SubState | grep -qx exited; then - return 0 # Success. - fi - - sleep 1 - done - - echo "snapd not seeded after ${i}s" - return 1 # Failed. -) - -cleanup() { - echo "" - if [ "${FAIL}" = "1" ]; then - echo "Test failed" - exit 1 - fi - - echo "Test passed" - exit 0 -} - -FAIL=1 -trap cleanup EXIT HUP INT TERM - # Refresh apt apt-get update -# Wait for snapd seeding -waitSnapdSeed +# Install dependencies +apt-get install --no-install-recommends --yes attr # Install LXD -snap remove lxd || true -snap install lxd --channel=latest/edge -snap set lxd shiftfs.enable=true -apt-get install --no-install-recommends --yes attr -lxd waitready --timeout=300 +install_lxd # Configure LXD +snap set lxd shiftfs.enable=true lxd init --auto # Test set -x -lxc launch ubuntu:20.04 c1 +lxc launch ubuntu-daily:22.04 c1 sleep 10 lxc exec c1 -- apt-get update lxc exec c1 -- apt-get install --no-install-recommends --yes attr fuse2fs @@ -76,40 +45,47 @@ lxc exec c1 -- mknod /dev/mknod-test c 1 3 lxc exec c1 -- mknod /root/mknod-test1 c 1 3 ## bpf (needs 5.9 or higher) -KMAJ="$(uname -r | cut -d. -f1)" -KMIN="$(uname -r | cut -d. -f2)" -if [ "${KMAJ}" -gt 5 ] || [ "${KMAJ}" -eq 5 ] && [ "${KMIN}" -ge 9 ]; then +if runsMinimumKernel 5.9; then lxc config set c1 security.syscalls.intercept.bpf=true security.syscalls.intercept.bpf.devices=true lxc restart c1 -f else echo "Skipping security.syscalls.intercept.bpf config as the kernel is too old" fi -## mount -truncate -s 10G loop.img -LOOP=$(losetup -f --show loop.img) -lxc config device add c1 loop unix-block source="${LOOP}" path=/dev/sda -lxc exec c1 -- mkfs.ext4 /dev/sda -! lxc exec c1 -- mount /dev/sda /mnt || false -lxc config set c1 security.syscalls.intercept.mount=true - -lxc config set c1 security.syscalls.intercept.mount.allowed=ext4 -lxc restart c1 -f -lxc exec c1 -- mount /dev/sda /mnt -[ "$(lxc exec c1 -- stat --format=%u:%g /mnt)" = "65534:65534" ] -lxc exec c1 -- umount /mnt - -lxc config set c1 security.syscalls.intercept.mount.shift=true -lxc exec c1 -- mount /dev/sda /mnt -[ "$(lxc exec c1 -- stat --format=%u:%g /mnt)" = "0:0" ] -lxc exec c1 -- umount /mnt - -lxc config unset c1 security.syscalls.intercept.mount.allowed -lxc config set c1 security.syscalls.intercept.mount.fuse=ext4=fuse2fs -lxc restart c1 -f +if hasNeededAPIExtension container_syscall_intercept_mount; then + ## mount + truncate -s 10G loop.img + LOOP=$(losetup -f --show loop.img) + lxc config device add c1 loop unix-block source="${LOOP}" path=/dev/sda + lxc exec c1 -- mkfs.ext4 /dev/sda + ! lxc exec c1 -- mount /dev/sda /mnt || false + lxc config set c1 security.syscalls.intercept.mount=true -lxc exec c1 -- mount /dev/sda /mnt -[ "$(lxc exec c1 -- stat --format=%u:%g /mnt)" = "0:0" ] -lxc exec c1 -- umount /mnt + lxc config set c1 security.syscalls.intercept.mount.allowed=ext4 + lxc restart c1 -f + lxc exec c1 -- mount /dev/sda /mnt + [ "$(lxc exec c1 -- stat --format=%u:%g /mnt)" = "65534:65534" ] + lxc exec c1 -- umount /mnt + + lxc config set c1 security.syscalls.intercept.mount.shift=true + lxc exec c1 -- mount /dev/sda /mnt + [ "$(lxc exec c1 -- stat --format=%u:%g /mnt)" = "0:0" ] + lxc exec c1 -- umount /mnt + + if hasNeededAPIExtension container_syscall_intercept_mount_fuse; then + lxc config unset c1 security.syscalls.intercept.mount.allowed + lxc config set c1 security.syscalls.intercept.mount.fuse=ext4=fuse2fs + lxc restart c1 -f + + lxc exec c1 -- mount /dev/sda /mnt + [ "$(lxc exec c1 -- stat --format=%u:%g /mnt)" = "0:0" ] + lxc exec c1 -- umount /mnt + else + echo "Skipping mount fuse tests as the container_syscall_intercept_mount_fuse API extension is missing" + fi +else + echo "Skipping mount tests as the container_syscall_intercept_mount API extension is missing" +fi +# shellcheck disable=SC2034 FAIL=0 diff --git a/tests/main.sh b/tests/main.sh index bbc5c3eb5..292ea462e 100755 --- a/tests/main.sh +++ b/tests/main.sh @@ -1,20 +1,27 @@ #!/bin/sh +set -eu -# cgroup -./bin/openstack-run jammy default tests/cgroup -./bin/openstack-run jammy cgroup1 tests/cgroup -./bin/openstack-run jammy swapaccount tests/cgroup +for lxd_snap_channel in "latest/edge" "5.0/edge"; do + # cgroup + ./bin/openstack-run jammy default tests/cgroup "${lxd_snap_channel}" + # XXX: disable test with Jammy's GA kernel configured for cgroup1 + # https://github.com/canonical/lxd-ci/issues/7 + #./bin/openstack-run jammy cgroup1 tests/cgroup "${lxd_snap_channel}" + ./bin/openstack-run jammy swapaccount tests/cgroup "${lxd_snap_channel}" -# interception -./bin/openstack-run jammy default tests/interception + # network-bridge-firewall + ./bin/openstack-run jammy default tests/network-bridge-firewall "${lxd_snap_channel}" + ./bin/openstack-run jammy hwe tests/network-bridge-firewall "${lxd_snap_channel}" -# network-bridge-firewall -./bin/openstack-run jammy default tests/network-bridge-firewall + # pylxd + ./bin/openstack-run jammy default tests/pylxd "${lxd_snap_channel}" -# pylxd -./bin/openstack-run jammy default tests/pylxd latest/edge -./bin/openstack-run jammy default tests/pylxd 5.0/edge -./bin/openstack-run jammy default tests/pylxd 4.0/edge + # storage + ./bin/openstack-run jammy default tests/storage-disks-vm "${lxd_snap_channel}" +done -# storage -./bin/openstack-run jammy default tests/storage-disks-vm +# interception +./bin/openstack-run jammy default tests/interception "latest/edge" + +# pylxd +./bin/openstack-run jammy default tests/pylxd "4.0/edge" diff --git a/tests/network-bridge-firewall b/tests/network-bridge-firewall index f453994a5..d609c4b90 100755 --- a/tests/network-bridge-firewall +++ b/tests/network-bridge-firewall @@ -1,44 +1,14 @@ #!/bin/sh -set -eux - -waitSnapdSeed() ( - set +x - for i in $(seq 60); do # Wait up to 60s. - if systemctl show snapd.seeded.service --value --property SubState | grep -qx exited; then - return 0 # Success. - fi - - sleep 1 - done - - echo "snapd not seeded after ${i}s" - return 1 # Failed. -) - -cleanup() { - echo "" - if [ "${FAIL}" = "1" ]; then - echo "Test failed" - exit 1 - fi - - echo "Test passed" - exit 0 -} - -FAIL=1 -trap cleanup EXIT HUP INT TERM +set -eu # Refresh apt apt-get update -# Wait for snapd seeding -waitSnapdSeed - # Install LXD -snap remove lxd || true -snap install lxd --channel=latest/edge -lxd waitready --timeout=300 +install_lxd + +# Test +set -x # Configure LXD lxc storage create default zfs @@ -64,7 +34,7 @@ modprobe br_netfilter ip link add lxdbr0unmanaged type bridge firewallTests() { - lxc launch ubuntu:focal c1 + lxc launch ubuntu-daily:22.04 c1 sleep 10 managed=0 @@ -149,7 +119,7 @@ firewallTests() { } networkLimitsPriorityNftablesTest() { - lxc launch ubuntu:focal c1 + lxc launch ubuntu-daily:22.04 c1 sleep 10 prio=7 @@ -185,8 +155,12 @@ lxc info | grep 'firewall: nftables' lxc profile device add default eth0 nic network=lxdbr0 firewallTests -echo "=> Performing nftables network device limits.priority option test" -networkLimitsPriorityNftablesTest +if hasNeededAPIExtension instances_nic_limits_priority && runsMinimumKernel 5.17; then + echo "=> Performing nftables network device limits.priority option test" + networkLimitsPriorityNftablesTest +else + echo "=> Skipping nftables network device limits.priority option test" +fi echo "=> Performing nftables unmanaged bridge tests" ip a flush dev lxdbr0 # Clear duplicate address from lxdbr0. @@ -236,4 +210,5 @@ lxc storage delete default lxd shutdown iptables -D INPUT +# shellcheck disable=SC2034 FAIL=0 diff --git a/tests/pylxd b/tests/pylxd index c22a246ef..6497dd83a 100755 --- a/tests/pylxd +++ b/tests/pylxd @@ -1,48 +1,19 @@ #!/bin/sh set -eu -channel=${1:-latest/stable} +# Install LXD +install_lxd + +# Test +set -x export DEBIAN_FRONTEND=noninteractive export HOME=/root -waitSnapdSeed() ( - set +x - for i in $(seq 60); do # Wait up to 60s. - if systemctl show snapd.seeded.service --value --property SubState | grep -qx exited; then - return 0 # Success. - fi - - sleep 1 - done - - echo "snapd not seeded after ${i}s" - return 1 # Failed. -) - -cleanup() { - echo "" - if [ "${FAIL}" = "1" ]; then - echo "Test failed" - exit 1 - fi - - echo "Test passed" - exit 0 -} - -FAIL=1 -trap cleanup EXIT HUP INT TERM - -# Wait for snapd seeding -waitSnapdSeed - -# Install LXD -snap remove lxd || true -snap install lxd --channel="${channel}" -lxd waitready --timeout=300 - # Run the pylxd tests [ -d pylxd ] || git clone https://github.com/canonical/pylxd cd pylxd -integration/run-integration-tests && FAIL=0 +integration/run-integration-tests + +# shellcheck disable=SC2034 +FAIL=0 \ No newline at end of file diff --git a/tests/storage-disks-vm b/tests/storage-disks-vm index 3e72b5b8c..e1e954aca 100755 --- a/tests/storage-disks-vm +++ b/tests/storage-disks-vm @@ -1,58 +1,11 @@ #!/bin/sh -set -eux - -waitSnapdSeed() ( - set +x - for i in $(seq 60); do # Wait up to 60s. - if systemctl show snapd.seeded.service --value --property SubState | grep -qx exited; then - return 0 # Success. - fi - - sleep 1 - done - - echo "snapd not seeded after ${i}s" - return 1 # Failed. -) - -cleanup() { - echo "" - if [ "${FAIL}" = "1" ]; then - echo "Test failed" - exit 1 - fi - - echo "Test passed" - exit 0 -} - -FAIL=1 -trap cleanup EXIT HUP INT TERM - -# Wait for snapd seeding -waitSnapdSeed +set -eu # Install LXD -snap remove lxd || true -snap install lxd --channel=latest/edge -lxd waitready --timeout=300 - -waitVMAgent() ( - set +x - # shellcheck disable=SC3043 - local vmName="$1" - for i in $(seq 90) # Wait up to 90s. - do - if lxc info "${vmName}" | grep -qF 127.0.0.1; then - return 0 # Success. - fi - - sleep 1 - done - - echo "VM ${vmName} agent not running after ${i}s" - return 1 # Failed. -) +install_lxd + +# Test +set -x echo "==> Setup share directory" # Create directory for use as basis for restricted disk source tests. @@ -93,7 +46,7 @@ lxc profile device add default eth0 nic network=lxdbr0 lxc profile show default # Create instance and add check relative source paths are not allowed. -lxc init ubuntu:22.04 v1 --vm +lxc init ubuntu-daily:22.04 v1 --vm ! lxc config device add v1 d1 disk source=foo path=/mnt || false # Check adding a disk with a source path above the restricted parent source path isn't allowed. @@ -162,18 +115,22 @@ lxc restart -f v1 waitVMAgent v1 lxc exec v1 -- mokutil --sb-state | grep -Fx "SecureBoot disabled" -# Remove disk device restrictions and add a NVMe disk -lxc stop -f v1 -lxc config device remove v1 d1 -lxc project unset restricted restricted.devices.disk -lxc project unset restricted restricted.devices.disk.paths -lxc project unset restricted restricted - -# Add a NVMe disk and check if a NVMe controller is added to the VM -lxc config device add v1 nvme-ssd disk source="${loopdev}" io.bus=nvme -lxc start v1 -waitVMAgent v1 -lxc exec v1 -- lspci | grep -F "QEMU NVM Express Controller" +if hasNeededAPIExtension disk_io_bus; then + # Remove disk device restrictions and add a NVMe disk + lxc stop -f v1 + lxc config device remove v1 d1 + lxc project unset restricted restricted.devices.disk + lxc project unset restricted restricted.devices.disk.paths + lxc project unset restricted restricted + + # Add a NVMe disk and check if a NVMe controller is added to the VM + lxc config device add v1 nvme-ssd disk source="${loopdev}" io.bus=nvme + lxc start v1 + waitVMAgent v1 + lxc exec v1 -- lspci | grep -F "QEMU NVM Express Controller" +else + echo 'Skipping NVME test due to missing extension: "disk_io_bus"' +fi echo "==> Cleanup" lxc delete -f v1 @@ -194,4 +151,5 @@ rmdir "${testRoot}" losetup --detach "${loopdev}" rm "${loopimg}" +# shellcheck disable=SC2034 FAIL=0