diff --git a/.gitignore b/.gitignore new file mode 100644 index 000000000..89f9ac04a --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +out/ diff --git a/bin/helpers b/bin/helpers index 7085d17a8..e0ae96991 100644 --- a/bin/helpers +++ b/bin/helpers @@ -30,6 +30,56 @@ waitVMAgent() ( return 1 # Failed. ) +# waitInstanceReady: waits for the instance to be ready. +waitInstanceReady() ( + set +x + maxWait=90 + instName="$1" + instProj="${2:-}" + if [ -z "${instProj}" ]; then + # Find the currently selected project. + instProj="$(lxc project list -f csv | sed -n 's/^\([^(]\+\) (current),.*/\1/ p')" + fi + for _ in $(seq "${maxWait}"); do + if lxc info --project "${instProj}" "${instName}" | grep -qF "Status: READY"; then + return 0 # Success. + fi + + sleep 1 + done + + echo "Instance ${instName} (${instProj}) not ready after ${maxWait}s" + lxc list --project "${instProj}" "${instName}" + return 1 # Failed. +) + +# setupCloudInitReady: setup a cloud-init script to signal readiness. +setupCloudInitReady() { + instProj="${1}" + profile="${2:-default}" + + lxc profile set "${profile}" --project "${instProj}" cloud-init.user-data - << EOF +#cloud-config +write_files: + - content: | + #!/bin/sh + exec curl --unix-socket /dev/lxd/sock lxd/1.0 -X PATCH -d '{"state":"Ready"}' + path: /var/lib/cloud/scripts/per-boot/ready.sh + permissions: "0755" +EOF +} + +# enableSRIOV: enable SR-IOV on a NIC. +enableNICSRIOV() ( + set +x + parentNIC="${1}" + numVFS="${2:-"7"}" + + echo "${numVFS}" > "/sys/class/net/${parentNIC}/device/sriov_numvfs" + ip link set "${parentNIC}" up + sleep 10 + ethtool "${parentNIC}" +) # install_lxd: install LXD from a specific channel or `latest/edge` if none is provided. install_lxd() ( diff --git a/bin/openstack-run b/bin/openstack-run index 1a1ce5878..db2a8c399 100755 --- a/bin/openstack-run +++ b/bin/openstack-run @@ -94,7 +94,7 @@ if [ "${kernel}" != "default" ]; then fi # Connect and run something -echo "==> Running the job (${test_name})" >&2 +echo "==> Running the job ${test_name} against ${lxd_snap_channel}" >&2 sed -e "1 a LXD_SNAP_CHANNEL=${lxd_snap_channel}" -e "1 r bin/helpers" "${script}" > "${_script}" if echo "${IP}" | grep -q ":"; then scp -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no "${_script}" "ubuntu@[${IP}]:test-script" diff --git a/bin/testflinger-run b/bin/testflinger-run new file mode 100755 index 000000000..a2e1cbfce --- /dev/null +++ b/bin/testflinger-run @@ -0,0 +1,58 @@ +#!/bin/sh +set -eux + +serie="${1}" +kernel="${2}" +script="${3}" +lxd_snap_channel="${4}" +shift 4 +_script="$(mktemp)" +test_name="$(basename "${script}")" + +testflinger_yaml_job() { + cat << EOF +job_queue: anything +provision_data: + distro: ${serie} +test_data: + test_cmds: | + #!/bin/bash + set -eux + export SSH_OPTS="-q -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=30 -o ServerAliveCountMax=3" + SCP="scp \$SSH_OPTS" + SSH="ssh -n \$SSH_OPTS" + + # Get lxd-ci scripts + git clone https://github.com/canonical/lxd-ci.git + cd lxd-ci + + echo "Desired kernel: ${kernel}" + + # Copy test script to DUT + _script="\$(mktemp)" + sed -e "1 a LXD_SNAP_CHANNEL=${lxd_snap_channel}" -e "1 r bin/helpers" "${script}" > "\${_script}" + \$SCP "\${_script}" "ubuntu@\${DEVICE_IP}:test-script" + + # Run the test + \$SSH "ubuntu@\${DEVICE_IP}" -- sudo sh test-script +EOF +} + +setup_testflinger() { + # Test connectivity + if ! wget --method HEAD -qO /dev/null https://testflinger.canonical.com/agents; then + echo "Failed to connect to testflinger.canonical.com, make sure you are connected to the VPN" >&2 + exit 1 + fi + + command -v testflinger >/dev/null && return + + echo "Installing testflinger snap" >&2 + snap install testflinger-cli +} + +setup_testflinger + +# Submit the job +echo "==> Running the job ${test_name} against ${lxd_snap_channel}" >&2 +testflinger_yaml_job | testflinger submit --poll - diff --git a/tests/cluster b/tests/cluster index a7f137171..ca17893ac 100755 --- a/tests/cluster +++ b/tests/cluster @@ -1,5 +1,7 @@ -#!/bin/sh -eu -PREFIX="cluster-$(uuidgen)" +#!/bin/sh +set -eu + +PREFIX="cluster-$$" if [ -z "${1:-""}" ] || [ -z "${2:-""}" ] || [ -z "${3:-""}" ]; then echo "Usage: ${0} " @@ -31,21 +33,11 @@ cleanup() { trap cleanup EXIT HUP INT TERM # Launch the container -lxc launch images:ubuntu/jammy "${PREFIX}-1" -c security.nesting=true +lxc launch ubuntu-daily:22.04 "${PREFIX}-1" -c security.nesting=true sleep 10 -lxc file push - "${PREFIX}-1"/etc/apt/sources.list << EOF -deb http://us.archive.ubuntu.com/ubuntu jammy main universe -deb http://us.archive.ubuntu.com/ubuntu jammy-updates main universe -deb http://us.archive.ubuntu.com/ubuntu jammy-security main universe -EOF - -lxc exec "${PREFIX}-1" -- apt-get update -lxc exec "${PREFIX}-1" -- apt-get install --no-install-recommends snapd fuse3 curl --yes -lxc exec "${PREFIX}-1" -- snap install core20 -lxc exec "${PREFIX}-1" -- snap install core22 -lxc exec "${PREFIX}-1" -- snap install lxd --channel="$2" +lxc exec "${PREFIX}-1" -- snap refresh lxd --channel="$2" for i in $(seq 2 "$1"); do lxc copy "${PREFIX}-1" "${PREFIX}-$i" @@ -55,7 +47,6 @@ done for i in $(seq "$1"); do sleep 10 - # Configure the cluster if [ "$i" = "1" ]; then CLUSTER_IP=$(lxc exec "${PREFIX}-$i" -- ip -4 addr show dev eth0 scope global | grep inet | cut -d' ' -f6 | cut -d/ -f1) @@ -92,8 +83,8 @@ lxc exec "${PREFIX}-1" -- lxc cluster list # Test fan networking (intra fan from container and host, as well as external NAT comms) echo "==> Test fan networking" -lxc exec "${PREFIX}-1" -- lxc launch images:ubuntu/focal u1 -lxc exec "${PREFIX}-1" -- lxc launch images:ubuntu/focal u2 +lxc exec "${PREFIX}-1" -- lxc launch ubuntu-daily:22.04 u1 +lxc exec "${PREFIX}-1" -- lxc launch ubuntu-daily:22.04 u2 echo "==> Wait for addresses" sleep 10 @@ -121,3 +112,6 @@ sleep 5m echo "==> Validating the cluster" lxc exec "${PREFIX}-1" -- lxc info lxc exec "${PREFIX}-1" -- lxc cluster list + +# shellcheck disable=SC2034 +FAIL=0 diff --git a/tests/cpu-vm b/tests/cpu-vm index c4dfa979c..749d78f2c 100755 --- a/tests/cpu-vm +++ b/tests/cpu-vm @@ -1,64 +1,13 @@ #!/bin/sh set -eux -waitSnapdSeed() ( - set +x - for i in $(seq 60); do # Wait up to 60s. - if systemctl show snapd.seeded.service --value --property SubState | grep -qx exited; then - return 0 # Success. - fi - - sleep 1 - done - - echo "snapd not seeded after ${i}s" - return 1 # Failed. -) - -cleanup() { - echo "" - if [ "${FAIL}" = "1" ]; then - echo "Test failed" - exit 1 - fi - - echo "Test passed" - exit 0 -} - architecture="$(uname -m)" - if [ "${architecture}" != "x86_64" ] && [ "${architecture}" != "s390x" ]; then echo "Skipping test as CPU hotplugging not supported on ${architecture}" fi -FAIL=1 -trap cleanup EXIT HUP INT TERM - -# Wait for snapd seeding -waitSnapdSeed - # Install LXD -snap remove lxd || true -snap install lxd --channel=latest/edge -lxd waitready --timeout=300 - -waitVMAgent() ( - set +x - # shellcheck disable=SC3043 - local vmName="$1" - for i in $(seq 90) # Wait up to 90s. - do - if lxc info "${vmName}" | grep -qF 127.0.0.1; then - return 0 # Success. - fi - - sleep 1 - done - - echo "VM ${vmName} agent not running after ${i}s" - return 1 # Failed. -) +install_lxd # Configure LXD lxc network create lxdbr0 @@ -71,7 +20,7 @@ echo "==> Create storage pool using driver ${poolDriver}" lxc storage create "${poolName}" "${poolDriver}" echo "==> Create ephemeral VM and boot" -lxc launch images:ubuntu/22.04 v1 --vm -s "${poolName}" --ephemeral +lxc launch ubuntu-daily:22.04 v1 --vm -s "${poolName}" --ephemeral waitVMAgent v1 lxc info v1 @@ -123,4 +72,5 @@ lxc storage delete "${poolName}" echo "==> Deleting storage pool" lxc network delete lxdbr0 +# shellcheck disable=SC2034 FAIL=0 diff --git a/tests/devlxd-vm b/tests/devlxd-vm index e13786c9a..468c5090d 100755 --- a/tests/devlxd-vm +++ b/tests/devlxd-vm @@ -1,59 +1,14 @@ #!/bin/sh set -eux -waitSnapdSeed() ( - set +x - for i in $(seq 60); do # Wait up to 60s. - if systemctl show snapd.seeded.service --value --property SubState | grep -qx exited; then - return 0 # Success. - fi +# Refresh apt +apt-get update - sleep 1 - done - - echo "snapd not seeded after ${i}s" - return 1 # Failed. -) - -cleanup() { - echo "" - if [ "${FAIL}" = "1" ]; then - echo "Test failed" - exit 1 - fi - - echo "Test passed" - exit 0 -} - -FAIL=1 -trap cleanup EXIT HUP INT TERM - -# Wait for snapd seeding -waitSnapdSeed - -# Install LXD -snap remove lxd || true -snap install lxd --channel=latest/edge +# Install dependencies apt-get install --no-install-recommends --yes jq -lxd waitready --timeout=300 - -waitVMAgent() ( - set +x - # shellcheck disable=SC3043 - local vmName="$1" - for i in $(seq 90) # Wait up to 90s. - do - if lxc info "${vmName}" | grep -qF 127.0.0.1; then - return 0 # Success. - fi - - sleep 1 - done - echo "VM ${vmName} agent not running after ${i}s" - return 1 # Failed. -) +# Install LXD +install_lxd # Configure LXD lxc network create lxdbr0 @@ -66,14 +21,10 @@ echo "==> Create storage pool using driver ${poolDriver}" lxc storage create "${poolName}" "${poolDriver}" echo "==> Create VM and boot" -lxc init images:ubuntu/22.04/cloud v1 --vm -s "${poolName}" -lxc start v1 +lxc launch ubuntu-daily:22.04 v1 --vm -s "${poolName}" waitVMAgent v1 lxc info v1 -# Install curl -lxc exec v1 -- sh -c "apt-get update && apt-get install --no-install-recommends --yes curl" - echo "==> Checking devlxd is working" # devlxd is enabled by default and should work @@ -136,10 +87,9 @@ lxc start v1 waitVMAgent v1 # Configure to use the proxy -lxc exec v1 -- apt-get install --no-install-recommends snapd --yes -lxc exec v1 -- snap install lxd --channel=latest/edge +lxc exec v1 -- snap install lxd --channel="${LXD_SNAP_CHANNEL:-"latest/edge"}" lxc exec v1 -- /snap/bin/lxd init --auto -lxc exec v1 -- /snap/bin/lxc launch images:ubuntu/22.04/cloud v1v1 --vm +lxc exec v1 -- /snap/bin/lxc launch ubuntu-daily:22.04 v1v1 --vm sleep 30 lxc exec v1 -- /snap/bin/lxc info v1v1 | grep -F RUNNING @@ -153,4 +103,5 @@ lxc storage delete "${poolName}" echo "==> Deleting storage pool" lxc network delete lxdbr0 +# shellcheck disable=SC2034 FAIL=0 diff --git a/tests/docker b/tests/docker index 8cff7b22e..66ff3b5a1 100755 --- a/tests/docker +++ b/tests/docker @@ -1,48 +1,13 @@ -#!/bin/sh -eux -CNAME="docker-$(uuidgen)" - -waitSnapdSeed() ( - set +x - for i in $(seq 60); do # Wait up to 60s. - if systemctl show snapd.seeded.service --value --property SubState | grep -qx exited; then - return 0 # Success. - fi - - sleep 1 - done - - echo "snapd not seeded after ${i}s" - return 1 # Failed. -) - -cleanup() { - echo "" - if [ "${FAIL}" = "1" ]; then - echo "Test failed" - exit 1 - fi - - echo "Test passed" - exit 0 -} - -FAIL=1 -trap cleanup EXIT HUP INT TERM +#!/bin/sh +set -eux -# Wait for snapd seeding -waitSnapdSeed +CNAME="docker-$$" # Install LXD -snap remove lxd || true -snap install lxd --channel=latest/edge -apt-get install --no-install-recommends --yes jq -lxd waitready --timeout=300 - -lxd init --auto -set -x +install_lxd # Create the container -lxc launch images:ubuntu/bionic "${CNAME}" -c security.nesting=true "$@" +lxc launch ubuntu-daily:22.04 "${CNAME}" -c security.nesting=true "$@" ( cat << EOF @@ -52,9 +17,8 @@ sleep 10 set -eux # Install distro docker -apt-get update --yes --force-yes -apt-get install --no-install-recommends --yes --force-yes apparmor docker.io wget -systemctl start apparmor +apt-get update +apt-get install --no-install-recommends --yes --force-yes docker.io # Stop the distro docker systemctl stop docker.service @@ -85,4 +49,6 @@ EOF ) | lxc exec "${CNAME}" -- sh -eux lxc delete --force "${CNAME}" + +# shellcheck disable=SC2034 FAIL=0 diff --git a/tests/gpu-mig b/tests/gpu-mig new file mode 100755 index 000000000..ce537df34 --- /dev/null +++ b/tests/gpu-mig @@ -0,0 +1,72 @@ +#!/bin/sh +set -eu + +# Install LXD +install_lxd + +# Configure LXD +lxc storage create default zfs +lxc profile device add default root disk path=/ pool=default +lxc network create lxdbr0 +lxc profile device add default eth0 nic network=lxdbr0 name=eth0 + +# Confirm GPU is online +nvidia-smi + +# LXD resource API +lxc info --resources + +# Setup MIG +nvidia-smi -mig 1 +nvidia-smi mig -lgip +nvidia-smi mig -cgi 2g.10gb,1g.5gb,1g.5gb +nvidia-smi mig -lgi +nvidia-smi mig -lcip +nvidia-smi mig -cci 1g.5gb -gi 7 +nvidia-smi mig -cci 1g.5gb -gi 13 +nvidia-smi mig -cci 1c.2g.10gb,1c.2g.10gb -gi 5 +nvidia-smi + +UUIDS="$(nvidia-smi -L | sed -n "/(UUID: MIG-/ s/.* \(MIG-[^)]\+\))$/\1/p")" +UUID1="$(echo "$UUIDS" | sed -n '1p')" +UUID2="$(echo "$UUIDS" | sed -n '2p')" +UUID3="$(echo "$UUIDS" | sed -n '3p')" +UUID4="$(echo "$UUIDS" | sed -n '4p')" + +# Launch test containers +lxc init ubuntu-daily:22.04 nvidia-mig1 -c nvidia.runtime=true +lxc config device add nvidia-mig1 gpu0 gpu gputype=mig mig.uuid="$UUID1" pci=07:00.0 +lxc init ubuntu-daily:22.04 nvidia-mig2 -c nvidia.runtime=true +lxc config device add nvidia-mig2 gpu0 gpu gputype=mig mig.uuid="$UUID2" pci=07:00.0 +lxc init ubuntu-daily:22.04 nvidia-mig3 -c nvidia.runtime=true +lxc config device add nvidia-mig3 gpu0 gpu gputype=mig mig.uuid="$UUID3" pci=07:00.0 +lxc init ubuntu-daily:22.04 nvidia-mig4 -c nvidia.runtime=true +lxc config device add nvidia-mig4 gpu0 gpu gputype=mig mig.uuid="$UUID4" pci=07:00.0 +lxc start nvidia-mig1 +lxc exec nvidia-mig1 -- nvidia-smi +lxc start nvidia-mig2 +lxc exec nvidia-mig2 -- nvidia-smi +lxc start nvidia-mig3 +lxc exec nvidia-mig3 -- nvidia-smi +lxc start nvidia-mig4 +lxc exec nvidia-mig4 -- nvidia-smi + +lxc stop nvidia-mig4 +lxc config device add nvidia-mig4 gpu1 gpu gputype=mig mig.uuid="$UUID1" vendorid=10de productid=20f1 +lxc config device add nvidia-mig4 gpu2 gpu gputype=mig mig.uuid="$UUID2" vendorid=10de productid=20f1 +lxc start nvidia-mig4 +lxc exec nvidia-mig4 -- nvidia-smi + +# Wait for them to start and list +lxc list + +# Stop all instances +lxc stop --all + +# Cleanup MIG +nvidia-smi mig -dci +nvidia-smi mig -dgi +nvidia-smi -mig 0 + +# shellcheck disable=SC2034 +FAIL=0 diff --git a/tests/gpu-vm b/tests/gpu-vm new file mode 100755 index 000000000..ba5474f69 --- /dev/null +++ b/tests/gpu-vm @@ -0,0 +1,50 @@ +#!/bin/sh +set -eu + +# Install LXD +install_lxd + +# Configure LXD +lxc storage create default zfs +lxc profile device add default root disk path=/ pool=default +lxc network create lxdbr0 +lxc profile device add default eth0 nic network=lxdbr0 name=eth0 + +if [ "${1}" = "nvidia" ]; then + # Enable SR-IOV + /usr/lib/nvidia/sriov-manage -e ALL + + # Confirm GPU is online + nvidia-smi + + # Disable MIG if enabled + nvidia-smi -mig 0 +fi + +# LXD resource API +lxc info --resources + +# Launch test instances +for i in $(seq 1 4); do + lxc init ubuntu-daily:22.04 "v${i}" --vm -c security.secureboot=false + if [ "${1}" = "nvidia" ]; then + lxc config device add "v${i}" vgpu gpu gputype=mdev pci=0000:07:00.0 mdev=nvidia-468 + fi + lxc start "v${i}" +done + +# Wait for them to start and list +sleep 30 +lxc list + +if [ "${1}" = "nvidia" ]; then + # Validate NVIDIA vGPU + lxc exec v4 -- apt-get update + lxc exec v4 -- apt-get install --no-install-recommends --yes build-essential wget pciutils linux-headers-virtual + lxc exec v4 -- wget -6 http://canonical-lxd.stgraber.org/nvidia/v14.0/nvidia-guest.deb + lxc exec v4 -- apt-get install --yes /root/nvidia-guest.deb + lxc exec v4 -- nvidia-smi +fi + +# shellcheck disable=SC2034 +FAIL=0 diff --git a/tests/kernel b/tests/kernel index ad75fa4e9..4617f1be6 100755 --- a/tests/kernel +++ b/tests/kernel @@ -4,30 +4,13 @@ set -eu # Print kernel version echo "Starting test on: $(uname -a)" -waitSnapdSeed() ( - set +x - for i in $(seq 60); do # Wait up to 60s. - if systemctl show snapd.seeded.service --value --property SubState | grep -qx exited; then - return 0 # Success. - fi - - sleep 1 - done - - echo "snapd not seeded after ${i}s" - return 1 # Failed. -) - -# Refresh apt -apt-get update +# Install dependencies +apt-add-repository ppa:ubuntu-lxc/daily --yes +apt-get dist-upgrade --yes # Wait for snapd seeding waitSnapdSeed - -# Install dependencies snap install go --classic -apt-add-repository ppa:ubuntu-lxc/daily --yes -apt-get dist-upgrade --yes apt-get install --no-install-recommends --yes libudev-dev liblxc-dev liblz4-dev libacl1-dev libuv1-dev libselinux-dev libseccomp-dev tcl libsqlite3-dev libcap-dev libdbus-1-dev dh-autoreconf build-essential acl attr easy-rsa ebtables jq pkg-config socat sqlite3 dnsmasq-base dnsutils nftables gettext apt-get remove --purge --yes uidmap @@ -77,3 +60,6 @@ uname -a # LXD_SHIFTFS_DISABLE=1 because of kernel regression https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1990849 LXD_SKIP_STATIC=1 LXD_SKIP_TESTS=storage_buckets LXD_OFFLINE=0 LXD_TMPFS=1 LXD_SHIFTFS_DISABLE=1 LXD_VERBOSE=1 time -p ./main.sh + +# shellcheck disable=SC2034 +FAIL=0 diff --git a/tests/main-testflinger b/tests/main-testflinger new file mode 100755 index 000000000..c91cb8ba9 --- /dev/null +++ b/tests/main-testflinger @@ -0,0 +1,97 @@ +#!/bin/bash +set -eux + +lxd_snap_channel="${1}" +shift 1 + +runner="testflinger" +pids="" + +run_test() { + local serie="${1}" + local kernel="${2}" + local test_file="${3}" + local lxd_snap_channel="${4}" + shift 4 + + local test_name out_dir log_file lxd_snap_channel_name + test_name="$(basename "${test_file}")" + out_dir="out/${runner}/${lxd_snap_channel_name}" + log_file="${out_dir}/${serie}-${kernel}-${test_name}.log" + lxd_snap_channel_name="$(echo "${lxd_snap_channel}" | sed 's/[./]/-/g')" + mkdir -p "${out_dir}/" + "./bin/${runner}-run" "${serie}" "${kernel}" "${test_file}" "${lxd_snap_channel}" "${@}" > "${log_file}" & + pids="$! ${pids}" + + # Avoid sending too many simultaneous requests + sleep 10 +} + +# cgroup +run_test jammy default tests/cgroup "${lxd_snap_channel}" +# XXX: disable test with Jammy's GA kernel configured for cgroup1 +# https://github.com/canonical/lxd-ci/issues/7 +#run_test jammy cgroup1 tests/cgroup "${lxd_snap_channel}" +#run_test jammy swapaccount tests/cgroup "${lxd_snap_channel}" +run_test focal default tests/cgroup "${lxd_snap_channel}" +#run_test focal swapaccount tests/cgroup "${lxd_snap_channel}" + +# cluster +#run_test jammy default tests/cluster "${lxd_snap_channel}" 3 "5.0/stable" "${lxd_snap_channel}" + +# cpu-vm +run_test jammy default tests/cpu-vm "${lxd_snap_channel}" + +# devlxd-vm +run_test jammy default tests/devlxd-vm "${lxd_snap_channel}" + +# docker +# XXX: unable to pull docker image due to rate limiting done by docker hub +#run_test jammy default tests/docker "${lxd_snap_channel}" + +# gpu +# XXX: requires nvidia graphic card to be present in the host and to +# know which PCI address it has +#run_test jammy default tests/gpu-container "${lxd_snap_channel}" +# XXX: requires a specific nvidia graphic card to be present in the host +#run_test jammy default tests/gpu-mig "${lxd_snap_channel}" +# XXX: requires nvidia graphic card to be present in the host and a +# compatible driver to be installed +#run_test jammy default tests/gpu-vm "${lxd_snap_channel}" nvidia + +# interception +# XXX: doesn't work on 5.0 or earlier but there is no extension advertised +if echo "${lxd_snap_channel}" | grep -qE "^(latest|5\.[1-9].)/"; then + run_test jammy default tests/interception "${lxd_snap_channel}" +fi + +# network +run_test jammy default tests/network "${lxd_snap_channel}" +run_test jammy default tests/network-bridge-firewall "${lxd_snap_channel}" +#run_test jammy hwe tests/network-bridge-firewall "${lxd_snap_channel}" +run_test jammy default tests/network-ovn "${lxd_snap_channel}" +run_test jammy default tests/network-routed "${lxd_snap_channel}" +# XXX: requires SR-IOV capable NIC +#run_test jammy default tests/network-sriov "${lxd_snap_channel}" + +# pylxd +run_test jammy default tests/pylxd "${lxd_snap_channel}" + +# storage +run_test jammy default tests/storage-buckets "${lxd_snap_channel}" +run_test jammy default tests/storage-disks-vm "${lxd_snap_channel}" +run_test jammy default tests/storage-vm "${lxd_snap_channel}" +run_test jammy default tests/storage-volume-vm "${lxd_snap_channel}" + +# vm-nesting +run_test jammy default tests/vm-nesting "${lxd_snap_channel}" +#run_test jammy hwe tests/vm-nesting "${lxd_snap_channel}" + +# Wait for all tests to finish +errors=0 +for pid in ${pids}; do + echo "Waiting on ${pid}" + wait "${pid}" || errors="$((errors + 1))" +done + +echo "All done, errors: ${errors}" diff --git a/tests/network b/tests/network index 2d4d06fe6..6117ee3ea 100755 --- a/tests/network +++ b/tests/network @@ -1,115 +1,49 @@ #!/bin/sh set -eux -parentNIC="${1:-"enp5s0"}" - -waitSnapdSeed() ( - set +x - for i in $(seq 60); do # Wait up to 60s. - if systemctl show snapd.seeded.service --value --property SubState | grep -qx exited; then - return 0 # Success. - fi - - sleep 1 - done - - echo "snapd not seeded after ${i}s" - return 1 # Failed. -) - -waitInstanceReady() ( - set +x - # shellcheck disable=SC3043 - local instName="$1" - for i in $(seq 90) # Wait up to 90s. - do - if lxc info "${instName}" | grep -qF "Status: READY"; then - return 0 # Success. - fi - - sleep 1 - done - - echo "Instance ${instName} not ready after ${i}s" - return 1 # Failed. -) - -cleanup() { - echo "" - if [ "${FAIL}" = "1" ]; then - echo "Test failed" - exit 1 - fi - - echo "Test passed" - exit 0 -} - -FAIL=1 -trap cleanup EXIT HUP INT TERM - -# Wait for snapd seeding -waitSnapdSeed - # Install LXD -snap remove lxd || true -snap install lxd --channel=latest/edge -apt-get install --no-install-recommends --yes jq -lxd waitready --timeout=300 +install_lxd + +parentNIC="${1:-"enp5s0"}" # Enable SR-IOV on nic and bring up -echo 7 > "/sys/class/net/${parentNIC}/device/sriov_numvfs" -ip link set "${parentNIC}" up -sleep 10 -ethtool "${parentNIC}" -ip a +enableNICSRIOV "${parentNIC}" # Configure LXD lxc storage create default zfs lxc profile device add default root disk path=/ pool=default - -cat < VM on default VLAN with physical" -lxc init images:ubuntu/22.04/cloud v1-physical --vm -c limits.cpu=3 +lxc init ubuntu-daily:22.04 v1-physical --vm -c limits.cpu=3 lxc config device add v1-physical eth0 nic nictype=physical parent="${parentNIC}v1" name=eth0 lxc start v1-physical echo "==> Container on default VLAN with physical" -lxc init images:ubuntu/22.04/cloud c1-physical +lxc init ubuntu-daily:22.04 c1-physical lxc config device add c1-physical eth0 nic nictype=physical parent="${parentNIC}v2" name=eth0 lxc start c1-physical # Launch instances with macvlan NICs echo "==> VM on default VLAN with macvlan" -lxc init images:ubuntu/22.04/cloud v1-macvlan --vm -c limits.cpu=3 +lxc init ubuntu-daily:22.04 v1-macvlan --vm -c limits.cpu=3 lxc config device add v1-macvlan eth0 nic nictype=macvlan parent="${parentNIC}" name=eth0 lxc start v1-macvlan echo "==> Container on default VLAN with macvlan" -lxc init images:ubuntu/22.04/cloud c1-macvlan +lxc init ubuntu-daily:22.04 c1-macvlan lxc config device add c1-macvlan eth0 nic nictype=macvlan parent="${parentNIC}" name=eth0 lxc start c1-macvlan # Launch instances with sriov NICs echo "==> VM on default VLAN with sriov" -lxc init images:ubuntu/22.04/cloud v1-sriov --vm -c limits.cpu=3 +lxc init ubuntu-daily:22.04 v1-sriov --vm -c limits.cpu=3 lxc config device add v1-sriov eth0 nic nictype=sriov parent="${parentNIC}" name=eth0 lxc start v1-sriov echo "==> Container on default VLAN with sriov" -lxc init images:ubuntu/22.04/cloud c1-sriov +lxc init ubuntu-daily:22.04 c1-sriov lxc config device add c1-sriov eth0 nic nictype=sriov parent="${parentNIC}" name=eth0 lxc start c1-sriov @@ -276,4 +210,5 @@ lxc profile device remove default root lxc storage delete default lxc network delete lxdbr0 +# shellcheck disable=SC2034 FAIL=0 diff --git a/tests/network-ovn b/tests/network-ovn index e652af379..22e04fc7d 100755 --- a/tests/network-ovn +++ b/tests/network-ovn @@ -1,86 +1,11 @@ #!/bin/sh set -eux -waitSnapdSeed() ( - set +x - for i in $(seq 60); do # Wait up to 60s. - if systemctl show snapd.seeded.service --value --property SubState | grep -qx exited; then - return 0 # Success. - fi - - sleep 1 - done - - echo "snapd not seeded after ${i}s" - return 1 # Failed. -) - -waitInstanceReady() ( - set +x - # shellcheck disable=SC3043 - local maxWait=90 - # shellcheck disable=SC3043 - local instName="$1" - # shellcheck disable=SC3043 - local instProj="${2:-}" - if [ -z "${instProj}" ]; then - # Find the currently selected project. - instProj="$(lxc project list -f csv | sed -n 's/^\([^(]\+\) (current),.*/\1/ p')" - fi - for _ in $(seq "${maxWait}"); do - if lxc info --project "${instProj}" "${instName}" | grep -qF "Status: READY"; then - return 0 # Success. - fi - - sleep 1 - done - - echo "Instance ${instName} (${instProj}) not ready after ${maxWait}s" - lxc list --project "${instProj}" "${instName}" - return 1 # Failed. -) - -setupCloudInitReady() { - # shellcheck disable=SC3043 - local instProj="${1}" - # shellcheck disable=SC3043 - local profile="${2:-default}" - - cat < Performing network tests" for url in $(lxc query "/1.0/instances" | jq -r .[]); do @@ -106,24 +68,6 @@ networkTests() { return 0 } -FAIL=1 -trap cleanup EXIT HUP INT TERM - -# Wait for snapd seeding -waitSnapdSeed - -# Install LXD -snap remove lxd || true -snap install lxd --channel=latest/edge -apt-get install --no-install-recommends --yes jq -lxd waitready --timeout=300 - -# Enable SR-IOV on nic and bring up -echo 7 > "/sys/class/net/${parentNIC}/device/sriov_numvfs" -ip link set "${parentNIC}" up -sleep 10 -ethtool "${parentNIC}" - # Configure LXD lxc storage create default zfs lxc profile device add default root disk path=/ pool=default @@ -132,21 +76,21 @@ lxc profile device add default eth0 nic nictype=sriov parent="${parentNIC}" name # Launch a few VMs. # Do this first before containers to ensure VF free search handles VFs unbound from host. echo "==> VM on default VLAN" -lxc init images:ubuntu/20.04/cloud v1 --vm +lxc init ubuntu-daily:22.04 v1 --vm lxc start v1 echo "==> VM on default VLAN with filtering" -lxc init images:ubuntu/20.04/cloud v2 --vm +lxc init ubuntu-daily:22.04 v2 --vm lxc config device override v2 eth0 security.mac_filtering=true lxc start v2 echo "==> VM on alternate VLAN" -lxc init images:ubuntu/20.04/cloud v3 --vm +lxc init ubuntu-daily:22.04 v3 --vm lxc config device override v3 eth0 vlan=4000 lxc start v3 echo "==> VM on alternate VLAN with filtering" -lxc init images:ubuntu/20.04/cloud v4 --vm +lxc init ubuntu-daily:22.04 v4 --vm lxc config device override v4 eth0 vlan=4000 security.mac_filtering=true lxc start v4 @@ -165,21 +109,21 @@ lxc delete -f v2 v3 v4 # Launch a few containers. echo "==> Container on default VLAN" -lxc init images:ubuntu/20.04/cloud c1 +lxc init ubuntu-daily:22.04 c1 lxc start c1 echo "==> Container on default VLAN with filtering" -lxc init images:ubuntu/20.04/cloud c2 +lxc init ubuntu-daily:22.04 c2 lxc config device override c2 eth0 security.mac_filtering=true lxc start c2 echo "==> Container on alternate VLAN" -lxc init images:ubuntu/20.04/cloud c3 +lxc init ubuntu-daily:22.04 c3 lxc config device override c3 eth0 vlan=4000 lxc start c3 echo "==> Container on alternate VLAN with filtering" -lxc init images:ubuntu/20.04/cloud c4 +lxc init ubuntu-daily:22.04 c4 lxc config device override c4 eth0 vlan=4000 security.mac_filtering=true lxc start c4 @@ -196,4 +140,5 @@ lxc profile device remove default eth0 lxc profile device remove default root lxc storage delete default +# shellcheck disable=SC2034 FAIL=0 diff --git a/tests/pylxd b/tests/pylxd index 6497dd83a..025ecea31 100755 --- a/tests/pylxd +++ b/tests/pylxd @@ -16,4 +16,4 @@ cd pylxd integration/run-integration-tests # shellcheck disable=SC2034 -FAIL=0 \ No newline at end of file +FAIL=0 diff --git a/tests/storage-buckets b/tests/storage-buckets index b5d40d92a..a3dc6a3fb 100755 --- a/tests/storage-buckets +++ b/tests/storage-buckets @@ -1,47 +1,19 @@ #!/bin/sh set -eux -waitSnapdSeed() ( - set +x - for i in $(seq 60); do # Wait up to 60s. - if systemctl show snapd.seeded.service --value --property SubState | grep -qx exited; then - return 0 # Success. - fi - - sleep 1 - done - - echo "snapd not seeded after ${i}s" - return 1 # Failed. -) - -cleanup() { - echo "" - if [ "${FAIL}" = "1" ]; then - echo "Test failed" - exit 1 - fi - - echo "Test passed" - exit 0 -} +# Install LXD +install_lxd poolDriverList="${1:-dir btrfs lvm lvm-thin zfs ceph}" -FAIL=1 -trap cleanup EXIT HUP INT TERM - -# Refresh apt -apt-get update -# Wait for snapd seeding -waitSnapdSeed +if echo "${poolDriverList}" | grep -qwF "ceph"; then + snap set lxd ceph.external=true + lxd waitready --timeout=300 -# Install LXD -snap remove lxd || true -snap install lxd --channel=latest/edge -snap set lxd ceph.external=true -apt-get install --no-install-recommends --yes ceph-common -lxd waitready --timeout=300 + # Install dependencies + apt-get update + apt-get install --no-install-recommends --yes ceph-common +fi # Configure LXD lxc project switch default @@ -82,4 +54,5 @@ done lxc project switch default lxc project delete test +# shellcheck disable=SC2034 FAIL=0 diff --git a/tests/storage-vm b/tests/storage-vm index bfa4340d2..ad1529838 100755 --- a/tests/storage-vm +++ b/tests/storage-vm @@ -1,60 +1,10 @@ #!/bin/sh set -eux -waitSnapdSeed() ( - set +x - for i in $(seq 60); do # Wait up to 60s. - if systemctl show snapd.seeded.service --value --property SubState | grep -qx exited; then - return 0 # Success. - fi - - sleep 1 - done - - echo "snapd not seeded after ${i}s" - return 1 # Failed. -) - -cleanup() { - echo "" - if [ "${FAIL}" = "1" ]; then - echo "Test failed" - exit 1 - fi - - echo "Test passed" - exit 0 -} +# Install LXD +install_lxd poolDriverList="${1:-dir btrfs lvm lvm-thin zfs ceph}" -FAIL=1 -trap cleanup EXIT HUP INT TERM - -# Wait for snapd seeding -waitSnapdSeed - -# Install LXD -snap remove lxd || true -snap install lxd --channel=latest/edge -apt-get install --no-install-recommends --yes jq -lxd waitready --timeout=300 - -waitVMAgent() ( - set +x - # shellcheck disable=SC3043 - local vmName="$1" - for i in $(seq 90) # Wait up to 90s. - do - if lxc info "${vmName}" | grep -qF 127.0.0.1; then - return 0 # Success. - fi - - sleep 1 - done - - echo "VM ${vmName} agent not running after ${i}s" - return 1 # Failed. -) # Configure LXD lxc network create lxdbr0 @@ -436,4 +386,5 @@ echo "==> Delete network" lxc profile device remove default eth0 lxc network delete lxdbr0 +# shellcheck disable=SC2034 FAIL=0 diff --git a/tests/storage-volumes-vm b/tests/storage-volumes-vm index c7d91c281..2ff3ed6da 100755 --- a/tests/storage-volumes-vm +++ b/tests/storage-volumes-vm @@ -1,60 +1,14 @@ #!/bin/sh set -eux -waitSnapdSeed() ( - set +x - for i in $(seq 60); do # Wait up to 60s. - if systemctl show snapd.seeded.service --value --property SubState | grep -qx exited; then - return 0 # Success. - fi - - sleep 1 - done - - echo "snapd not seeded after ${i}s" - return 1 # Failed. -) - -cleanup() { - echo "" - if [ "${FAIL}" = "1" ]; then - echo "Test failed" - exit 1 - fi - - echo "Test passed" - exit 0 -} - -poolDriverList="${1:-dir btrfs lvm lvm-thin zfs ceph}" -FAIL=1 -trap cleanup EXIT HUP INT TERM - -# Wait for snapd seeding -waitSnapdSeed - # Install LXD +install_lxd + +# Install dependencies +apt-get update apt-get install --no-install-recommends --yes genisoimage -snap remove lxd || true -snap install lxd --channel=latest/edge -lxd waitready --timeout=300 - -waitVMAgent() ( - set +x - # shellcheck disable=SC3043 - local vmName="$1" - for i in $(seq 90) # Wait up to 90s. - do - if lxc info "${vmName}" | grep -qF 127.0.0.1; then - return 0 # Success. - fi - - sleep 1 - done - - echo "VM ${vmName} agent not running after ${i}s" - return 1 # Failed. -) + +poolDriverList="${1:-dir btrfs lvm lvm-thin zfs ceph}" # Configure LXD lxc project switch default @@ -81,8 +35,8 @@ do fi echo "==> Create VM" - lxc init images:ubuntu/20.04 v1 --vm -s "${poolName}" - lxc init images:ubuntu/20.04 v2 --vm -s "${poolName}" + lxc init ubuntu-daily:22.04 v1 --vm -s "${poolName}" + lxc init ubuntu-daily:22.04 v2 --vm -s "${poolName}" echo "==> Create custom block volume and attach it to VM" lxc storage volume create "${poolName}" vol1 --type=block size=10MB @@ -181,7 +135,7 @@ do lxc storage volume delete "${poolName}" images --project=default echo "==> Deleting VM" - lxc rm v1 + lxc delete v1 echo "==> Deleting storage pool and volumes" lxc storage volume rm "${poolName}" vol1 @@ -198,4 +152,5 @@ lxc project switch default lxc project delete test lxc network delete lxdbr0 +# shellcheck disable=SC2034 FAIL=0 diff --git a/tests/vm_nesting.sh b/tests/vm-nesting similarity index 76% rename from tests/vm_nesting.sh rename to tests/vm-nesting index 2649ba37b..7fb277333 100755 --- a/tests/vm_nesting.sh +++ b/tests/vm-nesting @@ -1,49 +1,15 @@ #!/bin/bash set -eux -waitSnapdSeed() ( - set +x - for i in $(seq 60); do # Wait up to 60s. - if systemctl show snapd.seeded.service --value --property SubState | grep -qx exited; then - return 0 # Success. - fi - - sleep 1 - done - - echo "snapd not seeded after ${i}s" - return 1 # Failed. -) - -cleanup() { - echo "" - if [ "${FAIL}" = "1" ]; then - echo "Test failed" - exit 1 - fi - - echo "Test passed" - exit 0 -} - -FAIL=1 -trap cleanup EXIT HUP INT TERM - -# Wait for snapd seeding. -waitSnapdSeed - # Install LXD. -snap remove lxd || true -snap install lxd --channel=latest/edge -lxd waitready --timeout=300 +install_lxd # Configure LXD. lxc project switch default lxc storage create default zfs size=30GiB lxc network create lxdbr0 -instanceImage="ubuntu:22.04" -snapChannel="latest/edge" +instanceImage="ubuntu-daily:22.04" function parallel() { seq "$1" | xargs -P "$1" -I "{}" "${@:2}" @@ -114,7 +80,7 @@ delete 10 init 5 --vm start 5 wait 5 -cmd 5 "snap wait system seed.loaded && snap refresh lxd --channel $snapChannel" +cmd 5 "snap wait system seed.loaded && snap refresh lxd --channel ${LXD_SNAP_CHANNEL:-"latest/edge"}" cmd 5 "lxd init --auto" cmd 5 "systemctl reload snap.lxd.daemon" cmd 5 "lxc launch ${instanceImage} nested --vm -c limits.memory=512MiB -d root,size=5GiB" @@ -128,7 +94,7 @@ device_add 5 vhost-net unix-char source=/dev/vhost-net device_add 5 vhost-vsock unix-char source=/dev/vhost-vsock device_add 5 vsock unix-char source=/dev/vsock start 5 -cmd 5 "snap wait system seed.loaded && snap refresh lxd --channel $snapChannel" +cmd 5 "snap wait system seed.loaded && snap refresh lxd --channel ${LXD_SNAP_CHANNEL:-"latest/edge"}" cmd 5 "lxd init --auto" cmd 5 "systemctl reload snap.lxd.daemon" cmd 5 "lxc launch ${instanceImage} nested --vm -c limits.memory=512MiB -d root,size=5GiB" @@ -137,4 +103,5 @@ delete 5 lxc network delete lxdbr0 lxc storage delete default +# shellcheck disable=SC2034 FAIL=0