Merge pull request #49 from simondeziel/waitInstance

Rework helper functions to wait for instance readiness

bin/helpers

@@ -15,30 +15,33 @@ waitSnapdSeed() (
   return 1 # Failed.
 )
 
-# waitVMAgent: waits for the VM agent to be running.
-waitVMAgent() (
+# waitInstanceReady: waits for the instance to be ready (processes count > 1).
+waitInstanceReady() (
   set +x
-  vmName="${1}"
+  maxWait=90
+  instName="${1}"
   instProj="${2:-}"
   if [ -z "${instProj}" ]; then
     # Find the currently selected project.
     instProj="$(lxc project list -f csv | sed -n 's/^\([^(]\+\) (current),.*/\1/ p')"
   fi
 
-  for i in $(seq 90); do
-    if lxc info --project "${instProj}" "${vmName}" | grep -qF 127.0.0.1; then
-      return 0 # Success.
-    fi
-
-    sleep 1
+  # Wait for the instance to report more than one process.
+  processes=0
+  for _ in $(seq "${maxWait}"); do
+      processes="$(lxc info --project "${instProj}" "${instName}" | awk '{if ($1 == "Processes:") print $2}')"
+      if [ "${processes}" -gt 1 ]; then
+          return 0 # Success.
+      fi
+      sleep 1
   done
 
-  echo "VM ${vmName} (${instProj}) agent not running after ${i}s"
+  echo "Instance ${instName} (${instProj}) not ready after ${maxWait}s"
   return 1 # Failed.
 )
 
-# waitInstanceReady: waits for the instance to be ready.
-waitInstanceReady() (
+# waitInstanceBooted: waits for the instance to be ready and fully booted.
+waitInstanceBooted() (
   set +x
   maxWait=90
   instName="$1"
@@ -48,17 +51,17 @@ waitInstanceReady() (
     instProj="$(lxc project list -f csv | sed -n 's/^\([^(]\+\) (current),.*/\1/ p')"
   fi
 
-  # For VMs, wait for the agent to be connected before trying to exec into it
-  if lxc info --project "${instProj}" "${instName}" | grep -qxF 'Type: virtual-machine'; then
-      waitVMAgent "${instName}" "${instProj}"
-  fi
+  # Wait for the instance to be ready
+  waitInstanceReady "${instName}" "${instProj}"
 
+  # Then wait for the boot sequence to complete.
+  sleep 1
   state="$(lxc exec --project "${instProj}" "${instName}" -- timeout "${maxWait}" systemctl is-system-running --wait || true)"
   if [ "${state}" = "running" ]; then
     return 0 # Success.
   fi
 
-  echo "Instance ${instName} (${instProj}) not ready after ${maxWait}s"
+  echo "Instance ${instName} (${instProj}) not booted after ${maxWait}s"
   lxc list --project "${instProj}" "${instName}"
   return 1 # Failed.
 )

tests/cgroup

@@ -1,6 +1,14 @@
 #!/bin/sh
 set -eu
 
+# linux-image-kvm doesn't support setting limits.egress/limits.ingress
+if uname -r | grep -- "-kvm$"; then
+    echo "The -kvm kernel flavor does not have the required extra modules needed by this test."
+    # shellcheck disable=SC2034
+    FAIL=0
+    exit 0
+fi
+
 # Install dependencies
 install_deps jq iperf3
 
@@ -60,6 +68,8 @@ lxc config set c1 limits.cpu=2-2
 [ "$(lxc exec c1 -- nproc)" = "1" ]
 
 lxc config unset c1 limits.cpu
+# XXX: avoid a race (https://github.com/canonical/lxd/issues/12659)
+sleep 2
 [ "$(lxc exec c1 -- nproc)" = "$(nproc)" ]
 
 lxc config set c1 limits.cpu.priority 5

tests/cpu-vm

@@ -21,7 +21,7 @@ lxc storage create "${poolName}" "${poolDriver}"
 
 echo "==> Create ephemeral VM and boot"
 lxc launch ubuntu-daily:22.04 v1 --vm -s "${poolName}" --ephemeral
-waitVMAgent v1
+waitInstanceReady v1
 lxc info v1
 
 # Get number of CPUs

tests/devlxd-vm

@@ -19,7 +19,7 @@ lxc storage create "${poolName}" "${poolDriver}"
 
 echo "==> Create VM and boot"
 lxc launch ubuntu-daily:22.04 v1 --vm -s "${poolName}"
-waitVMAgent v1
+waitInstanceReady v1
 lxc info v1
 
 echo "==> Checking devlxd is working"
@@ -33,7 +33,7 @@ lxc exec v1 -- curl -s --unix-socket /dev/lxd/sock http://custom.socket/1.0/meta
 # Run sync before forcefully restarting the VM otherwise the filesystem will be corrupted.
 lxc exec v1 -- "sync"
 lxc restart -f v1
-waitVMAgent v1
+waitInstanceReady v1
 
 # devlxd should be running after restart
 lxc exec v1 -- curl -s --unix-socket /dev/lxd/sock http://custom.socket/1.0 | jq
@@ -50,7 +50,7 @@ echo "==> Checking devlxd is not working"
 
 lxc exec v1 -- "sync"
 lxc restart -f v1
-waitVMAgent v1
+waitInstanceReady v1
 
 # devlxd should not be running after restart
 ! lxc exec v1 -- curl -s --unix-socket /dev/lxd/sock http://custom.socket/1.0 || false
@@ -81,7 +81,7 @@ sleep 5
 
 # Test nested VM functionality.
 lxc start v1
-waitVMAgent v1
+waitInstanceReady v1
 
 # Configure to use the proxy
 lxc exec v1 -- snap wait system seed.loaded

tests/network

@@ -52,12 +52,12 @@ lxc start c1-sriov
 # Wait for VMs to start.
 echo "==> Waiting for VMs to start"
 
-waitInstanceReady c1-physical
-waitInstanceReady c1-macvlan
-waitInstanceReady c1-sriov
-waitInstanceReady v1-physical
-waitInstanceReady v1-macvlan
-waitInstanceReady v1-sriov
+waitInstanceBooted c1-physical
+waitInstanceBooted c1-macvlan
+waitInstanceBooted c1-sriov
+waitInstanceBooted v1-physical
+waitInstanceBooted v1-macvlan
+waitInstanceBooted v1-sriov
 
 # Check that all instances have an IPv4 and IPv6 address
 networkTests() {
@@ -188,7 +188,7 @@ lxc config device set v1-physical eth0 nictype=bridged parent=lxdbr0 network= mt
 lxc start v1-physical
 
 # Wait for lxd-agent to rename the interface.
-waitInstanceReady v1-physical
+waitInstanceBooted v1-physical
 
 # Interface "eth0" should exist in the VM with the correct MTU.
 lxc exec v1-physical -- test -d /sys/class/net/eth0

tests/network-ovn

@@ -62,10 +62,10 @@ ovn_basic_tests() {
     lxc start 4u
 
     echo "==> Wait for addresses"
-    waitInstanceReady u1
-    waitInstanceReady u2
-    waitInstanceReady u3
-    waitInstanceReady 4u
+    waitInstanceBooted u1
+    waitInstanceBooted u2
+    waitInstanceBooted u3
+    waitInstanceBooted 4u
     lxc list
 
     echo "==> Testing connectivity"
@@ -123,8 +123,8 @@ ovn_basic_tests() {
     lxc start u2
 
     echo "==> Wait for new addresses"
-    waitInstanceReady u2
-    waitInstanceReady u3
+    waitInstanceBooted u2
+    waitInstanceBooted u3
     lxc list
 
     U2_IPV4="$(lxc list u2 -c4 --format=csv | cut -d' ' -f1)"
@@ -218,7 +218,7 @@ ovn_basic_tests() {
     lxc start u1 --project testovn
 
     # Test external IPs allocated and published using dnat.
-    waitInstanceReady u1 testovn
+    waitInstanceBooted u1 testovn
     U1_EXT_IPV4="$(lxc list u1 --project testovn -c4 --format=csv | cut -d' ' -f1)"
     U1_EXT_IPV6="$(lxc list u1 --project testovn -c6 --format=csv | cut -d' ' -f1)"
     ovn-nbctl --bare --format=csv --column=external_ip,logical_ip,type find nat | grep "${U1_EXT_IPV4},${U1_EXT_IPV4},dnat_and_snat"
@@ -413,8 +413,8 @@ ovn_basic_tests() {
     lxc start u3
 
     echo "==> Wait for addresses"
-    waitInstanceReady u2 testovn
-    waitInstanceReady u3 testovn
+    waitInstanceBooted u2 testovn
+    waitInstanceBooted u3 testovn
     lxc list
 
     echo "==> Testing connectivity"
@@ -529,7 +529,7 @@ ovn_basic_tests() {
 
     # Check connectivity back to uplink bridge without SNAT address specified.
     lxc launch "${instanceImage}" u1 -n ovn-virtual-network -s default
-    waitInstanceReady u1
+    waitInstanceBooted u1
     lxc exec u1 -- ping -nc1 -w5 192.0.2.1
     lxc exec u1 -- ping -nc1 -w5 2001:db8:1:1::1
 
@@ -699,7 +699,7 @@ ovn_forward_tests() {
 
     # Create instance connected to ovn-virtual-network.
     lxc launch "${instanceImage}" u1 -n ovn-virtual-network -s default
-    waitInstanceReady u1
+    waitInstanceBooted u1
     lxc list
     U1_IPV4="$(lxc list u1 -c4 --format=csv | cut -d' ' -f1)"
     U1_IPV6="$(lxc list u1 -c6 --format=csv | cut -d' ' -f1)"
@@ -743,7 +743,7 @@ EOF
 
     # Check TCP port forward of port 53 to a different host.
     lxc launch "${instanceImage}" u2 -n ovn-virtual-network -s default
-    waitInstanceReady u2
+    waitInstanceBooted u2
     lxc list
     U2_IPV4="$(lxc list u2 -c4 --format=csv | cut -d' ' -f1)"
     U2_IPV6="$(lxc list u2 -c6 --format=csv | cut -d' ' -f1)"
@@ -901,7 +901,7 @@ ovn_load_balancer_tests() {
 
     # Create instance connected to ovn-virtual-network.
     lxc launch "${instanceImage}" u1 -n ovn-virtual-network -s default
-    waitInstanceReady u1
+    waitInstanceBooted u1
     lxc list
     U1_IPV4="$(lxc list u1 -c4 --format=csv | cut -d' ' -f1)"
     U1_IPV6="$(lxc list u1 -c6 --format=csv | cut -d' ' -f1)"
@@ -963,7 +963,7 @@ EOF
 
     # Check changing DNS TCP forwarding towards u2.
     lxc launch "${instanceImage}" u2 -n ovn-virtual-network -s default
-    waitInstanceReady u2
+    waitInstanceBooted u2
     lxc list
     U2_IPV4="$(lxc list u2 -c4 --format=csv | cut -d' ' -f1)"
     U2_IPV6="$(lxc list u2 -c6 --format=csv | cut -d' ' -f1)"
@@ -1099,8 +1099,8 @@ ovn_peering_tests() {
         ipv6.routes.external=2001:db8:1:2::2/128
     lxc start ovn1 --project=ovn1
     lxc launch "${instanceImage}" ovn2 -n ovn2 -s default --project=ovn2
-    waitInstanceReady ovn1 ovn1
-    waitInstanceReady ovn2 ovn2
+    waitInstanceBooted ovn1 ovn1
+    waitInstanceBooted ovn2 ovn2
 
     # Add IPs from routes to ovn1 instance.
     lxc exec ovn1 --project=ovn1 -- ip a add 198.51.100.1/32 dev eth0
@@ -1168,7 +1168,7 @@ ovn_peering_tests() {
     ovn-nbctl list address_set | grep -F 2001:db8:1:2::2/128
 
     # Check security policies prevent spoofed packets using peer connection.
-    waitInstanceReady ovn1 ovn1
+    waitInstanceBooted ovn1 ovn1
     ovn1NICIPv4="$(lxc list ovn1 -c4 --format=csv --project=ovn1 | cut -d' ' -f1)"
     ovn1NICIPv6="$(lxc list ovn1 -c6 --format=csv --project=ovn1 | cut -d' ' -f1)"
     ovn2NICIPv4="$(lxc list ovn2 -c4 --format=csv --project=ovn2 | cut -d' ' -f1)"
@@ -1235,7 +1235,7 @@ ovn_peering_tests() {
     lxc delete -f ovn1 --project=ovn1 # Remove as we cleared its DHCP config.
     lxc network set ovn1 ipv4.address=192.0.2.1/24 ipv6.address=2001:db8:1:1::1/64 --project=ovn1
     lxc launch "${instanceImage}" ovn1 -n ovn1 -s default --project=ovn1
-    waitInstanceReady ovn1 ovn1
+    waitInstanceBooted ovn1 ovn1
     ovn1NICIPv4="$(lxc list ovn1 -c4 --format=csv --project=ovn1 | cut -d' ' -f1)"
     ovn1NICIPv6="$(lxc list ovn1 -c6 --format=csv --project=ovn1 | cut -d' ' -f1)"
     lxc exec ovn2 -T -n --project=ovn2 -- ping -nc1 -4 -w5 "${ovn1NICIPv4}"
@@ -1374,7 +1374,7 @@ ovn_dhcp_reservation_tests() {
 
     # Launch new dynamic IP instance and check its not allocated the reserved IP.
     lxc launch "${instanceImage}" u2 -s default -n ovn1
-    waitInstanceReady u2
+    waitInstanceBooted u2
     lxc list
     U2_IPV4="$(lxc list u2 -c4 --format=csv | cut -d' ' -f1)"
     [ "${U2_IPV4}" != "10.10.11.2" ] || false
@@ -1476,8 +1476,8 @@ ovn_nested_vlan_tests() {
     lxc exec u1 -- ip netns exec vlan_nesting2 ip address add fd42:4242:4242:1011::11/64 dev eth0.2 nodad
     lxc exec u1 -- ip netns exec vlan_nesting2 ip link set eth0.2 up
 
-    waitInstanceReady u1
-    waitInstanceReady u2
+    waitInstanceBooted u1
+    waitInstanceBooted u2
     lxc list
     lxc exec u1 -- ip netns exec vlan_nesting1 ip address
     lxc exec u1 -- ip netns exec vlan_nesting2 ip address
@@ -1557,8 +1557,8 @@ ovn_acl_tests() {
     lxc launch "${instanceImage}" c2 -n ovn0 -s default
 
     echo "==> Wait for addresses"
-    waitInstanceReady c1
-    waitInstanceReady c2
+    waitInstanceBooted c1
+    waitInstanceBooted c2
     lxc list
 
     # Check per-NIC ACL rules added.
@@ -1709,7 +1709,7 @@ ovn_acl_tests() {
     lxc start c1
 
     echo "==> Wait for addresses"
-    waitInstanceReady c1
+    waitInstanceBooted c1
     lxc list
 
     # Ping to c1 instance from c2 should be blocked.
@@ -1849,8 +1849,8 @@ ovn_acl_tests() {
     lxc start c1
 
     echo "==> Wait for addresses"
-    waitInstanceReady c1
-    waitInstanceReady c2
+    waitInstanceBooted c1
+    waitInstanceBooted c2
     lxc list
 
     # Check c2 can ping external.
@@ -1907,8 +1907,8 @@ ovn_l3only_tests() {
     lxc launch "${instanceImage}" u1 -s default -n ovn1
     lxc launch "${instanceImage}" u2 -s default -n ovn1
 
-    waitInstanceReady u1
-    waitInstanceReady u2
+    waitInstanceBooted u1
+    waitInstanceBooted u2
     lxc list
 
     U1_IPV4="$(lxc list u1 -c4 --format=csv | cut -d' ' -f1)"

tests/network-routed

@@ -52,7 +52,7 @@ EOF
 lxc start v1
 
 # Wait for VM to start.
-waitVMAgent v1
+waitInstanceReady v1
 sleep 10
 
 # Test ping to/from VM NIC.

tests/network-sriov

@@ -95,10 +95,10 @@ lxc config device override v4 eth0 vlan=4000 security.mac_filtering=true
 lxc start v4
 
 # Wait for VMs to start.
-waitVMAgent v1
-waitVMAgent v2
-waitVMAgent v3
-waitVMAgent v4
+waitInstanceReady v1
+waitInstanceReady v2
+waitInstanceReady v3
+waitInstanceReady v4
 
 lxc list
 networkTests