Test against both MicroOVN and OVN debs (#368)

Supersedes #330

.github/workflows/tests.yml

@@ -97,7 +97,8 @@ jobs:
           - lxd-user
           - interception
           - network-bridge-firewall
-          - network-ovn
+          - network-ovn ovn:deb
+          - network-ovn ovn:latest/edge
           - network-routed
           - pylxd
           - snapd
@@ -135,7 +136,9 @@ jobs:
           - test: network-bridge-firewall
             track: "4.0/candidate"
             os: 20.04
-          - test: network-ovn
+          - test: network-ovn ovn:deb
+            track: "4.0/candidate"
+          - test: network-ovn ovn:latest/edge
             track: "4.0/candidate"
           - test: storage-buckets
             track: "4.0/candidate"
@@ -174,7 +177,9 @@ jobs:
           - test: network-bridge-firewall
             track: "4.0/edge"
             os: 20.04
-          - test: network-ovn
+          - test: network-ovn ovn:deb
+            track: "4.0/edge"
+          - test: network-ovn ovn:latest/edge
             track: "4.0/edge"
           - test: storage-buckets
             track: "4.0/edge"
@@ -203,6 +208,10 @@ jobs:
             track: "5.0/candidate"
           - test: efi-vars-editor-vm
             track: "5.0/edge"
+          - test: network-ovn ovn:latest/edge
+            track: "5.0/candidate"
+          - test: network-ovn ovn:latest/edge
+            track: "5.0/edge"
           - test: vm-migration
             track: "5.0/candidate"
           - test: vm-migration
@@ -290,8 +299,14 @@ jobs:
             dst_track="${{ matrix.track }}"
             src_track="$(echo "${dst_track}" | cut -d/ -f1)/stable"
             EXTRA_ARGS="${EXTRA_ARGS:-3} ${src_track} ${{ matrix.track }}"
+          elif [ "${TEST_SCRIPT}" = "network-ovn" ]; then
+            if [ -n "${EXTRA_ARGS}" ]; then
+              # Strip the `ovn:` prefix
+              export OVN_SOURCE="${EXTRA_ARGS##ovn:}"
+              EXTRA_ARGS=""
+            fi
           fi
-          sudo --preserve-env=GITHUB_ACTIONS,GITHUB_STEP_SUMMARY,TEST_IMG ./bin/local-run "tests/${TEST_SCRIPT}" ${{ matrix.track }} ${EXTRA_ARGS:-}
+          sudo --preserve-env=GITHUB_ACTIONS,GITHUB_STEP_SUMMARY,TEST_IMG,OVN_SOURCE ./bin/local-run "tests/${TEST_SCRIPT}" ${{ matrix.track }} ${EXTRA_ARGS:-}
 
       # always update cache as we have our own logic of
       # cache invalidation and updates in addition to a date check

README.md

@@ -123,6 +123,19 @@ To test a custom snap of LXD, you can set the `LXD_SNAP_PATH` environment variab
 LXD_SNAP_PATH=/tmp/lxd_0+git.89550582_amd64.snap ./bin/local-run tests/interception latest/edge
 ```
 
+To run `tests/network-ovn` against various OVN implementation:
+
+```
+# Using the deb package from the base Os
+OVN_SOURCE=deb PURGE_LXD=1 ./bin/local-run tests/network-ovn latest/edge
+
+# Use numbered releases of MicroOVN
+OVN_SOURCE=22.03/edge PURGE_LXD=1 ./bin/local-run tests/network-ovn latest/edge
+OVN_SOURCE=24.03/edge PURGE_LXD=1 ./bin/local-run tests/network-ovn latest/edge
+
+# Using the `latest/edge` MicroOVN snap channel
+PURGE_LXD=1 ./bin/local-run tests/network-ovn latest/edge
+```
 
 # Running tests on OpenStack (ProdStack)
 

bin/helpers

@@ -141,6 +141,39 @@ install_deps() (
     fi
 )
 
+# install_ovn: install OVN packages or MicroOVN snap.
+install_ovn() (
+    if [ "${OVN_SOURCE:-latest/edge}" = "deb" ]; then
+        # Avoid clashing with the microovn snap
+        if snap list microovn 2>/dev/null; then
+            snap remove --purge microovn
+        fi
+
+        install_deps ovn-host ovn-central
+    elif ! snap list microovn 2>/dev/null; then
+        # Avoid clashing with the deb packages
+        apt-get autopurge --yes ovn-host ovn-central || true
+
+        snap install microovn --channel="${OVN_SOURCE:-latest/edge}" --cohort=+
+    fi
+)
+
+# configure_ovn: prepare OVN for use by LXD.
+configure_ovn() {
+    if [ "${OVN_SOURCE:-latest/edge}" = "deb" ]; then
+        ovs-vsctl set open_vswitch . \
+          external_ids:ovn-remote=unix:/var/run/ovn/ovnsb_db.sock \
+          external_ids:ovn-encap-type=geneve \
+          external_ids:ovn-encap-ip=127.0.0.1
+
+        # Empty controller log so ACL log checks are consistent over repeat runs.
+        echo "" > /var/log/ovn/ovn-controller.log
+    else
+        microovn status || microovn cluster bootstrap
+        lxc config set network.ovn.northbound_connection "ssl:127.0.0.1:6641"
+    fi
+}
+
 # install_lxd: install LXD from a specific channel or `latest/edge` if none is provided.
 # Optional argument: boolean which indicates whether to start the daemon. Default is true.
 install_lxd() (

bin/local-run

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 set -ux
 
 script="${1}"

tests/network-ovn

@@ -2,19 +2,16 @@
 set -eux
 
 # Install dependencies
-install_deps ovn-host ovn-central bind9-dnsutils jq
+install_deps bind9-dnsutils iputils-ping jq
+
+# Install OVN
+install_ovn
 
 # Install LXD
 install_lxd
 
 # Configure OVN.
-ovs-vsctl set open_vswitch . \
-  external_ids:ovn-remote=unix:/var/run/ovn/ovnsb_db.sock \
-  external_ids:ovn-encap-type=geneve \
-  external_ids:ovn-encap-ip=127.0.0.1
-
-# Empty controller log so ACL log checks are consistent over repeat runs.
-echo "" > /var/log/ovn/ovn-controller.log
+configure_ovn
 
 # Configure LXD.
 lxc project switch default
@@ -766,8 +763,8 @@ ovn_forward_tests() {
 
     echo "==> Check forward is working by probing TCP/22"
     # Relies on static route (above) rather than neighbour adverts see https://github.com/ovn-org/ovn/issues/124
-    nc -zv 192.0.2.1 22
-    nc -zv 2001:db8:1:2::1 22
+    timeout 5 grep -m1 ^SSH < /dev/tcp/192.0.2.1/22
+    timeout 5 grep -m1 ^SSH < /dev/tcp/2001:db8:1:2::1/22
 
     echo "==> Check forward is working by pinging it"
     # XXX: ping used to work but doesn't anymore due to a regression in OVN in Jammy (22.03.3-0ubuntu0.22.04.1?)
@@ -778,7 +775,7 @@ ovn_forward_tests() {
     lxc exec u1 -- systemctl mask dnsmasq
     lxc exec u1 -- apt-get update
     lxc exec u1 -- apt-get install --no-install-recommends --yes dnsmasq
-cat <<EOF | lxc exec u1 -- tee /etc/dnsmasq.d/lxd_test.conf
+    lxc exec u1 -- tee /etc/dnsmasq.d/lxd_test.conf << EOF
 interface=eth0
 bind-interfaces
 interface-name=u1.lxd,eth0
@@ -804,7 +801,7 @@ EOF
     lxc exec u2 -- systemctl mask dnsmasq
     lxc exec u2 -- apt-get update
     lxc exec u2 -- apt-get install --no-install-recommends --yes dnsmasq
-cat <<EOF | lxc exec u2 -- tee /etc/dnsmasq.d/lxd_test.conf
+    lxc exec u2 -- tee /etc/dnsmasq.d/lxd_test.conf << EOF
 interface=eth0
 bind-interfaces
 interface-name=u2.lxd,eth0
@@ -1042,7 +1039,7 @@ ovn_load_balancer_tests() {
     lxc exec u1 -- systemctl mask dnsmasq
     lxc exec u1 -- apt-get update
     lxc exec u1 -- apt-get install --no-install-recommends --yes dnsmasq
-cat <<EOF | lxc exec u1 -- tee /etc/dnsmasq.d/lxd_test.conf
+    lxc exec u1 -- tee /etc/dnsmasq.d/lxd_test.conf << EOF
 interface=eth0
 bind-interfaces
 interface-name=u1.lxd,eth0
@@ -1081,7 +1078,7 @@ EOF
     lxc exec u2 -- systemctl mask dnsmasq
     lxc exec u2 -- apt-get update
     lxc exec u2 -- apt-get install --no-install-recommends --yes dnsmasq
-cat <<EOF | lxc exec u2 -- tee /etc/dnsmasq.d/lxd_test.conf
+    lxc exec u2 -- tee /etc/dnsmasq.d/lxd_test.conf << EOF
 interface=eth0
 bind-interfaces
 interface-name=u2.lxd,eth0
@@ -1411,43 +1408,45 @@ ovn_peering_tests() {
     lxc exec ovn2 -T -n --project=ovn2 -- ping -nc1 -6 -w5 "${ovn1NICIPv6}"
 
     echo "==> Check that acl rule for ovn ingress has all the expected values"
-    ovn_logs="$(lxc network acl show-log ovn1 --project=ovn1)"
-
-    # XXX: unable to get logs from old OVN .deb from 20.04
-    if [ -z "${ovn_logs}" ]; then
-        . /etc/os-release
-        if [ "${VERSION_ID}" = "20.04" ]; then
-            echo "Ignoring network acl show-log not working on ${VERSION_ID}"
-        else
-            false
+    if [ "${OVN_SOURCE:-latest/edge}" = "deb" ]; then
+      ovn_logs="$(lxc network acl show-log ovn1 --project=ovn1)"
+
+      # XXX: unable to get logs from old OVN .deb from 20.04
+      if [ -z "${ovn_logs}" ]; then
+          . /etc/os-release
+          if [ "${VERSION_ID}" = "20.04" ]; then
+              echo "Ignoring network acl show-log not working on ${VERSION_ID}"
+          else
+              false
+          fi
+      fi
+
+      echo "${ovn_logs}" | while IFS= read -r logline; do
+        proto=$(echo "$logline" | jq -r '.proto')
+        src=$(echo "$logline" | jq -r '.src')
+        dst=$(echo "$logline" | jq -r '.dst')
+        icmp_type=$(echo "$logline" | jq -r '.icmp_type')
+        icmp_code=$(echo "$logline" | jq -r '.icmp_code')
+        action=$(echo "$logline" | jq -r '.action')
+
+        [ "$action" = "allow" ]
+
+        if [ "$proto" = "icmp" ]; then
+          echo "==> IPv4 ping from ovn2 to ovn1 allowed"
+          [ "$src" = "${ovn2NICIPv4}" ]
+          [ "$dst" = "${ovn1NICIPv4}" ]
+          [ "$icmp_type" = "8" ]
+          [ "$icmp_code" = "0" ]
+        elif [ "$proto" = "icmp6" ]; then
+          echo "==> IPv6 ping from ovn2 to ovn1 allowed"
+          [ "$src" = "${ovn2NICIPv6}" ]
+          [ "$dst" = "${ovn1NICIPv6}" ]
+          [ "$icmp_type" = "128" ]
+          [ "$icmp_code" = "0" ]
         fi
+      done
     fi
 
-    echo "${ovn_logs}" | while IFS= read -r logline; do
-      proto=$(echo "$logline" | jq -r '.proto')
-      src=$(echo "$logline" | jq -r '.src')
-      dst=$(echo "$logline" | jq -r '.dst')
-      icmp_type=$(echo "$logline" | jq -r '.icmp_type')
-      icmp_code=$(echo "$logline" | jq -r '.icmp_code')
-      action=$(echo "$logline" | jq -r '.action')
-
-      [ "$action" = "allow" ]
-
-      if [ "$proto" = "icmp" ]; then
-        echo "==> IPv4 ping from ovn2 to ovn1 allowed"
-        [ "$src" = "${ovn2NICIPv4}" ]
-        [ "$dst" = "${ovn1NICIPv4}" ]
-        [ "$icmp_type" = "8" ]
-        [ "$icmp_code" = "0" ]
-      elif [ "$proto" = "icmp6" ]; then
-        echo "==> IPv6 ping from ovn2 to ovn1 allowed"
-        [ "$src" = "${ovn2NICIPv6}" ]
-        [ "$dst" = "${ovn1NICIPv6}" ]
-        [ "$icmp_type" = "128" ]
-        [ "$icmp_code" = "0" ]
-      fi
-    done
-
     echo "==> Check cannot add an ACL to a network NIC that references a peer connection from another network"
     lxc network create ovn1b --type=ovn network=lxdbr0 --project=ovn1
     ! lxc network set ovn1b security.acls=ovn1 --project=ovn1 || false

tests/vm-migration

@@ -130,7 +130,9 @@ lxc exec member1 -- lxc config device add v1 vol1-disk disk pool=ceph source=vol
 # Start the VM.
 lxc exec member1 -- lxc start v1
 sleep 60
-lxc exec member1 -- lxc move v1 --target member2
+
+# Move the VM to member2 but with a timeout as this sometimes hangs indefinitely but when it works it's fast.
+timeout 600 lxc exec member1 -- lxc move v1 --target member2
 
 # shellcheck disable=SC2034
 FAIL=0