CI tweaks (#362)

.github/workflows/tests.yml

@@ -306,8 +306,8 @@ jobs:
       - name: Tmate debugging session (self-hosted)
         if: ${{ failure() && inputs.tmate-debug && inputs.self-hosted-runner }}
         uses: canonical/action-tmate@main
-        timeout-minutes: ${{ inputs.tmate-timeout }}
+        timeout-minutes: ${{ fromJSON(inputs.tmate-timeout) }}
       - name: Tmate debugging session (gh-hosted)
         if: ${{ failure() && inputs.tmate-debug && !inputs.self-hosted-runner }}
         uses: mxschmitt/action-tmate@v3
-        timeout-minutes: ${{ inputs.tmate-timeout }}
+        timeout-minutes: ${{ fromJSON(inputs.tmate-timeout) }}

bin/helpers

@@ -3,8 +3,8 @@
 # waitSnapdSeed: wait for snapd to be seeded.
 # Optional argument: timeout in seconds, defaults to 60.
 waitSnapdSeed() (
+  { set +x; } 2>/dev/null
   waitSecs="${1:-60}"
-  set +x
   if timeout "${waitSecs}" snap wait system seed.loaded; then
     return 0 # Success.
   fi
@@ -14,7 +14,7 @@ waitSnapdSeed() (
 
 # waitInstanceReady: waits for the instance to be ready (processes count > 1).
 waitInstanceReady() (
-  set +x
+  { set +x; } 2>/dev/null
   maxWait="${MAX_WAIT_SECONDS:-120}"
   instName="${1}"
   instProj="${2:-}"
@@ -39,7 +39,7 @@ waitInstanceReady() (
 
 # waitInstanceBooted: waits for the instance to be ready and fully booted.
 waitInstanceBooted() (
-  set +x
+  { set +x; } 2>/dev/null
   prefix="${WARNING_PREFIX:-::warning::}"
   maxWait=90
   instName="$1"
@@ -74,7 +74,7 @@ waitInstanceBooted() (
 # isSystemdClean: wait for instance to be booted and check for systemd failures.
 # ==> Returns 1 if systemd state is clean (no failure), 1 otherwise.
 isSystemdClean() (
-  set +x
+  { set +x; } 2>/dev/null
   instName="$1"
   instProj="${2:-}"
   if [ -z "${instProj}" ]; then
@@ -97,7 +97,7 @@ isSystemdClean() (
 
 # enableSRIOV: enable SR-IOV on a NIC.
 enableNICSRIOV() (
-  set +x
+  { set +x; } 2>/dev/null
   parentNIC="${1}"
   numVFS="${2:-"7"}"
 
@@ -119,7 +119,7 @@ enableNICSRIOV() (
 
 # install_deps: install dependencies if needed.
 install_deps() (
-    set +x
+    { set +x; } 2>/dev/null
     PKGS="${*}"
     missing=""
     PKGS_LIST="$(dpkg --get-selections | awk '{print $1}')"
@@ -191,6 +191,9 @@ install_lxd() (
         mount --bind "${LXD_AGENT_SIDELOAD_PATH}" /snap/lxd/current/bin/lxd-agent
     fi
 
+    # Enable debug logging if not done already
+    [ "$(snap get lxd daemon.debug 2>/dev/null)" = "true" ] || snap set lxd daemon.debug=true
+
     if [ "$start_daemon" = "true" ]; then
         lxd waitready --timeout=300
     fi

tests/cluster

@@ -148,10 +148,12 @@ fi
 
 echo "==> Upgrading the cluster"
 for i in $(seq "${SIZE}"); do
+    lxc exec "${PREFIX}-$i" -- snap wait system seed.loaded
     lxc exec "${PREFIX}-$i" -- snap refresh
+    # XXX: there should be no refresh ongoing but we've seen races before so let's collect evidences
+    lxc exec "${PREFIX}-$i" -- snap changes
     lxc exec "${PREFIX}-$i" -- snap switch lxd --channel="$3"
     if [ "$i" = "${SIZE}" ]; then
-        lxc exec "${PREFIX}-$i" -- snap wait system seed.loaded
         lxc exec "${PREFIX}-$i" -- timeout 10m snap refresh lxd
     fi
 done