Changes to run tests on UC22

* Wait for snap changes
* Wait for docker daemon after reboot
* Run docker commands with sudo
* Docker run in non-interactive mode

.github/workflows/testflinger/nvidia-job.yaml

@@ -22,15 +22,21 @@ test_data:
 
     SCRIPTS=./attachments/test/scripts
 
-    echo "Testing: DEVICE_IP = $DEVICE_IP"
-    # Setup the environment on the target device
+    # On Ubuntu Core, kernel, core, snapd snaps get refreshed right after first boot,
+    # causing unexpected errors and triggering a reboot
+    while ! ssh ubuntu@$DEVICE_IP "$(< $SCRIPTS/check-snap-changes.sh)"; do
+      echo "Wait for ssh server and/or snap changes..."
+      sleep 30
+    done
+
     ssh ubuntu@$DEVICE_IP "$(< $SCRIPTS/setup.sh)"
 
     # Reboot the device in background to avoid breaking the SSH connection prematurely
     ssh ubuntu@$DEVICE_IP "(sleep 3 && sudo reboot) &"
 
-    echo "Wait for the device to boot and start its SSH server"
-    $SCRIPTS/wait_for_port.sh $DEVICE_IP 22
-
-    # Run the tests
+    while ! ssh ubuntu@$DEVICE_IP "sudo docker version"; do
+      echo "Wait for ssh server and/or Docker daemon..."
+      sleep 30
+    done
+    
     ssh ubuntu@$DEVICE_IP "$(< $SCRIPTS/test.sh)"

.github/workflows/testflinger/scripts/check-snap-changes.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+# This script is adapted from
+# https://github.com/canonical/hwcert-jenkins-tools/blob/c5cf512d968100db90998abe61c474de0be681ca/scriptlets/check_for_snap_changes
+
+echo "Get snap changes"
+
+# list the snap changes on the device and store the output in a temp file
+OUTPUT=$(mktemp)
+snap changes > $OUTPUT
+
+RESULT=$?
+if [ ! "$RESULT" -eq 0 ]; then exit $RESULT; fi
+
+# tail -n +2: remove the header
+# awk 'NF {print $2}': print the second column on non-empty lines (i.e. the status)
+# grep -q -E "...": succeed when changes are still ongoing or pending
+cat $OUTPUT | \
+tail -n +2 | \
+awk 'NF {print $2}' | \
+grep -q -E "\b(Doing|Undoing|Wait|Do|Undo)\b"
+
+if [ "$?" -eq 0 ]; then
+    # changes are still ongoing or pending: display output as a diagnostic
+    cat "$OUTPUT" | grep -E "\b(Doing|Undoing|Wait|Do|Undo)\b"
+    rm "$OUTPUT"
+
+    exit 1
+fi
+
+echo "No ongoing or pending snap changes"
+rm "$OUTPUT"

.github/workflows/testflinger/scripts/setup.sh

@@ -19,7 +19,7 @@ install_docker() {
   sudo snap install docker --channel="$DOCKER_SNAP_CHANNEL"
 
   # check the installation
-  docker --version || exit 1
+  sudo docker --version || exit 1
 }
 
 setup_classic() {

.github/workflows/testflinger/scripts/test.sh

@@ -8,7 +8,7 @@ smi_test() {
   if [[ $ID == "ubuntu" ]]; then
     sudo docker run --rm --runtime=nvidia --gpus all --env PATH="${PATH}:/var/lib/snapd/hostfs/usr/bin" ubuntu nvidia-smi || exit 1
   elif [[ $ID == "ubuntu-core" ]]; then
-    sudo docker run --rm --runtime nvidia --gpus all -it ubuntu bash -c "/snap/docker/*/graphics/bin/nvidia-smi" || exit 1
+    sudo docker run --rm --runtime nvidia --gpus all ubuntu bash -c "/snap/docker/*/graphics/bin/nvidia-smi" || exit 1
   else
     echo "Unexpected operating system ID: $ID"
     exit 1