Storage operation integrity checks (#245)

This is just the first cut at storage operation integrity checks. I intend to implement your `lxc file mount` suggestion for containers in a future PR but if you prefer it to be bundled here, that's OK too.
canonical · Jul 19, 2024 · ef67875 · ef67875
2 parents 42da814 + f7e42c0
commit ef67875
Show file tree

Hide file tree

Showing 3 changed files with 64 additions and 10 deletions.
diff --git a/tests/container-copy b/tests/container-copy
@@ -33,7 +33,7 @@ lxc remote add target "${token}" --accept-certificate
 # Run the test for various filesystems.
 for fs in "xfs" "btrfs" "ext4" ; do
   echo "==> Testing with ${fs}"
-  mkfs.${fs} "${loopdev}"
+  mkfs."${fs}" "${loopdev}"
   disk_dir="$(mktemp -d "${test_dir}/XXX.disk")"
   mount "${loopdev}" "${disk_dir}"
   rm -rf "${disk_dir}/lost+found"
@@ -47,8 +47,22 @@ for fs in "xfs" "btrfs" "ext4" ; do
   # Copy the container to the remote LXD.
   lxc copy c1 target:c1 --stateless --refresh --mode=relay
 
-  # Ensure the container exists.
-  lxc config show target:c1 > /dev/null
+  # Test refreshing the copy
+  lxc start target:c1
+  printf "%s" "${fs}" | lxc file push - c1/root/my-fs
+  ! lxc exec target:c1 -- test -e /root/my-fs || false
+  lxc stop -f target:c1
+  lxc copy c1 target:c1 --stateless --refresh --mode=relay
+  lxc start target:c1
+  [ "$(lxc exec target:c1 -- cat /root/my-fs)" = "${fs}" ]
+  lxc stop -f target:c1
+
+  # Test appending to a file before refreshing the copy
+  echo "-${$}" | lxc exec c1 -- tee -a /root/my-fs
+  lxc copy c1 target:c1 --stateless --refresh --mode=relay
+  lxc start target:c1
+  [ "$(lxc exec target:c1 -- cat /root/my-fs)" = "${fs}-${$}" ]
+  lxc stop -f target:c1
 
   # Clean up the storage pool for this run.
   lxc delete c1 -f

diff --git a/tests/storage-vm b/tests/storage-vm
@@ -99,7 +99,7 @@ for poolDriver in $poolDriverList; do
         echo "==> Checking restore VM snapshot"
         lxc restore v1 snap0
         waitInstanceReady v1
-        lxc exec v1 -- grep -Fx "foo" /root/foo.txt
+        [ "$(lxc exec v1 -- cat /root/foo.txt)" = "foo" ]
 
         echo "==> Checking VM filesystem volume was restored too"
         nsenter --mount=/run/snapd/ns/lxd.mnt [ -f "/var/snap/lxd/common/lxd/storage-pools/${poolName}/virtual-machines/v1/foo" ]
@@ -122,7 +122,7 @@ for poolDriver in $poolDriverList; do
         lxc copy v1/snap0 v2
         lxc start v2
         waitInstanceReady v2
-        lxc exec v2 -- grep -Fx "foo" /root/foo.txt
+        [ "$(lxc exec v2 -- cat /root/foo.txt)" = "foo" ]
 
         if [ "${poolDriver}" != "powerflex" ]; then
                 echo "==> Checking VM snapshot copy root disk size is 3584MiB"
@@ -156,7 +156,7 @@ for poolDriver in $poolDriverList; do
         lxc copy v1/snap0 localhost:v2
         lxc start v2
         waitInstanceReady v2
-        lxc exec v2 -- grep -Fx "foo" /root/foo.txt
+        [ "$(lxc exec v2 -- cat /root/foo.txt)" = "foo" ]
 
         if [ "${poolDriver}" != "powerflex" ]; then
                 echo "==> Checking VM snapshot copy root disk size is 3584MiB"
@@ -185,7 +185,7 @@ for poolDriver in $poolDriverList; do
 
         echo "==> Checking VM can be refreshed remotely (different storage pool)"
         lxc snapshot v1
-        lxc copy v1 localhost:v2 --refresh
+        lxc copy v1 localhost:v2 --refresh -s "${poolName}2"
         [ "$(lxc query /1.0/instances/v2?recursion=1 | jq '.snapshots | length')" -eq "2" ]
         lxc rm v1/snap1
         lxc delete -f v2
@@ -194,7 +194,7 @@ for poolDriver in $poolDriverList; do
         lxc copy v1/snap0 localhost:v2 -s "${poolName}2"
         lxc start v2
         waitInstanceReady v2
-        lxc exec v2 -- grep -Fx "foo" /root/foo.txt
+        [ "$(lxc exec v2 -- cat /root/foo.txt)" = "foo" ]
 
         if [ "${poolDriver}" != "powerflex" ]; then
                 echo "==> Checking VM snapshot copy root disk size is 3584MiB"
@@ -317,8 +317,7 @@ for poolDriver in $poolDriverList; do
                 lxc info v1
 
                 echo "==> Ensure the VM volume ignores volume.block.filesystem"
-                volume_fs="$(lxc storage volume get "${poolName}" virtual-machine/v1 block.filesystem)"
-                echo "${volume_fs}" | grep -F "ext4"
+                [ "$(lxc storage volume get "${poolName}" virtual-machine/v1 block.filesystem)" = "ext4" ]
 
                 echo "==> Checking VM config disk filesystem is not XFS"
                 serverPID="$(lxc query /1.0 | jq .environment.server_pid)"
@@ -349,6 +348,10 @@ for poolDriver in $poolDriverList; do
                 echo "==> Checking VM root disk size is 8GiB"
                 [ "$(($(lxc exec v1 -- blockdev --getsize64 /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_lxd_root) / GiB))" -eq "8" ]
         fi
+        V1_ROOT_GPT_DISK_ID="$(lxc exec v1 -- fdisk --list /dev/sda | awk '/^Disk identifier:/ {print $3}')"
+        V1_EFI_PART="$(lxc exec v1 -- fdisk --list /dev/sda | awk '/EFI System$/ {print $1}')"
+        lxc exec v1 -- umount "${V1_EFI_PART}"
+        V1_EFI_SHA256SUM="$(lxc exec v1 -- sha256sum "${V1_EFI_PART}")"
         lxc stop -f v1
 
         echo "==> Copy to different storage pool with same driver and check size"
@@ -369,6 +372,21 @@ for poolDriver in $poolDriverList; do
         waitInstanceReady v2
         lxc info v2
 
+        echo "==> Check the EFI partition number and content did not change after the volume size override"
+        V2_EFI_PART="$(lxc exec v2 -- fdisk --list /dev/sda | awk '/EFI System$/ {print $1}')"
+        lxc exec v2 -- umount "${V2_EFI_PART}"
+        V2_EFI_SHA256SUM="$(lxc exec v2 -- sha256sum "${V2_EFI_PART}")"
+        [ "${V1_EFI_SHA256SUM}" = "${V2_EFI_SHA256SUM}" ]
+
+        echo "==> Check the GPT disk ID of the root disk did not change after the copy"
+        V2_ROOT_GPT_DISK_ID="$(lxc exec v2 -- fdisk --list /dev/sda | awk '/^Disk identifier:/ {print $3}')"
+        [ "${V1_ROOT_GPT_DISK_ID}" = "${V2_ROOT_GPT_DISK_ID}" ]
+
+        echo "==> Check the GPT backup table was copied at the new end of the disk"
+        # `fdisk --list` would print `The backup GPT table is corrupt, but the primary appears OK, so that will be used.` to stderr
+        lxc exec v2 -- fdisk --list /dev/sda | grep -xF 'Disklabel type: gpt'
+        [ "$(lxc exec v2 -- fdisk --list /dev/sda 2>&1 >/dev/null)" = "" ]
+
         if [ "${poolDriver}" != "powerflex" ]; then
                 echo "==> Checking copied VM root disk size is 3584MiB"
                 [ "$(($(lxc exec v2 -- blockdev --getsize64 /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_lxd_root) / MiB))" -eq "3584" ]
@@ -412,6 +430,21 @@ for poolDriver in $poolDriverList; do
         waitInstanceReady v2
         lxc info v2
 
+        echo "==> Check the EFI partition number and content did not change after the volume size override"
+        V2_EFI_PART="$(lxc exec v2 -- fdisk --list /dev/sda | awk '/EFI System$/ {print $1}')"
+        lxc exec v2 -- umount "${V2_EFI_PART}"
+        V2_EFI_SHA256SUM="$(lxc exec v2 -- sha256sum "${V2_EFI_PART}")"
+        [ "${V1_EFI_SHA256SUM}" = "${V2_EFI_SHA256SUM}" ]
+
+        echo "==> Check the GPT disk ID of the root disk did not change after the copy"
+        V2_ROOT_GPT_DISK_ID="$(lxc exec v2 -- fdisk --list /dev/sda | awk '/^Disk identifier:/ {print $3}')"
+        [ "${V1_ROOT_GPT_DISK_ID}" = "${V2_ROOT_GPT_DISK_ID}" ]
+
+        echo "==> Check the GPT backup table was copied at the new end of the disk"
+        # `fdisk --list` would print `The backup GPT table is corrupt, but the primary appears OK, so that will be used.` to stderr
+        lxc exec v2 -- fdisk --list /dev/sda | grep -xF 'Disklabel type: gpt'
+        [ "$(lxc exec v2 -- fdisk --list /dev/sda 2>&1 >/dev/null)" = "" ]
+
         if [ "${poolDriver}" != "powerflex" ]; then
                 echo "==> Checking copied VM root disk size is 5GiB"
                 [ "$(($(lxc exec v2 -- blockdev --getsize64 /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_lxd_root) / GiB))" -eq "5" ]

diff --git a/tests/storage-volumes-vm b/tests/storage-volumes-vm
@@ -149,9 +149,15 @@ do
     fi
 
 	echo "==> Start VM and check content"
+	lxc storage volume attach "${poolName}" vol1 v1
 	lxc start v1
 	waitInstanceReady v1
 
+	echo "==> Verify that block volumes vol1 and vol2 (export/import) are bit for bit identical"
+	lxc exec v1 -- cmp /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_lxd_vol1 /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_lxd_vol2
+	echo "==> Verify that block volumes vol1 and vol3 (optimized export/import) are bit for bit identical"
+	lxc exec v1 -- cmp /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_lxd_vol1 /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_lxd_vol3
+
 	# shellcheck disable=2016
 	lxc exec v1 -- /bin/sh -c 'mount /dev/disk/by-id/scsi-0QEMU_QEMU_HARDDISK_lxd_vol2 /mnt && [ $(cat /mnt/bar) = foo ] && umount /mnt'
 	# shellcheck disable=2016
@@ -181,6 +187,7 @@ do
 	lxc exec v1 -- "sync"
 	lxc stop -f v1
 	lxc delete -f v2
+	lxc storage volume detach "${poolName}" vol1 v1
 	lxc storage volume detach "${poolName}" vol2 v1
 	lxc storage volume detach "${poolName}" vol3 v1
 	lxc storage volume detach "${poolName}" vol6 v1 || true  # optional ISO