Turn microceph setup into a reusable helper (#372)

This is the first step toward not relying on GH actions to setup
MicroCeph's snap. It replaces the ad-hoc manual setup by a set of 2
helper functions.

For now, it's only used to replace the `ceph` VM from the
`tests/vm-migration` by a `microceph` snap deployed in the GHA runner
VM. For now it doesn't use the ephemeral disk but a loop-backed file.

bin/helpers

@@ -141,6 +141,62 @@ install_deps() (
     fi
 )
 
+# install_microceph: install MicroCeph snap.
+install_microceph() (
+    if snap list microceph 2>/dev/null; then
+        snap refresh microceph --channel="${MICROCEPH_SNAP_CHANNEL:-latest/edge}" --cohort=+
+    else
+        snap install microceph --channel="${MICROCEPH_SNAP_CHANNEL:-latest/edge}" --cohort=+
+    fi
+)
+
+# configure_microceph: prepare MicroCeph for use by LXD.
+configure_microceph() {
+    if [ -z "${CEPH_DISK}" ]; then
+        echo "Missing disk for use with MicroCeph" >&2
+        return 1
+    fi
+
+    if [ ! -b "${CEPH_DISK}" ]; then
+        echo "${CEPH_DISK} is not a block device" >&2
+        return 1
+    fi
+
+    if microceph status; then
+        return 0
+    fi
+
+    microceph cluster bootstrap
+    microceph.ceph config set global mon_allow_pool_size_one true
+    microceph.ceph config set global mon_allow_pool_delete true
+    microceph.ceph config set global osd_pool_default_size 1
+    microceph.ceph config set global osd_memory_target 939524096
+    microceph.ceph osd crush rule rm replicated_rule
+    microceph.ceph osd crush rule create-replicated replicated default osd
+    for flag in nosnaptrim nobackfill norebalance norecover noscrub nodeep-scrub; do
+        microceph.ceph osd set $flag
+    done
+
+    microceph disk add --wipe "${CEPH_DISK}"
+
+    microceph enable rgw
+    microceph.ceph osd pool create cephfs_meta 32
+    microceph.ceph osd pool create cephfs_data 32
+    microceph.ceph fs new cephfs cephfs_meta cephfs_data
+    microceph.ceph fs ls
+    sleep 30
+    microceph.ceph status
+    # Wait until there are no more "unknowns" pgs
+    for _ in $(seq 60); do
+      if microceph.ceph pg stat | grep -wF unknown; then
+        sleep 1
+      else
+        break
+      fi
+    done
+    microceph.ceph status
+}
+
 # install_ovn: install OVN packages or MicroOVN snap.
 install_ovn() (
     if [ "${OVN_SOURCE:-latest/edge}" = "deb" ]; then

tests/vm-migration

@@ -4,6 +4,18 @@ set -eux
 # Install dependencies
 install_deps jq
 
+# Install MicroCeph
+install_microceph
+
+# Disk for use by MicroCeph
+backing_file="$(mktemp)"
+truncate -s 20GiB "${backing_file}"
+CEPH_DISK="$(losetup --direct-io=on --show --find "${backing_file}")"
+export CEPH_DISK
+
+# Configure MicroCeph
+configure_microceph
+
 # Install LXD
 install_lxd
 
@@ -18,41 +30,6 @@ echo "==> Create storage pool using driver ${poolDriver}"
 lxc storage create "${poolName}" "${poolDriver}"
 lxc profile device add default root disk path="/" pool="${poolName}"
 
-# Create ceph node
-lxc init "${TEST_IMG:-ubuntu-minimal-daily:24.04}" ceph --vm -c limits.cpu=2 -c limits.memory=4GiB
-lxc storage volume create "${poolName}" ceph-disk size=20GiB --type=block
-lxc config device add ceph ceph-disk disk pool="${poolName}" source=ceph-disk
-lxc start ceph
-
-# Wait for snap in ceph instance.
-waitInstanceReady ceph
-# shellcheck disable=SC3044 # Ignore "declare is undefined" shellcheck error.
-lxc exec ceph -- sh -c "$(declare -f waitSnapdSeed); waitSnapdSeed"
-
-# Install and configure ceph
-lxc exec ceph -- snap install microceph --edge
-lxc exec ceph -- microceph cluster bootstrap
-lxc exec ceph -- microceph.ceph config set global osd_pool_default_size 1
-lxc exec ceph -- microceph.ceph config set global mon_allow_pool_delete true
-lxc exec ceph -- microceph.ceph config set global osd_memory_target 939524096
-lxc exec ceph -- microceph.ceph osd crush rule rm replicated_rule
-lxc exec ceph -- microceph.ceph osd crush rule create-replicated replicated default osd
-for flag in nosnaptrim noscrub nobackfill norebalance norecover noscrub nodeep-scrub; do
-    lxc exec ceph -- microceph.ceph osd set "${flag}"
-done
-lxc exec ceph -- microceph disk add /dev/sdb
-lxc exec ceph -- microceph.ceph osd pool create cephfs_meta 32
-lxc exec ceph -- microceph.ceph osd pool create cephfs_data 32
-lxc exec ceph -- microceph.ceph fs new cephfs cephfs_meta cephfs_data
-lxc exec ceph -- microceph.ceph fs ls
-for _ in $(seq 60); do
-  if lxc exec ceph -- sudo microceph.ceph pg stat | grep -wF unknown; then
-    sleep 1
-  else
-    break
-  fi
-done
-
 # Launch two instances for our LXD cluster and wait for them to be ready.
 lxc init "${TEST_IMG:-ubuntu-minimal-daily:24.04}" member1 --vm -c limits.memory=2GiB
 lxc init "${TEST_IMG:-ubuntu-minimal-daily:24.04}" member2 --vm -c limits.memory=2GiB
@@ -105,11 +82,9 @@ cluster:
   cluster_token: "${joinToken}"
 EOF
 
-# Copy the ceph config from the microceph node into each cluster member.
-rm -rf etc/ceph
-lxc file pull -r ceph/var/snap/microceph/current/conf etc/ceph
-lxc file push -r -p etc/ceph/conf/* member1/etc/ceph/
-lxc file push -r -p etc/ceph/conf/* member2/etc/ceph/
+# Copy the ceph config from the local microceph snap into each cluster member.
+lxc file push -r -p /var/snap/microceph/current/conf/* member1/etc/ceph/
+lxc file push -r -p /var/snap/microceph/current/conf/* member2/etc/ceph/
 lxc exec member1 -- chmod +x /etc/ceph
 lxc exec member2 -- chmod +x /etc/ceph