upgrade-test: do basic tests on old builds before upgrade

rpm-ostree to new version, then upgrade to another new update

Test steps:
- start old build, do basic tests, reboot;
- upgrade to new rpm-ostree, reboot;
- exported to a container image from current commit, then rebase
to it, reboot;
- then upgrade to another update, reboot.

.cci.jenkinsfile

@@ -77,4 +77,13 @@ cosaPod(runAsUser: 0, memory: "${mem}Mi", cpu: "${nhosts}") {
       archiveArtifacts allowEmptyArchive: true, artifacts: 'vmcheck-logs.tar.xz'
     }
   }
+  stage("kola-upgrades") {
+    unstash 'rpms'
+    shwrap("""
+      coreos-installer download -p qemu -f qcow2.xz -s testing --decompress
+      make -C tests/kolainst upgrades-install
+      mv rpm-ostree-[0-9]*.rpm rpm-ostree-libs-[0-9]*.rpm /usr/lib/coreos-assembler/tests/kola/rpm-ostree/upgrades/data/      
+    """)
+    kola(cosaDir: "${env.WORKSPACE}", extraArgs: "--qemu-image ./fedora-coreos*.qcow2 ext.rpm-ostree.upgrades.*")
+  }
 }

tests/kolainst/Makefile

@@ -22,3 +22,9 @@ install: all
 localinstall: all
 	rm -rf ../kola
 	make install KOLA_TESTDIR=../kola
+
+upgrades-install: all
+	install -d -m 0755 $(KOLA_TESTDIR)
+	rsync -prlv ./upgrades $(KOLA_TESTDIR)/
+	rsync -prlv ../common/*.sh $(KOLA_TESTDIR)/upgrades/data/
+	rsync -prlv rpm-repos/ $(KOLA_TESTDIR)/upgrades/data/rpm-repos/

tests/kolainst/upgrades/misc-tests

@@ -0,0 +1,163 @@
+#!/bin/bash
+## kola:
+##   # This test reboots a lot, generates container images, etc.
+##   timeoutMin: 40
+##   minMemory: 2048
+##   tags: needs-internet
+##   description: Do basic tests on old builds, then upgrade rpm-ostree
+##     to new version, then upgrade to another new update.
+
+# Start old build, do basic tests, reboot;
+# upgrade to new rpm-ostree, reboot;
+# exported to a container image from current commit, then rebase to it, reboot;
+# then upgrade to another update, reboot.
+
+
+set -xeuo pipefail
+
+. ${KOLA_EXT_DATA}/libtest.sh
+
+. /etc/os-release
+case $VERSION_ID in
+  39) kernel_release=6.5.6-300.fc39.x86_64
+    koji_kernel_url="https://koji.fedoraproject.org/koji/buildinfo?buildID=2302642"
+  ;;
+  *) echo "Unsupported Fedora version: $VERSION_ID"
+    exit 1
+  ;;
+esac
+
+get_deployment_root() {
+  local csum
+  local serial
+  local osname
+  csum=$(rpm-ostree status --json | jq -r '.deployments[0].checksum')
+  serial="$(rpm-ostree status --json | jq -r '.deployments[0].serial')"
+  osname="$(rpm-ostree status --json | jq -r '.deployments[0].osname')"
+  echo /ostree/deploy/$osname/deploy/$csum.$serial
+}
+
+misc_tests()
+{
+  # override kernel
+  # copy test code from test-override-kernel.sh
+  current=$(rpm-ostree status --json | jq -r '.deployments[0].checksum')
+  rpm-ostree db list "${current}" > current-dblist.txt
+  if grep -qF $kernel_release current-dblist.txt; then
+    echo "Should not find $kernel_release in current deployment"
+    exit 1
+  fi
+
+  grep -E '^ kernel-[0-9]' current-dblist.txt  | sed -e 's,^ *,,' > orig-kernel.txt
+  test "$(wc -l < orig-kernel.txt)" == "1"
+  orig_kernel=$(cat orig-kernel.txt)
+
+  rpm-ostree override replace $koji_kernel_url
+  new=$(rpm-ostree status --json | jq -r '.deployments[0].checksum')
+  rpm-ostree db list "${new}" > new-dblist.txt
+  if ! grep -qF $kernel_release new-dblist.txt; then
+    echo "Should find $kernel_release in the new deployment"
+    exit 1
+  fi
+
+  if grep -q -F -e "${orig_kernel}" new-dblist.txt; then
+    echo "Should not find ${orig_kernel} in the new deployment"
+    exit 1
+  fi
+  newroot=$(get_deployment_root)
+  find ${newroot}/usr/lib/modules -maxdepth 1 -type d > modules-dirs.txt
+  test "$(wc -l < modules-dirs.txt)" == "2"
+  if ! grep -qF $kernel_release modules-dirs.txt; then
+    echo "Should find $kernel_release in ${newroot}/usr/lib/modules"
+    exit 1
+  fi
+
+  rpm-ostree kargs --append foo=bar
+
+  touch /etc/foobar.conf
+  rpm-ostree initramfs --enable --arg=-I --arg=/etc/foobar.conf
+
+  rpm-ostree override remove moby-engine
+  rpm-ostree install --cache-only ${KOLA_EXT_DATA}/rpm-repos/0/packages/x86_64/foo-1.2-3.x86_64.rpm
+}
+
+do_checking()
+{
+  rpm-ostree status
+
+  test "$(uname -r)" == $kernel_release
+
+  cat /proc/cmdline > cmdlinekargs.txt
+  if ! grep foo=bar cmdlinekargs.txt; then
+    echo "can not find kernel parameter foo=bar"
+    exit 1
+  fi
+  lsinitrd "/usr/lib/modules/$(uname -r)/initramfs.img" > lsinitrd.txt
+  if ! grep etc/foobar.conf lsinitrd.txt; then
+    echo "can not find file expected to be included in initramfs.img"
+    exit 1
+  fi
+  if rpm -q moby-engine 2>/dev/null; then
+    echo "found package expected to be removed"
+    exit 1
+  fi
+  rpm -q foo
+}
+
+cd "$(mktemp -d)"
+
+image=containers-storage:localhost/fcos
+image_pull=ostree-unverified-image:$image
+
+case "${AUTOPKGTEST_REBOOT_MARK:-}" in
+  "")
+    rpm-ostree --version
+    systemctl mask --now zincati
+
+    misc_tests
+    /tmp/autopkgtest-reboot 1
+    ;;
+  "1")
+    do_checking
+    echo "ok misc tests"
+
+    # update rpm-ostree
+    rpm-ostree override replace ${KOLA_EXT_DATA}/rpm-ostree*
+    /tmp/autopkgtest-reboot 2
+    ;;
+  "2")
+    do_checking
+    echo "ok upgrade to new rpm-ostree"
+
+    # remove foo as workaround, see https://github.com/coreos/rpm-ostree/issues/4805
+    rpm-ostree uninstall foo
+    checksum=$(rpm-ostree status --json | jq -r '.deployments[0].checksum')
+    ostree container encapsulate --repo=/ostree/repo ${checksum} ${image}
+
+    # rebase to container image
+    rpm-ostree rebase ${image_pull}
+    /tmp/autopkgtest-reboot 3    
+    ;;
+  "3")
+    rpm-ostree status
+
+    booted_image=$(rpm-ostree status --json | jq -r '.deployments[0]["container-image-reference"]')
+    test "${image_pull}" = "$booted_image"
+    echo "ok rebase container image"
+
+    cat > Containerfile << EOF
+FROM localhost/fcos
+RUN rpm-ostree install vim-filesystem && rpm-ostree cleanup -m
+EOF
+
+    podman build --net=host -t localhost/fcos-derived --squash .
+    rpm-ostree rebase "${image_pull}-derived"
+    /tmp/autopkgtest-reboot 4
+    ;;
+  "4")
+    rpm-ostree status
+    rpm -q vim-filesystem
+    echo "ok upgrade to another container image"
+    ;;
+  *) echo "unexpected mark: ${AUTOPKGTEST_REBOOT_MARK}"; exit 1;;
+esac