Enhance 2 upgrade tests

- Do client-side layering tests on old build, then upgrade
rpm-ostree to new version, then upgrade to another new update.
- `e2e-upgrades`: Start old build with upgraded rpm-ostree,
do layering tests, then upgrade to another new update.

.cci.jenkinsfile

@@ -77,4 +77,12 @@ cosaPod(runAsUser: 0, memory: "${mem}Mi", cpu: "${nhosts}") {
       archiveArtifacts allowEmptyArchive: true, artifacts: 'vmcheck-logs.tar.xz'
     }
   }
+  stage("kola-upgrades") {
+    unstash 'rpms'
+    shwrap("""
+      coreos-installer download -p qemu -f qcow2.xz -s testing --decompress
+      mv rpm-ostree-[0-9]*.rpm rpm-ostree-libs-[0-9]*.rpm /usr/lib/coreos-assembler/tests/kola/rpm-ostree/upgrades/data/      
+    """)
+    kola(cosaDir: "${env.WORKSPACE}", extraArgs: "--qemu-image ./fedora-coreos*.qcow2 --tag rpm-ostree-upgrade ext.rpm-ostree.destructive.client-layering-upgrade")
+  }
 }

ci/prow/kola/upgrades

@@ -25,6 +25,13 @@ case "${AUTOPKGTEST_REBOOT_MARK:-}" in
     rpm -q moby-engine
     test '!' -f /etc/somenewfile
 
+    # do client layerring tests using new rpm-ostree
+    rpm-ostree install vim-filesystem
+    rpm-ostree override remove vim-minimal
+    rpm-ostree kargs --append foo=bar
+    touch /etc/foobar.conf
+    rpm-ostree initramfs --enable --arg=-I --arg=/etc/foobar.conf
+
     upgrade_image=$(cat /etc/upgrade-image)
     rpm-ostree rebase ${upgrade_image}
     /tmp/autopkgtest-reboot 1
@@ -43,6 +50,17 @@ case "${AUTOPKGTEST_REBOOT_MARK:-}" in
     fi
     test -f /etc/somenewfile
 
+    rpm -q vim-filesystem
+    if rpm -q vim-minimal 2>/dev/null; then
+      echo "found package expected to be removed"
+      exit 1
+    fi
+    cat /proc/cmdline > cmdlinekargs.txt
+    grep foo=bar cmdlinekargs.txt
+
+    lsinitrd "/usr/lib/modules/$(uname -r)/initramfs.img" > lsinitrd.txt
+    grep etc/foobar.conf lsinitrd.txt
+
     echo "ok e2e upgrade"
     ;;
   *) echo "unexpected mark: ${AUTOPKGTEST_REBOOT_MARK}"; exit 1;;

tests/kolainst/destructive/client-layering-upgrade

@@ -0,0 +1,124 @@
+#!/bin/bash
+## kola:
+##   # This test reboots a lot, generates container images, etc.
+##   timeoutMin: 30
+##   minMemory: 2048
+##   requiredTag: rpm-ostree-upgrade
+##   tags: needs-internet
+##   description: Do client-side layering tests on old build, then upgrade 
+##     rpm-ostree to new version, then upgrade to another new update.
+
+# Start old build, do client-side layering tests, reboot;
+# upgrade to new rpm-ostree, reboot;
+# exported to a container image from current commit, then rebase to it, reboot;
+# then upgrade to another update, reboot.
+
+
+set -xeuo pipefail
+
+. ${KOLA_EXT_DATA}/libtest.sh
+
+. /etc/os-release
+case $VERSION_ID in
+  39) kernel_release=6.5.6-300.fc39.x86_64
+    koji_kernel_url="https://koji.fedoraproject.org/koji/buildinfo?buildID=2302642"
+  ;;
+  *) echo "Unsupported Fedora version: $VERSION_ID"
+    exit 1
+  ;;
+esac
+
+# Do client-side layering tests includes:
+# kernel override, initramfs args, initramfs-etc, layering, overrides
+client_side_layering_tests()
+{
+  # override kernel
+  # copy test code from test-override-kernel.sh
+  current=$(rpm-ostree status --json | jq -r '.deployments[0].checksum')
+  rpm-ostree db list "${current}" > current-dblist.txt
+  if grep -qF $kernel_release current-dblist.txt; then
+    echo "Should not find $kernel_release in current deployment"
+    exit 1
+  fi
+
+  grep -E '^ kernel-[0-9]' current-dblist.txt  | sed -e 's,^ *,,' > orig-kernel.txt
+  test "$(wc -l < orig-kernel.txt)" == "1"
+  #orig_kernel=$(cat orig-kernel.txt)
+
+  rpm-ostree override replace $koji_kernel_url
+
+  rpm-ostree kargs --append foo=bar
+
+  touch /etc/foobar.conf
+  rpm-ostree initramfs --enable --arg=-I --arg=/etc/foobar.conf
+
+  rpm-ostree override remove moby-engine
+  rpm-ostree install --cache-only ${KOLA_EXT_DATA}/rpm-repos/0/packages/x86_64/foo-1.2-3.x86_64.rpm
+}
+
+# Check client-side layering tests results
+client_side_layering_checking()
+{
+  rpm-ostree status
+
+  test "$(uname -r)" == $kernel_release
+
+  cat /proc/cmdline > cmdlinekargs.txt
+  grep "foo=bar" cmdlinekargs.txt
+
+  lsinitrd "/usr/lib/modules/$(uname -r)/initramfs.img" > lsinitrd.txt
+  grep "etc/foobar.conf" lsinitrd.txt
+
+  if rpm -q moby-engine 2>/dev/null; then
+    echo "found package expected to be removed"
+    exit 1
+  fi
+  rpm -q foo
+}
+
+cd "$(mktemp -d)"
+
+case "${AUTOPKGTEST_REBOOT_MARK:-}" in
+  "")
+    rpm-ostree --version
+    systemctl mask --now zincati
+
+    client_side_layering_tests
+    /tmp/autopkgtest-reboot 1
+    ;;
+  "1")
+    client_side_layering_checking
+    echo "ok client-side layering tests"
+
+    # update rpm-ostree
+    rpm-ostree override replace ${KOLA_EXT_DATA}/rpm-ostree*
+    /tmp/autopkgtest-reboot 2
+    ;;
+  "2")
+    client_side_layering_checking
+    echo "ok upgrade to new rpm-ostree"
+
+    checksum=$(rpm-ostree status --json | jq -r '.deployments[0]."base-checksum"')
+    ostree refs ${checksum} --create kola
+    ostree commit -b kola --tree=ref=$checksum
+    rpm-ostree rebase :kola
+
+    /tmp/autopkgtest-reboot 3    
+    ;;
+  "3")
+    rpm-ostree status
+    echo "ok rebase"
+
+    rpm-ostree install vim-filesystem
+    ostree commit --base=kola -b kola-upgrade
+
+    rpm-ostree rebase :kola-upgrade
+    /tmp/autopkgtest-reboot 4
+    ;;
+  "4")
+    rpm-ostree status
+    rpm -q vim-filesystem
+    echo "ok upgrade to another update"
+    ;;
+  *) echo "unexpected mark: ${AUTOPKGTEST_REBOOT_MARK}"; exit 1;;
+esac