test: refactor monolithic test_smoke into multiple tests

.tekton/bootc-image-builder-pull-request.yaml

@@ -258,7 +258,7 @@ spec:
             - name: name
               value: buildah
             - name: bundle
-              value: quay.io/redhat-appstudio-tekton-catalog/task-buildah:0.1@sha256:9e7be6d3f2cde33fabfda9d454757530ce7f73e2414c00da1a3ebabf95fc52c5
+              value: quay.io/redhat-appstudio-tekton-catalog/task-buildah:0.1@sha256:d95e898c518093c24bc7dacd25aa98a39996262a6b34650e403b60b1c8c10fda
             - name: kind
               value: task
           resolver: bundles
@@ -295,7 +295,7 @@ spec:
             - name: name
               value: buildah-remote
             - name: bundle
-              value: quay.io/redhat-appstudio-tekton-catalog/task-buildah-remote:0.1@sha256:a0ed8d0f575244757c598acb228039211ad0051a7b7879630838260022a7b771
+              value: quay.io/redhat-appstudio-tekton-catalog/task-buildah-remote:0.1@sha256:e6818104e9b71c97737886f8ff1fb43df6c472915369cfd2dbea7fc3c1f06c29
             - name: kind
               value: task
           resolver: bundles
@@ -347,7 +347,7 @@ spec:
             - name: name
               value: inspect-image
             - name: bundle
-              value: quay.io/redhat-appstudio-tekton-catalog/task-inspect-image:0.1@sha256:ff8a4844ac36a8cbeecf02874e73952f314089cf497037ed762fa01dde397225
+              value: quay.io/redhat-appstudio-tekton-catalog/task-inspect-image:0.1@sha256:ab050621d5f6addf0b6c07451e91a6b134b4d5f402c780abdbf9842a8ff18504
             - name: kind
               value: task
           resolver: bundles
@@ -368,7 +368,7 @@ spec:
             - name: name
               value: deprecated-image-check
             - name: bundle
-              value: quay.io/redhat-appstudio-tekton-catalog/task-deprecated-image-check:0.3@sha256:61f9b273432c6d087393de6ab972a2f7af1c385a1cc36447b215dabf5d9ba0de
+              value: quay.io/redhat-appstudio-tekton-catalog/task-deprecated-image-check:0.3@sha256:d87f8c50a674f57527a0c4f3df6d9093941a2ae84739b55368b3c11702ce340c
             - name: kind
               value: task
           resolver: bundles
@@ -390,7 +390,7 @@ spec:
             - name: name
               value: clair-scan
             - name: bundle
-              value: quay.io/redhat-appstudio-tekton-catalog/task-clair-scan:0.1@sha256:fe27ccc0603b63b313329b76aa888523dabf7368fc25e976fcccc4d52bb3a5e3
+              value: quay.io/redhat-appstudio-tekton-catalog/task-clair-scan:0.1@sha256:fbe1ab58531d856fba360060d3884a0606310a966e2d01ba9305da9eb01ab916
             - name: kind
               value: task
           resolver: bundles
@@ -407,7 +407,7 @@ spec:
             - name: name
               value: sast-snyk-check
             - name: bundle
-              value: quay.io/redhat-appstudio-tekton-catalog/task-sast-snyk-check:0.1@sha256:339903f4a95b45b3f2dd4e6941b84679eaa70683bde83f46b0d835680c7aae8b
+              value: quay.io/redhat-appstudio-tekton-catalog/task-sast-snyk-check:0.1@sha256:eee508768b14655275fbcc2f42f9da1ab553b872dcbe113b0896aa9bcf7e1adf
             - name: kind
               value: task
           resolver: bundles
@@ -432,7 +432,7 @@ spec:
             - name: name
               value: clamav-scan
             - name: bundle
-              value: quay.io/redhat-appstudio-tekton-catalog/task-clamav-scan:0.1@sha256:de8c855a71d5bb6b838246f5c5a1a420747379b9cca047ce79b89c9b53812267
+              value: quay.io/redhat-appstudio-tekton-catalog/task-clamav-scan:0.1@sha256:d742ad90474ecccd066bbd98657885602e32c9f76d26991726b63a44f11feab2
             - name: kind
               value: task
           resolver: bundles
@@ -454,7 +454,7 @@ spec:
             - name: name
               value: sbom-json-check
             - name: bundle
-              value: quay.io/redhat-appstudio-tekton-catalog/task-sbom-json-check:0.1@sha256:563150a37f3802ab481be73b562f02f5a26ac3a11f116f247096620fe7a1ca48
+              value: quay.io/redhat-appstudio-tekton-catalog/task-sbom-json-check:0.1@sha256:717e6e33f02dbe1a28fb743f32699e002c944680c251a50b644f27becb9208e9
             - name: kind
               value: task
           resolver: bundles

.tekton/bootc-image-builder-push.yaml

@@ -255,7 +255,7 @@ spec:
             - name: name
               value: buildah
             - name: bundle
-              value: quay.io/redhat-appstudio-tekton-catalog/task-buildah:0.1@sha256:9e7be6d3f2cde33fabfda9d454757530ce7f73e2414c00da1a3ebabf95fc52c5
+              value: quay.io/redhat-appstudio-tekton-catalog/task-buildah:0.1@sha256:d95e898c518093c24bc7dacd25aa98a39996262a6b34650e403b60b1c8c10fda
             - name: kind
               value: task
           resolver: bundles
@@ -292,7 +292,7 @@ spec:
             - name: name
               value: buildah-remote
             - name: bundle
-              value: quay.io/redhat-appstudio-tekton-catalog/task-buildah-remote:0.1@sha256:a0ed8d0f575244757c598acb228039211ad0051a7b7879630838260022a7b771
+              value: quay.io/redhat-appstudio-tekton-catalog/task-buildah-remote:0.1@sha256:e6818104e9b71c97737886f8ff1fb43df6c472915369cfd2dbea7fc3c1f06c29
             - name: kind
               value: task
           resolver: bundles
@@ -344,7 +344,7 @@ spec:
             - name: name
               value: inspect-image
             - name: bundle
-              value: quay.io/redhat-appstudio-tekton-catalog/task-inspect-image:0.1@sha256:ff8a4844ac36a8cbeecf02874e73952f314089cf497037ed762fa01dde397225
+              value: quay.io/redhat-appstudio-tekton-catalog/task-inspect-image:0.1@sha256:ab050621d5f6addf0b6c07451e91a6b134b4d5f402c780abdbf9842a8ff18504
             - name: kind
               value: task
           resolver: bundles
@@ -365,7 +365,7 @@ spec:
             - name: name
               value: deprecated-image-check
             - name: bundle
-              value: quay.io/redhat-appstudio-tekton-catalog/task-deprecated-image-check:0.3@sha256:61f9b273432c6d087393de6ab972a2f7af1c385a1cc36447b215dabf5d9ba0de
+              value: quay.io/redhat-appstudio-tekton-catalog/task-deprecated-image-check:0.3@sha256:d87f8c50a674f57527a0c4f3df6d9093941a2ae84739b55368b3c11702ce340c
             - name: kind
               value: task
           resolver: bundles
@@ -387,7 +387,7 @@ spec:
             - name: name
               value: clair-scan
             - name: bundle
-              value: quay.io/redhat-appstudio-tekton-catalog/task-clair-scan:0.1@sha256:fe27ccc0603b63b313329b76aa888523dabf7368fc25e976fcccc4d52bb3a5e3
+              value: quay.io/redhat-appstudio-tekton-catalog/task-clair-scan:0.1@sha256:fbe1ab58531d856fba360060d3884a0606310a966e2d01ba9305da9eb01ab916
             - name: kind
               value: task
           resolver: bundles
@@ -404,7 +404,7 @@ spec:
             - name: name
               value: sast-snyk-check
             - name: bundle
-              value: quay.io/redhat-appstudio-tekton-catalog/task-sast-snyk-check:0.1@sha256:339903f4a95b45b3f2dd4e6941b84679eaa70683bde83f46b0d835680c7aae8b
+              value: quay.io/redhat-appstudio-tekton-catalog/task-sast-snyk-check:0.1@sha256:eee508768b14655275fbcc2f42f9da1ab553b872dcbe113b0896aa9bcf7e1adf
             - name: kind
               value: task
           resolver: bundles
@@ -429,7 +429,7 @@ spec:
             - name: name
               value: clamav-scan
             - name: bundle
-              value: quay.io/redhat-appstudio-tekton-catalog/task-clamav-scan:0.1@sha256:de8c855a71d5bb6b838246f5c5a1a420747379b9cca047ce79b89c9b53812267
+              value: quay.io/redhat-appstudio-tekton-catalog/task-clamav-scan:0.1@sha256:d742ad90474ecccd066bbd98657885602e32c9f76d26991726b63a44f11feab2
             - name: kind
               value: task
           resolver: bundles
@@ -451,7 +451,7 @@ spec:
             - name: name
               value: sbom-json-check
             - name: bundle
-              value: quay.io/redhat-appstudio-tekton-catalog/task-sbom-json-check:0.1@sha256:563150a37f3802ab481be73b562f02f5a26ac3a11f116f247096620fe7a1ca48
+              value: quay.io/redhat-appstudio-tekton-catalog/task-sbom-json-check:0.1@sha256:717e6e33f02dbe1a28fb743f32699e002c944680c251a50b644f27becb9208e9
             - name: kind
               value: task
           resolver: bundles

test/test_smoke.py

@@ -1,3 +1,4 @@
+import collections
 import json
 import os
 import pathlib
@@ -11,39 +12,102 @@
 from vm import VM
 
 
-@pytest.fixture(name="output_path")
-def output_path_fixture(tmp_path):
-    output_path = tmp_path / "output"
-    output_path.mkdir(exist_ok=True)
-    return output_path
+if not testutil.has_executable("podman"):
+    pytest.skip("no podman, skipping integration tests that required podman", allow_module_level=True)
+
+if os.getuid() != 0:
+    pytest.skip("tests require root to run", allow_module_level=True)
+
+# building an ELN image needs x86_64-v3 to work, we use avx2 as a proxy
+# to detect if we have x86-64-v3 (not perfect but should be good enough)
+if not testutil.has_x86_64_v3_cpu():
+    pytest.skip("need x86_64-v3 capable CPU", allow_module_level=True)
+
+
+@pytest.fixture(name="build_container", scope="session")
+def build_container_fixture():
+    """Build a container from the Containerfile and returns the name"""
+    container_tag = "bootc-image-builder-test"
+    subprocess.check_call([
+        "podman", "build",
+        "-f", "Containerfile",
+        "-t", container_tag,
+    ])
+    return container_tag
 
 
-@pytest.fixture(name="config_json")
-def config_json_fixture(output_path):
+@pytest.fixture(name="build_image", scope="session")
+def build_image_fixture(tmpdir_factory, build_container):
+    """
+    Build an image inside the passed build_container and return a
+    named tuple with the resulting image path and user/password
+    """
+    username = "test"
+    password = "password"
     CFG = {
         "blueprint": {
             "customizations": {
                 "user": [
                     {
-                        "name": "test",
-                        "password": "password",
+                        "name": username,
+                        "password": password,
                         "groups": ["wheel"],
                     },
                 ],
             },
         },
     }
+    output_path = pathlib.Path(tmpdir_factory.mktemp("data")) / "output"
+    output_path.mkdir(exist_ok=True)
+
     config_json_path = output_path / "config.json"
     config_json_path.write_text(json.dumps(CFG), encoding="utf-8")
-    return config_json_path
+
+    cursor = testutil.journal_cursor()
+    # run container to deploy an image into output/qcow2/disk.qcow2
+    subprocess.check_call([
+        "podman", "run", "--rm",
+        "--privileged",
+        "--security-opt", "label=type:unconfined_t",
+        "-v", f"{output_path}:/output",
+        build_container,
+        "quay.io/centos-bootc/fedora-bootc:eln",
+        "--config", "/output/config.json",
+    ])
+    journal_output = testutil.journal_after_cursor(cursor)
+    generated_img = pathlib.Path(output_path) / "qcow2/disk.qcow2"
+
+    ImageFixtureResult = collections.namedtuple(
+        "BuildImage", ["img_path", "username", "password", "journal_output"])
+    return ImageFixtureResult(generated_img, username, password, journal_output)
+
+
+def test_container_builds(build_container):
+    output = subprocess.check_output([
+        "podman", "images", "-n", build_container], encoding="utf-8")
+    assert build_container in output
+
+
+def test_image_is_generated(build_image):
+    assert build_image.img_path.exists(), "output file missing, dir "\
+        f"content: {os.listdir(os.fspath(build_image.img_path))}"
+
+
+def test_image_boots(build_image):
+    with VM(build_image.img_path) as test_vm:
+        exit_status, _ = test_vm.run("true", user=build_image.username, password=build_image.password)
+        assert exit_status == 0
+        exit_status, output = test_vm.run("echo hello", user="test", password="password")
+        assert exit_status == 0
+        assert "hello" in output
 
 
 def log_has_osbuild_selinux_denials(log):
     OSBUID_SELINUX_DENIALS_RE = re.compile(r"(?ms)avc:\ +denied.*osbuild")
     return re.search(OSBUID_SELINUX_DENIALS_RE, log)
 
 
-def test_osbuild_selinux_denails_re_works():
+def test_osbuild_selinux_denials_re_works():
     fake_log = (
         'Dec 05 07:19:39 other log msg\n'
         'Dec 05 07:19:39 fedora audit: SELINUX_ERR'
@@ -60,45 +124,13 @@ def test_osbuild_selinux_denails_re_works():
     assert not log_has_osbuild_selinux_denials("some\nrandom\nlogs")
 
 
-@pytest.mark.skipif(os.getuid() != 0, reason="needs root")
-@pytest.mark.skipif(not testutil.has_executable("podman"), reason="need podman")
-def test_smoke(output_path, config_json):
-    # build local container
-    subprocess.check_call([
-        "podman", "build",
-        "-f", "Containerfile",
-        "-t", "bootc-image-builder-test",
-    ])
-    cursor = testutil.journal_cursor()
-    # and run container to deploy an image into output/disk.qcow2
-    subprocess.check_call([
-        "podman", "run", "--rm",
-        "--privileged",
-        "--security-opt", "label=type:unconfined_t",
-        "-v", f"{output_path}:/output",
-        "bootc-image-builder-test",
-        "quay.io/centos-bootc/fedora-bootc:eln",
-        "--config", "/output/config.json",
-    ])
-    generated_img = pathlib.Path(output_path) / "qcow2/disk.qcow2"
-    assert generated_img.exists(), f"output file missing, dir content: {os.listdir(os.fspath(output_path))}"
+def has_selinux():
+    return testutil.has_executable("selinuxenabled") and subprocess.run("selinuxenabled").returncode == 0
 
-    # check that there are no selinux denials
-    journal_output = testutil.journal_after_cursor(cursor)
-    assert journal_output != ""
-    if testutil.has_executable("selinuxenabled") and subprocess.run("selinuxenabled").returncode == 0:
-        assert not log_has_osbuild_selinux_denials(journal_output), f"denials in log {journal_output}"
-    else:
-        print("WARNING: selinux not enabled, cannot check for denials")
-
-    # building an ELN image needs x86_64-v3 to work, we use avx2 as a proxy
-    # to detect if we have x86-64-v3 (not perfect but should be good enough)
-    if " avx2 " not in pathlib.Path("/proc/cpuinfo").read_text():
-        print("WARNING: no x86_64-v3 cpu detected, skipping VM boot test")
-    else:
-        with VM(generated_img) as test_vm:
-            exit_status, _ = test_vm.run("true", user="test", password="password")
-            assert exit_status == 0
-            exit_status, output = test_vm.run("echo hello", user="test", password="password")
-            assert exit_status == 0
-            assert "hello" in output
+
+@pytest.mark.skipif(not has_selinux(), reason="selinux not enabled")
+def test_image_build_without_se_linux_denials(build_image):
+    # the journal always contains logs from the image building
+    assert build_image.journal_output != ""
+    assert not log_has_osbuild_selinux_denials(build_image.journal_output), \
+        f"denials in log {build_image.journal_output}"

test/testutil.py

@@ -1,3 +1,4 @@
+import pathlib
 import socket
 import shutil
 import subprocess
@@ -39,3 +40,10 @@ def wait_ssh_ready(port, sleep, max_wait_sec):
                 pass
             time.sleep(sleep)
     raise ConnectionRefusedError(f"cannot connect to port {port} after {max_wait_sec}s")
+
+
+def has_x86_64_v3_cpu():
+    # x86_64-v3 has multiple features, see
+    # https://en.wikipedia.org/wiki/X86-64#Microarchitecture_levels
+    # but "avx2" is probably a good enough proxy
+    return " avx2 " in pathlib.Path("/proc/cpuinfo").read_text()

test/testutil_test.py

@@ -24,6 +24,7 @@ def test_wait_ssh_ready_sleeps_no_connection(mocked_sleep, free_port):
     assert mocked_sleep.call_args_list == [call(0.1), call(0.1), call(0.1)]
 
 
+@pytest.mark.skipif(not has_executable("nc"), reason="needs nc")
 def test_wait_ssh_ready_sleeps_wrong_reply(free_port, tmp_path):
     with contextlib.ExitStack() as cm:
         p = subprocess.Popen(