Merge pull request #25 from cevich/pbsaio

Add all-in-one image and build automation

.cirrus.yml

@@ -85,8 +85,8 @@ cron_image_build_task:
     only_if: $CIRRUS_CRON == 'cron_image_build_task'
     gce_instance: *build_push
     env:
-        CONTAINERS_USERNAME: ENCRYPTED[f94aa9610f678dc79ca45d49ee4c41a43da9468094883eb386ea907f6218cd49df61f892105109da8b5309523db3ed0b]
-        CONTAINERS_PASSWORD: ENCRYPTED[84a2784130e2c359afa70ad0575b04f448d248ca947d130d3450eb01676e7f934b6de621a167edf56cd4901396dfe7e2]
+        CONTAINERS_USERNAME: &cntu ENCRYPTED[f94aa9610f678dc79ca45d49ee4c41a43da9468094883eb386ea907f6218cd49df61f892105109da8b5309523db3ed0b]
+        CONTAINERS_PASSWORD: &cntp ENCRYPTED[84a2784130e2c359afa70ad0575b04f448d248ca947d130d3450eb01676e7f934b6de621a167edf56cd4901396dfe7e2]
         PODMAN_USERNAME: ENCRYPTED[c7c6506427eeecce7c709a94fb7547987545cb4ba7e607e249444b3588a41069ad116781f3187018c12c6fff0fd425d7]
         PODMAN_PASSWORD: ENCRYPTED[f7c321e7dfb017e4111e0fc3c0f7eb2e743d11f4eddca5cf209c2f25e2c778eb33ab746b6ef91233e570ac3d547a86f0]
         BUILDAH_USERNAME: ENCRYPTED[58742c385f0938a25cd523837bee50bf40db7c2523dc4506b9a1c3d72233e828ad9527ca638c18eb825d2ceef6d5b31d]
@@ -96,6 +96,32 @@ cron_image_build_task:
     matrix: *pbs_matrix
     script: *pbs_script
 
+test_aio_image_build_task:
+    alias: test_aio_image_build
+    name: "Test build AIO image"
+    only_if: *is_pr
+    skip: "!changesInclude('.cirrus.yml', 'ci/aio_build_push.sh', 'ci/tag_version.sh', 'aio/**/*')"
+    depends_on:
+        - test_build-push
+    gce_instance: *build_push
+    env:
+        ARCHES: amd64
+        DRYRUN: 1  # Don't actually push anything, only build.
+        A_DEBUG: 1
+    build_script: &aio_script |
+        source /etc/automation_environment
+        ./ci/aio_build_push.sh ${CIRRUS_REPO_CLONE_URL}
+    test_script: ./aio/test.sh
+
+cron_aio_build_task:
+    alias: cron_aio_build
+    only_if: $CIRRUS_CRON == 'cron_aio_build_task'
+    gce_instance: *build_push
+    env:
+        CONTAINERS_USERNAME: *cntu
+        CONTAINERS_PASSWORD: *cntp
+    build_script: *aio_script
+
 # This task is critical.  It updates the "last-used by" timestamp stored
 # in metadata for all VM images.  This mechanism functions in tandem with
 # an out-of-band pruning operation to remove disused VM images.
@@ -125,6 +151,7 @@ success_task:
         - validate
         - test_build-push
         - test_image_build
+        - test_aio_image_build
         - meta
     container:
         <<: *ci_container

README.md

@@ -35,6 +35,11 @@ or `skopeo`:
   * `quay.io/containers/*:latest` and `quay.io/*/stable:latest` -
     Built daily using the same `Containerfile` as above.  The tool versions
     will remain the "latest" available in Fedora.
+  * `quay.io/containers/aio:latest` and `quay.io/containers/aio:<date stamp>` -
+    "All In One" image containing Podman, Buildah, and Skopeo.  Built weekly
+    using a similar `Containerfile` as the Podman and Buildah images.  It's a
+    smaller, minimal image, intended to be used as a base-image for development
+    containers or CI/automation.
   * `quay.io/*/testing:latest` - This image is built daily, using the
     latest tooling version available in the Fedora `updates-testing` repository.
   * `quay.io/*/upstream:latest` - This image is built daily using the latest
@@ -54,3 +59,7 @@ or `skopeo`:
 ## Skopeo Sample Usage
 
 [Please see the subdirectory README.md](https://github.com/containers/image_build/blob/main/skopeo/README.md)
+
+## All In One Sample Usage
+
+[Please see the subdirectory README.md](https://github.com/containers/image_build/blob/main/aio/README.md)

aio/Containerfile

@@ -0,0 +1,68 @@
+# aio/Containerfile
+#
+# Build an all in one Podman, Buildah, Skopeo container
+# image from the latest stable version of Podman on the
+# Fedoras Updates System.
+# https://bodhi.fedoraproject.org/updates/?search=podman
+# https://bodhi.fedoraproject.org/updates/?search=buildah
+# https://bodhi.fedoraproject.org/updates/?search=skopeo
+# This image is intended to be used as-is, or as a base-
+# image for development work or use in CI/CD systems.
+
+FROM registry.fedoraproject.org/fedora-minimal:latest
+
+# When building for multiple-architectures in parallel using emulation
+# it's really easy for one/more dnf processes to timeout or mis-count
+# the minimum download rates.  Bump both to be extremely forgiving of
+# an overworked host.
+RUN echo -e "\n\n# Added during image build" >> /etc/dnf/dnf.conf && \
+    echo -e "minrate=100\ntimeout=60\n" >> /etc/dnf/dnf.conf
+
+RUN microdnf -y makecache && \
+    microdnf -y update && \
+    microdnf -y install podman buildah skopeo fuse-overlayfs openssh-clients \
+        --exclude "container-selinux,qemu-*" && \
+    rpm --setcaps shadow-utils 2>/dev/null && \
+    microdnf clean all && \
+    rm -rf /var/cache /var/log/dnf* /var/log/yum.*
+
+# It's assumed `user` will end up with UID/GID 1000
+RUN useradd user && \
+    echo -e "user:1:999\nuser:1001:64535" > /etc/subuid && \
+    echo -e "user:1:999\nuser:1001:64535" > /etc/subgid
+
+ADD /containers.conf /etc/containers/containers.conf
+ADD /user-containers.conf /home/user/.config/containers/containers.conf
+
+RUN mkdir -p /home/user/.local/share/containers && \
+    mkdir -p /home/user/.config/containers && \
+    chown user:user -R /home/user && \
+    chmod 644 /etc/containers/containers.conf
+
+# Copy & modify the defaults to provide reference if runtime changes needed.
+# Changes here are required for running with fuse-overlay storage inside container.
+RUN sed -e 's|^#mount_program|mount_program|g' \
+        -e '/additionalimage.*/a "/var/lib/shared",' \
+        -e 's|^mountopt[[:space:]]*=.*$|mountopt = "nodev,fsync=0"|g' \
+        /usr/share/containers/storage.conf \
+        > /etc/containers/storage.conf
+
+# Setup internal Podman to pass subscriptions down from host to internal container
+RUN printf '/run/secrets/etc-pki-entitlement:/run/secrets/etc-pki-entitlement\n/run/secrets/rhsm:/run/secrets/rhsm\n' > /etc/containers/mounts.conf
+
+# Note VOLUME options must always happen after the chown call above
+# RUN commands can not modify existing volumes
+VOLUME /var/lib/containers
+VOLUME /home/user/.local/share/containers
+
+RUN mkdir -p /var/lib/shared/overlay-images \
+             /var/lib/shared/overlay-layers \
+             /var/lib/shared/vfs-images \
+             /var/lib/shared/vfs-layers && \
+    touch /var/lib/shared/overlay-images/images.lock && \
+    touch /var/lib/shared/overlay-layers/layers.lock && \
+    touch /var/lib/shared/vfs-images/images.lock && \
+    touch /var/lib/shared/vfs-layers/layers.lock
+
+ENV _CONTAINERS_USERNS_CONFIGURED="" \
+    BUILDAH_ISOLATION=chroot

aio/README.md

@@ -0,0 +1,56 @@
+[comment]: <> (***ATTENTION*** ***WARNING*** ***ALERT*** ***CAUTION*** ***DANGER***)
+[comment]: <> ()
+[comment]: <> (ANY changes made below, once committed/merged must)
+[comment]: <> (be manually copy/pasted -in markdown- into the description)
+[comment]: <> (field on Quay at the following locations:)
+[comment]: <> ()
+[comment]: <> (https://quay.io/repository/containers/aio)
+[comment]: <> ()
+[comment]: <> (***ATTENTION*** ***WARNING*** ***ALERT*** ***CAUTION*** ***DANGER***)
+
+![PODMAN logo](https://raw.githubusercontent.com/containers/common/main/logos/podman-logo-full-vert.png)
+![buildah logo](https://cdn.rawgit.com/containers/buildah/main/logos/buildah-logo_large.png)
+<img src="https://cdn.rawgit.com/containers/skopeo/main/docs/skopeo.svg" width="250">
+
+# All In One: Podman, Buildah and Skopeo Image
+
+## Build information
+
+Please see the [containers/image_build repo. README.md for build
+details](https://github.com/containers/image_build/blob/main/README.md).
+
+## Sample Usage
+
+Running as 'root' inside the container:
+
+```
+# Create a directory on the host to mount the container's
+# /var/lib/container directory to so containers can be
+# run within the container.
+mkdir /var/lib/mycontainers
+
+# Run a shell in the container, will full nested container run and build
+# possibilities:
+podman run -it --net=host --security-opt label=disable --privileged \
+    --security-opt seccomp=unconfined --device /dev/fuse:rw \
+    -v /var/lib/mycontainers:/var/lib/containers:Z \
+    quay.io/containers/aio:latest
+```
+
+Running rootless inside the container:
+```
+mkdir $HOME/mycontainers
+
+# Run a shell in the container, will full nested container run and build
+# possibilities:
+podman run -it --net=host --security-opt label=disable --privileged \
+    --security-opt seccomp=unconfined --device /dev/fuse:rw \
+    --user user --userns=keep-id:uid=1000,gid=1000 \
+    -v $HOME/mycontainers:/home/user/.local/share/containers:Z \
+    quay.io/containers/aio:latest
+```
+
+**Note:** If you encounter a `fuse: device not found` error when running the container image, it is likely that
+the fuse kernel module has not been loaded on your host system.  Use the command `modprobe fuse` to load the
+module and then run the container image.  To enable this automatically at boot time, you can add a configuration
+file to `/etc/modules.load.d`.  See `man modules-load.d` for more details.

aio/containers.conf

@@ -0,0 +1,13 @@
+[containers]
+netns="host"
+userns="host"
+ipcns="host"
+utsns="host"
+cgroupns="host"
+cgroups="disabled"
+log_driver = "k8s-file"
+
+[engine]
+cgroup_manager = "cgroupfs"
+events_logger="file"
+runtime="crun"

aio/test.sh

@@ -0,0 +1,87 @@
+#!/bin/bash
+
+# This script is not intended for humans.  It's meant to be run
+# by a CI system after a test-build of the
+# quay.io/containers/aio:latest manifest list.
+
+set -eo pipefail
+
+if [[ -r "/etc/automation_environment" ]]; then
+    source /etc/automation_environment  # defines AUTOMATION_LIB_PATH
+    #shellcheck disable=SC1090,SC2154
+    source "$AUTOMATION_LIB_PATH/common_lib.sh"
+    dbg "Using automation common library version $(<$AUTOMATION_LIB_PATH/../AUTOMATION_VERSION)"
+else
+    echo "Expecting to find automation common library installed."
+    exit 1
+fi
+
+FQIN="quay.io/containers/aio:latest"
+FQIN_FILE="$(basename $FQIN | tr ':' '-').tar"
+
+# msg() doesn't support a prefix, nor show file/line-no.
+# Abuse warn() to print testing messages and make them stand-out.
+WARNING_MSG_PREFIX="***** TEST:"
+msg() { warn "$1"; }
+
+# These tests need to be run rootless, assume the environment is disposable.
+# N/B: This condition does not return!
+if [[ "$UID" -eq 0 ]]; then
+    msg "Check that $FQIN exists in local storage"
+    showrun podman manifest exists $FQIN
+
+    msg "Verify manifest-list contains image for amd64 architecture"
+    arches=$(showrun podman manifest inspect $FQIN | showrun jq -r -e '.manifests[].platform.architecture')
+    showrun grep -F -x -q 'amd64' <<<"$arches"
+
+    msg "Verify skopeo can inspect the local manifest list"
+    showrun skopeo inspect --raw containers-storage:$FQIN | jq .
+
+    msg "Setting up for rootless testing"
+    TESTUSER="testuser$RANDOM"
+    showrun useradd "$TESTUSER"
+    export TUHOME="/home/$TESTUSER"
+    showrun podman save -o "$TUHOME/$FQIN_FILE" "$FQIN"
+    showrun chown $TESTUSER:$TESTUSER "$TUHOME/$FQIN_FILE"
+    (umask 077; showrun mkdir -p "/root/.ssh")
+    (umask 077; showrun ssh-keyscan localhost >> "/root/.ssh/known_hosts")
+    showrun ssh-keygen -t rsa -P "" -f "/root/.ssh/id_rsa"
+    (umask 077; showrun mkdir -p "$TUHOME/.ssh")
+    showrun cp "/root/.ssh/id_rsa.pub" "$TUHOME/.ssh/authorized_keys"
+    showrun chown -R $TESTUSER:$TESTUSER "$TUHOME/.ssh"
+    showrun chmod 0600 "$TUHOME/.ssh/authorized_keys"
+    # $SCRIPT_PATH/$SCRIPT_FILENAME defined by automation library
+    # shellcheck disable=SC2154
+    showrun exec ssh $TESTUSER@localhost $SCRIPT_PATH/$SCRIPT_FILENAME
+fi
+
+# SCRIPT_FILENAME defined by automation library
+# shellcheck disable=SC2154
+TMPD=$(mktemp -p '' -d ${SCRIPT_FILENAME}_XXXXX_tmp)
+trap "podman unshare rm -rf '$TMPD'" EXIT
+
+msg "Loading test image"
+showrun podman load -i $HOME/$FQIN_FILE
+
+# These tests come directly from the aio/README.md examples
+mkdir $TMPD/cntr_storage
+mkdir $TMPD/context
+echo -e 'FROM registry.fedoraproject.org/fedora-minimal:latest\nENV TESTING=true' > $TMPD/context/Containerfile
+for tool in buildah podman; do
+    msg "Verify $tool can create a simple image as root inside $FQIN"
+    showrun podman unshare rm -rf $TMPD/cntr_storage/* $TMPD/cntr_storage/.??*
+    showrun podman run -i --rm --net=host --security-opt label=disable --privileged \
+        --security-opt seccomp=unconfined --device /dev/fuse:rw \
+        -v $TMPD/cntr_storage:/var/lib/containers:Z \
+        -v $TMPD/context:/root/context:Z \
+        $FQIN $tool build -t root_testimage /root/context
+
+    msg "Verify $tool can create a simple image as rootless inside $FQIN"
+    showrun podman unshare rm -rf $TMPD/cntr_storage/* $TMPD/cntr_storage/.??*
+    showrun podman run -i --rm --net=host --security-opt label=disable --privileged \
+        --security-opt seccomp=unconfined --device /dev/fuse:rw \
+        --user user --userns=keep-id:uid=1000,gid=1000 \
+        -v $TMPD/cntr_storage:/home/user/.local/share/containers:Z \
+        -v $TMPD/context:/home/user/context:Z \
+        $FQIN $tool build -t rootless_testimage /home/user/context
+done

aio/user-containers.conf

@@ -0,0 +1,5 @@
+[containers]
+volumes = [
+	"/proc:/proc",
+]
+default_sysctls = []