From 4148c637da460e649484e3829cb24e5b660cc7f4 Mon Sep 17 00:00:00 2001 From: Allison Karlitskaya Date: Mon, 18 Mar 2024 11:06:18 +0100 Subject: [PATCH] Rewrite integration test in pytest Move it from tasks/ into test/ as it involves more than the tasks container, and could e.g. also grow integration tests for metrics. Also set up ruff and mypy static code checks, and add a pyproject.toml configuration for these. --- .github/workflows/tests.yml | 15 +- Makefile | 2 + ansible/roles/webhook/tasks/main.yml | 2 +- pyproject.toml | 64 +++ tasks/README.md | 17 +- tasks/mock-github | 2 +- tasks/run-local.sh | 615 ------------------------ test/conftest.py | 26 + test/test_deployment.py | 682 +++++++++++++++++++++++++++ 9 files changed, 796 insertions(+), 629 deletions(-) create mode 100644 pyproject.toml delete mode 100755 tasks/run-local.sh create mode 100644 test/conftest.py create mode 100644 test/test_deployment.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 85a6abd6..167510e9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,9 +10,11 @@ jobs: - name: Install dependencies run: | sudo apt-get update - sudo apt-get install -y make python3-pyflakes python3-pycodestyle + sudo apt-get install -y make python3-pyflakes python3-pycodestyle python3-pip python3-pytest + # `pip install .[test]` does not work properly on Ubuntu 22.04 + sudo pip install ruff mypy types-PyYAML - - name: Run unit tests + - name: Run lint tests run: make check tasks: @@ -35,6 +37,11 @@ jobs: git config user.email github-actions@github.com git rebase origin/main + - name: Install test dependencies + run: | + sudo apt-get update + sudo apt-get install -y make python3-pytest + # HACK: Ubuntu 22.04 has podman 3.4, which isn't compatible with podman-remote 4 in our tasks container # This PPA is a backport of podman 4.3 from Debian 12; drop this when moving `runs-on:` to ubuntu-24.04 - name: Update to newer podman @@ -57,6 +64,6 @@ jobs: - name: Test local deployment run: | - echo '${{ secrets.GITHUB_TOKEN }}' > ~/.config/github-token + echo '${{ secrets.GITHUB_TOKEN }}' > github-token PRN=$(echo "$GITHUB_REF" | cut -f3 -d '/') - tasks/run-local.sh -p $PRN -t ~/.config/github-token + python3 -m pytest -vv --pr $PRN --pr-repository '${{ github.repository }}' --github-token=github-token diff --git a/Makefile b/Makefile index 1d38f611..e62f593e 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,8 @@ all: check: python3 -m pyflakes tasks tasks/container/webhook python3 -m pycodestyle --max-line-length=120 --ignore=E722 tasks tasks/container/webhook + if command -v mypy >/dev/null && pip show types-PyYAML >/dev/null; then mypy test; else echo "SKIP: mypy or types-PyYAML not installed"; fi + if command -v ruff >/dev/null; then ruff check test; else echo "SKIP: ruff not installed"; fi TAG := $(shell date --iso-8601) TASK_SECRETS := /var/lib/cockpit-secrets/tasks diff --git a/ansible/roles/webhook/tasks/main.yml b/ansible/roles/webhook/tasks/main.yml index 2d5a2e1e..a0fb54ed 100644 --- a/ansible/roles/webhook/tasks/main.yml +++ b/ansible/roles/webhook/tasks/main.yml @@ -7,7 +7,7 @@ dest: /run/cockpit-tasks-webhook.yaml mode: preserve -# keep this in sync with tasks/run-local.sh +# keep this in sync with test/test_deployment.py - name: Generate flat files from RabbitMQ config map shell: | rm -rf /etc/rabbitmq diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..fac4ffcb --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,64 @@ +[project] +name = "cockpituous" +description = "Cockpit CI infrastructure" +classifiers = [ + "License :: OSI Approved :: GNU Lesser General Public License v3 or later (LGPLv3+)" +] + +dependencies = [ + "pyyaml", +] + +version = "1" + +[project.optional-dependencies] +test = [ + "ruff", + "mypy", + "pytest", + "types-PyYAML", +] + +[tool.setuptools] +packages = [] + +[tool.pytest.ini_options] +addopts = "-m 'not shell'" +markers = [ + "shell: interactive shell for development, skipped on normal runs", +] + +[tool.ruff] +exclude = [ + ".git/", +] +line-length = 118 +src = [] + +[tool.ruff.lint] +select = [ + "A", # flake8-builtins + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "D300", # pydocstyle: Forbid ''' in docstrings + "DTZ", # flake8-datetimez + "E", # pycodestyle + "EXE", # flake8-executable + "F", # pyflakes + "FBT", # flake8-boolean-trap + "G", # flake8-logging-format + "I", # isort + "ICN", # flake8-import-conventions + "ISC", # flake8-implicit-str-concat + "PIE", # flake8-pie + "PLE", # pylint errors + "PGH", # pygrep-hooks + "PT", # flake8-pytest-style + "RSE", # flake8-raise + "RUF", # ruff rules + "T10", # flake8-debugger + "TCH", # flake8-type-checking + "UP032", # f-string + "W", # warnings (mostly whitespace) + "YTT", # flake8-2020 +] diff --git a/tasks/README.md b/tasks/README.md index 769eec30..71f7590b 100644 --- a/tasks/README.md +++ b/tasks/README.md @@ -60,29 +60,30 @@ Some helpful commands: # Deploying locally for development, integration tests -For hacking on the webhook, task container, bots infrastructure,, or validating +For hacking on the webhook, task container, bots infrastructure, or validating new container images, you can also run a [podman pod](http://docs.podman.io/en/latest/pod.html) locally with RabbitMQ, webhook, minio S3, and tasks containers. Without arguments this will run some purely local integration tests: - tasks/run-local.sh + pytest -This will also generate the secrets in a temporary directory, unless they -already exist in `tasks/credentials/`. By default this will use the +This will also generate the secrets in a temporary directory. +By default this will use the [`ghcr.io/cockpit-project/tasks:latest`](https://ghcr.io/cockpit-project/tasks) -container, but you can run a different tag by setting `$TASKS_TAG`. +container, but you can run a different image by setting `$TASKS_IMAGE`. You can also test the whole GitHub → webhook → tasks → GitHub status workflow -on some cockpituous PR with specifying the PR number and a GitHub token: +on some cockpituous PR with specifying the PR number, your GitHub token, and +optionally a non-default repository for testing against a fork: - tasks/run-local.sh -p 123 -t ~/.config/cockpit-dev/github-token + pytest -vvsk test_real_pr --pr 123 --pr-repository yourfork/cockpituous --github-token=/home/user/.config/cockpit-dev/github-token This will run tests-scan/tests-trigger on the given PR and trigger an [unit-tests](../.cockpit-ci/run) test which simply does `make check`. You can get an interactive shell with - tasks/run-local.sh -i + pytest -sm shell to run things manually. For example, use `publish-queue` to inject a job into AMQP, or run `job-runner` or some bots command. diff --git a/tasks/mock-github b/tasks/mock-github index 1c381bca..cd8fddbf 100755 --- a/tasks/mock-github +++ b/tasks/mock-github @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # Mock GitHub API server for testing an opened PR or an issue for an image-refresh -# You can run this manually in `tasks/run-local.sh -i` with `podman cp` and running +# You can run this manually in `pytest -sm shell` with `podman cp` and running # cd bots # PYTHONPATH=. ./mock-github cockpit-project/bots $(git rev-parse HEAD) & # export GITHUB_API=http://127.0.0.7:8443 diff --git a/tasks/run-local.sh b/tasks/run-local.sh deleted file mode 100755 index 3e02cffb..00000000 --- a/tasks/run-local.sh +++ /dev/null @@ -1,615 +0,0 @@ -#!/bin/sh -# Run a local pod with a AMQP and a tasks container -# You can run against a tasks container tag different than latest by setting "$TASKS_TAG" -set -eu - -MYDIR=$(realpath $(dirname $0)) -ROOTDIR=$(dirname $MYDIR) -DATADIR=$ROOTDIR/local-data -RABBITMQ_CONFIG=$DATADIR/rabbitmq-config -SECRETS=$DATADIR/secrets -IMAGE_PORT=${IMAGE_PORT:-8080} -S3_PORT=${S3_PORT:-9000} -# S3 address from inside cockpituous pod -S3_URL_POD=https://localhost.localdomain:9000 -# S3 address from host -S3_URL_HOST=https://localhost.localdomain:$S3_PORT -# AMQP address from inside the cockpituous pod -AMQP_POD=localhost:5671 -# mock GitHub API running in tasks pod -GHAPI_URL_POD="http://127.0.0.7:8443" - -# CLI option defaults/values -PR= -PR_REPO=cockpit-project/cockpituous -TOKEN= -INTERACTIVE= - -assert_in() { - if ! echo "$2" | grep -q "$1"; then - echo "ERROR: did not find '$1' in '$2'" >&2 - exit 1 - fi -} - -assert_not_in() { - if echo "$2" | grep -q "$1"; then - echo "ERROR: unexpectedly found '$1' in '$2'" >&2 - exit 1 - fi -} - -parse_options() { - while getopts "his:t:p:r:" opt "$@"; do - case $opt in - h) - echo '-p run unit tests in the local deployment against a real PR' - echo "-r run unit tests in the local deployment against an owner/repo other than $PR_REPO" - echo '-t supply a token which will be copied into the webhook secrets' - echo "-i interactive mode: disable cockpit-tasks script, no automatic shutdown" - exit 0 - ;; - p) PR="$OPTARG" ;; - r) PR_REPO="$OPTARG" ;; - i) INTERACTIVE="1" ;; - t) - if [ ! -e "$OPTARG" ]; then - echo $OPTARG does not exist - exit 1 - fi - TOKEN="$OPTARG" - ;; - esac - done -} - -# initialize DATADIR and config files -setup_config() { - # clean up data dir from previous round - rm -rf "$DATADIR" - - # generate flat files from RabbitMQ config map - mkdir -p $RABBITMQ_CONFIG - python3 - < webhook/.config--github-token - - # minio S3 key - mkdir tasks/s3-keys - echo 'cockpituous foobarfoo' > tasks/s3-keys/localhost.localdomain - ) - - # start podman API - systemctl $([ $(id -u) -eq 0 ] || echo "--user") start podman.socket - - # need to make files world-readable, as containers run as different user 1111 - chmod -R go+rX "$SECRETS" - # for the same reason, make podman socket accessible to that container user - # the directory is only accessible for the user, so 666 permissions don't hurt - chmod o+rw ${XDG_RUNTIME_DIR:-/run}/podman/podman.sock - fi -} - -create_job_runner_config() { - # we never want to push to real GitHub branches in this test - if [ "$1" = "mock" ]; then - forge_opts="api-url = '$GHAPI_URL_POD'" - # needs to run in pod network so that it can access GITHUB_API_POD - run_args="'--pod=cockpituous', '--env=GITHUB_API=$GHAPI_URL_POD'" - elif [ "$1" = "real" ]; then - forge_opts="" - run_args="" - else - echo "ERROR: unknown job-runner config $1" >&2 - exit 1 - fi - - cat < $SECRETS/tasks/job-runner.toml -[logs] -driver='s3' - -[forge.github] -token = [{file="/run/secrets/webhook/.config--github-token"}] -$forge_opts - -[logs.s3] -url = '$S3_URL_POD/logs' -ca = [{file='/run/secrets/webhook/ca.pem'}] -key = [{file="/run/secrets/tasks/s3-keys/localhost.localdomain"}] - -[container] -command = ['podman-remote', '--url=unix:///podman.sock'] -run-args = [ - '--security-opt=label=disable', - '--volume=$MYDIR/mock-git-push:/usr/local/bin/git:ro', - '--env=COCKPIT_IMAGE_UPLOAD_STORE=$S3_URL_POD/images/', - '--env=GIT_AUTHOR_*', - '--env=GIT_COMMITTER_*', - $run_args -] - -[container.secrets] -# these are *host* paths, this is podman-remote -image-upload=[ - '--volume=$SECRETS/tasks/s3-keys:/run/secrets/s3-keys:ro', - '--env=COCKPIT_S3_KEY_DIR=/run/secrets/s3-keys', - '--volume=$SECRETS/webhook/ca.pem:/run/secrets/ca.pem:ro', - '--env=COCKPIT_CA_PEM=/run/secrets/ca.pem', -] -github-token=[ - '--volume=$SECRETS/webhook/.config--github-token:/run/secrets/github-token:ro', - '--env=COCKPIT_GITHUB_TOKEN_FILE=/run/secrets/github-token', -] -EOF -} - -launch_containers() { - cleanup() { - if [ $? -ne 0 ] && [ -z "$INTERACTIVE" ] && [ -t 0 ]; then - echo "Test failure; investigate, and press Enter to shut down" - read - fi - podman pod rm -f cockpituous - } - - trap cleanup EXIT INT QUIT PIPE - - # start podman and run RabbitMQ in the background - podman run -d --name cockpituous-rabbitmq --pod=new:cockpituous \ - --security-opt=label=disable \ - --publish $IMAGE_PORT:8080 \ - --publish $S3_PORT:9000 \ - --publish 9001:9001 \ - -v "$RABBITMQ_CONFIG":/etc/rabbitmq:ro \ - -v "$SECRETS"/webhook:/run/secrets/webhook:ro \ - docker.io/rabbitmq - - # S3 - local admin_password="$(dd if=/dev/urandom bs=10 count=1 status=none | base64)" - podman run -d --name cockpituous-s3 --pod=cockpituous \ - --security-opt=label=disable \ - -e MINIO_ROOT_USER="minioadmin" \ - -e MINIO_ROOT_PASSWORD="$admin_password" \ - -v "$SECRETS"/tasks/s3-server.key:/root/.minio/certs/private.key:ro \ - -v "$SECRETS"/tasks/s3-server.pem:/root/.minio/certs/public.crt:ro \ - quay.io/minio/minio server /data --console-address :9001 - # wait until it started, create bucket - podman run -d --interactive --name cockpituous-mc --pod=cockpituous \ - --security-opt=label=disable \ - -v "$SECRETS"/ca.pem:/etc/pki/ca-trust/source/anchors/ca.pem:ro \ - --entrypoint /bin/sh quay.io/minio/mc - read s3user s3key < "$SECRETS/tasks/s3-keys/localhost.localdomain" - podman exec -i cockpituous-mc /bin/sh <> /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem -until mc alias set minio '$S3_URL_HOST' minioadmin '$admin_password'; do sleep 1; done -mc mb minio/images -mc mb minio/logs -mc anonymous set download minio/images -mc anonymous set download minio/logs -mc admin user add minio/ $s3user $s3key -mc admin policy attach minio/ readwrite --user $s3user -EOF - unset s3key - - # scanning actual cockpit PRs interferes with automatic tests; but do this in interactive mode to have a complete deployment - if [ -n "$INTERACTIVE" ]; then - [ -z "$TOKEN" ] || cp -fv "$TOKEN" "$SECRETS"/webhook/.config--github-token - podman run -d --name cockpituous-webhook --pod=cockpituous --user user \ - --security-opt=label=disable \ - -v "$SECRETS"/webhook:/run/secrets/webhook:ro \ - --env=AMQP_SERVER=$AMQP_POD \ - --env=COCKPIT_GITHUB_TOKEN_FILE=/run/secrets/webhook/.config--github-token \ - --env=COCKPIT_GITHUB_WEBHOOK_TOKEN_FILE=/run/secrets/webhook/.config--github-webhook-token \ - ghcr.io/cockpit-project/tasks:${TASKS_TAG:-latest} webhook - fi - - # wait until AMQP initialized - sleep 5 - until podman exec -i cockpituous-rabbitmq timeout 5 rabbitmqctl list_queues; do - echo "waiting for RabbitMQ to come up..." - sleep 3 - done - - # Run tasks container in the background - # use bash as pid 1 to mop up zombies - # we always want to upload images to our local S3 store - podman run -d -it --name cockpituous-tasks --pod=cockpituous \ - --security-opt=label=disable \ - -v "$SECRETS"/tasks:/run/secrets/tasks:ro \ - -v "$SECRETS"/webhook:/run/secrets/webhook:ro \ - -v "${XDG_RUNTIME_DIR:-/run}/podman/podman.sock:/podman.sock" \ - --env=COCKPIT_GITHUB_TOKEN_FILE=/run/secrets/webhook/.config--github-token \ - --env=COCKPIT_CA_PEM=/run/secrets/webhook/ca.pem \ - --env=COCKPIT_BOTS_REPO=${COCKPIT_BOTS_REPO:-} \ - --env=COCKPIT_BOTS_BRANCH=${COCKPIT_BOTS_BRANCH:-} \ - --env=COCKPIT_TESTMAP_INJECT=main/unit-tests \ - --env=JOB_RUNNER_CONFIG=/run/secrets/tasks/job-runner.toml \ - --env=AMQP_SERVER=$AMQP_POD \ - --env=S3_LOGS_URL=$S3_URL_POD/logs/ \ - --env=COCKPIT_S3_KEY_DIR=/run/secrets/tasks/s3-keys \ - --env=COCKPIT_IMAGE_UPLOAD_STORE=$S3_URL_POD/images/ \ - --env=COCKPIT_IMAGES_DATA_DIR=/cache/images \ - --env=GIT_COMMITTER_NAME=Cockpituous \ - --env=GIT_COMMITTER_EMAIL=cockpituous@cockpit-project.org \ - --env=GIT_AUTHOR_NAME=Cockpituous \ - --env=GIT_AUTHOR_EMAIL=cockpituous@cockpit-project.org \ - --env=SKIP_STATIC_CHECK=1 \ - ghcr.io/cockpit-project/tasks:${TASKS_TAG:-latest} bash - - # check out the correct bots, as part of what cockpit-tasks would usually do - podman exec -i cockpituous-tasks sh -euc \ - 'git clone --quiet --depth=1 -b "${COCKPIT_BOTS_BRANCH:-main}" "${COCKPIT_BOTS_REPO:-https://github.com/cockpit-project/bots}"' -} - -cleanup_containers() { - echo "Cleaning up..." - - # clean up token, so that image-prune does not try to use it - rm "$SECRETS"/webhook/.config--github-token - - # revert podman socket permission change - chmod o-rw ${XDG_RUNTIME_DIR:-run}/podman/podman.sock - - podman stop --time=0 cockpituous-tasks -} - -test_image() { - # test image upload - podman exec -i cockpituous-tasks timeout 30 sh -euxc ' - cd bots - - # fake an image - echo world > /cache/images/testimage - NAME="testimage-$(sha256sum /cache/images/testimage | cut -f1 -d\ ).qcow2" - mv /cache/images/testimage /cache/images/$NAME - ln -s $NAME images/testimage - - # test image-upload to S3 - ./image-upload --store '$S3_URL_POD'/images/ testimage - # S3 store received this - python3 -m lib.s3 ls '$S3_URL_POD'/images/ | grep -q "testimage.*qcow" - ' - - # validate image downloading from S3 - podman exec -i cockpituous-tasks sh -euxc ' - rm --verbose /cache/images/testimage* - cd bots - ./image-download --store '$S3_URL_POD'/images/ testimage - grep -q "^world" /cache/images/testimage-*.qcow2 - rm --verbose /cache/images/testimage* - ' - - # validate image pruning on s3 - podman exec -i cockpituous-tasks sh -euxc ' - cd bots - rm images/testimage - ./image-prune --s3 '$S3_URL_POD'/images/ --force --checkout-only - # S3 store removed it - [ -z "$(python3 -m lib.s3 ls '$S3_URL_POD'/images/ | grep testimage)" ] - ' -} - -test_mock_pr() { - podman cp "$MYDIR/mock-github" cockpituous-tasks:/work/bots/mock-github - create_job_runner_config mock - - # test mock PR against our checkout, so that cloning will work - SHA=$(podman exec -i cockpituous-tasks git -C bots rev-parse HEAD) - - podman exec -i cockpituous-tasks sh -euxc " - cd bots - - # start mock GH server - PYTHONPATH=. ./mock-github --log /tmp/mock.log cockpit-project/bots $SHA & - GH_MOCK_PID=\$! - export GITHUB_API=$GHAPI_URL_POD - until curl --silent --fail \$GITHUB_API/repos/cockpit-project/bots; do sleep 0.1; done - - # simulate GitHub webhook event, put that into the webhook queue - PYTHONPATH=. ./mock-github --print-pr-event cockpit-project/bots $SHA | \ - ./publish-queue --amqp $AMQP_POD --create --queue webhook - - ./inspect-queue --amqp $AMQP_POD - - # first run-queue processes webhook → tests-scan → public queue - ./run-queue --amqp $AMQP_POD - ./inspect-queue --amqp $AMQP_POD - - # second run-queue actually runs the test - ./run-queue --amqp $AMQP_POD - - kill \$GH_MOCK_PID - " - - LOGS_URL="$S3_URL_HOST/logs/" - CURL="curl --cacert $SECRETS/ca.pem --silent --fail --show-error" - LOG_MATCH="$($CURL $LOGS_URL| grep -o "pull-1-[[:alnum:]-]*-unit-tests/log<")" - LOG="$($CURL "${LOGS_URL}${LOG_MATCH%<}")" - echo "--------------- mock PR test log -----------------" - echo "$LOG" - echo "--------------- mock PR test log end -------------" - assert_in 'Test run finished, return code: 0\|Job ran successfully' "$LOG" - assert_in 'Running on:.*cockpituous' "$LOG" - - # 3 status updates posted - # FIXME: assert JSON more precisely once we rewrite in Python - GH_MOCK_LOG="$(podman exec cockpituous-tasks cat /tmp/mock.log)" - assert_in "POST /repos/cockpit-project/bots/statuses/$SHA .*description.*Not yet tested" "$GH_MOCK_LOG" - assert_in "POST /repos/cockpit-project/bots/statuses/$SHA .*description.*In progress \\[cockpituous\\]" "$GH_MOCK_LOG" - assert_in "POST /repos/cockpit-project/bots/statuses/$SHA .*state.*success" "$GH_MOCK_LOG" -} - -test_mock_cross_project_pr() { - podman cp "$MYDIR/mock-github" cockpituous-tasks:/work/bots/mock-github - create_job_runner_config mock - - # test mock PR against our checkout, so that cloning will work - SHA=$(podman exec -i cockpituous-tasks git -C bots rev-parse HEAD) - - podman exec -i cockpituous-tasks sh -euxc " - cd bots - - # start mock GH server - PYTHONPATH=. ./mock-github --log /tmp/mock.log cockpit-project/bots $SHA & - GH_MOCK_PID=\$! - export GITHUB_API=$GHAPI_URL_POD - until curl --silent --fail \$GITHUB_API/repos/cockpit-project/bots; do sleep 0.1; done - - # simulate GitHub webhook event, put that into the webhook queue - PYTHONPATH=. ./mock-github --print-pr-event cockpit-project/bots $SHA | \ - ./publish-queue --amqp $AMQP_POD --create --queue webhook - - ./inspect-queue --amqp $AMQP_POD - - # cross-project test request - export COCKPIT_TESTMAP_INJECT=main/unit-tests@cockpit-project/cockpituous - - # first run-queue processes webhook → tests-scan → public queue - ./run-queue --amqp $AMQP_POD - ./inspect-queue --amqp $AMQP_POD - - # second run-queue actually runs the test - ./run-queue --amqp $AMQP_POD - - kill \$GH_MOCK_PID - " - set -x - - LOGS_URL="$S3_URL_HOST/logs/" - CURL="curl --cacert $SECRETS/ca.pem --silent --fail --show-error" - LOG_MATCH="$($CURL $LOGS_URL| grep -o "pull-1-[[:alnum:]-]*-unit-tests-cockpit-project-cockpituous/log<")" - LOG_URL="${LOGS_URL}${LOG_MATCH%<}" - LOG="$($CURL "$LOG_URL")" - echo "--------------- mock PR test log -----------------" - echo "$LOG" - echo "--------------- mock PR test log end -------------" - assert_in 'Test run finished, return code: 0\|Job ran successfully' "$LOG" - assert_in 'Running on:.*cockpituous' "$LOG" - - # validate test attachment - BOGUS_LOG=$($CURL "${LOG_URL%/log}/bogus.log") - assert_in 'heisenberg compensator' "$BOGUS_LOG" - - # 3 status updates posted to bots project (the PR we are testing) - # FIXME: assert JSON more precisely once we rewrite in Python - GH_MOCK_LOG="$(podman exec cockpituous-tasks cat /tmp/mock.log)" - assert_in "POST /repos/cockpit-project/bots/statuses/$SHA .*description.*Not yet tested" "$GH_MOCK_LOG" - # old tests-invoke says "Testing in progress", job-runner says "In progress" - assert_in "POST /repos/cockpit-project/bots/statuses/$SHA .*description.*[iI]n progress \\[cockpituous\\]" "$GH_MOCK_LOG" - assert_in "POST /repos/cockpit-project/bots/statuses/$SHA .*state.*success" "$GH_MOCK_LOG" - - # did not post status to cockpituous - assert_not_in "POST /repos/cockpit-project/cockpituous/statuses" "$GH_MOCK_LOG" -} - -test_pr() { - # need to use real GitHub token for this - [ -z "$TOKEN" ] || cp -fv "$TOKEN" "$SECRETS"/webhook/.config--github-token - create_job_runner_config real - - # run the main loop in the background; we could do this with a single run-queue invocation, - # but we want to test the cockpit-tasks script - podman exec -i --env=SLUMBER=1 cockpituous-tasks cockpit-tasks & - TASKS_PID=$! - - podman exec -i cockpituous-tasks sh -euxc " - cd bots - - ./tests-scan -p $PR --amqp '$AMQP_POD' --repo $PR_REPO; - for retry in \$(seq 10); do - ./tests-scan --repo $PR_REPO --human-readable --dry; - OUT=\$(./tests-scan --repo $PR_REPO -p $PR --human-readable --dry); - [ \"\${OUT%unit-tests*}\" = \"\$OUT\" ] || break; - echo waiting until the status is visible; - sleep 10; - done; - ./inspect-queue --amqp $AMQP_POD;" - - LOGS_URL="$S3_URL_HOST/logs/" - CURL="curl --cacert $SECRETS/ca.pem --silent --fail --show-error" - - # wait until the unit-test got run and published, i.e. until the non-chunked raw log file exists - for retry in $(seq 60); do - LOG_MATCH="$($CURL $LOGS_URL| grep -o "pull-${PR}-[[:alnum:]-]*-unit-tests/log<")" && break - echo waiting for unit-tests run to finish... - sleep 10 - done - - # tell the tasks container iteration that we are done - kill -TERM $TASKS_PID - wait $TASKS_PID || true - - LOG_PATH="${LOG_MATCH%<}" - - # spot-checks that it produced sensible logs in S3 - LOG_URL="$LOGS_URL$LOG_PATH" - LOG="$($CURL $LOG_URL)" - LOG_HTML="$($CURL ${LOG_URL}.html)" - echo "--------------- test log -----------------" - echo "$LOG" - echo "--------------- test log end -------------" - assert_in '' "$LOG_HTML" - assert_in 'Running on:.*cockpituous' "$LOG" - assert_in 'Test run finished, return code: 0\|Job ran successfully' "$LOG" - # validate test attachment if we ran cockpituous' own tests - if [ "${PR_REPO%/cockpituous}" != "$PR_REPO" ]; then - BOGUS_LOG=$($CURL ${LOG_URL%/log}/bogus.log) - assert_in 'heisenberg compensator' "$BOGUS_LOG" - BOGUS_SUBDIR_FILE=$($CURL ${LOG_URL%/log}/data/subdir-file.txt) - assert_in 'subdir-file' "$BOGUS_SUBDIR_FILE" - fi -} - -test_mock_image_refresh() { - podman cp "$MYDIR/mock-github" cockpituous-tasks:/work/bots/mock-github - podman cp "$MYDIR/mock-git-push" cockpituous-tasks:/usr/local/bin/git - create_job_runner_config mock - - # test mock PR against our checkout, so that cloning will work - SHA=$(podman exec -i cockpituous-tasks git -C bots rev-parse HEAD) - - podman exec -i cockpituous-tasks sh -euxc " - cd bots - # start mock GH server - PYTHONPATH=. ./mock-github --log /tmp/mock.log cockpit-project/bots $SHA & - GH_MOCK_PID=\$! - export GITHUB_API=$GHAPI_URL_POD - until curl --silent --fail \$GITHUB_API/repos/cockpit-project/bots; do sleep 0.1; done - - # simulate GitHub webhook event, put that into the webhook queue - PYTHONPATH=. ./mock-github --print-image-refresh-event cockpit-project/bots $SHA | \ - ./publish-queue --amqp $AMQP_POD --create --queue webhook - - ./inspect-queue --amqp $AMQP_POD - - # first run-queue processes webhook → issue-scan → public queue - ./run-queue --amqp $AMQP_POD - ./inspect-queue --amqp $AMQP_POD - - # second run-queue actually runs the image refresh - ./run-queue --amqp $AMQP_POD - - kill \$GH_MOCK_PID - " - - # successful refresh log - LOGS_URL="$S3_URL_HOST/logs/" - CURL="curl --cacert $SECRETS/ca.pem --silent --fail --show-error" - LOG_MATCH="$($CURL $LOGS_URL| grep -o "image-refresh-foonux-[[:alnum:]-]*/log<")" - LOG="$($CURL "${LOGS_URL}${LOG_MATCH%<}")" - echo "--------------- mock image-refresh test log -----------------" - echo "$LOG" - echo "--------------- mock image-refresh test log end -------------" - assert_in 'Running on:.*cockpituous' "$LOG" - assert_in './image-create.*foonux' "$LOG" - assert_in "Uploading to $S3_URL_POD/images/foonux.*qcow2" "$LOG" - assert_in 'Success.' "$LOG" - - # branch was (mock) pushed - PUSH_LOG_MATCH="$($CURL $LOGS_URL| grep -o "image-refresh-foonux-[[:alnum:]-]*/git-push.log<")" - PUSH_LOG="$($CURL "${LOGS_URL}${PUSH_LOG_MATCH%<}")" - assert_in 'push origin +HEAD:refs/heads/image-refresh-foonux-' "$PUSH_LOG" - podman exec -i -u root cockpituous-tasks rm /usr/local/bin/git - - podman exec -i cockpituous-tasks sh -euxc ' - # image is on the S3 server - cd bots - name=$(python3 -m lib.s3 ls '$S3_URL_POD'/images/ | grep -o "foonux.*qcow2") - - # download image (it was not pushed to git, so need to use --state) - rm -f /cache/images/foonux* - ./image-download --store $COCKPIT_IMAGE_UPLOAD_STORE --state "$name" - - # validate image contents - qemu-img convert /cache/images/foonux-*.qcow2 /tmp/foonux.raw - grep "^fakeimage" /tmp/foonux.raw - rm /tmp/foonux.raw - ' - - # status updates posted to original bots SHA on which the image got triggered - # FIXME: assert JSON more precisely once we rewrite in Python (unpredictable JSON order) - GH_MOCK_LOG="$(podman exec cockpituous-tasks cat /tmp/mock.log)" - assert_in "POST /repos/cockpit-project/bots/statuses/$SHA .*image-refresh/foonux" "$GH_MOCK_LOG" - assert_in "POST /repos/cockpit-project/bots/statuses/$SHA .*In progress \\[cockpituous\\]" "$GH_MOCK_LOG" - assert_in "POST /repos/cockpit-project/bots/statuses/$SHA .*success" "$GH_MOCK_LOG" - - # and forwarded to the converted PR (new SHA) - assert_in "POST /repos/cockpit-project/bots/statuses/a1b2c3 .*success.*Forwarded status.*target_url" "$GH_MOCK_LOG" - - # posts new comment with log - assert_in "POST /repos/cockpit-project/bots/issues/2/comments .*Success. Log: https.*" "$GH_MOCK_LOG" - -} - -test_queue() { - # tasks can connect to queue - OUT=$(podman exec -i cockpituous-tasks bots/inspect-queue --amqp $AMQP_POD) - echo "$OUT" | grep -q 'queue public does not exist' -} - -test_podman() { - # tasks can connect to host's podman service - # this will be covered implicitly by job-runner, but as a more basal plumbing test this is easier to debug - out="$(podman exec -i cockpituous-tasks podman-remote --url unix:///podman.sock ps)" - assert_in 'cockpituous-tasks' "$out" - out="$(podman exec -i cockpituous-tasks podman-remote --url unix:///podman.sock run -it --rm ghcr.io/cockpit-project/tasks:latest whoami)" - assert_in '^user' "$out" -} - -# -# main -# - -parse_options "$@" -setup_config -launch_containers - -# Follow the output -podman logs -f cockpituous-tasks & - -if [ -n "$INTERACTIVE" ]; then - echo "Starting a tasks container shell; exit it to clean up the deployment" - podman exec -it cockpituous-tasks bash -else - # tests which don't need GitHub interaction - test_image - test_queue - test_podman - # "almost" end-to-end, starting with GitHub webhook JSON payload injection; fully localy, no privs - test_mock_pr - test_mock_cross_project_pr - # similar structure for issue-scan for an image refresh - test_mock_image_refresh - # if we have a PR number, run a unit test inside local deployment, and update PR status - [ -z "$PR" ] || test_pr -fi - -cleanup_containers -# bring logs -f to the foreground -wait diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 00000000..98f5f731 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,26 @@ +import os +import subprocess +import sys + + +def pytest_addoption(parser): + parser.addoption('--github-token', metavar='PATH', + help='path to real GitHub token, for testing real PR or shell mode') + parser.addoption('--pr', metavar='NUMBER', type=int, + help='run unit tests in the local deployment against a real PR') + parser.addoption('--pr-repository', metavar='OWNER/REPO', default='cockpit-project/cockpituous', + help='run --pr against owner/repo other than %(default)s') + + +def pytest_exception_interact(node, call, report): + if report.failed: + if os.isatty(0): + print('Test failure; investigate, and press Enter to shut down') + input() + else: + print('\n\n---------- cockpit-tasks log ------------') + sys.stdout.flush() + subprocess.run(['podman', 'exec', '-i', f'cockpituous-tasks-{os.getpid()}', + 'cat', '/tmp/cockpit-tasks.log']) + print('-----------------------------------------') + sys.stdout.flush() diff --git a/test/test_deployment.py b/test/test_deployment.py new file mode 100644 index 00000000..64c7ea45 --- /dev/null +++ b/test/test_deployment.py @@ -0,0 +1,682 @@ +import json +import os +import re +import shutil +import ssl +import subprocess +import textwrap +import time +import urllib.request +from pathlib import Path +from typing import Generator + +import pytest +import yaml + +TASKS_IMAGE = os.environ.get('TASKS_IMAGE', 'ghcr.io/cockpit-project/tasks:latest') +ROOT_DIR = Path(__file__).parent.parent +PODMAN_SOCKET = Path(os.getenv('XDG_RUNTIME_DIR', '/run'), 'podman', 'podman.sock') +# AMQP address from inside the cockpituous pod +AMQP_POD = 'localhost:5671' +# S3 address from inside cockpituous pod +S3_URL_POD = 'https://localhost.localdomain:9000' +# mock GitHub API running in tasks pod +GHAPI_URL_POD = 'http://127.0.0.7:8443' + + +# +# Deployment configuration and secrets (global, session scope) +# + +class Config: + rabbitmq: Path + secrets: Path + webhook: Path + tasks: Path + + +@pytest.fixture(scope='session') +def config(tmp_path_factory) -> Config: + configdir = tmp_path_factory.mktemp('config') + config = Config() + + # generate flat files from RabbitMQ config map; keep in sync with ansible/roles/webhook/tasks/main.yml + config.rabbitmq = configdir / 'rabbitmq-config' + config.rabbitmq.mkdir(parents=True) + + for doc in yaml.full_load_all((ROOT_DIR / 'tasks/cockpit-tasks-webhook.yaml').read_text()): + if doc['metadata']['name'] == 'amqp-config': + files = doc['data'] + for name, contents in files.items(): + (config.rabbitmq / name).write_text(contents) + break + else: + raise ValueError('amqp-config not found in the webhook task') + + config.secrets = configdir / 'secrets' + + # webhook secrets + os.makedirs(config.secrets) + subprocess.run(ROOT_DIR / 'tasks/credentials/generate-ca.sh', cwd=config.secrets, check=True) + config.webhook = config.secrets / 'webhook' + config.webhook.mkdir() + subprocess.run(ROOT_DIR / 'tasks/credentials/webhook/generate.sh', cwd=config.webhook, check=True) + + # default to dummy token, tests need to opt into real one with user_github_token + (config.webhook / '.config--github-token').write_text('0123abc') + + # tasks secrets + config.tasks = config.secrets / 'tasks' + config.tasks.mkdir() + subprocess.run(ROOT_DIR / 'local-s3/generate-s3-cert.sh', cwd=config.tasks, check=True) + # minio S3 key + (config.tasks / 's3-keys').mkdir() + (config.tasks / 's3-keys/localhost.localdomain').write_text('cockpituous foobarfoo') + + # need to make secrets world-readable, as containers run as non-root + subprocess.run(['chmod', '-R', 'go+rX', configdir], check=True) + + # start podman API + user_opt = [] if os.geteuid() == 0 else ['--user'] + subprocess.run(['systemctl', *user_opt, 'start', 'podman.socket'], check=True) + # make podman socket accessible to the container user + # the socket's directory is only accessible for the user, so 666 permissions don't hurt + PODMAN_SOCKET.chmod(0o666) + + return config + + +@pytest.fixture() +def user_github_token(config: Config, request) -> None: + if request.config.getoption('github_token'): + shutil.copy(request.config.getoption('--github-token'), config.webhook / '.config--github-token') + return None # silence ruff PT004 + + +# +# Container deployment +# + +class PodData: + pod: str + # container names + rabbitmq: str + s3: str + mc: str + tasks: str + webhook: str | None # only in "shell" marker + # forwarded ports + host_port_s3: int + + +@pytest.fixture(scope='session') +def pod(config: Config, pytestconfig) -> Generator[PodData, None, None]: + """Deployment pod definition""" + + launch_args = ['--stop-timeout=0', '--security-opt=label=disable'] + + # we want to have useful pod/container names for interactive debugging and log dumping, but still allow + # parallel tests (with e.g. xdist), so disambiguate them with the pid + test_instance = str(os.getpid()) + data = PodData() + data.pod = f'cockpituous-{test_instance}' + + # RabbitMQ, also defines/starts pod + data.rabbitmq = f'cockpituous-rabbitmq-{test_instance}' + subprocess.run(['podman', 'run', '-d', '--name', data.rabbitmq, f'--pod=new:{data.pod}', *launch_args, + # you can set 9000:9000 to make S3 log URLs work; but it breaks parallel tests + '--publish', '9000', + '-v', f'{config.rabbitmq}:/etc/rabbitmq:ro', + '-v', f'{config.webhook}:/run/secrets/webhook:ro', + 'docker.io/rabbitmq'], + check=True) + + # minio S3 store + data.s3 = f'cockpituous-s3-{test_instance}' + subprocess.run(['podman', 'run', '-d', '--name', data.s3, f'--pod={data.pod}', *launch_args, + '-v', f'{config.tasks}/s3-server.key:/root/.minio/certs/private.key:ro', + '-v', f'{config.tasks}/s3-server.pem:/root/.minio/certs/public.crt:ro', + '-e', 'MINIO_ROOT_USER=minioadmin', + '-e', 'MINIO_ROOT_PASSWORD=minioadmin', + 'quay.io/minio/minio', 'server', '/data', '--console-address', ':9001'], + check=True) + + proc = subprocess.run(['podman', 'port', data.s3, '9000'], capture_output=True, text=True, check=True) + # looks like "0.0.0.0:12345" + data.host_port_s3 = int(proc.stdout.strip().split(':')[-1]) + + # minio S3 console + data.mc = f'cockpituous-mc-{test_instance}' + subprocess.run(['podman', 'run', '-d', '--interactive', '--name', data.mc, f'--pod={data.pod}', *launch_args, + '--entrypoint', '/bin/sh', + '-v', f'{config.secrets}/ca.pem:/etc/pki/ca-trust/source/anchors/ca.pem:ro', + 'quay.io/minio/mc'], + check=True) + + # wait until S3 started, create bucket + (s3user, s3key) = (config.tasks / 's3-keys/localhost.localdomain').read_text().strip().split() + exec_c(data.mc, f''' +set -e +cat /etc/pki/ca-trust/source/anchors/ca.pem >> /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem +until mc alias set minio '{S3_URL_POD}' minioadmin minioadmin; do sleep 1; done +mc mb minio/images +mc mb minio/logs +mc anonymous set download minio/images +mc anonymous set download minio/logs +mc admin user add minio/ {s3user} {s3key} +mc admin policy attach minio/ readwrite --user {s3user} +''') + + # tasks + data.tasks = f'cockpituous-tasks-{test_instance}' + subprocess.run(['podman', 'run', '-d', '--interactive', '--name', data.tasks, f'--pod={data.pod}', *launch_args, + '-v', f'{PODMAN_SOCKET}:/podman.sock', + '-v', f'{config.webhook}:/run/secrets/webhook:ro', + '-v', f'{config.tasks}:/run/secrets/tasks:ro', + '-e', 'COCKPIT_GITHUB_TOKEN_FILE=/run/secrets/webhook/.config--github-token', + '-e', 'COCKPIT_CA_PEM=/run/secrets/webhook/ca.pem', + '-e', f'COCKPIT_BOTS_REPO={os.getenv("COCKPIT_BOTS_REPO", "")}', + '-e', f'COCKPIT_BOTS_BRANCH={os.getenv("COCKPIT_BOTS_BRANCH", "")}', + '-e', 'COCKPIT_TESTMAP_INJECT=main/unit-tests', + '-e', 'JOB_RUNNER_CONFIG=/run/secrets/tasks/job-runner.toml', + '-e', f'AMQP_SERVER={AMQP_POD}', + '-e', f'S3_LOGS_URL={S3_URL_POD}/logs/', + '-e', 'COCKPIT_S3_KEY_DIR=/run/secrets/tasks/s3-keys', + '-e', f'COCKPIT_IMAGE_UPLOAD_STORE={S3_URL_POD}/images/', + '-e', 'COCKPIT_IMAGES_DATA_DIR=/cache/images', + '-e', 'GIT_COMMITTER_NAME=Cockpituous', + '-e', 'GIT_COMMITTER_EMAIL=cockpituous@cockpit-project.org', + '-e', 'GIT_AUTHOR_NAME=Cockpituous', + '-e', 'GIT_AUTHOR_EMAIL=cockpituous@cockpit-project.org', + '-e', 'SLUMBER=1', + TASKS_IMAGE], + check=True) + + # check out the correct bots, as part of what cockpit-tasks would usually do + exec_c(data.tasks, + f'git clone --quiet --depth=1 -b {os.getenv("COCKPIT_BOTS_BRANCH", "main")}' + f' {os.getenv("COCKPIT_BOTS_REPO", "https://github.com/cockpit-project/bots")} bots') + + # scanning/queueing actual cockpit PRs interferes with automatic tests; but do this in + # shell mode to have a complete deployment + if 'not shell' not in pytestconfig.option.markexpr: + data.webhook = f'cockpituous-webhook-{test_instance}' + subprocess.run(['podman', 'run', '-d', '--name', data.webhook, f'--pod={data.pod}', *launch_args, + '-v', f'{config.webhook}:/run/secrets/webhook:ro', + '-e', f'AMQP_SERVER={AMQP_POD}', + '-e', 'COCKPIT_GITHUB_TOKEN_FILE=/run/secrets/webhook/.config--github-token', + '-e', 'COCKPIT_GITHUB_WEBHOOK_TOKEN_FILE=/run/secrets/webhook/.config--github-webhook-token', + TASKS_IMAGE, 'webhook'], + check=True) + + # wait until RabbitMQ initialized + exec_c(data.tasks, f'until bots/inspect-queue --amqp {AMQP_POD}; do sleep 1; done') + + yield data + + subprocess.run(['podman', 'pod', 'rm', '-f', data.pod], check=True) + + +# +# Utilities +# + +def exec_c(container: str, command: str, timeout=None, *, capture: bool = False, + exec_opts: list[str] | None = None) -> str | None: + """Run shell command in a container + + Assert that it succeeds. + Return stdout if capture is True. stderr is not captured. + + Default timeout is 5s. + """ + res = subprocess.run( + ['podman', 'exec', '-i', *(exec_opts or []), container, 'sh', '-ec', command], + check=True, + stdout=subprocess.PIPE if capture else None, + timeout=5 if timeout is None else timeout) + + return res.stdout.decode() if capture else None + + +def exec_c_out(container: str, command: str, timeout=None) -> str: + """Run shell command in a container + + Assert that it succeeds. + Return stdout. stderr is not captured. + + Default timeout is 5s. + """ + out = exec_c(container, command, timeout, capture=True) + assert out is not None # mypy + return out + + +def get_s3(config: Config, pod: PodData, path: str) -> str: + """Return the content of an S3 object""" + + s3_base = f'https://localhost.localdomain:{pod.host_port_s3}' + + context = ssl.create_default_context(cafile=config.secrets / 'ca.pem') + with urllib.request.urlopen(f'{s3_base}/{path}', context=context) as f: + return f.read().decode() + + +# +# Per test fixtures +# + +@pytest.fixture() +def bots_sha(pod: PodData) -> str: + """Return the SHA of the bots checkout""" + + return exec_c_out(pod.tasks, 'git -C bots rev-parse HEAD').strip() + + +@pytest.fixture() +def mock_github(pod: PodData, bots_sha: str) -> Generator[str, None, None]: + """Start mock GitHub API server + + Return environment shell command for using it + """ + subprocess.run(['podman', 'cp', str(ROOT_DIR / 'tasks/mock-github'), f'{pod.tasks}:/work/bots/mock-github'], + check=True) + mock_pid = exec_c_out(pod.tasks, f''' + cd bots + PYTHONPATH=. nohup ./mock-github --log /tmp/mock.log cockpit-project/bots {bots_sha} \ + /tmp/mock.out 2>&1 & + echo $!''').strip() + # wait until it started + exec_c_out(pod.tasks, f'curl --retry 5 --retry-connrefused --fail {GHAPI_URL_POD}/repos/cockpit-project/bots') + + yield f'export GITHUB_API={GHAPI_URL_POD}; SHA={bots_sha}' + + exec_c_out(pod.tasks, f'while kill {mock_pid}; do sleep 0.1; done') + + +@pytest.fixture() +def mock_git_push(pod: PodData) -> Generator[None, None, None]: + """Install tasks/mock-git-push""" + + subprocess.run(['podman', 'cp', str(ROOT_DIR / 'tasks/mock-git-push'), f'{pod.tasks}:/usr/local/bin/git'], + check=True) + + yield None + + exec_c(pod.tasks, 'rm /usr/local/bin/git', exec_opts=['--user', 'root']) + + +def generate_config(config: Config, forge_opts: str, run_args: str) -> None: + conf = textwrap.dedent(f'''\ + [logs] + driver='s3' + + [forge.github] + token = [{{file="/run/secrets/webhook/.config--github-token"}}] + {forge_opts} + + [logs.s3] + url = '{S3_URL_POD}/logs' + ca = [{{file='/run/secrets/webhook/ca.pem'}}] + key = [{{file="/run/secrets/tasks/s3-keys/localhost.localdomain"}}] + + [container] + command = ['podman-remote', '--url=unix:///podman.sock'] + run-args = [ + '--security-opt=label=disable', + '--volume={ROOT_DIR / 'tasks'}/mock-git-push:/usr/local/bin/git:ro', + '--env=COCKPIT_IMAGE_UPLOAD_STORE={S3_URL_POD}/images/', + '--env=GIT_AUTHOR_*', + '--env=GIT_COMMITTER_*', + {run_args} + ] + + [container.secrets] + # these are *host* paths, this is podman-remote + image-upload=[ + '--volume={config.tasks}/s3-keys:/run/secrets/s3-keys:ro', + '--env=COCKPIT_S3_KEY_DIR=/run/secrets/s3-keys', + '--volume={config.webhook}/ca.pem:/run/secrets/ca.pem:ro', + '--env=COCKPIT_CA_PEM=/run/secrets/ca.pem', + ] + github-token=[ + '--volume={config.webhook}/.config--github-token:/run/secrets/github-token:ro', + '--env=COCKPIT_GITHUB_TOKEN_FILE=/run/secrets/github-token', + ] + ''') + + (config.tasks / 'job-runner.toml').write_text(conf) + + +@pytest.fixture() +def mock_runner_config(config: Config, pod: PodData) -> None: + generate_config(config, + forge_opts=f'api-url = "{GHAPI_URL_POD}"', + run_args=f'"--pod={pod.pod}", "--env=GITHUB_API={GHAPI_URL_POD}"') + return None # silence ruff PT004 + + +@pytest.fixture() +def real_runner_config(config: Config) -> None: + generate_config(config, forge_opts='', run_args='') + return None # silence ruff PT004 + + +# +# Integration tests +# + +def test_podman(pod: PodData) -> None: + """tasks can connect to host's podman service + + This is covered implicitly by job-runner, but as a more basal plumbing test + this is easier to debug. + """ + assert 'cockpituous-tasks' in exec_c_out(pod.tasks, 'podman-remote --url unix:///podman.sock ps') + out = exec_c_out(pod.tasks, f'podman-remote --url unix:///podman.sock run -i --rm {TASKS_IMAGE} whoami') + assert out.strip() == 'user' + + +def test_images(pod: PodData) -> None: + """test image upload/download/prune""" + + exec_c(pod.tasks, f''' + cd bots + + # fake an image + echo world > /cache/images/testimage + NAME="testimage-$(sha256sum /cache/images/testimage | cut -f1 -d' ').qcow2" + mv /cache/images/testimage /cache/images/$NAME + ln -s $NAME images/testimage + + # test image-upload to S3 + ./image-upload --store {S3_URL_POD}/images/ testimage + ''') + # S3 store received this + out = exec_c_out(pod.tasks, f'cd bots; python3 -m lib.s3 ls {S3_URL_POD}/images/') + assert re.search(r'testimage-[0-9a-f]+\.qcow2', out) + + # image downloading from S3 + exec_c(pod.tasks, f''' + rm --verbose /cache/images/testimage* + cd bots + ./image-download --store {S3_URL_POD}/images/ testimage + grep -q "^world" /cache/images/testimage-*.qcow2 + rm --verbose /cache/images/testimage* + ''') + + # image pruning on s3 + exec_c(pod.tasks, f''' + cd bots + rm images/testimage + ./image-prune --s3 {S3_URL_POD}/images/ --force --checkout-only + ''') + # S3 store removed it + assert 'testimage' not in exec_c_out(pod.tasks, f'cd bots; python3 -m lib.s3 ls {S3_URL_POD}/images/') + + +def test_queue(pod: PodData) -> None: + """tasks can connect to AMQP""" + + out = exec_c_out(pod.tasks, f'bots/inspect-queue --amqp {AMQP_POD}') + # this depends on whether the test runs first or not + assert 'queue public does not exist' in out or 'queue public is empty' in out + + +def test_mock_pr(config: Config, pod: PodData, bots_sha: str, mock_github, mock_runner_config) -> None: + """almost end-to-end PR test + + Starting with GitHub webhook JSON payload injection; fully local, no privileges needed. + """ + exec_c(pod.tasks, f'''set -ex + {mock_github} + + cd bots + + # simulate GitHub webhook event, put that into the webhook queue + PYTHONPATH=. ./mock-github --print-pr-event cockpit-project/bots $SHA | \ + ./publish-queue --amqp {AMQP_POD} --create --queue webhook + + ./inspect-queue --amqp {AMQP_POD} + + # first run-queue processes webhook → tests-scan → public queue + ./run-queue --amqp {AMQP_POD} + ./inspect-queue --amqp {AMQP_POD} + + # second run-queue actually runs the test + ./run-queue --amqp {AMQP_POD} + ''', timeout=360) + + # check log in S3 + # looks like pull-1-a4d25bb9-20240315-135902-unit-tests/log + m = re.search(r'pull-1-[a-z0-9-]*-unit-tests/log(?=<)', get_s3(config, pod, 'logs/')) + assert m + slug = m.group(0) + log = get_s3(config, pod, f'logs/{slug}') + assert 'Job ran successfully' in log + assert re.search(r'Running on:\s+cockpituous', log) + + # 3 status updates posted + gh_mock_log = exec_c_out(pod.tasks, 'cat /tmp/mock.log') + jsons = re.findall(f'POST /repos/cockpit-project/bots/statuses/{bots_sha} (.*)$', gh_mock_log, re.M) + assert len(jsons) == 3 + assert json.loads(jsons[0]) == {"state": "pending", "description": "Not yet tested", "context": "unit-tests"} + assert json.loads(jsons[1]) == { + "state": "pending", + "description": f"In progress [{pod.pod}]", + "context": "unit-tests", + "target_url": f"https://localhost.localdomain:9000/logs/{slug}.html" + } + assert json.loads(jsons[2]) == { + "state": "success", + "description": f"Success [{pod.pod}]", + "context": "unit-tests", + "target_url": f"https://localhost.localdomain:9000/logs/{slug}.html" + } + + +def test_mock_cross_project_pr(config: Config, pod: PodData, bots_sha: str, + mock_github, mock_runner_config) -> None: + """almost end-to-end PR cross-project test + + Starting with GitHub webhook JSON payload injection; fully local, no privileges needed. + """ + exec_c(pod.tasks, f'''set -ex + {mock_github} + + cd bots + + # simulate GitHub webhook event, put that into the webhook queue + PYTHONPATH=. ./mock-github --print-pr-event cockpit-project/bots $SHA | \ + ./publish-queue --amqp {AMQP_POD} --create --queue webhook + + ./inspect-queue --amqp {AMQP_POD} + + # cross-project test request + export COCKPIT_TESTMAP_INJECT=main/unit-tests@cockpit-project/cockpituous + + # first run-queue processes webhook → tests-scan → public queue + ./run-queue --amqp {AMQP_POD} + ./inspect-queue --amqp {AMQP_POD} + + # second run-queue actually runs the test + ./run-queue --amqp {AMQP_POD} + ''', timeout=360) + + # check log in S3 + # looks like pull-1-a4d25bb9-20240315-135902-unit-tests.../log + m = re.search(r'pull-1-[a-z0-9-]*-unit-tests-cockpit-project-cockpituous(?=/)', get_s3(config, pod, 'logs/')) + assert m + slug = m.group(0) + log = get_s3(config, pod, f'logs/{slug}/log') + assert 'Job ran successfully' in log + assert re.search(r'Running on:\s+cockpituous', log) + + # validate test attachment + assert 'heisenberg compensator' in get_s3(config, pod, f'logs/{slug}/bogus.log') + + # 3 status updates posted to bots project (the PR we are testing) + gh_mock_log = exec_c_out(pod.tasks, 'cat /tmp/mock.log') + context = "unit-tests@cockpit-project/cockpituous" + jsons = re.findall(f'POST /repos/cockpit-project/bots/statuses/{bots_sha} (.*)$', gh_mock_log, re.M) + assert len(jsons) == 3 + assert json.loads(jsons[0]) == {"state": "pending", "description": "Not yet tested", "context": context} + assert json.loads(jsons[1]) == { + "state": "pending", + "description": f"In progress [{pod.pod}]", + "context": context, + "target_url": f"https://localhost.localdomain:9000/logs/{slug}/log.html" + } + assert json.loads(jsons[2]) == { + "state": "success", + "description": f"Success [{pod.pod}]", + "context": context, + "target_url": f"https://localhost.localdomain:9000/logs/{slug}/log.html" + } + + +def test_mock_image_refresh(config: Config, pod: PodData, bots_sha: str, + mock_github, mock_git_push, mock_runner_config) -> None: + """almost end-to-end PR image refresh + + Starting with GitHub webhook JSON payload injection; fully local, no privileges needed. + """ + exec_c(pod.tasks, f'''set -ex + {mock_github} + cd bots + + # simulate GitHub webhook event, put that into the webhook queue + PYTHONPATH=. ./mock-github --print-image-refresh-event cockpit-project/bots $SHA | \ + ./publish-queue --amqp {AMQP_POD} --create --queue webhook + + ./inspect-queue --amqp {AMQP_POD} + + # first run-queue processes webhook → issue-scan → public queue + ./run-queue --amqp {AMQP_POD} + ./inspect-queue --amqp {AMQP_POD} + + # second run-queue actually runs the image refresh + ./run-queue --amqp {AMQP_POD} + ''', timeout=360) + + # check log in S3 + m = re.search(r'image-refresh-foonux-[a-z0-9-]*(?=/)', get_s3(config, pod, 'logs/')) + assert m + slug = m.group(0) + log = get_s3(config, pod, f'logs/{slug}/log') + assert re.search(r'Running on:\s+cockpituous', log) + assert './image-refresh --verbose --issue=2 foonux\n' in log + assert f'Uploading to {S3_URL_POD}/images/foonux' in log + assert 'Success.' in log + + # branch was (mock) pushed + push_log = get_s3(config, pod, f'logs/{slug}/git-push.log') + assert 'push origin +HEAD:refs/heads/image-refresh-foonux-' in push_log + + exec_c(pod.tasks, f'''set -ex + cd bots + # image is on the S3 server + name=$(python3 -m lib.s3 ls {S3_URL_POD}/images/ | grep -o "foonux.*qcow2") + + # download image (it was not pushed to git, so need to use --state) + rm -f /cache/images/foonux* + ./image-download --store $COCKPIT_IMAGE_UPLOAD_STORE --state "$name" + + # validate image contents + qemu-img convert /cache/images/foonux-*.qcow2 /tmp/foonux.raw + grep "^fakeimage" /tmp/foonux.raw + rm /tmp/foonux.raw + ''') + + # status updates posted to original bots SHA on which the image got triggered + gh_mock_log = exec_c_out(pod.tasks, 'cat /tmp/mock.log') + jsons = re.findall(f'POST /repos/cockpit-project/bots/statuses/{bots_sha} (.*)$', gh_mock_log, re.M) + assert len(jsons) == 2 + assert json.loads(jsons[0]) == { + "context": "image-refresh/foonux", + "state": "pending", + "description": f"In progress [{pod.pod}]", + "target_url": f"https://localhost.localdomain:9000/logs/{slug}/log.html", + } + assert json.loads(jsons[1]) == { + "context": "image-refresh/foonux", + "state": "success", + "description": f"Success [{pod.pod}]", + "target_url": f"https://localhost.localdomain:9000/logs/{slug}/log.html", + } + + # and forwarded to the converted PR (new SHA) + assert re.search(r"POST /repos/cockpit-project/bots/statuses/a1b2c3 .*success.*Forwarded status.*target_url", + gh_mock_log) + + # posted new comment with log + assert re.search(r"POST /repos/cockpit-project/bots/issues/2/comments .*Success. Log: https.*", gh_mock_log) + + +def test_real_pr(config: Config, request, pod: PodData, bots_sha: str, real_runner_config, + user_github_token) -> None: + """full end-to-end PR test + + Requires --pr and --github-token. + """ + pr = request.config.getoption('--pr', skip=True) + pr_repo = request.config.getoption('--pr-repository') + + # run the main loop in the background; we could do this with a single run-queue invocation, + # but we want to test the cockpit-tasks script + tasks_pid = exec_c_out( + pod.tasks, '(nohup cockpit-tasks /tmp/cockpit-tasks.log 2>&1 & echo $!)' + ).strip() + + # wait until test status appears + exec_c(pod.tasks, f'''set -ex + cd bots + ./tests-scan -p {pr} --amqp {AMQP_POD} --repo {pr_repo} + for retry in $(seq 10); do + OUT=$(./tests-scan --repo {pr_repo} -p {pr} --human-readable --dry) + [ "${{OUT%unit-tests*}}" = "$OUT" ] || break + echo waiting until the status is visible + sleep 10 + done''', timeout=360) + + # wait until the unit-test got run and published, i.e. until the non-chunked raw log file exists + for _retry in range(60): + logs_dir = get_s3(config, pod, 'logs/') + m = re.search(f'pull-{pr}-[a-z0-9-]*-unit-tests/log(?=<)', logs_dir) + if m: + log_name = m.group(0) + break + print('waiting for unit-tests run to finish...') + time.sleep(10) + else: + raise SystemError('unit-tests run did not finish') + + # tell the tasks container iteration that we are done, and wait for it to finish + exec_c(pod.tasks, + f'set -ex; kill {tasks_pid}; while kill -0 {tasks_pid} 2>/dev/null; do sleep 1; done', + timeout=360) + + # spot-check the log + log = get_s3(config, pod, f'logs/{log_name}') + print(f'----- PR unit-tests log -----\n{log}\n-----------------') + assert re.search(r'Running on:\s+cockpituous', log) + assert 'Job ran successfully' in log + assert '' in get_s3(config, pod, f'logs/{log_name}.html') + + # validate test attachment if we ran cockpituous' own tests + if pr_repo.endswith('/cockpituous'): + print(f'----- S3 logs/ dir -----\n{logs_dir}\n-----------------') + slug = os.path.dirname(log_name) # strip off '/log' + assert 'heisenberg compensator' in get_s3(config, pod, f'logs/{slug}/bogus.log') + assert 'subdir-file' in get_s3(config, pod, f'logs/{slug}/data/subdir-file.txt') + + +# +# Interactive scenarios +# + +@pytest.mark.shell() +def test_shell(pod: PodData, user_github_token) -> None: + """interactive shell for development; run with `pytest -sm shell`""" + + subprocess.run(["podman", "exec", "-it", pod.tasks, "bash"])