Add an alternative standalone, container-based dev environment (and fix a few things exposed)

bcd

@@ -0,0 +1,125 @@
+#!/usr/bin/python
+
+import argparse
+import os
+import subprocess
+import sys
+
+CONTAINERS = (
+    "database",
+    "waiverdb",
+    "greenwave",
+    "rabbitmq",
+    "ipsilon",
+    "bodhi",
+)
+
+
+def ansible(args):
+    """Run an ansible playbook command based on the parser name."""
+    # this is the subcommand that was run - 'run', 'stop' etc.
+    here = os.path.abspath(os.path.dirname(__file__))
+    ret = subprocess.run(
+        (
+            "ansible-playbook",
+            f"{here}/devel/ansible-podman/playbook.yml",
+            f"-e bodhi_dev_{args.subcommand}=true"
+        )
+    )
+    sys.exit(ret.returncode)
+
+
+def logs(args):
+    fullc = f"bodhi-dev-{args.container}"
+    ret = subprocess.run(("podman", "logs", fullc))
+    sys.exit(ret.returncode)
+
+
+def shell(args):
+    fullc = f"bodhi-dev-{args.container}"
+    ret = subprocess.run(("podman", "exec", "-it", fullc, "/bin/bash"))
+    sys.exit(ret.returncode)
+
+
+def parse_args():
+    """Parse arguments with argparse."""
+    parser = argparse.ArgumentParser(
+        description=(
+            "Bodhi Container Development environment. Controls a complete Bodhi development "
+            "environment in Podman containers orchestrated by Ansible."
+        )
+    )
+    subparsers = parser.add_subparsers(dest="subcommand")
+    subparsers.required = True
+    parser_run = subparsers.add_parser(
+        "run",
+        description="Prepare and run the environment",
+        aliases=["up"]
+    )
+    parser_run.set_defaults(func=ansible)
+    parser_stop = subparsers.add_parser(
+        "stop",
+        description="Stop the environment (does not remove containers)",
+        aliases=["halt"]
+    )
+    parser_stop.set_defaults(func=ansible)
+    parser_remove = subparsers.add_parser(
+        "remove",
+        description="Stop and remove all containers",
+        aliases=["destroy"]
+    )
+    parser_remove.set_defaults(func=ansible)
+    parser_cis = subparsers.add_parser(
+        "cis",
+        description="Clear Ipsilon sessions (to allow you to log in as a different user)"
+    )
+    parser_cis.set_defaults(func=ansible)
+    parser_shell = subparsers.add_parser(
+        "shell",
+        description="Open a shell in a container. Container must be running"
+    )
+    parser_shell.add_argument(
+        "container",
+        help="The container to open a shell in (default: bodhi)",
+        default="bodhi",
+        nargs='?',
+        choices=CONTAINERS
+    )
+    parser_shell.set_defaults(func=shell)
+    parser_logs = subparsers.add_parser(
+        "logs",
+        description="Show logs of the specified container (does not work on bodhi, use journalctl)"
+    )
+    parser_logs.add_argument(
+        "container",
+        help="The container to show logs for",
+        choices=[cont for cont in CONTAINERS if cont != "bodhi"],
+    )
+    parser_logs.set_defaults(func=logs)
+    parser_prep = subparsers.add_parser(
+        "prep",
+        description="Run preparation steps only"
+    )
+    parser_prep.set_defaults(func=ansible)
+    parser_start = subparsers.add_parser(
+        "start",
+        description="Start containers only (do not run prep, will fail if prep has not already run)"
+    )
+    parser_start.set_defaults(func=ansible)
+    return parser.parse_args()
+
+
+def main():
+    """Main loop."""
+    try:
+        args = parse_args()
+        args.func(args)
+    except KeyboardInterrupt:
+        sys.stderr.write("Interrupted, exiting...\n")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()
+
+# vim: set textwidth=100 ts=8 et sw=4:

bodhi-server/bodhi/server/ffmarkdown.py

@@ -41,9 +41,6 @@
 
 
 BUGZILLA_RE = r'([a-zA-Z]+)(#[0-9]{5,})'
-UPDATE_RE = (r'(?:(?<!\S)|('
-             + escape(config['base_address'])
-             + r'updates/))([A-Z\-]+-\d{4}-[^\W_]{10})(?:(?=[\.,;:])|(?!\S))')
 
 
 def user_url(name: str) -> str:
@@ -204,7 +201,15 @@ def extendMarkdown(self, md: markdown.Markdown) -> None:
         Args:
             md: An instance of the Markdown class.
         """
+        # it is intentional that update_re is defined here, not in
+        # global scope at the top of this file. When testing, defining
+        # it early results in config['base_address'] being read before
+        # the test setup method has changed the config to use the
+        # values from testing.ini and the value may not be as expected
+        update_re = (r'(?:(?<!\S)|('
+                     + escape(config['base_address'])
+                     + r'updates/))([A-Z\-]+-\d{4}-[^\W_]{10})(?:(?=[\.,;:])|(?!\S))')
         md.inlinePatterns.register(MentionProcessor(MENTION_RE, md), 'mention', 175)
         md.inlinePatterns.register(BugzillaProcessor(BUGZILLA_RE, md), 'bugzilla', 175)
-        md.inlinePatterns.register(UpdateProcessor(UPDATE_RE, md), 'update', 175)
+        md.inlinePatterns.register(UpdateProcessor(update_re, md), 'update', 175)
         md.postprocessors.register(SurroundPostprocessor(md), 'surround', 175)

bodhi-server/bodhi/server/validators.py

@@ -609,6 +609,14 @@ def validate_acls(request, **kwargs):
             committers = ['ralph', 'bowlofeggs', 'guest']
             if config['acl_dummy_committer']:
                 committers.append(config['acl_dummy_committer'])
+            # let's also assume the update's owner can edit it
+            update = request.validated.get('update')
+            if not update:
+                alias = request.validated.get('edited')
+                if alias:
+                    update = Update.get(alias)
+            if update:
+                committers.append(update.user.name)
             if user.name in committers:
                 has_access = True
             people = committers

bodhi-server/tests/views/test_generic.py

@@ -227,8 +227,10 @@ def test_markdown_with_update_alias(self):
     def test_markdown_with_update_link(self):
         """Update link should be converted to alias."""
         res = self.app.get('/markdown', {
-            'text': 'See https://bodhi-dev.example.com/updates/FEDORA-2019-1a2b3c4d5e.',
+            'text': f'See {config["base_address"]}updates/FEDORA-2019-1a2b3c4d5e.',
         }, status=200)
+        # the server name changing to 'localhost' is just an effect
+        # of using webtest, don't panic
         assert res.json_body['html'] == \
             ('<div class="markdown"><p>See '
              '<a href="http://localhost/updates/FEDORA-2019-1a2b3c4d5e">'

devel/ansible-podman/containers/bodhi/Containerfile

@@ -0,0 +1,95 @@
+FROM quay.io/fedora/fedora:latest
+LABEL \
+    summary="Bodhi development environment" \
+    description="Distribution software update management system" \
+    maintainer="Red Hat, Inc." \
+    license="GPLv2+" \
+    url="https://github.com/fedora-infra/bodhi" \
+    vcs-type="git" \
+    io.k8s.display-name="Bodhi DE"
+MAINTAINER adamwill
+RUN set -exo pipefail \
+    # to get bodhi's runtime deps installed, install the official
+    # packages then remove them (in a single step so we don't get a
+    # layer with them included)
+    && dnf install -y --setopt install_weak_deps=false --nodocs bodhi-server bodhi-client python3-bodhi-messages \
+    && dnf remove -y --noautoremove bodhi-server bodhi-client python3-bodhi-messages \
+    # install test and dev env deps (and any new deps not yet in the
+    # package)
+    && dnf install -y --setopt install_weak_deps=false --nodocs \
+        # missing runtime deps
+        # there are errors in the logs if this isn't installed
+        procps-ng \
+        # dev env creation and control deps
+        poetry \
+        postgresql \
+        python3-pip \
+        sed \
+        systemd \
+        # dev env QOL and debug deps
+        bash-completion \
+        htop \
+        httpie \
+        nano \
+        nmap-ncat \
+        pcp-system-tools \
+        python3-ipdb \
+        python3-pydocstyle \
+        screen \
+        tmux \
+        tree \
+        vim-enhanced \
+        # doc build deps
+        graphviz \
+        make \
+        python3-sqlalchemy_schemadisplay \
+        python3-sphinx \
+        # test deps
+        createrepo_c \
+        pre-commit \
+        python3-createrepo_c \
+        python3-diff-cover \
+        python3-pytest \
+        python3-pytest-cov \
+        python3-pytest-mock \
+        python3-responses \
+        python3-webtest \
+        # pre-commit uses flake8, mypy, pydocstyle and ruff-pre-commit,
+        # but it always pulls them from repos, never uses packages
+    && dnf clean all \
+    && rm -rf /var/cache/* /var/log/dnf*
+
+# note we use the devel/ directory as the build context so we can access
+# development.ini.example here
+COPY ./ansible-podman/containers/bodhi/celery.service /etc/systemd/system/celery.service
+COPY ./ansible-podman/containers/bodhi/bodhi.service /etc/systemd/system/bodhi.service
+COPY ./ansible-podman/containers/bodhi/motd /etc/motd
+COPY ./ansible-podman/containers/bodhi/bashrc /root/.bashrc
+COPY ./ansible-podman/containers/bodhi/vimrc /root/.vimrc
+COPY ./ansible-podman/containers/bodhi/config.toml /etc/fedora-messaging/config.toml
+COPY ./ansible-podman/containers/bodhi/printer.toml /etc/fedora-messaging/printer.toml
+COPY ./ansible-podman/containers/bodhi/remote.toml /etc/fedora-messaging/remote.toml
+COPY ./ansible-podman/containers/bodhi/bodhi-wait.py /usr/local/bin/bodhi-wait.py
+COPY ./development.ini.example /etc/bodhi/production.ini
+
+RUN set -ex \
+    && sed -i -e "s,celery_config.*,celery_config = /bodhi/bodhi-server/celeryconfig.py,g" /etc/bodhi/production.ini \
+    && sed -i -e "s,pungi.basepath.*,pungi.basepath = /bodhi/devel/ci/integration/bodhi/,g" /etc/bodhi/production.ini \
+    && sed -i -e "s,base_address.*,base_address = http://localhost.localdomain:6543/,g" /etc/bodhi/production.ini \
+    && sed -i -e "s,cors_origins_rw.*,cors_origins_rw = *,g" /etc/bodhi/production.ini \
+    && sed -i -e "s,cors_connect_src.*,cors_connect_src = *,g" /etc/bodhi/production.ini \
+    && sed -i -e "s,openid.provider.*,openid.provider = http://localhost.localdomain:6546/openid/,g" /etc/bodhi/production.ini \
+    && sed -i -e "s,openid.url.*,openid.url = http://localhost.localdomain:6546/,g" /etc/bodhi/production.ini \
+    && sed -i -e "s,openid_template.*,openid_template = {username}.localdomain,g" /etc/bodhi/production.ini \
+    && sed -i -e "s,oidc.fedora.client_id.*,oidc.fedora.client_id = integration-tests,g" /etc/bodhi/production.ini \
+    && sed -i -e "s,oidc.fedora.client_secret.*,oidc.fedora.client_secret = integration-tests,g" /etc/bodhi/production.ini \
+    && sed -i -e "s,oidc.fedora.server_metadata_url.*,oidc.fedora.server_metadata_url = http://localhost.localdomain:6546/openidc/.well-known/openid-configuration,g" /etc/bodhi/production.ini
+
+RUN pip install pyramid_debugtoolbar
+RUN ln -s /usr/bin/true /usr/bin/pungi-koji
+RUN mkdir -p /srv/{composes/final,composes/stage}
+RUN systemctl enable [email protected] [email protected] celery.service bodhi.service
+RUN systemctl disable pmcd.service pmie.service pmlogger.service pmlogger_farm.service
+RUN poetry config virtualenvs.create false
+EXPOSE 6543
+CMD [ "/usr/sbin/init" ]

devel/ansible-podman/containers/bodhi/bashrc

@@ -0,0 +1,59 @@
+# .bashrc
+
+# Source global definitions
+if [ -f /etc/bashrc ]; then
+        . /etc/bashrc
+fi
+
+# Uncomment the following line if you don't like systemctl's auto-paging feature:
+# export SYSTEMD_PAGER=
+
+shopt -s expand_aliases
+alias bdocs="make -C /bodhi/docs clean && make -C /bodhi/docs html && make -C /bodhi/docs man"
+alias blog="journalctl -u bodhi -u fm-consumer@config"
+alias brestart="systemctl restart bodhi.service celery.service [email protected] [email protected] && echo 'The application is running on http://localhost.localdomain:6543'"
+alias bstart="systemctl start bodhi.service celery.service [email protected] [email protected] && echo 'The application is running on http://localhost.localdomain:6543'"
+alias bstop="systemctl stop bodhi.service celery.service [email protected] [email protected]"
+alias blint="pre-commit run -a"
+alias bmessages="journalctl -u fm-consumer@printer -f"
+
+
+function bresetdb {
+    bstop;
+    psql -U postgres -h localhost -c "DROP DATABASE bodhi2";
+    createdb -U postgres -h localhost bodhi2;
+    if [ ! -f "/tmp/bodhi2.dump.xz" ] ; then
+        curl -o /tmp/bodhi2.dump.xz https://infrastructure.fedoraproject.org/infra/db-dumps/bodhi2.dump.xz
+    fi
+    xzcat /tmp/bodhi2.dump.xz | psql -U postgres -h localhost bodhi2;
+    pushd /bodhi/bodhi-server;
+    # we call 'python3' explicitly to dodge some options in the
+    # shebang which break finding our bodhi modules
+    python3 /usr/bin/alembic upgrade head;
+    popd;
+    bstart;
+}
+
+
+function btest {
+    find /bodhi -name "*.pyc" -delete;
+    pushd /bodhi
+    blint || return $?
+    bdocs || return $?
+    for module in bodhi-messages bodhi-client bodhi-server; do
+        pushd $module
+        python3 -m pytest $@ tests || return $?
+        popd
+    done
+    diff-cover bodhi-*/coverage.xml --compare-branch=develop --fail-under=100
+    popd
+}
+
+
+export BODHI_URL="http://localhost.localdomain:6543/"
+export BODHI_OPENID_PROVIDER="http://localhost.localdomain:6546/openidc"
+export PYTHONWARNINGS="once"
+export BODHI_CI_ARCHIVE_PATH="/bodhi-ci-test_results/"
+
+cat /etc/motd
+cd /bodhi

devel/ansible-podman/containers/bodhi/bodhi-wait.py

@@ -0,0 +1,29 @@
+#!/usr/bin/python3
+
+import time
+
+from bodhi.server.config import config
+from sqlalchemy import engine_from_config
+from sqlalchemy.exc import OperationalError
+
+config.load_config()
+engine = engine_from_config(config)
+
+
+# stolen from waiverdb, GPLv2+, thanks Dan Callaghan
+def wait_for_db():
+    poll_interval = 10  # seconds
+    while True:
+        try:
+            engine.connect()
+        except OperationalError as e:
+            print('Failed to connect to database: {}'.format(e))
+            print(f'Sleeping for {poll_interval} seconds...')
+            time.sleep(poll_interval)
+            print('Retrying...')
+        else:
+            break
+
+
+if __name__ == '__main__':
+    wait_for_db()

devel/ansible-podman/containers/bodhi/bodhi.service

@@ -0,0 +1,20 @@
+[Unit]
+Description=bodhi
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Environment=PYTHONWARNINGS=once
+WorkingDirectory=/bodhi/bodhi-server
+ExecStartPre=/usr/bin/poetry -C /bodhi/bodhi-messages install --only-root
+ExecStartPre=/usr/bin/poetry -C /bodhi/bodhi-client install --only-root
+ExecStartPre=/usr/bin/poetry -C /bodhi/bodhi-server install --only-root
+ExecStartPre=/usr/local/bin/bodhi-wait.py
+# we don't run alembic and pserve directly from /usr/bin as some
+# options in their shebangs break finding our bodhi modules
+ExecStartPre=python3 -m alembic -c alembic.ini upgrade head
+ExecStart=python3 -m pyramid.scripts.pserve /etc/bodhi/production.ini --reload
+TimeoutStartSec=600
+
+[Install]
+WantedBy=multi-user.target

devel/ansible-podman/containers/bodhi/celery.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=celery
+After=bodhi.service
+Wants=bodhi.service
+
+[Service]
+Environment=BODHI_CONFIG=/etc/bodhi/development.ini
+WorkingDirectory=/bodhi/bodhi-server
+ExecStart=/usr/bin/poetry run celery -A bodhi.server.tasks.app worker -l info -Q celery,has_koji_mount -B
+
+[Install]
+WantedBy=multi-user.target