diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9c862fd..d46cd09 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,7 +21,7 @@ jobs: - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: - python-version: 3.7 + python-version: "3.10" - name: Install dependencies # https://docs.github.com/en/free-pro-team@latest/actions/guides/building-and-testing-python#installing-dependencies diff --git a/config/complete/orderly-web.yml b/config/complete/orderly-web.yml index e0bdbcc..6ab9f38 100644 --- a/config/complete/orderly-web.yml +++ b/config/complete/orderly-web.yml @@ -69,15 +69,27 @@ volumes: # Optional: to migrate the underlying orderly metadata to outpack metadata outpack: + repo: mrcide server: - repo: mrcide name: outpack_server tag: main migrate: - repo: mrcide name: outpack.orderly tag: main +# Optional: include Packit app +packit: + repo: mrcide + api: + name: packit-api + tag: main + app: + name: packit + tag: mrc-4261 + db: + name: packit-db + tag: main + ## Redis configuration redis: image: diff --git a/config/montagu/orderly-web.yml b/config/montagu/orderly-web.yml index 5010a49..999c3c6 100644 --- a/config/montagu/orderly-web.yml +++ b/config/montagu/orderly-web.yml @@ -21,12 +21,11 @@ volumes: # Optional: to migrate the underlying orderly metadata to outpack metadata outpack: + repo: mrcide server: - repo: mrcide name: outpack_server tag: main migrate: - repo: mrcide name: outpack.orderly tag: main diff --git a/config/noproxy/orderly-web.yml b/config/noproxy/orderly-web.yml index 16b02de..ca3521a 100644 --- a/config/noproxy/orderly-web.yml +++ b/config/noproxy/orderly-web.yml @@ -1,4 +1,3 @@ -## Prefix for container names; we'll use {container_prefix}-(orderly,web) container_prefix: orderly-web ## The name of the docker network that containers will be attached to. diff --git a/config/packit/orderly-web.yml b/config/packit/orderly-web.yml new file mode 100644 index 0000000..ca17a72 --- /dev/null +++ b/config/packit/orderly-web.yml @@ -0,0 +1,148 @@ +## Prefix for container names; we'll use {container_prefix}-(container_name) +container_prefix: orderly-web + +## The name of the docker network that containers will be attached to. +## If you want to proxy OrderlyWeb to the host, you will need to +## arrange a proxy on this network, or use dev_mode in the web section +## below. +network: orderly_web_network + +## Names of the docker volumes to use: +## +## orderly: stores the orderly archive +## proxy_logs: stores logs from the reverse proxy (only used if proxy is given) +## documents: stores static documentation available through the web app +## +## (More volumes are anticipated as the tool develops) +volumes: + orderly: orderly_web_volume + proxy_logs: orderly_web_proxy_logs + documents: orderly_web_documents + redis: orderly_web_redis_data + outpack: orderly_web_outpack + +## Redis configuration +redis: + image: + name: redis + tag: "5.0" + volume: orderly_web_redis_data + + +## Orderly configuration +orderly: + ## Image to use for orderly. This should build off of + ## vimc/orderly.server but can be extended to use whatever packages + ## you need. The components repo, image and tag will be assembled as + ## /: for the full docker image reference. + image: + repo: vimc + name: orderly.server + tag: master + worker_name: orderly.server + ## Initial data source for the orderly reports. This section is + ## optional - if not present, it is up to you to initialise the + ## orderly volume (in the volumes section above) with appropriate + ## data (if data is not present, orderly will not start). This + ## section only has an effect if the volume is empty. + initial: + ## Source must be one of "clone" or "demo" + source: clone + ## If source is "clone", then "url" must be given. If using a + ## private repo, then use an ssh url and provide ssh keys in the + ## "ssh" section. + url: https://github.com/reside-ic/orderly-example + ## Number of workers to create to run orderly jobs + workers: 1 + expose: true + env: + ORDERLY_API_SERVER_IDENTITY: production + +## Api and Website configuration +web: + ## Image to use for the web components. As for the orderly + ## configuration these will be assembled as /:. In + ## addition, a second image with the database migration support will + ## be used as /:. It is not expected (unlike + ## the orderly image) that the 'name' and 'migrate' will need to be + ## changed often, and it is expected that a web image will go + ## together with corresponding migration and admin images. + image: + repo: vimc + name: orderly-web + tag: master + migrate: orderlyweb-migrate + admin: orderly-web-user-cli + ## Public-facing url for the whole web service, including protocol + ## (ideally https://), hostname and port (if not using standard + ## ports). Here, we're going to use the same as for the proxy but + ## if you are using an external proxy then you'd use its hostname + ## and port. + url: https://localhost + ## If dev_mode is true then the port is exposed to the host (as + ## plain http). Do not use in production. The port is attached + ## only to the localhost and will not be available from other + ## machines + dev_mode: true + ## Port to use for the web service + port: 8888 + ## Name of the web service (affects the UI) + name: OrderlyWeb + ## Email address of the web service + email: admin@example.com + ## Authentication configuration + auth: + ## Name of the github organisation if using github auth + github_org: vimc + ## Name of the team within this github organisation if using + ## github auth (cuurrently must be "" - VIMC + github_team: "" + ## If using github auth you will need to register a github oauth app + ## https://developer.github.com/apps/building-oauth-apps/creating-an-oauth-app/ + ## The "Authorization callback URL" must be set to the public facing url + ## that you configured above + github_oauth: + id: "notarealid" + secret: "notarealsecret" + ## Enable or disable fine grained permissions + fine_grained: true + ## Enable montagu authentication provider + montagu: false + +# Optional: to notify a Slack channel during deployment, provide a webhook url +slack: + webhook_url: https://hooks.slack.com/services/T000/B000/XXXX + +## If running a proxy directly, fill this section in. Otherwise you +## are responsible for proxying the application out of the docker +## network +proxy: + enabled: true + hostname: localhost + port_http: 80 + port_https: 443 + image: + repo: vimc + name: orderly-web-proxy + tag: master + +outpack: + repo: mrcide + server: + name: outpack_server + tag: main + migrate: + name: outpack.orderly + tag: main + +packit: + repo: mrcide + api: + name: packit-api + tag: master + app: + name: packit + tag: mrc-4261 + db: + name: packit-db + tag: main diff --git a/config/vault/orderly-web.yml b/config/vault/orderly-web.yml index 21ff2dc..8d386da 100644 --- a/config/vault/orderly-web.yml +++ b/config/vault/orderly-web.yml @@ -119,4 +119,4 @@ proxy: image: repo: vimc name: orderly-web-proxy - tag: mrc-211 + tag: master diff --git a/orderly_web/config.py b/orderly_web/config.py index 083b307..c0c1181 100644 --- a/orderly_web/config.py +++ b/orderly_web/config.py @@ -182,7 +182,7 @@ def __init__(self, path, dat): self.volumes["outpack"] = config.config_string(dat, ["volumes", "outpack"]) self.outpack_repo = config.config_string( - dat, ["outpack", "server", "repo"]) + dat, ["outpack", "repo"]) self.outpack_name = config.config_string( dat, ["outpack", "server", "name"]) self.outpack_tag = config.config_string( @@ -191,14 +191,12 @@ def __init__(self, path, dat): self.outpack_repo, self.outpack_name, self.outpack_tag) - self.outpack_migrate_repo = config.config_string( - dat, ["outpack", "migrate", "repo"]) self.outpack_migrate_name = config.config_string( dat, ["outpack", "migrate", "name"]) self.outpack_migrate_tag = config.config_string( dat, ["outpack", "migrate", "tag"]) self.outpack_migrate_ref = constellation.ImageReference( - self.outpack_migrate_repo, self.outpack_migrate_name, + self.outpack_repo, self.outpack_migrate_name, self.outpack_migrate_tag) self.containers["outpack-server"] = "outpack-server" @@ -206,6 +204,45 @@ def __init__(self, path, dat): self.containers["outpack-migrate"] = "outpack-migrate" self.images["outpack-migrate"] = self.outpack_migrate_ref + # 8. Packit + if "packit" in dat and not self.outpack_enabled: + print("Ignoring Packit configuration as outpack is not enabled") + self.packit_enabled = "packit" in dat and self.outpack_enabled + if self.packit_enabled: + self.packit_repo = config.config_string( + dat, ["packit", "repo"]) + + self.packit_db_name = config.config_string( + dat, ["packit", "db", "name"]) + self.packit_db_tag = config.config_string( + dat, ["packit", "db", "tag"]) + self.packit_db_ref = constellation.ImageReference( + self.packit_repo, self.packit_db_name, + self.packit_db_tag) + + self.packit_api_name = config.config_string( + dat, ["packit", "api", "name"]) + self.packit_api_tag = config.config_string( + dat, ["packit", "api", "tag"]) + self.packit_api_ref = constellation.ImageReference( + self.packit_repo, self.packit_api_name, + self.packit_api_tag) + + self.packit_app_name = config.config_string( + dat, ["packit", "app", "name"]) + self.packit_app_tag = config.config_string( + dat, ["packit", "app", "tag"]) + self.packit_app_ref = constellation.ImageReference( + self.packit_repo, self.packit_app_name, + self.packit_app_tag) + + self.containers["packit-db"] = "packit-db" + self.images["packit-db"] = self.packit_db_ref + self.containers["packit-api"] = "packit-api" + self.images["packit-api"] = self.packit_api_ref + self.containers["packit"] = "packit" + self.images["packit"] = self.packit_app_ref + self.non_constellation_images = { "admin": self.admin_ref, "migrate": self.migrate_ref diff --git a/orderly_web/constellation.py b/orderly_web/constellation.py index df14be4..69e4bd0 100644 --- a/orderly_web/constellation.py +++ b/orderly_web/constellation.py @@ -17,16 +17,27 @@ def orderly_constellation(cfg): web = web_container(cfg) containers = [redis, orderly, worker, web] - if cfg.proxy_enabled: - proxy = proxy_container(cfg, web) - containers.append(proxy) - if cfg.outpack_enabled: outpack_migrate = outpack_migrate_container(cfg) containers.append(outpack_migrate) outpack_server = outpack_server_container(cfg) containers.append(outpack_server) + if cfg.packit_enabled: + packit_db = packit_db_container(cfg) + containers.append(packit_db) + packit_api = packit_api_container(cfg) + containers.append(packit_api) + packit = packit_container(cfg) + containers.append(packit) + + if cfg.proxy_enabled: + if cfg.packit_enabled: + proxy = proxy_container(cfg, web, packit_api, packit) + else: + proxy = proxy_container(cfg, web) + containers.append(proxy) + obj = constellation.Constellation("orderly-web", cfg.container_prefix, containers, cfg.network, cfg.volumes, data=cfg, vault_config=cfg.vault) @@ -51,6 +62,53 @@ def outpack_migrate_container(cfg): return outpack_migrate +def packit_db_container(cfg): + name = cfg.containers["packit-db"] + packit_db = constellation.ConstellationContainer( + name, cfg.packit_db_ref, configure=packit_db_configure) + return packit_db + + +def packit_db_configure(container, cfg): + docker_util.exec_safely(container, ["wait-for-db"]) + docker_util.exec_safely(container, + ["psql", "-U", "packituser", "-d", + "packit", "-a", "-f", + "/packit-schema/schema.sql"]) + + +def packit_api_container(cfg): + name = cfg.containers["packit-api"] + packit_api = constellation.ConstellationContainer( + name, cfg.packit_api_ref, configure=packit_api_configure) + return packit_api + + +def packit_api_configure(container, cfg): + print("[web] Configuring Packit API container") + outpack_container = cfg.containers["outpack-server"] + packit_db_container = cfg.containers["packit-db"] + url = "jdbc:postgresql://{}-{}:5432/packit?stringtype=unspecified" + opts = { + "db.url": url.format(cfg.container_prefix, + packit_db_container), + "db.user": "packituser", + "db.password": "changeme", + "outpack.server.url": "http://{}-{}:8000".format(cfg.container_prefix, + outpack_container) + } + txt = "".join(["{}={}\n".format(k, v) for k, v in opts.items()]) + docker_util.string_into_container( + txt, container, "/etc/packit/config.properties") + + +def packit_container(cfg): + name = cfg.containers["packit"] + packit = constellation.ConstellationContainer( + name, cfg.packit_app_ref) + return packit + + def redis_container(cfg): redis_name = cfg.containers["redis"] redis_mounts = [constellation.ConstellationMount("redis", "/data")] @@ -288,13 +346,27 @@ def web_start(container): docker_util.exec_safely(container, ["touch", "/etc/orderly/web/go_signal"]) -def proxy_container(cfg, web_container): +def proxy_container(cfg, web, packit_api=None, packit=None): print("[proxy] Creating proxy container") proxy_name = cfg.containers["proxy"] web_addr = "{}:{}".format( - web_container.name_external(cfg.container_prefix), cfg.web_port) + web.name_external(cfg.container_prefix), cfg.web_port) + if packit_api is not None: + packit_api_addr = "{}:8080".format( + packit_api.name_external(cfg.container_prefix)) + else: + # this is a bit hacky, but if Packit not available, just pass + # the OW address. this will just result in all proxy routes + # being mapped to OW and is easier than writing conditional + # logic in the nginx proxy scripts + packit_api_addr = web_addr + if packit is not None: + packit_addr = packit.name_external(cfg.container_prefix) + else: + packit_addr = web_addr proxy_args = [cfg.proxy_hostname, str(cfg.proxy_port_http), - str(cfg.proxy_port_https), web_addr] + str(cfg.proxy_port_https), web_addr, packit_api_addr, + packit_addr] proxy_mounts = [constellation.ConstellationMount( "proxy_logs", "/var/log/nginx")] proxy_ports = [cfg.proxy_port_http, cfg.proxy_port_https] diff --git a/proxy/bin/orderly-web-proxy b/proxy/bin/orderly-web-proxy index 1b439a0..422ffef 100755 --- a/proxy/bin/orderly-web-proxy +++ b/proxy/bin/orderly-web-proxy @@ -1,21 +1,23 @@ #!/usr/bin/env bash set -eu -if [ "$#" -eq 4 ]; then +if [ "$#" -eq 6 ]; then export HTTP_HOST=$1 export HTTP_PORT=$2 export HTTPS_PORT=$3 export ORDERLY_WEB=$4 + export PACKIT_API=$5 + export PACKIT=$6 else - echo "Usage: HOSTNAME PORT_HTTP PORT_HTTPS ORDERLY_WEB" - echo "e.g. docker run ... montagu.vaccineimpact.org 80 443 orderly" + echo "Usage: HOSTNAME PORT_HTTP PORT_HTTPS ORDERLY_WEB PACKIT_API PACKIT" + echo "e.g. docker run ... montagu.vaccineimpact.org 80 443 orderly packit_api packit" exit 1 fi echo "We will listen on ports $HTTP_PORT (http) and $HTTPS_PORT (https)" echo "with hostname $HTTP_HOST, proxying orderly web from $ORDERLY_WEB" -envsubst '$HTTP_HOST,$HTTP_PORT,$HTTPS_PORT,$ORDERLY_WEB' \ +envsubst '$HTTP_HOST,$HTTP_PORT,$HTTPS_PORT,$ORDERLY_WEB, $PACKIT_API, $PACKIT' \ < /etc/nginx/nginx.conf.template > /etc/nginx/nginx.conf # These paths must match the paths as used in the nginx.conf diff --git a/proxy/nginx.conf b/proxy/nginx.conf index 81e17e2..4dc1f70 100644 --- a/proxy/nginx.conf +++ b/proxy/nginx.conf @@ -68,6 +68,14 @@ http { root /usr/share/nginx/html; + location /packit/api/ { + proxy_pass http://${PACKIT_API}/; + } + + location /packit/ { + proxy_pass http://${PACKIT}/; + } + location / { proxy_pass http://${ORDERLY_WEB}/; } diff --git a/setup.py b/setup.py index 8bcfac6..172f53c 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ "Pillow"] setup(name="orderly_web", - version="0.2.2", + version="1.0.0", description="Deploy scripts for OrderlyWeb", long_description=long_description, classifiers=[ diff --git a/test/test_config.py b/test/test_config.py index 1cd3920..4199fbe 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -3,9 +3,7 @@ import pytest import shutil import tempfile -import vault_dev import yaml -import os from orderly_web.config import * @@ -56,7 +54,7 @@ def test_example_config(): assert cfg.proxy_enabled assert cfg.proxy_ssl_self_signed - assert str(cfg.images["proxy"]) == "vimc/orderly-web-proxy:master" + assert str(cfg.images["proxy"]) == "vimc/orderly-web-proxy:mrc-4255" assert cfg.orderly_expose assert cfg.orderly_initial_source == "clone" @@ -263,11 +261,10 @@ def test_outpack_volume_required_if_enabled(): def test_outpack_config(): - options = {"outpack": {"migrate": {"repo": "mrcide", - "name": "outpack.orderly", + options = {"outpack": {"repo": "mrcide", + "migrate": {"name": "outpack.orderly", "tag": "main"}, - "server": {"repo": "mrcide", - "name": "outpack_server", + "server": {"name": "outpack_server", "tag": "main"} }, "volumes": {"outpack": "outpack_vol"}} @@ -285,6 +282,22 @@ def test_outpack_disabled_if_no_config(): assert cfg.outpack_enabled is False +def test_packit_config(): + cfg = build_config("config/packit") + assert cfg.packit_api_ref is not None + assert cfg.containers["packit-api"] == "packit-api" + assert cfg.packit_db_ref is not None + assert cfg.containers["packit-db"] == "packit-db" + assert cfg.packit_app_ref is not None + assert cfg.containers["packit"] == "packit" + assert cfg.packit_enabled is True + + +def test_packit_disabled_if_no_config(): + cfg = build_config("config/basic") + assert cfg.packit_enabled is False + + def read_file(path): with open(path, "r") as f: return f.read() diff --git a/test/test_integration.py b/test/test_integration.py index d342be9..47ab230 100644 --- a/test/test_integration.py +++ b/test/test_integration.py @@ -430,11 +430,10 @@ def test_can_start_with_prepared_volume(): def test_can_start_with_outpack(): path = "config/basic" - options = {"outpack": {"migrate": {"repo": "mrcide", - "name": "outpack.orderly", + options = {"outpack": {"repo": "mrcide", + "migrate": {"name": "outpack.orderly", "tag": "main"}, - "server": {"repo": "mrcide", - "name": "outpack_server", + "server": {"name": "outpack_server", "tag": "main"} }, "volumes": {"outpack": "outpack_vol"}} @@ -453,6 +452,18 @@ def test_can_start_with_outpack(): orderly_web.stop(path, kill=True, volumes=True, network=True) +def test_can_start_with_packit(): + path = "config/packit" + cfg = build_config(path) + try: + orderly_web.start(path) + assert docker_util.container_exists("orderly-web-packit-db") + assert docker_util.container_exists("orderly-web-packit-api") + assert docker_util.container_exists("orderly-web-packit") + finally: + orderly_web.stop(path, kill=True, volumes=True, network=True) + + def test_notifies_slack_on_success(): with patch.object(Notifier, 'post', return_value=None) as mock_notify: