diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3032521..d8b4e55 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -38,8 +38,7 @@ jobs: - name: Pull images run: | docker pull mher/flower:0.9.5 - docker pull nginx/nginx-prometheus-exporter:0.10.0 - docker pull nginx/nginx-prometheus-exporter:0.4.1 + docker pull nginx/nginx-prometheus-exporter:1.3.0 docker pull reachfive/fake-smtp-server:latest docker pull redis docker pull redis:5.0 diff --git a/config/basic/montagu.yml b/config/basic/montagu.yml index 90cb445..e517529 100644 --- a/config/basic/montagu.yml +++ b/config/basic/montagu.yml @@ -72,7 +72,7 @@ proxy: metrics: repo: nginx name: nginx-prometheus-exporter - tag: 0.10.0 + tag: 1.3.0 contrib: name: montagu-contrib-portal tag: master diff --git a/config/ci/montagu.yml b/config/ci/montagu.yml index 1385d7b..7e00495 100644 --- a/config/ci/montagu.yml +++ b/config/ci/montagu.yml @@ -12,7 +12,7 @@ vault: ## a list, which is currently ## ## azure, github, gcp, kubernetes, ldap, mfa, okta - method: github + method: token ## Prefix for container names; we'll use {container_prefix}-(container_name) container_prefix: montagu @@ -89,7 +89,7 @@ proxy: metrics: repo: nginx name: nginx-prometheus-exporter - tag: 0.10.0 + tag: 1.3.0 contrib: name: montagu-contrib-portal tag: master diff --git a/config/complete/montagu.yml b/config/complete/montagu.yml index e25a84b..22a7853 100644 --- a/config/complete/montagu.yml +++ b/config/complete/montagu.yml @@ -82,7 +82,7 @@ proxy: metrics: repo: nginx name: nginx-prometheus-exporter - tag: 0.4.1 + tag: 1.3.0 ## This section describes how to get the certificate in. We ## support two sources: ## diff --git a/pyproject.toml b/pyproject.toml index a03207b..a290cf9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,7 +89,7 @@ style = [ ] fmt = [ "black {args:.}", - "ruff --fix {args:.}", + "ruff check --fix {args:.}", "style", ] all = [ @@ -105,6 +105,8 @@ skip-string-normalization = true [tool.ruff] target-version = "py37" line-length = 120 + +[tool.ruff.lint] select = [ "A", "ARG", @@ -149,13 +151,13 @@ unfixable = [ "F401", ] -[tool.ruff.isort] +[tool.ruff.lint.isort] known-first-party = ["montagu_deploy"] -[tool.ruff.flake8-tidy-imports] +[tool.ruff.lint.flake8-tidy-imports] ban-relative-imports = "all" -[tool.ruff.per-file-ignores] +[tool.ruff.lint.per-file-ignores] # Tests can use magic values, assertions, and relative imports "tests/**/*" = ["PLR2004", "S101", "TID252"] diff --git a/src/montagu_deploy/cli.py b/src/montagu_deploy/cli.py index 9ee1872..072456f 100644 --- a/src/montagu_deploy/cli.py +++ b/src/montagu_deploy/cli.py @@ -26,7 +26,7 @@ import montagu_deploy.__about__ as about from montagu_deploy.config import MontaguConfig -from montagu_deploy.montagu_constellation import MontaguConstellation +from montagu_deploy.montagu_constellation import montagu_constellation def main(argv=None): @@ -35,7 +35,7 @@ def main(argv=None): return about.__version__ else: cfg = MontaguConfig(path, extra, options) - obj = MontaguConstellation(cfg) + obj = montagu_constellation(cfg) if args.action == "start": montagu_start(obj, args) elif args.action == "status": diff --git a/src/montagu_deploy/montagu_constellation.py b/src/montagu_deploy/montagu_constellation.py index 7d767dd..ca7653e 100644 --- a/src/montagu_deploy/montagu_constellation.py +++ b/src/montagu_deploy/montagu_constellation.py @@ -9,44 +9,26 @@ from montagu_deploy import database -class MontaguConstellation: - def __init__(self, cfg): - api = api_container(cfg) - db = db_container(cfg) - admin = admin_container(cfg) - contrib = contrib_container(cfg) - proxy = proxy_container(cfg) - mq = mq_container(cfg) - flower = flower_container(cfg) - task_queue = task_queue_container(cfg) - - containers = [api, db, admin, contrib, proxy, mq, flower, task_queue] - - if cfg.fake_smtp_ref: - fake_smtp = fake_smtp_container(cfg) - containers.append(fake_smtp) - - self.cfg = cfg - self.obj = constellation.Constellation( - "montagu", cfg.container_prefix, containers, cfg.network, cfg.volumes, data=cfg, vault_config=cfg.vault - ) - - def start(self, **kwargs): - self.obj.start(**kwargs) - # The proxy metrics container cannot be started via constellation, because - # it has to belong to the same network as the proxy as soon as it is started - # and constellation starts containers on the 'none' network. So we provide - # start/stop/status methods for the metrics container that mimic the - # constellation behaviour - start_proxy_metrics(self.cfg) - - def stop(self, **kwargs): - stop_proxy_metrics(self.cfg) - self.obj.stop(**kwargs) - - def status(self): - self.obj.status() - status_proxy_metrics(self.cfg) +def montagu_constellation(cfg): + containers = [ + api_container(cfg), + db_container(cfg), + admin_container(cfg), + contrib_container(cfg), + proxy_container(cfg), + proxy_metrics_container(cfg), + mq_container(cfg), + flower_container(cfg), + task_queue_container(cfg), + ] + + if cfg.fake_smtp_ref: + fake_smtp = fake_smtp_container(cfg) + containers.append(fake_smtp) + + return constellation.Constellation( + "montagu", cfg.container_prefix, containers, cfg.network, cfg.volumes, data=cfg, vault_config=cfg.vault + ) def admin_container(cfg): @@ -250,40 +232,11 @@ def proxy_configure(container, cfg): docker_util.string_into_container(cfg.dhparam, container, join(ssl_path, "dhparam.pem")) -def start_proxy_metrics(cfg): - name = "{}-{}".format(cfg.container_prefix, cfg.containers["metrics"]) +def proxy_metrics_container(cfg): proxy_name = cfg.containers["proxy"] - image = str(cfg.proxy_metrics_ref) - print("Starting {} ({})".format(cfg.containers["metrics"], image)) - docker.from_env().containers.run( - image, - restart_policy={"Name": "always"}, - ports={"9113/tcp": 9113}, - command=f'-nginx.scrape-uri "http://{proxy_name}/basic_status"', - network=cfg.network, - name=name, - detach=True, + return constellation.ConstellationContainer( + cfg.containers["metrics"], + cfg.proxy_metrics_ref, + ports=[9113], + args=["-nginx.scrape-uri", f"http://{proxy_name}/basic_status"], ) - - -def stop_proxy_metrics(cfg): - name = "{}-{}".format(cfg.container_prefix, cfg.containers["metrics"]) - container = get_container(name) - if container: - print(f"Killing '{name}'") - container.remove(force=True) - - -def status_proxy_metrics(cfg): - name = "{}-{}".format(cfg.container_prefix, cfg.containers["metrics"]) - container = get_container(name) - status = container.status if container else "missing" - print(" - {} ({}): {}".format(cfg.containers["metrics"], name, status)) - - -def get_container(name): - client = docker.client.from_env() - try: - return client.containers.get(name) - except docker.errors.NotFound: - return None diff --git a/tests/test_config.py b/tests/test_config.py index feb2962..869e5dc 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -38,7 +38,7 @@ def test_config_basic(): assert str(cfg.images["mq"]) == "docker.io/redis:latest" assert str(cfg.images["flower"]) == "mher/flower:0.9.5" assert str(cfg.images["task_queue"]) == "vimc/task-queue-worker:master" - assert str(cfg.images["metrics"]) == "nginx/nginx-prometheus-exporter:0.10.0" + assert str(cfg.images["metrics"]) == "nginx/nginx-prometheus-exporter:1.3.0" assert str(cfg.images["db_migrate"]) == "vimc/montagu-migrate:master" assert str(cfg.images["fake_smtp"]) == "reachfive/fake-smtp-server:latest" diff --git a/tests/test_constellation.py b/tests/test_constellation.py index c40ce20..2cf37d8 100644 --- a/tests/test_constellation.py +++ b/tests/test_constellation.py @@ -2,145 +2,159 @@ from constellation import docker_util from src.montagu_deploy.config import MontaguConfig -from src.montagu_deploy.montagu_constellation import MontaguConstellation +from src.montagu_deploy.montagu_constellation import montagu_constellation from tests.utils import http_get def test_start_and_stop(): cfg = MontaguConfig("config/basic") - obj = MontaguConstellation(cfg) + obj = montagu_constellation(cfg) - obj.start() + try: + obj.start() - cl = docker.client.from_env() + cl = docker.client.from_env() - assert docker_util.network_exists(cfg.network) - assert docker_util.volume_exists(cfg.volumes["db"]) - assert docker_util.volume_exists(cfg.volumes["burden_estimates"]) - assert docker_util.volume_exists(cfg.volumes["emails"]) - assert docker_util.volume_exists(cfg.volumes["mq"]) - assert docker_util.volume_exists(cfg.volumes["templates"]) - assert docker_util.volume_exists(cfg.volumes["guidance"]) + assert docker_util.network_exists(cfg.network) + assert docker_util.volume_exists(cfg.volumes["db"]) + assert docker_util.volume_exists(cfg.volumes["burden_estimates"]) + assert docker_util.volume_exists(cfg.volumes["emails"]) + assert docker_util.volume_exists(cfg.volumes["mq"]) + assert docker_util.volume_exists(cfg.volumes["templates"]) + assert docker_util.volume_exists(cfg.volumes["guidance"]) - assert docker_util.container_exists("montagu-api") - assert docker_util.container_exists("montagu-db") - assert docker_util.container_exists("montagu-proxy") - assert docker_util.container_exists("montagu-proxy-metrics") - assert docker_util.container_exists("montagu-admin") - assert docker_util.container_exists("montagu-contrib") - assert docker_util.container_exists("montagu-mq") - assert docker_util.container_exists("montagu-flower") - assert docker_util.container_exists("montagu-task-queue") - assert docker_util.container_exists("montagu-fake-smtp") + assert docker_util.container_exists("montagu-api") + assert docker_util.container_exists("montagu-db") + assert docker_util.container_exists("montagu-proxy") + assert docker_util.container_exists("montagu-proxy-metrics") + assert docker_util.container_exists("montagu-admin") + assert docker_util.container_exists("montagu-contrib") + assert docker_util.container_exists("montagu-mq") + assert docker_util.container_exists("montagu-flower") + assert docker_util.container_exists("montagu-task-queue") + assert docker_util.container_exists("montagu-fake-smtp") - containers = cl.containers.list() - assert len(containers) == 10 + containers = cl.containers.list() + print([c.name for c in containers]) + assert len(containers) == 10 - obj.stop(kill=True, remove_volumes=True) + finally: + obj.stop(kill=True, remove_volumes=True) def test_api_configured(): cfg = MontaguConfig("config/basic") - obj = MontaguConstellation(cfg) - - obj.start() + obj = montagu_constellation(cfg) - api = get_container(cfg, "api") - api_config = docker_util.string_from_container(api, "/etc/montagu/api/config.properties").split("\n") + try: + obj.start() - assert "app.url=https://localhost/api" in api_config - assert "db.host=db" in api_config - assert "db.username=api" in api_config - assert "db.password=apipassword" in api_config - assert "allow.localhost=False" in api_config - assert "upload.dir=/upload_dir" in api_config - assert "email.mode=real" not in api_config + api = get_container(cfg, "api") + api_config = docker_util.string_from_container(api, "/etc/montagu/api/config.properties").split("\n") - res = http_get("https://localhost/api/v1") - assert '"status": "success"' in res + assert "app.url=https://localhost/api" in api_config + assert "db.host=db" in api_config + assert "db.username=api" in api_config + assert "db.password=apipassword" in api_config + assert "allow.localhost=False" in api_config + assert "upload.dir=/upload_dir" in api_config + assert "email.mode=real" not in api_config - obj.stop(kill=True, remove_volumes=True) + res = http_get("https://localhost/api/v1") + assert '"status": "success"' in res + finally: + obj.stop(kill=True, remove_volumes=True) cfg = MontaguConfig("config/complete") - obj = MontaguConstellation(cfg) + obj = montagu_constellation(cfg) - obj.start() - api = get_container(cfg, "api") - api_config = docker_util.string_from_container(api, "/etc/montagu/api/config.properties").split("\n") - assert "email.mode=real" in api_config - assert "email.password=changeme" in api_config - assert "flow.url=fakeurl" in api_config + try: + obj.start() + api = get_container(cfg, "api") + api_config = docker_util.string_from_container(api, "/etc/montagu/api/config.properties").split("\n") + assert "email.mode=real" in api_config + assert "email.password=changeme" in api_config + assert "flow.url=fakeurl" in api_config - obj.stop(kill=True, remove_volumes=True) + finally: + obj.stop(kill=True, remove_volumes=True) def test_proxy_configured_self_signed(): cfg = MontaguConfig("config/basic") - obj = MontaguConstellation(cfg) + obj = montagu_constellation(cfg) - obj.start() + try: + obj.start() - api = get_container(cfg, "proxy") - cert = docker_util.string_from_container(api, "/etc/montagu/proxy/certificate.pem") - key = docker_util.string_from_container(api, "/etc/montagu/proxy/ssl_key.pem") - param = docker_util.string_from_container(api, "/etc/montagu/proxy/dhparam.pem") - assert cert is not None - assert key is not None - assert param is not None + api = get_container(cfg, "proxy") + cert = docker_util.string_from_container(api, "/etc/montagu/proxy/certificate.pem") + key = docker_util.string_from_container(api, "/etc/montagu/proxy/ssl_key.pem") + param = docker_util.string_from_container(api, "/etc/montagu/proxy/dhparam.pem") + assert cert is not None + assert key is not None + assert param is not None - res = http_get("https://localhost") - assert "Montagu" in res + res = http_get("https://localhost") + assert "Montagu" in res - obj.stop(kill=True, remove_volumes=True) + finally: + obj.stop(kill=True, remove_volumes=True) def test_db_configured(): cfg = MontaguConfig("config/complete") - obj = MontaguConstellation(cfg) + obj = montagu_constellation(cfg) - obj.start() + try: + obj.start() - db = get_container(cfg, "db") - res = docker_util.exec_safely(db, f'psql -U {cfg.db_root_user} -d postgres -c "\\du"') - res = res.output.decode("UTF-8") + db = get_container(cfg, "db") + res = docker_util.exec_safely(db, f'psql -U {cfg.db_root_user} -d postgres -c "\\du"') + res = res.output.decode("UTF-8") - for u in cfg.db_users: - assert u in res + for u in cfg.db_users: + assert u in res - query = "SELECT * FROM pg_replication_slots WHERE slot_name = 'barman'" - res = docker_util.exec_safely(db, f'psql -U {cfg.db_root_user} -d postgres -c "{query}"') - res = res.output.decode("UTF-8") + query = "SELECT * FROM pg_replication_slots WHERE slot_name = 'barman'" + res = docker_util.exec_safely(db, f'psql -U {cfg.db_root_user} -d postgres -c "{query}"') + res = res.output.decode("UTF-8") - assert "barman" in res + assert "barman" in res - obj.stop(kill=True, remove_volumes=True) + finally: + obj.stop(kill=True, remove_volumes=True) def test_proxy_configured_ssl(): cfg = MontaguConfig("config/complete") - obj = MontaguConstellation(cfg) + obj = montagu_constellation(cfg) - obj.start() + try: + obj.start() - api = get_container(cfg, "proxy") - cert = docker_util.string_from_container(api, "/etc/montagu/proxy/certificate.pem") - key = docker_util.string_from_container(api, "/etc/montagu/proxy/ssl_key.pem") - param = docker_util.string_from_container(api, "/etc/montagu/proxy/dhparam.pem") - assert cert == "cert" - assert key == "k3y" - assert param == "param" + api = get_container(cfg, "proxy") + cert = docker_util.string_from_container(api, "/etc/montagu/proxy/certificate.pem") + key = docker_util.string_from_container(api, "/etc/montagu/proxy/ssl_key.pem") + param = docker_util.string_from_container(api, "/etc/montagu/proxy/dhparam.pem") + assert cert == "cert" + assert key == "k3y" + assert param == "param" - obj.stop(kill=True, remove_volumes=True) + finally: + obj.stop(kill=True, remove_volumes=True) def test_metrics(): cfg = MontaguConfig("config/basic") - obj = MontaguConstellation(cfg) + obj = montagu_constellation(cfg) - obj.start() - http_get("http://localhost:9113/metrics") + try: + obj.start() + http_get("http://localhost:9113/metrics") - obj.stop(kill=True) + finally: + obj.stop(kill=True) def get_container(cfg, name): diff --git a/tests/test_integration.py b/tests/test_integration.py index 36b69f0..0c81bd9 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -55,15 +55,20 @@ def test_task_queue(): cfg = MontaguConfig(path) try: youtrack_token = os.environ["YOUTRACK_TOKEN"] - os.environ["VAULT_AUTH_GITHUB_TOKEN"] = os.environ["VAULT_TOKEN"] - with vault_dev.server() as s: + with vault_dev.Server(export_token=True) as s: cl = s.client() - enable_github_login(cl) cl.write("secret/youtrack/token", value=youtrack_token) vault_addr = f"http://localhost:{s.port}" orderly_web.start(orderly_config_path) - cli.main(["start", path, f"--option=vault.addr={vault_addr}"]) + cli.main( + [ + "start", + path, + f"--option=vault.addr={vault_addr}", + f"--option=vault.auth.args.token={s.token}", + ] + ) # wait for API to be ready http_get("https://localhost/api/v1") @@ -101,21 +106,3 @@ def add_task_queue_user(cfg, orderly_config_path): orderly_web.admin.grant( orderly_config_path, "montagu-task@imperial.ac.uk", ["*/reports.run", "*/reports.review", "*/reports.read"] ) - - -def enable_github_login(cl, path="github"): - cl.sys.enable_auth_method(method_type="github", path=path) - policy = """ - path "secret/*" { - capabilities = ["read", "list"] - } - """ - - cl.sys.create_or_update_policy( - name="secret-reader", - policy=policy, - ) - - cl.auth.github.map_team(team_name="robots", policies=["secret-reader"], mount_point=path) - - cl.auth.github.configure(organization="vimc", mount_point=path)