refactor: read config options from environment variables

.env.example

@@ -1,26 +1,24 @@
-SERVICE_URL=http://localhost:8080
-SERVICE_API_URL=https://localhost:8080/api
-SERVICE_NAME=Abrechnung
-DB_HOST=abrechnung_postgres
-DB_USER=abrechnung
-DB_NAME=abrechnung
-DB_PASSWORD=replaceme # e.g. pwgen -s 64 1
+ABRECHNUNG_SERVICE__URL=http://localhost:8080
+ABRECHNUNG_SERVICE__API_URL=https://localhost:8080/api
+ABRECHNUNG_SERVICE__NAME=Abrechnung
+ABRECHNUNG_DATABASE__HOST=abrechnung_postgres
+ABRECHNUNG_DATABASE__USER=abrechnung
+ABRECHNUNG_DATABASE__DBNAME=abrechnung
+ABRECHNUNG_DATABASE__PASSWORD=replaceme # e.g. pwgen -s 64 1
 
-ABRECHNUNG_SECRET=replaceme # pwgen -s 64 1
-ABRECHNUNG_PORT=8080
-ABRECHNUNG_ID=default
+ABRECHNUNG_API__SECRET_KEY=replaceme # pwgen -s 64 1
+ABRECHNUNG_API__PORT=8080
+ABRECHNUNG_API__ID=default
 
-REGISTRATION_ENABLED=false
-REGISTRATION_VALID_EMAIL_DOMAINS=sft.lol,sft.mx
-REGISTRATION_ALLOW_GUEST_USERS=true
+ABRECHNUNG_REGISTRATION__ENABLED=false
 
-ABRECHNUNG_EMAIL=[email protected]
-SMTP_HOST=mail
-SMTP_PORT=1025
-SMTP_MODE=smtp
-#SMTP_MODE=smtp-starttls # use this in production, remove line above
-SMTP_USER=username
-SMTP_PASSWORD=replaceme
+ABRECHNUNG_EMAIL__ADDRESS=[email protected]
+ABRECHNUNG_EMAIL__HOST=mail
+ABRECHNUNG_EMAIL__PORT=1025
+ABRECHNUNG_EMAIL__MODE=smtp
+#ABRECHNUNG_EMAIL__MODE=smtp-starttls # use this in production, remove line above
+ABRECHNUNG_EMAIL__AUTH__USERNAME=username
+ABRECHNUNG_EMAIL__AUTH__PASSWORD=replaceme
 
 POSTGRES_USER=abrechnung
 POSTGRES_PASSWORD=replaceme # use the same as DB_PASSWORD

abrechnung/config.py

@@ -1,9 +1,14 @@
 from datetime import timedelta
 from pathlib import Path
-from typing import List, Optional
+from typing import List, Literal, Optional, Tuple, Type
 
 import yaml
-from pydantic import BaseModel
+from pydantic import BaseModel, EmailStr
+from pydantic_settings import (
+    BaseSettings,
+    PydanticBaseSettingsSource,
+    SettingsConfigDict,
+)
 
 from abrechnung.framework.database import DatabaseConfig
 
@@ -39,14 +44,16 @@ class AuthConfig(BaseModel):
         username: str
         password: str
 
-    address: str
+    address: EmailStr
     host: str
     port: int
-    mode: str = "smtp"  # oneof "local" "smtp-ssl" "smtp-starttls" "smtp"
+    mode: Literal["local", "smtp-ssl", "smtp", "smtp-starttls"] = "smtp"
     auth: Optional[AuthConfig] = None
 
 
-class Config(BaseModel):
+class Config(BaseSettings):
+    model_config = SettingsConfigDict(env_prefix="ABRECHNUNG_", env_nested_delimiter="__")
+
     service: ServiceConfig
     api: ApiConfig
     database: DatabaseConfig
@@ -55,8 +62,20 @@ class Config(BaseModel):
     demo: DemoConfig = DemoConfig()
     registration: RegistrationConfig = RegistrationConfig()
 
+    @classmethod
+    def settings_customise_sources(
+        cls,
+        settings_cls: Type[BaseSettings],
+        init_settings: PydanticBaseSettingsSource,
+        env_settings: PydanticBaseSettingsSource,
+        dotenv_settings: PydanticBaseSettingsSource,
+        file_secret_settings: PydanticBaseSettingsSource,
+    ) -> Tuple[PydanticBaseSettingsSource, ...]:
+        return env_settings, init_settings, dotenv_settings, file_secret_settings
+
 
 def read_config(config_path: Path) -> Config:
     content = config_path.read_text("utf-8")
-    config = Config(**yaml.safe_load(content))
+    loaded = yaml.safe_load(content)
+    config = Config(**loaded)
     return config

docker-compose.base.yaml

@@ -18,5 +18,3 @@ services:
     command: cron
     healthcheck:
       disable: true
-
-

docker-compose.devel.yaml

@@ -2,13 +2,13 @@ services:
   api:
     build:
       context: .
-      dockerfile: docker/Dockerfile-devel
-      target: api
+      dockerfile: docker/Dockerfile-api
     extends:
       file: docker-compose.base.yaml
       service: api
+    env_file: .env
     environment:
-      DB_HOST: postgres
+      ABRECHNUNG_DATABASE__HOST: postgres
     depends_on:
       postgres:
         condition: service_healthy
@@ -40,30 +40,38 @@ services:
       - "8080:80"
 
   mailer:
+    build:
+      context: .
+      dockerfile: docker/Dockerfile-api
     extends:
       file: docker-compose.base.yaml
       service: mailer
+    env_file: .env
     environment:
-      SMTP_HOST: mail
-      SMTP_PORT: 1025
-      SMTP_MODE: smtp
-      DB_HOST: postgres
-
+      ABRECHNUNG_DATABASE__HOST: postgres
+      ABRECHNUNG_EMAIL__HOST: mail
+      ABRECHNUNG_EMAIL__PORT: 1025
+      ABRECHNUNG_EMAIL__MODE: smtp
     depends_on:
       api:
         condition: service_healthy
     links:
       - postgres
       - "mailhog:mail"
+
   cron:
+    build:
+      context: .
+      dockerfile: docker/Dockerfile-api
     extends:
       file: docker-compose.base.yaml
       service: cron
+    env_file: .env
     environment:
-      SMTP_HOST: mail
-      SMTP_PORT: 1025
-      SMTP_MODE: smtp
-      DB_HOST: postgres
+      ABRECHNUNG_DATABASE__HOST: postgres
+      ABRECHNUNG_EMAIL__HOST: mail
+      ABRECHNUNG_EMAIL__PORT: 1025
+      ABRECHNUNG_EMAIL__MODE: smtp
     links:
       - postgres
       - "mailhog:mail"

docker/Dockerfile-api

@@ -8,7 +8,7 @@ RUN /opt/abrechnung-venv/bin/python3 -m pip install /src
 FROM python:3.10-alpine
 RUN addgroup -S abrechnung && adduser -S abrechnung -G abrechnung && apk add --no-cache curl
 COPY --from=builder /opt/abrechnung-venv/ /opt/abrechnung-venv/
-ADD --chmod=644 --chown=abrechnung:abrechnung config/abrechnung.yaml /etc/abrechnung/abrechnung.yaml
+ADD --chmod=644 --chown=abrechnung:abrechnung docker/abrechnung.yaml /etc/abrechnung/abrechnung.yaml
 ADD --chmod=755 ./docker/entrypoint.py /
 COPY --chown=abrechnung:abrechnung ./docker/crontab /var/spool/cron/crontabs/abrechnung
 USER abrechnung

docker/Dockerfile-devel

@@ -1,13 +1,4 @@
 # syntax=docker/dockerfile:1.3
-FROM python:3.10-alpine as api
-RUN addgroup -S abrechnung && adduser -S abrechnung -G abrechnung \
-  && apk add --no-cache curl
-ADD . /usr/share/abrechnung
-RUN pip install --editable /usr/share/abrechnung
-ADD --chmod=755 ./docker/entrypoint.py /
-COPY --chown=abrechnung:abrechnung ./docker/crontab /var/spool/cron/crontabs/abrechnung
-ENTRYPOINT ["/entrypoint.py"]
-
 FROM node:lts as build
 ADD frontend/ /build/
 RUN cd /build/ && npm install && npx nx build web 

docker/abrechnung.yaml

@@ -0,0 +1,22 @@
+service:
+  url: "https://localhost"
+  api_url: "https://localhost/api"
+  name: "Abrechnung"
+
+database:
+  host: "0.0.0.0"
+  user: "abrechnung"
+  dbname: "abrechnung"
+
+api:
+  host: "0.0.0.0"
+  port: 8080
+  id: default
+
+registration:
+  enabled: false
+
+email:
+  host: "localhost"
+  port: 587
+  mode: "smtp-starttls"

docker/entrypoint.py

@@ -1,73 +1,9 @@
 import subprocess
 import sys
-from os import execlp, execvp, getenv, makedirs
-
-from yaml import dump, safe_load
-
-
-def to_bool(data: str):
-    return data.lower() in [
-        "true",
-        "1",
-        "t",
-        "y",
-        "yes",
-        "on",
-    ]
-
+from os import execlp, execvp
 
 abrechnung_venv_python = "/opt/abrechnung-venv/bin/python3"
 
-print("generating config")
-config = {}
-filename = "/etc/abrechnung/abrechnung.yaml"
-with open(filename, "r", encoding="utf-8") as filehandle:
-    config = safe_load(filehandle)
-
-if not "service" in config:
-    config["service"] = {}
-if not "database" in config:
-    config["database"] = {}
-if not "registration" in config:
-    config["registration"] = {}
-if not "email" in config:
-    config["email"] = {}
-
-config["service"]["url"] = getenv("SERVICE_URL", "https://localhost")
-config["service"]["api_url"] = getenv("SERVICE_API_URL", "https://localhost/api")
-config["service"]["name"] = getenv("SERVICE_NAME", "Abrechnung")
-config["database"]["host"] = getenv("DB_HOST")
-config["database"]["user"] = getenv("DB_USER")
-config["database"]["dbname"] = getenv("DB_NAME")
-config["database"]["password"] = getenv("DB_PASSWORD")
-
-config["api"]["secret_key"] = getenv("ABRECHNUNG_SECRET")
-config["api"]["host"] = "0.0.0.0"
-config["api"]["port"] = int(getenv("ABRECHNUNG_PORT", "8080"))
-config["api"]["id"] = getenv("ABRECHNUNG_ID", "default")
-
-config["registration"]["allow_guest_users"] = to_bool(getenv("REGISTRATION_ALLOW_GUEST_USERS", "false"))
-config["registration"]["enabled"] = to_bool(getenv("REGISTRATION_ENABLED", "false"))
-config["registration"]["valid_email_domains"] = getenv("REGISTRATION_VALID_EMAIL_DOMAINS", "false").split(",")
-
-config["email"]["address"] = getenv("ABRECHNUNG_EMAIL", "")
-config["email"]["host"] = getenv("SMTP_HOST", "localhost")
-config["email"]["port"] = int(getenv("SMTP_PORT", "587"))
-config["email"]["mode"] = getenv("SMTP_MODE", "smtp-starttls")
-
-if getenv("SMTP_USER", None):
-    if not "auth" in config["email"]:
-        config["email"]["auth"] = dict()
-
-    config["email"]["auth"]["username"] = getenv("SMTP_USER", None)
-    config["email"]["auth"]["password"] = getenv("SMTP_PASSWORD", None)
-
-output = dump(config)
-makedirs("/etc/abrechnung/", exist_ok=True)
-with open("/etc/abrechnung/abrechnung.yaml", "w", encoding="utf-8") as file:
-    file.write(output)
-print("config done")
-
 if sys.argv[1] == "api":
     print("migrating ...")
     subprocess.run([abrechnung_venv_python, "-m", "abrechnung", "-vvv", "db", "migrate"], check=True, stdout=sys.stdout)

docs/usage/configuration.rst

@@ -98,6 +98,21 @@ without authentication.
 
 .. _abrechnung-config-all-options:
 
+Configuration via Environment Variables
+----------------------
+
+All of the configuration options set in the config yaml file can also be set via environment variables.
+The respective environment variable name for a config variable is in the pattern ``ABRECHNUNG_<config section>__<variable name in capslock>``.
+
+E.g. to set the email auth username from the config yaml as below we'd use the environment variable ``ABRECHNUNG_EMAIL__AUTH__USERNAME``.
+
+.. code-block:: yaml
+
+  email:
+    auth:
+      username: "..."
+
+
 Frontend Configuration
 -------------------------
 

docs/usage/installation.rst

@@ -58,18 +58,3 @@ Then a simple simple ::
   docker-compose -f docker-compose.prod.yaml up 
 
 Should suffice to get you up and running.
-
-Pip
----------------
-
-TODO
-
-From Release Tarball
---------------------
-
-TODO
-
-From Source
----------------
-
-TODO

pyproject.toml

@@ -22,6 +22,7 @@ dependencies = [
     "typer~=0.9.0",
     "fastapi==0.108.0",
     "pydantic[email]~=2.4.0",
+    "pydantic-settings==2.1.0",
     "uvicorn[standard]~=0.23.0",
     "python-jose[cryptography]~=3.3.0",
     "asyncpg~=0.28.0",
@@ -35,6 +36,7 @@ dependencies = [
 test = [
     "aiosmtpd~=1.4",
     "pytest",
+    "pytest-asyncio",
     "pytest-cov",
     "httpx~=0.23",
 ]

tests/test_config.py

@@ -0,0 +1,30 @@
+import os
+import tempfile
+from pathlib import Path
+
+from abrechnung.config import read_config
+
+docker_base_config = (Path(__file__).parent.parent / "docker" / "abrechnung.yaml").read_text()
+
+
+def test_config_load_from_env():
+    os.environ["ABRECHNUNG_SERVICE__NAME"] = "my abrechnung"
+    os.environ["ABRECHNUNG_API__SECRET_KEY"] = "secret"
+    os.environ["ABRECHNUNG_DATABASE__HOST"] = "localhost"
+    os.environ["ABRECHNUNG_DATABASE__DBNAME"] = "abrechnung"
+    os.environ["ABRECHNUNG_DATABASE__PASSWORD"] = "password"
+    os.environ["ABRECHNUNG_DATABASE__USER"] = "abrechnung"
+    os.environ["ABRECHNUNG_EMAIL__ADDRESS"] = "[email protected]"
+    with tempfile.NamedTemporaryFile() as f:
+        filename = Path(f.name)
+        filename.write_text(docker_base_config, "utf-8")
+
+        loaded_cfg = read_config(filename)
+
+        assert loaded_cfg.service.name == "my abrechnung"
+        assert loaded_cfg.api.secret_key == "secret"
+        assert loaded_cfg.database.user == "abrechnung"
+        assert loaded_cfg.database.host == "localhost"
+        assert loaded_cfg.database.dbname == "abrechnung"
+        assert loaded_cfg.database.password == "password"
+        assert loaded_cfg.email.address == "[email protected]"