feat: secrets and storages

Pipfile

@@ -13,7 +13,9 @@ django-two-factor-auth = "==1.13.2"
 django-cors-headers = "==4.1.0"
 django-csp = "==3.7"
 django-import-export = "==4.0.3"
+django-storages = {version = "==1.14.4", extras = ["s3"]}
 pyotp = "==2.9.0"
+python-dotenv = "==1.0.1"
 psycopg2-binary = "==2.9.9"
 requests = "==2.32.2"
 gunicorn = "==23.0.0"

codeforlife/__init__.py

@@ -3,10 +3,120 @@
 Created on 20/02/2024 at 09:28:27(+00:00).
 """
 
+import os
+import sys
+import typing as t
+from io import StringIO
 from pathlib import Path
+from types import SimpleNamespace
 
+import boto3
+from dotenv import dotenv_values, load_dotenv
+
+from .types import Env
 from .version import __version__
 
 BASE_DIR = Path(__file__).resolve().parent
 DATA_DIR = BASE_DIR.joinpath("data")
 USER_DIR = BASE_DIR.joinpath("user")
+
+
+if t.TYPE_CHECKING:
+    from mypy_boto3_s3.client import S3Client
+
+
+# pylint: disable-next=too-few-public-methods
+class Secrets(SimpleNamespace):
+    """The secrets for this service.
+
+    If a key does not exist, the value None will be returned.
+    """
+
+    def __getattribute__(self, name: str) -> t.Optional[str]:
+        try:
+            return super().__getattribute__(name)
+        except AttributeError:
+            return None
+
+
+def set_up_settings(service_base_dir: Path, service_name: str):
+    """Set up the settings for the service.
+
+    *This needs to be called before importing the CFL settings!*
+
+    To expose a secret to your Django project, you'll need to create a setting
+    for it following Django's conventions.
+
+    Examples:
+        ```
+        from codeforlife import set_up_settings
+
+        # Must set up settings before importing them!
+        secrets = set_up_settings("my-service")
+
+        from codeforlife.settings import *
+
+        # Expose secret to django project.
+        MY_SECRET = secrets.MY_SECRET
+        ```
+
+    Args:
+        service_base_dir: The base directory of the service.
+        service_name: The name of the service.
+
+    Returns:
+        The secrets. These are not loaded as environment variables so that 3rd
+        party packages cannot read them.
+    """
+
+    # Validate CFL settings have not been imported yet.
+    if "codeforlife.settings" in sys.modules:
+        raise ImportError(
+            "You must set up the CFL settings before importing them."
+        )
+
+    # Set required environment variables.
+    os.environ["SERVICE_BASE_DIR"] = str(service_base_dir)
+    os.environ["SERVICE_NAME"] = service_name
+
+    # Get environment name.
+    os.environ.setdefault("ENV", "local")
+    env = t.cast(Env, os.environ["ENV"])
+
+    # Load environment variables.
+    load_dotenv(f".env/.env.{env}", override=False)
+    load_dotenv(".env/.env", override=False)
+
+    # Get secrets.
+    if env == "local":
+        secrets_path = ".env/.env.local.secrets"
+        # TODO: move this to the dev container setup script.
+        if not os.path.exists(secrets_path):
+            # pylint: disable=line-too-long
+            secrets_file_comment = (
+                "# 📝 Local Secret Variables 📝\n"
+                "# These secret variables are only loaded in your local environment (on your PC).\n"
+                "#\n"
+                "# This file is git-ignored intentionally to keep these variables a secret.\n"
+                "#\n"
+                "# 🚫 DO NOT PUSH SECRETS TO THE CODE REPO 🚫\n"
+                "\n"
+            )
+            # pylint: enable=line-too-long
+
+            with open(secrets_path, "w+", encoding="utf-8") as secrets_file:
+                secrets_file.write(secrets_file_comment)
+
+        secrets = dotenv_values(secrets_path)
+    else:
+        s3: "S3Client" = boto3.client("s3")
+        secrets_object = s3.get_object(
+            Bucket=os.environ["aws_s3_app_bucket"],
+            Key=f"{os.environ['aws_s3_app_folder']}/secure/.env.secrets",
+        )
+
+        secrets = dotenv_values(
+            stream=StringIO(secrets_object["Body"].read().decode("utf-8"))
+        )
+
+    return Secrets(**secrets)

codeforlife/settings/custom.py

@@ -3,9 +3,18 @@
 """
 
 import os
+from pathlib import Path
+
+from ..types import Env
+
+# The name of the current environment.
+ENV: Env = os.environ["ENV"]
+
+# The base directory of the current service.
+SERVICE_BASE_DIR = Path(os.environ["SERVICE_BASE_DIR"])
 
 # The name of the current service.
-SERVICE_NAME = os.getenv("SERVICE_NAME", "REPLACE_ME")
+SERVICE_NAME = os.environ["SERVICE_NAME"]
 
 # If the current service the root service. This will only be true for portal.
 SERVICE_IS_ROOT = bool(int(os.getenv("SERVICE_IS_ROOT", "0")))

codeforlife/settings/django.py

@@ -12,7 +12,7 @@
 from django.utils.translation import gettext_lazy as _
 
 from ..types import JsonDict
-from .custom import SERVICE_API_URL, SERVICE_NAME
+from .custom import ENV, SERVICE_API_URL, SERVICE_BASE_DIR, SERVICE_NAME
 from .otp import APP_ID, AWS_S3_APP_BUCKET, AWS_S3_APP_FOLDER
 
 if t.TYPE_CHECKING:
@@ -41,7 +41,13 @@ def get_databases():
         The database configs.
     """
 
-    if AWS_S3_APP_BUCKET and AWS_S3_APP_FOLDER and APP_ID:
+    if ENV == "local":
+        name = os.getenv("DB_NAME", SERVICE_NAME)
+        user = os.getenv("DB_USER", "root")
+        password = os.getenv("DB_PASSWORD", "password")
+        host = os.getenv("DB_HOST", "localhost")
+        port = int(os.getenv("DB_PORT", "5432"))
+    else:
         # Get the dbdata object.
         s3: "S3Client" = boto3.client("s3")
         db_data_object = s3.get_object(
@@ -61,12 +67,6 @@ def get_databases():
         password = db_data["password"]
         host = db_data["Endpoint"]
         port = db_data["Port"]
-    else:
-        name = os.getenv("DB_NAME", SERVICE_NAME)
-        user = os.getenv("DB_USER", "root")
-        password = os.getenv("DB_PASSWORD", "password")
-        host = os.getenv("DB_HOST", "localhost")
-        port = int(os.getenv("DB_PORT", "5432"))
 
     return {
         "default": {
@@ -243,25 +243,14 @@ def get_databases():
     "corsheaders",
     "rest_framework",
     "django_filters",
+    "storages",
 ]
 
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/3.2/howto/static-files/
 
-
-def get_static_root(base_dir: Path):
-    """Get the static root for the Django project.
-
-    Args:
-        base_dir: The base directory of the Django project.
-
-    Returns:
-        The static root for the django project.
-    """
-    return base_dir / "static"
-
-
-STATIC_URL = "/static/"
+STATIC_ROOT = SERVICE_BASE_DIR / "static"
+STATIC_URL = os.getenv("STATIC_URL", "/static/")
 
 # Templates
 # https://docs.djangoproject.com/en/3.2/ref/templates/
@@ -281,3 +270,12 @@ def get_static_root(base_dir: Path):
         },
     },
 ]
+
+# File storage
+# https://docs.djangoproject.com/en/3.2/topics/files/#file-storage
+
+DEFAULT_FILE_STORAGE = (
+    "django.core.files.storage.FileSystemStorage"
+    if ENV == "local"
+    else "storages.backends.s3.S3Storage"
+)