Skip to content

Commit

Permalink
feat: secrets and storages (#148)
Browse files Browse the repository at this point in the history
* feat: secrets and storages

* cast env

* fix: key errors

* fix: urls

* AWS_S3_OBJECT_PARAMETERS

* fix: types

* fix: linting

* refresh pipfile.lock

* fix: imports
  • Loading branch information
SKairinos authored Dec 6, 2024
1 parent 4a1bb70 commit b960b60
Show file tree
Hide file tree
Showing 11 changed files with 360 additions and 191 deletions.
2 changes: 2 additions & 0 deletions Pipfile
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@ django-two-factor-auth = "==1.13.2"
django-cors-headers = "==4.1.0"
django-csp = "==3.7"
django-import-export = "==4.0.3"
django-storages = {version = "==1.14.4", extras = ["s3"]}
pyotp = "==2.9.0"
python-dotenv = "==1.0.1"
psycopg2-binary = "==2.9.9"
requests = "==2.32.2"
gunicorn = "==23.0.0"
Expand Down
194 changes: 107 additions & 87 deletions Pipfile.lock

Large diffs are not rendered by default.

113 changes: 113 additions & 0 deletions codeforlife/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,123 @@
Created on 20/02/2024 at 09:28:27(+00:00).
"""

import os
import sys
import typing as t
from io import StringIO
from pathlib import Path
from types import SimpleNamespace

from .types import Env
from .version import __version__

BASE_DIR = Path(__file__).resolve().parent
DATA_DIR = BASE_DIR.joinpath("data")
USER_DIR = BASE_DIR.joinpath("user")


if t.TYPE_CHECKING:
from mypy_boto3_s3.client import S3Client


# pylint: disable-next=too-few-public-methods
class Secrets(SimpleNamespace):
"""The secrets for this service.
If a key does not exist, the value None will be returned.
"""

def __getattribute__(self, name: str) -> t.Optional[str]:
try:
return super().__getattribute__(name)
except AttributeError:
return None


def set_up_settings(service_base_dir: Path, service_name: str):
"""Set up the settings for the service.
*This needs to be called before importing the CFL settings!*
To expose a secret to your Django project, you'll need to create a setting
for it following Django's conventions.
Examples:
```
from codeforlife import set_up_settings
# Must set up settings before importing them!
secrets = set_up_settings("my-service")
from codeforlife.settings import *
# Expose secret to django project.
MY_SECRET = secrets.MY_SECRET
```
Args:
service_base_dir: The base directory of the service.
service_name: The name of the service.
Returns:
The secrets. These are not loaded as environment variables so that 3rd
party packages cannot read them.
"""

# Validate CFL settings have not been imported yet.
if "codeforlife.settings" in sys.modules:
raise ImportError(
"You must set up the CFL settings before importing them."
)

# pylint: disable-next=import-outside-toplevel
from dotenv import dotenv_values, load_dotenv

# Set required environment variables.
os.environ["SERVICE_BASE_DIR"] = str(service_base_dir)
os.environ["SERVICE_NAME"] = service_name

# Get environment name.
os.environ.setdefault("ENV", "local")
env = t.cast(Env, os.environ["ENV"])

# Load environment variables.
load_dotenv(f".env/.env.{env}", override=False)
load_dotenv(".env/.env", override=False)

# Get secrets.
if env == "local":
secrets_path = ".env/.env.local.secrets"
# TODO: move this to the dev container setup script.
if not os.path.exists(secrets_path):
# pylint: disable=line-too-long
secrets_file_comment = (
"# 📝 Local Secret Variables 📝\n"
"# These secret variables are only loaded in your local environment (on your PC).\n"
"#\n"
"# This file is git-ignored intentionally to keep these variables a secret.\n"
"#\n"
"# 🚫 DO NOT PUSH SECRETS TO THE CODE REPO 🚫\n"
"\n"
)
# pylint: enable=line-too-long

with open(secrets_path, "w+", encoding="utf-8") as secrets_file:
secrets_file.write(secrets_file_comment)

secrets = dotenv_values(secrets_path)
else:
# pylint: disable-next=import-outside-toplevel
import boto3

s3: "S3Client" = boto3.client("s3")
secrets_object = s3.get_object(
Bucket=os.environ["aws_s3_app_bucket"],
Key=f"{os.environ['aws_s3_app_folder']}/secure/.env.secrets",
)

secrets = dotenv_values(
stream=StringIO(secrets_object["Body"].read().decode("utf-8"))
)

return Secrets(**secrets)
28 changes: 13 additions & 15 deletions codeforlife/settings/custom.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,20 @@
"""

import os
import typing as t
from pathlib import Path

from ..types import Env

# The name of the current environment.
ENV = t.cast(Env, os.getenv("ENV", "local"))

# The base directory of the current service.
SERVICE_BASE_DIR = Path(os.getenv("SERVICE_BASE_DIR", "/"))

# The name of the current service.
SERVICE_NAME = os.getenv("SERVICE_NAME", "REPLACE_ME")

# If the current service the root service. This will only be true for portal.
SERVICE_IS_ROOT = bool(int(os.getenv("SERVICE_IS_ROOT", "0")))

# The protocol, domain and port of the current service.
SERVICE_PROTOCOL = os.getenv("SERVICE_PROTOCOL", "http")
SERVICE_DOMAIN = os.getenv("SERVICE_DOMAIN", "localhost")
Expand All @@ -18,18 +25,9 @@
# The base url of the current service.
# The root service does not need its name included in the base url.
SERVICE_BASE_URL = f"{SERVICE_PROTOCOL}://{SERVICE_DOMAIN}:{SERVICE_PORT}"
if not SERVICE_IS_ROOT:
SERVICE_BASE_URL += f"/{SERVICE_NAME}"

# The api url of the current service.
SERVICE_API_URL = f"{SERVICE_BASE_URL}/api"

# The website url of the current service.
SERVICE_SITE_URL = (
"http://localhost:5173"
if SERVICE_DOMAIN == "localhost"
else SERVICE_BASE_URL
)

# The frontend url of the current service.
SERVICE_SITE_URL = os.getenv("SERVICE_SITE_URL", "http://localhost:5173")

# The authorization bearer token used to authenticate with Dotdigital.
MAIL_AUTH = os.getenv("MAIL_AUTH", "REPLACE_ME")
Expand Down
57 changes: 27 additions & 30 deletions codeforlife/settings/django.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,12 @@
import json
import os
import typing as t
from pathlib import Path

import boto3
from django.utils.translation import gettext_lazy as _

from ..types import JsonDict
from .custom import SERVICE_API_URL, SERVICE_NAME
from .custom import ENV, SERVICE_BASE_DIR, SERVICE_BASE_URL, SERVICE_NAME
from .otp import APP_ID, AWS_S3_APP_BUCKET, AWS_S3_APP_FOLDER

if t.TYPE_CHECKING:
Expand Down Expand Up @@ -41,11 +40,17 @@ def get_databases():
The database configs.
"""

if AWS_S3_APP_BUCKET and AWS_S3_APP_FOLDER and APP_ID:
if ENV == "local":
name = os.getenv("DB_NAME", SERVICE_NAME)
user = os.getenv("DB_USER", "root")
password = os.getenv("DB_PASSWORD", "password")
host = os.getenv("DB_HOST", "localhost")
port = int(os.getenv("DB_PORT", "5432"))
else:
# Get the dbdata object.
s3: "S3Client" = boto3.client("s3")
db_data_object = s3.get_object(
Bucket=AWS_S3_APP_BUCKET,
Bucket=t.cast(str, AWS_S3_APP_BUCKET),
Key=f"{AWS_S3_APP_FOLDER}/dbMetadata/{APP_ID}/app.dbdata",
)

Expand All @@ -56,17 +61,11 @@ def get_databases():
if not db_data or db_data["DBEngine"] != "postgres":
raise ConnectionAbortedError("Invalid database data.")

name = db_data["Database"]
user = db_data["user"]
password = db_data["password"]
host = db_data["Endpoint"]
port = db_data["Port"]
else:
name = os.getenv("DB_NAME", SERVICE_NAME)
user = os.getenv("DB_USER", "root")
password = os.getenv("DB_PASSWORD", "password")
host = os.getenv("DB_HOST", "localhost")
port = int(os.getenv("DB_PORT", "5432"))
name = t.cast(str, db_data["Database"])
user = t.cast(str, db_data["user"])
password = t.cast(str, db_data["password"])
host = t.cast(str, db_data["Endpoint"])
port = t.cast(int, db_data["Port"])

return {
"default": {
Expand Down Expand Up @@ -104,7 +103,7 @@ def get_databases():
# Auth
# https://docs.djangoproject.com/en/3.2/topics/auth/default/

LOGIN_URL = f"{SERVICE_API_URL}/session/expired/"
LOGIN_URL = f"{SERVICE_BASE_URL}/session/expired/"

# Authentication backends
# https://docs.djangoproject.com/en/3.2/ref/settings/#authentication-backends
Expand Down Expand Up @@ -243,25 +242,14 @@ def get_databases():
"corsheaders",
"rest_framework",
"django_filters",
"storages",
]

# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/3.2/howto/static-files/


def get_static_root(base_dir: Path):
"""Get the static root for the Django project.
Args:
base_dir: The base directory of the Django project.
Returns:
The static root for the django project.
"""
return base_dir / "static"


STATIC_URL = "/static/"
STATIC_ROOT = SERVICE_BASE_DIR / "static"
STATIC_URL = os.getenv("STATIC_URL", "/static/")

# Templates
# https://docs.djangoproject.com/en/3.2/ref/templates/
Expand All @@ -281,3 +269,12 @@ def get_static_root(base_dir: Path):
},
},
]

# File storage
# https://docs.djangoproject.com/en/3.2/topics/files/#file-storage

DEFAULT_FILE_STORAGE = (
"django.core.files.storage.FileSystemStorage"
if ENV == "local"
else "storages.backends.s3.S3Storage"
)
45 changes: 45 additions & 0 deletions codeforlife/settings/third_party.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@
This file contains custom settings defined by third party extensions.
"""

import json
import os

from .django import DEBUG

# CORS
Expand All @@ -23,3 +26,45 @@
],
"DEFAULT_PAGINATION_CLASS": "codeforlife.pagination.LimitOffsetPagination",
}

# Django storages
# https://django-storages.readthedocs.io/en/latest/backends/amazon-S3.html#settings

AWS_STORAGE_BUCKET_NAME = os.getenv("AWS_STORAGE_BUCKET_NAME")
AWS_S3_OBJECT_PARAMETERS = json.loads(
os.getenv("AWS_S3_OBJECT_PARAMETERS", "{}")
)
AWS_DEFAULT_ACL = os.getenv("AWS_DEFAULT_ACL")
AWS_QUERYSTRING_AUTH = bool(int(os.getenv("AWS_QUERYSTRING_AUTH", "1")))
AWS_S3_MAX_MEMORY_SIZE = int(os.getenv("AWS_S3_MAX_MEMORY_SIZE", "0"))
AWS_QUERYSTRING_EXPIRE = int(os.getenv("AWS_QUERYSTRING_EXPIRE", "3600"))
AWS_S3_URL_PROTOCOL = os.getenv("AWS_S3_URL_PROTOCOL", "https:")
AWS_S3_FILE_OVERWRITE = bool(int(os.getenv("AWS_S3_FILE_OVERWRITE", "1")))
AWS_LOCATION = os.getenv("AWS_LOCATION", "")
AWS_IS_GZIPPED = bool(int(os.getenv("AWS_IS_GZIPPED", "0")))
GZIP_CONTENT_TYPES = os.getenv(
"GZIP_CONTENT_TYPES",
"("
+ ",".join(
[
"text/css",
"text/javascript",
"application/javascript",
"application/x-javascript",
"image/svg+xml",
]
)
+ ")",
)
AWS_S3_REGION_NAME = os.getenv("AWS_S3_REGION_NAME")
AWS_S3_USE_SSL = bool(int(os.getenv("AWS_S3_USE_SSL", "1")))
AWS_S3_VERIFY = os.getenv("AWS_S3_VERIFY")
AWS_S3_ENDPOINT_URL = os.getenv("AWS_S3_ENDPOINT_URL")
AWS_S3_ADDRESSING_STYLE = os.getenv("AWS_S3_ADDRESSING_STYLE")
AWS_S3_PROXIES = os.getenv("AWS_S3_PROXIES")
AWS_S3_TRANSFER_CONFIG = os.getenv("AWS_S3_TRANSFER_CONFIG")
AWS_S3_CUSTOM_DOMAIN = os.getenv("AWS_S3_CUSTOM_DOMAIN")
AWS_CLOUDFRONT_KEY = os.getenv("AWS_CLOUDFRONT_KEY")
AWS_CLOUDFRONT_KEY_ID = os.getenv("AWS_CLOUDFRONT_KEY_ID")
AWS_S3_SIGNATURE_VERSION = os.getenv("AWS_S3_SIGNATURE_VERSION")
AWS_S3_CLIENT_CONFIG = os.getenv("AWS_S3_CLIENT_CONFIG")
2 changes: 2 additions & 0 deletions codeforlife/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@

import typing as t

Env = t.Literal["local", "development", "staging", "production"]

Args = t.Tuple[t.Any, ...]
KwArgs = t.Dict[str, t.Any]

Expand Down
Loading

0 comments on commit b960b60

Please sign in to comment.