diff --git a/Pipfile b/Pipfile index ac9c1e3..de4f3ff 100644 --- a/Pipfile +++ b/Pipfile @@ -13,7 +13,9 @@ django-two-factor-auth = "==1.13.2" django-cors-headers = "==4.1.0" django-csp = "==3.7" django-import-export = "==4.0.3" +django-storages = {version = "==1.14.4", extras = ["s3"]} pyotp = "==2.9.0" +python-dotenv = "==1.0.1" psycopg2-binary = "==2.9.9" requests = "==2.32.2" gunicorn = "==23.0.0" diff --git a/Pipfile.lock b/Pipfile.lock index 3dc0cbb..d054c1b 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "95bf4e62cd12de65fdc81a5326dc1bbfa6a6807c50067ed0c2dcd8fa7b6bd35e" + "sha256": "172ff0e75008059427fdc3d00096720a746c11fc54340cea0363e723992b3fca" }, "pipfile-spec": 6, "requires": { @@ -317,6 +317,17 @@ ], "version": "==2.0.0" }, + "django-storages": { + "extras": [ + "s3" + ], + "hashes": [ + "sha256:69aca94d26e6714d14ad63f33d13619e697508ee33ede184e462ed766dc2a73f", + "sha256:d61930acb4a25e3aebebc6addaf946a3b1df31c803a6bf1af2f31c9047febaa3" + ], + "markers": "python_version >= '3.7'", + "version": "==1.14.4" + }, "django-treebeard": { "hashes": [ "sha256:83aebc34a9f06de7daaec330d858d1c47887e81be3da77e3541fe7368196dd8a" @@ -735,6 +746,15 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.9.0.post0" }, + "python-dotenv": { + "hashes": [ + "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca", + "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a" + ], + "index": "pypi", + "markers": "python_version >= '3.8'", + "version": "==1.0.1" + }, "pytz": { "hashes": [ "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a", @@ -895,11 +915,11 @@ }, "six": { "hashes": [ - "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", - "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", + "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.16.0" + "version": "==1.17.0" }, "sqlparse": { "hashes": [ @@ -1026,11 +1046,11 @@ }, "botocore-stubs": { "hashes": [ - "sha256:7e938bb169c28faf05ce14e67bb0b5e5583092ab6ccc9d3d68d698530edb6584", - "sha256:c5f7208b20ae19400fa73eb569017f1e372990f7a5505a72116ed6420904f666" + "sha256:617508d023e0bc98901e0189b794c4b3f289c1747c7cc410173ad698c819a716", + "sha256:c977a049481d50a14bf2db0ef15020b76734ff628d4b8e0e77b8d1c65318369e" ], "markers": "python_version >= '3.8'", - "version": "==1.35.71" + "version": "==1.35.76" }, "certifi": { "hashes": [ @@ -1379,17 +1399,17 @@ }, "mypy-boto3-dynamodb": { "hashes": [ - "sha256:187915c781f352bc79d35b08a094605515ecc54f30107f629972c3358b864a5c", - "sha256:92eac35c49e9f3ff23a4ad6dee5dc54e410e0c49a98b4d93493c7000ebe74568" + "sha256:a815d044b8f5f4ba308ea3114916565fbd932fcaf218f8d0288b2840415f9c46", + "sha256:b693b459abb1910cbb28f3a478ced8c6e6515f1bf136b45aca1a76b6146b5adb" ], - "version": "==1.35.60" + "version": "==1.35.74" }, "mypy-boto3-ec2": { "hashes": [ - "sha256:93f9ddadac303d63f34cd4c0a60a682c008d655d3b2cfa74d1234fbde9a0b401", - "sha256:d5b27b79b1749fb10a4eb9508069995a8e4bf2614f4e171224637596403e42c8" + "sha256:3206cd6da473647cdefa5dcec4121b4a83778f49ee540ca4b8aeb6c337975b69", + "sha256:d2ff43ad1c42655cbcbb06d11dff74b3827503d80a99a78098ab52ba0fbb7235" ], - "version": "==1.35.70" + "version": "==1.35.72" }, "mypy-boto3-lambda": { "hashes": [ @@ -1400,17 +1420,17 @@ }, "mypy-boto3-rds": { "hashes": [ - "sha256:0850cb5bddda1853c6ba44bb8dc1bf0d303ea4729f8cdf982d0e4d91f08ab2d9", - "sha256:7bfeadfbd361aaf53a5f161c571886d3cadbdf05c15591761280fe6f079ab273" + "sha256:4c345e616a7767953284a0d54ab6dbabd8b068fe353b34194b79364b47176b61", + "sha256:acd87fdfd12cc8f7298f586734f5c5e7afb0fcd6da8154a10cccb5730cc5c799" ], - "version": "==1.35.66" + "version": "==1.35.72" }, "mypy-boto3-s3": { "hashes": [ - "sha256:11a34259983e09d67e4d3a322fd47904a006bbfff19984e4e36a77e30f2014bb", - "sha256:97f7944a84a4a49282825bef1483a25680dcdce75da6017745d709d2cf2aa1c0" + "sha256:35f9ae109c3cb64ac6b44596dffc429058085ddb82f4daaf5be0a39e5cc1b576", + "sha256:6cf1f034985fe610754c3e6ef287490629870d508ada13b7d61e7b9aaeb46108" ], - "version": "==1.35.69" + "version": "==1.35.76" }, "mypy-boto3-sqs": { "hashes": [ @@ -1606,11 +1626,11 @@ }, "types-awscrt": { "hashes": [ - "sha256:0d362a5d62d68ca4216f458172f41c1123ec04791d68364de8ee8b61b528b262", - "sha256:a20b425dabb258bc3d07a5e7de503fd9558dd1542d72de796e74e402c6d493b2" + "sha256:043c0ae0fe5d272618294cbeaf1c349a654a9f7c00121be64d27486933ac4a26", + "sha256:cc0057885cb7ce1e66856123a4c2861b051e9f0716b1767ad72bfe4ca26bbcd4" ], "markers": "python_version >= '3.8'", - "version": "==0.23.1" + "version": "==0.23.3" }, "types-pytz": { "hashes": [ diff --git a/codeforlife/__init__.py b/codeforlife/__init__.py index cce0923..c585df8 100644 --- a/codeforlife/__init__.py +++ b/codeforlife/__init__.py @@ -3,10 +3,120 @@ Created on 20/02/2024 at 09:28:27(+00:00). """ +import os +import sys +import typing as t +from io import StringIO from pathlib import Path +from types import SimpleNamespace +import boto3 +from dotenv import dotenv_values, load_dotenv + +from .types import Env from .version import __version__ BASE_DIR = Path(__file__).resolve().parent DATA_DIR = BASE_DIR.joinpath("data") USER_DIR = BASE_DIR.joinpath("user") + + +if t.TYPE_CHECKING: + from mypy_boto3_s3.client import S3Client + + +# pylint: disable-next=too-few-public-methods +class Secrets(SimpleNamespace): + """The secrets for this service. + + If a key does not exist, the value None will be returned. + """ + + def __getattribute__(self, name: str) -> t.Optional[str]: + try: + return super().__getattribute__(name) + except AttributeError: + return None + + +def set_up_settings(service_base_dir: Path, service_name: str): + """Set up the settings for the service. + + *This needs to be called before importing the CFL settings!* + + To expose a secret to your Django project, you'll need to create a setting + for it following Django's conventions. + + Examples: + ``` + from codeforlife import set_up_settings + + # Must set up settings before importing them! + secrets = set_up_settings("my-service") + + from codeforlife.settings import * + + # Expose secret to django project. + MY_SECRET = secrets.MY_SECRET + ``` + + Args: + service_base_dir: The base directory of the service. + service_name: The name of the service. + + Returns: + The secrets. These are not loaded as environment variables so that 3rd + party packages cannot read them. + """ + + # Validate CFL settings have not been imported yet. + if "codeforlife.settings" in sys.modules: + raise ImportError( + "You must set up the CFL settings before importing them." + ) + + # Set required environment variables. + os.environ["SERVICE_BASE_DIR"] = str(service_base_dir) + os.environ["SERVICE_NAME"] = service_name + + # Get environment name. + os.environ.setdefault("ENV", "local") + env = t.cast(Env, os.environ["ENV"]) + + # Load environment variables. + load_dotenv(f".env/.env.{env}", override=False) + load_dotenv(".env/.env", override=False) + + # Get secrets. + if env == "local": + secrets_path = ".env/.env.local.secrets" + # TODO: move this to the dev container setup script. + if not os.path.exists(secrets_path): + # pylint: disable=line-too-long + secrets_file_comment = ( + "# 📝 Local Secret Variables 📝\n" + "# These secret variables are only loaded in your local environment (on your PC).\n" + "#\n" + "# This file is git-ignored intentionally to keep these variables a secret.\n" + "#\n" + "# 🚫 DO NOT PUSH SECRETS TO THE CODE REPO 🚫\n" + "\n" + ) + # pylint: enable=line-too-long + + with open(secrets_path, "w+", encoding="utf-8") as secrets_file: + secrets_file.write(secrets_file_comment) + + secrets = dotenv_values(secrets_path) + else: + s3: "S3Client" = boto3.client("s3") + secrets_object = s3.get_object( + Bucket=os.environ["aws_s3_app_bucket"], + Key=f"{os.environ['aws_s3_app_folder']}/secure/.env.secrets", + ) + + secrets = dotenv_values( + stream=StringIO(secrets_object["Body"].read().decode("utf-8")) + ) + + return Secrets(**secrets) diff --git a/codeforlife/settings/custom.py b/codeforlife/settings/custom.py index f90ac72..8f73e8f 100644 --- a/codeforlife/settings/custom.py +++ b/codeforlife/settings/custom.py @@ -3,9 +3,18 @@ """ import os +from pathlib import Path + +from ..types import Env + +# The name of the current environment. +ENV: Env = os.environ["ENV"] + +# The base directory of the current service. +SERVICE_BASE_DIR = Path(os.environ["SERVICE_BASE_DIR"]) # The name of the current service. -SERVICE_NAME = os.getenv("SERVICE_NAME", "REPLACE_ME") +SERVICE_NAME = os.environ["SERVICE_NAME"] # If the current service the root service. This will only be true for portal. SERVICE_IS_ROOT = bool(int(os.getenv("SERVICE_IS_ROOT", "0"))) diff --git a/codeforlife/settings/django.py b/codeforlife/settings/django.py index 62bd8e8..a23cb35 100644 --- a/codeforlife/settings/django.py +++ b/codeforlife/settings/django.py @@ -12,7 +12,7 @@ from django.utils.translation import gettext_lazy as _ from ..types import JsonDict -from .custom import SERVICE_API_URL, SERVICE_NAME +from .custom import ENV, SERVICE_API_URL, SERVICE_BASE_DIR, SERVICE_NAME from .otp import APP_ID, AWS_S3_APP_BUCKET, AWS_S3_APP_FOLDER if t.TYPE_CHECKING: @@ -41,7 +41,13 @@ def get_databases(): The database configs. """ - if AWS_S3_APP_BUCKET and AWS_S3_APP_FOLDER and APP_ID: + if ENV == "local": + name = os.getenv("DB_NAME", SERVICE_NAME) + user = os.getenv("DB_USER", "root") + password = os.getenv("DB_PASSWORD", "password") + host = os.getenv("DB_HOST", "localhost") + port = int(os.getenv("DB_PORT", "5432")) + else: # Get the dbdata object. s3: "S3Client" = boto3.client("s3") db_data_object = s3.get_object( @@ -61,12 +67,6 @@ def get_databases(): password = db_data["password"] host = db_data["Endpoint"] port = db_data["Port"] - else: - name = os.getenv("DB_NAME", SERVICE_NAME) - user = os.getenv("DB_USER", "root") - password = os.getenv("DB_PASSWORD", "password") - host = os.getenv("DB_HOST", "localhost") - port = int(os.getenv("DB_PORT", "5432")) return { "default": { @@ -243,25 +243,14 @@ def get_databases(): "corsheaders", "rest_framework", "django_filters", + "storages", ] # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/3.2/howto/static-files/ - -def get_static_root(base_dir: Path): - """Get the static root for the Django project. - - Args: - base_dir: The base directory of the Django project. - - Returns: - The static root for the django project. - """ - return base_dir / "static" - - -STATIC_URL = "/static/" +STATIC_ROOT = SERVICE_BASE_DIR / "static" +STATIC_URL = os.getenv("STATIC_URL", "/static/") # Templates # https://docs.djangoproject.com/en/3.2/ref/templates/ @@ -281,3 +270,12 @@ def get_static_root(base_dir: Path): }, }, ] + +# File storage +# https://docs.djangoproject.com/en/3.2/topics/files/#file-storage + +DEFAULT_FILE_STORAGE = ( + "django.core.files.storage.FileSystemStorage" + if ENV == "local" + else "storages.backends.s3.S3Storage" +) diff --git a/codeforlife/settings/third_party.py b/codeforlife/settings/third_party.py index 700f6ee..891b758 100644 --- a/codeforlife/settings/third_party.py +++ b/codeforlife/settings/third_party.py @@ -2,6 +2,8 @@ This file contains custom settings defined by third party extensions. """ +import os + from .django import DEBUG # CORS @@ -23,3 +25,33 @@ ], "DEFAULT_PAGINATION_CLASS": "codeforlife.pagination.LimitOffsetPagination", } + +# Django storages +# https://django-storages.readthedocs.io/en/latest/backends/amazon-S3.html#settings + +AWS_STORAGE_BUCKET_NAME = os.getenv("AWS_STORAGE_BUCKET_NAME") +AWS_S3_OBJECT_PARAMETERS = os.getenv("AWS_S3_OBJECT_PARAMETERS", {}) +AWS_DEFAULT_ACL = os.getenv("AWS_DEFAULT_ACL") +AWS_QUERYSTRING_AUTH = bool(int(os.getenv("AWS_QUERYSTRING_AUTH", "1"))) +AWS_S3_MAX_MEMORY_SIZE = int(os.getenv("AWS_S3_MAX_MEMORY_SIZE", "0")) +AWS_QUERYSTRING_EXPIRE = int(os.getenv("AWS_QUERYSTRING_EXPIRE", "3600")) +AWS_S3_URL_PROTOCOL = os.getenv("AWS_S3_URL_PROTOCOL", "https:") +AWS_S3_FILE_OVERWRITE = bool(int(os.getenv("AWS_S3_FILE_OVERWRITE", "1"))) +AWS_LOCATION = os.getenv("AWS_LOCATION", "") +AWS_IS_GZIPPED = bool(int(os.getenv("AWS_IS_GZIPPED", "0"))) +GZIP_CONTENT_TYPES = os.getenv( + "GZIP_CONTENT_TYPES", + "(text/css,text/javascript,application/javascript,application/x-javascript,image/svg+xml)", +) +AWS_S3_REGION_NAME = os.getenv("AWS_S3_REGION_NAME") +AWS_S3_USE_SSL = bool(int(os.getenv("AWS_S3_USE_SSL", "1"))) +AWS_S3_VERIFY = os.getenv("AWS_S3_VERIFY") +AWS_S3_ENDPOINT_URL = os.getenv("AWS_S3_ENDPOINT_URL") +AWS_S3_ADDRESSING_STYLE = os.getenv("AWS_S3_ADDRESSING_STYLE") +AWS_S3_PROXIES = os.getenv("AWS_S3_PROXIES") +AWS_S3_TRANSFER_CONFIG = os.getenv("AWS_S3_TRANSFER_CONFIG") +AWS_S3_CUSTOM_DOMAIN = os.getenv("AWS_S3_CUSTOM_DOMAIN") +AWS_CLOUDFRONT_KEY = os.getenv("AWS_CLOUDFRONT_KEY") +AWS_CLOUDFRONT_KEY_ID = os.getenv("AWS_CLOUDFRONT_KEY_ID") +AWS_S3_SIGNATURE_VERSION = os.getenv("AWS_S3_SIGNATURE_VERSION") +AWS_S3_CLIENT_CONFIG = os.getenv("AWS_S3_CLIENT_CONFIG") diff --git a/codeforlife/types.py b/codeforlife/types.py index 68524a1..6d2f9b1 100644 --- a/codeforlife/types.py +++ b/codeforlife/types.py @@ -7,6 +7,8 @@ import typing as t +Env = t.Literal["local", "development", "staging", "production"] + Args = t.Tuple[t.Any, ...] KwArgs = t.Dict[str, t.Any]