From a3b60cbfb6ae5dad4d903d2c4a194aa3a7015a6c Mon Sep 17 00:00:00 2001 From: Faysal Ishtiaq Date: Sat, 29 Oct 2022 14:07:43 +0200 Subject: [PATCH 1/4] override settings from deployment file --- deployment.yml | 4 ++++ src/climsoft_api/api/s3_files/router.py | 10 ++++++---- src/climsoft_api/api/upload/router.py | 11 ++++++----- src/climsoft_api/main.py | 1 + src/climsoft_api/services/file_upload_service.py | 12 ++++++------ src/climsoft_api/utils/deployment.py | 13 +++++++++++++ src/climsoft_api/utils/s3.py | 4 +--- 7 files changed, 37 insertions(+), 18 deletions(-) diff --git a/deployment.yml b/deployment.yml index 15657f2..0caf353 100644 --- a/deployment.yml +++ b/deployment.yml @@ -2,3 +2,7 @@ test: NAME: Climsoft Test DATABASE_URI: "mysql+mysqldb://root:password@mariadb/climsoft" AUTH_ENABLED: false + S3_BUCKET: climsoft-paper-archive + AWS_REGION: eu-west-2 + AWS_ACCESS_KEY_ID: ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: SECRET_ACCESS_KEY diff --git a/src/climsoft_api/api/s3_files/router.py b/src/climsoft_api/api/s3_files/router.py index 5e6d2ed..caa82a9 100644 --- a/src/climsoft_api/api/s3_files/router.py +++ b/src/climsoft_api/api/s3_files/router.py @@ -1,18 +1,20 @@ from climsoft_api.config import settings from climsoft_api.utils.s3 import get_s3_client -from fastapi import APIRouter +from fastapi import APIRouter, Request from fastapi.responses import Response from climsoft_api.utils.exception import handle_exceptions +from climsoft_api.utils.deployment import override_settings router = APIRouter() @router.get("/s3/image/{object_key}") @handle_exceptions -def get_s3_object(object_key): - s3_client = get_s3_client() +def get_s3_object(object_key, request: Request): + _settings = override_settings(request.state.settings_override) + s3_client = get_s3_client(_settings) response = s3_client.get_object( - Bucket=settings.S3_BUCKET, + Bucket=_settings.S3_BUCKET, Key=object_key ) return Response(response["Body"].read(), diff --git a/src/climsoft_api/api/upload/router.py b/src/climsoft_api/api/upload/router.py index da236d5..2985d5f 100644 --- a/src/climsoft_api/api/upload/router.py +++ b/src/climsoft_api/api/upload/router.py @@ -1,14 +1,14 @@ +import json import logging from climsoft_api.api.upload.schema import ( FileUploadedToDiskResponse, FileUploadedToS3Response ) -from climsoft_api.config import settings from climsoft_api.services import file_upload_service from climsoft_api.utils.response import get_success_response, get_error_response -from fastapi import APIRouter, UploadFile, File +from fastapi import APIRouter, UploadFile, File, Request from climsoft_api.utils.response import translate_schema - +from climsoft_api.utils.deployment import override_settings router = APIRouter() @@ -19,7 +19,8 @@ @router.post( "/file-upload/image" ) -async def upload_image(file: UploadFile = File(...)): +async def upload_image(request: Request, file: UploadFile = File(...)): + _settings = override_settings(request.state.settings_override) try: contents = await file.read() file_type = file.content_type @@ -27,7 +28,7 @@ async def upload_image(file: UploadFile = File(...)): raise TypeError(_("Only image files are supported.")) filepath = file_upload_service.save_file( - settings.FILE_STORAGE, + _settings, contents, file_type ) diff --git a/src/climsoft_api/main.py b/src/climsoft_api/main.py index f2f53e3..2b84a09 100644 --- a/src/climsoft_api/main.py +++ b/src/climsoft_api/main.py @@ -39,6 +39,7 @@ def get_app(config=None): async def db_session_middleware(request: Request, call_next): try: request.state.get_session = get_session_local(config) + request.state.settings_override = deployment_configs[config] response = await call_next(request) except Exception as exc: logging.exception(exc) diff --git a/src/climsoft_api/services/file_upload_service.py b/src/climsoft_api/services/file_upload_service.py index 9102a3f..ea487d5 100644 --- a/src/climsoft_api/services/file_upload_service.py +++ b/src/climsoft_api/services/file_upload_service.py @@ -5,18 +5,18 @@ from climsoft_api.utils.s3 import get_s3_client -def save_file(storage, file, file_type): +def save_file(settings, file, file_type): file_name = f"{uuid.uuid4().hex}.{file_type.split('/')[-1]}" - if storage == "disk": + if settings.storage == "disk": return save_file_to_disk(file, file_name) - elif storage == "s3": - return save_file_to_s3(file, file_name) + elif settings.storage == "s3": + return save_file_to_s3(settings, file, file_name) else: raise NotImplemented() -def save_file_to_s3(file, file_name): - s3_client = get_s3_client() +def save_file_to_s3(settings, file, file_name): + s3_client = get_s3_client(settings) s3_client.upload_fileobj( io.BytesIO(file), settings.S3_BUCKET, diff --git a/src/climsoft_api/utils/deployment.py b/src/climsoft_api/utils/deployment.py index 845dec7..aee4dd3 100644 --- a/src/climsoft_api/utils/deployment.py +++ b/src/climsoft_api/utils/deployment.py @@ -1,6 +1,10 @@ +import copy + import yaml from pathlib import Path from typing import Dict +from pydantic import BaseSettings +from climsoft_api.config import settings, Settings deployment_config_file = Path.resolve(Path("./deployment.yml")) @@ -12,3 +16,12 @@ def load_deployment_configs() -> Dict[str, Dict[str, str]]: with open(deployment_config_file, "r") as stream: deployment_configs = yaml.safe_load(stream=stream) return deployment_configs + + +def override_settings(overrides: Dict[str, str]) -> Settings: + if overrides.get("NAME"): + overrides.pop("NAME") + settings_copy = copy.deepcopy(settings) + for k, v in overrides.items(): + setattr(settings_copy, k, v) + return settings_copy diff --git a/src/climsoft_api/utils/s3.py b/src/climsoft_api/utils/s3.py index 25b6d61..8995f62 100644 --- a/src/climsoft_api/utils/s3.py +++ b/src/climsoft_api/utils/s3.py @@ -1,15 +1,13 @@ import boto3 -from climsoft_api.config import settings -def get_s3_client(): +def get_s3_client(settings): s3_client = boto3.client( 's3', aws_access_key_id=settings.AWS_ACCESS_KEY_ID, aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, region_name=settings.AWS_REGION ) - return s3_client From d8e060d48f779e84a933c1ef0400fd891f0388b7 Mon Sep 17 00:00:00 2001 From: Faysal Ishtiaq Date: Sat, 29 Oct 2022 14:16:23 +0200 Subject: [PATCH 2/4] use original settings on attribute error --- src/climsoft_api/api/s3_files/router.py | 6 +++++- src/climsoft_api/api/upload/router.py | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/climsoft_api/api/s3_files/router.py b/src/climsoft_api/api/s3_files/router.py index caa82a9..16c27b3 100644 --- a/src/climsoft_api/api/s3_files/router.py +++ b/src/climsoft_api/api/s3_files/router.py @@ -11,7 +11,11 @@ @router.get("/s3/image/{object_key}") @handle_exceptions def get_s3_object(object_key, request: Request): - _settings = override_settings(request.state.settings_override) + try: + _settings = override_settings(request.state.settings_override) + except AttributeError: + _settings = settings + s3_client = get_s3_client(_settings) response = s3_client.get_object( Bucket=_settings.S3_BUCKET, diff --git a/src/climsoft_api/api/upload/router.py b/src/climsoft_api/api/upload/router.py index 2985d5f..c2dda01 100644 --- a/src/climsoft_api/api/upload/router.py +++ b/src/climsoft_api/api/upload/router.py @@ -9,6 +9,7 @@ from fastapi import APIRouter, UploadFile, File, Request from climsoft_api.utils.response import translate_schema from climsoft_api.utils.deployment import override_settings +from climsoft_api.config import settings router = APIRouter() @@ -20,7 +21,10 @@ "/file-upload/image" ) async def upload_image(request: Request, file: UploadFile = File(...)): - _settings = override_settings(request.state.settings_override) + try: + _settings = override_settings(request.state.settings_override) + except AttributeError: + _settings = settings try: contents = await file.read() file_type = file.content_type From 1a49526896d14cf3f93e05eeec0991cb4327a6f0 Mon Sep 17 00:00:00 2001 From: Faysal Ishtiaq Date: Sat, 29 Oct 2022 14:22:39 +0200 Subject: [PATCH 3/4] put none if required --- src/climsoft_api/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/climsoft_api/main.py b/src/climsoft_api/main.py index 2b84a09..2eae435 100644 --- a/src/climsoft_api/main.py +++ b/src/climsoft_api/main.py @@ -39,7 +39,7 @@ def get_app(config=None): async def db_session_middleware(request: Request, call_next): try: request.state.get_session = get_session_local(config) - request.state.settings_override = deployment_configs[config] + request.state.settings_override = deployment_configs.get(config) response = await call_next(request) except Exception as exc: logging.exception(exc) From ec27344c8bd098374e92e1fcb13726d7aee990c8 Mon Sep 17 00:00:00 2001 From: Faysal Ishtiaq Date: Sun, 6 Nov 2022 12:33:23 +0100 Subject: [PATCH 4/4] remove deployment from git tracking --- deployment.yml | 8 -------- src/climsoft_api/api/s3_files/router.py | 17 ++++++++++------- src/climsoft_api/config.py | 2 +- .../services/file_upload_service.py | 14 ++++++++------ 4 files changed, 19 insertions(+), 22 deletions(-) delete mode 100644 deployment.yml diff --git a/deployment.yml b/deployment.yml deleted file mode 100644 index 0caf353..0000000 --- a/deployment.yml +++ /dev/null @@ -1,8 +0,0 @@ -test: - NAME: Climsoft Test - DATABASE_URI: "mysql+mysqldb://root:password@mariadb/climsoft" - AUTH_ENABLED: false - S3_BUCKET: climsoft-paper-archive - AWS_REGION: eu-west-2 - AWS_ACCESS_KEY_ID: ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY: SECRET_ACCESS_KEY diff --git a/src/climsoft_api/api/s3_files/router.py b/src/climsoft_api/api/s3_files/router.py index 5d99245..4a07a99 100644 --- a/src/climsoft_api/api/s3_files/router.py +++ b/src/climsoft_api/api/s3_files/router.py @@ -1,3 +1,6 @@ +import minio +import urllib3.response + from climsoft_api.config import settings from climsoft_api.utils.s3 import get_minio_client from fastapi import APIRouter, Request @@ -8,18 +11,18 @@ router = APIRouter() -@router.get("/s3/image/{object_key}") -@handle_exceptions +@router.get("/cloud-storage/image/{object_key}") def get_s3_object(object_key, request: Request): try: _settings = override_settings(request.state.settings_override) except AttributeError: _settings = settings - s3_client = get_minio_client(_settings) - response = s3_client.get_object( - Bucket=_settings.S3_BUCKET, - Key=object_key + client: minio.Minio = get_minio_client(_settings) + response: urllib3.response.HTTPResponse = client.get_object( + bucket_name=_settings.S3_BUCKET, + object_name=object_key ) - return Response(response["Body"].read(), + + return Response(response.read(), media_type=f"image/{object_key.split('.')[-1]}") diff --git a/src/climsoft_api/config.py b/src/climsoft_api/config.py index 8651837..11f12a2 100644 --- a/src/climsoft_api/config.py +++ b/src/climsoft_api/config.py @@ -11,7 +11,7 @@ class Settings(BaseSettings): DATABASE_URI: AnyUrl = "mysql+mysqldb://root:password@mariadb/climsoft" FILE_STORAGE: str = "disk" UPLOAD_DIR: str = "/climsoft_uploads" - S3_BUCKET: str = "climsoft-paper-archive" + S3_BUCKET: str = "s3-bucket-name" AWS_REGION: str = "eu-west-2" AWS_ACCESS_KEY_ID: str = "replace it" AWS_SECRET_ACCESS_KEY: str = "replace it" diff --git a/src/climsoft_api/services/file_upload_service.py b/src/climsoft_api/services/file_upload_service.py index 0b81c90..c1f0a0c 100644 --- a/src/climsoft_api/services/file_upload_service.py +++ b/src/climsoft_api/services/file_upload_service.py @@ -3,13 +3,14 @@ from pathlib import Path import minio from climsoft_api.utils.s3 import get_minio_client +from climsoft_api.config import Settings -def save_file(settings, file, file_type): +def save_file(settings: Settings, file, file_type): file_name = f"{uuid.uuid4().hex}.{file_type.split('/')[-1]}" - if settings.storage == "disk": + if settings.FILE_STORAGE == "disk": return save_file_to_disk(settings, file, file_name) - elif settings.storage == "s3": + elif settings.FILE_STORAGE == "cloud_storage": return save_file_to_cloud_storage(settings, file, file_name) else: raise NotImplemented() @@ -17,15 +18,16 @@ def save_file(settings, file, file_type): def save_file_to_cloud_storage(settings, file, file_name): client: minio.Minio = get_minio_client(settings) + octet_stream = io.BytesIO(file) client.put_object( bucket_name=settings.S3_BUCKET, object_name=file_name, - data=io.BytesIO(file), - length=-1 + data=octet_stream, + length=len(octet_stream.getbuffer()) ) return { - "storage": "s3", + "storage": "cloud_storage", "object_key": file_name }