From 0add09953f25e07c1249b9447cc58e2f23e64001 Mon Sep 17 00:00:00 2001 From: Benjamin Bolte Date: Sat, 15 Jun 2024 18:02:27 -0700 Subject: [PATCH] some changes --- README.md | 23 +++++++ linguaphoto/crud/base.py | 83 +++++++++++++++++++++++-- linguaphoto/crud/images.py | 45 ++++++++++++++ linguaphoto/crud/robots.py | 48 -------------- linguaphoto/crud/users.py | 5 +- linguaphoto/db.py | 51 +++++++++++---- linguaphoto/requirements-dev.txt | 3 + linguaphoto/requirements.txt | 2 +- linguaphoto/routers/users.py | 7 ++- linguaphoto/settings/configs/local.yaml | 5 ++ linguaphoto/settings/environment.py | 17 ++++- scripts/docker-compose.yml | 5 ++ tests/test_users.py | 17 ++--- 13 files changed, 227 insertions(+), 84 deletions(-) create mode 100644 linguaphoto/crud/images.py delete mode 100644 linguaphoto/crud/robots.py diff --git a/README.md b/README.md index 3b26972..87b3eae 100644 --- a/README.md +++ b/README.md @@ -1 +1,24 @@ # LinguaPhoto + +## Getting Started + +First, start localstack, which lets you run AWS locally. + +Next, create tables using the command: + +```bash +python -m linguaphoto.db +``` + +Run the backend using the command: + +```bash +uvicorn linguaphoto.main:app --reload +``` + +Finally, run the frontend using the command: + +```bash +cd frontend +npm start +``` diff --git a/linguaphoto/crud/base.py b/linguaphoto/crud/base.py index 47c789c..14ca71a 100644 --- a/linguaphoto/crud/base.py +++ b/linguaphoto/crud/base.py @@ -1,5 +1,6 @@ """Defines the base CRUD interface.""" +import asyncio import itertools import logging from typing import Any, AsyncContextManager, Literal, Self @@ -7,7 +8,9 @@ import aioboto3 from botocore.exceptions import ClientError from redis.asyncio import Redis +from types_aiobotocore_cloudfront.client import CloudFrontClient from types_aiobotocore_dynamodb.service_resource import DynamoDBServiceResource +from types_aiobotocore_s3.client import S3Client from linguaphoto.settings import settings @@ -20,6 +23,8 @@ def __init__(self) -> None: self.__db: DynamoDBServiceResource | None = None self.__kv: Redis | None = None + self.__s3: S3Client | None = None + self.__cf: CloudFrontClient | None = None @property def db(self) -> DynamoDBServiceResource: @@ -27,23 +32,75 @@ def db(self) -> DynamoDBServiceResource: raise RuntimeError("Must call __aenter__ first!") return self.__db - async def __aenter__(self) -> Self: - session = aioboto3.Session() + @property + def kv(self) -> Redis: + if self.__kv is None: + raise RuntimeError("Must call __aenter__ first!") + return self.__kv + + @property + def s3(self) -> S3Client: + if self.__s3 is None: + raise RuntimeError("Must call __aenter__ first!") + return self.__s3 + + @property + def cf(self) -> CloudFrontClient: + if self.__cf is None: + raise RuntimeError("Must call __aenter__ first!") + return self.__cf + + async def _init_dynamodb(self, session: aioboto3.Session) -> Self: db = session.resource("dynamodb") - db = await db.__aenter__() + await db.__aenter__() self.__db = db + return self + + async def _init_cloudfront(self, session: aioboto3.Session) -> Self: + cf = session.client("cloudfront") + await cf.__aenter__() + self.__cf = cf + return self + + async def _init_s3(self, session: aioboto3.Session) -> Self: + s3 = session.client("s3") + await s3.__aenter__() + self.__s3 = s3 + return self - self.kv = Redis( + async def _init_redis(self) -> Self: + kv = Redis( host=settings.redis.host, password=settings.redis.password, port=settings.redis.port, db=settings.redis.db, ) + await kv.__aenter__() + self.__kv = kv + return self + + async def __aenter__(self) -> Self: + session = aioboto3.Session() + + await asyncio.gather( + self._init_dynamodb(session), + self._init_cloudfront(session), + self._init_s3(session), + self._init_redis(), + ) + return self async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: # noqa: ANN401 - if self.__db is not None: - await self.__db.__aexit__(exc_type, exc_val, exc_tb) + await asyncio.gather( + self.db.__aexit__(exc_type, exc_val, exc_tb), + self.cf.__aexit__(exc_type, exc_val, exc_tb), + self.kv.__aexit__(exc_type, exc_val, exc_tb), + self.s3.__aexit__(exc_type, exc_val, exc_tb), + ) + self.__db = None + self.__kv = None + self.__s3 = None async def _create_dynamodb_table( self, @@ -87,3 +144,17 @@ async def _create_dynamodb_table( BillingMode="PAY_PER_REQUEST", ) await table.wait_until_exists() + + async def _delete_dynamodb_table(self, name: str) -> None: + """Deletes a table in the Dynamo database. + + Args: + name: Name of the table. + """ + try: + table = await self.db.Table(name) + await table.delete() + await table.wait_until_not_exists() + except ClientError: + logger.info("Table %s does not exist", name) + return diff --git a/linguaphoto/crud/images.py b/linguaphoto/crud/images.py new file mode 100644 index 0000000..a82943b --- /dev/null +++ b/linguaphoto/crud/images.py @@ -0,0 +1,45 @@ +"""Defines CRUD interface for images API.""" + +import uuid + +from linguaphoto.crud.base import BaseCrud +from linguaphoto.settings import settings + + +class ImagesCrud(BaseCrud): + async def add_image(self, image_id: uuid.UUID, user_id: uuid.UUID, image: bytes) -> None: + # Adds the image to the database. + table = await self.db.Table("Images") + await table.put_item(Item={"image_id": str(image_id), "user_id": str(user_id)}) + + # Uploads the image to S3. + key = f"{user_id}/{image_id}" + await self.s3.put_object(Bucket=settings.aws.image_bucket_id, Key=key, Body=image) + + async def delete_image(self, image_id: uuid.UUID, user_id: uuid.UUID) -> None: + # Deletes the image from the database. + table = await self.db.Table("Images") + await table.delete_item(Key={"image_id": str(image_id), "user_id": str(user_id)}) + + # Deletes the image from S3. + key = f"{user_id}/{image_id}.webp" + await self.s3.delete_object(Bucket=settings.aws.image_bucket_id, Key=key) + + async def get_image_url(self, image_id: uuid.UUID, user_id: uuid.UUID) -> str | None: + key = f"{user_id}/{image_id}.webp" + + # If CloudFront is enabled, gets the image URL from CloudFront. + if settings.aws.cloudfront_url is not None: + return await self.cf.generate_presigned_url( + "get_object", + Params={"Bucket": settings.aws.image_bucket_id, "Key": key}, + ExpiresIn=3600, + ) + + # Gets the image URL from S3. + url = await self.s3.generate_presigned_url( + "get_object", + Params={"Bucket": settings.aws.image_bucket_id, "Key": key}, + ExpiresIn=3600, + ) + return url diff --git a/linguaphoto/crud/robots.py b/linguaphoto/crud/robots.py deleted file mode 100644 index aaa392e..0000000 --- a/linguaphoto/crud/robots.py +++ /dev/null @@ -1,48 +0,0 @@ -"""Defines CRUD interface for robot API.""" - -import logging - -from linguaphoto.crud.base import BaseCrud -from linguaphoto.model import Part, Robot - -logger = logging.getLogger(__name__) - - -class RobotCrud(BaseCrud): - async def add_robot(self, robot: Robot) -> None: - table = await self.db.Table("Robots") - await table.put_item(Item=robot.model_dump()) - - async def add_part(self, part: Part) -> None: - table = await self.db.Table("Parts") - await table.put_item(Item=part.model_dump()) - - async def list_robots(self) -> list[Robot]: - table = await self.db.Table("Robots") - return [Robot.model_validate(robot) for robot in (await table.scan())["Items"]] - - async def get_robot(self, robot_id: str) -> Robot | None: - table = await self.db.Table("Robots") - robot_dict = await table.get_item(Key={"robot_id": robot_id}) - if "Item" not in robot_dict: - return None - return Robot.model_validate(robot_dict["Item"]) - - async def delete_robot(self, robot_id: str) -> None: - table = await self.db.Table("Robots") - await table.delete_item(Key={"robot_id": robot_id}) - - async def list_parts(self) -> list[Part]: - table = await self.db.Table("Parts") - return [Part.model_validate(part) for part in (await table.scan())["Items"]] - - async def get_part(self, part_id: str) -> Part | None: - table = await self.db.Table("Parts") - part_dict = await table.get_item(Key={"part_id": part_id}) - if "Item" not in part_dict: - return None - return Part.model_validate(part_dict["Item"]) - - async def delete_part(self, part_id: str) -> None: - table = await self.db.Table("Parts") - await table.delete_item(Key={"part_id": part_id}) diff --git a/linguaphoto/crud/users.py b/linguaphoto/crud/users.py index 031c952..7a44e4b 100644 --- a/linguaphoto/crud/users.py +++ b/linguaphoto/crud/users.py @@ -14,7 +14,10 @@ class UserCrud(BaseCrud): async def add_user(self, user: User) -> None: table = await self.db.Table("Users") - await table.put_item(Item=user.model_dump()) + await table.put_item( + Item=user.model_dump(), + ConditionExpression="attribute_not_exists(email) AND attribute_not_exists(username)", + ) async def get_user(self, user_id: uuid.UUID) -> User | None: table = await self.db.Table("Users") diff --git a/linguaphoto/db.py b/linguaphoto/db.py index 3be4bb3..8399988 100644 --- a/linguaphoto/db.py +++ b/linguaphoto/db.py @@ -4,14 +4,15 @@ import logging from typing import AsyncGenerator, Self +import argparse from linguaphoto.crud.base import BaseCrud -from linguaphoto.crud.robots import RobotCrud +from linguaphoto.crud.images import ImagesCrud from linguaphoto.crud.users import UserCrud class Crud( UserCrud, - RobotCrud, + ImagesCrud, BaseCrud, ): """Composes the various CRUD classes into a single class.""" @@ -22,11 +23,12 @@ async def get(cls) -> AsyncGenerator[Self, None]: yield crud -async def create_tables(crud: Crud | None = None) -> None: +async def create_tables(crud: Crud | None = None, deletion_protection: bool = False) -> None: """Initializes all of the database tables. Args: crud: The top-level CRUD class. + deletion_protection: Whether to enable deletion protection on the tables. """ logging.basicConfig(level=logging.INFO) @@ -35,17 +37,44 @@ async def create_tables(crud: Crud | None = None) -> None: await create_tables(crud) else: - await crud._create_dynamodb_table( - name="Users", - keys=[ - ("user_id", "S", "HASH"), - ], - gsis=[ - ("emailIndex", "email", "S", "HASH"), - ], + await asyncio.gather( + crud._create_dynamodb_table( + name="Users", + keys=[ + ("user_id", "S", "HASH"), + ], + gsis=[ + ("emailIndex", "email", "S", "HASH"), + ("usernameIndex", "username", "S", "HASH"), + ], + deletion_protection=deletion_protection, + ), + crud._create_dynamodb_table( + name="Images", + keys=[ + ("image_id", "S", "HASH"), + ], + ), ) +async def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("action", choices=["create", "delete", "populate"]) + args = parser.parse_args() + + async with Crud() as crud: + match args.action: + case "create": + await create_tables(crud) + case "delete": + await delete_tables(crud) + case "populate": + await populate_with_dummy_data(crud) + case _: + raise ValueError(f"Invalid action: {args.action}") + + if __name__ == "__main__": # python -m linguaphoto.db asyncio.run(create_tables()) diff --git a/linguaphoto/requirements-dev.txt b/linguaphoto/requirements-dev.txt index 279bf6a..3f08f1d 100644 --- a/linguaphoto/requirements-dev.txt +++ b/linguaphoto/requirements-dev.txt @@ -20,3 +20,6 @@ s3fs # Types types-requests + +# AWS +localstack diff --git a/linguaphoto/requirements.txt b/linguaphoto/requirements.txt index f0493c5..b872e86 100644 --- a/linguaphoto/requirements.txt +++ b/linguaphoto/requirements.txt @@ -20,7 +20,7 @@ python-multipart uvicorn[standard] # Types -types-aioboto3[dynamodb] +types-aioboto3[dynamodb,cloudfront,s3] # AI dependencies numpy diff --git a/linguaphoto/routers/users.py b/linguaphoto/routers/users.py index 55958cb..7044aff 100644 --- a/linguaphoto/routers/users.py +++ b/linguaphoto/routers/users.py @@ -13,6 +13,7 @@ from linguaphoto.crypto import get_new_api_key, get_new_user_id from linguaphoto.db import Crud from linguaphoto.model import User +from linguaphoto.settings import settings logger = logging.getLogger(__name__) @@ -117,7 +118,9 @@ async def google_login_endpoint( data: GoogleLogin, crud: Annotated[Crud, Depends(Crud.get)], ) -> UserLoginResponse: - """Uses Google OAuth to create an API token that lasts for a week (i.e. 604800 seconds).""" + if (test_user := settings.user.test_user) is not None and data.token == test_user.google_token: + return await get_login_response(test_user.email, settings.user.auth_lifetime_seconds, crud) + try: idinfo = await get_google_user_info(data.token) email = idinfo["email"] @@ -126,7 +129,7 @@ async def google_login_endpoint( if idinfo.get("email_verified") is not True: raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail="Google email not verified") - return await get_login_response(email, 604800, crud) + return await get_login_response(email, settings.user.auth_lifetime_seconds, crud) class UserInfoResponse(BaseModel): diff --git a/linguaphoto/settings/configs/local.yaml b/linguaphoto/settings/configs/local.yaml index 81f54a4..2102846 100644 --- a/linguaphoto/settings/configs/local.yaml +++ b/linguaphoto/settings/configs/local.yaml @@ -2,3 +2,8 @@ crypto: jwt_secret: fakeJwtSecret site: homepage: http://127.0.0.1:3000 + image_bucket_id: linguaphoto-images +user: + test_user: + email: test@example.com + google_token: fakeGoogleToken diff --git a/linguaphoto/settings/environment.py b/linguaphoto/settings/environment.py index 9957abd..aba54c3 100644 --- a/linguaphoto/settings/environment.py +++ b/linguaphoto/settings/environment.py @@ -21,10 +21,16 @@ class CryptoSettings: algorithm: str = field(default="HS256") +@dataclass +class TestUserSettings: + email: str = field(default=MISSING) + google_token: str = field(default=MISSING) + + @dataclass class UserSettings: - authorized_emails: list[str] | None = field(default=None) - admin_emails: list[str] = field(default_factory=lambda: []) + test_user: TestUserSettings | None = field(default_factory=TestUserSettings) + auth_lifetime_seconds: int = field(default=604800) # 1 week @dataclass @@ -33,10 +39,17 @@ class SiteSettings: image_url: str | None = field(default=None) +@dataclass +class AWSSettings: + image_bucket_id: str = field(default=MISSING) + cloudfront_url: str | None = field(default=None) + + @dataclass class EnvironmentSettings: redis: RedisSettings = field(default_factory=RedisSettings) user: UserSettings = field(default_factory=UserSettings) crypto: CryptoSettings = field(default_factory=CryptoSettings) site: SiteSettings = field(default_factory=SiteSettings) + aws: AWSSettings = field(default_factory=AWSSettings) debug: bool = field(default=False) diff --git a/scripts/docker-compose.yml b/scripts/docker-compose.yml index 017ff31..a8b6cf0 100644 --- a/scripts/docker-compose.yml +++ b/scripts/docker-compose.yml @@ -13,3 +13,8 @@ services: - "8001:8001" environment: - DYNAMO_ENDPOINT=http://dynamodb:8000 + + redis: + image: redis + ports: + - "6379:6379" diff --git a/tests/test_users.py b/tests/test_users.py index 7550b6c..9ead2f4 100644 --- a/tests/test_users.py +++ b/tests/test_users.py @@ -6,28 +6,19 @@ from pytest_mock.plugin import MockType from linguaphoto.db import create_tables -from linguaphoto.utils.email import OneTimePassPayload def test_user_auth_functions(app_client: TestClient, mock_send_email: MockType) -> None: asyncio.run(create_tables()) - test_email = "test@example.com" - login_url = "/" + test_username = "testusername" + test_password = "ccccc@#$bhui1324frhnund!!@#$" - # Sends the one-time password to the test email. - response = app_client.post("/users/login", json={"email": test_email, "login_url": login_url, "lifetime": 3600}) + # Attempts to log in before creating the user. + response = app_client.post("/users/login", json={"username": test_username, "password": test_password}) assert response.status_code == 200, response.json() assert mock_send_email.call_count == 1 - # Uses the one-time pass to get an API key. We need to make a new OTP - # manually because we can't send emails in unit tests. - otp = OneTimePassPayload(email=test_email, lifetime=3600) - response = app_client.post("/users/otp", json={"payload": otp.encode()}) - assert response.status_code == 200, response.json() - response_data = response.json() - api_key = response_data["api_key"] - # Checks that without the API key we get a 401 response. response = app_client.get("/users/me") assert response.status_code == 401, response.json()