From ada73ab1d8870b1725b60dc224deb87726121b18 Mon Sep 17 00:00:00 2001 From: Pradip-p Date: Tue, 5 Nov 2024 11:06:54 +0545 Subject: [PATCH 1/2] fix(s3-config): update S3 bucket name & path prefix to mimic with dev & prd --- src/backend/app/config.py | 3 ++- src/backend/app/projects/image_processing.py | 10 ++++------ src/backend/app/projects/project_logic.py | 10 +++++++--- src/backend/app/projects/project_routes.py | 6 +++--- src/backend/app/projects/project_schemas.py | 5 +++-- src/backend/app/s3.py | 2 +- src/backend/app/waypoints/waypoint_routes.py | 4 +++- 7 files changed, 23 insertions(+), 17 deletions(-) diff --git a/src/backend/app/config.py b/src/backend/app/config.py index 2b5ce89c..e69aff29 100644 --- a/src/backend/app/config.py +++ b/src/backend/app/config.py @@ -86,7 +86,8 @@ def assemble_db_connection(cls, v: Optional[str], info: ValidationInfo) -> Any: S3_ENDPOINT: str = "http://s3:9000" S3_ACCESS_KEY: Optional[str] = "" S3_SECRET_KEY: Optional[str] = "" - S3_BUCKET_NAME: str = "dtm-data" + S3_BUCKET_NAME: str = "dronetm" + S3_PATH_PREFIX: str = "dtm-data" S3_DOWNLOAD_ROOT: Optional[str] = None ALGORITHM: str = "HS256" diff --git a/src/backend/app/projects/image_processing.py b/src/backend/app/projects/image_processing.py index c44723c9..41a87b81 100644 --- a/src/backend/app/projects/image_processing.py +++ b/src/backend/app/projects/image_processing.py @@ -63,7 +63,7 @@ def download_images_from_s3(self, bucket_name, local_dir): :param local_dir: Local directory to save the images. :return: List of local image file paths. """ - prefix = f"projects/{self.project_id}/{self.task_id}" + prefix = f"{settings.S3_PATH_PREFIX}/projects/{self.project_id}/{self.task_id}" objects = list_objects_from_bucket(bucket_name, prefix) @@ -167,7 +167,7 @@ def process_images_from_s3(self, bucket_name, name=None, options=[], webhook=Non ) # Upload the results into s3 - s3_path = f"projects/{self.project_id}/{self.task_id}/assets.zip" + s3_path = f"{settings.S3_PATH_PREFIX}/projects/{self.project_id}/{self.task_id}/assets.zip" add_file_to_bucket(bucket_name, path_to_download, s3_path) # now update the task as completed in Db. # Call the async function using asyncio @@ -228,7 +228,7 @@ async def download_and_upload_assets_from_odm_to_s3( assets_path = task.download_zip(output_file_path) # Upload the results into S3 (Minio) - s3_path = f"projects/{dtm_project_id}/{dtm_task_id}/assets.zip" + s3_path = f"{settings.S3_PATH_PREFIX}/projects/{dtm_project_id}/{dtm_task_id}/assets.zip" log.info(f"Uploading {output_file_path} to S3 path: {s3_path}") add_file_to_bucket(settings.S3_BUCKET_NAME, assets_path, s3_path) @@ -249,9 +249,7 @@ async def download_and_upload_assets_from_odm_to_s3( log.info(f"Orthophoto found at {orthophoto_path}") # Upload the orthophoto to S3 - s3_ortho_path = ( - f"projects/{dtm_project_id}/{dtm_task_id}/orthophoto/odm_orthophoto.tif" - ) + s3_ortho_path = f"{settings.S3_PATH_PREFIX}/projects/{dtm_project_id}/{dtm_task_id}/orthophoto/odm_orthophoto.tif" log.info(f"Uploading orthophoto to S3 path: {s3_ortho_path}") add_file_to_bucket(settings.S3_BUCKET_NAME, orthophoto_path, s3_ortho_path) diff --git a/src/backend/app/projects/project_logic.py b/src/backend/app/projects/project_logic.py index 2dcb2c3d..21ac73e7 100644 --- a/src/backend/app/projects/project_logic.py +++ b/src/backend/app/projects/project_logic.py @@ -69,7 +69,7 @@ async def upload_file_to_s3( str: The S3 URL for the uploaded file. """ # Define the S3 file path - file_path = f"/projects/{project_id}/{file_name}" + file_path = f"{settings.S3_PATH_PREFIX}/projects/{project_id}/{file_name}" # Read the file bytes file_bytes = await file.read() @@ -227,7 +227,9 @@ def get_project_info_from_s3(project_id: uuid.UUID, task_id: uuid.UUID): """ try: # Prefix for the images - images_prefix = f"projects/{project_id}/{task_id}/images/" + images_prefix = ( + f"{settings.S3_PATH_PREFIX}/projects/{project_id}/{task_id}/images/" + ) # List and count the images objects = list_objects_from_bucket( @@ -241,7 +243,9 @@ def get_project_info_from_s3(project_id: uuid.UUID, task_id: uuid.UUID): # Generate a presigned URL for the assets ZIP file try: # Check if the object exists - assets_path = f"projects/{project_id}/{task_id}/assets.zip" + assets_path = ( + f"{settings.S3_PATH_PREFIX}/projects/{project_id}/{task_id}/assets.zip" + ) get_object_metadata(settings.S3_BUCKET_NAME, assets_path) # If it exists, generate the presigned URL diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py index 44360d9a..be5c5954 100644 --- a/src/backend/app/projects/project_routes.py +++ b/src/backend/app/projects/project_routes.py @@ -315,11 +315,11 @@ async def generate_presigned_url( # Process each image in the request for image in data.image_name: - image_path = f"projects/{data.project_id}/{data.task_id}/images/{image}" + image_path = f"{settings.S3_PATH_PREFIX}/projects/{data.project_id}/{data.task_id}/images/{image}" # If replace_existing is True, delete the image first if replace_existing: - image_dir = f"projects/{data.project_id}/{data.task_id}/images/" + image_dir = f"{settings.S3_PATH_PREFIX}/projects/{data.project_id}/{data.task_id}/images/" try: # Prepare the list of objects to delete (recursively if necessary) delete_object_list = map( @@ -605,7 +605,7 @@ async def get_orthophoto_tile( :return: PNG image tile. """ try: - cog_path = get_cog_path("dtm-data", project_id, task_id) + cog_path = get_cog_path(settings.S3_BUCKET_NAME, project_id, task_id) with Reader(cog_path) as tiff: try: img = tiff.tile(int(x), int(y), int(z), tilesize=256, expression=None) diff --git a/src/backend/app/projects/project_schemas.py b/src/backend/app/projects/project_schemas.py index bd394db6..0413295f 100644 --- a/src/backend/app/projects/project_schemas.py +++ b/src/backend/app/projects/project_schemas.py @@ -542,8 +542,9 @@ def set_image_url(cls, values): """Set image_url before rendering the model.""" project_id = values.id if project_id: - image_dir = f"projects/{project_id}/map_screenshot.png" - # values.image_url = get_image_dir_url(settings.S3_BUCKET_NAME, image_dir) + image_dir = ( + f"{settings.S3_PATH_PREFIX}/projects/{project_id}/map_screenshot.png" + ) values.image_url = get_presigned_url(settings.S3_BUCKET_NAME, image_dir, 5) return values diff --git a/src/backend/app/s3.py b/src/backend/app/s3.py index 63aabe97..beddc447 100644 --- a/src/backend/app/s3.py +++ b/src/backend/app/s3.py @@ -227,7 +227,7 @@ def get_cog_path(bucket_name: str, project_id: str, task_id: str): str: The presigned URL to access the COG file. """ # Construct the S3 path for the COG file - s3_path = f"projects/{project_id}/{task_id}/orthophoto/odm_orthophoto.tif" + s3_path = f"{settings.S3_PATH_PREFIX}/projects/{project_id}/{task_id}/orthophoto/odm_orthophoto.tif" # Get the presigned URL return get_presigned_url(bucket_name, s3_path) diff --git a/src/backend/app/waypoints/waypoint_routes.py b/src/backend/app/waypoints/waypoint_routes.py index b87399af..71cc9d01 100644 --- a/src/backend/app/waypoints/waypoint_routes.py +++ b/src/backend/app/waypoints/waypoint_routes.py @@ -123,7 +123,9 @@ async def get_task_waypoint( dem_path = f"/tmp/{uuid.uuid4()}/dem.tif" try: get_file_from_bucket( - settings.S3_BUCKET_NAME, f"projects/{project_id}/dem.tif", dem_path + settings.S3_BUCKET_NAME, + f"{settings.S3_PATH_PREFIX}/projects/{project_id}/dem.tif", + dem_path, ) # TODO: Do this with inmemory data outfile_with_elevation = "/tmp/output_file_with_elevation.geojson" From e33c3eee550fef69168ea29fbee2d9f2e982746f Mon Sep 17 00:00:00 2001 From: Pradip-p Date: Tue, 5 Nov 2024 11:47:20 +0545 Subject: [PATCH 2/2] refac: s3 bucket path name --- src/backend/app/config.py | 3 +-- src/backend/app/projects/image_processing.py | 10 ++++++---- src/backend/app/projects/project_logic.py | 10 +++------- src/backend/app/projects/project_routes.py | 8 ++++++-- src/backend/app/projects/project_schemas.py | 4 +--- src/backend/app/s3.py | 2 +- src/backend/app/waypoints/waypoint_routes.py | 2 +- 7 files changed, 19 insertions(+), 20 deletions(-) diff --git a/src/backend/app/config.py b/src/backend/app/config.py index e69aff29..2b5ce89c 100644 --- a/src/backend/app/config.py +++ b/src/backend/app/config.py @@ -86,8 +86,7 @@ def assemble_db_connection(cls, v: Optional[str], info: ValidationInfo) -> Any: S3_ENDPOINT: str = "http://s3:9000" S3_ACCESS_KEY: Optional[str] = "" S3_SECRET_KEY: Optional[str] = "" - S3_BUCKET_NAME: str = "dronetm" - S3_PATH_PREFIX: str = "dtm-data" + S3_BUCKET_NAME: str = "dtm-data" S3_DOWNLOAD_ROOT: Optional[str] = None ALGORITHM: str = "HS256" diff --git a/src/backend/app/projects/image_processing.py b/src/backend/app/projects/image_processing.py index 41a87b81..16f55ffd 100644 --- a/src/backend/app/projects/image_processing.py +++ b/src/backend/app/projects/image_processing.py @@ -63,7 +63,7 @@ def download_images_from_s3(self, bucket_name, local_dir): :param local_dir: Local directory to save the images. :return: List of local image file paths. """ - prefix = f"{settings.S3_PATH_PREFIX}/projects/{self.project_id}/{self.task_id}" + prefix = f"dtm-data/projects/{self.project_id}/{self.task_id}" objects = list_objects_from_bucket(bucket_name, prefix) @@ -167,7 +167,9 @@ def process_images_from_s3(self, bucket_name, name=None, options=[], webhook=Non ) # Upload the results into s3 - s3_path = f"{settings.S3_PATH_PREFIX}/projects/{self.project_id}/{self.task_id}/assets.zip" + s3_path = ( + f"dtm-data/projects/{self.project_id}/{self.task_id}/assets.zip" + ) add_file_to_bucket(bucket_name, path_to_download, s3_path) # now update the task as completed in Db. # Call the async function using asyncio @@ -228,7 +230,7 @@ async def download_and_upload_assets_from_odm_to_s3( assets_path = task.download_zip(output_file_path) # Upload the results into S3 (Minio) - s3_path = f"{settings.S3_PATH_PREFIX}/projects/{dtm_project_id}/{dtm_task_id}/assets.zip" + s3_path = f"dtm-data/projects/{dtm_project_id}/{dtm_task_id}/assets.zip" log.info(f"Uploading {output_file_path} to S3 path: {s3_path}") add_file_to_bucket(settings.S3_BUCKET_NAME, assets_path, s3_path) @@ -249,7 +251,7 @@ async def download_and_upload_assets_from_odm_to_s3( log.info(f"Orthophoto found at {orthophoto_path}") # Upload the orthophoto to S3 - s3_ortho_path = f"{settings.S3_PATH_PREFIX}/projects/{dtm_project_id}/{dtm_task_id}/orthophoto/odm_orthophoto.tif" + s3_ortho_path = f"dtm-data/projects/{dtm_project_id}/{dtm_task_id}/orthophoto/odm_orthophoto.tif" log.info(f"Uploading orthophoto to S3 path: {s3_ortho_path}") add_file_to_bucket(settings.S3_BUCKET_NAME, orthophoto_path, s3_ortho_path) diff --git a/src/backend/app/projects/project_logic.py b/src/backend/app/projects/project_logic.py index 21ac73e7..984e76ef 100644 --- a/src/backend/app/projects/project_logic.py +++ b/src/backend/app/projects/project_logic.py @@ -69,7 +69,7 @@ async def upload_file_to_s3( str: The S3 URL for the uploaded file. """ # Define the S3 file path - file_path = f"{settings.S3_PATH_PREFIX}/projects/{project_id}/{file_name}" + file_path = f"dtm-data/projects/{project_id}/{file_name}" # Read the file bytes file_bytes = await file.read() @@ -227,9 +227,7 @@ def get_project_info_from_s3(project_id: uuid.UUID, task_id: uuid.UUID): """ try: # Prefix for the images - images_prefix = ( - f"{settings.S3_PATH_PREFIX}/projects/{project_id}/{task_id}/images/" - ) + images_prefix = f"dtm-data/projects/{project_id}/{task_id}/images/" # List and count the images objects = list_objects_from_bucket( @@ -243,9 +241,7 @@ def get_project_info_from_s3(project_id: uuid.UUID, task_id: uuid.UUID): # Generate a presigned URL for the assets ZIP file try: # Check if the object exists - assets_path = ( - f"{settings.S3_PATH_PREFIX}/projects/{project_id}/{task_id}/assets.zip" - ) + assets_path = f"dtm-data/projects/{project_id}/{task_id}/assets.zip" get_object_metadata(settings.S3_BUCKET_NAME, assets_path) # If it exists, generate the presigned URL diff --git a/src/backend/app/projects/project_routes.py b/src/backend/app/projects/project_routes.py index be5c5954..7c09f6c8 100644 --- a/src/backend/app/projects/project_routes.py +++ b/src/backend/app/projects/project_routes.py @@ -315,11 +315,15 @@ async def generate_presigned_url( # Process each image in the request for image in data.image_name: - image_path = f"{settings.S3_PATH_PREFIX}/projects/{data.project_id}/{data.task_id}/images/{image}" + image_path = ( + f"dtm-data/projects/{data.project_id}/{data.task_id}/images/{image}" + ) # If replace_existing is True, delete the image first if replace_existing: - image_dir = f"{settings.S3_PATH_PREFIX}/projects/{data.project_id}/{data.task_id}/images/" + image_dir = ( + f"dtm-data/projects/{data.project_id}/{data.task_id}/images/" + ) try: # Prepare the list of objects to delete (recursively if necessary) delete_object_list = map( diff --git a/src/backend/app/projects/project_schemas.py b/src/backend/app/projects/project_schemas.py index 0413295f..ad183a45 100644 --- a/src/backend/app/projects/project_schemas.py +++ b/src/backend/app/projects/project_schemas.py @@ -542,9 +542,7 @@ def set_image_url(cls, values): """Set image_url before rendering the model.""" project_id = values.id if project_id: - image_dir = ( - f"{settings.S3_PATH_PREFIX}/projects/{project_id}/map_screenshot.png" - ) + image_dir = f"dtm-data/projects/{project_id}/map_screenshot.png" values.image_url = get_presigned_url(settings.S3_BUCKET_NAME, image_dir, 5) return values diff --git a/src/backend/app/s3.py b/src/backend/app/s3.py index beddc447..22d83113 100644 --- a/src/backend/app/s3.py +++ b/src/backend/app/s3.py @@ -227,7 +227,7 @@ def get_cog_path(bucket_name: str, project_id: str, task_id: str): str: The presigned URL to access the COG file. """ # Construct the S3 path for the COG file - s3_path = f"{settings.S3_PATH_PREFIX}/projects/{project_id}/{task_id}/orthophoto/odm_orthophoto.tif" + s3_path = f"dtm-data/projects/{project_id}/{task_id}/orthophoto/odm_orthophoto.tif" # Get the presigned URL return get_presigned_url(bucket_name, s3_path) diff --git a/src/backend/app/waypoints/waypoint_routes.py b/src/backend/app/waypoints/waypoint_routes.py index 71cc9d01..c4317103 100644 --- a/src/backend/app/waypoints/waypoint_routes.py +++ b/src/backend/app/waypoints/waypoint_routes.py @@ -124,7 +124,7 @@ async def get_task_waypoint( try: get_file_from_bucket( settings.S3_BUCKET_NAME, - f"{settings.S3_PATH_PREFIX}/projects/{project_id}/dem.tif", + f"dtm-data/projects/{project_id}/dem.tif", dem_path, ) # TODO: Do this with inmemory data