From 93f22524b4ed6d0b98886ab1ed04b92572d9568b Mon Sep 17 00:00:00 2001
From: OriginalByteMe <noahrijkaard@gmail.com>
Date: Sun, 5 Nov 2023 23:51:05 +0800
Subject: [PATCH 01/14] modify cutout to use envionrmental variables

---
 app/cutout.py | 30 ++++++++++++++++++++++++------
 1 file changed, 24 insertions(+), 6 deletions(-)

diff --git a/app/cutout.py b/app/cutout.py
index 1847352..86b7c07 100644
--- a/app/cutout.py
+++ b/app/cutout.py
@@ -1,13 +1,31 @@
 import cv2
 import numpy as np
 import os
+from s3_handler import Boto3Client
+from dino import Dino
+from segment import Segmenter
 
+HOME = os.path.abspath(os.path.join(os.getcwd(), os.pardir))
 
 class CutoutCreator:
-  def __init__(self, image_folder):
-    self.image_folder = image_folder
-
-  def create_cutouts(self, image_name, masks, output_folder,bucket_name, s3):
+  def __init__(self, grounding_dino_config_path, grounding_dino_checkpoint_path):
+    self.dino = Dino(classes=['person', 'nose', 'chair', 'shoe', 'ear', 'hat'],
+        box_threshold=0.35,
+        text_threshold=0.25,
+        model_config_path=grounding_dino_config_path,
+        model_checkpoint_path=grounding_dino_checkpoint_path)
+    self.s3 = Boto3Client()
+
+  def create_cutouts(self, image_name, sam_checkpoint_path):
+    
+    # Download image from s3
+    image_path = self.s3.download_from_s3(os.path.join(HOME, "data"), "cutout-image-store", f"images/{image_name}")
+    
+    image = cv2.imread(image_path)
+    segment = Segmenter(sam_encoder_version="vit_h", sam_checkpoint_path=sam_checkpoint_path)
+    detections = self.dino.predict(image)
+    
+    masks = segment.segment(image, detections.xyxy)
     # Load the image
     image_path = os.path.join(self.image_folder, image_name)
     for item in os.listdir(self.image_folder):
@@ -29,10 +47,10 @@ def create_cutouts(self, image_name, masks, output_folder,bucket_name, s3):
 
       # Save the cutout
       cutout_name = f"{image_name}_cutout_{i}.png"
-      cutout_path = os.path.join(output_folder, cutout_name)
+      cutout_path = os.path.join("cutouts", cutout_name)
       cv2.imwrite(cutout_path, cutout)
 
       # Upload the cutout to S3
       with open(cutout_path, "rb") as f:
-        s3.upload_to_s3(bucket_name, f.read(), f"cutouts/{image_name}/{cutout_name}")
+        self.s3.upload_to_s3(f.read(), f"cutouts/{image_name}/{cutout_name}")
 

From 18daff06dde1e36113deb35bb3b605f1bd87e8f0 Mon Sep 17 00:00:00 2001
From: OriginalByteMe <noahrijkaard@gmail.com>
Date: Sun, 5 Nov 2023 23:51:30 +0800
Subject: [PATCH 02/14] Simplify s3 inputs to have auto bucket

---
 app/s3_handler.py | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/app/s3_handler.py b/app/s3_handler.py
index f92f97b..0c26a93 100644
--- a/app/s3_handler.py
+++ b/app/s3_handler.py
@@ -12,27 +12,25 @@ def __init__(self):
             region_name=os.environ["AWS_REGION"],
         )
 
-    def download_from_s3(self, save_path, bucket_name, key):
+    def download_from_s3(self, save_path, image_name):
         s3_client = boto3.client("s3")
-
-        file_name = key.split("/")[-1]
-        file_path = os.path.join(save_path, file_name)
+        file_path = os.path.join(save_path, image_name)
         try:
-            s3_client.download_file(bucket_name, key, file_path)
+            s3_client.download_file(os.environ["CUTOUT_BUCKET"], f"cutouts/{image_name}/cutout-{image_name}", file_path)
         except ClientError as e:
             print("BOTO error: ",e)
             return None
 
         return file_path
 
-    def upload_to_s3(self, bucket_name, file_body, key):
-        self.s3.put_object(Body=file_body, Bucket=bucket_name, Key=key)
+    def upload_to_s3(self, file_body, image_name):
+        self.s3.put_object(Body=file_body, Bucket=os.environ["CUTOUT_BUCKET"], Key=f"images/{image_name}")
 
-    def generate_presigned_url(self, bucket_name, key, expiration=3600):
+    def generate_presigned_url(self, image_name, expiration=3600):
         try:
             response = self.s3.generate_presigned_url(
                 "get_object",
-                Params={"Bucket": bucket_name, "Key": key},
+                Params={"Bucket": os.environ["CUTOUT_BUCKET"], "Key": f"cutouts/{image_name}/cutout-{image_name}"},
                 ExpiresIn=expiration,
             )
         except ClientError as e:

From 1e32d62570e8bb9a3a4a0a5642e43c523e9199c0 Mon Sep 17 00:00:00 2001
From: OriginalByteMe <noahrijkaard@gmail.com>
Date: Mon, 6 Nov 2023 00:18:34 +0800
Subject: [PATCH 03/14] Fix api endpoint errors for s3 api's

---
 app/grounded_cutouts.py | 103 ++++++++++++++++++++++------------------
 1 file changed, 58 insertions(+), 45 deletions(-)

diff --git a/app/grounded_cutouts.py b/app/grounded_cutouts.py
index 79faeb9..09d6f9e 100644
--- a/app/grounded_cutouts.py
+++ b/app/grounded_cutouts.py
@@ -1,13 +1,16 @@
 import os
-import modal
+from modal import asgi_app, Secret, Stub, Mount, Image
 import cv2
+from fastapi import FastAPI, File, UploadFile
+from fastapi.responses import FileResponse
 
-stub = modal.Stub(name="cutout_generator")
 
-img_volume = modal.NetworkFileSystem.persisted("image-storage-vol")
-cutout_volume = modal.NetworkFileSystem.persisted("cutout-storage-vol")
-local_packages = modal.Mount.from_local_python_packages("cutout", "dino", "segment", "s3_handler")
-cutout_generator_image = modal.Image.from_registry("nvcr.io/nvidia/pytorch:22.12-py3").pip_install( "segment-anything", "opencv-python", "botocore", "boto3").run_commands(
+app = FastAPI()
+
+stub = Stub(name="cutout_generator")
+
+local_packages = Mount.from_local_python_packages("cutout", "dino", "segment", "s3_handler", "fastapi", "starlette")
+cutout_generator_image = Image.from_registry("nvcr.io/nvidia/pytorch:22.12-py3").pip_install( "segment-anything", "opencv-python", "botocore", "boto3").run_commands(
   "apt-get update",
   "apt-get install -y git wget libgl1-mesa-glx libglib2.0-0",
   "echo $CUDA_HOME",
@@ -21,51 +24,61 @@
   "pip install -q supervision==0.6.0",
   "wget -q https://github.com/IDEA-Research/GroundingDINO/releases/download/v0.1.0-alpha/groundingdino_swint_ogc.pth -P weights/",
   "wget -q https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth -P weights/",
-  "wget -q https://media.roboflow.com/notebooks/examples/dog.jpeg -P data/",
+  "wget -q https://media.roboflow.com/notebooks/examples/dog.jpeg -P images/",
   "ls -F",
   "ls -F GroundingDINO/groundingdino/config",
   "ls -F GroundingDINO/groundingdino/models/GroundingDINO/"
 )
-# Setup paths
+
 HOME = os.path.abspath(os.path.join(os.getcwd(), os.pardir))
 GROUNDING_DINO_CONFIG_PATH = os.path.join(HOME, "GroundingDINO/groundingdino/config/GroundingDINO_SwinT_OGC.py")
 GROUNDING_DINO_CHECKPOINT_PATH = os.path.join(HOME, "weights", "groundingdino_swint_ogc.pth")
 SAM_CHECKPOINT_PATH = os.path.join(HOME, "weights", "sam_vit_h_4b8939.pth")
 
-@stub.function(image=cutout_generator_image, mounts=[local_packages], gpu="T4", secret=modal.Secret.from_name("my-aws-secret"), network_file_systems={"/images": img_volume, "/cutouts": cutout_volume})
-@modal.web_endpoint()
-def main(image_name):
-  # Import relevant classes
-  from dino import Dino
-  from segment import Segmenter
-  from cutout import CutoutCreator
-  from s3_handler import Boto3Client
-  SOURCE_IMAGE_PATH = os.path.join(HOME, "data", image_name)
-  print(f"SOURCE_IMAGE_PATH: {SOURCE_IMAGE_PATH}")
-  SAVE_IMAGE_PATH = os.path.join(HOME, "data")
-  OUTPUT_CUTOUT_PATH = os.path.join(HOME, "cutouts")
-  dino = Dino(classes=['person', 'nose', 'chair', 'shoe', 'ear', 'hat'],
-      box_threshold=0.35,
-      text_threshold=0.25,
-      model_config_path=GROUNDING_DINO_CONFIG_PATH,
-      model_checkpoint_path=GROUNDING_DINO_CHECKPOINT_PATH)
-  
-  segment = Segmenter(sam_encoder_version="vit_h", sam_checkpoint_path=SAM_CHECKPOINT_PATH)
-  
-  cutout = CutoutCreator(image_folder=SAVE_IMAGE_PATH)
-  
-  s3 = Boto3Client()
-  
-  s3.download_from_s3(SAVE_IMAGE_PATH, "cutout-image-store", f"images/{image_name}")
-  
-  image = cv2.imread(SOURCE_IMAGE_PATH)
-  
-  # Run the DINO model on the image
-  detections = dino.predict(image)
-  
-  detections.mask = segment.segment(image, detections.xyxy)
-  
-  cutout.create_cutouts(image_name, detections.mask, OUTPUT_CUTOUT_PATH, "cutout-image-store", s3)
-  
-  return "Success"  
-  
+@stub.function(mounts=[Mount.from_local_python_packages("s3_handler")], secret=Secret.from_name("my-aws-secret"))
+@app.post("/upload-image")
+async def upload_image_to_s3(image: UploadFile = File(...)):
+    from s3_handler import Boto3Client
+    s3_client = Boto3Client()
+    s3_client.upload_to_s3(image.file, image.filename)
+    return {"message": "Image uploaded successfully"}
+
+@stub.function(mounts=[Mount.from_local_python_packages("s3_handler")], secret=Secret.from_name("my-aws-secret"))
+@app.get("/create-presigned-cutout/{image_name}")
+async def create_presigned_cutout(image_name: str):
+    from s3_handler import Boto3Client
+    s3_client = Boto3Client()
+    presigned_url = s3_client.generate_presigned_url(image_name)
+    return {"presigned_url": presigned_url}
+
+@stub.function(image=cutout_generator_image, mounts=[local_packages], gpu="T4", secret=Secret.from_name("my-aws-secret"))
+@app.get("/create-cutouts/{image_name}")
+async def create_cutouts(image_name: str):
+    from cutout import CutoutCreator
+    OUTPUT_CUTOUT_PATH = os.path.join(HOME, "cutouts")
+
+    cutout = CutoutCreator(grounding_dino_checkpoint_path=GROUNDING_DINO_CHECKPOINT_PATH, grounding_dino_config_path=GROUNDING_DINO_CONFIG_PATH)
+    cutout.create_cutouts(image_name, SAM_CHECKPOINT_PATH )
+    return FileResponse(os.path.join(OUTPUT_CUTOUT_PATH, f"{image_name.split('.')[0]}_cutouts.zip"))
+
+# @app.post("/create-cutouts")
+# async def create_cutouts_from_image(image: UploadFile = File(...)):
+#     SOURCE_IMAGE_PATH = os.path.join(HOME, "data", image.filename)
+#     SAVE_IMAGE_PATH = os.path.join(HOME, "data")
+#     OUTPUT_CUTOUT_PATH = os.path.join(HOME, "cutouts")
+#     dino = Dino(classes=['person', 'nose', 'chair', 'shoe', 'ear', 'hat'],
+#         box_threshold=0.35,
+#         text_threshold=0.25,
+#         model_config_path=GROUNDING_DINO_CONFIG_PATH,
+#         model_checkpoint_path=GROUNDING_DINO_CHECKPOINT_PATH)
+
+#     segment = Segmenter(sam_encoder_version="vit_h", sam_checkpoint_path=SAM_CHECKPOINT_PATH)
+
+#     cutout = CutoutCreator(image_folder=SAVE_IMAGE_PATH)
+#     cutout.create_cutouts_from_file(image.file, dino, segment, OUTPUT_CUTOUT_PATH)
+#     return FileResponse(os.path.join(OUTPUT_CUTOUT_PATH, f"{image.filename.split('.')[0]}_cutouts.zip"))
+
+@stub.function(image=cutout_generator_image, mounts=[local_packages], secret=Secret.from_name("my-aws-secret"))
+@asgi_app()
+def cutout_app():
+  return app
\ No newline at end of file

From 35a758e2ba5eefe97c073a5382d114fcd4c535e4 Mon Sep 17 00:00:00 2001
From: OriginalByteMe <noahrijkaard@gmail.com>
Date: Mon, 6 Nov 2023 19:46:57 +0800
Subject: [PATCH 04/14] Fix api endpoint errors

---
 app/cutout.py           | 17 +++++++++--------
 app/grounded_cutouts.py | 30 ++++++------------------------
 2 files changed, 15 insertions(+), 32 deletions(-)

diff --git a/app/cutout.py b/app/cutout.py
index 86b7c07..c658f7f 100644
--- a/app/cutout.py
+++ b/app/cutout.py
@@ -19,19 +19,20 @@ def __init__(self, grounding_dino_config_path, grounding_dino_checkpoint_path):
   def create_cutouts(self, image_name, sam_checkpoint_path):
     
     # Download image from s3
-    image_path = self.s3.download_from_s3(os.path.join(HOME, "data"), "cutout-image-store", f"images/{image_name}")
-    
+    image_path = self.s3.download_from_s3(os.path.join(HOME, "data"), image_name)
+    if not os.path.exists(os.path.join(HOME, "cutouts")):
+      os.mkdir(os.path.join(HOME, "cutouts"))
     image = cv2.imread(image_path)
     segment = Segmenter(sam_encoder_version="vit_h", sam_checkpoint_path=sam_checkpoint_path)
     detections = self.dino.predict(image)
     
     masks = segment.segment(image, detections.xyxy)
     # Load the image
-    image_path = os.path.join(self.image_folder, image_name)
-    for item in os.listdir(self.image_folder):
-      print("Item: ",item)
+    # image_path = os.path.join(self.image_folder, image_name)
+    # for item in os.listdir(self.image_folder):
+    #   print("Item: ",item)
     if not os.path.exists(image_path):
-      print(f"Image {image_name} not found in folder {self.image_folder}")
+      print(f"Image {image_name} not found in folder {image_path}")
       return
 
     image = cv2.imread(image_path)
@@ -47,10 +48,10 @@ def create_cutouts(self, image_name, sam_checkpoint_path):
 
       # Save the cutout
       cutout_name = f"{image_name}_cutout_{i}.png"
-      cutout_path = os.path.join("cutouts", cutout_name)
+      cutout_path = os.path.join(HOME, "cutouts", cutout_name)
       cv2.imwrite(cutout_path, cutout)
 
       # Upload the cutout to S3
       with open(cutout_path, "rb") as f:
-        self.s3.upload_to_s3(f.read(), f"cutouts/{image_name}/{cutout_name}")
+        self.s3.upload_to_s3(f.read(), "cutouts",f"{image_name}/{cutout_name}")
 
diff --git a/app/grounded_cutouts.py b/app/grounded_cutouts.py
index 09d6f9e..bcac95c 100644
--- a/app/grounded_cutouts.py
+++ b/app/grounded_cutouts.py
@@ -4,7 +4,6 @@
 from fastapi import FastAPI, File, UploadFile
 from fastapi.responses import FileResponse
 
-
 app = FastAPI()
 
 stub = Stub(name="cutout_generator")
@@ -40,7 +39,7 @@
 async def upload_image_to_s3(image: UploadFile = File(...)):
     from s3_handler import Boto3Client
     s3_client = Boto3Client()
-    s3_client.upload_to_s3(image.file, image.filename)
+    s3_client.upload_to_s3(image.file,"images", image.filename)
     return {"message": "Image uploaded successfully"}
 
 @stub.function(mounts=[Mount.from_local_python_packages("s3_handler")], secret=Secret.from_name("my-aws-secret"))
@@ -55,30 +54,13 @@ async def create_presigned_cutout(image_name: str):
 @app.get("/create-cutouts/{image_name}")
 async def create_cutouts(image_name: str):
     from cutout import CutoutCreator
-    OUTPUT_CUTOUT_PATH = os.path.join(HOME, "cutouts")
-
+    from s3_handler import Boto3Client
+    s3 = Boto3Client()
     cutout = CutoutCreator(grounding_dino_checkpoint_path=GROUNDING_DINO_CHECKPOINT_PATH, grounding_dino_config_path=GROUNDING_DINO_CONFIG_PATH)
     cutout.create_cutouts(image_name, SAM_CHECKPOINT_PATH )
-    return FileResponse(os.path.join(OUTPUT_CUTOUT_PATH, f"{image_name.split('.')[0]}_cutouts.zip"))
-
-# @app.post("/create-cutouts")
-# async def create_cutouts_from_image(image: UploadFile = File(...)):
-#     SOURCE_IMAGE_PATH = os.path.join(HOME, "data", image.filename)
-#     SAVE_IMAGE_PATH = os.path.join(HOME, "data")
-#     OUTPUT_CUTOUT_PATH = os.path.join(HOME, "cutouts")
-#     dino = Dino(classes=['person', 'nose', 'chair', 'shoe', 'ear', 'hat'],
-#         box_threshold=0.35,
-#         text_threshold=0.25,
-#         model_config_path=GROUNDING_DINO_CONFIG_PATH,
-#         model_checkpoint_path=GROUNDING_DINO_CHECKPOINT_PATH)
-
-#     segment = Segmenter(sam_encoder_version="vit_h", sam_checkpoint_path=SAM_CHECKPOINT_PATH)
-
-#     cutout = CutoutCreator(image_folder=SAVE_IMAGE_PATH)
-#     cutout.create_cutouts_from_file(image.file, dino, segment, OUTPUT_CUTOUT_PATH)
-#     return FileResponse(os.path.join(OUTPUT_CUTOUT_PATH, f"{image.filename.split('.')[0]}_cutouts.zip"))
+    return s3.generate_presigned_urls(f"cutouts/{image_name}")
 
-@stub.function(image=cutout_generator_image, mounts=[local_packages], secret=Secret.from_name("my-aws-secret"))
+@stub.function(image=cutout_generator_image,gpu="T4", mounts=[local_packages], secret=Secret.from_name("my-aws-secret"))
 @asgi_app()
 def cutout_app():
-  return app
\ No newline at end of file
+  return app

From c6c5b547fce2955cb0ba8b655810464ed3ec6e84 Mon Sep 17 00:00:00 2001
From: OriginalByteMe <noahrijkaard@gmail.com>
Date: Mon, 6 Nov 2023 19:47:18 +0800
Subject: [PATCH 05/14] Return list of presigned urls

---
 app/s3_handler.py | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/app/s3_handler.py b/app/s3_handler.py
index 0c26a93..ebd5448 100644
--- a/app/s3_handler.py
+++ b/app/s3_handler.py
@@ -16,25 +16,31 @@ def download_from_s3(self, save_path, image_name):
         s3_client = boto3.client("s3")
         file_path = os.path.join(save_path, image_name)
         try:
-            s3_client.download_file(os.environ["CUTOUT_BUCKET"], f"cutouts/{image_name}/cutout-{image_name}", file_path)
+            s3_client.download_file(os.environ["CUTOUT_BUCKET"], f"images/{image_name}", file_path)
         except ClientError as e:
             print("BOTO error: ",e)
+            print(f"File {image_name} not found in bucket {os.environ['CUTOUT_BUCKET']}")
             return None
 
         return file_path
 
-    def upload_to_s3(self, file_body, image_name):
-        self.s3.put_object(Body=file_body, Bucket=os.environ["CUTOUT_BUCKET"], Key=f"images/{image_name}")
+    def upload_to_s3(self, file_body, folder, image_name):
+        self.s3.put_object(Body=file_body, Bucket=os.environ["CUTOUT_BUCKET"], Key=f"{folder}/{image_name}")
 
-    def generate_presigned_url(self, image_name, expiration=3600):
+    def generate_presigned_urls(self, folder, expiration=3600):
         try:
-            response = self.s3.generate_presigned_url(
-                "get_object",
-                Params={"Bucket": os.environ["CUTOUT_BUCKET"], "Key": f"cutouts/{image_name}/cutout-{image_name}"},
-                ExpiresIn=expiration,
-            )
+            response = self.s3.list_objects_v2(Bucket=os.environ["CUTOUT_BUCKET"], Prefix=folder)
+            urls = []
+            for obj in response.get('Contents', []):
+                key = obj['Key']
+                url = self.s3.generate_presigned_url(
+                    "get_object",
+                    Params={"Bucket": os.environ["CUTOUT_BUCKET"], "Key": key},
+                    ExpiresIn=expiration,
+                )
+                urls.append(url)
         except ClientError as e:
             print(e)
             return None
 
-        return response
+        return urls

From 409989aa3717c000c0219d590559a4c189fa1f9b Mon Sep 17 00:00:00 2001
From: OriginalByteMe <noahrijkaard@gmail.com>
Date: Mon, 6 Nov 2023 23:39:56 +0800
Subject: [PATCH 06/14] remove unused api endpoint

---
 app/grounded_cutouts.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/app/grounded_cutouts.py b/app/grounded_cutouts.py
index bcac95c..331eb74 100644
--- a/app/grounded_cutouts.py
+++ b/app/grounded_cutouts.py
@@ -42,14 +42,6 @@ async def upload_image_to_s3(image: UploadFile = File(...)):
     s3_client.upload_to_s3(image.file,"images", image.filename)
     return {"message": "Image uploaded successfully"}
 
-@stub.function(mounts=[Mount.from_local_python_packages("s3_handler")], secret=Secret.from_name("my-aws-secret"))
-@app.get("/create-presigned-cutout/{image_name}")
-async def create_presigned_cutout(image_name: str):
-    from s3_handler import Boto3Client
-    s3_client = Boto3Client()
-    presigned_url = s3_client.generate_presigned_url(image_name)
-    return {"presigned_url": presigned_url}
-
 @stub.function(image=cutout_generator_image, mounts=[local_packages], gpu="T4", secret=Secret.from_name("my-aws-secret"))
 @app.get("/create-cutouts/{image_name}")
 async def create_cutouts(image_name: str):

From 8da7c1cb1b221743a86aa9e3fcbec501a926fa7a Mon Sep 17 00:00:00 2001
From: OriginalByteMe <noahrijkaard@gmail.com>
Date: Mon, 6 Nov 2023 23:40:31 +0800
Subject: [PATCH 07/14] Add functionality of creating annotated image

---
 app/cutout.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/app/cutout.py b/app/cutout.py
index c658f7f..af67133 100644
--- a/app/cutout.py
+++ b/app/cutout.py
@@ -4,6 +4,10 @@
 from s3_handler import Boto3Client
 from dino import Dino
 from segment import Segmenter
+from typing import Dict
+import supervision as sv
+import io
+from PIL import Image
 
 HOME = os.path.abspath(os.path.join(os.getcwd(), os.pardir))
 
@@ -15,7 +19,17 @@ def __init__(self, grounding_dino_config_path, grounding_dino_checkpoint_path):
         model_config_path=grounding_dino_config_path,
         model_checkpoint_path=grounding_dino_checkpoint_path)
     self.s3 = Boto3Client()
+    self.mask_annotator = sv.MaskAnnotator()
 
+  def create_annotated_image(self, image, image_name, detections: Dict[str, list]):
+      annotated_image = self.mask_annotator.annotate(scene=image, detections=detections)
+      # Convert annotated image to bytes
+      img_bytes = io.BytesIO()
+      Image.fromarray(np.uint8(annotated_image)).save(img_bytes, format='PNG')
+      img_bytes.seek(0)
+      # Upload bytes to S3
+      self.s3.upload_to_s3(img_bytes.read(), "cutouts", f"{image_name}_annotated.png")
+  
   def create_cutouts(self, image_name, sam_checkpoint_path):
     
     # Download image from s3
@@ -55,3 +69,6 @@ def create_cutouts(self, image_name, sam_checkpoint_path):
       with open(cutout_path, "rb") as f:
         self.s3.upload_to_s3(f.read(), "cutouts",f"{image_name}/{cutout_name}")
 
+    # Create annotated image
+    # self.create_annotated_image(image, f"{image_name}_{i}", detections)
+

From 40e5aff51a2477819bf76f2e395550630dd89c4c Mon Sep 17 00:00:00 2001
From: OriginalByteMe <noahrijkaard@gmail.com>
Date: Mon, 6 Nov 2023 23:57:13 +0800
Subject: [PATCH 08/14] Add formatting + docs

---
 app/cutout.py           | 137 ++++++++++++++++++++++++----------------
 app/dino.py             |  68 ++++++++++++++------
 app/grounded_cutouts.py | 119 ++++++++++++++++++++++++----------
 app/s3_handler.py       |  24 +++++--
 app/segment.py          |  35 +++++-----
 5 files changed, 253 insertions(+), 130 deletions(-)

diff --git a/app/cutout.py b/app/cutout.py
index af67133..054da01 100644
--- a/app/cutout.py
+++ b/app/cutout.py
@@ -1,74 +1,99 @@
+from typing import Dict
+import os
+import io
 import cv2
 import numpy as np
-import os
 from s3_handler import Boto3Client
 from dino import Dino
 from segment import Segmenter
-from typing import Dict
-import supervision as sv
-import io
 from PIL import Image
+import supervision as sv
 
 HOME = os.path.abspath(os.path.join(os.getcwd(), os.pardir))
 
 class CutoutCreator:
-  def __init__(self, grounding_dino_config_path, grounding_dino_checkpoint_path):
-    self.dino = Dino(classes=['person', 'nose', 'chair', 'shoe', 'ear', 'hat'],
-        box_threshold=0.35,
-        text_threshold=0.25,
-        model_config_path=grounding_dino_config_path,
-        model_checkpoint_path=grounding_dino_checkpoint_path)
-    self.s3 = Boto3Client()
-    self.mask_annotator = sv.MaskAnnotator()
+    """
+    A class for creating cutouts from an image and uploading them to S3.
+
+    Attributes:
+      dino: A Dino object for object detection.
+      s3: A Boto3Client object for uploading to S3.
+      mask_annotator: A MaskAnnotator object for annotating images with masks.
+    """
+    def __init__(self, classes: str, grounding_dino_config_path: str, grounding_dino_checkpoint_path: str):
+        self.dino = Dino(
+            classes=classes,
+            box_threshold=0.35,
+            text_threshold=0.25,
+            model_config_path=grounding_dino_config_path,
+            model_checkpoint_path=grounding_dino_checkpoint_path,
+        )
+        self.s3 = Boto3Client()
+        self.mask_annotator = sv.MaskAnnotator()
+
+    def create_annotated_image(self, image, image_name, detections: Dict[str, list]):
+        """Create a highlighted annotated image from an image and detections.
+
+        Args:
+            image (File): Image to be used for creating the annotated image.
+            image_name (string): name of image
+            detections (Dict[str, list]): annotations for the image
+        """
+        annotated_image = self.mask_annotator.annotate(
+            scene=image, detections=detections
+        )
+        # Convert annotated image to bytes
+        img_bytes = io.BytesIO()
+        Image.fromarray(np.uint8(annotated_image)).save(img_bytes, format="PNG")
+        img_bytes.seek(0)
+        # Upload bytes to S3
+        self.s3.upload_to_s3(img_bytes.read(), "cutouts", f"{image_name}_annotated.png")
+
+    def create_cutouts(self, image_name, sam_checkpoint_path):
+        """Create cutouts from an image and upload them to S3.
 
-  def create_annotated_image(self, image, image_name, detections: Dict[str, list]):
-      annotated_image = self.mask_annotator.annotate(scene=image, detections=detections)
-      # Convert annotated image to bytes
-      img_bytes = io.BytesIO()
-      Image.fromarray(np.uint8(annotated_image)).save(img_bytes, format='PNG')
-      img_bytes.seek(0)
-      # Upload bytes to S3
-      self.s3.upload_to_s3(img_bytes.read(), "cutouts", f"{image_name}_annotated.png")
-  
-  def create_cutouts(self, image_name, sam_checkpoint_path):
-    
-    # Download image from s3
-    image_path = self.s3.download_from_s3(os.path.join(HOME, "data"), image_name)
-    if not os.path.exists(os.path.join(HOME, "cutouts")):
-      os.mkdir(os.path.join(HOME, "cutouts"))
-    image = cv2.imread(image_path)
-    segment = Segmenter(sam_encoder_version="vit_h", sam_checkpoint_path=sam_checkpoint_path)
-    detections = self.dino.predict(image)
-    
-    masks = segment.segment(image, detections.xyxy)
-    # Load the image
-    # image_path = os.path.join(self.image_folder, image_name)
-    # for item in os.listdir(self.image_folder):
-    #   print("Item: ",item)
-    if not os.path.exists(image_path):
-      print(f"Image {image_name} not found in folder {image_path}")
-      return
+        Args:
+            image_name (string): name of image
+            sam_checkpoint_path (string): path to sam checkpoint
+        """
+        # Download image from s3
+        image_path = self.s3.download_from_s3(os.path.join(HOME, "data"), image_name)
+        if not os.path.exists(os.path.join(HOME, "cutouts")):
+            os.mkdir(os.path.join(HOME, "cutouts"))
+        image = cv2.imread(image_path)
+        segment = Segmenter(
+            sam_encoder_version="vit_h", sam_checkpoint_path=sam_checkpoint_path
+        )
+        detections = self.dino.predict(image)
 
-    image = cv2.imread(image_path)
+        masks = segment.segment(image, detections.xyxy)
+        # Load the image
+        # image_path = os.path.join(self.image_folder, image_name)
+        # for item in os.listdir(self.image_folder):
+        #   print("Item: ",item)
+        if not os.path.exists(image_path):
+            print(f"Image {image_name} not found in folder {image_path}")
+            return
 
-    # Apply each mask to the image
-    for i, mask in enumerate(masks):
-      # Ensure the mask is a boolean array
-      mask = mask.astype(bool)
+        image = cv2.imread(image_path)
 
-      # Apply the mask to create a cutout
-      cutout = np.zeros_like(image)
-      cutout[mask] = image[mask]
+        # Apply each mask to the image
+        for i, mask in enumerate(masks):
+            # Ensure the mask is a boolean array
+            mask = mask.astype(bool)
 
-      # Save the cutout
-      cutout_name = f"{image_name}_cutout_{i}.png"
-      cutout_path = os.path.join(HOME, "cutouts", cutout_name)
-      cv2.imwrite(cutout_path, cutout)
+            # Apply the mask to create a cutout
+            cutout = np.zeros_like(image)
+            cutout[mask] = image[mask]
 
-      # Upload the cutout to S3
-      with open(cutout_path, "rb") as f:
-        self.s3.upload_to_s3(f.read(), "cutouts",f"{image_name}/{cutout_name}")
+            # Save the cutout
+            cutout_name = f"{image_name}_cutout_{i}.png"
+            cutout_path = os.path.join(HOME, "cutouts", cutout_name)
+            cv2.imwrite(cutout_path, cutout)
 
-    # Create annotated image
-    # self.create_annotated_image(image, f"{image_name}_{i}", detections)
+            # Upload the cutout to S3
+            with open(cutout_path, "rb") as f:
+                self.s3.upload_to_s3(f.read(), "cutouts", f"{image_name}/{cutout_name}")
 
+        # Create annotated image
+        # self.create_annotated_image(image, f"{image_name}_{i}", detections)
diff --git a/app/dino.py b/app/dino.py
index b46199b..69a5f8c 100644
--- a/app/dino.py
+++ b/app/dino.py
@@ -4,25 +4,55 @@
 
 
 class Dino:
-  def __init__(self, classes, box_threshold, text_threshold, model_config_path, model_checkpoint_path):
-    self.classes = classes
-    self.box_threshold = box_threshold
-    self.text_threshold = text_threshold
-    self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
-    self.grounding_dino_model = Model(model_config_path=model_config_path, model_checkpoint_path=model_checkpoint_path)
-
-  def enhance_class_name(self, class_names: List[str]) -> List[str]:
-    return [
-        f"all {class_name}s"
-        for class_name
-        in class_names
-    ]
-
-  def predict(self, image):
-    detections = self.grounding_dino_model.predict_with_classes(image=image, classes=self.enhance_class_name(class_names=self.classes), box_threshold=self.box_threshold, text_threshold=self.text_threshold)
-    detections = detections[detections.class_id != None]
-    return detections
-  
+    """ A class for object detection using GroundingDINO.
+    """
+    def __init__(
+        self,
+        classes,
+        box_threshold,
+        text_threshold,
+        model_config_path,
+        model_checkpoint_path,
+    ):
+        self.classes = classes
+        self.box_threshold = box_threshold
+        self.text_threshold = text_threshold
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        self.grounding_dino_model = Model(
+            model_config_path=model_config_path,
+            model_checkpoint_path=model_checkpoint_path,
+        )
+
+    def enhance_class_name(self, class_names: List[str]) -> List[str]:
+        """Enhance class names for GroundingDINO.
+
+        Args:
+            class_names (List[str]): List of class names.
+
+        Returns:
+            List[str]: List of class names with "all" prepended.
+        """
+        return [f"all {class_name}s" for class_name in class_names]
+
+    def predict(self, image):
+        """Predict objects in an image.
+
+        Args:
+            image (File): Image to be used for object detection.
+
+        Returns:
+            Dict[str, list]: Dictionary of objects detected in the image.
+        """
+        detections = self.grounding_dino_model.predict_with_classes(
+            image=image,
+            classes=self.enhance_class_name(class_names=self.classes),
+            box_threshold=self.box_threshold,
+            text_threshold=self.text_threshold,
+        )
+        detections = detections[detections.class_id is not None]
+        return detections
+
+
 # Example usage
 # dino = Dino(classes=['person', 'nose', 'chair', 'shoe', 'ear', 'hat'],
 #       box_threshold=0.35,
diff --git a/app/grounded_cutouts.py b/app/grounded_cutouts.py
index 331eb74..a5d0e0f 100644
--- a/app/grounded_cutouts.py
+++ b/app/grounded_cutouts.py
@@ -1,58 +1,111 @@
 import os
 from modal import asgi_app, Secret, Stub, Mount, Image
-import cv2
-from fastapi import FastAPI, File, UploadFile
-from fastapi.responses import FileResponse
+from fastapi import FastAPI, File, UploadFile, Body
+from typing import List
 
 app = FastAPI()
 
 stub = Stub(name="cutout_generator")
 
-local_packages = Mount.from_local_python_packages("cutout", "dino", "segment", "s3_handler", "fastapi", "starlette")
-cutout_generator_image = Image.from_registry("nvcr.io/nvidia/pytorch:22.12-py3").pip_install( "segment-anything", "opencv-python", "botocore", "boto3").run_commands(
-  "apt-get update",
-  "apt-get install -y git wget libgl1-mesa-glx libglib2.0-0",
-  "echo $CUDA_HOME",
-  "git clone https://github.com/IDEA-Research/GroundingDINO.git",
-  "pip install -q -e GroundingDINO/",
-  "mkdir -p /weights",
-  "mkdir -p /data",
-  "pip uninstall -y supervision",
-  "pip uninstall -y opencv-python",
-"pip install opencv-python==4.8.0.74",
-  "pip install -q supervision==0.6.0",
-  "wget -q https://github.com/IDEA-Research/GroundingDINO/releases/download/v0.1.0-alpha/groundingdino_swint_ogc.pth -P weights/",
-  "wget -q https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth -P weights/",
-  "wget -q https://media.roboflow.com/notebooks/examples/dog.jpeg -P images/",
-  "ls -F",
-  "ls -F GroundingDINO/groundingdino/config",
-  "ls -F GroundingDINO/groundingdino/models/GroundingDINO/"
+local_packages = Mount.from_local_python_packages(
+    "cutout", "dino", "segment", "s3_handler", "fastapi", "starlette"
+)
+cutout_generator_image = (
+    Image.from_registry("nvcr.io/nvidia/pytorch:22.12-py3")
+    .pip_install("segment-anything", "opencv-python", "botocore", "boto3")
+    .run_commands(
+        "apt-get update",
+        "apt-get install -y git wget libgl1-mesa-glx libglib2.0-0",
+        "echo $CUDA_HOME",
+        "git clone https://github.com/IDEA-Research/GroundingDINO.git",
+        "pip install -q -e GroundingDINO/",
+        "mkdir -p /weights",
+        "mkdir -p /data",
+        "pip uninstall -y supervision",
+        "pip uninstall -y opencv-python",
+        "pip install opencv-python==4.8.0.74",
+        "pip install -q supervision==0.6.0",
+        "wget -q https://github.com/IDEA-Research/GroundingDINO/releases/download/v0.1.0-alpha/groundingdino_swint_ogc.pth -P weights/",
+        "wget -q https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth -P weights/",
+        "wget -q https://media.roboflow.com/notebooks/examples/dog.jpeg -P images/",
+        "ls -F",
+        "ls -F GroundingDINO/groundingdino/config",
+        "ls -F GroundingDINO/groundingdino/models/GroundingDINO/",
+    )
 )
 
 HOME = os.path.abspath(os.path.join(os.getcwd(), os.pardir))
-GROUNDING_DINO_CONFIG_PATH = os.path.join(HOME, "GroundingDINO/groundingdino/config/GroundingDINO_SwinT_OGC.py")
-GROUNDING_DINO_CHECKPOINT_PATH = os.path.join(HOME, "weights", "groundingdino_swint_ogc.pth")
+GROUNDING_DINO_CONFIG_PATH = os.path.join(
+    HOME, "GroundingDINO/groundingdino/config/GroundingDINO_SwinT_OGC.py"
+)
+GROUNDING_DINO_CHECKPOINT_PATH = os.path.join(
+    HOME, "weights", "groundingdino_swint_ogc.pth"
+)
 SAM_CHECKPOINT_PATH = os.path.join(HOME, "weights", "sam_vit_h_4b8939.pth")
 
-@stub.function(mounts=[Mount.from_local_python_packages("s3_handler")], secret=Secret.from_name("my-aws-secret"))
+
+@stub.function(
+    mounts=[Mount.from_local_python_packages("s3_handler")],
+    secret=Secret.from_name("my-aws-secret"),
+)
 @app.post("/upload-image")
 async def upload_image_to_s3(image: UploadFile = File(...)):
+    """Upload an image to S3.
+
+    Args:
+        image (UploadFile, optional): File to upload to s3 . Defaults to File(...).
+
+    Returns:
+        str: Message indicating whether the upload was successful.
+    """
     from s3_handler import Boto3Client
+
     s3_client = Boto3Client()
-    s3_client.upload_to_s3(image.file,"images", image.filename)
+    s3_client.upload_to_s3(image.file, "images", image.filename)
     return {"message": "Image uploaded successfully"}
 
-@stub.function(image=cutout_generator_image, mounts=[local_packages], gpu="T4", secret=Secret.from_name("my-aws-secret"))
-@app.get("/create-cutouts/{image_name}")
-async def create_cutouts(image_name: str):
+
+@stub.function(
+    image=cutout_generator_image,
+    mounts=[local_packages],
+    gpu="T4",
+    secret=Secret.from_name("my-aws-secret"),
+)
+@app.post("/create-cutouts/{image_name}")
+async def create_cutouts(image_name: str, classes: List[str] = Body(...)):
+    """Create cutouts from an image and upload them to S3.
+
+    Args:
+        image_name (str): Name of image to create cutouts from.
+        classes (List[str], optional): A list of classes for the AI to detect for. Defaults to Body(...).
+
+    Returns:
+        _type_: _description_
+    """
     from cutout import CutoutCreator
     from s3_handler import Boto3Client
+
     s3 = Boto3Client()
-    cutout = CutoutCreator(grounding_dino_checkpoint_path=GROUNDING_DINO_CHECKPOINT_PATH, grounding_dino_config_path=GROUNDING_DINO_CONFIG_PATH)
-    cutout.create_cutouts(image_name, SAM_CHECKPOINT_PATH )
+    cutout = CutoutCreator(
+        classes=classes,
+        grounding_dino_checkpoint_path=GROUNDING_DINO_CHECKPOINT_PATH,
+        grounding_dino_config_path=GROUNDING_DINO_CONFIG_PATH,
+    )
+    cutout.create_cutouts(image_name, SAM_CHECKPOINT_PATH)
     return s3.generate_presigned_urls(f"cutouts/{image_name}")
 
-@stub.function(image=cutout_generator_image,gpu="T4", mounts=[local_packages], secret=Secret.from_name("my-aws-secret"))
+
+@stub.function(
+    image=cutout_generator_image,
+    gpu="T4",
+    mounts=[local_packages],
+    secret=Secret.from_name("my-aws-secret"),
+)
 @asgi_app()
 def cutout_app():
-  return app
+    """Create a FastAPI app for creating cutouts.
+
+    Returns:
+        FastAPI: FastAPI app for creating cutouts.
+    """
+    return app
diff --git a/app/s3_handler.py b/app/s3_handler.py
index ebd5448..5481641 100644
--- a/app/s3_handler.py
+++ b/app/s3_handler.py
@@ -16,23 +16,33 @@ def download_from_s3(self, save_path, image_name):
         s3_client = boto3.client("s3")
         file_path = os.path.join(save_path, image_name)
         try:
-            s3_client.download_file(os.environ["CUTOUT_BUCKET"], f"images/{image_name}", file_path)
+            s3_client.download_file(
+                os.environ["CUTOUT_BUCKET"], f"images/{image_name}", file_path
+            )
         except ClientError as e:
-            print("BOTO error: ",e)
-            print(f"File {image_name} not found in bucket {os.environ['CUTOUT_BUCKET']}")
+            print("BOTO error: ", e)
+            print(
+                f"File {image_name} not found in bucket {os.environ['CUTOUT_BUCKET']}"
+            )
             return None
 
         return file_path
 
     def upload_to_s3(self, file_body, folder, image_name):
-        self.s3.put_object(Body=file_body, Bucket=os.environ["CUTOUT_BUCKET"], Key=f"{folder}/{image_name}")
+        self.s3.put_object(
+            Body=file_body,
+            Bucket=os.environ["CUTOUT_BUCKET"],
+            Key=f"{folder}/{image_name}",
+        )
 
     def generate_presigned_urls(self, folder, expiration=3600):
         try:
-            response = self.s3.list_objects_v2(Bucket=os.environ["CUTOUT_BUCKET"], Prefix=folder)
+            response = self.s3.list_objects_v2(
+                Bucket=os.environ["CUTOUT_BUCKET"], Prefix=folder
+            )
             urls = []
-            for obj in response.get('Contents', []):
-                key = obj['Key']
+            for obj in response.get("Contents", []):
+                key = obj["Key"]
                 url = self.s3.generate_presigned_url(
                     "get_object",
                     Params={"Bucket": os.environ["CUTOUT_BUCKET"], "Key": key},
diff --git a/app/segment.py b/app/segment.py
index 3d61359..33e7a88 100644
--- a/app/segment.py
+++ b/app/segment.py
@@ -4,19 +4,24 @@
 
 
 class Segmenter:
-  def __init__(self, sam_encoder_version: str, sam_checkpoint_path: str, ):
-    self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
-    self.sam = sam_model_registry[sam_encoder_version](checkpoint=sam_checkpoint_path).to(device=self.device)
-    self.sam_predictor = SamPredictor(self.sam)
+    def __init__(
+        self,
+        sam_encoder_version: str,
+        sam_checkpoint_path: str,
+    ):
+        self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        self.sam = sam_model_registry[sam_encoder_version](
+            checkpoint=sam_checkpoint_path
+        ).to(device=self.device)
+        self.sam_predictor = SamPredictor(self.sam)
 
-  def segment(self, image: np.ndarray, xyxy: np.ndarray) -> np.ndarray:
-    self.sam_predictor.set_image(image)
-    result_masks = []
-    for box in xyxy:
-      masks, scores, logits = self.sam_predictor.predict(
-        box=box,
-        multimask_output=True
-      )
-      index = np.argmax(scores)
-      result_masks.append(masks[index])
-    return np.array(result_masks)
\ No newline at end of file
+    def segment(self, image: np.ndarray, xyxy: np.ndarray) -> np.ndarray:
+        self.sam_predictor.set_image(image)
+        result_masks = []
+        for box in xyxy:
+            masks, scores, logits = self.sam_predictor.predict(
+                box=box, multimask_output=True
+            )
+            index = np.argmax(scores)
+            result_masks.append(masks[index])
+        return np.array(result_masks)

From becaa572e22931e387d01438771452a0d70e9994 Mon Sep 17 00:00:00 2001
From: OriginalByteMe <noahrijkaard@gmail.com>
Date: Tue, 7 Nov 2023 13:07:49 +0800
Subject: [PATCH 09/14] Add ability for mass image processing

---
 app/grounded_cutouts.py | 41 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)

diff --git a/app/grounded_cutouts.py b/app/grounded_cutouts.py
index a5d0e0f..18bb492 100644
--- a/app/grounded_cutouts.py
+++ b/app/grounded_cutouts.py
@@ -64,6 +64,20 @@ async def upload_image_to_s3(image: UploadFile = File(...)):
     s3_client.upload_to_s3(image.file, "images", image.filename)
     return {"message": "Image uploaded successfully"}
 
+@app.get("/generate-presigned-urls/{image_name}")
+async def generate_presigned_urls(image_name: str):
+    """Generate presigned urls for the cutouts of an image.
+
+    Args:
+        image_name (str): Name of image to generate presigned urls for.
+
+    Returns:
+        List[str]: List of presigned urls for the cutouts of an image.
+    """
+    from s3_handler import Boto3Client
+
+    s3_client = Boto3Client()
+    return s3_client.generate_presigned_urls(f"cutouts/{image_name}")
 
 @stub.function(
     image=cutout_generator_image,
@@ -94,6 +108,33 @@ async def create_cutouts(image_name: str, classes: List[str] = Body(...)):
     cutout.create_cutouts(image_name, SAM_CHECKPOINT_PATH)
     return s3.generate_presigned_urls(f"cutouts/{image_name}")
 
+@app.post("/create-cutouts")
+async def create_cutouts(image_names: List[str] = Body(...), classes: List[str] = Body(...)):
+    """Create cutouts from multiple images and upload them to S3.
+
+    Args:
+        image_names (List[str]): List of image names to create cutouts from.
+        classes (List[str], optional): A list of classes for the AI to detect for. Defaults to Body(...).
+
+    Returns:
+        Dict[str, List[str]]: A dictionary where the keys are the image names and the values are the lists of presigned URLs for the cutouts.
+    """
+    from cutout import CutoutCreator
+    from s3_handler import Boto3Client
+
+    s3 = Boto3Client()
+    cutout = CutoutCreator(
+        classes=classes,
+        grounding_dino_checkpoint_path=GROUNDING_DINO_CHECKPOINT_PATH,
+        grounding_dino_config_path=GROUNDING_DINO_CONFIG_PATH,
+    )
+
+    result = {}
+    for image_name in image_names:
+        cutout.create_cutouts(image_name, SAM_CHECKPOINT_PATH)
+        result[image_name] = s3.generate_presigned_urls(f"cutouts/{image_name}")
+
+    return result
 
 @stub.function(
     image=cutout_generator_image,

From ccf2074bd8ea6d9e164d4e39cf42da4f9a57fbb3 Mon Sep 17 00:00:00 2001
From: OriginalByteMe <noahrijkaard@gmail.com>
Date: Wed, 15 Nov 2023 20:35:30 +0800
Subject: [PATCH 10/14] Add CORS middleware and handle exceptions in S3 upload

---
 app/grounded_cutouts.py | 30 ++++++++++++++++++++++++++----
 app/s3_handler.py       | 25 ++++++++++++++++++-------
 2 files changed, 44 insertions(+), 11 deletions(-)

diff --git a/app/grounded_cutouts.py b/app/grounded_cutouts.py
index 18bb492..f54d324 100644
--- a/app/grounded_cutouts.py
+++ b/app/grounded_cutouts.py
@@ -1,12 +1,26 @@
 import os
 from modal import asgi_app, Secret, Stub, Mount, Image
-from fastapi import FastAPI, File, UploadFile, Body
+from fastapi import FastAPI, File, UploadFile, Body, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
 from typing import List
 
 app = FastAPI()
 
 stub = Stub(name="cutout_generator")
 
+origins = [
+    "http://localhost:3000",  # localdevelopment
+    "https://cutouts.noahrijkaard.com",  # main website
+]
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=origins,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
 local_packages = Mount.from_local_python_packages(
     "cutout", "dino", "segment", "s3_handler", "fastapi", "starlette"
 )
@@ -59,10 +73,18 @@ async def upload_image_to_s3(image: UploadFile = File(...)):
         str: Message indicating whether the upload was successful.
     """
     from s3_handler import Boto3Client
-
+    from botocore.exceptions import BotoCoreError, NoCredentialsError
+    
     s3_client = Boto3Client()
-    s3_client.upload_to_s3(image.file, "images", image.filename)
-    return {"message": "Image uploaded successfully"}
+    try:
+        s3_client.upload_to_s3(image.file, "images", image.filename)
+    except NoCredentialsError as e:
+        raise HTTPException(status_code=401, detail="No AWS credentials found") from e
+    except BotoCoreError as e:
+        raise HTTPException(status_code=500, detail=str(e)) from e
+    except Exception as e:
+        raise HTTPException(status_code=500, detail="An error occurred while uploading the image") from e
+    return {"message": "Image uploaded successfully", "status_code": 200}
 
 @app.get("/generate-presigned-urls/{image_name}")
 async def generate_presigned_urls(image_name: str):
diff --git a/app/s3_handler.py b/app/s3_handler.py
index 5481641..b27501e 100644
--- a/app/s3_handler.py
+++ b/app/s3_handler.py
@@ -1,7 +1,7 @@
 import os
 import boto3
-from botocore.exceptions import ClientError
-
+import logging
+from botocore.exceptions import ClientError, BotoCoreError, NoCredentialsError
 
 class Boto3Client:
     def __init__(self):
@@ -29,11 +29,22 @@ def download_from_s3(self, save_path, image_name):
         return file_path
 
     def upload_to_s3(self, file_body, folder, image_name):
-        self.s3.put_object(
-            Body=file_body,
-            Bucket=os.environ["CUTOUT_BUCKET"],
-            Key=f"{folder}/{image_name}",
-        )
+        try:
+            self.s3.put_object(
+                Body=file_body,
+                Bucket=os.environ["CUTOUT_BUCKET"],
+                Key=f"{folder}/{image_name}",
+            )
+            logging.info(f"Successfully uploaded {image_name} to {folder}")
+        except NoCredentialsError as e:
+            logging.error("No AWS credentials found")
+            raise
+        except BotoCoreError as e:
+            logging.error(f"An error occurred with Boto3: {e}")
+            raise
+        except Exception as e:
+            logging.error(f"An error occurred while uploading the image: {e}")
+            raise
 
     def generate_presigned_urls(self, folder, expiration=3600):
         try:

From 4fdc00d42a5b7c7615200d85129dbc0a29d5ba7d Mon Sep 17 00:00:00 2001
From: OriginalByteMe <noahrijkaard@gmail.com>
Date: Thu, 16 Nov 2023 00:16:02 +0800
Subject: [PATCH 11/14] Create S3 handler app for modal

---
 app/s3_handler/app.py        | 119 +++++++++++++++++++++++++++++++++++
 app/s3_handler/s3_handler.py |  90 ++++++++++++++++++++++++++
 2 files changed, 209 insertions(+)
 create mode 100644 app/s3_handler/app.py
 create mode 100644 app/s3_handler/s3_handler.py

diff --git a/app/s3_handler/app.py b/app/s3_handler/app.py
new file mode 100644
index 0000000..f4656ce
--- /dev/null
+++ b/app/s3_handler/app.py
@@ -0,0 +1,119 @@
+from modal import Mount, Image, Secret, Stub, asgi_app
+import os
+import logging
+from fastapi import FastAPI, File, UploadFile, Body, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
+
+
+stub = Stub(name="s3_handler")
+
+app = FastAPI()
+
+origins = [
+    "http://localhost:3000",  # localdevelopment
+    "https://cutouts.noahrijkaard.com",  # main website
+]
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=origins,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# ================================================
+#  API Endpoints
+# ================================================
+@app.post("/upload-image")
+async def upload_image_to_s3(image: UploadFile = File(...)):
+    """Upload an image to S3.
+
+    Args:
+        image (UploadFile, optional): File to upload to s3 . Defaults to File(...).
+
+    Returns:
+        str: Message indicating whether the upload was successful.
+    """
+    from s3_handler import Boto3Client
+    from botocore.exceptions import BotoCoreError, NoCredentialsError
+    
+    s3_client = Boto3Client()
+    try:
+        s3_client.upload_to_s3(image.file, "images", image.filename)
+    except NoCredentialsError as e:
+        raise HTTPException(status_code=401, detail="No AWS credentials found") from e
+    except BotoCoreError as e:
+        raise HTTPException(status_code=500, detail=str(e)) from e
+    except Exception as e:
+        raise HTTPException(status_code=500, detail="An error occurred while uploading the image") from e
+    return {"message": "Image uploaded successfully", "status_code": 200}
+
+@app.get("/generate-presigned-urls/{image_name}")
+async def generate_presigned_urls(image_name: str):
+    """Generate presigned urls for the cutouts of an image.
+
+    Args:
+        image_name (str): Name of image to generate presigned urls for.
+
+    Returns:
+        List[str]: List of presigned urls for the cutouts of an image.
+    """
+    from s3_handler import Boto3Client
+
+    s3_client = Boto3Client()
+    return s3_client.generate_presigned_urls(f"cutouts/{image_name}")
+
+
+@app.get('/get-image/{image_name}')
+async def get_image(image_name: str):
+    """Get an image from S3.
+
+    Args:
+        image_name (str): Name of image to get.
+
+    Returns:
+        FileResponse: FileResponse object containing the image.
+    """
+    from s3_handler import Boto3Client
+    from fastapi.responses import FileResponse
+
+    s3_client = Boto3Client()
+    data = s3_client.generate_presigned_url_with_metadata("images", image_name)
+    if data is None:
+        raise HTTPException(status_code=404, detail="Image not found")
+    return data
+
+
+@stub.function(
+    image=Image.debian_slim().pip_install("boto3", "fastapi", "starlette", "uvicorn", "python-multipart", "pydantic", "requests", "httpx", "httpcore", "httpx[http2]", "httpx[http1]"), mounts=[Mount.from_local_python_packages("s3_handler")], secret=Secret.from_name("my-aws-secret")
+)
+
+@asgi_app()
+def main():
+    return app
+
+# =================================
+#  Modal s3 functions
+# =================================
+# @stub.function(
+#     image=Image.debian_slim().pip_install("boto3", "fastapi", "starlette", "uvicorn", "python-multipart", "pydantic", "requests", "httpx", "httpcore", "httpx[http2]", "httpx[http1]"), mounts=[Mount.from_local_python_packages("s3_handler")], secret=Secret.from_name("my-aws-secret")
+# )
+# def upload_to_s3(file_body, folder, image_name):
+#     from s3_handler import Boto3Client
+#     from botocore.exceptions import ClientError, BotoCoreError, NoCredentialsError
+
+#     s3_client = Boto3Client()
+    
+#     try:
+#         s3_client.upload_to_s3(file_body, folder, image_name)
+#         logging.info(f"Successfully uploaded {image_name} to {folder}")
+#     except NoCredentialsError as e:
+#         logging.error("No AWS credentials found")
+#         raise
+#     except BotoCoreError as e:
+#         logging.error(f"An error occurred with Boto3: {e}")
+#         raise
+#     except Exception as e:
+#         logging.error(f"An error occurred while uploading the image: {e}")
+#         raise
diff --git a/app/s3_handler/s3_handler.py b/app/s3_handler/s3_handler.py
new file mode 100644
index 0000000..2cd5bb5
--- /dev/null
+++ b/app/s3_handler/s3_handler.py
@@ -0,0 +1,90 @@
+import os
+import boto3
+import logging
+from botocore.exceptions import ClientError, BotoCoreError, NoCredentialsError
+
+
+class Boto3Client:
+    def __init__(self):
+        self.s3 = boto3.client(
+            "s3",
+            aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"],
+            aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"],
+            region_name=os.environ["AWS_REGION"],
+        )
+
+    def download_from_s3(self, save_path, image_name):
+        s3_client = boto3.client("s3")
+        file_path = os.path.join(save_path, image_name)
+        try:
+            s3_client.download_file(
+                os.environ["CUTOUT_BUCKET"], f"images/{image_name}", file_path
+            )
+        except ClientError as e:
+            print("BOTO error: ", e)
+            print(
+                f"File {image_name} not found in bucket {os.environ['CUTOUT_BUCKET']}"
+            )
+            return None
+
+        return file_path
+
+    def upload_to_s3(self, file_body, folder, image_name):
+        try:
+            self.s3.put_object(
+                Body=file_body,
+                Bucket=os.environ["CUTOUT_BUCKET"],
+                Key=f"{folder}/{image_name}",
+            )
+            logging.info(f"Successfully uploaded {image_name} to {folder}")
+        except NoCredentialsError as e:
+            logging.error("No AWS credentials found")
+            raise
+        except BotoCoreError as e:
+            logging.error(f"An error occurred with Boto3: {e}")
+            raise
+        except Exception as e:
+            logging.error(f"An error occurred while uploading the image: {e}")
+            raise
+
+    def generate_presigned_urls(self, folder, expiration=3600):
+        try:
+            response = self.s3.list_objects_v2(
+                Bucket=os.environ["CUTOUT_BUCKET"], Prefix=folder
+            )
+            urls = []
+            for obj in response.get("Contents", []):
+                key = obj["Key"]
+                url = self.s3.generate_presigned_url(
+                    "get_object",
+                    Params={"Bucket": os.environ["CUTOUT_BUCKET"], "Key": key},
+                    ExpiresIn=expiration,
+                )
+                # Get object metadata
+                metadata = self.s3.head_object(
+                    Bucket=os.environ["CUTOUT_BUCKET"], Key=key
+                )["Metadata"]
+                urls.append((url, metadata))
+        except ClientError as e:
+            print(e)
+            return None
+
+        return urls
+
+    def generate_presigned_url_with_metadata(self, folder, key, expiration=3600):
+        try:
+            # Generate presigned URL
+            url = self.s3.generate_presigned_url(
+                "get_object",
+                Params={"Bucket": os.environ["CUTOUT_BUCKET"], "Key": f"{folder}/{key}"},
+                ExpiresIn=expiration,
+            )
+            # Get object metadata
+            metadata = self.s3.head_object(
+                Bucket=os.environ["CUTOUT_BUCKET"], Key=f"{folder}/{key}"
+            )["Metadata"]
+        except ClientError as e:
+            logging.error("An error occurred: %s", e)
+            return None
+
+        return url, metadata

From f4dd8470c293836e85254c177597577c05a5859f Mon Sep 17 00:00:00 2001
From: OriginalByteMe <noahrijkaard@gmail.com>
Date: Thu, 16 Nov 2023 00:17:12 +0800
Subject: [PATCH 12/14] Add fastapi and starlette to cutout generator image and
 remove unused code

---
 app/grounded_cutouts.py | 55 +++--------------------------------------
 app/s3_handler.py       | 25 ++++++++++++++++++-
 2 files changed, 27 insertions(+), 53 deletions(-)

diff --git a/app/grounded_cutouts.py b/app/grounded_cutouts.py
index f54d324..646b5b7 100644
--- a/app/grounded_cutouts.py
+++ b/app/grounded_cutouts.py
@@ -22,11 +22,11 @@
 )
 
 local_packages = Mount.from_local_python_packages(
-    "cutout", "dino", "segment", "s3_handler", "fastapi", "starlette"
+    "cutout", "dino", "segment", "s3_handler"
 )
 cutout_generator_image = (
     Image.from_registry("nvcr.io/nvidia/pytorch:22.12-py3")
-    .pip_install("segment-anything", "opencv-python", "botocore", "boto3")
+    .pip_install("segment-anything", "opencv-python","botocore", "boto3", "fastapi", "starlette")
     .run_commands(
         "apt-get update",
         "apt-get install -y git wget libgl1-mesa-glx libglib2.0-0",
@@ -58,55 +58,6 @@
 SAM_CHECKPOINT_PATH = os.path.join(HOME, "weights", "sam_vit_h_4b8939.pth")
 
 
-@stub.function(
-    mounts=[Mount.from_local_python_packages("s3_handler")],
-    secret=Secret.from_name("my-aws-secret"),
-)
-@app.post("/upload-image")
-async def upload_image_to_s3(image: UploadFile = File(...)):
-    """Upload an image to S3.
-
-    Args:
-        image (UploadFile, optional): File to upload to s3 . Defaults to File(...).
-
-    Returns:
-        str: Message indicating whether the upload was successful.
-    """
-    from s3_handler import Boto3Client
-    from botocore.exceptions import BotoCoreError, NoCredentialsError
-    
-    s3_client = Boto3Client()
-    try:
-        s3_client.upload_to_s3(image.file, "images", image.filename)
-    except NoCredentialsError as e:
-        raise HTTPException(status_code=401, detail="No AWS credentials found") from e
-    except BotoCoreError as e:
-        raise HTTPException(status_code=500, detail=str(e)) from e
-    except Exception as e:
-        raise HTTPException(status_code=500, detail="An error occurred while uploading the image") from e
-    return {"message": "Image uploaded successfully", "status_code": 200}
-
-@app.get("/generate-presigned-urls/{image_name}")
-async def generate_presigned_urls(image_name: str):
-    """Generate presigned urls for the cutouts of an image.
-
-    Args:
-        image_name (str): Name of image to generate presigned urls for.
-
-    Returns:
-        List[str]: List of presigned urls for the cutouts of an image.
-    """
-    from s3_handler import Boto3Client
-
-    s3_client = Boto3Client()
-    return s3_client.generate_presigned_urls(f"cutouts/{image_name}")
-
-@stub.function(
-    image=cutout_generator_image,
-    mounts=[local_packages],
-    gpu="T4",
-    secret=Secret.from_name("my-aws-secret"),
-)
 @app.post("/create-cutouts/{image_name}")
 async def create_cutouts(image_name: str, classes: List[str] = Body(...)):
     """Create cutouts from an image and upload them to S3.
@@ -131,7 +82,7 @@ async def create_cutouts(image_name: str, classes: List[str] = Body(...)):
     return s3.generate_presigned_urls(f"cutouts/{image_name}")
 
 @app.post("/create-cutouts")
-async def create_cutouts(image_names: List[str] = Body(...), classes: List[str] = Body(...)):
+async def create_all_cutouts(image_names: List[str] = Body(...), classes: List[str] = Body(...)):
     """Create cutouts from multiple images and upload them to S3.
 
     Args:
diff --git a/app/s3_handler.py b/app/s3_handler.py
index b27501e..2cd5bb5 100644
--- a/app/s3_handler.py
+++ b/app/s3_handler.py
@@ -3,6 +3,7 @@
 import logging
 from botocore.exceptions import ClientError, BotoCoreError, NoCredentialsError
 
+
 class Boto3Client:
     def __init__(self):
         self.s3 = boto3.client(
@@ -59,9 +60,31 @@ def generate_presigned_urls(self, folder, expiration=3600):
                     Params={"Bucket": os.environ["CUTOUT_BUCKET"], "Key": key},
                     ExpiresIn=expiration,
                 )
-                urls.append(url)
+                # Get object metadata
+                metadata = self.s3.head_object(
+                    Bucket=os.environ["CUTOUT_BUCKET"], Key=key
+                )["Metadata"]
+                urls.append((url, metadata))
         except ClientError as e:
             print(e)
             return None
 
         return urls
+
+    def generate_presigned_url_with_metadata(self, folder, key, expiration=3600):
+        try:
+            # Generate presigned URL
+            url = self.s3.generate_presigned_url(
+                "get_object",
+                Params={"Bucket": os.environ["CUTOUT_BUCKET"], "Key": f"{folder}/{key}"},
+                ExpiresIn=expiration,
+            )
+            # Get object metadata
+            metadata = self.s3.head_object(
+                Bucket=os.environ["CUTOUT_BUCKET"], Key=f"{folder}/{key}"
+            )["Metadata"]
+        except ClientError as e:
+            logging.error("An error occurred: %s", e)
+            return None
+
+        return url, metadata

From aeb6a7d53cbbc762864a16e7eb63afff0c1898bf Mon Sep 17 00:00:00 2001
From: OriginalByteMe <noahrijkaard@gmail.com>
Date: Thu, 16 Nov 2023 11:07:44 +0800
Subject: [PATCH 13/14] Fix class_id comparison in Dino detection

---
 app/dino.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/dino.py b/app/dino.py
index 69a5f8c..1e1528f 100644
--- a/app/dino.py
+++ b/app/dino.py
@@ -49,7 +49,7 @@ def predict(self, image):
             box_threshold=self.box_threshold,
             text_threshold=self.text_threshold,
         )
-        detections = detections[detections.class_id is not None]
+        detections = detections[detections.class_id != None]
         return detections
 
 

From b488edd7c6f7f69f1e0dcf92d785c24550e77ea6 Mon Sep 17 00:00:00 2001
From: OriginalByteMe <noahrijkaard@gmail.com>
Date: Thu, 16 Nov 2023 11:09:53 +0800
Subject: [PATCH 14/14] Add logging and request parsing to create_cutouts
 endpoint

---
 app/grounded_cutouts.py | 57 ++++++++++++++++++++++++++++++++++-------
 1 file changed, 48 insertions(+), 9 deletions(-)

diff --git a/app/grounded_cutouts.py b/app/grounded_cutouts.py
index 646b5b7..562425d 100644
--- a/app/grounded_cutouts.py
+++ b/app/grounded_cutouts.py
@@ -3,6 +3,28 @@
 from fastapi import FastAPI, File, UploadFile, Body, HTTPException
 from fastapi.middleware.cors import CORSMiddleware
 from typing import List
+import logging
+import json
+from starlette.requests import Request
+
+#======================
+# Logging
+#======================
+# Create a custom logger
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
+# Create handlers
+c_handler = logging.StreamHandler()
+c_handler.setLevel(logging.DEBUG)
+
+# Create formatters and add it to handlers
+c_format = logging.Formatter('%(name)s - %(levelname)s - %(message)s')
+c_handler.setFormatter(c_format)
+
+# Add handlers to the logger
+logger.addHandler(c_handler)
+
 
 app = FastAPI()
 
@@ -59,7 +81,7 @@
 
 
 @app.post("/create-cutouts/{image_name}")
-async def create_cutouts(image_name: str, classes: List[str] = Body(...)):
+async def create_cutouts(image_name: str, request: Request):
     """Create cutouts from an image and upload them to S3.
 
     Args:
@@ -69,17 +91,34 @@ async def create_cutouts(image_name: str, classes: List[str] = Body(...)):
     Returns:
         _type_: _description_
     """
+    # Parse the request body as JSON
+    data = await request.json()
+
+    # Get the classes from the JSON data
+    classes = data.get('classes', [])
     from cutout import CutoutCreator
     from s3_handler import Boto3Client
 
-    s3 = Boto3Client()
-    cutout = CutoutCreator(
-        classes=classes,
-        grounding_dino_checkpoint_path=GROUNDING_DINO_CHECKPOINT_PATH,
-        grounding_dino_config_path=GROUNDING_DINO_CONFIG_PATH,
-    )
-    cutout.create_cutouts(image_name, SAM_CHECKPOINT_PATH)
-    return s3.generate_presigned_urls(f"cutouts/{image_name}")
+    try:
+        logger.info(f"Creating cutouts for image {image_name}")
+        logger.info(f"Classes: {classes}")
+        s3 = Boto3Client()
+        cutout = CutoutCreator(
+            classes=classes,
+            grounding_dino_checkpoint_path=GROUNDING_DINO_CHECKPOINT_PATH,
+            grounding_dino_config_path=GROUNDING_DINO_CONFIG_PATH,
+        )
+        print(f"CREATING CUTOUTS FOR IMAGE {image_name}")
+        cutout.create_cutouts(image_name, SAM_CHECKPOINT_PATH)
+        logger.info(f"Cutouts created for image {image_name}")
+        urls = s3.generate_presigned_urls(f"cutouts/{image_name}")
+        logger.info(f"Presigned URLs generated for cutouts of image {image_name}")
+        return urls
+    except Exception as e:
+        logger.error(f"An error occurred while creating cutouts for image {image_name}: {e}")
+        raise
+
+    return urls
 
 @app.post("/create-cutouts")
 async def create_all_cutouts(image_names: List[str] = Body(...), classes: List[str] = Body(...)):