EXAMPLE: Optimization kitchen sink

.github/actions/deploy_infrastructure/action.yml

@@ -76,14 +76,18 @@ runs:
         command: up
         stack-name: ${{ inputs.environment }}
 
-    # Copy shared database_client.py files
-    - name: Copy Shared File
+    - name: Copy common db files
       shell: bash
       run: |
-        mkdir -p cerulean_cloud/cloud_function_ais_analysis/cerulean_cloud/
-        cp cerulean_cloud/database_client.py cerulean_cloud/cloud_function_ais_analysis/cerulean_cloud/database_client.py
-        cp cerulean_cloud/database_schema.py cerulean_cloud/cloud_function_ais_analysis/cerulean_cloud/database_schema.py
-        cp cerulean_cloud/__init__.py cerulean_cloud/cloud_function_ais_analysis/cerulean_cloud/__init__.py
+        mkdir -p cerulean_cloud/cloud_function_ais_analysis/cerulean_cloud/common
+        cp \
+          cerulean_cloud/__init__.py \
+          cerulean_cloud/cloud_function_ais_analysis/cerulean_cloud/
+        cp \
+          cerulean_cloud/common/__init__.py \
+          cerulean_cloud/common/database_client.py \
+          cerulean_cloud/common/database_schema.py \
+          cerulean_cloud/cloud_function_ais_analysis/cerulean_cloud/common/
 
     - name: Deploy Infrastructure
       uses: pulumi/actions@v5

Dockerfiles/Dockerfile.cloud_run_offset

@@ -18,14 +18,16 @@ RUN pip install \
       -r requirements.txt
 
 RUN pip install \
-       --isolated \
-       --no-cache-dir \
-       --disable-pip-version-check \
-       'cyclonedx-bom<4.0.0' \
-      && pip freeze --path ./site-packages | cyclonedx-py -r -i - --purl-bom-ref --format xml -o sbom.xml --force \
-      && rm requirements.txt
-
-COPY cerulean_cloud/ /app/cerulean_cloud/
+      --isolated \
+      --no-cache-dir \
+      --disable-pip-version-check \
+      'cyclonedx-bom<4.0.0'
+RUN pip freeze --path ./site-packages | cyclonedx-py -r -i - --purl-bom-ref --format xml -o sbom.xml --force
+RUN rm requirements.txt
+
+COPY cerulean_cloud/__init__.py /app/cerulean_cloud/
+COPY cerulean_cloud/common /app/cerulean_cloud/common
+COPY cerulean_cloud/cloud_run_offset_tiles /app/cerulean_cloud/cloud_run_offset_tiles
 
 FROM gcr.io/distroless/python3-debian11:debug-nonroot AS final
 

Dockerfiles/Dockerfile.cloud_run_orchestrator

@@ -1,7 +1,6 @@
 FROM python:3.9-slim AS build-env
 
 WORKDIR /app
-COPY cerulean_cloud/ /app/cerulean_cloud/
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
@@ -14,16 +13,20 @@ RUN pip install \
       --no-warn-script-location \
       --isolated \
       --target ./site-packages \
-      -r ./requirements.txt \
-      && pip install \
-       --isolated \
-       --no-cache-dir \
-       --disable-pip-version-check \
-       'cyclonedx-bom<4.0.0' \
-      && pip freeze --path ./site-packages | cyclonedx-py -r -i - --purl-bom-ref --format xml -o sbom.xml --force \
-      && rm requirements.txt
-
-COPY cerulean_cloud/ /app/cerulean_cloud/
+      -r ./requirements.txt
+RUN pip install \
+      --isolated \
+      --no-cache-dir \
+      --disable-pip-version-check \
+      'cyclonedx-bom<4.0.0'
+RUN pip freeze --path ./site-packages | cyclonedx-py -r -i - --purl-bom-ref --format xml -o sbom.xml --force
+RUN rm requirements.txt
+
+COPY cerulean_cloud/__init__.py /app/cerulean_cloud/
+COPY cerulean_cloud/common /app/cerulean_cloud/common
+COPY cerulean_cloud/cloud_run_orchestrator /app/cerulean_cloud/cloud_run_orchestrator
+COPY cerulean_cloud/cloud_run_offset_tiles /app/cerulean_cloud/cloud_run_offset_tiles
+COPY cerulean_cloud/cloud_function_ais_analysis /app/cerulean_cloud/cloud_function_ais_analysis
 
 FROM gcr.io/distroless/python3-debian11:debug-nonroot AS final
 

Dockerfiles/Dockerfile.cloud_run_tipg

@@ -16,16 +16,19 @@ RUN pip install \
       --no-cache-dir \
       --upgrade \
       --target ./site-packages \
-      -r ./requirements.txt \
-      && pip install \
-       --isolated \
-       --no-cache-dir \
-       --disable-pip-version-check \
-       'cyclonedx-bom<4.0.0' \
-      && pip freeze --path ./site-packages | cyclonedx-py -r -i - --purl-bom-ref --format xml -o sbom.xml --force \
-      && rm requirements.txt
+      -r ./requirements.txt
 
-COPY cerulean_cloud/ /app/cerulean_cloud/
+RUN pip install \
+      --isolated \
+      --no-cache-dir \
+      --disable-pip-version-check \
+      'cyclonedx-bom<4.0.0'
+RUN pip freeze --path ./site-packages | cyclonedx-py -r -i - --purl-bom-ref --format xml -o sbom.xml --force
+RUN rm requirements.txt
+
+COPY cerulean_cloud/__init__.py /app/cerulean_cloud/
+COPY cerulean_cloud/common /app/cerulean_cloud/common
+COPY cerulean_cloud/cloud_run_tipg /app/cerulean_cloud/cloud_run_tipg
 
 FROM gcr.io/distroless/python3-debian11:debug-nonroot AS final
 

Dockerfiles/Dockerfile.titiler

@@ -5,8 +5,8 @@ WORKDIR /var/task
 
 COPY cerulean_cloud/titiler_sentinel/requirements.txt ./requirements.txt
 COPY cerulean_cloud/titiler_sentinel/handler.py handler.py
-COPY cerulean_cloud/auth.py auth.py
 COPY cerulean_cloud/titiler_sentinel/templates/ ./templates
+COPY cerulean_cloud/common/auth.py auth.py
 
 # Use after upgrade of Pulumi Docker to >= 4
 # COPY --chown=root:root --chmod=400 Dockerfiles/scripts/titiler-entrypoint.sh /entrypoint.sh

alembic/versions/5e03ce584f3c_add_eez.py

@@ -12,7 +12,7 @@
 from shapely.geometry import MultiPolygon, shape
 from sqlalchemy import orm
 
-import cerulean_cloud.database_schema as database_schema
+import cerulean_cloud.common.database_schema as database_schema
 from alembic import op
 
 # revision identifiers, used by Alembic.

alembic/versions/c0bd1215a3ca_add_iho.py

@@ -12,7 +12,7 @@
 from shapely.geometry import MultiPolygon, shape
 from sqlalchemy import orm
 
-import cerulean_cloud.database_schema as database_schema
+import cerulean_cloud.common.database_schema as database_schema
 from alembic import op
 
 # revision identifiers, used by Alembic.

alembic/versions/c941681a050d_add_initial_records.py

@@ -10,7 +10,7 @@
 
 from sqlalchemy import orm
 
-import cerulean_cloud.database_schema as database_schema
+import cerulean_cloud.common.database_schema as database_schema
 from alembic import op
 
 # revision identifiers, used by Alembic.

alembic/versions/f9b7166c86b7_add_mpa.py

@@ -12,7 +12,7 @@
 from shapely.geometry import MultiPolygon, shape
 from sqlalchemy import orm
 
-import cerulean_cloud.database_schema as database_schema
+import cerulean_cloud.common.database_schema as database_schema
 from alembic import op
 
 # revision identifiers, used by Alembic.

cerulean_cloud/cloud_function_ais_analysis/main.py

@@ -11,7 +11,7 @@
 from shapely import wkb
 from utils.analyzer import ASA_MAPPING
 
-from cerulean_cloud.database_client import DatabaseClient, get_engine
+from cerulean_cloud.common.database_client import DatabaseClient, get_engine
 
 
 def verify_api_key(request):

cerulean_cloud/cloud_run_offset_tiles/handler.py

@@ -9,12 +9,12 @@
 from fastapi_utils.timing import add_timing_middleware, record_timing
 from starlette.requests import Request
 
-from cerulean_cloud.auth import api_key_auth
 from cerulean_cloud.cloud_run_offset_tiles.schema import (
     InferenceResultStack,
     PredictPayload,
 )
-from cerulean_cloud.models import get_model
+from cerulean_cloud.common.auth import api_key_auth
+from cerulean_cloud.common.models import get_model
 
 # mypy: ignore-errors
 

cerulean_cloud/cloud_run_offset_tiles/requirements.txt

@@ -3,8 +3,9 @@ starlette-cramjam==0.1.0
 uvicorn[standard]
 rasterio==1.3.0
 setuptools==59.5.0
-torch==1.11.0
-torchvision==0.12.0
+-f https://download.pytorch.org/whl/cpu/torch_stable.html
+torch==1.11.0+cpu
+torchvision==0.12.0+cpu
 numpy<2.0.0
 typing-inspect
 fastapi-utils

cerulean_cloud/cloud_run_orchestrator/handler.py

@@ -23,18 +23,18 @@
 from global_land_mask import globe
 from shapely.geometry import shape
 
-from cerulean_cloud.auth import api_key_auth
 from cerulean_cloud.cloud_function_ais_analysis.queuer import add_to_asa_queue
 from cerulean_cloud.cloud_run_orchestrator.clients import CloudRunInferenceClient
 from cerulean_cloud.cloud_run_orchestrator.schema import (
     OrchestratorInput,
     OrchestratorResult,
 )
-from cerulean_cloud.database_client import DatabaseClient, get_engine
-from cerulean_cloud.models import get_model
-from cerulean_cloud.roda_sentinelhub_client import RodaSentinelHubClient
-from cerulean_cloud.tiling import TMS, offset_bounds_from_base_tiles
-from cerulean_cloud.titiler_client import TitilerClient
+from cerulean_cloud.common.auth import api_key_auth
+from cerulean_cloud.common.database_client import DatabaseClient, get_engine
+from cerulean_cloud.common.models import get_model
+from cerulean_cloud.common.roda_sentinelhub_client import RodaSentinelHubClient
+from cerulean_cloud.common.tiling import TMS, offset_bounds_from_base_tiles
+from cerulean_cloud.common.titiler_client import TitilerClient
 
 app = FastAPI(title="Cloud Run orchestrator", dependencies=[Depends(api_key_auth)])
 # Allow CORS for local debugging

cerulean_cloud/cloud_run_orchestrator/requirements.txt

@@ -24,8 +24,9 @@ pygeos
 networkx
 google-cloud-tasks
 protobuf
-torch==1.11.0
-torchvision==0.12.0
+-f https://download.pytorch.org/whl/cpu/torch_stable.html
+torch==1.11.0+cpu
+torchvision==0.12.0+cpu
 scipy
 scikit-image
 pydantic<2.0

cerulean_cloud/common/__init__.py

@@ -0,0 +1 @@
+"""Common modules for Cerulean Cloud."""

cerulean_cloud/database_client.py → cerulean_cloud/common/database_client.py

@@ -10,7 +10,7 @@
 from sqlalchemy import and_, or_, select, update
 from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
 
-import cerulean_cloud.database_schema as db
+import cerulean_cloud.common.database_schema as db
 
 
 class InstanceNotFoundError(Exception):

cerulean_cloud/roda_sentinelhub_client.py → cerulean_cloud/common/roda_sentinelhub_client.py

@@ -1,4 +1,5 @@
 """client code to interact with roda http proxy to s3 s1 bucket"""
+
 import posixpath
 import urllib.parse as urlib
 from typing import Dict

cerulean_cloud/titiler_client.py → cerulean_cloud/common/titiler_client.py

@@ -10,7 +10,7 @@
 from rasterio.io import MemoryFile
 from rasterio.plot import reshape_as_image
 
-from cerulean_cloud.tiling import TMS
+from cerulean_cloud.common.tiling import TMS
 
 TMS_TITLE = TMS.identifier
 

cerulean_cloud/titiler_sentinel/handler.py

@@ -28,7 +28,6 @@
 import os
 from typing import Dict
 
-from auth import api_key_auth
 from fastapi import Depends, FastAPI, Query
 from mangum import Mangum
 from rio_tiler_pds.errors import InvalidSentinelSceneId
@@ -42,6 +41,8 @@
 from titiler.core.errors import DEFAULT_STATUS_CODES, add_exception_handlers
 from titiler.core.factory import MultiBandTilerFactory
 
+from cerulean_cloud.common.auth import api_key_auth
+
 app = FastAPI(title="Sentinel-1 API")
 add_exception_handlers(app, DEFAULT_STATUS_CODES)
 add_exception_handlers(

images/cloud_run_images.py

@@ -65,6 +65,9 @@ def construct_name(resource_name: str) -> str:
     build=docker.DockerBuildArgs(
         context="../",
         dockerfile="../Dockerfiles/Dockerfile.cloud_run_offset",
+        cache_from=docker.CacheFromArgs(
+            images=[cloud_run_offset_tile_image_url.apply(lambda url: f"{url}:latest")]
+        ),
         target="final",
     ),
     image_name=cloud_run_offset_tile_image_url,
@@ -75,6 +78,9 @@ def construct_name(resource_name: str) -> str:
     build=docker.DockerBuildArgs(
         context="../",
         dockerfile="../Dockerfiles/Dockerfile.cloud_run_orchestrator",
+        cache_from=docker.CacheFromArgs(
+            images=[cloud_run_orchestrator_image_url.apply(lambda url: f"{url}:latest")]
+        ),
         target="final",
     ),
     image_name=cloud_run_orchestrator_image_url,
@@ -85,6 +91,9 @@ def construct_name(resource_name: str) -> str:
     build=docker.DockerBuildArgs(
         context="../",
         dockerfile="../Dockerfiles/Dockerfile.cloud_run_tipg",
+        cache_from=docker.CacheFromArgs(
+            images=[cloud_run_tipg_image_url.apply(lambda url: f"{url}:latest")]
+        ),
         target="final",
     ),
     image_name=cloud_run_tipg_image_url,

stack/cloud_function_ais_analysis.py

@@ -1,7 +1,5 @@
 """cloud function to find slick culprits from AIS tracks"""
 
-import time
-
 import database
 import git
 import pulumi
@@ -58,7 +56,7 @@
 # source code. ("main.py" and "requirements.txt".)
 source_archive_object = storage.BucketObject(
     construct_name("source-cf-ais"),
-    name=f"handler.py-{time.time():f}",
+    name="handler.py",
     bucket=bucket.name,
     source=archive,
 )

stack/cloud_function_historical_run.py

@@ -1,7 +1,5 @@
 """cloud function to select appropriate scenes (over water and IW) from SNS notification"""
 
-import time
-
 import cloud_function_scene_relevancy
 import cloud_run_orchestrator
 import database
@@ -37,7 +35,7 @@
 # source code. ("main.py" and "requirements.txt".)
 source_archive_object = storage.BucketObject(
     construct_name("source-cf-historical-run"),
-    name=f"handler.py-{time.time():f}",
+    name="handler.py",
     bucket=cloud_function_scene_relevancy.bucket.name,
     source=archive,
 )

stack/cloud_function_scene_relevancy.py

@@ -1,7 +1,5 @@
 """cloud function to select appropriate scenes (over water and IW) from SNS notification"""
 
-import time
-
 import cloud_run_orchestrator
 import database
 import pulumi
@@ -60,7 +58,7 @@
 # source code. ("main.py" and "requirements.txt".)
 source_archive_object = storage.BucketObject(
     construct_name("source-cf-sr"),
-    name=f"handler.py-{time.time():f}",
+    name="handler.py",
     bucket=bucket.name,
     source=archive,
 )

stack/utils.py

@@ -126,9 +126,7 @@ def create_zip(
     :param compression: The compression type to use for the zip file (default is ZIP_DEFLATED)
     """
     if not zip_filepath:
-        _, zip_filepath = mkstemp(
-            suffix=".zip",
-        )
+        _, zip_filepath = mkstemp(suffix=".zip")
     else:
         zip_filepath = os.path.abspath(zip_filepath)
 
@@ -142,7 +140,21 @@ def create_zip(
                 ):
                     # Store the file relative to the directory specified
                     archive_name = os.path.relpath(full_path, dir_to_zip)
-                    zipf.write(full_path, archive_name)
+
+                    # Create ZipInfo object to set the date_time manually
+                    zinfo = zipfile.ZipInfo.from_file(full_path, archive_name)
+                    # Set date_time to January 1, 1980
+                    zinfo.date_time = (1980, 1, 1, 0, 0, 0)
+
+                    # Write file data to zip in chunks to be memory efficient
+                    with open(full_path, "rb") as f:
+                        with zipf.open(zinfo, "w") as zf:
+                            while True:
+                                chunk = f.read(1024 * 1024)  # Read 1MB at a time
+                                if not chunk:
+                                    break
+                                zf.write(chunk)
+
     return zip_filepath