Optimize docker image contents

.github/actions/deploy_infrastructure/action.yml

@@ -76,14 +76,18 @@ runs:
         command: up
         stack-name: ${{ inputs.environment }}
 
-    # Copy shared database_client.py files
-    - name: Copy Shared File
+    - name: Copy common db files
       shell: bash
       run: |
-        mkdir -p cerulean_cloud/cloud_function_ais_analysis/cerulean_cloud/
-        cp cerulean_cloud/database_client.py cerulean_cloud/cloud_function_ais_analysis/cerulean_cloud/database_client.py
-        cp cerulean_cloud/database_schema.py cerulean_cloud/cloud_function_ais_analysis/cerulean_cloud/database_schema.py
-        cp cerulean_cloud/__init__.py cerulean_cloud/cloud_function_ais_analysis/cerulean_cloud/__init__.py
+        mkdir -p cerulean_cloud/cloud_function_ais_analysis/cerulean_cloud/common
+        cp \
+          cerulean_cloud/__init__.py \
+          cerulean_cloud/cloud_function_ais_analysis/cerulean_cloud/
+        cp \
+          cerulean_cloud/common/__init__.py \
+          cerulean_cloud/common/database_client.py \
+          cerulean_cloud/common/database_schema.py \
+          cerulean_cloud/cloud_function_ais_analysis/cerulean_cloud/common/
 
     - name: Deploy Infrastructure
       uses: pulumi/actions@v5

Dockerfiles/Dockerfile.cloud_run_offset

@@ -18,14 +18,16 @@ RUN pip install \
       -r requirements.txt
 
 RUN pip install \
-       --isolated \
-       --no-cache-dir \
-       --disable-pip-version-check \
-       'cyclonedx-bom<4.0.0' \
-      && pip freeze --path ./site-packages | cyclonedx-py -r -i - --purl-bom-ref --format xml -o sbom.xml --force \
-      && rm requirements.txt
-
-COPY cerulean_cloud/ /app/cerulean_cloud/
+      --isolated \
+      --no-cache-dir \
+      --disable-pip-version-check \
+      'cyclonedx-bom<4.0.0'
+RUN pip freeze --path ./site-packages | cyclonedx-py -r -i - --purl-bom-ref --format xml -o sbom.xml --force
+RUN rm requirements.txt
+
+COPY cerulean_cloud/__init__.py /app/cerulean_cloud/
+COPY cerulean_cloud/common /app/cerulean_cloud/common
+COPY cerulean_cloud/cloud_run_offset_tiles /app/cerulean_cloud/cloud_run_offset_tiles
 
 FROM gcr.io/distroless/python3-debian11:debug-nonroot AS final
 

Dockerfiles/Dockerfile.cloud_run_orchestrator

@@ -1,7 +1,6 @@
 FROM python:3.9-slim AS build-env
 
 WORKDIR /app
-COPY cerulean_cloud/ /app/cerulean_cloud/
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
@@ -14,16 +13,20 @@ RUN pip install \
       --no-warn-script-location \
       --isolated \
       --target ./site-packages \
-      -r ./requirements.txt \
-      && pip install \
-       --isolated \
-       --no-cache-dir \
-       --disable-pip-version-check \
-       'cyclonedx-bom<4.0.0' \
-      && pip freeze --path ./site-packages | cyclonedx-py -r -i - --purl-bom-ref --format xml -o sbom.xml --force \
-      && rm requirements.txt
-
-COPY cerulean_cloud/ /app/cerulean_cloud/
+      -r ./requirements.txt
+RUN pip install \
+      --isolated \
+      --no-cache-dir \
+      --disable-pip-version-check \
+      'cyclonedx-bom<4.0.0'
+RUN pip freeze --path ./site-packages | cyclonedx-py -r -i - --purl-bom-ref --format xml -o sbom.xml --force
+RUN rm requirements.txt
+
+COPY cerulean_cloud/__init__.py /app/cerulean_cloud/
+COPY cerulean_cloud/common /app/cerulean_cloud/common
+COPY cerulean_cloud/cloud_run_orchestrator /app/cerulean_cloud/cloud_run_orchestrator
+COPY cerulean_cloud/cloud_run_offset_tiles /app/cerulean_cloud/cloud_run_offset_tiles
+COPY cerulean_cloud/cloud_function_ais_analysis /app/cerulean_cloud/cloud_function_ais_analysis
 
 FROM gcr.io/distroless/python3-debian11:debug-nonroot AS final
 

Dockerfiles/Dockerfile.cloud_run_tipg

@@ -16,16 +16,19 @@ RUN pip install \
       --no-cache-dir \
       --upgrade \
       --target ./site-packages \
-      -r ./requirements.txt \
-      && pip install \
-       --isolated \
-       --no-cache-dir \
-       --disable-pip-version-check \
-       'cyclonedx-bom<4.0.0' \
-      && pip freeze --path ./site-packages | cyclonedx-py -r -i - --purl-bom-ref --format xml -o sbom.xml --force \
-      && rm requirements.txt
+      -r ./requirements.txt
 
-COPY cerulean_cloud/ /app/cerulean_cloud/
+RUN pip install \
+      --isolated \
+      --no-cache-dir \
+      --disable-pip-version-check \
+      'cyclonedx-bom<4.0.0'
+RUN pip freeze --path ./site-packages | cyclonedx-py -r -i - --purl-bom-ref --format xml -o sbom.xml --force
+RUN rm requirements.txt
+
+COPY cerulean_cloud/__init__.py /app/cerulean_cloud/
+COPY cerulean_cloud/common /app/cerulean_cloud/common
+COPY cerulean_cloud/cloud_run_tipg /app/cerulean_cloud/cloud_run_tipg
 
 FROM gcr.io/distroless/python3-debian11:debug-nonroot AS final
 

Dockerfiles/Dockerfile.titiler

@@ -5,8 +5,8 @@ WORKDIR /var/task
 
 COPY cerulean_cloud/titiler_sentinel/requirements.txt ./requirements.txt
 COPY cerulean_cloud/titiler_sentinel/handler.py handler.py
-COPY cerulean_cloud/auth.py auth.py
 COPY cerulean_cloud/titiler_sentinel/templates/ ./templates
+COPY cerulean_cloud/common/auth.py auth.py
 
 # Use after upgrade of Pulumi Docker to >= 4
 # COPY --chown=root:root --chmod=400 Dockerfiles/scripts/titiler-entrypoint.sh /entrypoint.sh

alembic/versions/5e03ce584f3c_add_eez.py

@@ -12,7 +12,7 @@
 from shapely.geometry import MultiPolygon, shape
 from sqlalchemy import orm
 
-import cerulean_cloud.database_schema as database_schema
+import cerulean_cloud.common.database_schema as database_schema
 from alembic import op
 
 # revision identifiers, used by Alembic.

alembic/versions/c0bd1215a3ca_add_iho.py

@@ -12,7 +12,7 @@
 from shapely.geometry import MultiPolygon, shape
 from sqlalchemy import orm
 
-import cerulean_cloud.database_schema as database_schema
+import cerulean_cloud.common.database_schema as database_schema
 from alembic import op
 
 # revision identifiers, used by Alembic.

alembic/versions/c941681a050d_add_initial_records.py

@@ -10,7 +10,7 @@
 
 from sqlalchemy import orm
 
-import cerulean_cloud.database_schema as database_schema
+import cerulean_cloud.common.database_schema as database_schema
 from alembic import op
 
 # revision identifiers, used by Alembic.

alembic/versions/f9b7166c86b7_add_mpa.py

@@ -12,7 +12,7 @@
 from shapely.geometry import MultiPolygon, shape
 from sqlalchemy import orm
 
-import cerulean_cloud.database_schema as database_schema
+import cerulean_cloud.common.database_schema as database_schema
 from alembic import op
 
 # revision identifiers, used by Alembic.

cerulean_cloud/cloud_function_ais_analysis/main.py

@@ -11,7 +11,7 @@
 from shapely import wkb
 from utils.analyzer import ASA_MAPPING
 
-from cerulean_cloud.database_client import DatabaseClient, get_engine
+from cerulean_cloud.common.database_client import DatabaseClient, get_engine
 
 
 def verify_api_key(request):

cerulean_cloud/cloud_run_offset_tiles/handler.py

@@ -9,12 +9,12 @@
 from fastapi_utils.timing import add_timing_middleware, record_timing
 from starlette.requests import Request
 
-from cerulean_cloud.auth import api_key_auth
 from cerulean_cloud.cloud_run_offset_tiles.schema import (
     InferenceResultStack,
     PredictPayload,
 )
-from cerulean_cloud.models import get_model
+from cerulean_cloud.common.auth import api_key_auth
+from cerulean_cloud.common.models import get_model
 
 # mypy: ignore-errors
 

cerulean_cloud/cloud_run_orchestrator/handler.py

@@ -23,18 +23,18 @@
 from global_land_mask import globe
 from shapely.geometry import shape
 
-from cerulean_cloud.auth import api_key_auth
 from cerulean_cloud.cloud_function_ais_analysis.queuer import add_to_asa_queue
 from cerulean_cloud.cloud_run_orchestrator.clients import CloudRunInferenceClient
 from cerulean_cloud.cloud_run_orchestrator.schema import (
     OrchestratorInput,
     OrchestratorResult,
 )
-from cerulean_cloud.database_client import DatabaseClient, get_engine
-from cerulean_cloud.models import get_model
-from cerulean_cloud.roda_sentinelhub_client import RodaSentinelHubClient
-from cerulean_cloud.tiling import TMS, offset_bounds_from_base_tiles
-from cerulean_cloud.titiler_client import TitilerClient
+from cerulean_cloud.common.auth import api_key_auth
+from cerulean_cloud.common.database_client import DatabaseClient, get_engine
+from cerulean_cloud.common.models import get_model
+from cerulean_cloud.common.roda_sentinelhub_client import RodaSentinelHubClient
+from cerulean_cloud.common.tiling import TMS, offset_bounds_from_base_tiles
+from cerulean_cloud.common.titiler_client import TitilerClient
 
 app = FastAPI(title="Cloud Run orchestrator", dependencies=[Depends(api_key_auth)])
 # Allow CORS for local debugging

cerulean_cloud/common/__init__.py

@@ -0,0 +1 @@
+"""Common modules for Cerulean Cloud."""

cerulean_cloud/database_client.py → cerulean_cloud/common/database_client.py

@@ -10,7 +10,7 @@
 from sqlalchemy import and_, or_, select, update
 from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
 
-import cerulean_cloud.database_schema as db
+import cerulean_cloud.common.database_schema as db
 
 
 class InstanceNotFoundError(Exception):

cerulean_cloud/roda_sentinelhub_client.py → cerulean_cloud/common/roda_sentinelhub_client.py

@@ -1,4 +1,5 @@
 """client code to interact with roda http proxy to s3 s1 bucket"""
+
 import posixpath
 import urllib.parse as urlib
 from typing import Dict

cerulean_cloud/titiler_client.py → cerulean_cloud/common/titiler_client.py

@@ -10,7 +10,7 @@
 from rasterio.io import MemoryFile
 from rasterio.plot import reshape_as_image
 
-from cerulean_cloud.tiling import TMS
+from cerulean_cloud.common.tiling import TMS
 
 TMS_TITLE = TMS.identifier
 

cerulean_cloud/titiler_sentinel/handler.py

@@ -28,7 +28,6 @@
 import os
 from typing import Dict
 
-from auth import api_key_auth
 from fastapi import Depends, FastAPI, Query
 from mangum import Mangum
 from rio_tiler_pds.errors import InvalidSentinelSceneId
@@ -42,6 +41,8 @@
 from titiler.core.errors import DEFAULT_STATUS_CODES, add_exception_handlers
 from titiler.core.factory import MultiBandTilerFactory
 
+from cerulean_cloud.common.auth import api_key_auth
+
 app = FastAPI(title="Sentinel-1 API")
 add_exception_handlers(app, DEFAULT_STATUS_CODES)
 add_exception_handlers(

test/test_cerulean_cloud/test_cloud_run_offset_tiles.py

@@ -7,10 +7,10 @@
 import torchvision  # noqa necessary for torch.jit.load of icevision mrcnn model
 from rasterio.plot import reshape_as_raster
 
-import cerulean_cloud.models as models
+import cerulean_cloud.common.models as models
 from cerulean_cloud.cloud_run_offset_tiles.schema import InferenceInput, PredictPayload
-from cerulean_cloud.tiling import TMS
-from cerulean_cloud.titiler_client import TitilerClient
+from cerulean_cloud.common.tiling import TMS
+from cerulean_cloud.common.titiler_client import TitilerClient
 
 
 @pytest.mark.skip

test/test_cerulean_cloud/test_cloud_run_orchestrator.py

@@ -28,10 +28,10 @@
     offset_group_shape_from_base_tiles,
 )
 from cerulean_cloud.cloud_run_orchestrator.schema import OrchestratorInput
-from cerulean_cloud.models import BaseModel, b64_image_to_array
-from cerulean_cloud.roda_sentinelhub_client import RodaSentinelHubClient
-from cerulean_cloud.tiling import TMS, offset_bounds_from_base_tiles
-from cerulean_cloud.titiler_client import TitilerClient
+from cerulean_cloud.common.models import BaseModel, b64_image_to_array
+from cerulean_cloud.common.roda_sentinelhub_client import RodaSentinelHubClient
+from cerulean_cloud.common.tiling import TMS, offset_bounds_from_base_tiles
+from cerulean_cloud.common.titiler_client import TitilerClient
 
 S1_ID = "S1A_IW_GRDH_1SDV_20200729T034859_20200729T034924_033664_03E6D3_93EF"
 
@@ -45,9 +45,7 @@ def get_base_tile_3band(*args, **kwargs):
 
 
 @pytest.mark.skip
-@patch.object(
-    cerulean_cloud.titiler_client.TitilerClient, "get_base_tile", get_base_tile_3band
-)
+@patch.object(TitilerClient, "get_base_tile", get_base_tile_3band)
 def test_create_fixture_inference(
     url="https://0xshe4bmk8.execute-api.eu-central-1.amazonaws.com/",
     inference_url="https://cerulean-cloud-staging-cr-offset-tiles-49b-5qkjkyomta-ew.a.run.app",
@@ -143,14 +141,10 @@ async def mock_get_product_info(*args, **kwargs):
 
 
 @pytest.mark.skip
+@patch.object(TitilerClient, "get_statistics", mock_get_statistics)
+@patch.object(TitilerClient, "get_bounds", mock_get_bounds)
 @patch.object(
-    cerulean_cloud.titiler_client.TitilerClient, "get_statistics", mock_get_statistics
-)
-@patch.object(
-    cerulean_cloud.titiler_client.TitilerClient, "get_bounds", mock_get_bounds
-)
-@patch.object(
-    cerulean_cloud.roda_sentinelhub_client.RodaSentinelHubClient,
+    RodaSentinelHubClient,
     "get_product_info",
     mock_get_product_info,
 )

test/test_cerulean_cloud/test_database_client.py

@@ -11,9 +11,9 @@
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import Session, sessionmaker
 
-import cerulean_cloud.database_schema as database_schema
-from cerulean_cloud.database_client import DatabaseClient, get_engine
-from cerulean_cloud.titiler_client import TitilerClient
+from cerulean_cloud.common import database_schema
+from cerulean_cloud.common.database_client import DatabaseClient, get_engine
+from cerulean_cloud.common.titiler_client import TitilerClient
 
 
 def test_get_engine(postgresql):

test/test_cerulean_cloud/test_inference_client.py

@@ -7,7 +7,6 @@
 import rasterio
 from rasterio.plot import reshape_as_image
 
-import cerulean_cloud.titiler_client
 from cerulean_cloud.cloud_run_offset_tiles.schema import (
     InferenceResult,
     InferenceResultStack,
@@ -19,8 +18,8 @@
     get_ship_density,
     handle_aux_datasets,
 )
-from cerulean_cloud.tiling import TMS
-from cerulean_cloud.titiler_client import TitilerClient
+from cerulean_cloud.common.tiling import TMS
+from cerulean_cloud.common.titiler_client import TitilerClient
 
 
 def get_mock_layer(short_name, source_url=""):
@@ -77,9 +76,7 @@ def fixture_cloud_inference_tile(httpx_mock):
     )
 
 
-@patch.object(
-    cerulean_cloud.titiler_client.TitilerClient, "get_base_tile", mock_get_base_tile
-)
+@patch.object(TitilerClient, "get_base_tile", mock_get_base_tile)
 @pytest.mark.skip
 @pytest.mark.asyncio
 async def test_get_tile_inference(fixture_cloud_inference_tile, httpx_mock):

test/test_cerulean_cloud/test_roda_sentinelhub_client.py

@@ -2,7 +2,7 @@
 
 import pytest
 
-from cerulean_cloud.roda_sentinelhub_client import RodaSentinelHubClient
+from cerulean_cloud.common.roda_sentinelhub_client import RodaSentinelHubClient
 
 S1_IDS = [
     "S1A_IW_GRDH_1SDV_20200729T034859_20200729T034924_033664_03E6D3_93EF",

test/test_cerulean_cloud/test_tiling.py

@@ -4,7 +4,7 @@
 import shapely.geometry
 import supermercado
 
-from cerulean_cloud.tiling import TMS, offset_bounds_from_base_tiles
+from cerulean_cloud.common.tiling import TMS, offset_bounds_from_base_tiles
 
 
 @pytest.fixture

test/test_cerulean_cloud/test_titiler_client.py

@@ -1,8 +1,8 @@
 import mercantile
 import pytest
 
-from cerulean_cloud.tiling import adjacent_tile, pixel_to_location
-from cerulean_cloud.titiler_client import TMS_TITLE, TitilerClient
+from cerulean_cloud.common.tiling import adjacent_tile, pixel_to_location
+from cerulean_cloud.common.titiler_client import TMS_TITLE, TitilerClient
 
 
 @pytest.fixture