From 16ca6514e2c5e3c869b260c65e0b27444deb9cd4 Mon Sep 17 00:00:00 2001 From: Jona Date: Wed, 1 Nov 2023 22:29:13 -0400 Subject: [PATCH 1/9] Accept large numbers of requests in Scene Relevancy --- cerulean_cloud/cloud_function_scene_relevancy/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cerulean_cloud/cloud_function_scene_relevancy/main.py b/cerulean_cloud/cloud_function_scene_relevancy/main.py index e6e88fda..f90d7916 100644 --- a/cerulean_cloud/cloud_function_scene_relevancy/main.py +++ b/cerulean_cloud/cloud_function_scene_relevancy/main.py @@ -107,7 +107,7 @@ def handle_notification(request_json, ocean_poly): """handle notification""" filtered_scenes = [] for r in request_json.get("Records"): - sns = request_json["Records"][0]["Sns"] + sns = r["Sns"] msg = json.loads(sns["Message"]) scene_poly = sh.polygon.Polygon(msg["footprint"]["coordinates"][0][0]) From c6227d721b5c650557ab017f1e4590f1186ed8c4 Mon Sep 17 00:00:00 2001 From: Jona Date: Wed, 1 Nov 2023 23:48:41 -0400 Subject: [PATCH 2/9] Add test example --- notebooks/run_sr_on_scene_ids.ipynb | 37 +++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 notebooks/run_sr_on_scene_ids.ipynb diff --git a/notebooks/run_sr_on_scene_ids.ipynb b/notebooks/run_sr_on_scene_ids.ipynb new file mode 100644 index 00000000..310a9ad0 --- /dev/null +++ b/notebooks/run_sr_on_scene_ids.ipynb @@ -0,0 +1,37 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import httpx\n", + "import json\n", + "def stage(scene_ids, API_KEY):\n", + " URL = \"https://europe-west1-cerulean-338116.cloudfunctions.net/cerulean-cloud-staging-cloud-function-sr\"\n", + " payload = {\n", + " \"Records\": [{\n", + " \"Sns\": {\n", + " \"Message\": json.dumps({\n", + " \"id\": s,\n", + " \"footprint\": {\"coordinates\": [[[[0, 0], [0, 1], [1, 1], [1, 0], [0, 0]]]]}})}}\n", + " for s in scene_ids]}\n", + "\n", + " orchestrator_result = httpx.post(\n", + " URL,\n", + " json=payload,\n", + " timeout=100,\n", + " headers={\"Authorization\": f\"Bearer {API_KEY}\", \"Content-Type\": \"application/json\"})\n", + " print(orchestrator_result)" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 18d5795915fec01a293f9f632d74363ba4b453f0 Mon Sep 17 00:00:00 2001 From: Jona Date: Fri, 3 Nov 2023 23:53:41 -0400 Subject: [PATCH 3/9] Change flow parameters: max instance count, max concurrency, max dispatch, max connections Load model in ONCE per container, make accessible to multiple concurrent requests Attempt some cleanup to avoid memory leak in orchestrator --- .../cloud_run_offset_tiles/handler.py | 26 +- .../cloud_run_orchestrator/clients.py | 87 +++--- .../cloud_run_orchestrator/handler.py | 43 ++- .../cloud_run_orchestrator/merging.py | 8 + notebooks/run_sr_on_scene_ids.ipynb | 256 ++++++++++++++++++ stack/cloud_function_ais_analysis.py | 2 +- stack/cloud_function_scene_relevancy.py | 2 +- stack/cloud_run_offset_tile.py | 3 + stack/cloud_run_orchestrator.py | 2 + stack/database.py | 2 +- stack/utils.py | 5 +- 11 files changed, 345 insertions(+), 91 deletions(-) diff --git a/cerulean_cloud/cloud_run_offset_tiles/handler.py b/cerulean_cloud/cloud_run_offset_tiles/handler.py index 16cec7ff..fcd50d9b 100644 --- a/cerulean_cloud/cloud_run_offset_tiles/handler.py +++ b/cerulean_cloud/cloud_run_offset_tiles/handler.py @@ -2,7 +2,6 @@ Ref: https://github.com/python-engineer/ml-deployment/tree/main/google-cloud-run """ from base64 import b64decode, b64encode -from functools import lru_cache from typing import Dict, List, Tuple, Union import geojson @@ -33,17 +32,17 @@ app.add_middleware(CORSMiddleware, allow_origins=["*"]) add_timing_middleware(app, prefix="app") +MODEL = None -def load_tracing_model(savepath): - """load tracing model. a tracing model must be applied to the same batch dimensions the model was trained on.""" - tracing_model = torch.jit.load(savepath, map_location="cpu") - return tracing_model - -@lru_cache() -def get_model(): - """load model""" - return load_tracing_model("cerulean_cloud/cloud_run_offset_tiles/model/model.pt") +def load_model(): + """Load the model into the global variable.""" + global MODEL + if MODEL is None: + # You should specify the correct path to your model file + model_path = "cerulean_cloud/cloud_run_offset_tiles/model/model.pt" + MODEL = torch.jit.load(model_path, map_location="cpu") + return MODEL def logits_to_classes(out_batch_logits): @@ -164,11 +163,10 @@ def _predict( tags=["Run inference"], response_model=InferenceResultStack, ) -def predict( - request: Request, payload: PredictPayload, model=Depends(get_model) -) -> Dict: - """predict""" +def predict(request: Request, payload: PredictPayload) -> Dict: + """Run prediction using the loaded model.""" record_timing(request, note="Started") + model = load_model() results = _predict(payload.inf_stack, model, payload.inf_parms) record_timing(request, note="Finished inference") diff --git a/cerulean_cloud/cloud_run_orchestrator/clients.py b/cerulean_cloud/cloud_run_orchestrator/clients.py index cbd8ece1..9e087fda 100644 --- a/cerulean_cloud/cloud_run_orchestrator/clients.py +++ b/cerulean_cloud/cloud_run_orchestrator/clients.py @@ -1,5 +1,4 @@ """Clients for other cloud run functions""" -import asyncio import json import os import zipfile @@ -70,76 +69,70 @@ def __init__( self.inference_parms = inference_parms async def get_base_tile_inference( - self, tile: morecantile.Tile, semaphore: asyncio.Semaphore, rescale=(0, 255) + self, tile: morecantile.Tile, rescale=(0, 255) ) -> InferenceResultStack: """fetch inference for base tiles""" - async with semaphore: - img_array = await self.titiler_client.get_base_tile( - sceneid=self.sceneid, tile=tile, scale=self.scale, rescale=rescale - ) + img_array = await self.titiler_client.get_base_tile( + sceneid=self.sceneid, tile=tile, scale=self.scale, rescale=rescale + ) - img_array = reshape_as_raster(img_array) + img_array = reshape_as_raster(img_array) - bounds = list(TMS.bounds(tile)) + bounds = list(TMS.bounds(tile)) - with self.aux_datasets.open() as src: - window = rasterio.windows.from_bounds(*bounds, transform=src.transform) - height, width = img_array.shape[1:] - aux_ds = src.read(window=window, out_shape=(height, width)) + with self.aux_datasets.open() as src: + window = rasterio.windows.from_bounds(*bounds, transform=src.transform) + height, width = img_array.shape[1:] + aux_ds = src.read(window=window, out_shape=(height, width)) - img_array = np.concatenate([img_array[0:1, :, :], aux_ds], axis=0) + img_array = np.concatenate([img_array[0:1, :, :], aux_ds], axis=0) - encoded = img_array_to_b64_image(img_array) + encoded = img_array_to_b64_image(img_array) - inf_stack = [InferenceInput(image=encoded, bounds=TMS.bounds(tile))] - payload = PredictPayload( - inf_stack=inf_stack, inf_parms=self.inference_parms - ) - res = await self.client.post( - self.url + "/predict", json=payload.dict(), timeout=None - ) + inf_stack = [InferenceInput(image=encoded, bounds=TMS.bounds(tile))] + payload = PredictPayload(inf_stack=inf_stack, inf_parms=self.inference_parms) + res = await self.client.post( + self.url + "/predict", json=payload.dict(), timeout=None + ) if res.status_code == 200: return InferenceResultStack(**res.json()) else: - print(f"XXX Received unexpected status code: {res.status_code}") - print(f"XXX HTTP content: {res.content}") print(f"XXX Issue was found in: {self.sceneid}") - return InferenceResultStack(stack=[]) + raise Exception( + f"XXX Received unexpected status code: {res.status_code} {res.content}" + ) async def get_offset_tile_inference( - self, bounds: List[float], semaphore: asyncio.Semaphore, rescale=(0, 255) + self, bounds: List[float], rescale=(0, 255) ) -> InferenceResultStack: """fetch inference for offset tiles""" - async with semaphore: - hw = self.scale * 256 - img_array = await self.titiler_client.get_offset_tile( - self.sceneid, *bounds, width=hw, height=hw, rescale=rescale - ) - img_array = reshape_as_raster(img_array) - with self.aux_datasets.open() as src: - window = rasterio.windows.from_bounds(*bounds, transform=src.transform) - height, width = img_array.shape[1:] - aux_ds = src.read(window=window, out_shape=(height, width)) + hw = self.scale * 256 + img_array = await self.titiler_client.get_offset_tile( + self.sceneid, *bounds, width=hw, height=hw, rescale=rescale + ) + img_array = reshape_as_raster(img_array) + with self.aux_datasets.open() as src: + window = rasterio.windows.from_bounds(*bounds, transform=src.transform) + height, width = img_array.shape[1:] + aux_ds = src.read(window=window, out_shape=(height, width)) - img_array = np.concatenate([img_array[0:1, :, :], aux_ds], axis=0) + img_array = np.concatenate([img_array[0:1, :, :], aux_ds], axis=0) - encoded = img_array_to_b64_image(img_array) + encoded = img_array_to_b64_image(img_array) - inf_stack = [InferenceInput(image=encoded, bounds=bounds)] + inf_stack = [InferenceInput(image=encoded, bounds=bounds)] - payload = PredictPayload( - inf_stack=inf_stack, inf_parms=self.inference_parms - ) - res = await self.client.post( - self.url + "/predict", json=payload.dict(), timeout=None - ) + payload = PredictPayload(inf_stack=inf_stack, inf_parms=self.inference_parms) + res = await self.client.post( + self.url + "/predict", json=payload.dict(), timeout=None + ) if res.status_code == 200: return InferenceResultStack(**res.json()) else: - print(f"XXX Received unexpected status code: {res.status_code}") - print(f"XXX HTTP content: {res.content}") print(f"XXX Issue was found in: {self.sceneid}") - return InferenceResultStack(stack=[]) + raise Exception( + f"XXX Received unexpected status code: {res.status_code} {res.content}" + ) def get_scene_date_month(scene_id: str) -> str: diff --git a/cerulean_cloud/cloud_run_orchestrator/handler.py b/cerulean_cloud/cloud_run_orchestrator/handler.py index e7075ab8..d0ad3daf 100644 --- a/cerulean_cloud/cloud_run_orchestrator/handler.py +++ b/cerulean_cloud/cloud_run_orchestrator/handler.py @@ -10,7 +10,6 @@ """ import asyncio import os -import traceback import urllib.parse as urlparse from base64 import b64decode # , b64encode from datetime import datetime, timedelta @@ -245,14 +244,13 @@ def flatten_feature_list( return flat_list -async def perform_inference(tiles, inference_func, semaphore_value, description): +async def perform_inference(tiles, inference_func, description): """ Perform inference on a set of tiles asynchronously. Parameters: - tiles or bounds (list): List of tiles to perform inference on. (depends on inference_func) - inference_func (function): Asynchronous function to call for inference. - - semaphore_value (int): Maximum number of concurrent tasks. - description (str): Description of the inference task for logging. Returns: @@ -263,22 +261,11 @@ async def perform_inference(tiles, inference_func, semaphore_value, description) - Prints traceback of exceptions to the console. """ print(f"Inference on {description}!") - semaphore = asyncio.Semaphore(value=semaphore_value) inferences = await asyncio.gather( - *[ - inference_func(tile, rescale=(0, 255), semaphore=semaphore) - for tile in tiles - ], - return_exceptions=True, + *[inference_func(tile, rescale=(0, 255)) for tile in tiles], + return_exceptions=False, # This raises exceptions ) - clean_inferences = [] - for res in inferences: - if isinstance(res, Exception): - print(f"WARNING: Exception occurred during {description} inference: {res}") - traceback.print_tb(res.__traceback__) - else: - clean_inferences.append(res) - return clean_inferences + return inferences async def _orchestrate( @@ -396,17 +383,21 @@ async def _orchestrate( base_tiles_inference = await perform_inference( base_tiles, cloud_run_inference.get_base_tile_inference, - 20, + 100, "base tiles", ) offset_tiles_inference = await perform_inference( offset_tiles_bounds, cloud_run_inference.get_offset_tile_inference, - 20, + 100, "offset tiles", ) + # Clean up potentially memory heavy assets + del base_tiles + del offset_tiles_bounds + if model.type == "MASKRCNN": out_fc = geojson.FeatureCollection( features=flatten_feature_list(base_tiles_inference) @@ -414,6 +405,10 @@ async def _orchestrate( out_fc_offset = geojson.FeatureCollection( features=flatten_feature_list(offset_tiles_inference) ) + + # Clean up potentially memory heavy assets + del base_tiles_inference + del offset_tiles_inference elif model.type == "UNET": # print("Loading all tiles into memory for merge!") # ds_base_tiles = [] @@ -495,11 +490,9 @@ async def _orchestrate( buffered_gdf["geometry"] = buffered_gdf.to_crs( "EPSG:3857" ).buffer(LAND_MASK_BUFFER_M) + landmask = get_landmask_gdf() intersecting_land = gpd.sjoin( - get_landmask_gdf(), - buffered_gdf, - how="inner", - predicate="intersects", + landmask, buffered_gdf, how="inner", predicate="intersects" ) if not intersecting_land.empty: feat["properties"]["inf_idx"] = 0 @@ -531,6 +524,10 @@ async def _orchestrate( ntiles=ntiles, noffsettiles=noffsettiles, ) + + # Clean up potentially memory heavy assets + del out_fc + del out_fc_offset else: print("WARNING: Operating as a DRY RUN!!") orchestrator_result = OrchestratorResult( diff --git a/cerulean_cloud/cloud_run_orchestrator/merging.py b/cerulean_cloud/cloud_run_orchestrator/merging.py index 8b36db52..f515fc4c 100644 --- a/cerulean_cloud/cloud_run_orchestrator/merging.py +++ b/cerulean_cloud/cloud_run_orchestrator/merging.py @@ -105,6 +105,14 @@ def merge_inferences( # Reproject the GeoDataFrame back to WGS 84 CRS result = dissolved_gdf.to_crs(crs=4326) + # Clean up potentially memory heavy assets + del dissolved_gdf + del concat_gdf + del final_gdf + del joined + del base_gdf + del offset_gdf + return result.__geo_interface__ else: # If one of the FeatureCollections is empty, return an empty FeatureCollection diff --git a/notebooks/run_sr_on_scene_ids.ipynb b/notebooks/run_sr_on_scene_ids.ipynb index 310a9ad0..c9ba40a8 100644 --- a/notebooks/run_sr_on_scene_ids.ipynb +++ b/notebooks/run_sr_on_scene_ids.ipynb @@ -1,5 +1,261 @@ { "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "slicks = [\"S1A_IW_GRDH_1SDV_20200727T053553_20200727T053622_033636_03E5FE_2D39\",\n", + "\"S1B_IW_GRDH_1SDV_20200727T092212_20200727T092243_022655_02AFFE_98C7\",\n", + "\"S1A_IW_GRDH_1SDV_20200729T034859_20200729T034924_033664_03E6D3_93EF\",\n", + "\"S1A_IW_GRDH_1SDV_20200729T095401_20200729T095430_033668_03E6EE_2611\",\n", + "\"S1B_IW_GRDH_1SDV_20200731T055735_20200731T055800_022711_02B1B2_854D\",\n", + "\"S1B_IW_GRDH_1SSV_20200801T105756_20200801T105824_022728_02B236_F52F\",\n", + "\"S1A_IW_GRDH_1SDV_20200802T062802_20200802T062827_033724_03E89F_6485\",\n", + "\"S1B_IW_GRDH_1SDV_20200803T061857_20200803T061922_022755_02B2F6_ED3D\",\n", + "\"S1B_IW_GRDH_1SDV_20200803T062422_20200803T062447_022755_02B2F6_27F0\",\n", + "\"S1B_IW_GRDH_1SDV_20200803T173558_20200803T173623_022762_02B332_E4F2\",\n", + "\"S1B_IW_GRDH_1SDV_20200803T173623_20200803T173648_022762_02B332_AA8F\",\n", + "\"S1A_IW_GRDH_1SDV_20200805T220025_20200805T220050_033777_03EA60_3031\",\n", + "\"S1A_IW_GRDH_1SDV_20200806T152803_20200806T152828_033788_03EAB9_DCCA\",\n", + "\"S1B_IW_GRDH_1SDV_20200806T162023_20200806T162048_022805_02B484_06D3\",\n", + "\"S1A_IW_GRDH_1SDV_20200806T223947_20200806T224012_033792_03EAE3_9E54\",\n", + "\"S1B_IW_GRDH_1SDV_20200807T025207_20200807T025232_022811_02B4B5_9FCE\",\n", + "\"S1A_IW_GRDH_1SDV_20200808T222606_20200808T222633_033821_03EBDD_2485\",\n", + "\"S1B_IW_GRDH_1SDV_20200810T172848_20200810T172904_022864_02B66F_595C\",\n", + "\"S1A_IW_GRDH_1SDV_20200811T060256_20200811T060321_033855_03ED02_07D6\",\n", + "\"S1A_IW_GRDH_1SDV_20200811T060641_20200811T060706_033855_03ED02_3E59\",\n", + "\"S1A_IW_GRDH_1SDV_20200812T125006_20200812T125026_033874_03EDB4_721F\",\n", + "\"S1B_IW_GRDH_1SDV_20200812T170559_20200812T170624_022893_02B744_B462\",\n", + "\"S1A_IW_GRDH_1SDV_20200814T105729_20200814T105754_033902_03EE9F_3456\",\n", + "\"S1A_IW_GRDH_1SDV_20200816T061108_20200816T061133_033928_03EF95_6194\",\n", + "\"S1A_IW_GRDH_1SDV_20200817T163009_20200817T163034_033949_03F053_8FD9\",\n", + "\"S1A_IW_GRDH_1SDV_20200817T163009_20200817T163025_033949_03F053_C800\",\n", + "\"S1B_IW_GRDH_1SDV_20200819T054843_20200819T054908_022988_02BA34_E26A\",\n", + "\"S1A_IW_GRDH_1SDV_20200819T062057_20200819T062126_033972_03F134_D4C1\",\n", + "\"S1A_IW_GRDH_1SDV_20200819T142908_20200819T142933_033977_03F162_C177\",\n", + "\"S1B_IW_GRDH_1SDV_20200820T092213_20200820T092238_023005_02BABB_74F3\",\n", + "\"S1A_IW_GRDH_1SDV_20200820T165606_20200820T165631_033993_03F1E0_DF57\",\n", + "\"S1B_IW_GRDH_1SDV_20200823T094743_20200823T094812_023049_02BC37_604E\",\n", + "\"S1A_IW_GRDH_1SDV_20200826T062739_20200826T062804_034074_03F4B5_151E\",\n", + "\"S1B_IW_GRDH_1SDV_20200826T164820_20200826T164845_023097_02BDAD_5CDF\",\n", + "\"S1A_IW_GRDH_1SDV_20200826T174457_20200826T174524_034081_03F4F8_1812\",\n", + "\"S1B_IW_GRDH_1SDV_20200827T061833_20200827T061858_023105_02BDDE_2D7D\",\n", + "\"S1A_IW_GRDH_1SDV_20200828T061109_20200828T061134_034103_03F5BD_EBE2\",\n", + "\"S1A_IW_GRDH_1SDV_20200829T065329_20200829T065354_034118_03F64E_29F9\",\n", + "\"S1A_IW_GRDH_1SDV_20200829T094838_20200829T094903_034120_03F65E_0A81\",\n", + "\"S1A_IW_GRDH_1SDV_20200830T071607_20200830T071636_034133_03F6D5_6592\",\n", + "\"S1B_IW_GRDH_1SDV_20200830T080240_20200830T080309_023150_02BF42_95BC\",\n", + "\"S1B_IW_GRDH_1SDV_20200901T074652_20200901T074721_023179_02C030_41EF\",\n", + "\"S1B_IW_GRDH_1SDV_20200901T174057_20200901T174130_023185_02C064_C8EC\",\n", + "\"S1A_IW_GRDH_1SDV_20200904T224710_20200904T224739_034215_03F9B3_86D7\",\n", + "\"S1A_IW_GRDH_1SDV_20200905T110703_20200905T110728_034222_03F9FE_64E8\",\n", + "\"S1A_IW_GRDH_1SDV_20200905T214055_20200905T214120_034229_03FA38_ED1A\",\n", + "\"S1B_IW_GRDH_1SDV_20200906T175227_20200906T175252_023258_02C2A8_872C\",\n", + "\"S1A_IW_GRDH_1SDV_20200907T230944_20200907T231009_034259_03FB3E_965C\",\n", + "\"S1B_IW_GRDH_1SDV_20200908T172846_20200908T172911_023287_02C398_079F\",\n", + "\"S1B_IW_GRDH_1SDV_20200909T034850_20200909T034915_023293_02C3CD_F327\",\n", + "\"S1B_IW_GRDH_1SDV_20200909T050510_20200909T050539_023294_02C3D3_5BFE\",\n", + "\"S1B_IW_GRDH_1SDV_20200909T181350_20200909T181415_023302_02C415_31CE\",\n", + "\"S1A_IW_GRDH_1SDV_20200910T220117_20200910T220142_034302_03FCBE_63C9\",\n", + "\"S1A_IW_GRDH_1SDV_20200911T152715_20200911T152740_034313_03FD2A_FDFB\",\n", + "\"S1A_IW_GRDH_1SDV_20200914T155236_20200914T155301_034357_03FEB5_3FF5\",\n", + "\"S1A_IW_GRDH_1SDV_20200915T095403_20200915T095432_034368_03FF15_9322\",\n", + "\"S1B_IW_GRDH_1SDV_20200916T045603_20200916T045633_023396_02C70E_91BE\",\n", + "\"S1A_IW_GRDH_1SDV_20200916T224804_20200916T224829_034390_03FFDA_C6FB\",\n", + "\"S1B_IW_GRDH_1SDV_20200918T045856_20200918T045921_023425_02C7F1_107A\",\n", + "\"S1A_IW_GRDH_1SDV_20200918T163714_20200918T163739_034415_0400BB_0649\",\n", + "\"S1B_IW_GRDH_1SDV_20200918T214237_20200918T214302_023435_02C842_FBF3\",\n", + "\"S1A_IW_GRDH_1SDV_20200918T223129_20200918T223154_034419_0400E1_96AB\",\n", + "\"S1B_IW_GRDH_1SDV_20200919T054009_20200919T054034_023440_02C861_C12B\",\n", + "\"S1B_IW_GRDH_1SDV_20200919T054012_20200919T054037_023440_02C861_9310\",\n", + "\"S1B_IW_GRDH_1SDV_20200919T054419_20200919T054444_023440_02C861_A430\",\n", + "\"S1B_IW_GRDH_1SDV_20200920T173001_20200920T173026_023462_02C912_6B7E\",\n", + "\"S1A_IW_GRDH_1SDV_20200921T104144_20200921T104209_034456_04023C_C472\",\n", + "\"S1A_IW_GRDH_1SDV_20200924T110557_20200924T110622_034500_0403C8_453D\",\n", + "\"S1A_IW_GRDH_1SDV_20200927T095703_20200927T095728_034543_040541_5DFB\",\n", + "\"S1A_IW_GRDH_1SDV_20200927T234107_20200927T234136_034551_04058A_14EF\",\n", + "\"S1A_IW_GRDH_1SDV_20200929T050747_20200929T050812_034569_040632_42A5\",\n", + "\"S1B_IW_GRDH_1SDV_20200930T045856_20200930T045921_023600_02CD6C_6A16\",\n", + "\"S1B_IW_GRDH_1SDV_20201001T053714_20201001T053739_023615_02CDE1_C3BD\",\n", + "\"S1A_IW_GRDH_1SDV_20201001T105756_20201001T105821_034602_04074E_5D47\",\n", + "\"S1B_IW_GRDH_1SDV_20201001T164706_20201001T164731_023622_02CE1A_5A7E\",\n", + "\"S1B_IW_GRDH_1SDV_20201002T172847_20201002T172912_023637_02CE8C_FC31\",\n", + "\"S1B_IW_GRDH_1SDV_20201005T064329_20201005T064358_023674_02CFB0_F7B2\",\n", + "\"S1B_IW_GRDH_1SDV_20201006T020955_20201006T021020_023686_02D017_0A1B\",\n", + "\"S1B_IW_GRDH_1SDV_20201006T115402_20201006T115425_023692_02D04C_FE86\",\n", + "\"S1A_IW_GRDH_1SDV_20201008T140751_20201008T140816_034706_040AEE_EAD2\",\n", + "\"S1A_IW_GRDH_1SDV_20201009T201613_20201009T201642_034724_040B9A_67CD\",\n", + "\"S1A_IW_GRDH_1SDV_20201013T213406_20201013T213431_034783_040DAD_B3DD\",\n", + "\"S1A_IW_GRDH_1SDV_20201014T221637_20201014T221702_034798_040E34_A2D7\",\n", + "\"S1B_IW_GRDH_1SDV_20201015T050510_20201015T050539_023819_02D437_8BEC\",\n", + "\"S1A_IW_GRDH_1SDV_20201018T142910_20201018T142935_034852_04102A_7930\",\n", + "\"S1B_IW_GRDH_1SDV_20201018T183430_20201018T183455_023871_02D5CE_E8CD\",\n", + "\"S1A_IW_GRDH_1SDV_20201021T145027_20201021T145052_034896_041191_73FA\",\n", + "\"S1B_IW_GRDH_1SDV_20201021T172050_20201021T172115_023914_02D741_0719\",\n", + "\"S1B_IW_GRDH_1SDV_20201022T033938_20201022T034003_023920_02D76C_177F\",\n", + "\"S1A_IW_GRDH_1SDV_20201023T051313_20201023T051338_034919_041264_CD66\",\n", + "\"S1A_IW_GRDH_1SDV_20201023T051329_20201023T051354_034919_041264_545A\",\n", + "\"S1B_IW_GRDH_1SDV_20201023T170409_20201023T170433_023943_02D81C_C8C1\",\n", + "\"S1A_IW_GRDH_1SDV_20201024T115458_20201024T115523_034938_041306_3757\",\n", + "\"S1B_IW_GRDH_1SDV_20201025T164846_20201025T164911_023972_02D908_B5CD\",\n", + "\"S1A_IW_GRDH_1SDV_20201027T044047_20201027T044112_034977_04145C_816C\",\n", + "\"S1A_IW_GRDH_1SDV_20201028T111704_20201028T111729_034995_041508_D5D9\",\n", + "\"S1B_IW_GRDH_1SDV_20201103T081047_20201103T081116_024098_02DCED_4307\",\n", + "\"S1A_IW_GRDH_1SDV_20201103T121212_20201103T121237_035084_041809_3140\",\n", + "\"S1A_IW_GRDH_1SDV_20201104T175743_20201104T175808_035102_0418B1_CFD4\",\n", + "\"S1A_IW_GRDH_1SDV_20201106T155808_20201106T155833_035130_04199D_4800\",\n", + "\"S1B_IW_GRDH_1SDV_20201106T181706_20201106T181731_024148_02DE74_A73C\",\n", + "\"S1B_IW_GRDH_1SDV_20201107T173257_20201107T173322_024162_02DEE1_CE9F\",\n", + "\"S1A_IW_GRDH_1SDV_20201107T202510_20201107T202538_035147_041A3A_6755\",\n", + "\"S1B_IW_GRDH_1SDV_20201108T034825_20201108T034850_024168_02DF16_86C2\",\n", + "\"S1B_IW_GRDH_1SDV_20201109T102532_20201109T102557_024186_02DFA7_FD74\",\n", + "\"S1A_IW_GRDH_1SDV_20201109T220028_20201109T220053_035177_041B4C_E413\",\n", + "\"S1A_IW_GRDH_1SDV_20201110T055833_20201110T055858_035182_041B7B_5F7D\",\n", + "\"S1B_IW_GRDH_1SDV_20201112T045635_20201112T045700_024227_02E0E5_11BA\",\n", + "\"S1B_IW_GRDH_1SDV_20201112T105616_20201112T105635_024231_02E10A_8A74\",\n", + "\"S1A_IW_GRDH_1SDV_20201112T222309_20201112T222338_035221_041CD2_BD9D\",\n", + "\"S1B_IW_GRDH_1SDV_20201114T013822_20201114T013850_024254_02E1E1_4638\",\n", + "\"S1A_IW_GRDH_1SDV_20201114T034910_20201114T034935_035239_041D79_AA81\",\n", + "\"S1B_IW_GRDH_1SDV_20201115T180127_20201115T180152_024279_02E2A2_784D\",\n", + "\"S1A_IW_GRDH_1SDV_20201116T052225_20201116T052250_035269_041E82_12BD\",\n", + "\"S1A_IW_GRDH_1SDV_20201116T175357_20201116T175422_035277_041EC7_0A33\",\n", + "\"S1A_IW_GRDH_1SDV_20201117T205332_20201117T205401_035293_041F57_5883\",\n", + "\"S1B_IW_GRDH_1SDV_20201119T154826_20201119T154851_024336_02E455_8DB6\",\n", + "\"S1B_IW_GRDH_1SDV_20201123T115402_20201123T115425_024392_02E625_5AFA\",\n", + "\"S1A_IW_GRDH_1SDV_20201124T040611_20201124T040636_035385_042277_9B55\",\n", + "\"S1A_IW_GRDH_1SDV_20201124T040816_20201124T040841_035385_042277_817C\",\n", + "\"S1A_IW_GRDH_1SDV_20201202T154342_20201202T154407_035509_0426CE_9477\",\n", + "\"S1A_IW_GRDH_1SDV_20201203T020715_20201203T020740_035515_0426FB_EE3B\",\n", + "\"S1B_IW_GRDH_1SDV_20201203T080555_20201203T080623_024535_02EAA8_1CB3\",\n", + "\"S1A_IW_GRDH_1SDV_20201208T180156_20201208T180221_035598_0429D7_BBDB\",\n", + "\"S1A_IW_GRDH_1SDV_20201209T025943_20201209T030008_035603_042A0A_899D\",\n", + "\"S1A_IW_GRDH_1SDV_20201210T224256_20201210T224325_035630_042AF4_1998\",\n", + "\"S1A_IW_GRDH_1SDV_20201211T115403_20201211T115432_035638_042B31_BAD4\",\n", + "\"S1B_IW_GRDH_1SDV_20201212T100558_20201212T100623_024668_02EEFE_89B4\",\n", + "\"S1B_IW_GRDH_1SDV_20201213T082213_20201213T082238_024681_02EF68_CBA2\",\n", + "\"S1A_IW_GRDH_1SDV_20201220T095521_20201220T095546_035768_042FAC_052A\",\n", + "\"S1A_IW_GRDH_1SDV_20201220T234106_20201220T234134_035776_042FF9_A66B\",\n", + "\"S1A_IW_GRDH_1SDV_20201223T115432_20201223T115457_035813_04313B_3F2D\",\n", + "\"S1A_IW_GRDH_1SDV_20201224T104941_20201224T105010_035826_0431AC_AAED\",\n", + "\"S1A_IW_GRDH_1SDV_20201228T223602_20201228T223631_035892_043418_A489\",\n", + "\"S1B_IW_GRDH_1SDV_20201229T054850_20201229T054915_024913_02F6E8_4CB9\",\n", + "\"S1B_IW_GRDH_1SDV_20201230T032144_20201230T032213_024926_02F74E_477A\",\n", + "\"S1B_IW_GRDH_1SDV_20210103T151943_20210103T152008_024992_02F972_74D5\",\n", + "\"S1A_IW_GRDH_1SDV_20210107T211923_20210107T211948_036037_043910_955C\",\n", + "\"S1B_IW_GRDH_1SDV_20210108T060259_20210108T060324_025059_02FB91_5DF2\",\n", + "\"S1B_IW_GRDH_1SDV_20210108T152831_20210108T152900_025065_02FBC1_5A3D\",\n", + "\"S1B_IW_GRDH_1SDV_20210109T051051_20210109T051116_025073_02FC03_DDA5\",\n", + "\"S1A_IW_GRDH_1SDV_20210111T005559_20210111T005624_036083_043AB6_3173\",\n", + "\"S1A_IW_GRDH_1SDV_20210114T080140_20210114T080205_036131_043C63_3037\",\n", + "\"S1A_IW_GRDH_1SDV_20210114T102514_20210114T102542_036132_043C73_5FD7\",\n", + "\"S1B_IW_GRDH_1SSV_20210116T105755_20210116T105824_025178_02FF7A_F2F3\",\n", + "\"S1B_IW_GRDH_1SDV_20210116T174610_20210116T174635_025183_02FF94_7B48\",\n", + "\"S1A_IW_GRDH_1SDV_20210117T063647_20210117T063712_036174_043DC7_6CC8\",\n", + "\"S1A_IW_GRDH_1SDV_20210120T220140_20210120T220208_036227_043FC4_B850\",\n", + "\"S1A_IW_GRDH_1SDV_20210123T040814_20210123T040839_036260_0440DA_D0AD\",\n", + "\"S1A_IW_GRDH_1SDV_20210125T004125_20210125T004148_036287_0441CB_9D3B\",\n", + "\"S1A_IW_GRDH_1SDV_20210127T110726_20210127T110751_036322_044313_DE5C\",\n", + "\"S1A_IW_GRDH_1SDV_20210130T180909_20210130T180938_036371_0444B1_B3A6\",\n", + "\"S1A_IW_GRDH_1SDV_20210205T230221_20210205T230246_036461_0447D3_8CCD\",\n", + "\"S1B_IW_GRDH_1SDV_20210207T034025_20210207T034050_025495_03099F_537F\",\n", + "\"S1A_IW_GRDH_1SDV_20210208T215134_20210208T215159_036504_04495A_8ACE\",\n", + "\"S1B_IW_GRDH_1SDV_20210210T040747_20210210T040812_025539_030B0E_215C\",\n", + "\"S1A_IW_GRDH_1SDV_20210213T111635_20210213T111700_036570_044BAA_87EE\",\n", + "\"S1A_IW_GRDH_1SDV_20210213T111815_20210213T111840_036571_044BAA_EFFF\",\n", + "\"S1A_IW_GRDH_1SDV_20210216T054436_20210216T054501_036611_044D0A_F4BB\",\n", + "\"S1B_IW_GRDH_1SDV_20210221T214324_20210221T214349_025710_0310A1_708C\",\n", + "\"S1A_IW_GRDH_1SDV_20210222T213607_20210222T213632_036708_04506F_C174\",\n", + "\"S1A_IW_GRDH_1SDV_20210225T215524_20210225T215549_036752_0451F9_2D87\",\n", + "\"S1A_IW_GRDH_1SDV_20210228T164718_20210228T164743_036793_04535E_6075\",\n", + "\"S1A_IW_GRDH_1SDV_20210302T220541_20210302T220610_036825_04547E_3617\",\n", + "\"S1A_IW_GRDH_1SDV_20210304T215133_20210304T215158_036854_045580_A745\",\n", + "\"S1A_IW_GRDH_1SDV_20210305T183416_20210305T183441_036867_0455E9_224E\",\n", + "\"S1A_IW_GRDH_1SDV_20210306T173830_20210306T173855_036881_045669_EB98\",\n", + "\"S1A_IW_GRDH_1SDV_20210308T211922_20210308T211947_036912_04578C_32C3\",\n", + "\"S1A_IW_GRDH_1SDV_20210316T092832_20210316T092857_037021_045B62_8A50\",\n", + "\"S1B_IW_GRDH_1SSV_20210317T105754_20210317T105823_026053_031BD4_B5B5\",\n", + "\"S1A_IW_GRDH_1SDV_20210318T155714_20210318T155739_037055_045C7B_7BE8\",\n", + "\"S1A_IW_GRDH_1SDV_20210318T213402_20210318T213427_037058_045C9B_87A6\",\n", + "\"S1A_IW_GRDH_1SDV_20210321T180419_20210321T180444_037100_045E1E_F51F\",\n", + "\"S1B_IW_GRDH_1SDV_20210330T182527_20210330T182552_026248_0321F8_623A\",\n", + "\"S1B_IW_GRDH_1SDV_20210403T161317_20210403T161342_026305_0323BE_A751\",\n", + "\"S1A_IW_GRDH_1SDV_20210405T054621_20210405T054646_037311_046559_AD20\",\n", + "\"S1B_IW_GRDH_1SDV_20210407T154109_20210407T154134_026363_0325A8_F678\",\n", + "\"S1A_IW_GRDH_1SDV_20210409T110701_20210409T110726_037372_046780_6870\",\n", + "\"S1A_IW_GRDH_1SDV_20210409T110726_20210409T110751_037372_046780_4FAF\",\n", + "\"S1A_IW_GRDH_1SDV_20210410T023915_20210410T023940_037382_0467CD_AAE7\",\n", + "\"S1B_IW_GRDH_1SDV_20210410T050251_20210410T050316_026400_0326C5_08FD\",\n", + "\"S1A_IW_GRDH_1SDV_20210410T115430_20210410T115455_037388_0467FA_6BBC\",\n", + "\"S1B_IW_GRDH_1SDV_20210410T174454_20210410T174519_026408_032703_72DA\",\n", + "\"S1A_IW_GRDH_1SDV_20210410T223057_20210410T223126_037394_046832_0D96\",\n", + "\"S1B_IW_GRDH_1SDV_20210412T073925_20210412T073959_026431_0327BC_3F90\",\n", + "\"S1B_IW_GRDH_1SDV_20210412T154848_20210412T154913_026436_0327E9_9B9B\",\n", + "\"S1B_IW_GRDH_1SDV_20210413T070653_20210413T070718_026445_032837_1AA4\",\n", + "\"S1A_IW_GRDH_1SDV_20210414T111636_20210414T111701_037445_046A11_081C\",\n", + "\"S1A_IW_GRDH_1SDV_20210419T134630_20210419T134705_037520_046C92_5C67\",\n", + "\"S1A_IW_GRDH_1SDV_20210422T133502_20210422T133531_037564_046E16_70CC\",\n", + "\"S1A_IW_GRDH_1SDV_20210422T183442_20210422T183507_037567_046E30_59A7\",\n", + "\"S1A_IW_GRDH_1SDV_20210425T140000_20210425T140029_037608_046F9E_855A\",\n", + "\"S1B_IW_GRDH_1SDV_20210426T102555_20210426T102620_026636_032E57_2730\",\n", + "\"S1B_IW_GRDH_1SDV_20210430T180919_20210430T180944_026700_033073_D411\",\n", + "\"S1A_IW_GRDH_1SDV_20210505T173742_20210505T173807_037756_0474BF_7479\",\n", + "\"S1A_IW_GRDH_1SDV_20210506T040030_20210506T040055_037762_0474EC_7C80\",\n", + "\"S1B_IW_GRDH_1SDV_20210507T052900_20210507T052929_026794_033366_C7A0\",\n", + "\"S1A_IW_GRDH_1SDV_20210508T053003_20210508T053032_037792_0475E5_7B3F\",\n", + "\"S1A_IW_GRDH_1SDV_20210518T163942_20210518T164007_037945_047A75_22BA\",\n", + "\"S1A_IW_GRDH_1SDV_20210519T044110_20210519T044135_037952_047AB2_C27D\",\n", + "\"S1A_IW_GRDH_1SDV_20210523T005625_20210523T005651_038008_047C68_FE94\",\n", + "\"S1A_IW_GRDH_1SDV_20210523T015814_20210523T015843_038009_047C6D_19FD\",\n", + "\"S1A_IW_GRDH_1SDV_20210524T185906_20210524T185931_038034_047D29_07CF\",\n", + "\"S1A_IW_GRDH_1SDV_20210527T224256_20210527T224325_038080_047E80_D5FD\",\n", + "\"S1A_IW_GRDH_1SDV_20210528T115433_20210528T115458_038088_047EBE_0C02\",\n", + "\"S1B_IW_GRDH_1SDV_20210529T100234_20210529T100259_027117_033D47_7A3C\",\n", + "\"S1A_IW_GRDH_1SDV_20210602T060243_20210602T060308_038157_0480DD_6BB7\",\n", + "\"S1B_IW_GRDH_1SDV_20210604T045512_20210604T045537_027202_033FD0_5BD8\",\n", + "\"S1A_IW_GRDH_1SDV_20210609T223219_20210609T223244_038269_048419_AA2B\",\n", + "\"S1A_IW_GRDH_1SDV_20210612T140003_20210612T140032_038308_048551_A942\",\n", + "\"S1A_IW_GRDH_1SDV_20210612T221018_20210612T221043_038313_04857A_F322\",\n", + "\"S1B_IW_GRDH_1SDV_20210613T233207_20210613T233232_027344_03440C_B11F\",\n", + "\"S1B_IW_GRDH_1SDV_20210615T151813_20210615T151838_027369_0344D0_5F31\",\n", + "\"S1B_IW_GRDH_1SDV_20210615T165449_20210615T165514_027370_0344D7_8940\",\n", + "\"S1A_IW_GRDH_1SDV_20210621T115524_20210621T115549_038438_048925_3E3E\",\n", + "\"S1A_IW_GRDH_1SDV_20210621T183420_20210621T183445_038442_048944_E1A2\",\n", + "\"S1A_IW_GRDH_1SDV_20210621T231513_20210621T231538_038444_048958_CA6F\",\n", + "\"S1B_IW_GRDH_1SDV_20210623T173003_20210623T173028_027487_034821_2CDD\",\n", + "\"S1A_IW_GRDH_1SDV_20210624T154204_20210624T154229_038484_048A93_87BF\",\n", + "\"S1A_IW_GRDH_1SDV_20210624T154344_20210624T154409_038484_048A93_8178\",\n", + "\"S1B_IW_GRDH_1SDV_20210626T051247_20210626T051312_027523_034911_53D5\",\n", + "\"S1B_IW_GRDH_1SDV_20210628T155657_20210628T155726_027559_034A2F_9F11\",\n", + "\"S1A_IW_GRDH_1SDV_20210629T061906_20210629T061931_038551_048C97_235C\",\n", + "\"S1A_IW_GRDH_1SDV_20210703T170534_20210703T170559_038616_048E78_ACC7\",\n", + "\"S1A_IW_GRDH_1SDV_20210703T223221_20210703T223246_038619_048E94_2418\",\n", + "\"S1A_IW_GRDH_1SDV_20210703T223311_20210703T223333_038619_048E94_E29D\",\n", + "\"S1A_IW_GRDH_1SDV_20210704T063601_20210704T063626_038624_048EB8_87AC\",\n", + "\"S1B_IW_GRDH_1SDV_20210704T164823_20210704T164848_027647_034CBC_1AE3\",\n", + "\"S1B_IW_GRDH_1SDV_20210705T044729_20210705T044754_027654_034CE9_B778\",\n", + "\"S1B_IW_GRDH_1SDV_20210707T172027_20210707T172050_027691_034E02_CA71\",\n", + "\"S1A_IW_GRDH_1SDV_20210707T215504_20210707T215529_038677_049070_19B8\",\n", + "\"S1A_IW_GRDH_1SDV_20210708T120037_20210708T120102_038686_0490AC_2F5F\",\n", + "\"S1A_IW_GRDH_1SDV_20210709T142502_20210709T142527_038702_04912B_0FDD\",\n", + "\"S1A_IW_GRDH_1SDV_20210709T160646_20210709T160711_038703_049131_D410\",\n", + "\"S1A_IW_GRDH_1SDV_20210714T050536_20210714T050601_038769_049323_84FF\",\n", + "\"S1A_IW_GRDH_1SDV_20210714T143337_20210714T143402_038775_04934E_6462\",\n", + "\"S1B_IW_GRDH_1SDV_20210715T160447_20210715T160512_027807_035175_C66C\",\n", + "\"S1A_IW_GRDH_1SDV_20210715T170149_20210715T170214_038791_0493BD_B473\",\n", + "\"S1A_IW_GRDH_1SDV_20210716T000518_20210716T000547_038795_0493DB_BE47\",\n", + "\"S1A_IW_GRDH_1SDV_20210717T164011_20210717T164036_038820_0494A3_1AC6\",\n", + "\"S1B_IW_GRDH_1SDV_20210719T040729_20210719T040758_027858_0352F9_CF72\",\n", + "\"S1A_IW_GRDH_1SDV_20210719T111616_20210719T111641_038845_04957D_9ECD\",\n", + "\"S1B_IW_GRDH_1SDV_20210721T055232_20210721T055257_027888_0353E2_9B6E\",\n", + "\"S1B_IW_GRDH_1SDV_20210722T032148_20210722T032217_027901_03544C_A13B\",\n", + "\"S1A_IW_GRDH_1SDV_20210723T060213_20210723T060243_038901_049711_A591\",\n", + "\"S1B_IW_GRDH_1SDV_20210723T164019_20210723T164044_027924_0354FB_5BDF\",\n", + "\"S1A_IW_GRDH_1SDV_20210724T000136_20210724T000201_038912_04976B_207D\",\n", + "\"S1A_IW_GRDH_1SDV_20210726T215115_20210726T215140_038954_0498A9_7F2B\",]" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/stack/cloud_function_ais_analysis.py b/stack/cloud_function_ais_analysis.py index cb2afeae..8339cd46 100644 --- a/stack/cloud_function_ais_analysis.py +++ b/stack/cloud_function_ais_analysis.py @@ -19,7 +19,7 @@ construct_name("queue-cloud-tasks-ais-analysis"), location=pulumi.Config("gcp").require("region"), rate_limits=cloudtasks.QueueRateLimitsArgs( - max_concurrent_dispatches=1, + max_concurrent_dispatches=200, max_dispatches_per_second=1, ), retry_config=cloudtasks.QueueRetryConfigArgs( diff --git a/stack/cloud_function_scene_relevancy.py b/stack/cloud_function_scene_relevancy.py index e02e2c08..9f39d7f8 100644 --- a/stack/cloud_function_scene_relevancy.py +++ b/stack/cloud_function_scene_relevancy.py @@ -21,7 +21,7 @@ construct_name("queue-cloud-run-orchestrator"), location=pulumi.Config("gcp").require("region"), rate_limits=cloudtasks.QueueRateLimitsArgs( - max_concurrent_dispatches=1, + max_concurrent_dispatches=50, max_dispatches_per_second=1, ), retry_config=cloudtasks.QueueRetryConfigArgs( diff --git a/stack/cloud_run_offset_tile.py b/stack/cloud_run_offset_tile.py index 26a75152..8965cadf 100644 --- a/stack/cloud_run_offset_tile.py +++ b/stack/cloud_run_offset_tile.py @@ -33,6 +33,9 @@ ), metadata=dict( name=service_name + "-" + cloud_run_images.cloud_run_offset_tile_sha, + annotations={ + "autoscaling.knative.dev/maxScale": "2000", + }, ), ), metadata=gcp.cloudrun.ServiceMetadataArgs( diff --git a/stack/cloud_run_orchestrator.py b/stack/cloud_run_orchestrator.py index 2414aa3e..f3a75a11 100644 --- a/stack/cloud_run_orchestrator.py +++ b/stack/cloud_run_orchestrator.py @@ -120,11 +120,13 @@ ), ], timeout_seconds=3540, + container_concurrency=15, ), metadata=dict( name=service_name + "-" + cloud_run_images.cloud_run_orchestrator_sha, annotations={ "run.googleapis.com/cloudsql-instances": instance.connection_name, + "autoscaling.knative.dev/maxScale": "5", }, ), ), diff --git a/stack/database.py b/stack/database.py index c7784ab5..ccf46d34 100644 --- a/stack/database.py +++ b/stack/database.py @@ -12,7 +12,7 @@ settings=gcp.sql.DatabaseInstanceSettingsArgs( tier=pulumi.Config("db").require("db-instance"), backup_configuration=dict(enabled=True), - database_flags=[dict(name="max_connections", value=200)], + database_flags=[dict(name="max_connections", value=500)], ), deletion_protection=pulumi.Config("db").require("deletion-protection"), ) diff --git a/stack/utils.py b/stack/utils.py index ffb4bf7c..e2971187 100644 --- a/stack/utils.py +++ b/stack/utils.py @@ -7,13 +7,10 @@ import pulumi from google.cloud import storage -project = pulumi.get_project() -stack = pulumi.get_stack() - def construct_name(resource_name: str) -> str: """construct resource names from project and stack""" - return f"{project}-{stack}-{resource_name}" + return f"{pulumi.get_project()}-{pulumi.get_stack()}-{resource_name}" def sha256sum(filename): From 23c7f08f06e92af20c28a2f9bb3be8d4b20c0fd9 Mon Sep 17 00:00:00 2001 From: Jona Date: Fri, 3 Nov 2023 23:54:44 -0400 Subject: [PATCH 4/9] UNDO prevent AOIs from populating database --- alembic/versions/5e03ce584f3c_add_eez.py | 1 + alembic/versions/c0bd1215a3ca_add_iho.py | 1 + alembic/versions/f9b7166c86b7_add_mpa.py | 1 + 3 files changed, 3 insertions(+) diff --git a/alembic/versions/5e03ce584f3c_add_eez.py b/alembic/versions/5e03ce584f3c_add_eez.py index e3d728df..f951dd2c 100644 --- a/alembic/versions/5e03ce584f3c_add_eez.py +++ b/alembic/versions/5e03ce584f3c_add_eez.py @@ -44,6 +44,7 @@ def upgrade() -> None: session = orm.Session(bind=bind) eez = get_eez_from_url() # geojson.load(open("EEZ_and_HighSeas_20230410.json")) + eez = {"features": []} # noqa for feat in eez.get("features"): sovereign_keys = [ k for k in list(feat["properties"].keys()) if k.startswith("SOVEREIGN") diff --git a/alembic/versions/c0bd1215a3ca_add_iho.py b/alembic/versions/c0bd1215a3ca_add_iho.py index df574d85..841805cb 100644 --- a/alembic/versions/c0bd1215a3ca_add_iho.py +++ b/alembic/versions/c0bd1215a3ca_add_iho.py @@ -36,6 +36,7 @@ def upgrade() -> None: session = orm.Session(bind=bind) iho = get_iho_from_url() + iho = {"features": []} # noqa for feat in iho.get("features"): with session.begin(): aoi_iho = database_schema.AoiIho( diff --git a/alembic/versions/f9b7166c86b7_add_mpa.py b/alembic/versions/f9b7166c86b7_add_mpa.py index a44e10d8..e2d7ac19 100644 --- a/alembic/versions/f9b7166c86b7_add_mpa.py +++ b/alembic/versions/f9b7166c86b7_add_mpa.py @@ -36,6 +36,7 @@ def upgrade() -> None: session = orm.Session(bind=bind) mpa = get_mpa_from_url() + mpa = {"features": []} # noqa for feat in mpa.get("features"): with session.begin(): aoi_mpa = database_schema.AoiMpa( From 268af148ce6bbc064611f3cdaaf9b6c4d23c3a36 Mon Sep 17 00:00:00 2001 From: Jona Date: Sat, 4 Nov 2023 00:35:01 -0400 Subject: [PATCH 5/9] oops D: --- .../cloud_run_orchestrator/handler.py | 2 - notebooks/run_sr_on_scene_ids.ipynb | 50 ++++++++++++++++++- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/cerulean_cloud/cloud_run_orchestrator/handler.py b/cerulean_cloud/cloud_run_orchestrator/handler.py index d0ad3daf..c2a5aeec 100644 --- a/cerulean_cloud/cloud_run_orchestrator/handler.py +++ b/cerulean_cloud/cloud_run_orchestrator/handler.py @@ -383,14 +383,12 @@ async def _orchestrate( base_tiles_inference = await perform_inference( base_tiles, cloud_run_inference.get_base_tile_inference, - 100, "base tiles", ) offset_tiles_inference = await perform_inference( offset_tiles_bounds, cloud_run_inference.get_offset_tile_inference, - 100, "offset tiles", ) diff --git a/notebooks/run_sr_on_scene_ids.ipynb b/notebooks/run_sr_on_scene_ids.ipynb index c9ba40a8..0d18a912 100644 --- a/notebooks/run_sr_on_scene_ids.ipynb +++ b/notebooks/run_sr_on_scene_ids.ipynb @@ -264,7 +264,7 @@ "source": [ "import httpx\n", "import json\n", - "def stage(scene_ids, API_KEY):\n", + "def stage(scene_ids, API_KEY=\"\"):\n", " URL = \"https://europe-west1-cerulean-338116.cloudfunctions.net/cerulean-cloud-staging-cloud-function-sr\"\n", " payload = {\n", " \"Records\": [{\n", @@ -279,13 +279,59 @@ " json=payload,\n", " timeout=100,\n", " headers={\"Authorization\": f\"Bearer {API_KEY}\", \"Content-Type\": \"application/json\"})\n", + " print(orchestrator_result)\n", + "def test(scene_ids, API_KEY=\"\"):\n", + " URL = \"https://europe-west1-cerulean-338116.cloudfunctions.net/cerulean-cloud-test-cloud-function-sr\"\n", + " payload = {\n", + " \"Records\": [{\n", + " \"Sns\": {\n", + " \"Message\": json.dumps({\n", + " \"id\": s,\n", + " \"footprint\": {\"coordinates\": [[[[0, 0], [0, 1], [1, 1], [1, 0], [0, 0]]]]}})}}\n", + " for s in scene_ids]}\n", + "\n", + " orchestrator_result = httpx.post(\n", + " URL,\n", + " json=payload,\n", + " timeout=100,\n", + " headers={\"Authorization\": f\"Bearer {API_KEY}\", \"Content-Type\": \"application/json\"})\n", " print(orchestrator_result)" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "test(slicks)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { + "kernelspec": { + "display_name": "pulumi_env", + "language": "python", + "name": "python3" + }, "language_info": { - "name": "python" + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.18" } }, "nbformat": 4, From 9b3812f011dea6e044ac619ed5d5637077dcb951 Mon Sep 17 00:00:00 2001 From: Jona Date: Mon, 6 Nov 2023 09:11:19 -0500 Subject: [PATCH 6/9] Missing scale parameter in offset? --- cerulean_cloud/cloud_run_orchestrator/clients.py | 12 ++++++++++-- cerulean_cloud/titiler_client.py | 2 ++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/cerulean_cloud/cloud_run_orchestrator/clients.py b/cerulean_cloud/cloud_run_orchestrator/clients.py index 9e087fda..5838b8d6 100644 --- a/cerulean_cloud/cloud_run_orchestrator/clients.py +++ b/cerulean_cloud/cloud_run_orchestrator/clients.py @@ -73,7 +73,10 @@ async def get_base_tile_inference( ) -> InferenceResultStack: """fetch inference for base tiles""" img_array = await self.titiler_client.get_base_tile( - sceneid=self.sceneid, tile=tile, scale=self.scale, rescale=rescale + sceneid=self.sceneid, + tile=tile, + scale=self.scale, + rescale=rescale, ) img_array = reshape_as_raster(img_array) @@ -108,7 +111,12 @@ async def get_offset_tile_inference( """fetch inference for offset tiles""" hw = self.scale * 256 img_array = await self.titiler_client.get_offset_tile( - self.sceneid, *bounds, width=hw, height=hw, rescale=rescale + self.sceneid, + *bounds, + width=hw, + height=hw, + scale=self.scale, + rescale=rescale, ) img_array = reshape_as_raster(img_array) with self.aux_datasets.open() as src: diff --git a/cerulean_cloud/titiler_client.py b/cerulean_cloud/titiler_client.py index 33d8a1da..cd643978 100644 --- a/cerulean_cloud/titiler_client.py +++ b/cerulean_cloud/titiler_client.py @@ -144,6 +144,7 @@ async def get_offset_tile( height: int = 256, band: str = "vv", img_format: str = "png", + scale: int = 1, rescale: Tuple[int, int] = (0, 255), ) -> np.ndarray: """get offset tile as numpy array (with bounds) @@ -169,6 +170,7 @@ async def get_offset_tile( ) url += f"?sceneid={sceneid}" url += f"&bands={band}" + url += f"&scale={scale}" url += f"&rescale={','.join([str(r) for r in rescale])}" resp = await self.client.get(url, timeout=self.timeout) From 5305624a47984c48511d169bf8faab53b35acb7d Mon Sep 17 00:00:00 2001 From: Jona Date: Mon, 6 Nov 2023 21:19:14 -0500 Subject: [PATCH 7/9] fix broken test --- test/test_cerulean_cloud/test_titiler_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_cerulean_cloud/test_titiler_client.py b/test/test_cerulean_cloud/test_titiler_client.py index 929b7265..ee058a38 100644 --- a/test/test_cerulean_cloud/test_titiler_client.py +++ b/test/test_cerulean_cloud/test_titiler_client.py @@ -116,7 +116,7 @@ async def test_offset_tile(titiler_client, tiles_s1_scene, httpx_mock): httpx_mock.add_response( method="GET", url=titiler_client.url - + f"crop/{minx},{miny},{maxx},{maxy}/256x256.png?sceneid={sceneid}&bands=vv&rescale=0,255", + + f"crop/{minx},{miny},{maxx},{maxy}/256x256.png?sceneid={sceneid}&bands=vv&scale=1&rescale=0,255", content=img_bytes, ) From 94a45f837737b0416a8e5e76a9db6c354a39b64c Mon Sep 17 00:00:00 2001 From: Jona Date: Mon, 6 Nov 2023 22:33:31 -0500 Subject: [PATCH 8/9] All GDFs need to be in the same CRS --- cerulean_cloud/cloud_run_orchestrator/merging.py | 2 ++ test/test_cerulean_cloud/fixtures/whole_world.geojson | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cerulean_cloud/cloud_run_orchestrator/merging.py b/cerulean_cloud/cloud_run_orchestrator/merging.py index 10aa8cbb..85a66543 100644 --- a/cerulean_cloud/cloud_run_orchestrator/merging.py +++ b/cerulean_cloud/cloud_run_orchestrator/merging.py @@ -51,6 +51,8 @@ def merge_inferences( return geojson.FeatureCollection(features=[]) # Concat the GeoDataFrames + target_crs = gdfs_for_processing[0].crs + gdfs_for_processing = [gdf.to_crs(target_crs) for gdf in gdfs_for_processing] concat_gdf = pd.concat(gdfs_for_processing, ignore_index=True) final_gdf = concat_gdf.copy() diff --git a/test/test_cerulean_cloud/fixtures/whole_world.geojson b/test/test_cerulean_cloud/fixtures/whole_world.geojson index 5b5cbe27..add66a7f 100644 --- a/test/test_cerulean_cloud/fixtures/whole_world.geojson +++ b/test/test_cerulean_cloud/fixtures/whole_world.geojson @@ -1,6 +1,6 @@ { "type": "FeatureCollection", "features": [ - {"type": "Feature", "geometry":{"type": "Polygon", "coordinates": [[[179.9, -89.9], [179.9, 89.9], [-179.9, 89.9], [-179.9, -89.9], [179.9, -89.9]]]}, "properties": {}} + {"type": "Feature", "geometry":{"type": "Polygon", "coordinates": [[[179.999, -89.999], [179.999, 89.999], [-179.999, 89.999], [-179.999, -89.999], [179.999, -89.999]]]}, "properties": {}} ] } \ No newline at end of file From 97f4f511e170cb710f623dbe409623d610573a6b Mon Sep 17 00:00:00 2001 From: Jona Date: Tue, 7 Nov 2023 10:09:28 -0500 Subject: [PATCH 9/9] Reverse TestDB changes --- alembic/versions/5e03ce584f3c_add_eez.py | 1 - alembic/versions/c0bd1215a3ca_add_iho.py | 1 - alembic/versions/f9b7166c86b7_add_mpa.py | 1 - 3 files changed, 3 deletions(-) diff --git a/alembic/versions/5e03ce584f3c_add_eez.py b/alembic/versions/5e03ce584f3c_add_eez.py index f951dd2c..e3d728df 100644 --- a/alembic/versions/5e03ce584f3c_add_eez.py +++ b/alembic/versions/5e03ce584f3c_add_eez.py @@ -44,7 +44,6 @@ def upgrade() -> None: session = orm.Session(bind=bind) eez = get_eez_from_url() # geojson.load(open("EEZ_and_HighSeas_20230410.json")) - eez = {"features": []} # noqa for feat in eez.get("features"): sovereign_keys = [ k for k in list(feat["properties"].keys()) if k.startswith("SOVEREIGN") diff --git a/alembic/versions/c0bd1215a3ca_add_iho.py b/alembic/versions/c0bd1215a3ca_add_iho.py index 841805cb..df574d85 100644 --- a/alembic/versions/c0bd1215a3ca_add_iho.py +++ b/alembic/versions/c0bd1215a3ca_add_iho.py @@ -36,7 +36,6 @@ def upgrade() -> None: session = orm.Session(bind=bind) iho = get_iho_from_url() - iho = {"features": []} # noqa for feat in iho.get("features"): with session.begin(): aoi_iho = database_schema.AoiIho( diff --git a/alembic/versions/f9b7166c86b7_add_mpa.py b/alembic/versions/f9b7166c86b7_add_mpa.py index e2d7ac19..a44e10d8 100644 --- a/alembic/versions/f9b7166c86b7_add_mpa.py +++ b/alembic/versions/f9b7166c86b7_add_mpa.py @@ -36,7 +36,6 @@ def upgrade() -> None: session = orm.Session(bind=bind) mpa = get_mpa_from_url() - mpa = {"features": []} # noqa for feat in mpa.get("features"): with session.begin(): aoi_mpa = database_schema.AoiMpa(