From f392dd1800a292b98a8e31f49e6c1ae513f15a6c Mon Sep 17 00:00:00 2001 From: Jona Date: Thu, 26 Oct 2023 23:22:34 -0400 Subject: [PATCH] Tweak logging --- .../cloud_function_ais_analysis/main.py | 8 +++++ .../cloud_run_orchestrator/handler.py | 34 ++++++------------- stack/cloud_function_ais_analysis.py | 2 +- 3 files changed, 20 insertions(+), 24 deletions(-) diff --git a/cerulean_cloud/cloud_function_ais_analysis/main.py b/cerulean_cloud/cloud_function_ais_analysis/main.py index cf916ba1..f032a59b 100644 --- a/cerulean_cloud/cloud_function_ais_analysis/main.py +++ b/cerulean_cloud/cloud_function_ais_analysis/main.py @@ -2,6 +2,7 @@ """ import asyncio +import logging import os from json import loads @@ -54,6 +55,7 @@ async def handle_aaa_request(request): request_json = request.get_json() if not request_json.get("dry_run"): scene_id = request_json.get("scene_id") + logging.info(f"Running AAA on scene_id: {scene_id}") db_engine = get_engine(db_url=os.getenv("DB_URL")) async with DatabaseClient(db_engine) as db_client: async with db_client.session.begin(): @@ -61,20 +63,26 @@ async def handle_aaa_request(request): slicks_without_sources = ( await db_client.get_slicks_without_sources_from_scene_id(scene_id) ) + logging.info(f"# Slicks found: {len(slicks_without_sources)}") if len(slicks_without_sources) > 0: ais_constructor = AISConstructor(s1) ais_constructor.retrieve_ais() # ais_constructor.add_infra() + logging.info("AIS retrieved") if ( ais_constructor.ais_gdf is not None and not ais_constructor.ais_gdf.empty ): + logging.info("AIS is not empty") ais_constructor.build_trajectories() ais_constructor.buffer_trajectories() for slick in slicks_without_sources: ais_associations = automatic_ais_analysis( ais_constructor, slick ) + logging.info( + f"{len(ais_associations)} found for Slick ID: {slick.id}" + ) if len(ais_associations) > 0: # XXX What to do if len(ais_associations)==0 and no sources are associated? # Then it will trigger another round of this process later! (unnecessary computation) diff --git a/cerulean_cloud/cloud_run_orchestrator/handler.py b/cerulean_cloud/cloud_run_orchestrator/handler.py index 028877a4..4eb42265 100644 --- a/cerulean_cloud/cloud_run_orchestrator/handler.py +++ b/cerulean_cloud/cloud_run_orchestrator/handler.py @@ -230,8 +230,6 @@ def flatten_feature_list( async def _orchestrate( payload, tiler, titiler_client, roda_sentinelhub_client, db_engine ): - logging.basicConfig(level=logging.INFO) - # Orchestrate inference start_time = datetime.now() logging.info(f"Orchestrating for sceneid {payload.sceneid}") @@ -246,11 +244,11 @@ async def _orchestrate( logging.info(f"scale: {scale}") if model.zoom_level != zoom: - logging.info( + logging.warning( f"WARNING: Model was trained on zoom level {model.zoom_level} but is being run on {zoom}" ) if model.tile_width_px != scale * 256: - logging.info( + logging.warning( f"WARNING: Model was trained on image tile of resolution {model.tile_width_px} but is being run on {scale*256}" ) @@ -430,22 +428,23 @@ async def _orchestrate( features=flatten_feature_list(offset_tiles_inference) ) except AttributeError as e: - logging.info(f"YYY error details: {e}") - logging.info(f"YYY base_tiles_inference: {base_tiles_inference}") - logging.info( + logging.debug(f"YYY error details: {e}") + logging.debug(f"YYY base_tiles_inference: {base_tiles_inference}") + logging.debug( f"YYY offset_tiles_inference: {offset_tiles_inference}" ) - logging.info( + logging.debug( f"YYY [r for r in base_tiles_inference]: {[r for r in base_tiles_inference]}" ) - logging.info( + logging.debug( f"YYY [r for r in offset_tiles_inference]: {[r for r in offset_tiles_inference]}" ) + raise e # XXXBUG ValueError: Cannot determine common CRS for concatenation inputs, got ['WGS 84 / UTM zone 28N', 'WGS 84 / UTM zone 29N']. Use `to_crs()` to transform geometries to the same CRS before merging." # Example: S1A_IW_GRDH_1SDV_20230727T185101_20230727T185126_049613_05F744_1E56 - logging.info(f"XXXDEBUG out_fc: {out_fc}") - logging.info(f"XXXDEBUG out_fc_offset: {out_fc_offset}") + logging.debug(f"out_fc: {out_fc}") + logging.debug(f"out_fc_offset: {out_fc_offset}") merged_inferences = merge_inferences( out_fc, out_fc_offset, @@ -456,17 +455,6 @@ async def _orchestrate( ) for feat in merged_inferences.get("features"): - try: - logging.info(f"XXX CHRISTIAN type(feat): {type(feat)}") - logging.info( - f"XXX CHRISTIAN type(feat['geometry']): {type(feat['geometry'])}" - ) - logging.info(f"XXX CHRISTIAN feat['geometry']: {feat['geometry']}") - logging.info( - f"XXX CHRISTIAN geojson.dumps(feat['geometry']): {geojson.dumps(feat['geometry'])}" - ) - except: # noqa - pass async with db_client.session.begin(): # mini_gdf = gpd.GeoDataframe(feat) # if mini_gdf.intersects(land): @@ -499,7 +487,7 @@ async def _orchestrate( noffsettiles=noffsettiles, ) else: - logging.info("DRY RUN!!") + logging.warning("WARNING: Operating as a DRY RUN!!") orchestrator_result = OrchestratorResult( classification_base=geojson.FeatureCollection(features=[]), classification_offset=geojson.FeatureCollection(features=[]), diff --git a/stack/cloud_function_ais_analysis.py b/stack/cloud_function_ais_analysis.py index 491f6ea5..cb2afeae 100644 --- a/stack/cloud_function_ais_analysis.py +++ b/stack/cloud_function_ais_analysis.py @@ -88,7 +88,7 @@ source_archive_object=source_archive_object.name, trigger_http=True, service_account_email=cloud_function_service_account.email, - available_memory_mb=1024, + available_memory_mb=2048, timeout=540, secret_environment_variables=[gfw_credentials], )