Skip to content

Commit

Permalink
Fix a bug in AAA add to queue and database construction
Browse files Browse the repository at this point in the history
  • Loading branch information
@jonaraphael
jonaraphael committed Oct 25, 2023
1 parent fd759b7 commit 2a4894e
Show file tree
Hide file tree
Showing 3 changed files with 90 additions and 38 deletions.
88 changes: 81 additions & 7 deletions alembic/versions/3736e85bc273_add_exposed_funcs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,6 @@ def upgrade() -> None:
OUT fill_factor double precision,
OUT s1_scene_id character varying,
OUT s1_geometry geography,
OUT cls_id integer,
OUT cls_short_name text,
OUT cls_long_name text,
OUT aoi_type_1_ids bigint[],
Expand All @@ -76,7 +75,31 @@ def upgrade() -> None:
IMMUTABLE PARALLEL SAFE
ROWS 1000
AS $BODY$
select distinct sp.*
select distinct
sp.id,
sp.linearity,
sp.slick_timestamp,
sp.geometry,
sp.active,
sp.orchestrator_run,
sp.create_time,
sp.inference_idx,
sp.cls,
sp.hitl_cls,
sp.machine_confidence,
sp.length,
sp.area,
sp.perimeter,
sp.centroid,
sp.polsby_popper,
sp.fill_factor,
sp.s1_scene_id,
sp.s1_geometry,
sp.cls_short_name,
sp.cls_long_name,
sp.aoi_type_1_ids,
sp.aoi_type_2_ids,
sp.aoi_type_3_ids
FROM public.slick_plus sp
LEFT JOIN slick_to_source sts ON sts.slick = sp.id AND source_id != 'NULL'
LEFT JOIN slick_to_aoi sta ON sta.slick = sp.id AND aoi_id != 'NULL'
Expand Down Expand Up @@ -110,7 +133,6 @@ def upgrade() -> None:
OUT fill_factor double precision,
OUT s1_scene_id character varying,
OUT s1_geometry geography,
OUT cls_id integer,
OUT cls_short_name text,
OUT cls_long_name text,
OUT aoi_type_1_ids bigint[],
Expand All @@ -123,7 +145,31 @@ def upgrade() -> None:
IMMUTABLE PARALLEL SAFE
ROWS 1000
AS $BODY$
select distinct sp.*
select distinct
sp.id,
sp.linearity,
sp.slick_timestamp,
sp.geometry,
sp.active,
sp.orchestrator_run,
sp.create_time,
sp.inference_idx,
sp.cls,
sp.hitl_cls,
sp.machine_confidence,
sp.length,
sp.area,
sp.perimeter,
sp.centroid,
sp.polsby_popper,
sp.fill_factor,
sp.s1_scene_id,
sp.s1_geometry,
sp.cls_short_name,
sp.cls_long_name,
sp.aoi_type_1_ids,
sp.aoi_type_2_ids,
sp.aoi_type_3_ids
FROM public.slick_plus sp
JOIN slick_to_source sts ON sts.slick = sp.id
WHERE sts.source = ANY(string_to_array(source_id, ',')::int[])
Expand Down Expand Up @@ -155,7 +201,6 @@ def upgrade() -> None:
OUT fill_factor double precision,
OUT s1_scene_id character varying,
OUT s1_geometry geography,
OUT cls_id integer,
OUT cls_short_name text,
OUT cls_long_name text,
OUT aoi_type_1_ids bigint[],
Expand All @@ -168,7 +213,31 @@ def upgrade() -> None:
IMMUTABLE PARALLEL SAFE
ROWS 1000
AS $BODY$
select distinct sp.*
select distinct
sp.id,
sp.linearity,
sp.slick_timestamp,
sp.geometry,
sp.active,
sp.orchestrator_run,
sp.create_time,
sp.inference_idx,
sp.cls,
sp.hitl_cls,
sp.machine_confidence,
sp.length,
sp.area,
sp.perimeter,
sp.centroid,
sp.polsby_popper,
sp.fill_factor,
sp.s1_scene_id,
sp.s1_geometry,
sp.cls_short_name,
sp.cls_long_name,
sp.aoi_type_1_ids,
sp.aoi_type_2_ids,
sp.aoi_type_3_ids
FROM public.slick_plus sp
JOIN slick_to_aoi sta ON sta.slick = sp.id
WHERE sta.aoi = ANY(string_to_array(aoi_id, ',')::int[]);
Expand Down Expand Up @@ -213,7 +282,12 @@ def downgrade() -> None:
)
op.execute(
"""
DROP FUNCTION public.get_slicks_by_aoi(text, integer);
DROP FUNCTION public.get_slicks_by_aoi(text);
"""
)
op.execute(
"""
DROP FUNCTION public.get_slicks_by_aoi_or_source(text, text, integer);
"""
)
get_history_slick = PGFunction(
Expand Down
36 changes: 7 additions & 29 deletions alembic/versions/c941681a050d_add_initial_records.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,50 +69,28 @@ def upgrade() -> None:
models = [
database_schema.Model(
type="MASKRCNN",
file_path="experiments/2023_06_26_09_34_18_4cls_rn34_pr512_px1024_720min_maskrcnn/scripting_cpu_model.pt",
file_path="experiments/2023_10_05_02_22_46_4cls_rnxt101_pr512_px1024_680min_maskrcnn_wd01/scripting_cpu_model.pt",
layers=["VV", "INFRA", "VESSEL"],
cls_map={
0: "BACKGROUND",
1: "INFRA",
2: "NATURAL",
3: "VESSEL",
}, # inference_idx maps to class table
name="Dummy for testing",
tile_width_m=20422,
name="ResNext 101 hires56",
tile_width_m=40844,
tile_width_px=512,
epochs=500,
epochs=122,
thresholds={
"pixel_nms_thresh": 0.4,
"bbox_score_thresh": 0.2,
"poly_score_thresh": 0.2,
"pixel_score_thresh": 0.2,
"groundtruth_dice_thresh": 0.0,
},
backbone_size=34,
pixel_f1=0.466,
instance_f1=0.455,
),
database_schema.Model(
type="MASKRCNN",
file_path="experiments/20_Jul_2022_00_14_15_icevision_maskrcnn/scripting_cpu_test_28_34_224_58.pt",
layers=["VV", "INFRA", "VESSEL"],
cls_map={
0: "BACKGROUND",
1: "INFRA",
2: "VESSEL",
3: "NATURAL",
}, # inference_idx maps to class table
name="OLD, Not working so well",
tile_width_m=20422,
tile_width_px=224,
thresholds={
"pixel_nms_thresh": 0.4,
"bbox_score_thresh": 0.2,
"poly_score_thresh": 0.2,
"pixel_score_thresh": 0.2,
"groundtruth_dice_thresh": 0.0,
},
backbone_size=34,
backbone_size=101,
pixel_f1=0.461,
instance_f1=0.47,
),
]
session.add_all(models)
Expand Down
4 changes: 2 additions & 2 deletions cerulean_cloud/cloud_function_ais_analysis/queuer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"""
import json
import os
from datetime import datetime
from datetime import datetime, timedelta, timezone

from google.cloud import tasks_v2
from google.protobuf import timestamp_pb2
Expand Down Expand Up @@ -56,7 +56,7 @@ def add_to_aaa_queue(scene_id):
# Each entry is another retry
ais_delays = [0, 3, 7] # TODO Magic number >>> Where should this live?
for delay in ais_delays:
d = datetime.now(tz=datetime.timezone.utc) + datetime.timedelta(days=delay)
d = datetime.now(tz=timezone.utc) + timedelta(days=delay)

# Create Timestamp protobuf.
timestamp = timestamp_pb2.Timestamp()
Expand Down

0 comments on commit 2a4894e

Please sign in to comment.