Merge branch 'develop' into 923_disp_s1_forward_processing

cluster_provisioning/dev-e2e/variables.tf

@@ -388,7 +388,7 @@ variable "pge_releases" {
     "dswx_hls" = "1.0.2"
     "cslc_s1"  = "2.1.1"
     "rtc_s1"   = "2.1.1"
-    "dswx_s1"  = "3.0.0-rc.2.1"
+    "dswx_s1"  = "3.0.0"
     "disp_s1"  = "3.0.0-rc.2.2"
     "dswx_ni"  = "4.0.0-er.2.0"
   }
@@ -513,8 +513,8 @@ variable "amis" {
     mozart    = "ami-024158daea696c58d" # mozart v4.25 - 240715
     metrics   = "ami-0ec9395bcd3c9b44c" # metrics v4.17 - 240715
     grq       = "ami-059583a64a2daab95" # grq v4.18 - 240715
-    factotum  = "ami-09a4f8d2ac5190255" # factotum v4.16 - 240715 
-    autoscale = "ami-09f953f51f1d84913" # verdi v4.16 patchdate - 240717
+    factotum  = "ami-09a4f8d2ac5190255" # factotum v4.16 - 240715
+    autoscale = "ami-05d1383b9765b51db" # verdi v4.16 patchdate - 240715
   }
 }
 

cluster_provisioning/dev-int/override.tf

@@ -175,7 +175,7 @@ variable "pge_releases" {
     "dswx_hls" = "1.0.2"
     "cslc_s1"  = "2.1.1"
     "rtc_s1"   = "2.1.1"
-    "dswx_s1"  = "3.0.0-rc.2.1"
+    "dswx_s1"  = "3.0.0"
     "disp_s1"  = "3.0.0-rc.2.2"
     "dswx_ni"  = "4.0.0-er.2.0"
   }

cluster_provisioning/dev/variables.tf

@@ -389,7 +389,7 @@ variable "pge_releases" {
     "dswx_hls" = "1.0.2"
     "cslc_s1"  = "2.1.1"
     "rtc_s1"   = "2.1.1"
-    "dswx_s1"  = "3.0.0-rc.2.1"
+    "dswx_s1"  = "3.0.0"
     "disp_s1"  = "3.0.0-rc.2.2"
     "dswx_ni"  = "4.0.0-er.2.0"
   }

cluster_provisioning/ebs-snapshot/variables.tf

@@ -33,7 +33,7 @@ variable "pge_releases" {
     "dswx_hls" = "1.0.2"
     "cslc_s1"  = "2.1.1"
     "rtc_s1"   = "2.1.1"
-    "dswx_s1"  = "3.0.0-rc.2.1"
+    "dswx_s1"  = "3.0.0"
     "disp_s1"  = "3.0.0-rc.2.2"
     "dswx_ni"  = "4.0.0-er.2.0"
   }

cluster_provisioning/int_pst_ops_devcommon_override.tf

@@ -6,25 +6,25 @@ variable "hysds_release" {
 }
 
 variable "lambda_package_release" {
-  default = "3.1.0-rc.4.0"
+  default = "3.0.0-rc.7.0"
 }
 
 variable "pcm_commons_branch" {
-  default = "3.1.0-rc.4.0"
+  default = "3.0.0-rc.7.0"
 }
 
 variable "pcm_branch" {
-  default = "3.1.0-rc.4.0"
+  default = "3.0.0-rc.7.0"
 }
 
 variable "product_delivery_branch" {
-  default = "3.1.0-rc.4.0"
+  default = "3.0.0-rc.7.0"
 }
 
 variable "bach_api_branch" {
-  default = "3.1.0-rc.4.0"
+  default = "3.0.0-rc.7.0"
 }
 
 variable "bach_ui_branch" {
-  default = "3.1.0-rc.4.0"
+  default = "3.0.0-rc.7.0"
 }

cluster_provisioning/modules/common/variables.tf

@@ -138,7 +138,6 @@ variable "amis" {
     grq       = "ami-059583a64a2daab95" # grq v4.18 - 240715
     factotum  = "ami-09a4f8d2ac5190255" # factotum v4.16 - 240715
     autoscale = "ami-05d1383b9765b51db" # verdi v4.16 patchdate - 240715
-    #autoscale = "ami-09f953f51f1d84913" # verdi v4.16 patchdate - 240717
   }
 }
 
@@ -593,7 +592,7 @@ variable "pge_releases" {
     "dswx_hls" = "1.0.2"
     "cslc_s1"  = "2.1.1"
     "rtc_s1"   = "2.1.1"
-    "dswx_s1"  = "3.0.0-rc.2.1"
+    "dswx_s1"  = "3.0.0"
     "disp_s1"  = "3.0.0-rc.2.2"
     "dswx_ni"  = "4.0.0-er.2.0"
   }

cluster_provisioning/modules/common/versions.tf

@@ -7,8 +7,7 @@ terraform {
     #   https://github.com/hashicorp/terraform-provider-aws/issues/10297
     aws = {
       source  = "hashicorp/aws"
-#      version = "2.63"
-      version = "2.70.1"
+      version = "3.75.2"
     }
     null = {
       source = "hashicorp/null"

conf/RunConfig.yaml.L3_DSWx_S1.jinja2.tmpl

@@ -61,7 +61,7 @@ RunConfig:
             worldcover_file_description: 'ESA WorldCover 10m 2020 v1.0'
             reference_water_file_description: 'JRC Global Surface Water - collection from 1984 to 2021'
             hand_file_description: 'ASF HAND GLO30'
-            shoreline_shapefile_description: 'NOAA GSHHS Level 1 resolution f - GSHHS_f_L1'
+            glad_classification_file_description: 'GLAD Global Land Cover 2020'
           static_ancillary_file_group:
             static_ancillary_inputs_flag: True
             {%- for type in data.static_ancillary_file_group.keys() %}
@@ -83,5 +83,11 @@ RunConfig:
             save_browse: True
             browse_image_height: 1024
             browse_image_width: 1024
+            flag_collapse_wtr_classes: True
+            exclude_inundated_vegetation: False
+            set_not_water_to_nodata: False
+            set_hand_mask_to_nodata: True
+            set_layover_shadow_to_nodata: True
+            set_ocean_masked_to_nodata: False
             save_tif_to_output: True
           log_file: {{ data.product_path_group.scratch_path }}/dswx-s1.log

conf/schema/RunConfig_schema.L3_DSWx_S1.yaml

@@ -75,6 +75,12 @@ RunConfig:
             # ESA WorldCover map description
             worldcover_file_description: str(required=False)
 
+            # GLAD classification map file
+            glad_classification_file: str(required=False)
+
+            # GLAD classification map file description
+            glad_classification_file_description: str(required=False)
+
             # NOAA GSHHS shapefile
             shoreline_shapefile: str(required=False)
 

conf/sds/rules/user_rules-cnm.json.tmpl

@@ -17,7 +17,7 @@
     {
       "enabled": true,
       "job_type": "hysds-io-send_notify_msg:__TAG__",
-      "kwargs": "{\"provider_name\":\"JPL-OPERA\",\"trace\":\"{{ TRACE }}\",\"checksum_type\":\"md5\",\"publisher_arn\":\"{{ PO_DAAC_PROXY }}\",\"sqs_queue_url\":\"{{ PO_DAAC_SQS_URL }}\",\"sqs_endpoint_url\":\"{{ PO_DAAC_ENDPOINT_URL }}\",\"staged_data_types\":{\"data\":[\"*B0*.tif\"],\"browse\":[\"*BROWSE.tif\",\"*.png\"],\"metadata\":[\"*.iso.xml\",\"*.md5\"]},\"use_s3_uri_structure\": \"{{ USE_S3_URI }}\",\"product_type_key\": \"CollectionName\",\"data_version\":\"0.0\",\"schema_version\":\"1.6.1\"}",
+      "kwargs": "{\"provider_name\":\"JPL-OPERA\",\"trace\":\"{{ TRACE }}\",\"checksum_type\":\"md5\",\"publisher_arn\":\"{{ PO_DAAC_PROXY }}\",\"sqs_queue_url\":\"{{ PO_DAAC_SQS_URL }}\",\"sqs_endpoint_url\":\"{{ PO_DAAC_ENDPOINT_URL }}\",\"staged_data_types\":{\"data\":[\"*B0*.tif\"],\"browse\":[\"*BROWSE.tif\",\"*.png\"],\"metadata\":[\"*.iso.xml\",\"*.md5\"]},\"use_s3_uri_structure\": \"{{ USE_S3_URI }}\",\"product_type_key\": \"CollectionName\",\"data_version\":\"1.0\",\"schema_version\":\"1.6.1\"}",
       "passthru_query": false,
       "priority": 0,
       "query_all": false,

conf/settings.yaml

@@ -23,8 +23,8 @@ CSLC_S1_STATIC_PRODUCT_VERSION: "1.0"
 RTC_S1_STATIC_PRODUCT_VERSION: "1.0"
 
 # DSWx-S1 & DISP-S1 in R3
-DSWX_S1_COLLECTION_NAME: "OPERA_L3_DSWX-S1_PROVISIONAL_V0"
-DSWX_S1_PRODUCT_VERSION: "0.4"
+DSWX_S1_COLLECTION_NAME: "OPERA_L3_DSWX-S1_V1"
+DSWX_S1_PRODUCT_VERSION: "1.0"
 DISP_S1_COLLECTION_NAME: "OPERA_L3_DISP-S1_PROVISIONAL_V0"
 DISP_S1_PRODUCT_VERSION: "0.3"
 
@@ -147,8 +147,6 @@ RTC_S1_STATIC:
     STDDEV_Y: 0.62
 
 DSWX_S1:
-  # Toggle application of ocean/shoreline masking during processing
-  APPLY_OCEAN_MASKING: !!bool false
   # Amount of margin in km to apply to staged ancillaries (DEM/Worldcover)
   ANCILLARY_MARGIN: 50
 

data_subscriber/rtc/rtc_download_job_submitter.py

@@ -1,4 +1,5 @@
 import asyncio
+import itertools
 import json
 import logging
 from functools import partial
@@ -8,6 +9,8 @@
 
 logger = logging.getLogger(__name__)
 
+flatten = itertools.chain.from_iterable
+
 
 async def example(batch_id_to_urls_map, args):
     """Sample usage function."""
@@ -33,6 +36,11 @@ def submit_rtc_download_job_submissions_tasks(batch_id_to_products_map, args, se
                 "metadata": {
                     "batch_id": mgrs_set_id_acquisition_ts_cycle_index,
                     "mgrs_set_id": mgrs_set_id,
+                    # for payload hash dedupe, include granule ID list (changes with improved coverage)
+                    "granule_ids": sorted({
+                        product["granule_id"]
+                        for product in flatten(batch_id_to_products_map[batch_id].values())
+                    })
                 }
             }
         }

docker/job-spec.json.SCIFLO_L3_DSWx_S1

@@ -10,8 +10,8 @@
   },
   "dependency_images": [
     {
-      "container_image_name": "opera_pge/dswx_s1:3.0.0-rc.2.1",
-      "container_image_url": "$CODE_BUCKET_URL/opera_pge-dswx_s1-3.0.0-rc.2.1.tar.gz",
+      "container_image_name": "opera_pge/dswx_s1:3.0.0",
+      "container_image_url": "$CODE_BUCKET_URL/opera_pge-dswx_s1-3.0.0.tar.gz",
       "container_mappings": {
         "$HOME/.netrc": ["/root/.netrc"],
         "$HOME/.aws": ["/root/.aws", "ro"]

docker/job-spec.json.pge_smoke_test

@@ -10,8 +10,8 @@
     },
     "dependency_images": [
         {
-          "container_image_name": "opera_pge/dswx_s1:3.0.0-rc.2.1",
-          "container_image_url": "$CODE_BUCKET_URL/opera_pge-dswx_s1-3.0.0-rc.2.1.tar.gz",
+          "container_image_name": "opera_pge/dswx_s1:3.0.0",
+          "container_image_url": "$CODE_BUCKET_URL/opera_pge-dswx_s1-3.0.0.tar.gz",
           "container_mappings": {
             "$HOME/.netrc": ["/root/.netrc"],
             "$HOME/.aws": ["/root/.aws", "ro"]

opera_chimera/configs/pge_configs/PGE_L3_DSWx_S1.yaml

@@ -20,8 +20,7 @@ runconfig:
     hand_file: __CHIMERA_VAL__
     worldcover_file: __CHIMERA_VAL__
     reference_water_file: __CHIMERA_VAL__
-    # TODO: enable if shoreline files are needed by DSWx-S1 SAS
-    # shoreline_shapefile: __CHIMERA_VAL__
+    glad_classification_file: __CHIMERA_VAL__
   static_ancillary_file_group:
     mgrs_database_file: __CHIMERA_VAL__
     mgrs_collection_database_file: __CHIMERA_VAL__
@@ -45,8 +44,6 @@ preconditions:
   - get_dswx_s1_num_workers
   - get_dswx_s1_algorithm_parameters
   - instantiate_algorithm_parameters_template
-  # TODO: enable if shoreline files are needed by DSWx-S1 SAS
-  #- get_shoreline_shapefiles
 
 # This lists all the postprocessor steps that this PGE will run after running the PGE.
 postprocess:
@@ -85,27 +82,18 @@ get_dswx_s1_dynamic_ancillary_maps:
     # TODO: The following water map is also available, need to confirm with ADT
     #       which will be used in the long run
     # s3_key: "2021/seasonality/seasonality_v1_4_2021.vrt"
+  glad_classification_file:
+    s3_bucket: "opera-land-cover"
+    s3_key: "GLCLU2000-2020/v2/2020/GLAD_2020_land_cover.vrt"
 
 get_dswx_s1_dem:
   # The s3 location containing the global DEM(s) to download regions from
   s3_bucket: "opera-dem"
   s3_key: "v1.1"
 
-get_shoreline_shapefiles:
-  # The s3 bucket containing the Shoreline shapefile data set
-  s3_bucket: "opera-dist2coast"
-  # Key paths to the files that comprise the Shoreline shapefile data set
-  # Typically should be a set of 4 files, one of which MUST be the .shp which
-  # gets configured within the PGE RunConfig
-  s3_keys:
-    - "GSHHS_f_L1.dbf"
-    - "GSHHS_f_L1.prj"
-    - "GSHHS_f_L1.shp"
-    - "GSHHS_f_L1.shx"
-
 get_dswx_s1_algorithm_parameters:
   s3_bucket: "opera-ancillaries"
-  s3_key: "algorithm_parameters/dswx_s1/dswx_s1_algorithm_parameters_calval_0.4.2.yaml.tmpl"
+  s3_key: "algorithm_parameters/dswx_s1/dswx_s1_algorithm_parameters_final_1.0.0.yaml.tmpl"
 
 instantiate_algorithm_parameters_template:
   template_mapping:

pge_smoke_test/run_pge_smoke_test.sh

@@ -23,7 +23,7 @@ date 2>&1
 
 echo "Working dir is ${WORKING_DIR}" 2>&1
 
-VERSION_TAG=$(docker images | grep ${PGE_NAME} -m 1 | xargs sh -c 'echo $1') 2>&1
+VERSION_TAG=$(docker images | grep opera_pge/${PGE_NAME} -m 1 | xargs sh -c 'echo $1') 2>&1
 STATUS=$?
 
 if [ $STATUS -ne 0 ]; then