RTC-S1 ISO XML Measured Parameters Rollout

.ci/scripts/rtc_s1/opera_pge_rtc_s1_delivery_5.2_final_runconfig.yaml

@@ -15,21 +15,16 @@ RunConfig:
             DynamicAncillaryFilesGroup:
                 AncillaryFileMap:
                     dem_file: /home/rtc_user/input_dir/dem.tif
-
                     burst_database_file: /home/rtc_user/input_dir/burst_db_0.2.0_230831-bbox-only.sqlite
 
             ProductPathGroup:
                 OutputProductPath: /home/rtc_user/output_dir
-
                 ScratchPath: /home/rtc_user/scratch_dir
 
             PrimaryExecutable:
                 ProductIdentifier: RTC_S1
-
                 ProductVersion: "1.0"
-
                 ProgramPath: conda
-
                 ProgramOptions:
                     - run
                     - --no-capture-output
@@ -38,23 +33,18 @@ RunConfig:
                     - rtc_s1.py
 
                 ErrorCodeBase: 300000
-
                 SchemaPath: /home/rtc_user/opera/pge/rtc_s1/schema/rtc_s1_sas_schema.yaml
-
                 IsoTemplatePath: /home/rtc_user/opera/pge/rtc_s1/templates/OPERA_ISO_metadata_L2_RTC_S1_template.xml.jinja2
-
+                IsoMeasuredParameterDescriptions: /home/rtc_user/opera/pge/rtc_s1/templates/rtc_s1_measured_parameters.yaml
                 DataValidityStartDate: 20140403
 
             QAExecutable:
                 Enabled: True
-
                 ProgramPath: /home/rtc_user/opera/.ci/scripts/rtc_s1/compare_rtc_s1_products.sh
-
                 ProgramOptions: []
 
             DebugLevelGroup:
                 DebugSwitch: False
-
                 ExecuteViaShell: False
 
         SAS:
@@ -95,29 +85,17 @@ RunConfig:
 
                     product_group:
                         product_version: "1.0"
-
                         product_path: /home/rtc_user/output_dir
-
                         scratch_path: /home/rtc_user/scratch_dir
-
                         output_dir: /home/rtc_user/output_dir
-
                         product_id:
-
                         rtc_s1_static_validity_start_date: 20140403
-
                         product_data_access: "https://search.asf.alaska.edu/#/?dataset=OPERA-S1&productTypes=RTC"
-
                         static_layers_data_access: "https://search.asf.alaska.edu/#/?dataset=OPERA-S1&productTypes=RTC-STATIC&operaBurstID={burst_id}&end={end_date}"
-
                         save_bursts: True
-
                         save_mosaics: False
-
                         save_browse: True
-
                         output_imagery_format: COG
-
                         save_metadata: True
 
                     primary_executable:
@@ -126,21 +104,15 @@ RunConfig:
                     processing:
 
                         check_ancillary_inputs_coverage: True
-
                         polarization: dual-pol
-
                         rtc:
                             output_type: gamma0
-
                         num_workers: 0
-
                         geocoding:
                             memory_mode: auto
-
                             estimated_geometric_accuracy_bias_x: -0.72
                             estimated_geometric_accuracy_bias_y: -0.67
                             estimated_geometric_accuracy_stddev_x: 0.70
                             estimated_geometric_accuracy_stddev_y: 0.62
-
                         mosaicking:
                             mosaic_mode: first

.ci/scripts/rtc_s1/opera_pge_rtc_s1_static_delivery_5.2_final_runconfig.yaml

@@ -15,21 +15,16 @@ RunConfig:
             DynamicAncillaryFilesGroup:
                 AncillaryFileMap:
                     dem_file: /home/rtc_user/input_dir/dem.tif
-
                     burst_database_file: /home/rtc_user/input_dir/burst_db_0.2.0_230831-bbox-only.sqlite
 
             ProductPathGroup:
                 OutputProductPath: /home/rtc_user/output_dir
-
                 ScratchPath: /home/rtc_user/scratch_dir
 
             PrimaryExecutable:
                 ProductIdentifier: RTC_S1
-
                 ProductVersion: "1.0"
-
                 ProgramPath: conda
-
                 ProgramOptions:
                     - run
                     - --no-capture-output
@@ -38,23 +33,18 @@ RunConfig:
                     - rtc_s1.py
 
                 ErrorCodeBase: 300000
-
                 SchemaPath: /home/rtc_user/opera/pge/rtc_s1/schema/rtc_s1_sas_schema.yaml
-
                 IsoTemplatePath: /home/rtc_user/opera/pge/rtc_s1/templates/OPERA_ISO_metadata_L2_RTC_S1_template.xml.jinja2
-
+                IsoMeasuredParameterDescriptions: /home/rtc_user/opera/pge/rtc_s1/templates/rtc_s1_measured_parameters.yaml
                 DataValidityStartDate: 20140403
 
             QAExecutable:
                 Enabled: True
-
                 ProgramPath: /home/rtc_user/opera/.ci/scripts/rtc_s1/compare_rtc_s1_static_products.sh
-
                 ProgramOptions: []
 
             DebugLevelGroup:
                 DebugSwitch: False
-
                 ExecuteViaShell: False
 
         SAS:
@@ -95,52 +85,33 @@ RunConfig:
 
                     product_group:
                         product_version: "1.0"
-
                         product_path: /home/rtc_user/output_dir
-
                         scratch_path: /home/rtc_user/scratch_dir
-
                         output_dir: /home/rtc_user/output_dir
-
                         product_id:
-
                         rtc_s1_static_validity_start_date: 20140403
-
                         product_data_access: "https://search.asf.alaska.edu/#/?dataset=OPERA-S1&productTypes=RTC"
-
                         static_layers_data_access: "https://search.asf.alaska.edu/#/?dataset=OPERA-S1&productTypes=RTC-STATIC&operaBurstID={burst_id}&end={end_date}"
-
                         save_bursts: True
-
                         save_mosaics: False
-
                         save_browse: True
-
                         output_imagery_format: COG
-
                         save_metadata: True
 
                     primary_executable:
                         product_type: RTC_S1_STATIC
 
                     processing:
-
                         check_ancillary_inputs_coverage: True
-
                         polarization: dual-pol
-
                         rtc:
                             output_type: gamma0
-
                         num_workers: 2
-
                         geocoding:
                             memory_mode: auto
-
                             estimated_geometric_accuracy_bias_x: -0.72
                             estimated_geometric_accuracy_bias_y: -0.67
                             estimated_geometric_accuracy_stddev_x: 0.70
                             estimated_geometric_accuracy_stddev_y: 0.62
-
                         mosaicking:
                             mosaic_mode: first

examples/rtc_s1_sample_runconfig-v2.1.1.yaml

@@ -92,6 +92,14 @@ RunConfig:
                 # Consult the Docker image build scripts for more info
                 IsoTemplatePath: /home/rtc_user/opera/pge/rtc_s1/templates/OPERA_ISO_metadata_L2_RTC_S1_template.xml.jinja2
 
+                # Path to a YAML file mapping Measured Parameter metadata names to descriptions
+                # used to supplement the ISO xml metadata file
+                # Path should correspond to the file system within the Docker
+                # container, and typically should reference a template file bundled
+                # with the opera_pge installation directory within the container
+                # Consult the Docker image build scripts for more info
+                IsoMeasuredParameterDescriptions: /home/rtc_user/opera/pge/rtc_s1/templates/rtc_s1_measured_parameters.yaml
+
                 # Date field which designates the point after which the
                 # RTC static layer product(s) should be considered valid.
                 # This field must be provided for RTC-S1 jobs when static layer

src/opera/pge/base/base_pge.py

@@ -9,6 +9,7 @@
 
 """
 
+import html
 import json
 import os
 from collections import OrderedDict
@@ -28,7 +29,9 @@
 from opera.util.logger import PgeLogger
 from opera.util.logger import default_log_file_name
 from opera.util.metfile import MetFile
-from opera.util.render_jinja2 import python_type_to_xml_type, guess_attribute_display_name
+from opera.util.render_jinja2 import (python_type_to_xml_type,
+                                      guess_attribute_display_name,
+                                      NumpyEncoder)
 from opera.util.run_utils import create_qa_command_line
 from opera.util.run_utils import create_sas_command_line
 from opera.util.run_utils import get_checksum
@@ -670,7 +673,7 @@ def augment_measured_parameters(self, measured_parameters):
                 value = value.tolist()
 
             if isinstance(value, (list, dict)):
-                value = json.dumps(value)
+                value = json.dumps(value, cls=NumpyEncoder)
 
             guessed_data_type = python_type_to_xml_type(value)
             guessed_attr_name = guess_attribute_display_name(name)
@@ -681,9 +684,15 @@ def augment_measured_parameters(self, measured_parameters):
             data_type = descriptions[name].get('attribute_data_type', guessed_data_type)
             attr_type = descriptions[name].get('attribute_type', "!Not Found!")
             attr_name = descriptions[name].get('display_name', guessed_attr_name)
+            escape_html = descriptions[name].get('escape_html', False)
 
-            augmented_parameters[name] = (dict(name=attr_name, value=value, attr_type=attr_type,
-                                               attr_description=attr_description, data_type=data_type))
+            if escape_html:
+                value = html.escape(value)
+
+            augmented_parameters[name] = (
+                dict(name=attr_name, value=value, attr_type=attr_type,
+                     attr_description=attr_description, data_type=data_type)
+            )
 
         return augmented_parameters
 

src/opera/pge/base/schema/iso_metadata_measured_parameters_config_schema.yaml

@@ -20,3 +20,7 @@ parameter:
   # handle some acronyms improperly (ie, DSWx, MGRS, RTC) and some desired display names retain some underscores. Use
   # this field to correct these errors where needed.
   display_name: str(required=False)
+
+  # OPTIONAL: Set this flag to true if the value of the attribute should be escaped for HTML characters prior to being
+  # written to the output XML. Applies to string and string-like attributes only.
+  escape_html: bool(required=False)

src/opera/pge/rtc_s1/rtc_s1_pge.py

@@ -15,6 +15,8 @@
 from os import walk
 from os.path import basename, getsize, join
 
+import yaml
+
 from opera.pge.base.base_pge import PgeExecutor
 from opera.pge.base.base_pge import PostProcessorMixin
 from opera.pge.base.base_pge import PreProcessorMixin
@@ -23,6 +25,7 @@
 from opera.util.error_codes import ErrorCode
 from opera.util.geo_utils import translate_utm_bbox_to_lat_lon
 from opera.util.h5_utils import get_rtc_s1_product_metadata
+from opera.util.h5_utils import MEASURED_PARAMETER_PATH_SEPARATOR
 from opera.util.input_validation import validate_slc_s1_inputs
 from opera.util.render_jinja2 import render_jinja2
 from opera.util.time import get_time_for_filename
@@ -695,6 +698,59 @@ def _qa_log_filename(self):
         """
         return self._ancillary_filename() + ".qa.log"
 
+    def augment_measured_parameters(self, measured_parameters):
+        """
+        Override of the augment_measured_parameters() method in Base PGE with an added
+        "preprocessing" step to handle the structure of HDF5 metadata. While GeoTIFF
+        metadata is a flat dictionary, HDF5 metadata is a nested dictionary structure,
+        wherein the variable "keys" can be arbitrarily deep into the structure and
+        the values likewise can be nested dictionaries.
+
+        The preprocessing step in this method selectively flattens the metadata
+        dictionary based on the "paths" provided in the variable keys of the configuration
+        YAML file. The result of this preprocessing is then safely passed to the base
+        method to get the correct structure expected by the Jinja template.
+
+        Parameters
+        ----------
+        measured_parameters : dict
+            The HDF5 metadata from the output product. See get_rtc_s1_product_metadata()
+
+        Returns
+        -------
+        augmented_parameters : dict
+            The metadata fields converted to a list with name, value, types, etc
+        """
+        descriptions_file = self.runconfig.iso_measured_parameter_descriptions
+
+        new_measured_parameters = {}
+
+        if descriptions_file:
+            with open(descriptions_file) as infile:
+                descriptions = yaml.safe_load(infile)
+        else:
+            msg = ('Measured parameters configuration is needed to extract the '
+                   'measured parameters attributes from the RTC-S1 metadata')
+            self.logger.critical(self.name, ErrorCode.ISO_METADATA_DESCRIPTIONS_CONFIG_NOT_FOUND, msg)
+
+        for parameter_var_name in descriptions:
+            key_path = parameter_var_name.split(MEASURED_PARAMETER_PATH_SEPARATOR)
+
+            mp = measured_parameters
+
+            while len(key_path) > 0:
+                try:
+                    mp = mp[key_path.pop(0)]
+                    new_measured_parameters[parameter_var_name] = mp
+                except KeyError as err:
+                    msg = (f'Measured parameters contains no entry for description '
+                           f'key "{err}"')
+                    self.logger.warning(self.name, ErrorCode.ISO_METADATA_NO_ENTRY_FOR_DESCRIPTION, msg)
+
+        augmented_parameters = super().augment_measured_parameters(new_measured_parameters)
+
+        return augmented_parameters
+
     def _collect_rtc_product_metadata(self, metadata_product):
         """
         Gathers the available metadata from an HDF5 product created by
@@ -753,6 +809,9 @@ def _collect_rtc_product_metadata(self, metadata_product):
             except ValueError as err:
                 self.logger.critical(self.name, ErrorCode.ISO_METADATA_RENDER_FAILED, str(err))
 
+        # Augment the metadata with descriptions from the measured parameter config for RTC-S1
+        output_product_metadata['MeasuredParameters'] = self.augment_measured_parameters(output_product_metadata)
+
         return output_product_metadata
 
     def _create_custom_metadata(self):