From 133c80131a8325f6da2df42e82746b348a3f378d Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Wed, 31 May 2023 16:32:11 -0700 Subject: [PATCH 01/11] rename init_ to create_ to reflect functionality updated docstring --- src/compass/s1_geocode_metadata.py | 28 +++++++++++++--------------- src/compass/s1_geocode_slc.py | 6 +++--- src/compass/utils/h5_helpers.py | 9 +++++---- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/src/compass/s1_geocode_metadata.py b/src/compass/s1_geocode_metadata.py index f0a64bb9..0129c436 100755 --- a/src/compass/s1_geocode_metadata.py +++ b/src/compass/s1_geocode_metadata.py @@ -17,7 +17,7 @@ from compass.utils.geo_runconfig import GeoRunConfig from compass.utils.h5_helpers import (algorithm_metadata_to_h5group, identity_to_h5group, - init_geocoded_dataset, + create_geocoded_dataset, metadata_to_h5group, DATA_PATH, METADATA_PATH, ROOT_PATH) from compass.utils.helpers import bursts_grouping_generator, get_module_name @@ -56,10 +56,10 @@ def _fix_layover_shadow_mask(static_layers_dict, h5_root, geo_grid): # delete existing and rewrite with masked data del h5_root[layover_shadow_path] - _ = init_geocoded_dataset(h5_root[DATA_PATH], dst_ds_name, geo_grid, - dtype=None, - description=np.string_(dst_ds_name), - data=temp_arr) + _ = create_geocoded_dataset(h5_root[DATA_PATH], dst_ds_name, geo_grid, + dtype=None, + description=np.string_(dst_ds_name), + data=temp_arr) def run(cfg, burst, fetch_from_scratch=False): @@ -170,9 +170,9 @@ def run(cfg, burst, fetch_from_scratch=False): dtype = np.byte # Create dataset with x/y coords/spacing and projection - topo_ds = init_geocoded_dataset(static_layer_data_group, - dataset_name, geo_grid, dtype, - np.string_(dataset_name)) + topo_ds = create_geocoded_dataset(static_layer_data_group, + dataset_name, geo_grid, dtype, + np.string_(dataset_name)) # Init output and input isce3.io.Raster objects for geocoding output_raster = isce3.io.Raster(f"IH5:::ID={topo_ds.id.id}".encode("utf-8"), @@ -291,13 +291,11 @@ def geocode_luts(geo_burst_h5, burst, cfg, dst_group_path, item_dict, for item_name, (rg_lut_grid, rg_lut_val, az_lut_grid, az_lut_val) in item_dict.items(): # prepare input dataset in output HDF5 - init_geocoded_dataset(dst_group, - item_name, - decimated_geogrid, - 'float32', - f'geocoded {item_name}') - - dst_dataset = geo_burst_h5[f'{dst_group_path}/{item_name}'] + dst_dataset = create_geocoded_dataset(dst_group, + item_name, + decimated_geogrid, + 'float32', + f'geocoded {item_name}') # prepare output raster geocoded_cal_lut_raster =\ diff --git a/src/compass/s1_geocode_slc.py b/src/compass/s1_geocode_slc.py index 1330d62d..71887402 100755 --- a/src/compass/s1_geocode_slc.py +++ b/src/compass/s1_geocode_slc.py @@ -24,7 +24,7 @@ corrections_to_h5group, flatten_metadata_to_h5group, identity_to_h5group, - init_geocoded_dataset, + create_geocoded_dataset, metadata_to_h5group, DATA_PATH, METADATA_PATH, ROOT_PATH) from compass.utils.helpers import bursts_grouping_generator, get_module_name @@ -58,8 +58,8 @@ def _init_geocoded_IH5_raster(dst_group: h5py.Group, dataset_name: str, isce3.geocode.geocode_slc can write to ''' # Init h5py.Dataset to be converted to IH5 raster object - dataset = init_geocoded_dataset(dst_group, dataset_name, geo_grid, ds_type, - desc) + dataset = create_geocoded_dataset(dst_group, dataset_name, geo_grid, ds_type, + desc) # Construct the output raster directly from HDF5 dataset geo_raster = isce3.io.Raster(f"IH5:::ID={dataset.id.id}".encode("utf-8"), diff --git a/src/compass/utils/h5_helpers.py b/src/compass/utils/h5_helpers.py index a1629c96..70e9ad65 100644 --- a/src/compass/utils/h5_helpers.py +++ b/src/compass/utils/h5_helpers.py @@ -79,11 +79,12 @@ def add_dataset_and_attrs(group, meta_item): val_ds.attrs[key] = _as_np_string_if_needed(val) -def init_geocoded_dataset(grid_group, dataset_name, geo_grid, dtype, - description, data=None): +def create_geocoded_dataset(grid_group, dataset_name, geo_grid, dtype, + description, data=None): ''' - Create and allocate dataset for isce.geocode.geocode_slc to write to that - is NC compliant + Create NC compliant dataset for isce.geocode.geocode_slc to write to, if + no data is provided, and if data is provided, then poplulate dataset with + said data. grid_group: h5py.Group h5py group where geocoded dataset will be created in From 98be0896a034da26990d4dc4243ae2036613cd8b Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Thu, 1 Jun 2023 17:58:20 -0700 Subject: [PATCH 02/11] correctly populate fill value in geocoded datasets rename grid to data in applicable dataset objects --- src/compass/s1_geocode_metadata.py | 2 +- src/compass/s1_geocode_slc.py | 8 ++--- src/compass/utils/h5_helpers.py | 52 ++++++++++++++++++++++++------ 3 files changed, 48 insertions(+), 14 deletions(-) diff --git a/src/compass/s1_geocode_metadata.py b/src/compass/s1_geocode_metadata.py index 0129c436..7793adc4 100755 --- a/src/compass/s1_geocode_metadata.py +++ b/src/compass/s1_geocode_metadata.py @@ -57,7 +57,7 @@ def _fix_layover_shadow_mask(static_layers_dict, h5_root, geo_grid): # delete existing and rewrite with masked data del h5_root[layover_shadow_path] _ = create_geocoded_dataset(h5_root[DATA_PATH], dst_ds_name, geo_grid, - dtype=None, + dtype=temp_arr.dtype, description=np.string_(dst_ds_name), data=temp_arr) diff --git a/src/compass/s1_geocode_slc.py b/src/compass/s1_geocode_slc.py index bad8f23d..76f6a3e1 100755 --- a/src/compass/s1_geocode_slc.py +++ b/src/compass/s1_geocode_slc.py @@ -58,8 +58,8 @@ def _init_geocoded_IH5_raster(dst_group: h5py.Group, dataset_name: str, isce3.geocode.geocode_slc can write to ''' # Init h5py.Dataset to be converted to IH5 raster object - dataset = create_geocoded_dataset(dst_group, dataset_name, geo_grid, ds_type, - desc) + dataset = create_geocoded_dataset(dst_group, dataset_name, geo_grid, + ds_type, desc) # Construct the output raster directly from HDF5 dataset geo_raster = isce3.io.Raster(f"IH5:::ID={dataset.id.id}".encode("utf-8"), @@ -165,7 +165,7 @@ def run(cfg: GeoRunConfig): ctype = h5py.h5t.py_create(np.complex64) ctype.commit(geo_burst_h5['/'].id, np.string_('complex64')) - grid_group = geo_burst_h5.require_group(DATA_PATH) + data_group = geo_burst_h5.require_group(DATA_PATH) check_eap = is_eap_correction_necessary(burst.ipf_version) for b in bursts: pol = b.polarization @@ -199,7 +199,7 @@ def run(cfg: GeoRunConfig): # Iterate over zipped names, types, and descriptions and create # raster objects geo_burst_raster, carrier_raster, flatten_phase_raster =\ - [_init_geocoded_IH5_raster(grid_group, ds_name, geo_grid, + [_init_geocoded_IH5_raster(data_group, ds_name, geo_grid, ds_type, ds_desc) for ds_name, ds_type, ds_desc in zip(ds_names, ds_types, ds_descrs)] diff --git a/src/compass/utils/h5_helpers.py b/src/compass/utils/h5_helpers.py index 70e9ad65..8ca320dc 100644 --- a/src/compass/utils/h5_helpers.py +++ b/src/compass/utils/h5_helpers.py @@ -79,14 +79,41 @@ def add_dataset_and_attrs(group, meta_item): val_ds.attrs[key] = _as_np_string_if_needed(val) -def create_geocoded_dataset(grid_group, dataset_name, geo_grid, dtype, +def _determine_fill_value(dtype): + ''' + Helper function to determine COMPASS specific fill values based on h5py + Dataset type (dtype) + ''' + # Possible float types and float fill + float_types = [np.double, np.single, np.float32, np.float64, 'float32', + 'float64'] + float_fill = np.nan + + # Possible complex types and complex fill + complex_types = [np.complex128, 'complex64', np.complex64, 'complex32'] + complex_fill = np.nan * (0 + 1j) + + # Possible int types and int fill + int_types = [np.byte, np.int8] + int_fill = 127 + + # Iterate through all types and their fill values, then return the fill + # value when a match is found + for types, fill_val in zip([float_types, complex_types, int_types], + [float_fill, complex_fill, int_fill]): + if any([dtype == t for t in types]): + print(dtype) + return fill_val + + +def create_geocoded_dataset(data_group, dataset_name, geo_grid, dtype, description, data=None): ''' Create NC compliant dataset for isce.geocode.geocode_slc to write to, if no data is provided, and if data is provided, then poplulate dataset with said data. - grid_group: h5py.Group + data_group: h5py.Group h5py group where geocoded dataset will be created in dataset_name: str Name of dataset to be created @@ -102,12 +129,18 @@ def create_geocoded_dataset(grid_group, dataset_name, geo_grid, dtype, cslc_ds: h5py.Dataset h5py dataset ready to be populated with geocoded dataset ''' + # Determine fill value of dataset + fill_val = _determine_fill_value(dtype) + if fill_val is None: + raise ValueError(f'Unexpected COMPASS geocoded dataset type: {dtype}') + shape = (geo_grid.length, geo_grid.width) if data is None: - cslc_ds = grid_group.require_dataset(dataset_name, dtype=dtype, - shape=shape) + cslc_ds = data_group.require_dataset(dataset_name, dtype=dtype, + shape=shape, fillvalue=fill_val) else: - cslc_ds = grid_group.create_dataset(dataset_name, data=data) + cslc_ds = data_group.create_dataset(dataset_name, data=data, + fillvalue=fill_val) cslc_ds.attrs['description'] = description @@ -125,9 +158,9 @@ def create_geocoded_dataset(grid_group, dataset_name, geo_grid, dtype, # following copied and pasted (and slightly modified) from: # https://github-fn.jpl.nasa.gov/isce-3/isce/wiki/CF-Conventions-and-Map-Projections - x_ds = grid_group.require_dataset('x_coordinates', dtype='float64', + x_ds = data_group.require_dataset('x_coordinates', dtype='float64', data=x_vect, shape=x_vect.shape) - y_ds = grid_group.require_dataset('y_coordinates', dtype='float64', + y_ds = data_group.require_dataset('y_coordinates', dtype='float64', data=y_vect, shape=y_vect.shape) # Mapping of dimension scales to datasets is not done automatically in HDF5 @@ -141,6 +174,7 @@ def create_geocoded_dataset(grid_group, dataset_name, geo_grid, dtype, # Associate grid mapping with data - projection created later cslc_ds.attrs['grid_mapping'] = np.string_("projection") + # Build list of metadata to be inserted to accompany dataset grid_meta_items = [ Meta('x_spacing', geo_grid.spacing_x, 'Spacing of the geographical grid along X-direction', @@ -150,14 +184,14 @@ def create_geocoded_dataset(grid_group, dataset_name, geo_grid, dtype, {'units': 'meters'}) ] for meta_item in grid_meta_items: - add_dataset_and_attrs(grid_group, meta_item) + add_dataset_and_attrs(data_group, meta_item) # Set up osr for wkt srs = osr.SpatialReference() srs.ImportFromEPSG(geo_grid.epsg) #Create a new single int dataset for projections - projection_ds = grid_group.require_dataset('projection', (), dtype='i') + projection_ds = data_group.require_dataset('projection', (), dtype='i') projection_ds[()] = geo_grid.epsg # WGS84 ellipsoid From 3bd2de784260ac7e4563ee3b07a86a7a1ef40923 Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Fri, 21 Jul 2023 15:39:58 -0700 Subject: [PATCH 03/11] flesh out docstring --- src/compass/utils/h5_helpers.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/compass/utils/h5_helpers.py b/src/compass/utils/h5_helpers.py index 4f25c11c..79c5d423 100644 --- a/src/compass/utils/h5_helpers.py +++ b/src/compass/utils/h5_helpers.py @@ -83,6 +83,16 @@ def _determine_fill_value(dtype): ''' Helper function to determine COMPASS specific fill values based on h5py Dataset type (dtype) + + Parameters + ---------- + dtype: type + Given numeric type whose corresponding fill value of same type is to be + determined + + Returns: + Fill value of type dtype. An exception is raised if no appropriate + value is found. ''' # Possible float types and float fill float_types = [np.double, np.single, np.float32, np.float64, 'float32', @@ -102,7 +112,6 @@ def _determine_fill_value(dtype): for types, fill_val in zip([float_types, complex_types, int_types], [float_fill, complex_fill, int_fill]): if any([dtype == t for t in types]): - print(dtype) return fill_val From b2db86638070865658ea42c509560afcbb918ed0 Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Fri, 21 Jul 2023 17:41:00 -0700 Subject: [PATCH 04/11] move fill value related class and function to own file make fill value logic more robust rename grid_group to data_group --- src/compass/s1_geocode_slc.py | 6 +-- src/compass/utils/browse_image.py | 4 +- src/compass/utils/fill_value.py | 88 +++++++++++++++++++++++++++++++ src/compass/utils/h5_helpers.py | 76 ++++++++++---------------- 4 files changed, 120 insertions(+), 54 deletions(-) create mode 100644 src/compass/utils/fill_value.py diff --git a/src/compass/s1_geocode_slc.py b/src/compass/s1_geocode_slc.py index bf8e24b7..3ad6f968 100755 --- a/src/compass/s1_geocode_slc.py +++ b/src/compass/s1_geocode_slc.py @@ -131,7 +131,7 @@ def run(cfg: GeoRunConfig): ctype = h5py.h5t.py_create(np.complex64) ctype.commit(geo_burst_h5['/'].id, np.string_('complex64')) - grid_group = geo_burst_h5.require_group(DATA_PATH) + data_group = geo_burst_h5.require_group(DATA_PATH) check_eap = is_eap_correction_necessary(burst.ipf_version) # Initialize source/radar and destination/geo dataset into lists @@ -169,7 +169,7 @@ def run(cfg: GeoRunConfig): rdr_data_blks.append(rdr_dataset.ReadAsArray()) # Prepare output dataset of current polarization in HDF5 - geo_ds = init_geocoded_dataset(grid_group, pol, geo_grid, + geo_ds = init_geocoded_dataset(data_group, pol, geo_grid, 'complex64', f'{pol} geocoded CSLC image', output_cfg=cfg.output_params) @@ -195,7 +195,7 @@ def run(cfg: GeoRunConfig): ((carrier_phase_data_blk, carrier_phase_ds), (flatten_phase_data_blk, flatten_phase_ds)) = \ [(np.full(out_shape, np.nan).astype(np.float64), - init_geocoded_dataset(grid_group, ds_name, geo_grid, + init_geocoded_dataset(data_group, ds_name, geo_grid, np.float64, ds_desc, output_cfg=cfg.output_params)) for ds_name, ds_desc in zip(phase_names, phase_descrs)] diff --git a/src/compass/utils/browse_image.py b/src/compass/utils/browse_image.py index ba0ef807..e05324aa 100644 --- a/src/compass/utils/browse_image.py +++ b/src/compass/utils/browse_image.py @@ -206,14 +206,14 @@ def make_browse_image(filename, path_h5, bursts, complex_to_real='amplitude', pe derived_netcdf_to_grid = f'{derived_ds_str}:NETCDF:{path_h5}:/{DATA_PATH}' with h5py.File(path_h5, 'r', swmr=True) as h5_obj: - grid_group = h5_obj[DATA_PATH] + data_group = h5_obj[DATA_PATH] for b in bursts: # get polarization to extract geocoded raster pol = b.polarization # compute browse shape - full_shape = grid_group[pol].shape + full_shape = data_group[pol].shape browse_h, browse_w = _scale_to_max_pixel_dimension(full_shape) # create in memory GDAL raster for GSLC as real value array diff --git a/src/compass/utils/fill_value.py b/src/compass/utils/fill_value.py new file mode 100644 index 00000000..01417bb9 --- /dev/null +++ b/src/compass/utils/fill_value.py @@ -0,0 +1,88 @@ +''' +Class and function for helping set and determine dataset fill values +''' +from dataclasses dataclass + +import numpy as np + + +@dataclass(frozen=True) +class FillValues: + ''' + Dataclass of fill values for float, complex, and int types with default + values. + ''' + # Catch all fill value for all float types (float32, float64, ...) + float_fill: np.single = np.nan + + # Catch all fill value for all complex types (complex64, complex128, ...) + complex_fill: np.csingle = np.nan * (0 + 1j) + + # Catch all fill value for all int types (int8, byte8, ...) + # Currently hard coded for int8/layover_shadow + int_fill: np.intc = 127 + + @classmethod + def from_user_defined_value(cls, user_defined_value): + ''' + Create and return a FillValues class object populated with all default + values populated to a single user defined value. + + Parameters + ---------- + user_defined_value: float + User defined value to be assigned to default value of all types + + Returns + ------- + FillValues + FillValues object with all default values set to user defined value + ''' + return cls(np.single(user_defined_value), + np.csingle(user_defined_value), + np.intc(user_defined_value) + + +def determine_fill_value(dtype, usr_fill_val=None): + ''' + Helper function to determine COMPASS specific fill values based on h5py + Dataset type (dtype) + + Parameters + ---------- + dtype: type + Given numeric type whose corresponding fill value of same type is to be + determined + usr_fill_val: float + User specified non-default dataset fill value + + Returns: + Fill value of type dtype. An exception is raised if no appropriate + value is found. + ''' + if usr_fill_val is None: + # Assign user provided non-default value to all fill values in + # FillValues object with correct typing. Logic below will return on + # accordingly. + fill_values = FillValues.from_user_defined_value(usr_fill_val) + else: + fill_values = FillValues() + + # Check if float type and return float fill + float_types = [np.double, np.single, np.float32, np.float64, 'float32', + 'float64'] + if dtype in float_types: + return fill_values.float_fill + + # Check if complex type and return complex fill + complex_types = [np.complex128, 'complex64', np.complex64, 'complex32'] + if dtype in complex_types: + return fill_values.complex_fill + + # Check if int type and return int fill + int_types = [np.byte, np.int8] + if dtype in int_types: + return fill_values.int_fill + + # No appropriate fill value found above. Raise exception. + raise ValueError(f'Unexpected COMPASS geocoded dataset type: {dtype}') diff --git a/src/compass/utils/h5_helpers.py b/src/compass/utils/h5_helpers.py index 79c5d423..13bd4c9e 100644 --- a/src/compass/utils/h5_helpers.py +++ b/src/compass/utils/h5_helpers.py @@ -14,6 +14,7 @@ import shapely import compass +from compass.utils.fill_value import determine_fill_value, FillValues TIME_STR_FMT = '%Y-%m-%d %H:%M:%S.%f' @@ -79,58 +80,26 @@ def add_dataset_and_attrs(group, meta_item): val_ds.attrs[key] = _as_np_string_if_needed(val) -def _determine_fill_value(dtype): - ''' - Helper function to determine COMPASS specific fill values based on h5py - Dataset type (dtype) - - Parameters - ---------- - dtype: type - Given numeric type whose corresponding fill value of same type is to be - determined - - Returns: - Fill value of type dtype. An exception is raised if no appropriate - value is found. - ''' - # Possible float types and float fill - float_types = [np.double, np.single, np.float32, np.float64, 'float32', - 'float64'] - float_fill = np.nan - - # Possible complex types and complex fill - complex_types = [np.complex128, 'complex64', np.complex64, 'complex32'] - complex_fill = np.nan * (0 + 1j) - - # Possible int types and int fill - int_types = [np.byte, np.int8] - int_fill = 127 - - # Iterate through all types and their fill values, then return the fill - # value when a match is found - for types, fill_val in zip([float_types, complex_types, int_types], - [float_fill, complex_fill, int_fill]): - if any([dtype == t for t in types]): - return fill_val - - -def init_geocoded_dataset(grid_group, dataset_name, geo_grid, dtype, - description, data=None, output_cfg=None): +def init_geocoded_dataset(data_group, dataset_name, geo_grid, dtype, + description, data=None, output_cfg=None, + fill_val=None): ''' Create and allocate dataset for isce.geocode.geocode_slc to write to that - is CF-compliant + is CF-compliant. If data parameter not provided, then an appropriate fill + value is found and used to fill dataset. Parameters ---------- - grid_group: h5py.Group + data_group: h5py.Group h5py group where geocoded dataset will be created in dataset_name: str Name of dataset to be created geo_grid: isce3.product.GeoGridParameters Geogrid of output - dtype: str - Data type of dataset to be geocoded + dtype: Union(str, type) + Data type of dataset to be geocoded to be passed to require_dataset. + require_dataset can take string values e.g. "float32" or types e.g. + numpy.float32 description: str Description of dataset to be geocoded data: np.ndarray @@ -138,6 +107,8 @@ def init_geocoded_dataset(grid_group, dataset_name, geo_grid, dtype, output_cfg: dict Optional dict containing output options in runconfig to apply to created datasets + fill_val: float + Optional value to fill an empty dataset Returns ------- @@ -158,10 +129,17 @@ def init_geocoded_dataset(grid_group, dataset_name, geo_grid, dtype, shape = (geo_grid.length, geo_grid.width) if data is None: - cslc_ds = grid_group.require_dataset(dataset_name, dtype=dtype, - shape=shape, **output_kwargs) + # Determine fill value of dataset + _fill_val = determine_fill_value(dtype) \ + if fill_val is None \ + else fill_val + # Create a dataset with shape and a fill value from above + cslc_ds = data_group.require_dataset(dataset_name, dtype=dtype, + shape=shape, fillvalue=_fill_val, + **output_kwargs) else: - cslc_ds = grid_group.create_dataset(dataset_name, data=data, + # Create a dataset with provided data + cslc_ds = data_group.create_dataset(dataset_name, data=data, **output_kwargs) cslc_ds.attrs['description'] = description @@ -180,9 +158,9 @@ def init_geocoded_dataset(grid_group, dataset_name, geo_grid, dtype, # following copied and pasted (and slightly modified) from: # https://github-fn.jpl.nasa.gov/isce-3/isce/wiki/CF-Conventions-and-Map-Projections - x_ds = grid_group.require_dataset('x_coordinates', dtype='float64', + x_ds = data_group.require_dataset('x_coordinates', dtype='float64', data=x_vect, shape=x_vect.shape) - y_ds = grid_group.require_dataset('y_coordinates', dtype='float64', + y_ds = data_group.require_dataset('y_coordinates', dtype='float64', data=y_vect, shape=y_vect.shape) # Mapping of dimension scales to datasets is not done automatically in HDF5 @@ -206,14 +184,14 @@ def init_geocoded_dataset(grid_group, dataset_name, geo_grid, dtype, {'units': 'meters'}) ] for meta_item in grid_meta_items: - add_dataset_and_attrs(grid_group, meta_item) + add_dataset_and_attrs(data_group, meta_item) # Set up osr for wkt srs = osr.SpatialReference() srs.ImportFromEPSG(geo_grid.epsg) #Create a new single int dataset for projections - projection_ds = grid_group.require_dataset('projection', (), dtype='i') + projection_ds = data_group.require_dataset('projection', (), dtype='i') projection_ds[()] = geo_grid.epsg # WGS84 ellipsoid From fd52ea59d97fd0414f03b81d69508f41005c132c Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Fri, 21 Jul 2023 17:46:17 -0700 Subject: [PATCH 05/11] fix imports and update fill value logic --- src/compass/utils/fill_value.py | 2 +- src/compass/utils/h5_helpers.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/compass/utils/fill_value.py b/src/compass/utils/fill_value.py index 01417bb9..38e511e4 100644 --- a/src/compass/utils/fill_value.py +++ b/src/compass/utils/fill_value.py @@ -1,7 +1,7 @@ ''' Class and function for helping set and determine dataset fill values ''' -from dataclasses dataclass +from dataclasses import dataclass import numpy as np diff --git a/src/compass/utils/h5_helpers.py b/src/compass/utils/h5_helpers.py index 13bd4c9e..f7dd18e5 100644 --- a/src/compass/utils/h5_helpers.py +++ b/src/compass/utils/h5_helpers.py @@ -130,9 +130,8 @@ def init_geocoded_dataset(data_group, dataset_name, geo_grid, dtype, shape = (geo_grid.length, geo_grid.width) if data is None: # Determine fill value of dataset - _fill_val = determine_fill_value(dtype) \ - if fill_val is None \ - else fill_val + _fill_val = determine_fill_value(dtype, fill_val) + # Create a dataset with shape and a fill value from above cslc_ds = data_group.require_dataset(dataset_name, dtype=dtype, shape=shape, fillvalue=_fill_val, From 9094ecbfbdc13db72536a89d448f589391cdd94f Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Sun, 23 Jul 2023 09:23:13 -0700 Subject: [PATCH 06/11] syntax fix --- src/compass/utils/fill_value.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compass/utils/fill_value.py b/src/compass/utils/fill_value.py index 38e511e4..6854aabb 100644 --- a/src/compass/utils/fill_value.py +++ b/src/compass/utils/fill_value.py @@ -40,7 +40,7 @@ def from_user_defined_value(cls, user_defined_value): ''' return cls(np.single(user_defined_value), np.csingle(user_defined_value), - np.intc(user_defined_value) + np.intc(user_defined_value)) def determine_fill_value(dtype, usr_fill_val=None): From 2bd57edada6dccf85a8436eba4b25eb91f42ce26 Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Mon, 24 Jul 2023 10:44:55 -0700 Subject: [PATCH 07/11] fix None logic --- src/compass/utils/fill_value.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/compass/utils/fill_value.py b/src/compass/utils/fill_value.py index 6854aabb..3bc1334c 100644 --- a/src/compass/utils/fill_value.py +++ b/src/compass/utils/fill_value.py @@ -61,12 +61,12 @@ def determine_fill_value(dtype, usr_fill_val=None): value is found. ''' if usr_fill_val is None: + fill_values = FillValues() + else: # Assign user provided non-default value to all fill values in # FillValues object with correct typing. Logic below will return on # accordingly. fill_values = FillValues.from_user_defined_value(usr_fill_val) - else: - fill_values = FillValues() # Check if float type and return float fill float_types = [np.double, np.single, np.float32, np.float64, 'float32', From c86fe322da14d9ded2b52d1431532860efe96252 Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Mon, 24 Jul 2023 11:18:29 -0700 Subject: [PATCH 08/11] address codacy issues --- src/compass/utils/h5_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compass/utils/h5_helpers.py b/src/compass/utils/h5_helpers.py index f7dd18e5..f3b4d87b 100644 --- a/src/compass/utils/h5_helpers.py +++ b/src/compass/utils/h5_helpers.py @@ -14,7 +14,7 @@ import shapely import compass -from compass.utils.fill_value import determine_fill_value, FillValues +from compass.utils.fill_value import determine_fill_value TIME_STR_FMT = '%Y-%m-%d %H:%M:%S.%f' From 94b6aa9054fd5fedfd15c048b18fbd454db3eab9 Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Mon, 24 Jul 2023 12:37:00 -0700 Subject: [PATCH 09/11] add comments for clarity --- src/compass/utils/h5_helpers.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/compass/utils/h5_helpers.py b/src/compass/utils/h5_helpers.py index f3b4d87b..44a34766 100644 --- a/src/compass/utils/h5_helpers.py +++ b/src/compass/utils/h5_helpers.py @@ -127,7 +127,12 @@ def init_geocoded_dataset(data_group, dataset_name, geo_grid, dtype, output_kwargs['compression_opts'] = output_cfg.compression_level output_kwargs['shuffle'] = output_cfg.shuffle + # Shape of dataset is defined by the geo grid shape = (geo_grid.length, geo_grid.width) + + # If data is None, create dataset to specified parameters and fill with + # specified fill value. If data is not None, create a dataset with + # provided data. if data is None: # Determine fill value of dataset _fill_val = determine_fill_value(dtype, fill_val) From 7d842c03d230521f403cbec7408398c62ac56dc9 Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Fri, 4 Aug 2023 13:06:51 -0700 Subject: [PATCH 10/11] use journal logging and fix docstring --- src/compass/utils/fill_value.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/compass/utils/fill_value.py b/src/compass/utils/fill_value.py index 3bc1334c..d31c20e9 100644 --- a/src/compass/utils/fill_value.py +++ b/src/compass/utils/fill_value.py @@ -3,6 +3,7 @@ ''' from dataclasses import dataclass +import journal import numpy as np @@ -56,7 +57,8 @@ def determine_fill_value(dtype, usr_fill_val=None): usr_fill_val: float User specified non-default dataset fill value - Returns: + Returns + ------- Fill value of type dtype. An exception is raised if no appropriate value is found. ''' @@ -85,4 +87,7 @@ def determine_fill_value(dtype, usr_fill_val=None): return fill_values.int_fill # No appropriate fill value found above. Raise exception. - raise ValueError(f'Unexpected COMPASS geocoded dataset type: {dtype}') + err_str = f'Unexpected COMPASS geocoded dataset type: {dtype}' + error_channel = journal.error('fill_value.determine_fill_value') + error_channel.log(err_str) + raise ValueError(err_str) From 1c52097fa84672593ea07ebcc373f7db6c0d7b41 Mon Sep 17 00:00:00 2001 From: Liang Yu Date: Mon, 7 Aug 2023 14:27:17 -0700 Subject: [PATCH 11/11] add fill value to dataset attribs --- src/compass/s1_geocode_metadata.py | 12 +++++------- src/compass/utils/h5_helpers.py | 8 +++++--- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/compass/s1_geocode_metadata.py b/src/compass/s1_geocode_metadata.py index 73775374..ccec8fba 100755 --- a/src/compass/s1_geocode_metadata.py +++ b/src/compass/s1_geocode_metadata.py @@ -64,7 +64,7 @@ def _fix_layover_shadow_mask(static_layers_dict, h5_root, geo_grid, del h5_root[layover_shadow_path] desc = 'Layover shadow mask. 0=no layover, no shadow; 1=shadow; 2=layover; 3=shadow and layover.' _ = init_geocoded_dataset(h5_root[DATA_PATH], dst_ds_name, geo_grid, - dtype=None, + dtype=np.int8, description=np.string_(desc), data=temp_arr, output_cfg=output_params) @@ -473,10 +473,8 @@ def geocode_noise_luts(geo_burst_h5, burst, cfg, for _, bursts in bursts_grouping_generator(cfg.bursts): burst = bursts[0] - # Generate required static layers - if cfg.rdr2geo_params.enabled: - s1_rdr2geo.run(cfg, save_in_scratch=True) + # Generate required un-geocoded static layers + s1_rdr2geo.run(cfg, save_in_scratch=True) - # Geocode static layers if needed - if cfg.rdr2geo_params.geocode_metadata_layers: - run(cfg, burst, fetch_from_scratch=True) + # Geocode static layers generated in step above + run(cfg, burst, fetch_from_scratch=True) diff --git a/src/compass/utils/h5_helpers.py b/src/compass/utils/h5_helpers.py index 34caa315..96eb719e 100644 --- a/src/compass/utils/h5_helpers.py +++ b/src/compass/utils/h5_helpers.py @@ -130,13 +130,14 @@ def init_geocoded_dataset(data_group, dataset_name, geo_grid, dtype, # Shape of dataset is defined by the geo grid shape = (geo_grid.length, geo_grid.width) + # Determine fill value of dataset to either correctly init empty dataset + # and/or populate dataset attribute + _fill_val = determine_fill_value(dtype, fill_val) + # If data is None, create dataset to specified parameters and fill with # specified fill value. If data is not None, create a dataset with # provided data. if data is None: - # Determine fill value of dataset - _fill_val = determine_fill_value(dtype, fill_val) - # Create a dataset with shape and a fill value from above cslc_ds = data_group.require_dataset(dataset_name, dtype=dtype, shape=shape, fillvalue=_fill_val, @@ -147,6 +148,7 @@ def init_geocoded_dataset(data_group, dataset_name, geo_grid, dtype, **output_kwargs) cslc_ds.attrs['description'] = description + cslc_ds.attrs['fill_value'] = _fill_val # Compute x scale dx = geo_grid.spacing_x