Skip to content

Commit

Permalink
Formatting the function to comply with precommit
Browse files Browse the repository at this point in the history
  • Loading branch information
@bryan-harter
bryan-harter committed Sep 12, 2023
1 parent 78937bb commit 3b979fa
Show file tree
Hide file tree
Showing 2 changed files with 74 additions and 64 deletions.
136 changes: 73 additions & 63 deletions cdflib/xarray/xarray_to_cdf.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,25 +12,26 @@
from cdflib.logging import logger



class ISTPError(Exception):
"""
Exception raised for ISTP Compliance Errors
"""
def __init__(self, message=""):

def __init__(self, message: str = ""):
super().__init__(message)

def _warn_or_except(message, exception=False):
if exception:
if 'ISTP' in message:

def _warn_or_except(message: str, exception: bool = False) -> None:
if exception:
if "ISTP" in message:
raise ISTPError(message)
else:
raise Exception(message)
else:
logger.warning(message)


def _dtype_to_cdf_type(var: xr.Dataset, terminate_on_warning: bool=False) -> Tuple[int, int]:
def _dtype_to_cdf_type(var: xr.Dataset, terminate_on_warning: bool = False) -> Tuple[int, int]:
epoch_regex_1 = re.compile("epoch$")
epoch_regex_2 = re.compile("epoch_[0-9]+$")
if epoch_regex_1.match(var.name.lower()) or epoch_regex_2.match(var.name.lower()):
Expand All @@ -56,7 +57,7 @@ def _dtype_to_cdf_type(var: xr.Dataset, terminate_on_warning: bool=False) -> Tup
except Exception as e:
_warn_or_except(
f"NOT SUPPORTED: Data in variable {var.name} has data type {var.dtype}. Attempting to convert it to strings ran into the error: {str(e)}",
terminate_on_warning
terminate_on_warning,
)
return 51, 1
elif var.dtype.type == np.datetime64:
Expand All @@ -81,58 +82,61 @@ def _dtype_to_fillval(dtype: np.dtype, terminate_on_warning: bool = False) -> Un


def _verify_depend_dimensions(
dataset: xr.Dataset, dimension_number: int, primary_variable_name: str, coordinate_variable_name: str,
terminate_on_warning: bool = False
dataset: xr.Dataset,
dimension_number: int,
primary_variable_name: str,
coordinate_variable_name: str,
terminate_on_warning: bool = False,
) -> bool:
primary_data = np.array(dataset[primary_variable_name])
coordinate_data = np.array(dataset[coordinate_variable_name])

if len(primary_data.shape) != 0 and len(coordinate_data.shape) == 0:
_warn_or_except(
f"ISTP Compliance Warning: {coordinate_variable_name} is listed as the DEPEND_{dimension_number} for variable {primary_variable_name}, but the dimensions do not match.",
terminate_on_warning
terminate_on_warning,
)
return False

if len(coordinate_data.shape) != 0 and len(primary_data.shape) == 0:
_warn_or_except(
f"ISTP Compliance Warning: {coordinate_variable_name} is listed as the DEPEND_{dimension_number} for variable {primary_variable_name}, but the dimensions do not match.",
terminate_on_warning
terminate_on_warning,
)
return False

if len(coordinate_data.shape) > 2:
_warn_or_except(
f"ISTP Compliance Warning: {coordinate_variable_name} has too many dimensions to be the DEPEND_{dimension_number} for variable {primary_variable_name}",
terminate_on_warning
terminate_on_warning,
)
return False
if len(coordinate_data.shape) == 2:
if primary_data.shape[0] != coordinate_data.shape[0]:
_warn_or_except(
f"ISTP Compliance Warning: {coordinate_variable_name} is listed as the DEPEND_{dimension_number} for variable {primary_variable_name}, but the Epoch dimensions do not match.",
terminate_on_warning
terminate_on_warning,
)
return False

if len(primary_data.shape) <= dimension_number:
_warn_or_except(
f"ISTP Compliance Warning: {coordinate_variable_name} is listed as the DEPEND_{dimension_number} for variable {primary_variable_name}, but {primary_variable_name} does not have that many dimensions",
terminate_on_warning
terminate_on_warning,
)
return False

if primary_data.shape[dimension_number] != coordinate_data.shape[-1]:
_warn_or_except(
f"ISTP Compliance Warning: {coordinate_variable_name} is listed as the DEPEND_{dimension_number} for variable {primary_variable_name}, but the dimensions do not match.",
terminate_on_warning
terminate_on_warning,
)
return False

return True


def _dimension_checker(dataset: xr.Dataset, terminate_on_warning: bool=False) -> List[str]:
def _dimension_checker(dataset: xr.Dataset, terminate_on_warning: bool = False) -> List[str]:
depend_regex = re.compile("depend_[0-9]+$")

# NOTE: This may add attributes to the dataset object!
Expand All @@ -153,15 +157,15 @@ def _dimension_checker(dataset: xr.Dataset, terminate_on_warning: bool=False) ->
if "VAR_TYPE" not in dataset[var].attrs:
_warn_or_except(
f"ISTP Compliance Warning: Variable {var} does not have an attribute VAR_TYPE to describe the variable. Attributes must be either data, support_data, metadata, or ignore_data.",
terminate_on_warning
terminate_on_warning,
)
var_type = None
else:
var_type = dataset[var].attrs["VAR_TYPE"]
if var_type.lower() not in ("data", "support_data", "metadata", "ignore_data"):
_warn_or_except(
f"ISTP Compliance Warning: Variable {var} attribute VAR_TYPE is not set to either data, support_data, metadata, or ignore_data.",
terminate_on_warning
terminate_on_warning,
)
var_type = None

Expand All @@ -175,7 +179,7 @@ def _dimension_checker(dataset: xr.Dataset, terminate_on_warning: bool=False) ->
else:
_warn_or_except(
f"ISTP Compliance Warning: variable {var} listed {dataset[var].attrs[att]} as its {att}. However, it was not found in the dataset.",
terminate_on_warning
terminate_on_warning,
)

# Determine potential dimension (non-epoch) variables
Expand All @@ -190,7 +194,7 @@ def _dimension_checker(dataset: xr.Dataset, terminate_on_warning: bool=False) ->
_warn_or_except(
f"ISTP Compliance Warning: variable {var} contains a dimension {dim} that is not defined in xarray. "
f"Specify one of the other xarray DataArrays as the DEPEND_{i} attribute.",
terminate_on_warning
terminate_on_warning,
)
i += 1

Expand All @@ -203,7 +207,7 @@ def _dimension_checker(dataset: xr.Dataset, terminate_on_warning: bool=False) ->


def _recheck_dimensions_after_epoch_checker(
dataset: xr.Dataset, time_varying_dimensions: List[str], dim_vars: List[str], terminate_on_warning: bool=False
dataset: xr.Dataset, time_varying_dimensions: List[str], dim_vars: List[str], terminate_on_warning: bool = False
) -> List[str]:
# We need to go back and take a look at the first dimensions of data that were not identified as time-varying
depend_dimension_list = []
Expand Down Expand Up @@ -269,21 +273,21 @@ def _epoch_checker(dataset: xr.Dataset, dim_vars: List[str], terminate_on_warnin
else:
_warn_or_except(
f'ISTP Compliance Warning: variable {var} contained a "record" dimension {potential_depend_0}, but they have different dimensions.',
terminate_on_warning
terminate_on_warning,
)
elif epoch_regex_1.match(var.lower()) or epoch_regex_2.match(var.lower()):
depend_0_list.append(potential_depend_0)
time_varying_dimensions.append(var)
else:
_warn_or_except(
f'ISTP Compliance Warning: variable {var} contained an "record" dimension {potential_depend_0}, but it was not found in the data set.',
terminate_on_warning
terminate_on_warning,
)

depend_0_list = list(set(depend_0_list))

if not depend_0_list:
_warn_or_except(f"ISTP Compliance Warning: No variable for the time dimension could be found.",terminate_on_warning)
_warn_or_except(f"ISTP Compliance Warning: No variable for the time dimension could be found.", terminate_on_warning)

epoch_found = False
for d in depend_0_list:
Expand All @@ -293,15 +297,19 @@ def _epoch_checker(dataset: xr.Dataset, dim_vars: List[str], terminate_on_warnin
if not epoch_found:
_warn_or_except(
f"ISTP Compliance Warning: There is no variable named Epoch. Epoch is the required name of a DEPEND_0 attribute.",
terminate_on_warning
terminate_on_warning,
)

return depend_0_list, time_varying_dimensions


def _add_depend_variables_to_dataset(
dataset: xr.Dataset, dim_vars: List[str], depend_0_vars: List[str], time_varying_dimensions: List[str],
terminate_on_warning: bool = False, auto_fix_depends: bool = True
dataset: xr.Dataset,
dim_vars: List[str],
depend_0_vars: List[str],
time_varying_dimensions: List[str],
terminate_on_warning: bool = False,
auto_fix_depends: bool = True,
) -> xr.Dataset:
data = (dataset, dataset.coords)

Expand Down Expand Up @@ -359,7 +367,7 @@ def _global_attribute_checker(dataset: xr.Dataset, terminate_on_warning: bool =
_warn_or_except(f"ISTP Compliance Warning: Missing dataset attribute {ga}.", terminate_on_warning)


def _variable_attribute_checker(dataset: xr.Dataset, epoch_list: List[str], terminate_on_warning) -> None:
def _variable_attribute_checker(dataset: xr.Dataset, epoch_list: List[str], terminate_on_warning: bool = False) -> None:
data = (dataset, dataset.coords)

for d in data:
Expand All @@ -373,23 +381,20 @@ def _variable_attribute_checker(dataset: xr.Dataset, epoch_list: List[str], term
if var_type.lower() not in ("data", "support_data", "metadata", "ignore_data"):
_warn_or_except(
f"ISTP Compliance Warning: VAR_TYPE for variable {var} is given a non-compliant value of {var_type}",
terminate_on_warning
terminate_on_warning,
)
var_type = ""

# Check for CATDESC
if "CATDESC" not in d[var].attrs:
_warn_or_except(f"ISTP Compliance Warning: CATDESC attribute is required for variable {var}",
terminate_on_warning)
_warn_or_except(f"ISTP Compliance Warning: CATDESC attribute is required for variable {var}", terminate_on_warning)

if "DISPLAY_TYPE" not in d[var].attrs:
if var_type.lower() == "data":
_warn_or_except(f"ISTP Compliance Warning: DISPLAY_TYPE not set for variable {var}",
terminate_on_warning)
_warn_or_except(f"ISTP Compliance Warning: DISPLAY_TYPE not set for variable {var}", terminate_on_warning)

if "FIELDNAM" not in d[var].attrs:
_warn_or_except(f"ISTP Compliance Warning: FIELDNAM attribute is required for variable {var}",
terminate_on_warning)
_warn_or_except(f"ISTP Compliance Warning: FIELDNAM attribute is required for variable {var}", terminate_on_warning)

if "FORMAT" not in d[var].attrs:
if "FORM_PTR" in d[var].attrs:
Expand All @@ -398,11 +403,13 @@ def _variable_attribute_checker(dataset: xr.Dataset, epoch_list: List[str], term
else:
_warn_or_except(
f"ISTP Compliance Warning: FORM_PTR for variable {var} does not point to an existing variable.",
terminate_on_warning
terminate_on_warning,
)
else:
_warn_or_except(f"ISTP Compliance Warning: FORMAT or FORM_PTR attribute is required for variable {var}",
terminate_on_warning)
_warn_or_except(
f"ISTP Compliance Warning: FORMAT or FORM_PTR attribute is required for variable {var}",
terminate_on_warning,
)

if "LABLAXIS" not in d[var].attrs:
if var_type.lower() == "data":
Expand All @@ -412,61 +419,62 @@ def _variable_attribute_checker(dataset: xr.Dataset, epoch_list: List[str], term
else:
_warn_or_except(
f"ISTP Compliance Warning: LABL_PTR_1 attribute for variable {var} does not point to an existing variable.",
terminate_on_warning
terminate_on_warning,
)
else:
_warn_or_except(f"ISTP Compliance Warning: LABLAXIS or LABL_PTR_1 attribute is required for variable {var}",
terminate_on_warning)
_warn_or_except(
f"ISTP Compliance Warning: LABLAXIS or LABL_PTR_1 attribute is required for variable {var}",
terminate_on_warning,
)

if "UNITS" not in d[var].attrs:
if var_type.lower() == "data" or var_type.lower() == "support_data":
if "UNIT_PTR" not in d[var].attrs:
_warn_or_except(f"ISTP Compliance Warning: UNITS or UNIT_PTR attribute is required for variable {var}",
terminate_on_warning)
_warn_or_except(
f"ISTP Compliance Warning: UNITS or UNIT_PTR attribute is required for variable {var}",
terminate_on_warning,
)
else:
if d[var].attrs["UNIT_PTR"] not in dataset:
_warn_or_except(
f"ISTP Compliance Warning: UNIT_PTR attribute for variable {var} does not point to an existing variable.",
terminate_on_warning
terminate_on_warning,
)

if "VALIDMIN" not in d[var].attrs:
if var_type.lower() == "data":
_warn_or_except(f"ISTP Compliance Warning: VALIDMIN required for variable {var}",
terminate_on_warning)
_warn_or_except(f"ISTP Compliance Warning: VALIDMIN required for variable {var}", terminate_on_warning)
elif var_type.lower() == "support_data":
if len(dataset[var].dims) > 0:
if dataset[var].dims[0] in epoch_list:
_warn_or_except(f"ISTP Compliance Warning: VALIDMIN required for variable {var}",
terminate_on_warning)
_warn_or_except(f"ISTP Compliance Warning: VALIDMIN required for variable {var}", terminate_on_warning)

if "VALIDMAX" not in d[var].attrs:
if var_type.lower() == "data":
_warn_or_except(f"ISTP Compliance Warning: VALIDMAX required for variable {var}",
terminate_on_warning)
_warn_or_except(f"ISTP Compliance Warning: VALIDMAX required for variable {var}", terminate_on_warning)
elif var_type.lower() == "support_data":
if len(dataset[var].dims) > 0:
if d[var].dims[0] in epoch_list:
_warn_or_except(f"ISTP Compliance Warning: VALIDMAX required for variable {var}",
terminate_on_warning)
_warn_or_except(f"ISTP Compliance Warning: VALIDMAX required for variable {var}", terminate_on_warning)

if "FILLVAL" not in d[var].attrs:
if var_type.lower() == "data":
_warn_or_except(f"ISTP Compliance Warning: FILLVAL required for variable {var}",
terminate_on_warning)
_warn_or_except(f"ISTP Compliance Warning: FILLVAL required for variable {var}", terminate_on_warning)
fillval = _dtype_to_fillval(d[var].dtype)
d[var].attrs["FILLVAL"] = fillval
_warn_or_except(f"ISTP Compliance Action: Automatically set FILLVAL to {fillval} for variable {var}",
terminate_on_warning)
_warn_or_except(
f"ISTP Compliance Action: Automatically set FILLVAL to {fillval} for variable {var}", terminate_on_warning
)
elif var_type.lower() == "support_data":
if len(dataset[var].dims) > 0:
if d[var].dims[0] in epoch_list:
_warn_or_except(f"ISTP Compliance Warning: FILLVAL required for variable {var}",
terminate_on_warning)
_warn_or_except(f"ISTP Compliance Warning: FILLVAL required for variable {var}", terminate_on_warning)
fillval = _dtype_to_fillval(d[var].dtype)
d[var].attrs["FILLVAL"] = fillval
_warn_or_except(f"ISTP Compliance Action: Automatically set FILLVAL to {fillval} for variable {var}",
terminate_on_warning)
_warn_or_except(
f"ISTP Compliance Action: Automatically set FILLVAL to {fillval} for variable {var}",
terminate_on_warning,
)


def _label_checker(dataset: xr.Dataset, terminate_on_warning: bool = False) -> List[str]:
Expand All @@ -485,7 +493,7 @@ def _label_checker(dataset: xr.Dataset, terminate_on_warning: bool = False) -> L
else:
_warn_or_except(
f"ISTP Compliance Warning: variable {var} listed {dataset[var].attrs[att]} as its {att}. However, it was not found in the dataset.",
terminate_on_warning
terminate_on_warning,
)

istp_label_list = list(set(istp_label_list))
Expand Down Expand Up @@ -694,7 +702,9 @@ def xarray_to_cdf(
dim_vars = _recheck_dimensions_after_epoch_checker(dataset, time_varying_dimensions, dim_vars, terminate_on_warning)

# This function will alter the attributes of the data variables if needed
dataset = _add_depend_variables_to_dataset(dataset, dim_vars, depend_0_vars, time_varying_dimensions, terminate_on_warning, auto_fix_depends)
dataset = _add_depend_variables_to_dataset(
dataset, dim_vars, depend_0_vars, time_varying_dimensions, terminate_on_warning, auto_fix_depends
)

_global_attribute_checker(dataset, terminate_on_warning)

Expand All @@ -714,7 +724,7 @@ def xarray_to_cdf(
i += 1
else:
glob_att_dict[ga] = {0: dataset.attrs[ga]}

x = CDF(file_name)
x.write_globalattrs(glob_att_dict)

Expand Down
2 changes: 1 addition & 1 deletion meta.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ package:
source:
git_url: https://github.com/MAVENSDC/cdflib.git
git_depth: 20
git_rev: 0.4.4
git_rev: 1.1.1

build:
number: 0
Expand Down

0 comments on commit 3b979fa

Please sign in to comment.