From 170ea554931b9c3516b2bafe2bea892f87b3a4e6 Mon Sep 17 00:00:00 2001 From: ellesmith88 Date: Tue, 4 Jan 2022 13:10:13 +0000 Subject: [PATCH 1/5] change calendar for reftime on decadal template --- generate_decadal_fix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generate_decadal_fix.py b/generate_decadal_fix.py index 6d879bba..9e918179 100644 --- a/generate_decadal_fix.py +++ b/generate_decadal_fix.py @@ -80,7 +80,7 @@ def arg_parse(): "encoding": { "dtype": "int32", "units": "days since 1850-01-01", - "calendar": "gregorian", + "calendar": "derive: daops.fix_utils.decadal_utils.get_sub_experiment_id", }, }, "source": { From 48cd52e71e0375826b9cc721b2c95bb480f12bef Mon Sep 17 00:00:00 2001 From: ellesmith88 Date: Mon, 10 Jan 2022 08:32:31 +0000 Subject: [PATCH 2/5] remove fill value fix --- dachar/fixes/attr_fixes.py | 13 ------------- tests/test_fixes/decadal_fixes/decadal.json | 9 --------- .../test_fixes/decadal_fixes/decadal_template.json | 9 --------- 3 files changed, 31 deletions(-) diff --git a/dachar/fixes/attr_fixes.py b/dachar/fixes/attr_fixes.py index 1c27d037..1040eb05 100644 --- a/dachar/fixes/attr_fixes.py +++ b/dachar/fixes/attr_fixes.py @@ -5,7 +5,6 @@ "GlobalAttrFix", "CheckAddGlobalAttrFix", "VarAttrFix", - "RemoveFillValuesFix", "RemoveCoordAttrFix", ] @@ -77,18 +76,6 @@ class VarAttrFix(_BaseDatasetFix): process_type = "post_processor" -class RemoveFillValuesFix(_BaseDatasetFix): - fix_id = "RemoveFillValuesFix" - title = "Remove the Fill Value from coordinate variables" - description = """ -"Remove the FillValue attributes from coordinate variables which are added during manipulation with xarray, as NaNs. -""" - category = "attr_fixes" - required_operands = [] - ref_implementation = "daops.data_utils.attr_utils.remove_fill_values" - process_type = "post_processor" - - class RemoveCoordAttrFix(_BaseDatasetFix): fix_id = "RemoveCoordAttrFix" title = "Remove the coordinate attribute added by xarray from specified variables" diff --git a/tests/test_fixes/decadal_fixes/decadal.json b/tests/test_fixes/decadal_fixes/decadal.json index f577fd9b..b99a6213 100644 --- a/tests/test_fixes/decadal_fixes/decadal.json +++ b/tests/test_fixes/decadal_fixes/decadal.json @@ -64,14 +64,5 @@ "version": "", "comments": "", "url":"https://github.com/cp4cds/c3s34g_master/tree/master/Decadal"} - }, - { - "fix_id": "RemoveFillValuesFix", - "operands": {}, - "source": { - "name": "ceda", - "version": "", - "comments": "", - "url":"https://github.com/cp4cds/c3s34g_master/tree/master/Decadal"} }] } diff --git a/tests/test_fixes/decadal_fixes/decadal_template.json b/tests/test_fixes/decadal_fixes/decadal_template.json index f10fe62c..d0b4e4c9 100644 --- a/tests/test_fixes/decadal_fixes/decadal_template.json +++ b/tests/test_fixes/decadal_fixes/decadal_template.json @@ -64,14 +64,5 @@ "version": "", "comments": "", "url":"https://github.com/cp4cds/c3s34g_master/tree/master/Decadal"} - }, - { - "fix_id": "RemoveFillValuesFix", - "operands": {}, - "source": { - "name": "ceda", - "version": "", - "comments": "", - "url":"https://github.com/cp4cds/c3s34g_master/tree/master/Decadal"} }] } From e862b063af9caf98202f7c1181f2d3a5c2ee7c9e Mon Sep 17 00:00:00 2001 From: Eleanor Smith Date: Mon, 10 Jan 2022 08:37:15 +0000 Subject: [PATCH 3/5] update wallclock for prcoessing --- dachar/etc/roocs.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dachar/etc/roocs.ini b/dachar/etc/roocs.ini index da9b953d..b8892807 100644 --- a/dachar/etc/roocs.ini +++ b/dachar/etc/roocs.ini @@ -5,9 +5,9 @@ extra_floats = concern_threshold [dachar:processing] queue = short-serial -wallclock_large = 23:59 +wallclock_large = 23:59:00 memory_large = 32000 -wallclock_small = 04:00 +wallclock_small = 04:00:00 memory_small = 4000 [dachar:output_paths] From bcc1747c49dcfc813e635b75e2fd186c09b02cfb Mon Sep 17 00:00:00 2001 From: Eleanor Smith Date: Mon, 10 Jan 2022 08:45:21 +0000 Subject: [PATCH 4/5] update derive function for reftime calendar --- generate_decadal_fix.py | 44 +++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/generate_decadal_fix.py b/generate_decadal_fix.py index 9e918179..f5d0bf83 100644 --- a/generate_decadal_fix.py +++ b/generate_decadal_fix.py @@ -27,15 +27,24 @@ def arg_parse(): parser = argparse.ArgumentParser() parser.add_argument( - "-f", "--fpath", type=str, required=True, help="Path to write the JSON fix file to." + "-f", + "--fpath", + type=str, + required=True, + help="Path to write the JSON fix file to.", ) parser.add_argument( - "-d", "--dsid", type=str, required=True, help="Dataset ID to create the fix template for." + "-d", + "--dsid", + type=str, + required=True, + help="Dataset ID to create the fix template for.", ) return parser.parse_args() + decadal_template = { "dataset_id": "", "fixes": [ @@ -80,7 +89,7 @@ def arg_parse(): "encoding": { "dtype": "int32", "units": "days since 1850-01-01", - "calendar": "derive: daops.fix_utils.decadal_utils.get_sub_experiment_id", + "calendar": "derive: daops.fix_utils.decadal_utils.get_time_calendar", }, }, "source": { @@ -100,7 +109,7 @@ def arg_parse(): "attrs": { "long_name": "Time elapsed since the start of the forecast", "standard_name": "forecast_period", - "units": "days" + "units": "days", }, "encoding": {"dtype": "double"}, }, @@ -129,19 +138,11 @@ def arg_parse(): "url": "https://github.com/cp4cds/c3s34g_master/tree/master/Decadal", }, }, - { - "fix_id": "RemoveFillValuesFix", - "operands": {}, - "source": { - "name": "ceda", - "version": "", - "comments": "", - "url": "https://github.com/cp4cds/c3s34g_master/tree/master/Decadal", - }, - }, { "fix_id": "RemoveCoordAttrFix", - "operands": {"var_ids": "derive: daops.fix_utils.decadal_utils.get_decadal_bnds_list"}, + "operands": { + "var_ids": "derive: daops.fix_utils.decadal_utils.get_decadal_bnds_list" + }, "source": { "name": "ceda", "version": "", @@ -156,7 +157,7 @@ def arg_parse(): def get_decadal_template(): "Takes a copy of the global template, and returns it." tmpl = copy.deepcopy(decadal_template) - return tmpl + return tmpl def get_ds_ids(fpath): @@ -181,10 +182,12 @@ def main(): dt["dataset_id"] = ds_id - if ds_id in get_ds_ids('./further_info_url-dsets.txt'): - global_attrs = dt['fixes'][1].get("operands").get("attrs") - global_attrs.update(further_info_url='derive: daops.fix_utils.decadal_utils.fix_further_info_url') - dt['fixes'][1]["operands"]["attrs"] = global_attrs + if ds_id in get_ds_ids("./further_info_url-dsets.txt"): + global_attrs = dt["fixes"][1].get("operands").get("attrs") + global_attrs.update( + further_info_url="derive: daops.fix_utils.decadal_utils.fix_further_info_url" + ) + dt["fixes"][1]["operands"]["attrs"] = global_attrs # save the template as a json file with open(fpath, "w") as fp: @@ -193,4 +196,3 @@ def main(): if __name__ == "__main__": main() - From 7ddfeec5bbd139ee57bbbe7c927c2f8998e74cf4 Mon Sep 17 00:00:00 2001 From: Eleanor Smith Date: Mon, 10 Jan 2022 15:32:41 +0000 Subject: [PATCH 5/5] fix failing test --- generate_decadal_fix.py | 5 ++++- tests/test_fixes/test_fix_class.py | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) mode change 100644 => 100755 generate_decadal_fix.py diff --git a/generate_decadal_fix.py b/generate_decadal_fix.py old mode 100644 new mode 100755 index f5d0bf83..dd1820f8 --- a/generate_decadal_fix.py +++ b/generate_decadal_fix.py @@ -14,7 +14,9 @@ import argparse import copy import json +import os import re +import sys from datetime import datetime import numpy as np @@ -182,7 +184,8 @@ def main(): dt["dataset_id"] = ds_id - if ds_id in get_ds_ids("./further_info_url-dsets.txt"): + pathname = os.path.dirname(sys.argv[0]) + if ds_id in get_ds_ids(os.path.join(pathname, "further_info_url-dsets.txt")): global_attrs = dt["fixes"][1].get("operands").get("attrs") global_attrs.update( further_info_url="derive: daops.fix_utils.decadal_utils.fix_further_info_url" diff --git a/tests/test_fixes/test_fix_class.py b/tests/test_fixes/test_fix_class.py index 4f1a3d34..f83d9212 100644 --- a/tests/test_fixes/test_fix_class.py +++ b/tests/test_fixes/test_fix_class.py @@ -18,7 +18,6 @@ def test_get_fix_categories(): "GlobalAttrFix", "CheckAddGlobalAttrFix", "VarAttrFix", - "RemoveFillValuesFix", "RemoveCoordAttrFix", ], "var_fixes": ["AddDataVarFix"],