Merge pull request #98 from roocs/fix_reftime_calendar

Fix reftime calendar and remove fill value fix
roocs · Jan 12, 2022 · 687b6ac · 687b6ac
2 parents dfc4d29 + 7ddfeec
commit 687b6ac
Show file tree

Hide file tree

Showing 6 changed files with 28 additions and 55 deletions.
diff --git a/dachar/etc/roocs.ini b/dachar/etc/roocs.ini
@@ -5,9 +5,9 @@ extra_floats = concern_threshold
 
 [dachar:processing]
 queue = short-serial
-wallclock_large = 23:59
+wallclock_large = 23:59:00
 memory_large = 32000
-wallclock_small = 04:00
+wallclock_small = 04:00:00
 memory_small = 4000
 
 [dachar:output_paths]

diff --git a/dachar/fixes/attr_fixes.py b/dachar/fixes/attr_fixes.py
@@ -5,7 +5,6 @@
     "GlobalAttrFix",
     "CheckAddGlobalAttrFix",
     "VarAttrFix",
-    "RemoveFillValuesFix",
     "RemoveCoordAttrFix",
 ]
 
@@ -77,18 +76,6 @@ class VarAttrFix(_BaseDatasetFix):
     process_type = "post_processor"
 
 
-class RemoveFillValuesFix(_BaseDatasetFix):
-    fix_id = "RemoveFillValuesFix"
-    title = "Remove the Fill Value from coordinate variables"
-    description = """
-"Remove the FillValue attributes from coordinate variables which are added during manipulation with xarray, as NaNs.
-"""
-    category = "attr_fixes"
-    required_operands = []
-    ref_implementation = "daops.data_utils.attr_utils.remove_fill_values"
-    process_type = "post_processor"
-
-
 class RemoveCoordAttrFix(_BaseDatasetFix):
     fix_id = "RemoveCoordAttrFix"
     title = "Remove the coordinate attribute added by xarray from specified variables"

diff --git a/generate_decadal_fix.py b/generate_decadal_fix.py
@@ -14,7 +14,9 @@
 import argparse
 import copy
 import json
+import os
 import re
+import sys
 from datetime import datetime
 
 import numpy as np
@@ -27,15 +29,24 @@ def arg_parse():
     parser = argparse.ArgumentParser()
 
     parser.add_argument(
-        "-f", "--fpath", type=str, required=True, help="Path to write the JSON fix file to."
+        "-f",
+        "--fpath",
+        type=str,
+        required=True,
+        help="Path to write the JSON fix file to.",
     )
 
     parser.add_argument(
-        "-d", "--dsid", type=str, required=True, help="Dataset ID to create the fix template for."
+        "-d",
+        "--dsid",
+        type=str,
+        required=True,
+        help="Dataset ID to create the fix template for.",
     )
 
     return parser.parse_args()
 
+
 decadal_template = {
     "dataset_id": "",
     "fixes": [
@@ -80,7 +91,7 @@ def arg_parse():
                 "encoding": {
                     "dtype": "int32",
                     "units": "days since 1850-01-01",
-                    "calendar": "gregorian",
+                    "calendar": "derive: daops.fix_utils.decadal_utils.get_time_calendar",
                 },
             },
             "source": {
@@ -100,7 +111,7 @@ def arg_parse():
                 "attrs": {
                     "long_name": "Time elapsed since the start of the forecast",
                     "standard_name": "forecast_period",
-                    "units": "days"
+                    "units": "days",
                 },
                 "encoding": {"dtype": "double"},
             },
@@ -129,19 +140,11 @@ def arg_parse():
                 "url": "https://github.com/cp4cds/c3s34g_master/tree/master/Decadal",
             },
         },
-        {
-            "fix_id": "RemoveFillValuesFix",
-            "operands": {},
-            "source": {
-                "name": "ceda",
-                "version": "",
-                "comments": "",
-                "url": "https://github.com/cp4cds/c3s34g_master/tree/master/Decadal",
-            },
-        },
         {
             "fix_id": "RemoveCoordAttrFix",
-            "operands": {"var_ids": "derive: daops.fix_utils.decadal_utils.get_decadal_bnds_list"},
+            "operands": {
+                "var_ids": "derive: daops.fix_utils.decadal_utils.get_decadal_bnds_list"
+            },
             "source": {
                 "name": "ceda",
                 "version": "",
@@ -156,7 +159,7 @@ def arg_parse():
 def get_decadal_template():
     "Takes a copy of the global template, and returns it."
     tmpl = copy.deepcopy(decadal_template)
-    return tmpl 
+    return tmpl
 
 
 def get_ds_ids(fpath):
@@ -181,10 +184,13 @@ def main():
 
     dt["dataset_id"] = ds_id
 
-    if ds_id in get_ds_ids('./further_info_url-dsets.txt'):
-        global_attrs = dt['fixes'][1].get("operands").get("attrs") 
-        global_attrs.update(further_info_url='derive: daops.fix_utils.decadal_utils.fix_further_info_url')
-        dt['fixes'][1]["operands"]["attrs"] = global_attrs
+    pathname = os.path.dirname(sys.argv[0])
+    if ds_id in get_ds_ids(os.path.join(pathname, "further_info_url-dsets.txt")):
+        global_attrs = dt["fixes"][1].get("operands").get("attrs")
+        global_attrs.update(
+            further_info_url="derive: daops.fix_utils.decadal_utils.fix_further_info_url"
+        )
+        dt["fixes"][1]["operands"]["attrs"] = global_attrs
 
     # save the template as a json file
     with open(fpath, "w") as fp:
@@ -193,4 +199,3 @@ def main():
 
 if __name__ == "__main__":
     main()
-
diff --git a/tests/test_fixes/decadal_fixes/decadal.json b/tests/test_fixes/decadal_fixes/decadal.json
@@ -64,14 +64,5 @@
             "version": "",
             "comments": "",
             "url":"https://github.com/cp4cds/c3s34g_master/tree/master/Decadal"}
-      },
-      {
-        "fix_id": "RemoveFillValuesFix",
-        "operands": {},
-        "source": {
-            "name": "ceda",
-            "version": "",
-            "comments": "",
-            "url":"https://github.com/cp4cds/c3s34g_master/tree/master/Decadal"}
       }]
 }
diff --git a/tests/test_fixes/decadal_fixes/decadal_template.json b/tests/test_fixes/decadal_fixes/decadal_template.json
@@ -64,14 +64,5 @@
             "version": "",
             "comments": "",
             "url":"https://github.com/cp4cds/c3s34g_master/tree/master/Decadal"}
-      },
-      {
-        "fix_id": "RemoveFillValuesFix",
-        "operands": {},
-        "source": {
-            "name": "ceda",
-            "version": "",
-            "comments": "",
-            "url":"https://github.com/cp4cds/c3s34g_master/tree/master/Decadal"}
       }]
 }
diff --git a/tests/test_fixes/test_fix_class.py b/tests/test_fixes/test_fix_class.py
@@ -18,7 +18,6 @@ def test_get_fix_categories():
             "GlobalAttrFix",
             "CheckAddGlobalAttrFix",
             "VarAttrFix",
-            "RemoveFillValuesFix",
             "RemoveCoordAttrFix",
         ],
         "var_fixes": ["AddDataVarFix"],