Fix time components (#245)

* added fix_time_components

* use fix_time_components

* added smoke test for 360-calendar

* using fix_parameters

* added smoke test for 360day calendar with workflow

rook/processes/wps_concat.py

@@ -1,10 +1,10 @@
 import logging
-import os
 
 from pywps import FORMATS, ComplexOutput, Format, LiteralInput, Process
 from pywps.app.Common import Metadata
-from pywps.app.exceptions import ProcessError
-from pywps.inout.outputs import MetaFile, MetaLink4
+
+# from pywps.app.exceptions import ProcessError
+# from pywps.inout.outputs import MetaFile, MetaLink4
 
 from ..director import wrap_director
 from ..utils.input_utils import parse_wps_input
@@ -137,7 +137,6 @@ def _handler(self, request, response):
         collection = parse_wps_input(
             request.inputs, "collection", as_sequence=True, must_exist=True
         )
-        # print(collection)
         inputs = {
             "collection": collection,
             "output_dir": self.workdir,
@@ -156,7 +155,6 @@ def _handler(self, request, response):
                 request.inputs, "dims", as_sequence=True, default=None
             ),
         }
-        # print(inputs)
 
         # Let the director manage the processing or redirection to original files
         director = wrap_director(collection, inputs, run_concat)

rook/processes/wps_subset.py

@@ -1,10 +1,10 @@
 import logging
-import os
 
 from pywps import FORMATS, ComplexOutput, Format, LiteralInput, Process
 from pywps.app.Common import Metadata
-from pywps.app.exceptions import ProcessError
-from pywps.inout.outputs import MetaFile, MetaLink4
+
+# from pywps.app.exceptions import ProcessError
+# from pywps.inout.outputs import MetaFile, MetaLink4
 
 from ..director import wrap_director
 from ..utils.input_utils import parse_wps_input

rook/utils/concat_utils.py

@@ -18,6 +18,7 @@
 from clisops.core.average import average_over_dims as average
 
 from .decadal_fixes import apply_decadal_fixes
+from .input_utils import fix_parameters
 
 coord_by_standard_name = {
     "realization": "realization",
@@ -125,6 +126,8 @@ def _concat(
 
 
 def run_concat(args):
+    args = fix_parameters(args)
+
     result = concat(**args)
     return result.file_uris
 

rook/utils/input_utils.py

@@ -5,6 +5,37 @@
 from roocs_utils.exceptions import InvalidProject
 from rook import CONFIG
 
+TC_ALL_MONTHS = "month:jan,feb,mar,apr,may,jun,jul,aug,sep,oct,nov,dec"
+TC_ALL_MONTHS_2 = "month:01,02,03,04,05,06,07,08,09,10,11,12"
+TC_ALL_DAYS = "day:01,02,03,04,05,06,07,08,09,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31"
+
+
+def fix_parameters(parameters):
+    if "time_components" in parameters:
+        parameters["time_components"] = fix_time_components(
+            parameters["time_components"]
+        )
+    return parameters
+
+
+def fix_time_components(tc):
+    # Remove redundant time-component parts to avoid for example issues with 360day calendars.
+    if not tc:
+        return None
+
+    tc_parts = tc.split("|")
+    new_tc_parts = []
+    for tc_part in tc_parts:
+        if tc_part == TC_ALL_MONTHS:
+            continue
+        if tc_part == TC_ALL_MONTHS_2:
+            continue
+        if tc_part == TC_ALL_DAYS:
+            continue
+        new_tc_parts.append(tc_part)
+    new_tc = "|".join(new_tc_parts)
+    return new_tc
+
 
 def parse_wps_input(inputs, key, as_sequence=False, must_exist=False, default=None):
     if not inputs.get(key):

rook/utils/subset_utils.py

@@ -1,6 +1,11 @@
+from .input_utils import fix_parameters
+
+
 def run_subset(args):
     # TODO: handle lazy load of daops
     from daops.ops.subset import subset
 
+    args = fix_parameters(args)
+
     result = subset(**args)
     return result.file_uris

tests/smoke/test_smoke_checks.py

@@ -1,4 +1,4 @@
-from pyparsing import dbl_slash_comment
+# from pyparsing import dbl_slash_comment
 from tests.smoke.utils import open_dataset
 
 from owslib.wps import ComplexDataInput
@@ -26,6 +26,10 @@
     "c3s-cmip6.CMIP.CSIRO-ARCCSS.ACCESS-CM2.historical.r1i1p1f1.Amon.ta.gn.v20191108"
 )
 
+C3S_CMIP6_360DAY_CALENDAR_COLLECTION = (
+    "c3s-cmip6.ScenarioMIP.MOHC.HadGEM3-GC31-LL.ssp245.r1i1p1f3.day.pr.gn.v20190908"
+)
+
 C3S_CMIP5_DAY_COLLECTION = (
     "c3s-cmip5.output1.IPSL.IPSL-CM5B-LR.historical.day.atmos.day.r1i1p1.tas.v20120718"
 )
@@ -120,6 +124,25 @@
     }
 )
 
+TC_ALL_DAYS = "day:01,02,03,04,05,06,07,08,09,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31"
+
+WF_C3S_CMIP6_360DAY_CALENDAR = json.dumps(
+    {
+        "doc": "subset on cmip6",
+        "inputs": {"ds": [C3S_CMIP6_360DAY_CALENDAR_COLLECTION]},
+        "outputs": {"output": "subset/output"},
+        "steps": {
+            "subset": {
+                "run": "subset",
+                "in": {
+                    "collection": "inputs/ds",
+                    "time": "2015/2015",
+                    "time_components": f"month:01,02,03|{TC_ALL_DAYS}",
+                },
+            },
+        },
+    }
+)
 
 WF_C3S_CORDEX = json.dumps(
     {
@@ -347,6 +370,19 @@ def test_smoke_execute_c3s_cmip6_subset_by_point(wps, tmp_path):
     assert "rlds" in ds.variables
 
 
+def test_smoke_execute_c3s_cmip6_360calendar_subset_by_point(wps, tmp_path):
+    inputs = [
+        ("collection", C3S_CMIP6_360DAY_CALENDAR_COLLECTION),
+        ("time", "2015/2015"),
+        ("time_components", f"month:jan,feb,mar|{TC_ALL_DAYS}"),
+    ]
+    urls = wps.execute("subset", inputs)
+    assert len(urls) == 1
+    assert "pr_day_HadGEM3-GC31-LL_ssp245_r1i1p1f3_gn_20150101-20150330.nc" in urls[0]
+    ds = open_dataset(urls[0], tmp_path)
+    assert "pr" in ds.variables
+
+
 def test_smoke_execute_c3s_cordex_subset_by_point(wps, tmp_path):
     inputs = [
         ("collection", C3S_CORDEX_MON_COLLECTION),
@@ -541,6 +577,15 @@ def test_smoke_execute_c3s_cmip6_regrid_orchestrate(wps):
     )
 
 
+def test_smoke_execute_c3s_cmip6_360day_calendar_orchestrate(wps):
+    inputs = [
+        ("workflow", ComplexDataInput(WF_C3S_CMIP6_360DAY_CALENDAR)),
+    ]
+    urls = wps.execute("orchestrate", inputs)
+    assert len(urls) == 1
+    assert "pr_day_HadGEM3-GC31-LL_ssp245_r1i1p1f3_gn_20150101-20150330.nc" in urls[0]
+
+
 def test_smoke_execute_c3s_cmip6_orchestrate_metadata(wps, tmp_path):
     inputs = [
         ("workflow", ComplexDataInput(WF_C3S_CMIP6)),

tests/test_utils.py

@@ -3,7 +3,11 @@
 
 from pywps.app import WPSRequest
 
-from rook.utils.input_utils import resolve_to_file_paths, parse_wps_input
+from rook.utils.input_utils import (
+    resolve_to_file_paths,
+    parse_wps_input,
+    fix_time_components,
+)
 from rook.utils.metalink_utils import build_metalink
 
 from .common import TESTS_HOME, MINI_ESGF_MASTER_DIR
@@ -124,3 +128,30 @@ def test_parse_wps_input():
         parse_wps_input(request.inputs, "time_components", default=None)
         == "year:1970,1980|month=01,02,03"
     )
+
+
+def test_fix_time_components():
+    assert (
+        fix_time_components("year:2000|month=01|day=01") == "year:2000|month=01|day=01"
+    )
+
+    all_days = "day:01,02,03,04,05,06,07,08,09,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31"
+    assert (
+        fix_time_components(f"year:2001,2002|month=02|{all_days}")
+        == "year:2001,2002|month=02"
+    )
+
+    all_months = "month:jan,feb,mar,apr,may,jun,jul,aug,sep,oct,nov,dec"
+    assert (
+        fix_time_components(f"year:2001,2002|{all_months}|{all_days}")
+        == "year:2001,2002"
+    )
+
+    all_months_2 = "month:01,02,03,04,05,06,07,08,09,10,11,12"
+    assert (
+        fix_time_components(f"year:2001,2002|{all_months_2}|day:01,31")
+        == "year:2001,2002|day:01,31"
+    )
+
+    assert fix_time_components("") is None
+    assert fix_time_components(None) is None