diff --git a/rook/processes/wps_concat.py b/rook/processes/wps_concat.py index 3ca3470b..d30c1870 100644 --- a/rook/processes/wps_concat.py +++ b/rook/processes/wps_concat.py @@ -1,10 +1,10 @@ import logging -import os from pywps import FORMATS, ComplexOutput, Format, LiteralInput, Process from pywps.app.Common import Metadata -from pywps.app.exceptions import ProcessError -from pywps.inout.outputs import MetaFile, MetaLink4 + +# from pywps.app.exceptions import ProcessError +# from pywps.inout.outputs import MetaFile, MetaLink4 from ..director import wrap_director from ..utils.input_utils import parse_wps_input @@ -137,7 +137,6 @@ def _handler(self, request, response): collection = parse_wps_input( request.inputs, "collection", as_sequence=True, must_exist=True ) - # print(collection) inputs = { "collection": collection, "output_dir": self.workdir, @@ -156,7 +155,6 @@ def _handler(self, request, response): request.inputs, "dims", as_sequence=True, default=None ), } - # print(inputs) # Let the director manage the processing or redirection to original files director = wrap_director(collection, inputs, run_concat) diff --git a/rook/processes/wps_subset.py b/rook/processes/wps_subset.py index 8ae5eeaf..bfad589b 100644 --- a/rook/processes/wps_subset.py +++ b/rook/processes/wps_subset.py @@ -1,10 +1,10 @@ import logging -import os from pywps import FORMATS, ComplexOutput, Format, LiteralInput, Process from pywps.app.Common import Metadata -from pywps.app.exceptions import ProcessError -from pywps.inout.outputs import MetaFile, MetaLink4 + +# from pywps.app.exceptions import ProcessError +# from pywps.inout.outputs import MetaFile, MetaLink4 from ..director import wrap_director from ..utils.input_utils import parse_wps_input diff --git a/rook/utils/concat_utils.py b/rook/utils/concat_utils.py index ffd33396..921ca1ae 100644 --- a/rook/utils/concat_utils.py +++ b/rook/utils/concat_utils.py @@ -18,6 +18,7 @@ from clisops.core.average import average_over_dims as average from .decadal_fixes import apply_decadal_fixes +from .input_utils import fix_parameters coord_by_standard_name = { "realization": "realization", @@ -125,6 +126,8 @@ def _concat( def run_concat(args): + args = fix_parameters(args) + result = concat(**args) return result.file_uris diff --git a/rook/utils/input_utils.py b/rook/utils/input_utils.py index 1451fad0..b34c1668 100644 --- a/rook/utils/input_utils.py +++ b/rook/utils/input_utils.py @@ -5,6 +5,37 @@ from roocs_utils.exceptions import InvalidProject from rook import CONFIG +TC_ALL_MONTHS = "month:jan,feb,mar,apr,may,jun,jul,aug,sep,oct,nov,dec" +TC_ALL_MONTHS_2 = "month:01,02,03,04,05,06,07,08,09,10,11,12" +TC_ALL_DAYS = "day:01,02,03,04,05,06,07,08,09,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31" + + +def fix_parameters(parameters): + if "time_components" in parameters: + parameters["time_components"] = fix_time_components( + parameters["time_components"] + ) + return parameters + + +def fix_time_components(tc): + # Remove redundant time-component parts to avoid for example issues with 360day calendars. + if not tc: + return None + + tc_parts = tc.split("|") + new_tc_parts = [] + for tc_part in tc_parts: + if tc_part == TC_ALL_MONTHS: + continue + if tc_part == TC_ALL_MONTHS_2: + continue + if tc_part == TC_ALL_DAYS: + continue + new_tc_parts.append(tc_part) + new_tc = "|".join(new_tc_parts) + return new_tc + def parse_wps_input(inputs, key, as_sequence=False, must_exist=False, default=None): if not inputs.get(key): diff --git a/rook/utils/subset_utils.py b/rook/utils/subset_utils.py index cfcb1c3c..291d1635 100644 --- a/rook/utils/subset_utils.py +++ b/rook/utils/subset_utils.py @@ -1,6 +1,11 @@ +from .input_utils import fix_parameters + + def run_subset(args): # TODO: handle lazy load of daops from daops.ops.subset import subset + args = fix_parameters(args) + result = subset(**args) return result.file_uris diff --git a/tests/smoke/test_smoke_checks.py b/tests/smoke/test_smoke_checks.py index 233e562e..1b63cfe2 100644 --- a/tests/smoke/test_smoke_checks.py +++ b/tests/smoke/test_smoke_checks.py @@ -1,4 +1,4 @@ -from pyparsing import dbl_slash_comment +# from pyparsing import dbl_slash_comment from tests.smoke.utils import open_dataset from owslib.wps import ComplexDataInput @@ -26,6 +26,10 @@ "c3s-cmip6.CMIP.CSIRO-ARCCSS.ACCESS-CM2.historical.r1i1p1f1.Amon.ta.gn.v20191108" ) +C3S_CMIP6_360DAY_CALENDAR_COLLECTION = ( + "c3s-cmip6.ScenarioMIP.MOHC.HadGEM3-GC31-LL.ssp245.r1i1p1f3.day.pr.gn.v20190908" +) + C3S_CMIP5_DAY_COLLECTION = ( "c3s-cmip5.output1.IPSL.IPSL-CM5B-LR.historical.day.atmos.day.r1i1p1.tas.v20120718" ) @@ -120,6 +124,25 @@ } ) +TC_ALL_DAYS = "day:01,02,03,04,05,06,07,08,09,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31" + +WF_C3S_CMIP6_360DAY_CALENDAR = json.dumps( + { + "doc": "subset on cmip6", + "inputs": {"ds": [C3S_CMIP6_360DAY_CALENDAR_COLLECTION]}, + "outputs": {"output": "subset/output"}, + "steps": { + "subset": { + "run": "subset", + "in": { + "collection": "inputs/ds", + "time": "2015/2015", + "time_components": f"month:01,02,03|{TC_ALL_DAYS}", + }, + }, + }, + } +) WF_C3S_CORDEX = json.dumps( { @@ -347,6 +370,19 @@ def test_smoke_execute_c3s_cmip6_subset_by_point(wps, tmp_path): assert "rlds" in ds.variables +def test_smoke_execute_c3s_cmip6_360calendar_subset_by_point(wps, tmp_path): + inputs = [ + ("collection", C3S_CMIP6_360DAY_CALENDAR_COLLECTION), + ("time", "2015/2015"), + ("time_components", f"month:jan,feb,mar|{TC_ALL_DAYS}"), + ] + urls = wps.execute("subset", inputs) + assert len(urls) == 1 + assert "pr_day_HadGEM3-GC31-LL_ssp245_r1i1p1f3_gn_20150101-20150330.nc" in urls[0] + ds = open_dataset(urls[0], tmp_path) + assert "pr" in ds.variables + + def test_smoke_execute_c3s_cordex_subset_by_point(wps, tmp_path): inputs = [ ("collection", C3S_CORDEX_MON_COLLECTION), @@ -541,6 +577,15 @@ def test_smoke_execute_c3s_cmip6_regrid_orchestrate(wps): ) +def test_smoke_execute_c3s_cmip6_360day_calendar_orchestrate(wps): + inputs = [ + ("workflow", ComplexDataInput(WF_C3S_CMIP6_360DAY_CALENDAR)), + ] + urls = wps.execute("orchestrate", inputs) + assert len(urls) == 1 + assert "pr_day_HadGEM3-GC31-LL_ssp245_r1i1p1f3_gn_20150101-20150330.nc" in urls[0] + + def test_smoke_execute_c3s_cmip6_orchestrate_metadata(wps, tmp_path): inputs = [ ("workflow", ComplexDataInput(WF_C3S_CMIP6)), diff --git a/tests/test_utils.py b/tests/test_utils.py index 46b44081..90d713bf 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -3,7 +3,11 @@ from pywps.app import WPSRequest -from rook.utils.input_utils import resolve_to_file_paths, parse_wps_input +from rook.utils.input_utils import ( + resolve_to_file_paths, + parse_wps_input, + fix_time_components, +) from rook.utils.metalink_utils import build_metalink from .common import TESTS_HOME, MINI_ESGF_MASTER_DIR @@ -124,3 +128,30 @@ def test_parse_wps_input(): parse_wps_input(request.inputs, "time_components", default=None) == "year:1970,1980|month=01,02,03" ) + + +def test_fix_time_components(): + assert ( + fix_time_components("year:2000|month=01|day=01") == "year:2000|month=01|day=01" + ) + + all_days = "day:01,02,03,04,05,06,07,08,09,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31" + assert ( + fix_time_components(f"year:2001,2002|month=02|{all_days}") + == "year:2001,2002|month=02" + ) + + all_months = "month:jan,feb,mar,apr,may,jun,jul,aug,sep,oct,nov,dec" + assert ( + fix_time_components(f"year:2001,2002|{all_months}|{all_days}") + == "year:2001,2002" + ) + + all_months_2 = "month:01,02,03,04,05,06,07,08,09,10,11,12" + assert ( + fix_time_components(f"year:2001,2002|{all_months_2}|day:01,31") + == "year:2001,2002|day:01,31" + ) + + assert fix_time_components("") is None + assert fix_time_components(None) is None