Dev regrid op

.github/workflows/main.yml

@@ -5,22 +5,32 @@ on: [ push, pull_request ]
 jobs:
   build:
 
+    name: Conda Build with Python${{ matrix.python-version }}
     runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: ["3.9", "3.10", "3.11"]
+    defaults:
+      run:
+        shell: bash -l {0}
     steps:
-    - name: Checkout repository and submodules
-      uses: actions/checkout@v2
-      with:
-        submodules: recursive
+    - uses: actions/checkout@v3
     - name: Install packages
       run: |
         sudo apt-get -y install pandoc graphviz
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+    - name: Setup Conda (Micromamba) with Python${{ matrix.python-version }}
+      uses: mamba-org/setup-micromamba@v1
       with:
-        python-version: ${{ matrix.python-version }}
+        cache-downloads: true
+        cache-environment: true
+        environment-file: environment.yml
+        create-args: >-
+          conda
+          python=${{ matrix.python-version }}
+    - name: Conda and Mamba versions
+      run: |
+        conda --version
+        echo "micromamba: $(micromamba --version)"
     - name: Install requirements 📦
       run: |
         python -m pip install --upgrade pip

environment.yml

@@ -4,34 +4,37 @@ channels:
 - defaults
 dependencies:
 - pip
-- python>=3.8
-- pywps>=4.5.2,<4.7
+- python >=3.9
+- pywps >=4.5.2,<4.7
 - jinja2
 - click
 - psutil
 - requests
 # daops
-- cftime>=1.2.1
-- xarray>=0.21
-- cf_xarray>=0.7
-- dask>=2021.12
-- netcdf4>=1.4
-- bottleneck>=1.3.1,<1.4
-- daops>=0.8.1,<0.9
-- clisops>=0.9.6,<0.11
-- roocs-utils>=0.6.4,<0.7
+- cftime >=1.2.1
+- xarray >=0.21,<2023.3.0 # https://github.com/pydata/xarray/issues/7794
+- xesmf >=0.8.2
+- cf_xarray >=0.7
+- dask >=2021.12
+- netcdf4 >=1.4
+- bottleneck >=1.3.1,<1.4
+- daops >=0.10.0,<0.11
+- clisops >=0.12.0,<0.13
+- roocs-utils >=0.6.4,<0.7
+- roocs-grids >=0.1.2
 # workflow
 - networkx
 # provenance
-- prov>=2.0.0
+- prov >=2.0.0
 - pydot
 - graphviz
 # catalog
 - intake
-- pandas>=1.3.5,<1.4
-- sqlalchemy>=1.4.45,<2.0
+- pandas >=1.3.5,<2.0
+- sqlalchemy >=1.4.45,<2.0
 - aiohttp
 # dashboard
-- bokeh>=2.4.2,<2.5
+- bokeh >=2.4.2,<2.5
 # tests
 - pytest
+- beautifulsoup4

requirements.txt

@@ -3,9 +3,12 @@ jinja2
 click
 psutil
 # daops
-daops>=0.8.1,<0.9
-clisops>=0.9.6,<0.11
+daops>=0.10.0,<0.11
+# daops @ git+https://github.com/roocs/daops.git@regrid-main-ce#egg=daops
+clisops>=0.12.0,<0.13
+# clisops @ git+https://github.com/roocs/clisops.git@regrid-main#egg=clisops
 roocs-utils>=0.6.4,<0.7
+roocs_grids>=0.1.2
 xarray>=0.21
 cf-xarray>=0.7
 dask[complete]
@@ -17,7 +20,7 @@ prov>=2.0.0
 pydot
 # catalog
 intake
-pandas>=1.3.5,<1.4
+pandas>=1.3.5,<2.0
 SQLAlchemy>=1.4.45,<2.0
 aiohttp
 # dashboard

rook/director/director.py

@@ -108,8 +108,8 @@ def _resolve(self):
         if self.inputs.get("apply_fixes") and self.requires_fixes():
             return
 
-        # TODO: quick fix for average and concat. Don't use original files for these operators.
-        if "dims" in self.inputs or "freq" in self.inputs:
+        # TODO: quick fix for average, regrid and concat. Don't use original files for these operators.
+        if "dims" in self.inputs or "freq" in self.inputs or "grid" in self.inputs:
             return
 
         # Finally, check if the subset requirements can align with whole datasets

rook/utils/regrid_utils.py

@@ -1,12 +1,8 @@
 def run_regrid(args):
-    from daops.ops.average import average_over_dims
+    from daops.ops.regrid import regrid
 
     args["apply_fixes"] = False
-    args["dims"] = ["latitude", "longitude"]
 
-    # remove regrid arguments
-    args.pop("method", None)
-    args.pop("grid", None)
+    result = regrid(**args)
 
-    result = average_over_dims(**args)
     return result.file_uris

tests/common.py

@@ -1,9 +1,8 @@
 import os
 
-# import tempfile
-
 from jinja2 import Template
 from pathlib import Path
+from bs4 import BeautifulSoup
 from pywps import get_ElementMakerForVersion
 from pywps.app.basic import get_xpath_ns
 from pywps.tests import WpsClient, WpsTestResponse
@@ -106,3 +105,10 @@ def get_output(doc):
             output[identifier_el.text] = data_el[0].text
 
     return output
+
+
+def extract_paths_from_metalink(path):
+    path = path.replace("file://", "")
+    doc = BeautifulSoup(open(path, "r").read(), "xml")
+    paths = [el.text.replace("file://", "") for el in doc.find_all("metaurl")]
+    return paths

tests/smoke/test_smoke_checks.py

@@ -98,6 +98,28 @@
     }
 )
 
+WF_C3S_CMIP6_REGRID = json.dumps(
+    {
+        "doc": "subset+regrid on cmip6",
+        "inputs": {"ds": [C3S_CMIP6_MON_COLLECTION]},
+        "outputs": {"output": "regrid/output"},
+        "steps": {
+            "subset": {
+                "run": "subset",
+                "in": {"collection": "inputs/ds", "time": "2016/2016"},
+            },
+            "regrid": {
+                "run": "regrid",
+                "in": {
+                    "collection": "subset/output",
+                    "method": "nearest_s2d",
+                    "grid": "1deg",
+                },
+            },
+        },
+    }
+)
+
 
 WF_C3S_CORDEX = json.dumps(
     {
@@ -450,6 +472,20 @@ def test_smoke_execute_c3s_cmip6_weighted_average(wps):
     )
 
 
+def test_smoke_execute_c3s_cmip6_regrid(wps):
+    inputs = [
+        ("collection", C3S_CMIP6_MON_COLLECTION),
+        ("grid", "auto"),
+        ("method", "nearest_s2d"),
+    ]
+    urls = wps.execute("regrid", inputs)
+    assert len(urls) == 1
+    assert (
+        "rlds_Amon_INM-CM5-0_ssp245_r1i1p1f1_gr_20150116-21001216_regrid-nearest_s2d-120x179_cells_grid.nc"
+        in urls[0]
+    )
+
+
 def test_smoke_execute_c3s_cmip5_orchestrate(wps):
     inputs = [
         ("workflow", ComplexDataInput(WF_C3S_CMIP5)),
@@ -479,6 +515,18 @@ def test_smoke_execute_c3s_cmip6_weighted_average_orchestrate(wps):
     )
 
 
+def test_smoke_execute_c3s_cmip6_regrid_orchestrate(wps):
+    inputs = [
+        ("workflow", ComplexDataInput(WF_C3S_CMIP6_REGRID)),
+    ]
+    urls = wps.execute("orchestrate", inputs)
+    assert len(urls) == 1
+    assert (
+        "rlds_Amon_INM-CM5-0_ssp245_r1i1p1f1_gr_20160116-20161216_regrid-nearest_s2d-180x360_cells_grid.nc"
+        in urls[0]
+    )
+
+
 def test_smoke_execute_c3s_cmip6_orchestrate_metadata(wps, tmp_path):
     inputs = [
         ("workflow", ComplexDataInput(WF_C3S_CMIP6)),

tests/test_wps_average_weighted.py

@@ -1,21 +1,13 @@
 import pytest
 
 import xarray as xr
-from bs4 import BeautifulSoup
 
 from pywps import Service
 from pywps.tests import assert_response_success, client_for
 
 from rook.processes.wps_average_weighted import WeightedAverage
 
-from .common import PYWPS_CFG, get_output
-
-
-def extract_paths_from_metalink(path):
-    path = path.replace("file://", "")
-    doc = BeautifulSoup(open(path, "r").read(), "xml")
-    paths = [el.text.replace("file://", "") for el in doc.find_all("metaurl")]
-    return paths
+from .common import PYWPS_CFG, get_output, extract_paths_from_metalink
 
 
 def assert_weighted_average(path):

tests/test_wps_regrid.py

@@ -1,16 +1,27 @@
 import pytest
 
+import xarray as xr
+
 from pywps import Service
 from pywps.tests import assert_response_success, client_for
 
 from rook.processes.wps_regrid import Regrid
 
-from .common import PYWPS_CFG, get_output
+from .common import PYWPS_CFG, get_output, extract_paths_from_metalink
+
+
+def assert_regrid(path):
+    assert "meta4" in path
+    paths = extract_paths_from_metalink(path)
+    assert len(paths) > 0
+    print(paths)
+    ds = xr.open_dataset(paths[0])
+    assert "time" in ds.coords
 
 
 def test_wps_regrid_cmip6():
     client = client_for(Service(processes=[Regrid()], cfgfiles=[PYWPS_CFG]))
-    datainputs = "collection=c3s-cmip6.ScenarioMIP.INM.INM-CM5-0.ssp245.r1i1p1f1.Amon.rlds.gr1.v20190619"
+    datainputs = "collection=CMIP6.CMIP.IPSL.IPSL-CM6A-LR.historical.r1i1p1f1.Amon.rlds.gr.v20180803"
     datainputs += ";method=nearest_s2d"
     datainputs += ";grid=auto"
     resp = client.get(
@@ -19,3 +30,4 @@ def test_wps_regrid_cmip6():
     print(resp)
     assert_response_success(resp)
     assert "output" in get_output(resp.xml)
+    assert_regrid(path=get_output(resp.xml)["output"])