support shapefile, customize drepr resources and updating climate example

Author: Binh Vu

drepr/Cargo.lock

@@ -1,7 +1,7 @@
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 [package]
 name = "engine"
-version = "1.0.6"   # ___PKG_VERSION___: DO NOT MODIFY the version here. Update it via version_manager.py!
+version = "1.0.7"   # ___PKG_VERSION___: DO NOT MODIFY the version here. Update it via version_manager.py!
 authors = ["Binh Vu <[email protected]>"]
 edition = "2018"
 

drepr/engine/Cargo.toml

@@ -13,7 +13,13 @@ pub enum Resource {
   #[serde(rename = "netcdf4")]
   NetCDF4(usize),
   #[serde(rename = "np-dict")]
-  NPDict(usize)
+  NPDict(usize),
+  #[serde(rename = "geotiff")]
+  GeoTIFF(usize),
+  #[serde(rename = "shapefile")]
+  Shapefile(usize),
+  #[serde(rename = "container")]
+  Container(usize)
 }
 
 #[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]

drepr/engine/src/lang/resource.rs

@@ -1,6 +1,8 @@
 from drepr.executors.readers.geotiff import GeoTIFFReader
 from drepr.executors.readers.netcdf import NetCDF4Reader
 from drepr.executors.readers.np_dict import NPDictReader
+from drepr.executors.readers.reader_container import ReaderContainer
+from drepr.executors.readers.shapefile import ShapefileReader
 from drepr.models import DRepr, ResourceType, RangeAlignment, PMap
 import numpy as np
 
@@ -56,6 +58,10 @@ def execute(cls, ds_model: DRepr, resource_file: str):
             reader = GeoTIFFReader.from_file(resource_file)
         elif resource.type == ResourceType.NPDict:
             reader = NPDictReader.from_file(resource_file)
+        elif resource.type == ResourceType.Shapefile:
+            reader = ShapefileReader.from_file(resource_file)
+        elif resource.type == ResourceType.Container:
+            reader = ReaderContainer.get_instance().get(resource_file)
         else:
             raise NotImplementedError()
 
@@ -64,12 +70,32 @@ def execute(cls, ds_model: DRepr, resource_file: str):
         for preprocess_fn in ds_model.preprocessing:
             if preprocess_fn.type == PreprocessingType.pmap:
                 steps = preprocess_fn.value.path.steps
-                assert all(isinstance(step, IndexExpr) for step in steps), "Range selection should use numpy map"
-
-                index = [step.val for step in steps]
+                n_range_exprs = sum(int(not isinstance(step, IndexExpr)) for step in steps)
                 fn = PyExec.compile(preprocess_fn.value.code)
-                value = fn.exec(reader.get_value(index), index, context)
-                reader.set_value(index, value)
+
+                if n_range_exprs == 0:
+                    index = [step.val for step in steps]
+                    value = fn.exec(reader.get_value(index), index, context)
+                    reader.set_value(index, value)
+                else:
+                    # TODO: improve it
+                    range_indices = [i for i in range(len(steps)) if isinstance(steps[i], RangeExpr)]
+                    index = [step.start if isinstance(step, RangeExpr) else step.val for step in steps]
+                    if n_range_exprs == 1:
+                        range_idx = range_indices[0]
+                        if steps[range_idx].end is None:
+                            if range_idx == 0:
+                                end = reader.len_range()
+                            else:
+                                end = len(reader.get_value(index[:range_idx]))
+                        else:
+                            end = steps[range_idx].end
+                        for i in range(steps[range_idx].start, end, steps[range_idx].step):
+                            index[range_idx] = i
+                            value = fn.exec(reader.get_value(index), index, context)
+                            reader.set_value(index, value)
+                    else:
+                        raise NotImplementedError()
             else:
                 raise Exception("You found a bug")
 
@@ -100,8 +126,10 @@ def execute(cls, ds_model: DRepr, resource_file: str):
                         count += 1
                     else:
                         step2dim.append(None)
+
                 new_attrs[f"dnode:{attr.id}"] = ArrayAttr(f"dnode:{attr.id}", attrs[attr.id], path, step2dim, nodata)
             else:
+                # TODO: this is wrong if the value are the list!
                 new_attrs[f"dnode:{attr.id}"] = ScalarAttr(f"dnode:{attr.id}", attrs[attr.id])
             # assert all(isinstance(step, IndexExpr) for step in attr.path.steps)
             # index = [step.val for step in steps]

pydrepr/drepr/executors/cf_convention_map/cf_convention_map.py

@@ -31,6 +31,9 @@ def from_file(cls, infile: str):
     def len(self) -> int:
         return self.data.shape[0] + 1
 
+    def len_range(self) -> int:
+        return self.data.shape[0]
+
     def select(self, steps: List[Union[IndexExpr, RangeExpr]]):
         if isinstance(steps[0], IndexExpr) and steps[0].val == '@':
             # select metadata

pydrepr/drepr/executors/cf_convention_map/geotiff_map.py

@@ -40,6 +40,9 @@ def len(self) -> int:
         # count the global metadata at '@'
         return len(self.dataset.variables.keys()) + 1
 
+    def len_range(self) -> int:
+        raise Exception("Error in your drepr model")
+
     def get_value(self, index: List[Index]):
         if index[0] == '@':
             # read metadata
@@ -71,7 +74,6 @@ def set_value(self, index: List[Index], value):
             ptr = ptr[index[i]]
         ptr[index[-1]] = value
 
-
     def select(self, steps: List[Union[IndexExpr, RangeExpr]]):
         # steps.0 must be index because there is no structure change
         assert isinstance(steps[0], IndexExpr)

pydrepr/drepr/executors/cf_convention_map/netcdf_map.py

@@ -33,4 +33,7 @@ def select(self, steps: List[Union[IndexExpr, RangeExpr]]):
         return self.data[steps[0].val]
 
     def len(self) -> int:
-        return len(self.data)
+        return len(self.data)
+
+    def len_range(self) -> int:
+        raise Exception("Error in your drepr model")

pydrepr/drepr/executors/readers/geotiff.py

@@ -35,3 +35,10 @@ def len(self) -> int:
         Get length of the current node.
         """
         pass
+
+    @abstractmethod
+    def len_range(self) -> int:
+        """
+        Get length of the current node (range only).
+        """
+        pass

pydrepr/drepr/executors/readers/netcdf.py

@@ -0,0 +1,26 @@
+from typing import List, Dict, Tuple, Callable, Any, Optional
+
+from drepr.executors.readers.ra_reader import NDArrayReader
+
+
+class ReaderContainer:
+    """
+    This contains the list of ndarray reader that can be injected by the users so that they can extend the
+    coverage of the system
+    """
+    instance = None
+
+    def __init__(self):
+        self.readers = {}
+
+    @staticmethod
+    def get_instance():
+        if ReaderContainer.instance is None:
+            ReaderContainer.instance = ReaderContainer()
+        return ReaderContainer.instance
+
+    def set(self, reader_id: str, reader: NDArrayReader):
+        self.readers[reader_id] = reader
+
+    def get(self, reader_id: str):
+        return self.readers[reader_id]