Feature/rework periodic value deserialisation

setup.py

@@ -70,6 +70,7 @@ def read(*names, **kwargs):
         # "oemof.solph>=0.5.1",
         "oemof.solph==0.5.2dev0",
         "pandas>=0.22",
+        "oemof.network==0.5.0a4",
         "paramiko",
         "toml",
     ],

src/oemof/tabular/__main__.py

@@ -8,6 +8,7 @@
 - https://docs.python.org/2/using/cmdline.html#cmdoption-m
 - https://docs.python.org/3/using/cmdline.html#cmdoption-m
 """
+
 from tabular.cli import main
 
 if __name__ == "__main__":

src/oemof/tabular/cli.py

@@ -2,6 +2,7 @@
 Module that contains the command line app.
 
 """
+
 import collections
 import copy
 

src/oemof/tabular/datapackage/aggregation.py

@@ -88,9 +88,9 @@ def temporal_skip(datapackage, n, path="/tmp", name=None, *args):
     r = Resource({"path": "data/temporal.csv"})
     r.infer()
 
-    r.descriptor[
-        "description"
-    ] = "Temporal selection based on skipped timesteps. Skipped n={}".format(n)
+    r.descriptor["description"] = (
+        "Temporal selection based on skipped timesteps. Skipped n={}".format(n)
+    )
 
     # Update meta-data of copied package
     cp = Package("datapackage.json")
@@ -212,9 +212,9 @@ def temporal_clustering(datapackage, n, path="/tmp", how="daily"):
     r = Resource({"path": "data/temporal.csv"})
     r.infer()
     # TODO: Add meta-data description
-    r.descriptor[
-        "description"
-    ] = "Temporal selection based on hierachical clustering..."
+    r.descriptor["description"] = (
+        "Temporal selection based on hierachical clustering..."
+    )
 
     # Update meta-data of copied package
     cp = Package("datapackage.json")

src/oemof/tabular/datapackage/reading.py

@@ -13,6 +13,7 @@
 import collections.abc as cabc
 import json
 import re
+import typing
 import warnings
 from decimal import Decimal
 from itertools import chain, groupby, repeat
@@ -117,14 +118,14 @@ def deserialize_energy_system(cls, path, typemap={}, attributemap={}):
         FLOW_TYPE: HSN,
     }
 
-    for k, v in default_typemap.items():
-        typemap[k] = typemap.get(k, v)
+    for k, value in default_typemap.items():
+        typemap[k] = typemap.get(k, value)
 
     if attributemap.get(object) is None:
         attributemap[object] = {"name": "label"}
 
-    for k, v in attributemap.items():
-        if v.get("name") is None:
+    for k, value in attributemap.items():
+        if value.get("name") is None:
             attributemap[k]["name"] = "label"
 
     package = dp.Package(path)
@@ -309,13 +310,15 @@ def create(cls, init, attributes):
 
     data["buses"] = {
         name: create(
-            mapping
-            if mapping
-            else raisestatement(
-                ValueError,
-                "Typemap is missing a mapping for '{}'.".format(
-                    bus.get("type", "bus")
-                ),
+            (
+                mapping
+                if mapping
+                else raisestatement(
+                    ValueError,
+                    "Typemap is missing a mapping for '{}'.".format(
+                        bus.get("type", "bus")
+                    ),
+                )
             ),
             {"label": name},
             bus["parameters"],
@@ -433,15 +436,108 @@ def create_periodic_values(values, periods_index):
 
         return periodic_values.tolist()
 
-    def create_yearly_values(values, years):
+    def create_yearly_values(
+        values: typing.Iterable[float], period_years: typing.Iterable[int]
+    ):
+        """
+        Creates a value for every year (between two periods)
+        Value of period is continued until next period
+        Parameters
+        ----------
+        values values to be interpolated
+        years years of periods
+
+        Returns list
+        -------
+
+        """
         results = pd.Series()
-        for i in range(len(years) - 1):
-            diff = years[i + 1] - years[i]
+        for i in range(len(period_years) - 1):
+            diff = period_years[i + 1] - period_years[i]
             period_results = pd.Series(repeat(values[i], diff))
             results = pd.concat([results, period_results])
         results = pd.concat([results, pd.Series(values[-1])])
         return results.tolist()
 
+    def convert_and_deserialize_dtypes(facade, period_data):
+        """
+        Depending on dtype and content.
+        Periodically changing values [given as array] are either unpacked into
+            - full periods
+            - yearly values (between periods)
+            - kept as periodical values
+
+        Decision happens based on
+            - value
+            - name
+            - entry in yearly/periodical values list.
+
+        Parameters
+        ----------
+        facade
+        period_data
+
+        Returns
+        -------
+        facade
+        """
+
+        yearly_values = ["fixed_costs", "marginal_costs"]
+        periodical_values = [
+            "capacity",
+            "capacity_cost",
+            "capacity_potential",
+            "storage_capacity",
+        ]
+
+        for value_name, value in facade.items():
+            if isinstance(value, Decimal):
+                facade[value_name] = float(value)
+            # check if multi-period and value is list
+            if period_data and isinstance(value, list):
+                # check if length of list equals number of periods
+                if len(value) == len(period_data["periods"]):
+                    if value_name in periodical_values:
+                        # special period parameters don't need to be
+                        # converted into timeseries
+                        facade[value_name] = [
+                            float(vv) if isinstance(vv, Decimal) else vv
+                            for vv in value
+                        ]
+                        continue
+                    elif value_name in yearly_values:
+                        # special period parameter need to be
+                        # converted into timeseries with value for each
+                        # year
+                        facade[value_name] = create_yearly_values(
+                            value, period_data["years"]
+                        )
+                        msg = (
+                            f"\nThe parameter '{value_name}' of a "
+                            f"'{facade['type']}' facade is converted "
+                            "into a yearly list. This might not be "
+                            "possible for every parameter and lead to "
+                            "ambiguous error messages.\nPlease be "
+                            "aware, when using this feature!"
+                        )
+                        warnings.warn(msg, UserWarning)
+
+                    else:
+                        # create timeseries with periodic values
+                        facade[value_name] = create_periodic_values(
+                            value, period_data["periods"]
+                        )
+                        msg = (
+                            f"\nThe parameter '{value_name}' of a "
+                            f"'{facade['type']}' facade is converted "
+                            "into a periodic timeseries. This might "
+                            "not be possible for every parameter and "
+                            "lead to ambiguous error messages.\nPlease"
+                            " be aware, when using this feature!"
+                        )
+                        warnings.warn(msg, UserWarning)
+        return facade
+
     facades = {}
     for r in package.resources:
         if all(
@@ -465,58 +561,14 @@ def create_yearly_values(values, years):
                 fk["fields"]: fk["reference"]
                 for fk in r.descriptor["schema"].get("foreignKeys", ())
             }
+
             for facade in facade_data:
                 # convert decimal to float
-                for f, v in facade.items():
-                    if isinstance(v, Decimal):
-                        facade[f] = float(v)
-                    # check if multi-period and value is list
-                    if period_data and isinstance(v, list):
-                        # check if length of list equals number of periods
-                        if len(v) == len(period_data["periods"]):
-                            if f in ["capacity_costs"]:
-                                # special period parameters don't need to be
-                                # converted into timeseries
-                                facade[f] = [
-                                    float(vv)
-                                    if isinstance(vv, Decimal)
-                                    else vv
-                                    for vv in v
-                                ]
-                                continue
-                            elif f in ["fixed_costs"]:
-                                # special period parameter need to be
-                                # converted into timeseries with value for each
-                                # year
-                                facade[f] = create_yearly_values(
-                                    v, period_data["years"]
-                                )
-                                msg = (
-                                    f"\nThe parameter '{f}' of a "
-                                    f"'{facade['type']}' facade is converted "
-                                    "into a yearly list. This might not be "
-                                    "possible for every parameter and lead to "
-                                    "ambiguous error messages.\nPlease be "
-                                    "aware, when using this feature!"
-                                )
-                                warnings.warn(msg, UserWarning)
-
-                            else:
-                                # create timeseries with periodic values
-                                facade[f] = create_periodic_values(
-                                    v, period_data["periods"]
-                                )
-                                msg = (
-                                    f"\nThe parameter '{f}' of a "
-                                    f"'{facade['type']}' facade is converted "
-                                    "into a periodic timeseries. This might "
-                                    "not be possible for every parameter and "
-                                    "lead to ambiguous error messages.\nPlease"
-                                    " be aware, when using this feature!"
-                                )
-                                warnings.warn(msg, UserWarning)
+
                 read_facade(
-                    facade,
+                    convert_and_deserialize_dtypes(
+                        facade=facade, period_data=period_data
+                    ),
                     facades,
                     create,
                     typemap,

src/oemof/tabular/examples/datapackages/dispatch/scripts/infer.py

@@ -2,6 +2,7 @@
 Run this script from the root directory of the datapackage to update
 or create meta data.
 """
+
 from oemof.tabular.datapackage import building
 
 # This part is for testing only: It allows to pass

src/oemof/tabular/examples/datapackages/dispatch_multi_period/scripts/infer.py

@@ -2,6 +2,7 @@
 Run this script from the root directory of the datapackage to update
 or create meta data.
 """
+
 from oemof.tabular.datapackage import building
 
 # This part is for testing only: It allows to pass

src/oemof/tabular/examples/datapackages/dispatch_multi_period_periodic_values/README.md

@@ -0,0 +1,5 @@
+# Dispatch example for oemof-tabular
+
+Run `scripts/infer.py` from the datapackage root directory to add the
+meta data file `datapackage.json` after updating the resources of the
+datapackage.

src/oemof/tabular/examples/datapackages/dispatch_multi_period_periodic_values/data/elements/bus.csv

@@ -0,0 +1,5 @@
+name;type;balanced
+bus0;bus;true
+bus1;bus;true
+heat-bus;bus;false
+gas-bus;bus;false

src/oemof/tabular/examples/datapackages/dispatch_multi_period_periodic_values/data/elements/dispatchable.csv

@@ -0,0 +1,4 @@
+name;type;carrier;tech;capacity;bus;marginal_cost;profile;output_parameters
+gas;dispatchable;gas;gt;100;bus1;40;1;{"full_load_time_max": 2000}
+coal;dispatchable;coal;st;100;bus0;40;1;{}
+lignite;dispatchable;lignite;st;50;bus0;20;1;{}

src/oemof/tabular/examples/datapackages/dispatch_multi_period_periodic_values/data/elements/link.csv

@@ -0,0 +1,3 @@
+name;type;capacity;capacity_cost;loss;from_bus;to_bus
+conn1;link;100;[10, 9, 8];0.05;bus0;bus1
+conn2;link;10;[0, 0, 0];0.05;heat-bus;gas-bus

src/oemof/tabular/examples/datapackages/dispatch_multi_period_periodic_values/data/elements/load.csv

@@ -0,0 +1,3 @@
+name;amount;profile;type;bus
+demand0;5000;electricity-load-profile;load;bus0
+demand1;1000;electricity-load-profile;load;bus1

src/oemof/tabular/examples/datapackages/dispatch_multi_period_periodic_values/data/elements/storage.csv

@@ -0,0 +1,3 @@
+name;carrier;tech;storage_capacity;capacity;capacity_cost;storage_capacity_initial;type;bus
+el-storage1;lithium;battery;100;[10,12,13];[10,12,13];0.5;storage;bus0
+el-storage2;lithium;battery;200;[10,12,13];[16,13,14];0.2;storage;bus0

src/oemof/tabular/examples/datapackages/dispatch_multi_period_periodic_values/data/elements/volatile.csv

@@ -0,0 +1,3 @@
+name;type;carrier;tech;capacity;capacity_cost;bus;marginal_cost;profile;output_parameters
+wind;volatile;wind;onshore;50;;bus0;[2, 1, 0];wind-profile;{}
+pv;volatile;solar;pv;20;;bus1;[2, 1, 0];pv-profile;{}

src/oemof/tabular/examples/datapackages/dispatch_multi_period_periodic_values/data/periods/periods.csv

@@ -0,0 +1,10 @@
+timeindex;periods; timeincrement
+2011-01-01T00:00:00Z;0;1
+2011-01-01T01:00:00Z;0;1
+2011-01-01T02:00:00Z;0;1
+2035-01-01T00:00:00Z;1;1
+2035-01-01T01:00:00Z;1;1
+2035-01-01T02:00:00Z;1;1
+2050-01-01T00:00:00Z;2;1
+2050-01-01T01:00:00Z;2;1
+2050-01-01T02:00:00Z;2;1

src/oemof/tabular/examples/datapackages/dispatch_multi_period_periodic_values/data/sequences/load_profile.csv

@@ -0,0 +1,10 @@
+timeindex;electricity-load-profile
+2011-01-01T00:00:00Z;0.000745659236
+2011-01-01T01:00:00Z;0.000709651546
+2011-01-01T02:00:00Z;0.00068564642
+2035-01-01T00:00:00Z;0.000745659236
+2035-01-01T01:00:00Z;0.000709651546
+2035-01-01T02:00:00Z;0.00068564642
+2050-01-01T00:00:00Z;0.000745659236
+2050-01-01T01:00:00Z;0.000709651546
+2050-01-01T02:00:00Z;0.00068564642

src/oemof/tabular/examples/datapackages/dispatch_multi_period_periodic_values/data/sequences/volatile_profile.csv

@@ -0,0 +1,10 @@
+timeindex;wind-profile;pv-profile
+2011-01-01T00:00:00Z;0.147532;0
+2011-01-01T01:00:00Z;0.184181;0
+2011-01-01T02:00:00Z;0.223937;0
+2035-01-01T00:00:00Z;0.147532;0
+2035-01-01T01:00:00Z;0.184181;0
+2035-01-01T02:00:00Z;0.223937;0
+2050-01-01T00:00:00Z;0.147532;0
+2050-01-01T01:00:00Z;0.184181;0
+2050-01-01T02:00:00Z;0.223937;0