Performance tuning

assume/common/base.py

@@ -9,6 +9,7 @@
 import numpy as np
 import pandas as pd
 
+from assume.common.fds import FastDatetimeSeries
 from assume.common.forecasts import Forecaster
 from assume.common.market_objects import MarketConfig, Orderbook, Product
 
@@ -55,7 +56,7 @@ def __init__(
         unit_operator: str,
         technology: str,
         bidding_strategies: dict[str, BaseStrategy],
-        index: pd.DatetimeIndex,
+        index: FastDatetimeSeries,
         node: str = "node0",
         forecaster: Forecaster = None,
         location: tuple[float, float] = (0.0, 0.0),
@@ -68,25 +69,28 @@ def __init__(
         self.location = location
         self.bidding_strategies: dict[str, BaseStrategy] = bidding_strategies
         self.index = index
-        self.outputs = defaultdict(lambda: pd.Series(0.0, index=self.index))
+        self.outputs = defaultdict(
+            lambda: FastDatetimeSeries(
+                self.index.start, self.index.end, self.index.freq
+            )
+        )
         # series does not like to convert from tensor to float otherwise
 
         # RL data stored as lists to simplify storing to the buffer
         self.outputs["rl_observations"] = []
         self.outputs["rl_actions"] = []
         self.outputs["rl_rewards"] = []
 
+        dt_index = self.index.get_date_list()
         # some data is stored as series to allow to store it in the outputs
-        self.outputs["actions"] = pd.Series(0.0, index=self.index, dtype=object)
-        self.outputs["exploration_noise"] = pd.Series(
-            0.0, index=self.index, dtype=object
-        )
-        self.outputs["reward"] = pd.Series(0.0, index=self.index, dtype=object)
+        self.outputs["actions"] = pd.Series(0.0, index=dt_index, dtype=object)
+        self.outputs["exploration_noise"] = pd.Series(0.0, index=dt_index, dtype=object)
+        self.outputs["reward"] = pd.Series(0.0, index=dt_index, dtype=object)
 
         if forecaster:
             self.forecaster = forecaster
         else:
-            self.forecaster = defaultdict(lambda: pd.Series(0.0, index=self.index))
+            self.forecaster = defaultdict(lambda: index.copy_empty())
 
     def calculate_bids(
         self,
@@ -128,7 +132,7 @@ def calculate_bids(
 
         return bids
 
-    def calculate_marginal_cost(self, start: pd.Timestamp, power: float) -> float:
+    def calculate_marginal_cost(self, start: datetime, power: float) -> float:
         """
         Calculates the marginal cost for the given power.
 
@@ -189,22 +193,20 @@ def calculate_generation_cost(
         """
 
         if start not in self.index:
-            start = self.index[0]
+            start = self.index.start
 
         product_type_mc = product_type + "_marginal_costs"
-        product_data = self.outputs[product_type].loc[start:end]
-
-        marginal_costs = product_data.index.map(
-            lambda t: self.calculate_marginal_cost(t, product_data.loc[t])
-        )
-        new_values = np.abs(marginal_costs * product_data.values)
-        self.outputs[product_type_mc].loc[start:end] = new_values
+        product_index = self.index.get_date_list(start, end)
+        for t in product_index:
+            value = self.outputs[product_type][t]
+            new_value = value * self.calculate_marginal_cost(t, value)
+            self.outputs[product_type_mc][t] = new_value
 
     def execute_current_dispatch(
         self,
-        start: pd.Timestamp,
-        end: pd.Timestamp,
-    ) -> pd.Series:
+        start: datetime,
+        end: datetime,
+    ) -> np.array:
         """
         Checks if the total dispatch plan is feasible.
 
@@ -218,7 +220,7 @@ def execute_current_dispatch(
         Returns:
             The volume of the unit within the given time range.
         """
-        return self.outputs["energy"][start:end]
+        return self.outputs["energy"][start, end]
 
     def get_output_before(self, dt: datetime, product_type: str = "energy") -> float:
         """
@@ -233,7 +235,7 @@ def get_output_before(self, dt: datetime, product_type: str = "energy") -> float
         Returns:
             The output before the given datetime.
         """
-        if dt - self.index.freq < self.index[0]:
+        if dt - self.index.freq < self.index.start:
             return 0
         else:
             return self.outputs[product_type].at[dt - self.index.freq]
@@ -322,8 +324,8 @@ class SupportsMinMax(BaseUnit):
     min_down_time: int = 0
 
     def calculate_min_max_power(
-        self, start: pd.Timestamp, end: pd.Timestamp, product_type: str = "energy"
-    ) -> tuple[pd.Series, pd.Series]:
+        self, start: datetime, end: datetime, product_type: str = "energy"
+    ) -> tuple[np.array, np.array]:
         """
         Calculates the min and max power for the given time period.
 
@@ -416,7 +418,7 @@ def get_operation_time(self, start: datetime) -> int:
         if len(arr) < 1:
             # before start of index
             return max_time
-        is_off = not arr.iloc[0]
+        is_off = not arr[0]
         run = 0
         for val in arr:
             if val == is_off:
@@ -440,7 +442,7 @@ def get_average_operation_times(self, start: datetime) -> tuple[float, float]:
         op_series = []
 
         before = start - self.index.freq
-        arr = self.outputs["energy"][self.index[0] : before][::-1] > 0
+        arr = self.outputs["energy"][self.index.start : before][::-1] > 0
 
         if len(arr) < 1:
             # before start of index
@@ -537,8 +539,8 @@ class SupportsMinMaxCharge(BaseUnit):
     efficiency_discharge: float
 
     def calculate_min_max_charge(
-        self, start: pd.Timestamp, end: pd.Timestamp, product_type="energy"
-    ) -> tuple[pd.Series, pd.Series]:
+        self, start: datetime, end: datetime, product_type="energy"
+    ) -> tuple[np.array, np.array]:
         """
         Calculates the min and max charging power for the given time period.
 
@@ -552,8 +554,8 @@ def calculate_min_max_charge(
         """
 
     def calculate_min_max_discharge(
-        self, start: pd.Timestamp, end: pd.Timestamp, product_type="energy"
-    ) -> tuple[pd.Series, pd.Series]:
+        self, start: datetime, end: datetime, product_type="energy"
+    ) -> tuple[FastDatetimeSeries, FastDatetimeSeries]:
         """
         Calculates the min and max discharging power for the given time period.
 
@@ -578,7 +580,7 @@ def get_soc_before(self, dt: datetime) -> float:
         Returns:
             float: The SoC before the given datetime.
         """
-        if dt - self.index.freq <= self.index[0]:
+        if dt - self.index.freq <= self.index.start:
             return self.initial_soc
         else:
             return self.outputs["soc"].at[dt - self.index.freq]

assume/common/fds.py

@@ -0,0 +1,144 @@
+# SPDX-FileCopyrightText: ASSUME Developers
+#
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+import math
+from datetime import datetime, timedelta
+
+import numpy as np
+import pandas as pd
+
+
+def freq_to_timedelta(freq: str) -> timedelta:
+    return timedelta(minutes=pd.tseries.frequencies.to_offset(freq).nanos / 1e9 // 60)
+
+
+class FastDatetimeSeries:
+    def __init__(self, start, end, freq, value=None, name=""):
+        self.start = start
+        self.end = end
+        self.freq = freq_to_timedelta(freq)
+        self.data = None
+        self.loc = self  # allow adjusting loc as well
+        self.at = self
+        if value is not None:
+            self.init_data(value)
+        # the name is not actually used other than for the compatibility of pandas import/export
+        self.name = name
+
+    def init_data(self, value=None):
+        if self.data is None:
+            self.data = self.get_numpy_date_range(
+                self.start, self.end, self.freq, value
+            )
+
+    def __getitem__(self, item: slice):
+        self.init_data()
+        if isinstance(item, slice):
+            start = self.get_idx_from_date(item.start)
+            # stop should be inclusive
+            # to comply with pandas behavior
+            stop = self.get_idx_from_date(item.stop) + 1
+            return self.data[start:stop]
+        elif isinstance(item, list):
+            if len(item) == len(self.data):
+                return self.data[item]
+            else:
+                return self.data[[self.get_idx_from_date(i) for i in item]]
+        else:
+            start = self.get_idx_from_date(item)
+
+            if start >= len(self.data):
+                return 0
+            else:
+                return self.data[start]
+
+    def __setitem__(self, item, value):
+        self.init_data()
+        if isinstance(item, slice):
+            start = self.get_idx_from_date(item.start)
+            stop = self.get_idx_from_date(item.stop)
+            self.data[start:stop] = value
+        elif isinstance(item, list | pd.Series):
+            if len(item) == len(self.data):
+                self.data[item] = value
+            else:
+                for i in item:
+                    start = self.get_idx_from_date(i)
+                    self.data[start] = value
+        else:
+            start = self.get_idx_from_date(item)
+            self.data[start] = value
+
+    def get_date_list(self, start=None, end=None):
+        self.init_data()
+        if start is None or start < self.start:
+            start = self.start
+        if end is None or end > self.end:
+            end = self.end
+        # takes next start value at most
+        start_idx = self.get_idx_from_date(start)
+        hour_count = self.get_idx_from_date(end) + 1 - start_idx
+        if hour_count < 0:
+            raise ValueError("start must be before end")
+
+        start = self.start + start_idx * self.freq
+
+        return [start + i * self.freq for i in range(hour_count)]
+
+    def get_idx_from_date(self, date: datetime):
+        if date is None:
+            return None
+        idx = (date - self.start) / self.freq
+        # todo check if modulo is 0 - else this is not a valid multiple of freq
+        # if idx < 0:
+        #    raise ValueError("date %s is before start %s", date, self.start)
+        return math.ceil(idx)
+
+    def get_numpy_date_range(self, start, end, freq, value=None):
+        """
+        Get the index of a date range.
+        """
+        if value is None:
+            value = 0.0
+        hour_count = self.get_idx_from_date(end) + 1
+        return np.full(hour_count, value)
+
+    def as_df(self, name, start=None, end=None):
+        self.init_data()
+        return pd.DataFrame(
+            self[start:end], index=self.get_date_list(start, end), columns=[self.name]
+        )
+
+    @staticmethod
+    def from_series(series):
+        if series.index.freq:
+            freq = series.index.freq
+        else:
+            freq = series.index[1] - series.index[0]
+        return FastDatetimeSeries(
+            series.index[0], series.index[-1], freq, series.values, series.name
+        )
+
+    def __truediv__(self, other: float):
+        self.init_data()
+        return self.data / other
+
+    def __mul__(self, other: float):
+        self.init_data()
+        return self.data * other
+
+    def __len__(self):
+        self.init_data()
+        return len(self.data)
+
+    def copy_empty(self, value=0.0, name=""):
+        return FastDatetimeSeries(self.start, self.end, self.freq, value, name)
+
+    def __contains__(self, other: datetime):
+        if self.start > other:
+            return False
+        if self.end < other:
+            return False
+        # TODO check if self.freq fits
+        return True