From ed3c86b8aa19bb258818d4d45692f27c6ce4d9a1 Mon Sep 17 00:00:00 2001
From: Christoph Toennis <christoph.toennis1988@gmail.com>
Date: Tue, 29 Oct 2024 11:32:19 +0100
Subject: [PATCH 01/30] Adding the chunk function

---
 src/ctapipe/monitoring/chunk_select.py | 78 ++++++++++++++++++++++++++
 1 file changed, 78 insertions(+)
 create mode 100644 src/ctapipe/monitoring/chunk_select.py

diff --git a/src/ctapipe/monitoring/chunk_select.py b/src/ctapipe/monitoring/chunk_select.py
new file mode 100644
index 00000000000..f90253ea527
--- /dev/null
+++ b/src/ctapipe/monitoring/chunk_select.py
@@ -0,0 +1,78 @@
+import numpy as np
+
+from ctapipe.core import Component, traits
+
+
+class ChunkFunction(Component):
+
+    """
+    Chunk Function for the gain and pedestals
+    Interpolates data so that for each time the value from the latest starting
+    valid chunk is given or the earliest available still valid chunk for any
+    pixels without valid data.
+
+    Parameters
+    ----------
+    input_table : astropy.table.Table
+        Table of calibration values, expected columns
+        are always ``start_time`` as ``Start_Time`` column,
+        ``end_time`` as ``End_Time`` column
+        other columns for the data that is to be selected
+    """
+
+    bounds_error = traits.Bool(
+        default_value=True,
+        help="If true, raises an exception when trying to extrapolate out of the given table",
+    ).tag(config=True)
+
+    extrapolate = traits.Bool(
+        help="If bounds_error is False, this flag will specify whether values outside"
+        "the available values are filled with nan (False) or extrapolated (True).",
+        default_value=False,
+    ).tag(config=True)
+
+    def __init__(
+        self,
+        input_table,
+        fill_value="extrapolate",
+    ):
+        input_table.sort("start_time")
+        self.start_times = input_table["start_time"]
+        self.end_times = input_table["end_time"]
+        self.values = input_table["values"]
+
+    def __call__(self, time):
+        if time < self.start_times[0]:
+            if self.bounds_error:
+                raise ValueError("below the interpolation range")
+
+            if self.extrapolate:
+                return self.values[0]
+
+            else:
+                a = np.empty(self.values[0].shape)
+                a[:] = np.nan
+                return a
+
+        elif time > self.end_times[-1]:
+            if self.bounds_error:
+                raise ValueError("above the interpolation range")
+
+            if self.extrapolate:
+                return self.values[-1]
+
+            else:
+                a = np.empty(self.values[0].shape)
+                a[:] = np.nan
+                return a
+
+        else:
+            i = np.searchsorted(
+                self.start_times, time, side="left"
+            )  # Latest valid chunk
+            j = np.searchsorted(
+                self.end_times, time, side="left"
+            )  # Earliest valid chunk
+            return np.where(
+                np.isnan(self.values[i - 1]), self.values[j], self.values[i - 1]
+            )  # Give value for latest chunk unless its nan. If nan give earliest chunk value

From 15efaeb02295e86901cb39800587bcde6768d1a8 Mon Sep 17 00:00:00 2001
From: Christoph Toennis <christoph.toennis1988@gmail.com>
Date: Tue, 29 Oct 2024 11:44:22 +0100
Subject: [PATCH 02/30] Reverting to the previous implementation

---
 src/ctapipe/monitoring/chunk_select.py  |  78 ----------
 src/ctapipe/monitoring/interpolation.py | 190 +++++++++++++++++++++++-
 2 files changed, 186 insertions(+), 82 deletions(-)
 delete mode 100644 src/ctapipe/monitoring/chunk_select.py

diff --git a/src/ctapipe/monitoring/chunk_select.py b/src/ctapipe/monitoring/chunk_select.py
deleted file mode 100644
index f90253ea527..00000000000
--- a/src/ctapipe/monitoring/chunk_select.py
+++ /dev/null
@@ -1,78 +0,0 @@
-import numpy as np
-
-from ctapipe.core import Component, traits
-
-
-class ChunkFunction(Component):
-
-    """
-    Chunk Function for the gain and pedestals
-    Interpolates data so that for each time the value from the latest starting
-    valid chunk is given or the earliest available still valid chunk for any
-    pixels without valid data.
-
-    Parameters
-    ----------
-    input_table : astropy.table.Table
-        Table of calibration values, expected columns
-        are always ``start_time`` as ``Start_Time`` column,
-        ``end_time`` as ``End_Time`` column
-        other columns for the data that is to be selected
-    """
-
-    bounds_error = traits.Bool(
-        default_value=True,
-        help="If true, raises an exception when trying to extrapolate out of the given table",
-    ).tag(config=True)
-
-    extrapolate = traits.Bool(
-        help="If bounds_error is False, this flag will specify whether values outside"
-        "the available values are filled with nan (False) or extrapolated (True).",
-        default_value=False,
-    ).tag(config=True)
-
-    def __init__(
-        self,
-        input_table,
-        fill_value="extrapolate",
-    ):
-        input_table.sort("start_time")
-        self.start_times = input_table["start_time"]
-        self.end_times = input_table["end_time"]
-        self.values = input_table["values"]
-
-    def __call__(self, time):
-        if time < self.start_times[0]:
-            if self.bounds_error:
-                raise ValueError("below the interpolation range")
-
-            if self.extrapolate:
-                return self.values[0]
-
-            else:
-                a = np.empty(self.values[0].shape)
-                a[:] = np.nan
-                return a
-
-        elif time > self.end_times[-1]:
-            if self.bounds_error:
-                raise ValueError("above the interpolation range")
-
-            if self.extrapolate:
-                return self.values[-1]
-
-            else:
-                a = np.empty(self.values[0].shape)
-                a[:] = np.nan
-                return a
-
-        else:
-            i = np.searchsorted(
-                self.start_times, time, side="left"
-            )  # Latest valid chunk
-            j = np.searchsorted(
-                self.end_times, time, side="left"
-            )  # Earliest valid chunk
-            return np.where(
-                np.isnan(self.values[i - 1]), self.values[j], self.values[i - 1]
-            )  # Give value for latest chunk unless its nan. If nan give earliest chunk value
diff --git a/src/ctapipe/monitoring/interpolation.py b/src/ctapipe/monitoring/interpolation.py
index 84064cbc1a3..7e216d487b7 100644
--- a/src/ctapipe/monitoring/interpolation.py
+++ b/src/ctapipe/monitoring/interpolation.py
@@ -9,10 +9,76 @@
 
 from ctapipe.core import Component, traits
 
-__all__ = [
-    "Interpolator",
-    "PointingInterpolator",
-]
+
+class ChunkFunction:
+
+    """
+    Chunk Interpolator for the gain and pedestals
+    Interpolates data so that for each time the value from the latest starting
+    valid chunk is given or the earliest available still valid chunk for any
+    pixels without valid data.
+
+    Parameters
+    ----------
+    values : None | np.array
+        Numpy array of the data that is to be interpolated.
+        The first dimension needs to be an index over time
+    times : None | np.array
+        Time values over which data are to be interpolated
+        need to be sorted and have same length as first dimension of values
+    """
+
+    def __init__(
+        self,
+        start_times,
+        end_times,
+        values,
+        bounds_error=True,
+        fill_value="extrapolate",
+        assume_sorted=True,
+        copy=False,
+    ):
+        self.values = values
+        self.start_times = start_times
+        self.end_times = end_times
+        self.bounds_error = bounds_error
+        self.fill_value = fill_value
+
+    def __call__(self, point):
+        if point < self.start_times[0]:
+            if self.bounds_error:
+                raise ValueError("below the interpolation range")
+
+            if self.fill_value == "extrapolate":
+                return self.values[0]
+
+            else:
+                a = np.empty(self.values[0].shape)
+                a[:] = np.nan
+                return a
+
+        elif point > self.end_times[-1]:
+            if self.bounds_error:
+                raise ValueError("above the interpolation range")
+
+            if self.fill_value == "extrapolate":
+                return self.values[-1]
+
+            else:
+                a = np.empty(self.values[0].shape)
+                a[:] = np.nan
+                return a
+
+        else:
+            i = np.searchsorted(
+                self.start_times, point, side="left"
+            )  # Latest valid chunk
+            j = np.searchsorted(
+                self.end_times, point, side="left"
+            )  # Earliest valid chunk
+            return np.where(
+                np.isnan(self.values[i - 1]), self.values[j], self.values[i - 1]
+            )  # Give value for latest chunk unless its nan. If nan give earliest chunk value
 
 
 class Interpolator(Component, metaclass=ABCMeta):
@@ -186,3 +252,119 @@ def add_table(self, tel_id, input_table):
         self._interpolators[tel_id] = {}
         self._interpolators[tel_id]["az"] = interp1d(mjd, az, **self.interp_options)
         self._interpolators[tel_id]["alt"] = interp1d(mjd, alt, **self.interp_options)
+
+
+class FlatFieldInterpolator(Interpolator):
+    """
+    Interpolator for flatfield data
+    """
+
+    telescope_data_group = "dl1/calibration/gain"  # TBD
+    required_columns = frozenset(["start_time", "end_time", "gain"])
+    expected_units = {"gain": u.one}
+
+    def __call__(self, tel_id, time):
+        """
+        Interpolate flatfield data for a given time and tel_id.
+
+        Parameters
+        ----------
+        tel_id : int
+            telescope id
+        time : astropy.time.Time
+            time for which to interpolate the calibration data
+
+        Returns
+        -------
+        ffield : array [float]
+            interpolated flatfield data
+        """
+
+        self._check_interpolators(tel_id)
+
+        ffield = self._interpolators[tel_id]["gain"](time)
+        return ffield
+
+    def add_table(self, tel_id, input_table):
+        """
+        Add a table to this interpolator
+
+        Parameters
+        ----------
+        tel_id : int
+            Telescope id
+        input_table : astropy.table.Table
+            Table of pointing values, expected columns
+            are ``time`` as ``Time`` column and "gain"
+            for the flatfield data
+        """
+
+        self._check_tables(input_table)
+
+        input_table = input_table.copy()
+        input_table.sort("start_time")
+        start_time = input_table["start_time"]
+        end_time = input_table["end_time"]
+        gain = input_table["gain"]
+        self._interpolators[tel_id] = {}
+        self._interpolators[tel_id]["gain"] = ChunkFunction(
+            start_time, end_time, gain, **self.interp_options
+        )
+
+
+class PedestalInterpolator(Interpolator):
+    """
+    Interpolator for Pedestal data
+    """
+
+    telescope_data_group = "dl1/calibration/pedestal"  # TBD
+    required_columns = frozenset(["start_time", "end_time", "pedestal"])
+    expected_units = {"pedestal": u.one}
+
+    def __call__(self, tel_id, time):
+        """
+        Interpolate pedestal or gain for a given time and tel_id.
+
+        Parameters
+        ----------
+        tel_id : int
+            telescope id
+        time : astropy.time.Time
+            time for which to interpolate the calibration data
+
+        Returns
+        -------
+        pedestal : array [float]
+            interpolated pedestal values
+        """
+
+        self._check_interpolators(tel_id)
+
+        pedestal = self._interpolators[tel_id]["pedestal"](time)
+        return pedestal
+
+    def add_table(self, tel_id, input_table):
+        """
+        Add a table to this interpolator
+
+        Parameters
+        ----------
+        tel_id : int
+            Telescope id
+        input_table : astropy.table.Table
+            Table of pointing values, expected columns
+            are ``time`` as ``Time`` column and "pedestal"
+            for the pedestal data
+        """
+
+        self._check_tables(input_table)
+
+        input_table = input_table.copy()
+        input_table.sort("start_time")
+        start_time = input_table["start_time"]
+        end_time = input_table["end_time"]
+        pedestal = input_table["pedestal"]
+        self._interpolators[tel_id] = {}
+        self._interpolators[tel_id]["pedestal"] = ChunkFunction(
+            start_time, end_time, pedestal, **self.interp_options
+        )

From 050f404bd9d761f416b882f41c08d55e1911dfea Mon Sep 17 00:00:00 2001
From: Christoph Toennis <christoph.toennis1988@gmail.com>
Date: Tue, 29 Oct 2024 11:59:34 +0100
Subject: [PATCH 03/30] Changing some variables

---
 src/ctapipe/monitoring/interpolation.py | 96 +++++--------------------
 1 file changed, 18 insertions(+), 78 deletions(-)

diff --git a/src/ctapipe/monitoring/interpolation.py b/src/ctapipe/monitoring/interpolation.py
index 7e216d487b7..d41d6cf6157 100644
--- a/src/ctapipe/monitoring/interpolation.py
+++ b/src/ctapipe/monitoring/interpolation.py
@@ -34,7 +34,7 @@ def __init__(
         end_times,
         values,
         bounds_error=True,
-        fill_value="extrapolate",
+        extrapolate=False,
         assume_sorted=True,
         copy=False,
     ):
@@ -42,14 +42,14 @@ def __init__(
         self.start_times = start_times
         self.end_times = end_times
         self.bounds_error = bounds_error
-        self.fill_value = fill_value
+        self.extrapolate = extrapolate
 
     def __call__(self, point):
         if point < self.start_times[0]:
             if self.bounds_error:
                 raise ValueError("below the interpolation range")
 
-            if self.fill_value == "extrapolate":
+            if self.extrapolate:
                 return self.values[0]
 
             else:
@@ -61,7 +61,7 @@ def __call__(self, point):
             if self.bounds_error:
                 raise ValueError("above the interpolation range")
 
-            if self.fill_value == "extrapolate":
+            if self.extrapolate:
                 return self.values[-1]
 
             else:
@@ -254,36 +254,34 @@ def add_table(self, tel_id, input_table):
         self._interpolators[tel_id]["alt"] = interp1d(mjd, alt, **self.interp_options)
 
 
-class FlatFieldInterpolator(Interpolator):
+class SimpleInterpolator(Interpolator):
     """
-    Interpolator for flatfield data
+    Simple interpolator for overlapping chunks of data
     """
 
-    telescope_data_group = "dl1/calibration/gain"  # TBD
-    required_columns = frozenset(["start_time", "end_time", "gain"])
-    expected_units = {"gain": u.one}
+    required_columns = frozenset(["start_time", "end_time", "values"])
 
     def __call__(self, tel_id, time):
         """
-        Interpolate flatfield data for a given time and tel_id.
+        Interpolate overlapping chunks of data for a given time and tel_id.
 
         Parameters
         ----------
         tel_id : int
             telescope id
         time : astropy.time.Time
-            time for which to interpolate the calibration data
+            time for which to interpolate the data
 
         Returns
         -------
-        ffield : array [float]
-            interpolated flatfield data
+        interpolated : array [float]
+            interpolated data
         """
 
         self._check_interpolators(tel_id)
 
-        ffield = self._interpolators[tel_id]["gain"](time)
-        return ffield
+        val = self._interpolators[tel_id]["value"](time)
+        return val
 
     def add_table(self, tel_id, input_table):
         """
@@ -295,8 +293,8 @@ def add_table(self, tel_id, input_table):
             Telescope id
         input_table : astropy.table.Table
             Table of pointing values, expected columns
-            are ``time`` as ``Time`` column and "gain"
-            for the flatfield data
+            are ``time`` as ``Time`` column and "values"
+            for the data
         """
 
         self._check_tables(input_table)
@@ -305,66 +303,8 @@ def add_table(self, tel_id, input_table):
         input_table.sort("start_time")
         start_time = input_table["start_time"]
         end_time = input_table["end_time"]
-        gain = input_table["gain"]
+        values = input_table["values"]
         self._interpolators[tel_id] = {}
-        self._interpolators[tel_id]["gain"] = ChunkFunction(
-            start_time, end_time, gain, **self.interp_options
-        )
-
-
-class PedestalInterpolator(Interpolator):
-    """
-    Interpolator for Pedestal data
-    """
-
-    telescope_data_group = "dl1/calibration/pedestal"  # TBD
-    required_columns = frozenset(["start_time", "end_time", "pedestal"])
-    expected_units = {"pedestal": u.one}
-
-    def __call__(self, tel_id, time):
-        """
-        Interpolate pedestal or gain for a given time and tel_id.
-
-        Parameters
-        ----------
-        tel_id : int
-            telescope id
-        time : astropy.time.Time
-            time for which to interpolate the calibration data
-
-        Returns
-        -------
-        pedestal : array [float]
-            interpolated pedestal values
-        """
-
-        self._check_interpolators(tel_id)
-
-        pedestal = self._interpolators[tel_id]["pedestal"](time)
-        return pedestal
-
-    def add_table(self, tel_id, input_table):
-        """
-        Add a table to this interpolator
-
-        Parameters
-        ----------
-        tel_id : int
-            Telescope id
-        input_table : astropy.table.Table
-            Table of pointing values, expected columns
-            are ``time`` as ``Time`` column and "pedestal"
-            for the pedestal data
-        """
-
-        self._check_tables(input_table)
-
-        input_table = input_table.copy()
-        input_table.sort("start_time")
-        start_time = input_table["start_time"]
-        end_time = input_table["end_time"]
-        pedestal = input_table["pedestal"]
-        self._interpolators[tel_id] = {}
-        self._interpolators[tel_id]["pedestal"] = ChunkFunction(
-            start_time, end_time, pedestal, **self.interp_options
+        self._interpolators[tel_id]["value"] = ChunkFunction(
+            start_time, end_time, values, **self.interp_options
         )

From 11ef1e2183c0089100aaa4b8216244f8032a2655 Mon Sep 17 00:00:00 2001
From: Christoph Toennis <christoph.toennis1988@gmail.com>
Date: Wed, 30 Oct 2024 16:20:42 +0100
Subject: [PATCH 04/30] Chaning to using scipy functions

---
 src/ctapipe/monitoring/interpolation.py       | 112 ++++++------------
 .../monitoring/tests/test_interpolator.py     |  53 ++++++++-
 2 files changed, 89 insertions(+), 76 deletions(-)

diff --git a/src/ctapipe/monitoring/interpolation.py b/src/ctapipe/monitoring/interpolation.py
index d41d6cf6157..fbf60db7941 100644
--- a/src/ctapipe/monitoring/interpolation.py
+++ b/src/ctapipe/monitoring/interpolation.py
@@ -9,76 +9,7 @@
 
 from ctapipe.core import Component, traits
 
-
-class ChunkFunction:
-
-    """
-    Chunk Interpolator for the gain and pedestals
-    Interpolates data so that for each time the value from the latest starting
-    valid chunk is given or the earliest available still valid chunk for any
-    pixels without valid data.
-
-    Parameters
-    ----------
-    values : None | np.array
-        Numpy array of the data that is to be interpolated.
-        The first dimension needs to be an index over time
-    times : None | np.array
-        Time values over which data are to be interpolated
-        need to be sorted and have same length as first dimension of values
-    """
-
-    def __init__(
-        self,
-        start_times,
-        end_times,
-        values,
-        bounds_error=True,
-        extrapolate=False,
-        assume_sorted=True,
-        copy=False,
-    ):
-        self.values = values
-        self.start_times = start_times
-        self.end_times = end_times
-        self.bounds_error = bounds_error
-        self.extrapolate = extrapolate
-
-    def __call__(self, point):
-        if point < self.start_times[0]:
-            if self.bounds_error:
-                raise ValueError("below the interpolation range")
-
-            if self.extrapolate:
-                return self.values[0]
-
-            else:
-                a = np.empty(self.values[0].shape)
-                a[:] = np.nan
-                return a
-
-        elif point > self.end_times[-1]:
-            if self.bounds_error:
-                raise ValueError("above the interpolation range")
-
-            if self.extrapolate:
-                return self.values[-1]
-
-            else:
-                a = np.empty(self.values[0].shape)
-                a[:] = np.nan
-                return a
-
-        else:
-            i = np.searchsorted(
-                self.start_times, point, side="left"
-            )  # Latest valid chunk
-            j = np.searchsorted(
-                self.end_times, point, side="left"
-            )  # Earliest valid chunk
-            return np.where(
-                np.isnan(self.values[i - 1]), self.values[j], self.values[i - 1]
-            )  # Give value for latest chunk unless its nan. If nan give earliest chunk value
+__all__ = ["PointingInterpolator", "SimpleInterpolator"]
 
 
 class Interpolator(Component, metaclass=ABCMeta):
@@ -301,10 +232,41 @@ def add_table(self, tel_id, input_table):
 
         input_table = input_table.copy()
         input_table.sort("start_time")
-        start_time = input_table["start_time"]
-        end_time = input_table["end_time"]
+        start_time = input_table["start_time"].to_value("mjd")
+        end_time = input_table["end_time"].to_value("mjd")
         values = input_table["values"]
-        self._interpolators[tel_id] = {}
-        self._interpolators[tel_id]["value"] = ChunkFunction(
-            start_time, end_time, values, **self.interp_options
+        start_interpolate = interp1d(
+            start_time, values, axis=0, kind="previous", fill_value="extrapolate"
+        )  #: This is giving the latest possibly valid chunk
+        start_time_interpolate = interp1d(
+            start_time, end_time, axis=0, kind="previous", fill_value="extrapolate"
+        )
+        end_interpolate = interp1d(
+            end_time, values, axis=0, kind="next", fill_value="extrapolate"
+        )  #: This is giving the earliest possibly valid chunk
+        end_time_interpolate = interp1d(
+            end_time, start_time, axis=0, kind="next", fill_value="extrapolate"
         )
+
+        def interpolate_chunk(time):
+            mjd = time.to_value("mjd")
+            early_value = end_interpolate(mjd)
+            early_start = end_time_interpolate(mjd)
+            late_value = start_interpolate(mjd)
+            late_end = start_time_interpolate(mjd)
+            if mjd > early_start:  #: check if the early chunk is valid
+                if mjd < late_end:  #: check if the late chunk is valid
+                    return np.where(
+                        np.isnan(early_value), late_value, early_value
+                    )  #: both chunks are valid, return as many non-nan values as possible, preferring the early chunk
+                else:
+                    return early_value  #: only the early chunk is valid
+            elif mjd < late_end:
+                return late_value  #: only the late chunk is valid
+            else:
+                raise (
+                    ValueError("No valid data available for the given time")
+                )  #: no chunk is valid
+
+        self._interpolators[tel_id] = {}
+        self._interpolators[tel_id]["value"] = interpolate_chunk
diff --git a/src/ctapipe/monitoring/tests/test_interpolator.py b/src/ctapipe/monitoring/tests/test_interpolator.py
index 782aeae7435..0ecde082341 100644
--- a/src/ctapipe/monitoring/tests/test_interpolator.py
+++ b/src/ctapipe/monitoring/tests/test_interpolator.py
@@ -5,11 +5,62 @@
 from astropy.table import Table
 from astropy.time import Time
 
-from ctapipe.monitoring.interpolation import PointingInterpolator
+from ctapipe.monitoring.interpolation import PointingInterpolator, SimpleInterpolator
 
 t0 = Time("2022-01-01T00:00:00")
 
 
+def test_chunk_selection():
+    table = Table(
+        {
+            "start_time": t0 + [0, 1, 2, 6] * u.s,
+            "end_time": t0 + [2, 3, 4, 8] * u.s,
+            "values": [[1, 2], [2, 1], [2, 3], [3, 2]],
+        },
+    )
+    interpolator = SimpleInterpolator()
+    interpolator.add_table(1, table)
+
+    val1 = interpolator(tel_id=1, time=t0 + 1.2 * u.s)
+    val2 = interpolator(tel_id=1, time=t0 + 1.7 * u.s)
+    val3 = interpolator(tel_id=1, time=t0 + 2.2 * u.s)
+
+    assert np.all(np.isclose(val1, [1, 2]))
+    assert np.all(np.isclose(val2, [1, 2]))
+    assert np.all(np.isclose(val3, [2, 1]))
+
+
+def test_nan_switch():
+    table = Table(
+        {
+            "start_time": t0 + [0, 1, 2, 6] * u.s,
+            "end_time": t0 + [2, 3, 4, 8] * u.s,
+            "values": [[1, np.nan], [2, 1], [2, 3], [3, 2]],
+        },
+    )
+    interpolator = SimpleInterpolator()
+    interpolator.add_table(1, table)
+
+    val = interpolator(tel_id=1, time=t0 + 1.2 * u.s)
+
+    assert np.all(np.isclose(val, [1, 1]))
+
+
+def test_no_valid_chunk():
+    table = Table(
+        {
+            "start_time": t0 + [0, 1, 2, 6] * u.s,
+            "end_time": t0 + [2, 3, 4, 8] * u.s,
+            "values": [[1, 2], [2, 1], [2, 3], [3, 2]],
+        },
+    )
+    interpolator = SimpleInterpolator()
+    interpolator.add_table(1, table)
+
+    with pytest.raises(ValueError, match="No valid data available for the given time"):
+        interpolator(tel_id=1, time=t0 + 5.2 * u.s)
+
+
 def test_azimuth_switchover():
     """Test pointing interpolation"""
 

From 7c08829d95422b4f8d62430a61ed065ebb914d05 Mon Sep 17 00:00:00 2001
From: Christoph Toennis <christoph.toennis1988@gmail.com>
Date: Wed, 30 Oct 2024 16:29:49 +0100
Subject: [PATCH 05/30] fixing docustring

---
 src/ctapipe/monitoring/interpolation.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/ctapipe/monitoring/interpolation.py b/src/ctapipe/monitoring/interpolation.py
index fbf60db7941..c5d0db01982 100644
--- a/src/ctapipe/monitoring/interpolation.py
+++ b/src/ctapipe/monitoring/interpolation.py
@@ -223,7 +223,7 @@ def add_table(self, tel_id, input_table):
         tel_id : int
             Telescope id
         input_table : astropy.table.Table
-            Table of pointing values, expected columns
+            Table of values to be interpolated, expected columns
             are ``time`` as ``Time`` column and "values"
             for the data
         """

From c674bcc46aa7aaf2431a9175b9f5f201cd52f23d Mon Sep 17 00:00:00 2001
From: Christoph Toennis <christoph.toennis1988@gmail.com>
Date: Wed, 30 Oct 2024 16:40:37 +0100
Subject: [PATCH 06/30] Updating docustrings further

---
 src/ctapipe/monitoring/interpolation.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/ctapipe/monitoring/interpolation.py b/src/ctapipe/monitoring/interpolation.py
index c5d0db01982..8d15a1e4409 100644
--- a/src/ctapipe/monitoring/interpolation.py
+++ b/src/ctapipe/monitoring/interpolation.py
@@ -224,8 +224,9 @@ def add_table(self, tel_id, input_table):
             Telescope id
         input_table : astropy.table.Table
             Table of values to be interpolated, expected columns
-            are ``time`` as ``Time`` column and "values"
-            for the data
+            are ``start_time`` as ``validity start Time`` column,
+            ``end_time`` as ``validity end Time``  and "values"
+            for the data of the chunks
         """
 
         self._check_tables(input_table)

From 7bd017df561c3892191b5a35089e37aae2216cf0 Mon Sep 17 00:00:00 2001
From: Christoph Toennis <christoph.toennis1988@gmail.com>
Date: Thu, 31 Oct 2024 11:13:13 +0100
Subject: [PATCH 07/30] simplifying chunk interpolation

---
 src/ctapipe/monitoring/interpolation.py | 33 +++++++++++++------------
 1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/src/ctapipe/monitoring/interpolation.py b/src/ctapipe/monitoring/interpolation.py
index 8d15a1e4409..a7e4a78762c 100644
--- a/src/ctapipe/monitoring/interpolation.py
+++ b/src/ctapipe/monitoring/interpolation.py
@@ -235,39 +235,40 @@ def add_table(self, tel_id, input_table):
         input_table.sort("start_time")
         start_time = input_table["start_time"].to_value("mjd")
         end_time = input_table["end_time"].to_value("mjd")
-        values = input_table["values"]
+        values = np.column_stack(
+            (input_table["values"], start_time, end_time)
+        )  # stack values and times together for interpolation and validity checks
         start_interpolate = interp1d(
             start_time, values, axis=0, kind="previous", fill_value="extrapolate"
         )  #: This is giving the latest possibly valid chunk
-        start_time_interpolate = interp1d(
-            start_time, end_time, axis=0, kind="previous", fill_value="extrapolate"
-        )
         end_interpolate = interp1d(
             end_time, values, axis=0, kind="next", fill_value="extrapolate"
         )  #: This is giving the earliest possibly valid chunk
-        end_time_interpolate = interp1d(
-            end_time, start_time, axis=0, kind="next", fill_value="extrapolate"
-        )
 
         def interpolate_chunk(time):
             mjd = time.to_value("mjd")
+
             early_value = end_interpolate(mjd)
-            early_start = end_time_interpolate(mjd)
+            early_start = early_value[-2]
+            early_end = early_value[-1]
+            early_value = early_value[:-2]
             late_value = start_interpolate(mjd)
-            late_end = start_time_interpolate(mjd)
-            if mjd > early_start:  #: check if the early chunk is valid
-                if mjd < late_end:  #: check if the late chunk is valid
+            late_start = late_value[-2]
+            late_end = late_value[-1]
+            late_value = late_value[:-2]
+            if early_start <= mjd <= early_end:  # check if the early chunk is valid
+                if late_start <= mjd <= late_end:  # check if the late chunk is valid
                     return np.where(
                         np.isnan(early_value), late_value, early_value
-                    )  #: both chunks are valid, return as many non-nan values as possible, preferring the early chunk
+                    )  # both chunks are valid, return as many non-nan values as possible, preferring the early chunk
                 else:
-                    return early_value  #: only the early chunk is valid
-            elif mjd < late_end:
-                return late_value  #: only the late chunk is valid
+                    return early_value  # only the early chunk is valid
+            elif late_start <= mjd <= late_end:
+                return late_value  # only the late chunk is valid
             else:
                 raise (
                     ValueError("No valid data available for the given time")
-                )  #: no chunk is valid
+                )  # no chunk is valid
 
         self._interpolators[tel_id] = {}
         self._interpolators[tel_id]["value"] = interpolate_chunk

From 7504c201313509a6e7f1361c1ba664e0d5afbad8 Mon Sep 17 00:00:00 2001
From: "mykhailo.dalchenko" <mykhailo.dalchenko@cta-consortium.org>
Date: Thu, 31 Oct 2024 12:14:45 +0100
Subject: [PATCH 08/30] Refactor ChunkInterpolator and its tests

---
 src/ctapipe/monitoring/interpolation.py       | 162 ++++++++++--------
 .../monitoring/tests/test_interpolator.py     | 105 +++++++++---
 2 files changed, 174 insertions(+), 93 deletions(-)

diff --git a/src/ctapipe/monitoring/interpolation.py b/src/ctapipe/monitoring/interpolation.py
index a7e4a78762c..2845710e285 100644
--- a/src/ctapipe/monitoring/interpolation.py
+++ b/src/ctapipe/monitoring/interpolation.py
@@ -4,12 +4,13 @@
 import astropy.units as u
 import numpy as np
 import tables
+from astropy.table import Table
 from astropy.time import Time
 from scipy.interpolate import interp1d
 
 from ctapipe.core import Component, traits
 
-__all__ = ["PointingInterpolator", "SimpleInterpolator"]
+__all__ = ["PointingInterpolator", "ChunkInterpolator"]
 
 
 class Interpolator(Component, metaclass=ABCMeta):
@@ -19,7 +20,7 @@ class Interpolator(Component, metaclass=ABCMeta):
     Parameters
     ----------
     h5file : None | tables.File
-        A open hdf5 file with read access.
+        An open hdf5 file with read access.
     """
 
     bounds_error = traits.Bool(
@@ -37,7 +38,7 @@ class Interpolator(Component, metaclass=ABCMeta):
     required_columns = set()
     expected_units = {}
 
-    def __init__(self, h5file=None, **kwargs):
+    def __init__(self, h5file: None | tables.File = None, **kwargs: Any) -> None:
         super().__init__(**kwargs)
 
         if h5file is not None and not isinstance(h5file, tables.File):
@@ -57,26 +58,25 @@ def __init__(self, h5file=None, **kwargs):
         self._interpolators = {}
 
     @abstractmethod
-    def add_table(self, tel_id, input_table):
+    def add_table(self, tel_id: int, input_table: Table) -> None:
         """
-        Add a table to this interpolator
+        Add a table to this interpolator.
         This method reads input tables and creates instances of the needed interpolators
         to be added to _interpolators. The first index of _interpolators needs to be
-        tel_id, the second needs to be the name of the parameter that is to be interpolated
+        tel_id, the second needs to be the name of the parameter that is to be interpolated.
 
         Parameters
         ----------
         tel_id : int
-            Telescope id
+            Telescope id.
         input_table : astropy.table.Table
             Table of pointing values, expected columns
             are always ``time`` as ``Time`` column and
-            other columns for the data that is to be interpolated
+            other columns for the data that is to be interpolated.
         """
-
         pass
 
-    def _check_tables(self, input_table):
+    def _check_tables(self, input_table: Table) -> None:
         missing = self.required_columns - set(input_table.colnames)
         if len(missing) > 0:
             raise ValueError(f"Table is missing required column(s): {missing}")
@@ -95,14 +95,14 @@ def _check_tables(self, input_table):
                         f"{col} must have units compatible with '{self.expected_units[col].name}'"
                     )
 
-    def _check_interpolators(self, tel_id):
+    def _check_interpolators(self, tel_id: int) -> None:
         if tel_id not in self._interpolators:
             if self.h5file is not None:
                 self._read_parameter_table(tel_id)  # might need to be removed
             else:
                 raise KeyError(f"No table available for tel_id {tel_id}")
 
-    def _read_parameter_table(self, tel_id):
+    def _read_parameter_table(self, tel_id: int) -> None:
         # prevent circular import between io and monitoring
         from ..io import read_table
 
@@ -115,30 +115,30 @@ def _read_parameter_table(self, tel_id):
 
 class PointingInterpolator(Interpolator):
     """
-    Interpolator for pointing and pointing correction data
+    Interpolator for pointing and pointing correction data.
     """
 
     telescope_data_group = "/dl0/monitoring/telescope/pointing"
     required_columns = frozenset(["time", "azimuth", "altitude"])
     expected_units = {"azimuth": u.rad, "altitude": u.rad}
 
-    def __call__(self, tel_id, time):
+    def __call__(self, tel_id: int, time: Time) -> tuple[u.Quantity, u.Quantity]:
         """
         Interpolate alt/az for given time and tel_id.
 
         Parameters
         ----------
         tel_id : int
-            telescope id
+            Telescope id.
         time : astropy.time.Time
-            time for which to interpolate the pointing
+            Time for which to interpolate the pointing.
 
         Returns
         -------
         altitude : astropy.units.Quantity[deg]
-            interpolated altitude angle
+            Interpolated altitude angle.
         azimuth : astropy.units.Quantity[deg]
-            interpolated azimuth angle
+            Interpolated azimuth angle.
         """
 
         self._check_interpolators(tel_id)
@@ -148,14 +148,14 @@ def __call__(self, tel_id, time):
         alt = u.Quantity(self._interpolators[tel_id]["alt"](mjd), u.rad, copy=False)
         return alt, az
 
-    def add_table(self, tel_id, input_table):
+    def add_table(self, tel_id: int, input_table: Table) -> None:
         """
-        Add a table to this interpolator
+        Add a table to this interpolator.
 
         Parameters
         ----------
         tel_id : int
-            Telescope id
+            Telescope id.
         input_table : astropy.table.Table
             Table of pointing values, expected columns
             are ``time`` as ``Time`` column, ``azimuth`` and ``altitude``
@@ -185,90 +185,106 @@ def add_table(self, tel_id, input_table):
         self._interpolators[tel_id]["alt"] = interp1d(mjd, alt, **self.interp_options)
 
 
-class SimpleInterpolator(Interpolator):
+class ChunkInterpolator(Interpolator):
     """
-    Simple interpolator for overlapping chunks of data
+    Simple interpolator for overlapping chunks of data.
     """
 
-    required_columns = frozenset(["start_time", "end_time", "values"])
+    required_columns = frozenset(["start_time", "end_time"])
 
-    def __call__(self, tel_id, time):
+    def __call__(
+        self, tel_id: int, time: Time, columns: str | list[str]
+    ) -> float | dict[str, float]:
         """
-        Interpolate overlapping chunks of data for a given time and tel_id.
+        Interpolate overlapping chunks of data for a given time, tel_id, and column(s).
 
         Parameters
         ----------
         tel_id : int
-            telescope id
+            Telescope id.
         time : astropy.time.Time
-            time for which to interpolate the data
+            Time for which to interpolate the data.
+        columns : str or list of str
+            Name(s) of the column(s) to interpolate.
 
         Returns
         -------
-        interpolated : array [float]
-            interpolated data
+        interpolated : float or dict
+            Interpolated data for the specified column(s).
         """
 
         self._check_interpolators(tel_id)
 
-        val = self._interpolators[tel_id]["value"](time)
-        return val
+        if isinstance(columns, str):
+            columns = [columns]
+
+        result = {}
+        mjd = time.to_value("mjd")
+        for column in columns:
+            if column not in self._interpolators[tel_id]:
+                raise ValueError(
+                    f"Column '{column}' not found in interpolators for tel_id {tel_id}"
+                )
+            result[column] = self._interpolators[tel_id][column](mjd)
 
-    def add_table(self, tel_id, input_table):
+        if len(result) == 1:
+            return result[columns[0]]
+        return result
+
+    def add_table(self, tel_id: int, input_table: Table, columns: list[str]) -> None:
         """
-        Add a table to this interpolator
+        Add a table to this interpolator for specific columns.
 
         Parameters
         ----------
         tel_id : int
-            Telescope id
+            Telescope id.
         input_table : astropy.table.Table
             Table of values to be interpolated, expected columns
             are ``start_time`` as ``validity start Time`` column,
-            ``end_time`` as ``validity end Time``  and "values"
-            for the data of the chunks
+            ``end_time`` as ``validity end Time`` and the specified columns
+            for the data of the chunks.
+        columns : list of str
+            Names of the columns to interpolate.
         """
 
+        required_columns = set(self.required_columns)
+        required_columns.update(columns)
+        self.required_columns = frozenset(required_columns)
         self._check_tables(input_table)
 
         input_table = input_table.copy()
         input_table.sort("start_time")
         start_time = input_table["start_time"].to_value("mjd")
         end_time = input_table["end_time"].to_value("mjd")
-        values = np.column_stack(
-            (input_table["values"], start_time, end_time)
-        )  # stack values and times together for interpolation and validity checks
-        start_interpolate = interp1d(
-            start_time, values, axis=0, kind="previous", fill_value="extrapolate"
-        )  #: This is giving the latest possibly valid chunk
-        end_interpolate = interp1d(
-            end_time, values, axis=0, kind="next", fill_value="extrapolate"
-        )  #: This is giving the earliest possibly valid chunk
-
-        def interpolate_chunk(time):
-            mjd = time.to_value("mjd")
-
-            early_value = end_interpolate(mjd)
-            early_start = early_value[-2]
-            early_end = early_value[-1]
-            early_value = early_value[:-2]
-            late_value = start_interpolate(mjd)
-            late_start = late_value[-2]
-            late_end = late_value[-1]
-            late_value = late_value[:-2]
-            if early_start <= mjd <= early_end:  # check if the early chunk is valid
-                if late_start <= mjd <= late_end:  # check if the late chunk is valid
-                    return np.where(
-                        np.isnan(early_value), late_value, early_value
-                    )  # both chunks are valid, return as many non-nan values as possible, preferring the early chunk
-                else:
-                    return early_value  # only the early chunk is valid
-            elif late_start <= mjd <= late_end:
-                return late_value  # only the late chunk is valid
-            else:
-                raise (
-                    ValueError("No valid data available for the given time")
-                )  # no chunk is valid
 
-        self._interpolators[tel_id] = {}
-        self._interpolators[tel_id]["value"] = interpolate_chunk
+        if tel_id not in self._interpolators:
+            self._interpolators[tel_id] = {}
+
+        for column in columns:
+            values = input_table[column]
+
+            def interpolate_chunk(
+                mjd: float, start_time=start_time, end_time=end_time, values=values
+            ) -> float:
+                # Find the index of the closest preceding start time
+                preceding_index = np.searchsorted(start_time, mjd, side="right") - 1
+                if preceding_index < 0:
+                    return np.nan
+
+                # Check if the time is within the valid range of the chunk
+                if start_time[preceding_index] <= mjd <= end_time[preceding_index]:
+                    value = values[preceding_index]
+                    if not np.isnan(value):
+                        return value
+
+                # If the closest preceding chunk has nan, check the next closest chunk
+                for i in range(preceding_index - 1, -1, -1):
+                    if start_time[i] <= mjd <= end_time[i]:
+                        value = values[i]
+                        if not np.isnan(value):
+                            return value
+
+                return np.nan
+
+            self._interpolators[tel_id][column] = interpolate_chunk
diff --git a/src/ctapipe/monitoring/tests/test_interpolator.py b/src/ctapipe/monitoring/tests/test_interpolator.py
index 0ecde082341..696f4f4f209 100644
--- a/src/ctapipe/monitoring/tests/test_interpolator.py
+++ b/src/ctapipe/monitoring/tests/test_interpolator.py
@@ -5,7 +5,7 @@
 from astropy.table import Table
 from astropy.time import Time
 
-from ctapipe.monitoring.interpolation import PointingInterpolator, SimpleInterpolator
+from ctapipe.monitoring.interpolation import ChunkInterpolator, PointingInterpolator
 
 t0 = Time("2022-01-01T00:00:00")
 
@@ -15,19 +15,49 @@ def test_chunk_selection():
         {
             "start_time": t0 + [0, 1, 2, 6] * u.s,
             "end_time": t0 + [2, 3, 4, 8] * u.s,
-            "values": [[1, 2], [2, 1], [2, 3], [3, 2]],
+            "values": [1, 2, 3, 4],
         },
     )
-    interpolator = SimpleInterpolator()
-    interpolator.add_table(1, table)
+    interpolator = ChunkInterpolator()
+    interpolator.add_table(1, table, ["values"])
+
+    val1 = interpolator(tel_id=1, time=t0 + 1.2 * u.s, columns="values")
+    val2 = interpolator(tel_id=1, time=t0 + 1.7 * u.s, columns="values")
+    val3 = interpolator(tel_id=1, time=t0 + 2.2 * u.s, columns="values")
+
+    assert np.isclose(val1, 2)
+    assert np.isclose(val2, 2)
+    assert np.isclose(val3, 3)
+
+
+def test_chunk_selection_multiple_columns():
+    table = Table(
+        {
+            "start_time": t0 + [0, 1, 2, 6] * u.s,
+            "end_time": t0 + [2, 3, 4, 8] * u.s,
+            "values1": [1, 2, 3, 4],
+            "values2": [10, 20, 30, 40],
+        },
+    )
+    interpolator = ChunkInterpolator()
+    interpolator.add_table(1, table, ["values1", "values2"])
 
-    val1 = interpolator(tel_id=1, time=t0 + 1.2 * u.s)
-    val2 = interpolator(tel_id=1, time=t0 + 1.7 * u.s)
-    val3 = interpolator(tel_id=1, time=t0 + 2.2 * u.s)
+    result1 = interpolator(
+        tel_id=1, time=t0 + 1.2 * u.s, columns=["values1", "values2"]
+    )
+    result2 = interpolator(
+        tel_id=1, time=t0 + 1.7 * u.s, columns=["values1", "values2"]
+    )
+    result3 = interpolator(
+        tel_id=1, time=t0 + 2.2 * u.s, columns=["values1", "values2"]
+    )
 
-    assert np.all(np.isclose(val1, [1, 2]))
-    assert np.all(np.isclose(val2, [1, 2]))
-    assert np.all(np.isclose(val3, [2, 1]))
+    assert np.isclose(result1["values1"], 2)
+    assert np.isclose(result1["values2"], 20)
+    assert np.isclose(result2["values1"], 2)
+    assert np.isclose(result2["values2"], 20)
+    assert np.isclose(result3["values1"], 3)
+    assert np.isclose(result3["values2"], 30)
 
 
 def test_nan_switch():
@@ -35,15 +65,33 @@ def test_nan_switch():
         {
             "start_time": t0 + [0, 1, 2, 6] * u.s,
             "end_time": t0 + [2, 3, 4, 8] * u.s,
-            "values": [[1, np.nan], [2, 1], [2, 3], [3, 2]],
+            "values": [1, np.nan, 3, 4],
         },
     )
-    interpolator = SimpleInterpolator()
-    interpolator.add_table(1, table)
+    interpolator = ChunkInterpolator()
+    interpolator.add_table(1, table, ["values"])
 
-    val = interpolator(tel_id=1, time=t0 + 1.2 * u.s)
+    val = interpolator(tel_id=1, time=t0 + 1.2 * u.s, columns="values")
 
-    assert np.all(np.isclose(val, [1, 1]))
+    assert np.isclose(val, 1)
+
+
+def test_nan_switch_multiple_columns():
+    table = Table(
+        {
+            "start_time": t0 + [0, 1, 2, 6] * u.s,
+            "end_time": t0 + [2, 3, 4, 8] * u.s,
+            "values1": [1, np.nan, 3, 4],
+            "values2": [10, 20, np.nan, 40],
+        },
+    )
+    interpolator = ChunkInterpolator()
+    interpolator.add_table(1, table, ["values1", "values2"])
+
+    result = interpolator(tel_id=1, time=t0 + 1.2 * u.s, columns=["values1", "values2"])
+
+    assert np.isclose(result["values1"], 1)
+    assert np.isclose(result["values2"], 20)
 
 
 def test_no_valid_chunk():
@@ -51,14 +99,31 @@ def test_no_valid_chunk():
         {
             "start_time": t0 + [0, 1, 2, 6] * u.s,
             "end_time": t0 + [2, 3, 4, 8] * u.s,
-            "values": [[1, 2], [2, 1], [2, 3], [3, 2]],
+            "values": [1, 2, 3, 4],
         },
     )
-    interpolator = SimpleInterpolator()
-    interpolator.add_table(1, table)
+    interpolator = ChunkInterpolator()
+    interpolator.add_table(1, table, ["values"])
+
+    val = interpolator(tel_id=1, time=t0 + 5.2 * u.s, columns="values")
+    assert np.isnan(val)
+
+
+def test_no_valid_chunk_multiple_columns():
+    table = Table(
+        {
+            "start_time": t0 + [0, 1, 2, 6] * u.s,
+            "end_time": t0 + [2, 3, 4, 8] * u.s,
+            "values1": [1, 2, 3, 4],
+            "values2": [10, 20, 30, 40],
+        },
+    )
+    interpolator = ChunkInterpolator()
+    interpolator.add_table(1, table, ["values1", "values2"])
 
-    with pytest.raises(ValueError, match="No valid data available for the given time"):
-        interpolator(tel_id=1, time=t0 + 5.2 * u.s)
+    result = interpolator(tel_id=1, time=t0 + 5.2 * u.s, columns=["values1", "values2"])
+    assert np.isnan(result["values1"])
+    assert np.isnan(result["values2"])
 
 
 def test_azimuth_switchover():

From 9766719ab861d64a372f6160ffc1ca6bb51827e9 Mon Sep 17 00:00:00 2001
From: Christoph Toennis <christoph.toennis1988@gmail.com>
Date: Thu, 31 Oct 2024 12:55:22 +0100
Subject: [PATCH 09/30] adding changelog

---
 docs/changes/2634.rst                   | 1 +
 src/ctapipe/monitoring/interpolation.py | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 docs/changes/2634.rst

diff --git a/docs/changes/2634.rst b/docs/changes/2634.rst
new file mode 100644
index 00000000000..f09235fb0bb
--- /dev/null
+++ b/docs/changes/2634.rst
@@ -0,0 +1 @@
+Add ChunkInterpolator to ctapipe.monitoring.interpolation as a tool to select data from chunks. The planned use for this is to select calibration data.
diff --git a/src/ctapipe/monitoring/interpolation.py b/src/ctapipe/monitoring/interpolation.py
index 2845710e285..2b3069b5f39 100644
--- a/src/ctapipe/monitoring/interpolation.py
+++ b/src/ctapipe/monitoring/interpolation.py
@@ -10,7 +10,7 @@
 
 from ctapipe.core import Component, traits
 
-__all__ = ["PointingInterpolator", "ChunkInterpolator"]
+__all__ = ["Interpolator", "PointingInterpolator", "ChunkInterpolator"]
 
 
 class Interpolator(Component, metaclass=ABCMeta):

From f22a0c068f2e5bf237dadf79215e423396d58bb7 Mon Sep 17 00:00:00 2001
From: Christoph Toennis <christoph.toennis1988@gmail.com>
Date: Thu, 31 Oct 2024 12:59:35 +0100
Subject: [PATCH 10/30] renaming changelog

---
 docs/changes/{2634.rst => 2634.feature.rst} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename docs/changes/{2634.rst => 2634.feature.rst} (100%)

diff --git a/docs/changes/2634.rst b/docs/changes/2634.feature.rst
similarity index 100%
rename from docs/changes/2634.rst
rename to docs/changes/2634.feature.rst

From 366a3f3efd2f5b715568b1eac1baf61c5f534c0c Mon Sep 17 00:00:00 2001
From: Christoph Toennis <christoph.toennis1988@gmail.com>
Date: Fri, 22 Nov 2024 14:18:15 +0100
Subject: [PATCH 11/30] Changing inheritance scheme

---
 src/ctapipe/monitoring/__init__.py            |  11 +-
 src/ctapipe/monitoring/interpolation.py       | 151 +++++++++++-------
 .../monitoring/tests/test_interpolator.py     |  10 +-
 3 files changed, 110 insertions(+), 62 deletions(-)

diff --git a/src/ctapipe/monitoring/__init__.py b/src/ctapipe/monitoring/__init__.py
index cb102f768cf..03d583abaab 100644
--- a/src/ctapipe/monitoring/__init__.py
+++ b/src/ctapipe/monitoring/__init__.py
@@ -2,7 +2,12 @@
 Module for handling monitoring data.
 """
 from .aggregator import PlainAggregator, SigmaClippingAggregator, StatisticsAggregator
-from .interpolation import Interpolator, PointingInterpolator
+from .interpolation import (
+    ChunkInterpolator,
+    LinearInterpolator,
+    MonitoringInterpolator,
+    PointingInterpolator,
+)
 from .outlier import (
     MedianOutlierDetector,
     OutlierDetector,
@@ -18,6 +23,8 @@
     "RangeOutlierDetector",
     "MedianOutlierDetector",
     "StdOutlierDetector",
-    "Interpolator",
+    "MonitoringInterpolator",
+    "LinearInterpolator",
     "PointingInterpolator",
+    "ChunkInterpolator",
 ]
diff --git a/src/ctapipe/monitoring/interpolation.py b/src/ctapipe/monitoring/interpolation.py
index 2b3069b5f39..94f27c13e98 100644
--- a/src/ctapipe/monitoring/interpolation.py
+++ b/src/ctapipe/monitoring/interpolation.py
@@ -1,4 +1,5 @@
 from abc import ABCMeta, abstractmethod
+from functools import partial
 from typing import Any
 
 import astropy.units as u
@@ -10,12 +11,17 @@
 
 from ctapipe.core import Component, traits
 
-__all__ = ["Interpolator", "PointingInterpolator", "ChunkInterpolator"]
+__all__ = [
+    "MonitoringInterpolator",
+    "LinearInterpolator",
+    "PointingInterpolator",
+    "ChunkInterpolator",
+]
 
 
-class Interpolator(Component, metaclass=ABCMeta):
+class MonitoringInterpolator(Component, metaclass=ABCMeta):
     """
-    Interpolator parent class.
+    MonitoringInterpolator parent class.
 
     Parameters
     ----------
@@ -23,20 +29,10 @@ class Interpolator(Component, metaclass=ABCMeta):
         An open hdf5 file with read access.
     """
 
-    bounds_error = traits.Bool(
-        default_value=True,
-        help="If true, raises an exception when trying to extrapolate out of the given table",
-    ).tag(config=True)
-
-    extrapolate = traits.Bool(
-        help="If bounds_error is False, this flag will specify whether values outside"
-        "the available values are filled with nan (False) or extrapolated (True).",
-        default_value=False,
-    ).tag(config=True)
-
     telescope_data_group = None
     required_columns = set()
     expected_units = {}
+    _interpolators = {}
 
     def __init__(self, h5file: None | tables.File = None, **kwargs: Any) -> None:
         super().__init__(**kwargs)
@@ -45,17 +41,20 @@ def __init__(self, h5file: None | tables.File = None, **kwargs: Any) -> None:
             raise TypeError("h5file must be a tables.File")
         self.h5file = h5file
 
-        self.interp_options: dict[str, Any] = dict(assume_sorted=True, copy=False)
-        if self.bounds_error:
-            self.interp_options["bounds_error"] = True
-        elif self.extrapolate:
-            self.interp_options["bounds_error"] = False
-            self.interp_options["fill_value"] = "extrapolate"
-        else:
-            self.interp_options["bounds_error"] = False
-            self.interp_options["fill_value"] = np.nan
+    @abstractmethod
+    def __call__(self, tel_id: int, time: Time):
+        """
+        Interpolates monitoring data for a given timestamp
+
+        Parameters
+        ----------
+        tel_id : int
+            Telescope id.
+        time : astropy.time.Time
+            Time for which to interpolate the monitoring data.
 
-        self._interpolators = {}
+        """
+        pass
 
     @abstractmethod
     def add_table(self, tel_id: int, input_table: Table) -> None:
@@ -113,7 +112,42 @@ def _read_parameter_table(self, tel_id: int) -> None:
         self.add_table(tel_id, input_table)
 
 
-class PointingInterpolator(Interpolator):
+class LinearInterpolator(MonitoringInterpolator):
+    """
+    LinearInterpolator parent class.
+
+    Parameters
+    ----------
+    h5file : None | tables.File
+        An open hdf5 file with read access.
+    """
+
+    bounds_error = traits.Bool(
+        default_value=True,
+        help="If true, raises an exception when trying to extrapolate out of the given table",
+    ).tag(config=True)
+
+    extrapolate = traits.Bool(
+        help="If bounds_error is False, this flag will specify whether values outside"
+        "the available values are filled with nan (False) or extrapolated (True).",
+        default_value=False,
+    ).tag(config=True)
+
+    def __init__(self, h5file: None | tables.File = None, **kwargs: Any) -> None:
+        super().__init__(h5file, **kwargs)
+
+        self.interp_options: dict[str, Any] = dict(assume_sorted=True, copy=False)
+        if self.bounds_error:
+            self.interp_options["bounds_error"] = True
+        elif self.extrapolate:
+            self.interp_options["bounds_error"] = False
+            self.interp_options["fill_value"] = "extrapolate"
+        else:
+            self.interp_options["bounds_error"] = False
+            self.interp_options["fill_value"] = np.nan
+
+
+class PointingInterpolator(LinearInterpolator):
     """
     Interpolator for pointing and pointing correction data.
     """
@@ -161,7 +195,6 @@ def add_table(self, tel_id: int, input_table: Table) -> None:
             are ``time`` as ``Time`` column, ``azimuth`` and ``altitude``
             as quantity columns for pointing and pointing correction data.
         """
-
         self._check_tables(input_table)
 
         if not isinstance(input_table["time"], Time):
@@ -185,12 +218,15 @@ def add_table(self, tel_id: int, input_table: Table) -> None:
         self._interpolators[tel_id]["alt"] = interp1d(mjd, alt, **self.interp_options)
 
 
-class ChunkInterpolator(Interpolator):
+class ChunkInterpolator(MonitoringInterpolator):
     """
     Simple interpolator for overlapping chunks of data.
     """
 
     required_columns = frozenset(["start_time", "end_time"])
+    start_time = {}
+    end_time = {}
+    values = {}
 
     def __call__(
         self, tel_id: int, time: Time, columns: str | list[str]
@@ -255,36 +291,41 @@ def add_table(self, tel_id: int, input_table: Table, columns: list[str]) -> None
 
         input_table = input_table.copy()
         input_table.sort("start_time")
-        start_time = input_table["start_time"].to_value("mjd")
-        end_time = input_table["end_time"].to_value("mjd")
 
         if tel_id not in self._interpolators:
             self._interpolators[tel_id] = {}
+            self.values[tel_id] = {}
+            self.start_time[tel_id] = {}
+            self.end_time[tel_id] = {}
 
         for column in columns:
-            values = input_table[column]
-
-            def interpolate_chunk(
-                mjd: float, start_time=start_time, end_time=end_time, values=values
-            ) -> float:
-                # Find the index of the closest preceding start time
-                preceding_index = np.searchsorted(start_time, mjd, side="right") - 1
-                if preceding_index < 0:
-                    return np.nan
-
-                # Check if the time is within the valid range of the chunk
-                if start_time[preceding_index] <= mjd <= end_time[preceding_index]:
-                    value = values[preceding_index]
-                    if not np.isnan(value):
-                        return value
-
-                # If the closest preceding chunk has nan, check the next closest chunk
-                for i in range(preceding_index - 1, -1, -1):
-                    if start_time[i] <= mjd <= end_time[i]:
-                        value = values[i]
-                        if not np.isnan(value):
-                            return value
-
-                return np.nan
-
-            self._interpolators[tel_id][column] = interpolate_chunk
+            self.values[tel_id][column] = input_table[column]
+            self.start_time[tel_id][column] = input_table["start_time"].to_value("mjd")
+            self.end_time[tel_id][column] = input_table["end_time"].to_value("mjd")
+            self._interpolators[tel_id][column] = partial(
+                self._interpolate_chunk, tel_id, column
+            )
+
+    def _interpolate_chunk(self, tel_id, column, mjd: float) -> float:
+        start_time = self.start_time[tel_id][column]
+        end_time = self.end_time[tel_id][column]
+        values = self.values[tel_id][column]
+        # Find the index of the closest preceding start time
+        preceding_index = np.searchsorted(start_time, mjd, side="right") - 1
+        if preceding_index < 0:
+            return np.nan
+
+        # Check if the time is within the valid range of the chunk
+        if start_time[preceding_index] <= mjd <= end_time[preceding_index]:
+            value = values[preceding_index]
+            if not np.isnan(value):
+                return value
+
+        # If the closest preceding chunk has nan, check the next closest chunk
+        for i in range(preceding_index - 1, -1, -1):
+            if start_time[i] <= mjd <= end_time[i]:
+                value = values[i]
+                if not np.isnan(value):
+                    return value
+
+        return np.nan
diff --git a/src/ctapipe/monitoring/tests/test_interpolator.py b/src/ctapipe/monitoring/tests/test_interpolator.py
index 696f4f4f209..9bee7076a09 100644
--- a/src/ctapipe/monitoring/tests/test_interpolator.py
+++ b/src/ctapipe/monitoring/tests/test_interpolator.py
@@ -173,7 +173,7 @@ def test_invalid_input():
     wrong_unit = Table(
         {
             "time": Time(1.7e9 + np.arange(3), format="unix"),
-            "azimuth": [1, 2, 3] * u.deg,
+            "azimuth": np.radians([1, 2, 3]) * u.rad,
             "altitude": [1, 2, 3],
         }
     )
@@ -188,8 +188,8 @@ def test_hdf5(tmp_path):
     table = Table(
         {
             "time": t0 + np.arange(0.0, 10.1, 2.0) * u.s,
-            "azimuth": np.linspace(0.0, 10.0, 6) * u.deg,
-            "altitude": np.linspace(70.0, 60.0, 6) * u.deg,
+            "azimuth": np.radians(np.linspace(0.0, 10.0, 6)) * u.rad,
+            "altitude": np.radians(np.linspace(70.0, 60.0, 6)) * u.rad,
         },
     )
 
@@ -198,8 +198,8 @@ def test_hdf5(tmp_path):
     with tables.open_file(path) as h5file:
         interpolator = PointingInterpolator(h5file)
         alt, az = interpolator(tel_id=1, time=t0 + 1 * u.s)
-        assert u.isclose(alt, 69 * u.deg)
-        assert u.isclose(az, 1 * u.deg)
+        assert u.isclose(alt, np.radians(69) * u.rad)
+        assert u.isclose(az, np.radians(1) * u.rad)
 
 
 def test_bounds():

From 386faa8b45ad994b9fe20d6386f553b81ea58d97 Mon Sep 17 00:00:00 2001
From: Christoph Toennis <christoph.toennis1988@gmail.com>
Date: Fri, 22 Nov 2024 15:51:34 +0100
Subject: [PATCH 12/30]  reverting some tests

---
 src/ctapipe/monitoring/interpolation.py           | 13 +++++++++++++
 src/ctapipe/monitoring/tests/test_interpolator.py | 10 +++++-----
 2 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/src/ctapipe/monitoring/interpolation.py b/src/ctapipe/monitoring/interpolation.py
index 94f27c13e98..885cfca7a31 100644
--- a/src/ctapipe/monitoring/interpolation.py
+++ b/src/ctapipe/monitoring/interpolation.py
@@ -307,6 +307,19 @@ def add_table(self, tel_id: int, input_table: Table, columns: list[str]) -> None
             )
 
     def _interpolate_chunk(self, tel_id, column, mjd: float) -> float:
+        """
+        Interpolates overlapping chunks of data preferring earlier chunks if valid
+
+        Parameters
+        ----------
+        tel_id : int
+            tel_id for which data is to be interpolated
+        column : str
+            name of the column for which data is to be interpolated
+        mjd : float
+            Time for which to interpolate the data.
+        """
+
         start_time = self.start_time[tel_id][column]
         end_time = self.end_time[tel_id][column]
         values = self.values[tel_id][column]
diff --git a/src/ctapipe/monitoring/tests/test_interpolator.py b/src/ctapipe/monitoring/tests/test_interpolator.py
index 9bee7076a09..696f4f4f209 100644
--- a/src/ctapipe/monitoring/tests/test_interpolator.py
+++ b/src/ctapipe/monitoring/tests/test_interpolator.py
@@ -173,7 +173,7 @@ def test_invalid_input():
     wrong_unit = Table(
         {
             "time": Time(1.7e9 + np.arange(3), format="unix"),
-            "azimuth": np.radians([1, 2, 3]) * u.rad,
+            "azimuth": [1, 2, 3] * u.deg,
             "altitude": [1, 2, 3],
         }
     )
@@ -188,8 +188,8 @@ def test_hdf5(tmp_path):
     table = Table(
         {
             "time": t0 + np.arange(0.0, 10.1, 2.0) * u.s,
-            "azimuth": np.radians(np.linspace(0.0, 10.0, 6)) * u.rad,
-            "altitude": np.radians(np.linspace(70.0, 60.0, 6)) * u.rad,
+            "azimuth": np.linspace(0.0, 10.0, 6) * u.deg,
+            "altitude": np.linspace(70.0, 60.0, 6) * u.deg,
         },
     )
 
@@ -198,8 +198,8 @@ def test_hdf5(tmp_path):
     with tables.open_file(path) as h5file:
         interpolator = PointingInterpolator(h5file)
         alt, az = interpolator(tel_id=1, time=t0 + 1 * u.s)
-        assert u.isclose(alt, np.radians(69) * u.rad)
-        assert u.isclose(az, np.radians(1) * u.rad)
+        assert u.isclose(alt, 69 * u.deg)
+        assert u.isclose(az, 1 * u.deg)
 
 
 def test_bounds():

From e5e2fde6c684724e50a4c37af0de459e060aba92 Mon Sep 17 00:00:00 2001
From: Christoph Toennis <christoph.toennis1988@gmail.com>
Date: Fri, 22 Nov 2024 16:58:10 +0100
Subject: [PATCH 13/30] documentation change

---
 src/ctapipe/monitoring/interpolation.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/ctapipe/monitoring/interpolation.py b/src/ctapipe/monitoring/interpolation.py
index 885cfca7a31..880acbdac54 100644
--- a/src/ctapipe/monitoring/interpolation.py
+++ b/src/ctapipe/monitoring/interpolation.py
@@ -214,8 +214,12 @@ def add_table(self, tel_id: int, input_table: Table) -> None:
         alt = input_table["altitude"].quantity.to_value(u.rad)
         mjd = input_table["time"].tai.mjd
         self._interpolators[tel_id] = {}
-        self._interpolators[tel_id]["az"] = interp1d(mjd, az, **self.interp_options)
-        self._interpolators[tel_id]["alt"] = interp1d(mjd, alt, **self.interp_options)
+        self._interpolators[tel_id]["az"] = interp1d(
+            mjd, az, kind="linear", **self.interp_options
+        )
+        self._interpolators[tel_id]["alt"] = interp1d(
+            mjd, alt, kind="linear", **self.interp_options
+        )
 
 
 class ChunkInterpolator(MonitoringInterpolator):

From 33377e5fa5cb6686dbab68f4c98a72c50451bed0 Mon Sep 17 00:00:00 2001
From: Christoph Toennis <christoph.toennis1988@gmail.com>
Date: Mon, 25 Nov 2024 14:56:25 +0100
Subject: [PATCH 14/30] fixing issue with class variable

---
 src/ctapipe/monitoring/interpolation.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/src/ctapipe/monitoring/interpolation.py b/src/ctapipe/monitoring/interpolation.py
index 880acbdac54..95e6456de1d 100644
--- a/src/ctapipe/monitoring/interpolation.py
+++ b/src/ctapipe/monitoring/interpolation.py
@@ -29,14 +29,14 @@ class MonitoringInterpolator(Component, metaclass=ABCMeta):
         An open hdf5 file with read access.
     """
 
-    telescope_data_group = None
-    required_columns = set()
-    expected_units = {}
-    _interpolators = {}
-
     def __init__(self, h5file: None | tables.File = None, **kwargs: Any) -> None:
         super().__init__(**kwargs)
 
+        self.telescope_data_group = None
+        self.required_columns = set()
+        self.expected_units = {}
+        self._interpolators = {}
+
         if h5file is not None and not isinstance(h5file, tables.File):
             raise TypeError("h5file must be a tables.File")
         self.h5file = h5file
@@ -152,9 +152,11 @@ class PointingInterpolator(LinearInterpolator):
     Interpolator for pointing and pointing correction data.
     """
 
-    telescope_data_group = "/dl0/monitoring/telescope/pointing"
-    required_columns = frozenset(["time", "azimuth", "altitude"])
-    expected_units = {"azimuth": u.rad, "altitude": u.rad}
+    def __init__(self, h5file: None | tables.File = None, **kwargs: Any) -> None:
+        super().__init__(h5file, **kwargs)
+        self.telescope_data_group = "/dl0/monitoring/telescope/pointing"
+        self.required_columns = frozenset(["time", "azimuth", "altitude"])
+        self.expected_units = {"azimuth": u.rad, "altitude": u.rad}
 
     def __call__(self, tel_id: int, time: Time) -> tuple[u.Quantity, u.Quantity]:
         """

From e735ec177fb225dd711bdd1b4408ebe45720994a Mon Sep 17 00:00:00 2001
From: Christoph Toennis <christoph.toennis1988@gmail.com>
Date: Wed, 27 Nov 2024 10:45:33 +0100
Subject: [PATCH 15/30] implementing reviewer comment

---
 src/ctapipe/monitoring/interpolation.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/ctapipe/monitoring/interpolation.py b/src/ctapipe/monitoring/interpolation.py
index 95e6456de1d..b11f774411e 100644
--- a/src/ctapipe/monitoring/interpolation.py
+++ b/src/ctapipe/monitoring/interpolation.py
@@ -229,10 +229,13 @@ class ChunkInterpolator(MonitoringInterpolator):
     Simple interpolator for overlapping chunks of data.
     """
 
-    required_columns = frozenset(["start_time", "end_time"])
-    start_time = {}
-    end_time = {}
-    values = {}
+    def __init__(self, h5file: None | tables.File = None, **kwargs: Any) -> None:
+        super().__init__(**kwargs)
+
+        self.required_columns = frozenset(["start_time", "end_time"])
+        self.start_time = {}
+        self.end_time = {}
+        self.values = {}
 
     def __call__(
         self, tel_id: int, time: Time, columns: str | list[str]

From 8a0c213e7bcd803415173a1131845f87dcfffa9a Mon Sep 17 00:00:00 2001
From: Christoph Toennis <christoph.toennis1988@gmail.com>
Date: Mon, 2 Dec 2024 17:34:50 +0100
Subject: [PATCH 16/30] moving some instance variales to class variables

---
 src/ctapipe/monitoring/interpolation.py | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/src/ctapipe/monitoring/interpolation.py b/src/ctapipe/monitoring/interpolation.py
index b11f774411e..235bfe88d1a 100644
--- a/src/ctapipe/monitoring/interpolation.py
+++ b/src/ctapipe/monitoring/interpolation.py
@@ -29,12 +29,13 @@ class MonitoringInterpolator(Component, metaclass=ABCMeta):
         An open hdf5 file with read access.
     """
 
+    telescope_data_group = None
+    required_columns = set()
+    expected_units = {}
+
     def __init__(self, h5file: None | tables.File = None, **kwargs: Any) -> None:
         super().__init__(**kwargs)
 
-        self.telescope_data_group = None
-        self.required_columns = set()
-        self.expected_units = {}
         self._interpolators = {}
 
         if h5file is not None and not isinstance(h5file, tables.File):
@@ -152,11 +153,12 @@ class PointingInterpolator(LinearInterpolator):
     Interpolator for pointing and pointing correction data.
     """
 
+    telescope_data_group = "/dl0/monitoring/telescope/pointing"
+    required_columns = frozenset(["time", "azimuth", "altitude"])
+    expected_units = {"azimuth": u.rad, "altitude": u.rad}
+
     def __init__(self, h5file: None | tables.File = None, **kwargs: Any) -> None:
         super().__init__(h5file, **kwargs)
-        self.telescope_data_group = "/dl0/monitoring/telescope/pointing"
-        self.required_columns = frozenset(["time", "azimuth", "altitude"])
-        self.expected_units = {"azimuth": u.rad, "altitude": u.rad}
 
     def __call__(self, tel_id: int, time: Time) -> tuple[u.Quantity, u.Quantity]:
         """
@@ -229,10 +231,11 @@ class ChunkInterpolator(MonitoringInterpolator):
     Simple interpolator for overlapping chunks of data.
     """
 
+    required_columns = frozenset(["start_time", "end_time"])
+
     def __init__(self, h5file: None | tables.File = None, **kwargs: Any) -> None:
         super().__init__(**kwargs)
 
-        self.required_columns = frozenset(["start_time", "end_time"])
         self.start_time = {}
         self.end_time = {}
         self.values = {}

From 3bad626c7689b79bc021a28a19ffaf81e92fe580 Mon Sep 17 00:00:00 2001
From: Christoph Toennis <christoph.toennis1988@gmail.com>
Date: Tue, 3 Dec 2024 15:11:50 +0100
Subject: [PATCH 17/30] removing unnecessary class variables in parent classes

---
 src/ctapipe/monitoring/interpolation.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/src/ctapipe/monitoring/interpolation.py b/src/ctapipe/monitoring/interpolation.py
index 235bfe88d1a..a3bde6672e7 100644
--- a/src/ctapipe/monitoring/interpolation.py
+++ b/src/ctapipe/monitoring/interpolation.py
@@ -29,10 +29,6 @@ class MonitoringInterpolator(Component, metaclass=ABCMeta):
         An open hdf5 file with read access.
     """
 
-    telescope_data_group = None
-    required_columns = set()
-    expected_units = {}
-
     def __init__(self, h5file: None | tables.File = None, **kwargs: Any) -> None:
         super().__init__(**kwargs)
 
@@ -157,9 +153,6 @@ class PointingInterpolator(LinearInterpolator):
     required_columns = frozenset(["time", "azimuth", "altitude"])
     expected_units = {"azimuth": u.rad, "altitude": u.rad}
 
-    def __init__(self, h5file: None | tables.File = None, **kwargs: Any) -> None:
-        super().__init__(h5file, **kwargs)
-
     def __call__(self, tel_id: int, time: Time) -> tuple[u.Quantity, u.Quantity]:
         """
         Interpolate alt/az for given time and tel_id.
@@ -232,6 +225,7 @@ class ChunkInterpolator(MonitoringInterpolator):
     """
 
     required_columns = frozenset(["start_time", "end_time"])
+    expected_units = {}
 
     def __init__(self, h5file: None | tables.File = None, **kwargs: Any) -> None:
         super().__init__(**kwargs)

From eaf34e24970089eea2b0c403fc14b6939449750b Mon Sep 17 00:00:00 2001
From: Christoph Toennis <christoph.toennis1988@gmail.com>
Date: Wed, 4 Dec 2024 11:15:10 +0100
Subject: [PATCH 18/30] moving ChunkInterpolator variables and making them
 mutable at first

---
 src/ctapipe/monitoring/interpolation.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/ctapipe/monitoring/interpolation.py b/src/ctapipe/monitoring/interpolation.py
index a3bde6672e7..2a7dbbd3224 100644
--- a/src/ctapipe/monitoring/interpolation.py
+++ b/src/ctapipe/monitoring/interpolation.py
@@ -224,12 +224,11 @@ class ChunkInterpolator(MonitoringInterpolator):
     Simple interpolator for overlapping chunks of data.
     """
 
-    required_columns = frozenset(["start_time", "end_time"])
-    expected_units = {}
-
     def __init__(self, h5file: None | tables.File = None, **kwargs: Any) -> None:
         super().__init__(**kwargs)
 
+        self.required_columns = ["start_time", "end_time"]
+        self.expected_units = {}
         self.start_time = {}
         self.end_time = {}
         self.values = {}

From 53ec34122d84831ae9e0a72648dbc5eb1de998ee Mon Sep 17 00:00:00 2001
From: Christoph Toennis <christoph.toennis1988@gmail.com>
Date: Wed, 4 Dec 2024 11:25:19 +0100
Subject: [PATCH 19/30] removing provate data definition from parent class

---
 src/ctapipe/monitoring/interpolation.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/ctapipe/monitoring/interpolation.py b/src/ctapipe/monitoring/interpolation.py
index 2a7dbbd3224..e5605ab0a77 100644
--- a/src/ctapipe/monitoring/interpolation.py
+++ b/src/ctapipe/monitoring/interpolation.py
@@ -32,8 +32,6 @@ class MonitoringInterpolator(Component, metaclass=ABCMeta):
     def __init__(self, h5file: None | tables.File = None, **kwargs: Any) -> None:
         super().__init__(**kwargs)
 
-        self._interpolators = {}
-
         if h5file is not None and not isinstance(h5file, tables.File):
             raise TypeError("h5file must be a tables.File")
         self.h5file = h5file
@@ -153,6 +151,11 @@ class PointingInterpolator(LinearInterpolator):
     required_columns = frozenset(["time", "azimuth", "altitude"])
     expected_units = {"azimuth": u.rad, "altitude": u.rad}
 
+    def __init__(self, h5file: None | tables.File = None, **kwargs: Any) -> None:
+        super().__init__(h5file=h5file, **kwargs)
+
+        self._interpolators = {}
+
     def __call__(self, tel_id: int, time: Time) -> tuple[u.Quantity, u.Quantity]:
         """
         Interpolate alt/az for given time and tel_id.
@@ -226,7 +229,7 @@ class ChunkInterpolator(MonitoringInterpolator):
 
     def __init__(self, h5file: None | tables.File = None, **kwargs: Any) -> None:
         super().__init__(**kwargs)
-
+        self._interpolators = {}
         self.required_columns = ["start_time", "end_time"]
         self.expected_units = {}
         self.start_time = {}

From e7aa23d6e1a831cc2f9cb08b1a716098c9336f85 Mon Sep 17 00:00:00 2001
From: Christoph Toennis <christoph.toennis1988@gmail.com>
Date: Wed, 4 Dec 2024 11:33:57 +0100
Subject: [PATCH 20/30] moving a variable

---
 src/ctapipe/monitoring/interpolation.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/src/ctapipe/monitoring/interpolation.py b/src/ctapipe/monitoring/interpolation.py
index e5605ab0a77..9f9f5f8751e 100644
--- a/src/ctapipe/monitoring/interpolation.py
+++ b/src/ctapipe/monitoring/interpolation.py
@@ -130,7 +130,7 @@ class LinearInterpolator(MonitoringInterpolator):
 
     def __init__(self, h5file: None | tables.File = None, **kwargs: Any) -> None:
         super().__init__(h5file, **kwargs)
-
+        self._interpolators = {}
         self.interp_options: dict[str, Any] = dict(assume_sorted=True, copy=False)
         if self.bounds_error:
             self.interp_options["bounds_error"] = True
@@ -151,11 +151,6 @@ class PointingInterpolator(LinearInterpolator):
     required_columns = frozenset(["time", "azimuth", "altitude"])
     expected_units = {"azimuth": u.rad, "altitude": u.rad}
 
-    def __init__(self, h5file: None | tables.File = None, **kwargs: Any) -> None:
-        super().__init__(h5file=h5file, **kwargs)
-
-        self._interpolators = {}
-
     def __call__(self, tel_id: int, time: Time) -> tuple[u.Quantity, u.Quantity]:
         """
         Interpolate alt/az for given time and tel_id.

From c3da3d87f726acd8c4098f029ad7701c658bd442 Mon Sep 17 00:00:00 2001
From: Christoph Toennis <christoph.toennis1988@gmail.com>
Date: Wed, 4 Dec 2024 13:54:25 +0100
Subject: [PATCH 21/30] putting required units on ChunkInterpolator

---
 src/ctapipe/monitoring/interpolation.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/ctapipe/monitoring/interpolation.py b/src/ctapipe/monitoring/interpolation.py
index 9f9f5f8751e..029a82a18c1 100644
--- a/src/ctapipe/monitoring/interpolation.py
+++ b/src/ctapipe/monitoring/interpolation.py
@@ -222,10 +222,11 @@ class ChunkInterpolator(MonitoringInterpolator):
     Simple interpolator for overlapping chunks of data.
     """
 
+    required_columns = ["start_time", "end_time"]
+
     def __init__(self, h5file: None | tables.File = None, **kwargs: Any) -> None:
         super().__init__(**kwargs)
         self._interpolators = {}
-        self.required_columns = ["start_time", "end_time"]
         self.expected_units = {}
         self.start_time = {}
         self.end_time = {}
@@ -290,6 +291,8 @@ def add_table(self, tel_id: int, input_table: Table, columns: list[str]) -> None
         required_columns = set(self.required_columns)
         required_columns.update(columns)
         self.required_columns = frozenset(required_columns)
+        for col in columns:
+            self.expected_units[col] = None
         self._check_tables(input_table)
 
         input_table = input_table.copy()

From 3c1f6857d749ba7a385ec9f6019866f5d856dee1 Mon Sep 17 00:00:00 2001
From: Christoph Toennis <christoph.toennis1988@gmail.com>
Date: Wed, 4 Dec 2024 14:41:32 +0100
Subject: [PATCH 22/30] implementing reviewer comment

---
 src/ctapipe/monitoring/interpolation.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/ctapipe/monitoring/interpolation.py b/src/ctapipe/monitoring/interpolation.py
index 029a82a18c1..4a3c8ceebb4 100644
--- a/src/ctapipe/monitoring/interpolation.py
+++ b/src/ctapipe/monitoring/interpolation.py
@@ -1,4 +1,5 @@
 from abc import ABCMeta, abstractmethod
+from copy import deepcopy
 from functools import partial
 from typing import Any
 
@@ -222,7 +223,7 @@ class ChunkInterpolator(MonitoringInterpolator):
     Simple interpolator for overlapping chunks of data.
     """
 
-    required_columns = ["start_time", "end_time"]
+    required_columns = frozenset(["start_time", "end_time"])
 
     def __init__(self, h5file: None | tables.File = None, **kwargs: Any) -> None:
         super().__init__(**kwargs)
@@ -288,7 +289,7 @@ def add_table(self, tel_id: int, input_table: Table, columns: list[str]) -> None
             Names of the columns to interpolate.
         """
 
-        required_columns = set(self.required_columns)
+        required_columns = set(deepcopy(self.required_columns))
         required_columns.update(columns)
         self.required_columns = frozenset(required_columns)
         for col in columns:

From c2530e49aca9ef191073d9d2b017f050b6c436fb Mon Sep 17 00:00:00 2001
From: Christoph Toennis <christoph.toennis1988@gmail.com>
Date: Mon, 9 Dec 2024 11:37:19 +0100
Subject: [PATCH 23/30] making required_columns an instance variable

---
 src/ctapipe/monitoring/interpolation.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/src/ctapipe/monitoring/interpolation.py b/src/ctapipe/monitoring/interpolation.py
index 4a3c8ceebb4..3560909b170 100644
--- a/src/ctapipe/monitoring/interpolation.py
+++ b/src/ctapipe/monitoring/interpolation.py
@@ -1,5 +1,4 @@
 from abc import ABCMeta, abstractmethod
-from copy import deepcopy
 from functools import partial
 from typing import Any
 
@@ -223,10 +222,9 @@ class ChunkInterpolator(MonitoringInterpolator):
     Simple interpolator for overlapping chunks of data.
     """
 
-    required_columns = frozenset(["start_time", "end_time"])
-
     def __init__(self, h5file: None | tables.File = None, **kwargs: Any) -> None:
         super().__init__(**kwargs)
+        self.required_columns = set(["start_time", "end_time"])
         self._interpolators = {}
         self.expected_units = {}
         self.start_time = {}
@@ -289,9 +287,8 @@ def add_table(self, tel_id: int, input_table: Table, columns: list[str]) -> None
             Names of the columns to interpolate.
         """
 
-        required_columns = set(deepcopy(self.required_columns))
-        required_columns.update(columns)
-        self.required_columns = frozenset(required_columns)
+        self.required_columns.update(columns)
+        self.required_columns = set(self.required_columns)
         for col in columns:
             self.expected_units[col] = None
         self._check_tables(input_table)

From 9a899dfa9f778c9abc803c3e1bdbaa22b4229829 Mon Sep 17 00:00:00 2001
From: Christoph Toennis <christoph.toennis1988@gmail.com>
Date: Mon, 9 Dec 2024 15:16:34 +0100
Subject: [PATCH 24/30] making subclasses to ChunkInterpolator

---
 src/ctapipe/monitoring/interpolation.py       |  40 +++----
 .../monitoring/tests/test_interpolator.py     | 104 ++++++++----------
 2 files changed, 62 insertions(+), 82 deletions(-)

diff --git a/src/ctapipe/monitoring/interpolation.py b/src/ctapipe/monitoring/interpolation.py
index 3560909b170..6fa8ca5ae99 100644
--- a/src/ctapipe/monitoring/interpolation.py
+++ b/src/ctapipe/monitoring/interpolation.py
@@ -224,16 +224,15 @@ class ChunkInterpolator(MonitoringInterpolator):
 
     def __init__(self, h5file: None | tables.File = None, **kwargs: Any) -> None:
         super().__init__(**kwargs)
-        self.required_columns = set(["start_time", "end_time"])
         self._interpolators = {}
-        self.expected_units = {}
         self.start_time = {}
         self.end_time = {}
         self.values = {}
+        self.columns = list(self.required_columns)  # these will be the data columns
+        self.columns.remove("start_time")
+        self.columns.remove("end_time")
 
-    def __call__(
-        self, tel_id: int, time: Time, columns: str | list[str]
-    ) -> float | dict[str, float]:
+    def __call__(self, tel_id: int, time: Time) -> float | dict[str, float]:
         """
         Interpolate overlapping chunks of data for a given time, tel_id, and column(s).
 
@@ -243,8 +242,6 @@ def __call__(
             Telescope id.
         time : astropy.time.Time
             Time for which to interpolate the data.
-        columns : str or list of str
-            Name(s) of the column(s) to interpolate.
 
         Returns
         -------
@@ -254,12 +251,9 @@ def __call__(
 
         self._check_interpolators(tel_id)
 
-        if isinstance(columns, str):
-            columns = [columns]
-
         result = {}
         mjd = time.to_value("mjd")
-        for column in columns:
+        for column in self.columns:
             if column not in self._interpolators[tel_id]:
                 raise ValueError(
                     f"Column '{column}' not found in interpolators for tel_id {tel_id}"
@@ -267,10 +261,10 @@ def __call__(
             result[column] = self._interpolators[tel_id][column](mjd)
 
         if len(result) == 1:
-            return result[columns[0]]
+            return result[self.columns[0]]
         return result
 
-    def add_table(self, tel_id: int, input_table: Table, columns: list[str]) -> None:
+    def add_table(self, tel_id: int, input_table: Table) -> None:
         """
         Add a table to this interpolator for specific columns.
 
@@ -283,14 +277,8 @@ def add_table(self, tel_id: int, input_table: Table, columns: list[str]) -> None
             are ``start_time`` as ``validity start Time`` column,
             ``end_time`` as ``validity end Time`` and the specified columns
             for the data of the chunks.
-        columns : list of str
-            Names of the columns to interpolate.
         """
 
-        self.required_columns.update(columns)
-        self.required_columns = set(self.required_columns)
-        for col in columns:
-            self.expected_units[col] = None
         self._check_tables(input_table)
 
         input_table = input_table.copy()
@@ -302,7 +290,7 @@ def add_table(self, tel_id: int, input_table: Table, columns: list[str]) -> None
             self.start_time[tel_id] = {}
             self.end_time[tel_id] = {}
 
-        for column in columns:
+        for column in self.columns:
             self.values[tel_id][column] = input_table[column]
             self.start_time[tel_id][column] = input_table["start_time"].to_value("mjd")
             self.end_time[tel_id][column] = input_table["end_time"].to_value("mjd")
@@ -318,8 +306,6 @@ def _interpolate_chunk(self, tel_id, column, mjd: float) -> float:
         ----------
         tel_id : int
             tel_id for which data is to be interpolated
-        column : str
-            name of the column for which data is to be interpolated
         mjd : float
             Time for which to interpolate the data.
         """
@@ -346,3 +332,13 @@ def _interpolate_chunk(self, tel_id, column, mjd: float) -> float:
                     return value
 
         return np.nan
+
+
+class FlatFieldInterpolator(ChunkInterpolator):
+    required_columns = frozenset(["start_time", "end_time", "relative_gain"])
+    expected_units = {"relative_gain": None}
+
+
+class PedestalInterpolator(ChunkInterpolator):
+    required_columns = frozenset(["start_time", "end_time", "pedestal"])
+    expected_units = {"pedestal": None}
diff --git a/src/ctapipe/monitoring/tests/test_interpolator.py b/src/ctapipe/monitoring/tests/test_interpolator.py
index 696f4f4f209..44fa206fdaa 100644
--- a/src/ctapipe/monitoring/tests/test_interpolator.py
+++ b/src/ctapipe/monitoring/tests/test_interpolator.py
@@ -5,125 +5,109 @@
 from astropy.table import Table
 from astropy.time import Time
 
-from ctapipe.monitoring.interpolation import ChunkInterpolator, PointingInterpolator
+from ctapipe.monitoring.interpolation import (
+    FlatFieldInterpolator,
+    PedestalInterpolator,
+    PointingInterpolator,
+)
 
 t0 = Time("2022-01-01T00:00:00")
 
 
 def test_chunk_selection():
-    table = Table(
+    table_ff = Table(
         {
             "start_time": t0 + [0, 1, 2, 6] * u.s,
             "end_time": t0 + [2, 3, 4, 8] * u.s,
-            "values": [1, 2, 3, 4],
+            "relative_gain": [1, 2, 3, 4],
         },
     )
-    interpolator = ChunkInterpolator()
-    interpolator.add_table(1, table, ["values"])
+    interpolator_ff = FlatFieldInterpolator()
+    interpolator_ff.add_table(1, table_ff)
 
-    val1 = interpolator(tel_id=1, time=t0 + 1.2 * u.s, columns="values")
-    val2 = interpolator(tel_id=1, time=t0 + 1.7 * u.s, columns="values")
-    val3 = interpolator(tel_id=1, time=t0 + 2.2 * u.s, columns="values")
+    val1 = interpolator_ff(tel_id=1, time=t0 + 1.2 * u.s)
+    val2 = interpolator_ff(tel_id=1, time=t0 + 1.7 * u.s)
+    val3 = interpolator_ff(tel_id=1, time=t0 + 2.2 * u.s)
 
     assert np.isclose(val1, 2)
     assert np.isclose(val2, 2)
     assert np.isclose(val3, 3)
 
-
-def test_chunk_selection_multiple_columns():
-    table = Table(
+    table_ped = Table(
         {
             "start_time": t0 + [0, 1, 2, 6] * u.s,
             "end_time": t0 + [2, 3, 4, 8] * u.s,
-            "values1": [1, 2, 3, 4],
-            "values2": [10, 20, 30, 40],
+            "pedestal": [1, 2, 3, 4],
         },
     )
-    interpolator = ChunkInterpolator()
-    interpolator.add_table(1, table, ["values1", "values2"])
+    interpolator_ped = PedestalInterpolator()
+    interpolator_ped.add_table(1, table_ped)
 
-    result1 = interpolator(
-        tel_id=1, time=t0 + 1.2 * u.s, columns=["values1", "values2"]
-    )
-    result2 = interpolator(
-        tel_id=1, time=t0 + 1.7 * u.s, columns=["values1", "values2"]
-    )
-    result3 = interpolator(
-        tel_id=1, time=t0 + 2.2 * u.s, columns=["values1", "values2"]
-    )
+    val1 = interpolator_ped(tel_id=1, time=t0 + 1.2 * u.s)
+    val2 = interpolator_ped(tel_id=1, time=t0 + 1.7 * u.s)
+    val3 = interpolator_ped(tel_id=1, time=t0 + 2.2 * u.s)
 
-    assert np.isclose(result1["values1"], 2)
-    assert np.isclose(result1["values2"], 20)
-    assert np.isclose(result2["values1"], 2)
-    assert np.isclose(result2["values2"], 20)
-    assert np.isclose(result3["values1"], 3)
-    assert np.isclose(result3["values2"], 30)
+    assert np.isclose(val1, 2)
+    assert np.isclose(val2, 2)
+    assert np.isclose(val3, 3)
 
 
 def test_nan_switch():
-    table = Table(
+    table_ff = Table(
         {
             "start_time": t0 + [0, 1, 2, 6] * u.s,
             "end_time": t0 + [2, 3, 4, 8] * u.s,
-            "values": [1, np.nan, 3, 4],
+            "relative_gain": [1, np.nan, 3, 4],
         },
     )
-    interpolator = ChunkInterpolator()
-    interpolator.add_table(1, table, ["values"])
+    interpolator_ff = FlatFieldInterpolator()
+    interpolator_ff.add_table(1, table_ff)
 
-    val = interpolator(tel_id=1, time=t0 + 1.2 * u.s, columns="values")
+    val = interpolator_ff(tel_id=1, time=t0 + 1.2 * u.s)
 
     assert np.isclose(val, 1)
 
-
-def test_nan_switch_multiple_columns():
-    table = Table(
+    table_ped = Table(
         {
             "start_time": t0 + [0, 1, 2, 6] * u.s,
             "end_time": t0 + [2, 3, 4, 8] * u.s,
-            "values1": [1, np.nan, 3, 4],
-            "values2": [10, 20, np.nan, 40],
+            "pedestal": [1, np.nan, 3, 4],
         },
     )
-    interpolator = ChunkInterpolator()
-    interpolator.add_table(1, table, ["values1", "values2"])
+    interpolator_ped = PedestalInterpolator()
+    interpolator_ped.add_table(1, table_ped)
 
-    result = interpolator(tel_id=1, time=t0 + 1.2 * u.s, columns=["values1", "values2"])
+    val = interpolator_ped(tel_id=1, time=t0 + 1.2 * u.s)
 
-    assert np.isclose(result["values1"], 1)
-    assert np.isclose(result["values2"], 20)
+    assert np.isclose(val, 1)
 
 
 def test_no_valid_chunk():
-    table = Table(
+    table_ff = Table(
         {
             "start_time": t0 + [0, 1, 2, 6] * u.s,
             "end_time": t0 + [2, 3, 4, 8] * u.s,
-            "values": [1, 2, 3, 4],
+            "relative_gain": [1, 2, 3, 4],
         },
     )
-    interpolator = ChunkInterpolator()
-    interpolator.add_table(1, table, ["values"])
+    interpolator_ff = FlatFieldInterpolator()
+    interpolator_ff.add_table(1, table_ff)
 
-    val = interpolator(tel_id=1, time=t0 + 5.2 * u.s, columns="values")
+    val = interpolator_ff(tel_id=1, time=t0 + 5.2 * u.s)
     assert np.isnan(val)
 
-
-def test_no_valid_chunk_multiple_columns():
-    table = Table(
+    table_ped = Table(
         {
             "start_time": t0 + [0, 1, 2, 6] * u.s,
             "end_time": t0 + [2, 3, 4, 8] * u.s,
-            "values1": [1, 2, 3, 4],
-            "values2": [10, 20, 30, 40],
+            "pedestal": [1, 2, 3, 4],
         },
     )
-    interpolator = ChunkInterpolator()
-    interpolator.add_table(1, table, ["values1", "values2"])
+    interpolator_ped = PedestalInterpolator()
+    interpolator_ped.add_table(1, table_ped)
 
-    result = interpolator(tel_id=1, time=t0 + 5.2 * u.s, columns=["values1", "values2"])
-    assert np.isnan(result["values1"])
-    assert np.isnan(result["values2"])
+    val = interpolator_ped(tel_id=1, time=t0 + 5.2 * u.s)
+    assert np.isnan(val)
 
 
 def test_azimuth_switchover():

From 0da6c5febd5847746ffff34d40522eb8b9e9eb90 Mon Sep 17 00:00:00 2001
From: Christoph Toennis <christoph.toennis1988@gmail.com>
Date: Mon, 9 Dec 2024 16:39:04 +0100
Subject: [PATCH 25/30] simplifying start_time and end_time

---
 src/ctapipe/monitoring/interpolation.py | 20 ++++++--------------
 1 file changed, 6 insertions(+), 14 deletions(-)

diff --git a/src/ctapipe/monitoring/interpolation.py b/src/ctapipe/monitoring/interpolation.py
index 6fa8ca5ae99..937031b3cc3 100644
--- a/src/ctapipe/monitoring/interpolation.py
+++ b/src/ctapipe/monitoring/interpolation.py
@@ -254,11 +254,7 @@ def __call__(self, tel_id: int, time: Time) -> float | dict[str, float]:
         result = {}
         mjd = time.to_value("mjd")
         for column in self.columns:
-            if column not in self._interpolators[tel_id]:
-                raise ValueError(
-                    f"Column '{column}' not found in interpolators for tel_id {tel_id}"
-                )
-            result[column] = self._interpolators[tel_id][column](mjd)
+            result[column] = self._interpolators[tel_id](column, mjd)
 
         if len(result) == 1:
             return result[self.columns[0]]
@@ -287,16 +283,12 @@ def add_table(self, tel_id: int, input_table: Table) -> None:
         if tel_id not in self._interpolators:
             self._interpolators[tel_id] = {}
             self.values[tel_id] = {}
-            self.start_time[tel_id] = {}
-            self.end_time[tel_id] = {}
+            self.start_time[tel_id] = input_table["start_time"].to_value("mjd")
+            self.end_time[tel_id] = input_table["end_time"].to_value("mjd")
+            self._interpolators[tel_id] = partial(self._interpolate_chunk, tel_id)
 
         for column in self.columns:
             self.values[tel_id][column] = input_table[column]
-            self.start_time[tel_id][column] = input_table["start_time"].to_value("mjd")
-            self.end_time[tel_id][column] = input_table["end_time"].to_value("mjd")
-            self._interpolators[tel_id][column] = partial(
-                self._interpolate_chunk, tel_id, column
-            )
 
     def _interpolate_chunk(self, tel_id, column, mjd: float) -> float:
         """
@@ -310,8 +302,8 @@ def _interpolate_chunk(self, tel_id, column, mjd: float) -> float:
             Time for which to interpolate the data.
         """
 
-        start_time = self.start_time[tel_id][column]
-        end_time = self.end_time[tel_id][column]
+        start_time = self.start_time[tel_id]
+        end_time = self.end_time[tel_id]
         values = self.values[tel_id][column]
         # Find the index of the closest preceding start time
         preceding_index = np.searchsorted(start_time, mjd, side="right") - 1

From cd35c8e38e82bea0d98e44f593bb2ad66768d48a Mon Sep 17 00:00:00 2001
From: Christoph Toennis <christoph.toennis1988@gmail.com>
Date: Tue, 10 Dec 2024 11:59:02 +0100
Subject: [PATCH 26/30] adding data groups

---
 src/ctapipe/monitoring/interpolation.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/ctapipe/monitoring/interpolation.py b/src/ctapipe/monitoring/interpolation.py
index 937031b3cc3..a8f7afde0e4 100644
--- a/src/ctapipe/monitoring/interpolation.py
+++ b/src/ctapipe/monitoring/interpolation.py
@@ -329,8 +329,10 @@ def _interpolate_chunk(self, tel_id, column, mjd: float) -> float:
 class FlatFieldInterpolator(ChunkInterpolator):
     required_columns = frozenset(["start_time", "end_time", "relative_gain"])
     expected_units = {"relative_gain": None}
+    telescope_data_group = "/dl1/monitoring/telescope/flatfield"
 
 
 class PedestalInterpolator(ChunkInterpolator):
     required_columns = frozenset(["start_time", "end_time", "pedestal"])
     expected_units = {"pedestal": None}
+    telescope_data_group = "/dl1/monitoring/telescope/pedestal"

From 4725326b2cab7aadc3c28f669051208c84061b44 Mon Sep 17 00:00:00 2001
From: Christoph Toennis <christoph.toennis1988@gmail.com>
Date: Thu, 12 Dec 2024 11:17:25 +0100
Subject: [PATCH 27/30] adding child classes, making chunk function take arrays

---
 src/ctapipe/monitoring/__init__.py            |  2 ++
 src/ctapipe/monitoring/interpolation.py       | 17 +++++++++-------
 .../monitoring/tests/test_interpolator.py     | 20 ++++++++++---------
 3 files changed, 23 insertions(+), 16 deletions(-)

diff --git a/src/ctapipe/monitoring/__init__.py b/src/ctapipe/monitoring/__init__.py
index 03d583abaab..d27ba445dca 100644
--- a/src/ctapipe/monitoring/__init__.py
+++ b/src/ctapipe/monitoring/__init__.py
@@ -27,4 +27,6 @@
     "LinearInterpolator",
     "PointingInterpolator",
     "ChunkInterpolator",
+    "FlatFieldInterpolator",
+    "PedestalInterpolator",
 ]
diff --git a/src/ctapipe/monitoring/interpolation.py b/src/ctapipe/monitoring/interpolation.py
index a8f7afde0e4..eb1617072fd 100644
--- a/src/ctapipe/monitoring/interpolation.py
+++ b/src/ctapipe/monitoring/interpolation.py
@@ -307,23 +307,26 @@ def _interpolate_chunk(self, tel_id, column, mjd: float) -> float:
         values = self.values[tel_id][column]
         # Find the index of the closest preceding start time
         preceding_index = np.searchsorted(start_time, mjd, side="right") - 1
+
         if preceding_index < 0:
             return np.nan
 
+        value = np.nan
+
         # Check if the time is within the valid range of the chunk
         if start_time[preceding_index] <= mjd <= end_time[preceding_index]:
             value = values[preceding_index]
-            if not np.isnan(value):
-                return value
 
-        # If the closest preceding chunk has nan, check the next closest chunk
+        # If an element in the closest preceding chunk has nan, check the next closest chunk
+
         for i in range(preceding_index - 1, -1, -1):
             if start_time[i] <= mjd <= end_time[i]:
-                value = values[i]
-                if not np.isnan(value):
-                    return value
+                if value is np.nan:
+                    value = values[i]
+                else:
+                    value = np.where(np.isnan(value), values[i], value)
 
-        return np.nan
+        return value
 
 
 class FlatFieldInterpolator(ChunkInterpolator):
diff --git a/src/ctapipe/monitoring/tests/test_interpolator.py b/src/ctapipe/monitoring/tests/test_interpolator.py
index 44fa206fdaa..86fd5498aee 100644
--- a/src/ctapipe/monitoring/tests/test_interpolator.py
+++ b/src/ctapipe/monitoring/tests/test_interpolator.py
@@ -14,12 +14,14 @@
 t0 = Time("2022-01-01T00:00:00")
 
 
-def test_chunk_selection():
+def test_chunk_selection(camera_geometry):
     table_ff = Table(
         {
             "start_time": t0 + [0, 1, 2, 6] * u.s,
             "end_time": t0 + [2, 3, 4, 8] * u.s,
-            "relative_gain": [1, 2, 3, 4],
+            "relative_gain": [
+                np.full((2, len(camera_geometry)), x) for x in [1, 2, 3, 4]
+            ],
         },
     )
     interpolator_ff = FlatFieldInterpolator()
@@ -29,15 +31,15 @@ def test_chunk_selection():
     val2 = interpolator_ff(tel_id=1, time=t0 + 1.7 * u.s)
     val3 = interpolator_ff(tel_id=1, time=t0 + 2.2 * u.s)
 
-    assert np.isclose(val1, 2)
-    assert np.isclose(val2, 2)
-    assert np.isclose(val3, 3)
+    assert np.all(np.isclose(val1, np.full((2, len(camera_geometry)), 2)))
+    assert np.all(np.isclose(val2, np.full((2, len(camera_geometry)), 2)))
+    assert np.all(np.isclose(val3, np.full((2, len(camera_geometry)), 3)))
 
     table_ped = Table(
         {
             "start_time": t0 + [0, 1, 2, 6] * u.s,
             "end_time": t0 + [2, 3, 4, 8] * u.s,
-            "pedestal": [1, 2, 3, 4],
+            "pedestal": [np.full((2, len(camera_geometry)), x) for x in [1, 2, 3, 4]],
         },
     )
     interpolator_ped = PedestalInterpolator()
@@ -47,9 +49,9 @@ def test_chunk_selection():
     val2 = interpolator_ped(tel_id=1, time=t0 + 1.7 * u.s)
     val3 = interpolator_ped(tel_id=1, time=t0 + 2.2 * u.s)
 
-    assert np.isclose(val1, 2)
-    assert np.isclose(val2, 2)
-    assert np.isclose(val3, 3)
+    assert np.all(np.isclose(val1, np.full((2, len(camera_geometry)), 2)))
+    assert np.all(np.isclose(val2, np.full((2, len(camera_geometry)), 2)))
+    assert np.all(np.isclose(val3, np.full((2, len(camera_geometry)), 3)))
 
 
 def test_nan_switch():

From 7ac13d9d993c98441754c822c121b6455d4803cf Mon Sep 17 00:00:00 2001
From: Christoph Toennis <christoph.toennis1988@gmail.com>
Date: Thu, 12 Dec 2024 11:40:12 +0100
Subject: [PATCH 28/30] making the nan switch test check if the switch is done
 element-wise

---
 .../monitoring/tests/test_interpolator.py     | 21 ++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/src/ctapipe/monitoring/tests/test_interpolator.py b/src/ctapipe/monitoring/tests/test_interpolator.py
index 86fd5498aee..0e2f0009ffd 100644
--- a/src/ctapipe/monitoring/tests/test_interpolator.py
+++ b/src/ctapipe/monitoring/tests/test_interpolator.py
@@ -54,12 +54,18 @@ def test_chunk_selection(camera_geometry):
     assert np.all(np.isclose(val3, np.full((2, len(camera_geometry)), 3)))
 
 
-def test_nan_switch():
+def test_nan_switch(camera_geometry):
+    data = np.array([np.full((2, len(camera_geometry)), x) for x in [1, 2, 3, 4]])
+    data[1][0, 0] = 5
+    data = np.where(
+        data > 4, np.nan, data
+    )  # this is a workaround to introduce a nan in the data
+
     table_ff = Table(
         {
             "start_time": t0 + [0, 1, 2, 6] * u.s,
             "end_time": t0 + [2, 3, 4, 8] * u.s,
-            "relative_gain": [1, np.nan, 3, 4],
+            "relative_gain": data,
         },
     )
     interpolator_ff = FlatFieldInterpolator()
@@ -67,13 +73,18 @@ def test_nan_switch():
 
     val = interpolator_ff(tel_id=1, time=t0 + 1.2 * u.s)
 
-    assert np.isclose(val, 1)
+    res = np.full((2, len(camera_geometry)), 2)
+    res[0][
+        0
+    ] = 1  # where the nan was introduced before we should now have the value from the earlier chunk
+
+    assert np.all(np.isclose(val, res))
 
     table_ped = Table(
         {
             "start_time": t0 + [0, 1, 2, 6] * u.s,
             "end_time": t0 + [2, 3, 4, 8] * u.s,
-            "pedestal": [1, np.nan, 3, 4],
+            "pedestal": data,
         },
     )
     interpolator_ped = PedestalInterpolator()
@@ -81,7 +92,7 @@ def test_nan_switch():
 
     val = interpolator_ped(tel_id=1, time=t0 + 1.2 * u.s)
 
-    assert np.isclose(val, 1)
+    assert np.all(np.isclose(val, res))
 
 
 def test_no_valid_chunk():

From 3b8b4a0b3eb7f5a7c71a774496e57f4c4351f242 Mon Sep 17 00:00:00 2001
From: Christoph Toennis <christoph.toennis1988@gmail.com>
Date: Fri, 13 Dec 2024 09:54:28 +0100
Subject: [PATCH 29/30] adding imports to __init__

---
 src/ctapipe/monitoring/__init__.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/ctapipe/monitoring/__init__.py b/src/ctapipe/monitoring/__init__.py
index d27ba445dca..563ab6d975b 100644
--- a/src/ctapipe/monitoring/__init__.py
+++ b/src/ctapipe/monitoring/__init__.py
@@ -4,8 +4,10 @@
 from .aggregator import PlainAggregator, SigmaClippingAggregator, StatisticsAggregator
 from .interpolation import (
     ChunkInterpolator,
+    FlatFieldInterpolator,
     LinearInterpolator,
     MonitoringInterpolator,
+    PedestalInterpolator,
     PointingInterpolator,
 )
 from .outlier import (

From 56afd670b83f1daa92293a5c78f7b5140de6a610 Mon Sep 17 00:00:00 2001
From: "mykhailo.dalchenko" <mykhailo.dalchenko@cta-consortium.org>
Date: Tue, 28 Jan 2025 18:13:59 +0100
Subject: [PATCH 30/30] Simplify logic in ChunkInterpolator - Move the
 _check_interpolators from MonitoringInterpolator to LinearInterpolator - Call
 _interpolate_chunk in directly __call__ of ChunkInterpolator

---
 src/ctapipe/monitoring/interpolation.py | 29 +++++++++++--------------
 1 file changed, 13 insertions(+), 16 deletions(-)

diff --git a/src/ctapipe/monitoring/interpolation.py b/src/ctapipe/monitoring/interpolation.py
index eb1617072fd..1425938efb4 100644
--- a/src/ctapipe/monitoring/interpolation.py
+++ b/src/ctapipe/monitoring/interpolation.py
@@ -1,5 +1,4 @@
 from abc import ABCMeta, abstractmethod
-from functools import partial
 from typing import Any
 
 import astropy.units as u
@@ -89,13 +88,6 @@ def _check_tables(self, input_table: Table) -> None:
                         f"{col} must have units compatible with '{self.expected_units[col].name}'"
                     )
 
-    def _check_interpolators(self, tel_id: int) -> None:
-        if tel_id not in self._interpolators:
-            if self.h5file is not None:
-                self._read_parameter_table(tel_id)  # might need to be removed
-            else:
-                raise KeyError(f"No table available for tel_id {tel_id}")
-
     def _read_parameter_table(self, tel_id: int) -> None:
         # prevent circular import between io and monitoring
         from ..io import read_table
@@ -141,6 +133,13 @@ def __init__(self, h5file: None | tables.File = None, **kwargs: Any) -> None:
             self.interp_options["bounds_error"] = False
             self.interp_options["fill_value"] = np.nan
 
+    def _check_interpolators(self, tel_id: int) -> None:
+        if tel_id not in self._interpolators:
+            if self.h5file is not None:
+                self._read_parameter_table(tel_id)  # might need to be removed
+            else:
+                raise KeyError(f"No table available for tel_id {tel_id}")
+
 
 class PointingInterpolator(LinearInterpolator):
     """
@@ -249,12 +248,13 @@ def __call__(self, tel_id: int, time: Time) -> float | dict[str, float]:
             Interpolated data for the specified column(s).
         """
 
-        self._check_interpolators(tel_id)
+        if tel_id not in self.values:
+            self._read_parameter_table(tel_id)
 
         result = {}
         mjd = time.to_value("mjd")
         for column in self.columns:
-            result[column] = self._interpolators[tel_id](column, mjd)
+            result[column] = self._interpolate_chunk(tel_id, column, mjd)
 
         if len(result) == 1:
             return result[self.columns[0]]
@@ -280,12 +280,9 @@ def add_table(self, tel_id: int, input_table: Table) -> None:
         input_table = input_table.copy()
         input_table.sort("start_time")
 
-        if tel_id not in self._interpolators:
-            self._interpolators[tel_id] = {}
-            self.values[tel_id] = {}
-            self.start_time[tel_id] = input_table["start_time"].to_value("mjd")
-            self.end_time[tel_id] = input_table["end_time"].to_value("mjd")
-            self._interpolators[tel_id] = partial(self._interpolate_chunk, tel_id)
+        self.values[tel_id] = {}
+        self.start_time[tel_id] = input_table["start_time"].to_value("mjd")
+        self.end_time[tel_id] = input_table["end_time"].to_value("mjd")
 
         for column in self.columns:
             self.values[tel_id][column] = input_table[column]