diff --git a/src/iblphotometry/processing.py b/src/iblphotometry/processing.py index 8ab3f48..52d3a2a 100644 --- a/src/iblphotometry/processing.py +++ b/src/iblphotometry/processing.py @@ -582,15 +582,29 @@ def fillnan_kde(y: np.ndarray, w: int = 25): def remove_outliers( - F: pd.Series, w_size: int = 1000, alpha: float = 0.005, w: int = 25 + F: pd.Series, + w_len: float = 60, + alpha: float = 0.005, + w: int = 25, + fs=None, + max_it=100, ): y, t = F.values, F.index.values + fs = 1 / np.median(np.diff(t)) if fs is None else fs + w_size = int(w_len * fs) + y = copy(y) outliers = detect_outliers(y, w_size=w_size, alpha=alpha) + j = 0 while len(outliers) > 0: y[outliers] = np.nan y = fillnan_kde(y, w=w) outliers = detect_outliers(y, w_size=w_size, alpha=alpha) + if j > max_it: + break + else: + j += 1 + return pd.Series(y, index=t) diff --git a/src/iblphotometry_tests/test_processing.py b/src/iblphotometry_tests/test_processing.py new file mode 100644 index 0000000..7045776 --- /dev/null +++ b/src/iblphotometry_tests/test_processing.py @@ -0,0 +1,34 @@ +import iblphotometry.io as fio +import iblphotometry.processing as processing +import pandas as pd + +from iblphotometry_tests.base_tests import PhotometryDataTestCase + + +class TestProcessing(PhotometryDataTestCase): + # think here about the possible use cases + + def test_processing(self): + self.set_paths('alejandro') + # get data + raw_dfs = fio.from_ibl_pqt( + self.paths['photometry_signal_pqt'], + self.paths['photometryROI_locations_pqt'], + ) + trials = pd.read_parquet(self.paths['trials_table_pqt']) + raw_df = raw_dfs['GCaMP']['DMS'] + + # bleach corrections + processing.lowpass_bleachcorrect(raw_df) + processing.exponential_bleachcorrect(raw_df) + + # outlier removal + processing.remove_outliers(raw_df) + processing.remove_spikes(raw_df) + + # other functions + processing.make_sliding_window(raw_df.values, 100, method='stride_tricks') + processing.make_sliding_window(raw_df.values, 100, method='window_generator') + processing.sliding_dFF(raw_df, w_len=60) + processing.sliding_z(raw_df, w_len=60) + processing.sliding_mad(raw_df, w_len=60)