Minor fixes

ibllib/io/extractors/biased_trials.py

@@ -97,7 +97,7 @@ class TrialsTableBiased(BaseBpodTrialsExtractor):
     save_names = ('_ibl_trials.table.pqt', None, None, '_ibl_wheel.timestamps.npy', '_ibl_wheel.position.npy',
                   '_ibl_wheelMoves.intervals.npy', '_ibl_wheelMoves.peakAmplitude.npy', None, None)
     var_names = ('table', 'stimOff_times', 'stimFreeze_times', 'wheel_timestamps', 'wheel_position', 'wheelMoves_intervals',
-                 'wheelMoves_peakAmplitude', 'peakVelocity_times', 'is_final_movement')
+                 'wheelMoves_peakAmplitude', 'wheelMoves_peakVelocity_times', 'is_final_movement')
 
     def _extract(self, extractor_classes=None, **kwargs):
         extractor_classes = extractor_classes or []
@@ -125,7 +125,7 @@ class TrialsTableEphys(BaseBpodTrialsExtractor):
                   '_ibl_wheelMoves.intervals.npy', '_ibl_wheelMoves.peakAmplitude.npy', None,
                   None, None, None, '_ibl_trials.quiescencePeriod.npy')
     var_names = ('table', 'stimOff_times', 'stimFreeze_times', 'wheel_timestamps', 'wheel_position', 'wheelMoves_intervals',
-                 'wheelMoves_peakAmplitude', 'peakVelocity_times', 'is_final_movement',
+                 'wheelMoves_peakAmplitude', 'wheelMoves_peakVelocity_times', 'is_final_movement',
                  'phase', 'position', 'quiescence')
 
     def _extract(self, extractor_classes=None, **kwargs):
@@ -152,12 +152,12 @@ class BiasedTrials(BaseBpodTrialsExtractor):
     save_names = ('_ibl_trials.goCueTrigger_times.npy', '_ibl_trials.stimOnTrigger_times.npy', None,
                   '_ibl_trials.stimOffTrigger_times.npy', None, None, '_ibl_trials.table.pqt',
                   '_ibl_trials.stimOff_times.npy', None, '_ibl_wheel.timestamps.npy', '_ibl_wheel.position.npy',
-                  '_ibl_wheelMoves.intervals.npy', '_ibl_wheelMoves.peakAmplitude.npy', None, None, '_ibl_trials.included.npy',
-                  None, None, '_ibl_trials.quiescencePeriod.npy')
+                  '_ibl_wheelMoves.intervals.npy', '_ibl_wheelMoves.peakAmplitude.npy', None, None,
+                  '_ibl_trials.included.npy', None, None, '_ibl_trials.quiescencePeriod.npy')
     var_names = ('goCueTrigger_times', 'stimOnTrigger_times', 'itiIn_times', 'stimOffTrigger_times', 'stimFreezeTrigger_times',
                  'errorCueTrigger_times', 'table', 'stimOff_times', 'stimFreeze_times', 'wheel_timestamps', 'wheel_position',
-                 'wheelMoves_intervals', 'wheelMoves_peakAmplitude', 'peakVelocity_times', 'is_final_movement', 'included',
-                 'phase', 'position', 'quiescence')
+                 'wheelMoves_intervals', 'wheelMoves_peakAmplitude', 'wheelMoves_peakVelocity_times', 'is_final_movement',
+                 'included', 'phase', 'position', 'quiescence')
 
     def _extract(self, extractor_classes=None, **kwargs) -> dict:
         extractor_classes = extractor_classes or []
@@ -182,8 +182,8 @@ class EphysTrials(BaseBpodTrialsExtractor):
                   '_ibl_trials.included.npy', None, None, '_ibl_trials.quiescencePeriod.npy')
     var_names = ('goCueTrigger_times', 'stimOnTrigger_times', 'itiIn_times', 'stimOffTrigger_times', 'stimFreezeTrigger_times',
                  'errorCueTrigger_times', 'table', 'stimOff_times', 'stimFreeze_times', 'wheel_timestamps', 'wheel_position',
-                 'wheelMoves_intervals', 'wheelMoves_peakAmplitude', 'peakVelocity_times', 'is_final_movement', 'included',
-                 'phase', 'position', 'quiescence')
+                 'wheelMoves_intervals', 'wheelMoves_peakAmplitude', 'wheelMoves_peakVelocity_times', 'is_final_movement',
+                 'included', 'phase', 'position', 'quiescence')
 
     def _extract(self, extractor_classes=None, **kwargs) -> dict:
         extractor_classes = extractor_classes or []

ibllib/io/extractors/ephys_fpga.py

@@ -585,13 +585,13 @@ class FpgaTrials(extractors_base.BaseExtractor):
                   '_ibl_trials.stimOff_times.npy', None, None, None, '_ibl_trials.quiescencePeriod.npy',
                   '_ibl_trials.table.pqt', '_ibl_wheel.timestamps.npy',
                   '_ibl_wheel.position.npy', '_ibl_wheelMoves.intervals.npy',
-                  '_ibl_wheelMoves.peakAmplitude.npy')
+                  '_ibl_wheelMoves.peakAmplitude.npy', None)
     var_names = ('goCueTrigger_times', 'stimOnTrigger_times',
                  'stimOffTrigger_times', 'stimFreezeTrigger_times', 'errorCueTrigger_times',
                  'errorCue_times', 'itiIn_times', 'stimFreeze_times', 'stimOff_times',
                  'valveOpen_times', 'phase', 'position', 'quiescence', 'table',
                  'wheel_timestamps', 'wheel_position',
-                 'wheelMoves_intervals', 'wheelMoves_peakAmplitude')
+                 'wheelMoves_intervals', 'wheelMoves_peakAmplitude', 'wheelMoves_peakVelocity_times')
 
     bpod_rsync_fields = ('intervals', 'response_times', 'goCueTrigger_times',
                          'stimOnTrigger_times', 'stimOffTrigger_times',

ibllib/io/extractors/mesoscope.py

@@ -4,8 +4,8 @@
 import numpy as np
 from scipy.signal import find_peaks
 import one.alf.io as alfio
-from one.util import ensure_list
 from one.alf.files import session_path_parts
+from iblutil.util import ensure_list
 import matplotlib.pyplot as plt
 from packaging import version
 

ibllib/io/extractors/training_trials.py

@@ -709,7 +709,7 @@ class TrialsTable(BaseBpodTrialsExtractor):
     save_names = ('_ibl_trials.table.pqt', None, None, '_ibl_wheel.timestamps.npy', '_ibl_wheel.position.npy',
                   '_ibl_wheelMoves.intervals.npy', '_ibl_wheelMoves.peakAmplitude.npy', None, None)
     var_names = ('table', 'stimOff_times', 'stimFreeze_times', 'wheel_timestamps', 'wheel_position', 'wheelMoves_intervals',
-                 'wheelMoves_peakAmplitude', 'peakVelocity_times', 'is_final_movement')
+                 'wheelMoves_peakAmplitude', 'wheelMoves_peakVelocity_times', 'is_final_movement')
 
     def _extract(self, extractor_classes=None, **kwargs):
         base = [Intervals, GoCueTimes, ResponseTimes, Choice, StimOnOffFreezeTimes, ContrastLR, FeedbackTimes, FeedbackType,
@@ -732,7 +732,7 @@ class TrainingTrials(BaseBpodTrialsExtractor):
     var_names = ('repNum', 'goCueTrigger_times', 'stimOnTrigger_times', 'itiIn_times', 'stimOffTrigger_times',
                  'stimFreezeTrigger_times', 'errorCueTrigger_times', 'table', 'stimOff_times', 'stimFreeze_times',
                  'wheel_timestamps', 'wheel_position', 'wheelMoves_intervals', 'wheelMoves_peakAmplitude',
-                 'peakVelocity_times', 'is_final_movement', 'phase', 'position', 'quiescence', 'pause_duration')
+                 'wheelMoves_peakVelocity_times', 'is_final_movement', 'phase', 'position', 'quiescence', 'pause_duration')
 
     def _extract(self) -> dict:
         base = [RepNum, GoCueTriggerTimes, StimOnTriggerTimes, ItiInTimes, StimOffTriggerTimes, StimFreezeTriggerTimes,

ibllib/io/extractors/training_wheel.py

@@ -330,11 +330,6 @@ def extract_first_movement_times(wheel_moves, trials, min_qt=None):
     gap between quiescence end and cue start, or during the quiescence period but sub-
     threshold).  The movement is sufficiently large if it is greater than or equal to THRESH.
 
-    :param wheel_moves:
-    :param trials: dictionary of trial data
-    :param min_qt:
-    :return: numpy array of
-
     Parameters
     ----------
     wheel_moves : dict
@@ -407,9 +402,9 @@ class Wheel(BaseBpodTrialsExtractor):
     save_names = ('_ibl_wheel.timestamps.npy', '_ibl_wheel.position.npy',
                   '_ibl_wheelMoves.intervals.npy', '_ibl_wheelMoves.peakAmplitude.npy', None,
                   '_ibl_trials.firstMovement_times.npy', None)
-    var_names = ('wheel_timestamps', 'wheel_position', 'wheelMoves_intervals',
-                 'wheelMoves_peakAmplitude', 'peakVelocity_times', 'firstMovement_times',
-                 'is_final_movement')
+    var_names = ('wheel_timestamps', 'wheel_position',
+                 'wheelMoves_intervals', 'wheelMoves_peakAmplitude', 'wheelMoves_peakVelocity_times',
+                 'firstMovement_times', 'is_final_movement')
 
     def _extract(self):
         ts, pos = get_wheel_position(self.session_path, self.bpod_trials, task_collection=self.task_collection)
@@ -425,6 +420,5 @@ def _extract(self):
         min_qt = self.settings.get('QUIESCENT_PERIOD', None)
 
         first_moves, is_final, _ = extract_first_movement_times(moves, trials, min_qt=min_qt)
-        output = (ts, pos, moves['intervals'], moves['peakAmplitude'],
-                  moves['peakVelocity_times'], first_moves, is_final)
+        output = (ts, pos, moves['intervals'], moves['peakAmplitude'], moves['peakVelocity_times'], first_moves, is_final)
         return output

ibllib/oneibl/data_handlers.py

@@ -15,10 +15,10 @@
 
 from one.api import ONE
 from one.webclient import AlyxClient
-from one.util import filter_datasets, ensure_list
+from one.util import filter_datasets
 from one.alf.files import add_uuid_string, session_path_parts
 from one.alf.cache import _make_datasets_df
-from iblutil.util import flatten
+from iblutil.util import flatten, ensure_list
 
 from ibllib.oneibl.registration import register_dataset, get_lab, get_local_data_repository
 from ibllib.oneibl.patcher import FTPPatcher, SDSCPatcher, SDSC_ROOT_PATH, SDSC_PATCH_PATH

ibllib/oneibl/patcher.py

@@ -33,14 +33,14 @@
 
 import globus_sdk
 import iblutil.io.params as iopar
+from iblutil.util import ensure_list
 from one.alf.files import get_session_path, add_uuid_string
 from one.alf.spec import is_uuid_string, is_uuid
 from one import params
 from one.webclient import AlyxClient
 from one.converters import path_from_dataset
 from one.remote import globus
 from one.remote.aws import url2uri
-from one.util import ensure_list
 
 from ibllib.oneibl.registration import register_dataset
 

ibllib/oneibl/registration.py

@@ -12,8 +12,9 @@
 from one.webclient import AlyxClient, no_cache
 from one.converters import ConversionMixin
 import one.alf.exceptions as alferr
-from one.util import datasets2records, ensure_list
 from one.api import ONE
+from one.util import datasets2records
+from iblutil.util import ensure_list
 
 import ibllib
 import ibllib.io.extractors.base

ibllib/pipes/base_tasks.py

@@ -4,8 +4,7 @@
 
 from packaging import version
 from one.webclient import no_cache
-from one.util import ensure_list
-from iblutil.util import flatten
+from iblutil.util import flatten, ensure_list
 import matplotlib.image
 from skimage.io import ImageCollection, imread
 
@@ -551,7 +550,7 @@ def register_snapshots(self, unlink=False, collection=None):
                 snapshot = self._save_as_png(snapshot_tif := snapshot)
                 if unlink:
                     snapshot_tif.unlink()
-            _logger.debug('Uploading "%s"...', snapshot.relative_to(self.session_path))
+            _logger.info('Uploading "%s"...', snapshot.relative_to(self.session_path))
             if snapshot.with_suffix('.txt').exists():
                 with open(snapshot.with_suffix('.txt'), 'r') as txt_file:
                     note['text'] = txt_file.read().strip()

ibllib/pipes/mesoscope_tasks.py

@@ -34,7 +34,7 @@
 from one.alf.spec import to_alf
 from one.alf.files import filename_parts, session_path_parts
 import one.alf.exceptions as alferr
-from iblutil.util import flatten
+from iblutil.util import flatten, ensure_list
 from iblatlas.atlas import ALLEN_CCF_LANDMARKS_MLAPDV_UM, MRITorontoAtlas
 
 from ibllib.pipes import base_tasks
@@ -489,8 +489,7 @@ def _consolidate_exptQC(exptQC):
 
         # Merge and make sure same indexes have same names across all files
         frameQC_names_list = [e['frameQC_names'] for e in exptQC]
-        frameQC_names_list = [{f: 0} if isinstance(f, str) else {f[i]: i for i in range(len(f))}
-                              for f in frameQC_names_list]
+        frameQC_names_list = [{k: i for i, k in enumerate(ensure_list(f))} for f in frameQC_names_list]
         frameQC_names = {k: v for d in frameQC_names_list for k, v in d.items()}
         for d in frameQC_names_list:
             for k, v in d.items():

ibllib/pipes/tasks.py

@@ -85,10 +85,9 @@
 from ibllib.oneibl import data_handlers
 from ibllib.oneibl.data_handlers import get_local_data_repository
 from ibllib.oneibl.registration import get_lab
-from iblutil.util import Bunch, flatten
+from iblutil.util import Bunch, flatten, ensure_list
 import one.params
 from one.api import ONE
-from one.util import ensure_list
 from one import webclient
 import one.alf.io as alfio
 

ibllib/qc/camera.py

@@ -579,9 +579,9 @@ def check_dropped_frames(self, threshold=.1):
         size_diff = int(self.data['count'].size - self.data['video']['length'])
         strict_increase = np.all(np.diff(self.data['count']) > 0)
         if not strict_increase:
-            n_effected = np.sum(np.invert(strict_increase))
+            n_affected = np.sum(np.invert(strict_increase))
             _log.info(f'frame count not strictly increasing: '
-                      f'{n_effected} frames effected ({n_effected / strict_increase.size:.2%})')
+                      f'{n_affected} frames affected ({n_affected / strict_increase.size:.2%})')
             return spec.QC.CRITICAL
         dropped = np.diff(self.data['count']).astype(int) - 1
         pct_dropped = (sum(dropped) / len(dropped) * 100)

ibllib/qc/task_extractors.py

@@ -15,7 +15,7 @@
                    'response_times', 'rewardVolume', 'stimFreezeTrigger_times',
                    'stimFreeze_times', 'stimOffTrigger_times', 'stimOff_times',
                    'stimOnTrigger_times', 'stimOn_times', 'valveOpen_times',
-                   'wheel_moves_intervals', 'wheel_moves_peak_amplitude',
+                   'wheelMoves_intervals', 'wheelMoves_peakAmplitude', 'wheelMoves_peakVelocity_times',
                    'wheel_position', 'wheel_timestamps']
 
 
@@ -56,14 +56,10 @@ def rename_data(data):
         if 'valveOpen_times' not in data:
             data['valveOpen_times'] = data['feedback_times'].copy()
             data['valveOpen_times'][~correct] = np.nan
-        if 'wheel_moves_intervals' not in data and 'wheelMoves_intervals' in data:
-            data['wheel_moves_intervals'] = data.pop('wheelMoves_intervals')
-        if 'wheel_moves_peak_amplitude' not in data and 'wheelMoves_peakAmplitude' in data:
-            data['wheel_moves_peak_amplitude'] = data.pop('wheelMoves_peakAmplitude')
         data['correct'] = correct
         diff_fields = list(set(REQUIRED_FIELDS).difference(set(data.keys())))
         for miss_field in diff_fields:
-            data[miss_field] = data['feedback_times'] * np.nan
+            data[miss_field] = None if miss_field.startswith('wheel') else data['feedback_times'] * np.nan
         if len(diff_fields):
             _logger.warning(f'QC extractor, missing fields filled with NaNs: {diff_fields}')
         return data

ibllib/qc/task_metrics.py

@@ -1111,9 +1111,8 @@ def check_n_trial_events(data, **_):
     # test errorCueTrigger_times separately
     # stimFreeze_times fails often due to TTL flicker
     exclude = ['camera_timestamps', 'errorCueTrigger_times', 'errorCue_times',
-               'firstMovement_times', 'peakVelocity_times', 'valveOpen_times',
-               'wheel_moves_peak_amplitude', 'wheel_moves_intervals', 'wheel_timestamps',
-               'wheel_intervals', 'stimFreeze_times']
+               'wheelMoves_peakVelocity_times', 'valveOpen_times', 'wheelMoves_peakAmplitude',
+               'wheelMoves_intervals', 'wheel_timestamps', 'stimFreeze_times']
     events = [k for k in data.keys() if k.endswith('_times') and k not in exclude]
     metric = np.zeros(data['intervals'].shape[0], dtype=bool)
 

ibllib/tests/extractors/test_extractors.py

@@ -679,7 +679,7 @@ def test_groom_pin_state(self):
             gpio['indices'][i + 1] = np.where(ts > rise + pulse_width)[0][0]
 
         gpio_, audio_, ts_ = camera.groom_pin_state(gpio, audio, ts)
-        self.assertEqual(audio, audio_, 'Audio dict shouldn\'t be effected')
+        self.assertEqual(audio, audio_, 'Audio dict shouldn\'t be affected')
         np.testing.assert_array_almost_equal(ts_[:4], [40., 40.016667, 40.033333, 40.05])
 
         # Broken TTLs + extra TTL

ibllib/tests/test_mesoscope.py

@@ -97,6 +97,36 @@ def test_get_default_tau(self):
             subject_detail['genotype'].pop(1)
             self.assertEqual(self.task.get_default_tau(), 1.5)  # return the default value
 
+    def test_consolidate_exptQC(self):
+        """Test for MesoscopePreprocess._consolidate_exptQC method."""
+        exptQC = [
+            {'frameQC_names': np.array(['ok', 'PMT off', 'galvos fault', 'high signal'], dtype=object),
+             'frameQC_frames': np.array([0, 0, 0, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 4])},
+            {'frameQC_names': np.array(['ok', 'PMT off', 'galvos fault', 'high signal'], dtype=object),
+             'frameQC_frames': np.zeros(50, dtype=int)}
+        ]
+
+        # Check concatinates frame QC arrays
+        frameQC, frameQC_names, bad_frames = self.task._consolidate_exptQC(exptQC)
+        expected_frames = np.r_[exptQC[0]['frameQC_frames'], exptQC[1]['frameQC_frames']]
+        np.testing.assert_array_equal(expected_frames, frameQC)
+        expected = [3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
+        np.testing.assert_array_equal(expected, bad_frames)
+        self.assertCountEqual(['qc_values', 'qc_labels'], frameQC_names.columns)
+        self.assertCountEqual(range(4), frameQC_names['qc_values'])
+        expected = ['ok', 'PMT off', 'galvos fault', 'high signal']
+        self.assertCountEqual(expected, frameQC_names['qc_labels'])
+
+        # Check with single str instead of array
+        exptQC[1]['frameQC_names'] = 'ok'
+        frameQC, frameQC_names, bad_frames = self.task._consolidate_exptQC(exptQC)
+        self.assertCountEqual(expected, frameQC_names['qc_labels'])
+        np.testing.assert_array_equal(expected_frames, frameQC)
+        # Check with inconsistent enumerations
+        exptQC[0]['frameQC_names'] = expected
+        exptQC[1]['frameQC_names'] = [*expected[-2:], *expected[:-2]]
+        self.assertRaises(IOError, self.task._consolidate_exptQC, exptQC)
+
     def test_setup_uncompressed(self):
         """Test set up behaviour when raw tifs present."""
         # Test signature when clobber = True

requirements.txt

@@ -24,7 +24,7 @@ tqdm>=4.32.1
 # ibl libraries
 iblatlas>=0.5.3
 ibl-neuropixel>=1.0.1
-iblutil>=1.11.0
+iblutil>=1.13.0
 mtscomp>=1.0.1
 ONE-api~=2.9.rc0
 phylib>=2.6.0