Bases: ABC
Bases: Task
Bases: Task
Bases: Task
Bases: Task
Bases: Task
Bases: Task
Bases: Task
Bases: Task
self.download_spike_sorting_object(obj='channels', missing='ignore', **kwargs)
channels = self._load_object(self.files['channels'], wildcards=self.one.wildcards)
if 'electrodeSites' in self.files: # if common dict keys, electrodeSites prevails
- channels = channels | self._load_object(self.files['electrodeSites'], wildcards=self.one.wildcards)
+ esites = channels | self._load_object(self.files['electrodeSites'], wildcards=self.one.wildcards)
+ if alfio.check_dimensions(esites) != 0:
+ esites = self._load_object(self.files['electrodeSites'], wildcards=self.one.wildcards)
+ esites['rawInd'] = np.arange(esites[list(esites.keys())[0]].shape[0])
if 'brainLocationIds_ccf_2017' not in channels:
_logger.debug(f"loading channels from alyx for {self.files['channels']}")
_channels, self.histology = _load_channel_locations_traj(
diff --git a/_modules/ibllib/oneibl/data_handlers.html b/_modules/ibllib/oneibl/data_handlers.html
index 89b320f5..47d8f5a8 100644
--- a/_modules/ibllib/oneibl/data_handlers.html
+++ b/_modules/ibllib/oneibl/data_handlers.html
@@ -722,7 +722,7 @@ Source code for ibllib.oneibl.data_handlers
one = one or self.one
session_datasets = one.list_datasets(one.path2eid(self.session_path), details=True)
dfs = [file.filter(session_datasets)[1] for file in self.signature['input_files']]
- return one._cache.datasets.iloc[0:0] if len(dfs) == 0 else pd.concat(dfs)
+ return one._cache.datasets.iloc[0:0] if len(dfs) == 0 else pd.concat(dfs).drop_duplicates()
procedures = list({*experiment_description_file.get('procedures', []), *(procedures or [])})
collections = session_params.get_task_collection(experiment_description_file)
+ # Read narrative.txt
+ if (narrative_file := ses_path.joinpath('narrative.txt')).exists():
+ with narrative_file.open('r') as f:
+ narrative = f.read()
+ else:
+ narrative = ''
+
# query Alyx endpoints for subject, error if not found
subject = self.assert_exists(subject, 'subjects')
@@ -421,6 +428,7 @@ Source code for ibllib.oneibl.registration
'end_time': self.ensure_ISO8601(end_time) if end_time else None,
'n_correct_trials': n_correct_trials,
'n_trials': n_trials,
+ 'narrative': narrative,
'json': json_field
}
session = self.one.alyx.rest('sessions', 'create', data=ses_)
@@ -436,6 +444,8 @@ Source code for ibllib.oneibl.registration
else: # if session exists update a few key fields
data = {'procedures': procedures, 'projects': projects,
'n_correct_trials': n_correct_trials, 'n_trials': n_trials}
+ if len(narrative) > 0:
+ data['narrative'] = narrative
if task_protocols:
data['task_protocol'] = '/'.join(task_protocols)
if end_time:
diff --git a/_modules/ibllib/pipes/local_server.html b/_modules/ibllib/pipes/local_server.html
index 91362f95..7a2bec67 100644
--- a/_modules/ibllib/pipes/local_server.html
+++ b/_modules/ibllib/pipes/local_server.html
@@ -128,6 +128,8 @@ Source code for ibllib.pipes.local_server
from one.api import ONE
from one.webclient import AlyxClient
from one.remote.globus import get_lab_from_endpoint_id, get_local_endpoint_id
+from one.alf.spec import is_session_path
+from one.alf.path import session_path_parts
from ibllib import __version__ as ibllib_version
from ibllib.pipes import tasks
@@ -220,11 +222,15 @@ Source code for ibllib.pipes.local_server
if not one:
one = ONE(cache_rest=None)
rc = IBLRegistrationClient(one=one)
- flag_files = list(Path(root_path).glob('**/raw_session.flag'))
+ flag_files = Path(root_path).glob('*/????-??-??/*/raw_session.flag')
+ flag_files = filter(lambda x: is_session_path(x.parent), flag_files)
pipes = []
all_datasets = []
for flag_file in flag_files:
session_path = flag_file.parent
+ if session_path_parts(session_path)[1] in ('test', 'test_subject'):
+ _logger.debug('skipping test session %s', session_path)
+ continue
_logger.info(f'creating session for {session_path}')
if dry:
continue
diff --git a/_modules/ibllib/pipes/mesoscope_tasks.html b/_modules/ibllib/pipes/mesoscope_tasks.html
index d368fdcb..70284bbb 100644
--- a/_modules/ibllib/pipes/mesoscope_tasks.html
+++ b/_modules/ibllib/pipes/mesoscope_tasks.html
@@ -942,8 +942,8 @@ Source code for ibllib.pipes.mesoscope_tasks
for m, collection in zip(all_meta, raw_image_collections):
badframes_path = self.session_path.joinpath(collection, 'badframes.mat')
if badframes_path.exists():
- raw_mat = loadmat(badframes_path, squeeze_me=True, simplify_cells=True)['badframes']
- badframes = np.r_[badframes, raw_mat + total_frames]
+ raw_mat = loadmat(badframes_path, squeeze_me=True, simplify_cells=True)
+ badframes = np.r_[badframes, raw_mat['badframes'].astype('uint32') + total_frames]
total_frames += m['nFrames']
if len(badframes) > 0 and use_badframes is True:
# The badframes array should always be a subset of the frameQC array
diff --git a/_modules/ibllib/pipes/tasks.html b/_modules/ibllib/pipes/tasks.html
index bb5ce5aa..f23dc326 100644
--- a/_modules/ibllib/pipes/tasks.html
+++ b/_modules/ibllib/pipes/tasks.html
@@ -225,9 +225,10 @@ Source code for ibllib.pipes.tasks
force = False # whether to re-download missing input files on local server if not present
job_size = 'small' # either 'small' or 'large', defines whether task should be run as part of the large or small job services
env = None # the environment name within which to run the task (NB: the env is not activated automatically!)
+ on_error = 'continue' # whether to raise an exception on error ('raise') or report the error and continue ('continue')
def __init__(self, session_path, parents=None, taskid=None, one=None,
- machine=None, clobber=True, location='server', scratch_folder=None, **kwargs):
+ machine=None, clobber=True, location='server', scratch_folder=None, on_error='continue', **kwargs):
"""
Base task class
:param session_path: session path
@@ -242,6 +243,7 @@ Source code for ibllib.pipes.tasks
:param scratch_folder: optional: Path where to write intermediate temporary data
:param args: running arguments
"""
+ self.on_error = on_error
self.taskid = taskid
self.one = one
self.session_path = session_path
@@ -381,10 +383,12 @@ Source code for ibllib.pipes.tasks
self.outputs = outputs if not self.outputs else self.outputs # ensure None if no inputs registered
else:
self.outputs.extend(ensure_list(outputs)) # Add output files to list of inputs to register
- except Exception:
+ except Exception as e:
_logger.error(traceback.format_exc())
_logger.info(f'Job {self.__class__} errored')
self.status = -1
+ if self.on_error == 'raise':
+ raise e
self.time_elapsed_secs = time.time() - start_time
# log the outputs
diff --git a/genindex.html b/genindex.html
index 80ffbfc4..8e27b0cf 100644
--- a/genindex.html
+++ b/genindex.html
@@ -3748,6 +3748,8 @@ N
O