From 616e479542b6c6e7e788406468f212aa36cfb745 Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Wed, 23 Oct 2019 07:33:04 -0400 Subject: [PATCH 01/12] Add SBRef and functional phase support to ReproIn. --- heudiconv/heuristics/reproin.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/heudiconv/heuristics/reproin.py b/heudiconv/heuristics/reproin.py index e26356eb..a02a2b80 100644 --- a/heudiconv/heuristics/reproin.py +++ b/heudiconv/heuristics/reproin.py @@ -126,7 +126,7 @@ import logging lgr = logging.getLogger('heudiconv') -# Terminology to hamornise and use to name variables etc +# Terminology to harmonise and use to name variables etc # experiment # subject # [session] @@ -262,7 +262,7 @@ # fix per accession yet # '23763823d2b9b4b09dafcadc8e8edf21': # [ - # ('anat-T1w_acq-MPRAGE', 'anat-T1w_acq-MPRAGE_run-06'), + # ('anat-T1w_acq-MPRAGE', 'anat-T1w_acq-MPRAGE_run-06'), # ('anat_T2w', 'anat_T2w_run-06'), # ('fmap_acq-3mm', 'fmap_acq-3mm_run-06'), # ], @@ -440,8 +440,8 @@ def infotodict(seqinfo): allowed template fields - follow python string module: - item: index within category - subject: participant id + item: index within category + subject: participant id seqitem: run number during scanning subindex: sub index within group session: scan index for longitudinal acq @@ -483,7 +483,6 @@ def infotodict(seqinfo): # 3 - Image IOD specific specialization (optional) dcm_image_iod_spec = s.image_type[2] image_type_seqtype = { - 'P': 'fmap', # phase 'FMRI': 'func', 'MPR': 'anat', # 'M': 'func', "magnitude" -- can be for scout, anat, bold, fmap @@ -541,18 +540,24 @@ def infotodict(seqinfo): # analyze s.protocol_name (series_id is based on it) for full name mapping etc if seqtype == 'func' and not seqtype_label: - if '_pace_' in series_spec: + if s.series_description.endswith('_SBRef'): + seqtype_label = 'sbref' + elif '_pace_' in series_spec: seqtype_label = 'pace' # or should it be part of seq- - else: + elif 'P' in s.image_type: + seqtype_label = 'phase' + elif 'M' in s.image_type: # assume bold by default seqtype_label = 'bold' + else: + seqtype_label = 'UNK' if seqtype == 'fmap' and not seqtype_label: if not dcm_image_iod_spec: raise ValueError("Do not know image data type yet to make decision") seqtype_label = { # might want explicit {file_index} ? - # _epi for pipolar fieldmaps, see + # _epi for pepolar fieldmaps, see # https://bids-specification.readthedocs.io/en/stable/04-modality-specific-files/01-magnetic-resonance-imaging-data.html#case-4-multiple-phase-encoded-directions-pepolar 'M': 'epi' if 'dir' in series_info else 'magnitude', 'P': 'phasediff', From 95e35632cfabdf8046e67eb8ad8928cfc3165514 Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Wed, 23 Oct 2019 07:55:41 -0400 Subject: [PATCH 02/12] Default func to BOLD. --- heudiconv/heuristics/reproin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/heudiconv/heuristics/reproin.py b/heudiconv/heuristics/reproin.py index a02a2b80..749b3c72 100644 --- a/heudiconv/heuristics/reproin.py +++ b/heudiconv/heuristics/reproin.py @@ -547,10 +547,10 @@ def infotodict(seqinfo): elif 'P' in s.image_type: seqtype_label = 'phase' elif 'M' in s.image_type: - # assume bold by default seqtype_label = 'bold' else: - seqtype_label = 'UNK' + # assume bold by default + seqtype_label = 'bold' if seqtype == 'fmap' and not seqtype_label: if not dcm_image_iod_spec: From a38583ffba09e4eb373450b0e42a851be54b9a06 Mon Sep 17 00:00:00 2001 From: Taylor Salo Date: Tue, 3 Dec 2019 07:33:20 -0500 Subject: [PATCH 03/12] Revert typo fixes. --- heudiconv/heuristics/reproin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/heudiconv/heuristics/reproin.py b/heudiconv/heuristics/reproin.py index 749b3c72..581012e4 100644 --- a/heudiconv/heuristics/reproin.py +++ b/heudiconv/heuristics/reproin.py @@ -126,7 +126,7 @@ import logging lgr = logging.getLogger('heudiconv') -# Terminology to harmonise and use to name variables etc +# Terminology to hamornise and use to name variables etc # experiment # subject # [session] @@ -557,7 +557,7 @@ def infotodict(seqinfo): raise ValueError("Do not know image data type yet to make decision") seqtype_label = { # might want explicit {file_index} ? - # _epi for pepolar fieldmaps, see + # _epi for pipolar fieldmaps, see # https://bids-specification.readthedocs.io/en/stable/04-modality-specific-files/01-magnetic-resonance-imaging-data.html#case-4-multiple-phase-encoded-directions-pepolar 'M': 'epi' if 'dir' in series_info else 'magnitude', 'P': 'phasediff', From 22644357720d947ecff38de4009bed947e1f7654 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 24 Mar 2020 22:01:49 -0400 Subject: [PATCH 04/12] BF: reproin - make _SBRef considered regardless of seqtype I think it would be a more correct behavior. Either it is func, or dwi, or likely anything else, we should get _sbref seqtype if it is present in series_description (not present in protocol name). --- heudiconv/heuristics/reproin.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/heudiconv/heuristics/reproin.py b/heudiconv/heuristics/reproin.py index af7e7dc6..c3bc0e4d 100644 --- a/heudiconv/heuristics/reproin.py +++ b/heudiconv/heuristics/reproin.py @@ -541,9 +541,7 @@ def infotodict(seqinfo): # analyze s.protocol_name (series_id is based on it) for full name mapping etc if seqtype == 'func' and not seqtype_label: - if s.series_description.endswith('_SBRef'): - seqtype_label = 'sbref' - elif '_pace_' in series_spec: + if '_pace_' in series_spec: seqtype_label = 'pace' # or should it be part of seq- elif 'P' in s.image_type: seqtype_label = 'phase' @@ -569,6 +567,11 @@ def infotodict(seqinfo): if seqtype == 'dwi' and not seqtype_label: seqtype_label = 'dwi' + # Some generic overrides, regardless what seq name says, because those + # come "complementary" + if s.series_description.endswith('_SBRef'): + seqtype_label = 'sbref' + run = series_info.get('run') if run is not None: # so we have an indicator for a run From be7ca60610be2854167ceb5e59bc13dffd61dea2 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 24 Mar 2020 22:03:25 -0400 Subject: [PATCH 05/12] RF: reproin - adjusted the comment about P and M not decided at image_type_seqtype --- heudiconv/heuristics/reproin.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/heudiconv/heuristics/reproin.py b/heudiconv/heuristics/reproin.py index c3bc0e4d..b180f7aa 100644 --- a/heudiconv/heuristics/reproin.py +++ b/heudiconv/heuristics/reproin.py @@ -484,9 +484,10 @@ def infotodict(seqinfo): # 3 - Image IOD specific specialization (optional) dcm_image_iod_spec = s.image_type[2] image_type_seqtype = { + # Note: P and M are too generic to make a decision here, could be + # for different seqtypes (bold, fmap, etc) 'FMRI': 'func', 'MPR': 'anat', - # 'M': 'func', "magnitude" -- can be for scout, anat, bold, fmap 'DIFFUSION': 'dwi', 'MIP_SAG': 'anat', # angiography 'MIP_COR': 'anat', # angiography From 6b717338bf0a17e6a711842ec36f6d04d4e0eb5f Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 24 Mar 2020 22:13:57 -0400 Subject: [PATCH 06/12] DOC: reproin - add TODO about phase1,2 etc --- heudiconv/heuristics/reproin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/heudiconv/heuristics/reproin.py b/heudiconv/heuristics/reproin.py index b180f7aa..71ba86ef 100644 --- a/heudiconv/heuristics/reproin.py +++ b/heudiconv/heuristics/reproin.py @@ -553,6 +553,7 @@ def infotodict(seqinfo): seqtype_label = 'bold' if seqtype == 'fmap' and not seqtype_label: + # TODO: support phase1 phase2 like in "Case 2: Two phase images ..." if not dcm_image_iod_spec: raise ValueError("Do not know image data type yet to make decision") seqtype_label = { From 5b7240440ba3ae1ca21319f69dc694e87901cb4e Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Tue, 24 Mar 2020 22:30:26 -0400 Subject: [PATCH 07/12] RF+ENH: reproin - place all logic for "not seqtype_label" together Also issue a warning if none was figured out - I think this should not happen. It should make code a bit easier to read --- heudiconv/heuristics/reproin.py | 69 +++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 30 deletions(-) diff --git a/heudiconv/heuristics/reproin.py b/heudiconv/heuristics/reproin.py index 71ba86ef..34016c28 100644 --- a/heudiconv/heuristics/reproin.py +++ b/heudiconv/heuristics/reproin.py @@ -540,40 +540,49 @@ def infotodict(seqinfo): # prefix = '' prefix = '' + # + # Figure out the seqtype_label (BIDS _suffix) + # + # If none was provided -- let's deduce it from the information we find: # analyze s.protocol_name (series_id is based on it) for full name mapping etc - if seqtype == 'func' and not seqtype_label: - if '_pace_' in series_spec: - seqtype_label = 'pace' # or should it be part of seq- - elif 'P' in s.image_type: - seqtype_label = 'phase' - elif 'M' in s.image_type: - seqtype_label = 'bold' - else: - # assume bold by default - seqtype_label = 'bold' - - if seqtype == 'fmap' and not seqtype_label: - # TODO: support phase1 phase2 like in "Case 2: Two phase images ..." - if not dcm_image_iod_spec: - raise ValueError("Do not know image data type yet to make decision") - seqtype_label = { - # might want explicit {file_index} ? - # _epi for pepolar fieldmaps, see - # https://bids-specification.readthedocs.io/en/stable/04-modality-specific-files/01-magnetic-resonance-imaging-data.html#case-4-multiple-phase-encoded-directions-pepolar - 'M': 'epi' if 'dir' in series_info else 'magnitude', - 'P': 'phasediff', - 'DIFFUSION': 'epi', # according to KODI those DWI are the EPIs we need - }[dcm_image_iod_spec] - - # label for dwi as well - if seqtype == 'dwi' and not seqtype_label: - seqtype_label = 'dwi' - - # Some generic overrides, regardless what seq name says, because those - # come "complementary" + if not seqtype_label: + if seqtype == 'func': + if '_pace_' in series_spec: + seqtype_label = 'pace' # or should it be part of seq- + elif 'P' in s.image_type: + seqtype_label = 'phase' + elif 'M' in s.image_type: + seqtype_label = 'bold' + else: + # assume bold by default + seqtype_label = 'bold' + elif seqtype == 'fmap': + # TODO: support phase1 phase2 like in "Case 2: Two phase images ..." + if not dcm_image_iod_spec: + raise ValueError("Do not know image data type yet to make decision") + seqtype_label = { + # might want explicit {file_index} ? + # _epi for pepolar fieldmaps, see + # https://bids-specification.readthedocs.io/en/stable/04-modality-specific-files/01-magnetic-resonance-imaging-data.html#case-4-multiple-phase-encoded-directions-pepolar + 'M': 'epi' if 'dir' in series_info else 'magnitude', + 'P': 'phasediff', + 'DIFFUSION': 'epi', # according to KODI those DWI are the EPIs we need + }[dcm_image_iod_spec] + elif seqtype == 'dwi': + # label for dwi as well + seqtype_label = 'dwi' + + # + # Even if seqtype_label was provided, for some data we might need to override, + # since they are complementary files produced along-side with original + # ones. + # if s.series_description.endswith('_SBRef'): seqtype_label = 'sbref' + if not seqtype_label: + lgr.warning("We ended up with an empty label/suffix for %r", series_spec) + run = series_info.get('run') if run is not None: # so we have an indicator for a run From 0eb5fbd9f40bad75ad5f081598dd66fe3aa7b0b3 Mon Sep 17 00:00:00 2001 From: pvelasco Date: Fri, 21 Feb 2020 12:03:10 -0500 Subject: [PATCH 08/12] Make sure we don't add _rec-magnitude for sbref when there are no phase images --- heudiconv/convert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/heudiconv/convert.py b/heudiconv/convert.py index e3390593..e19c1796 100644 --- a/heudiconv/convert.py +++ b/heudiconv/convert.py @@ -550,7 +550,7 @@ def save_converted_files(res, item_dicoms, bids_options, outtype, prefix, outnam # _sbref sequences reconstructing magnitude and phase generate # two NIfTI files IN THE SAME SERIES, so we cannot just add # the suffix, if we want to be bids compliant: - if bids_file and this_prefix_basename.endswith('_sbref'): + if bids_file and this_prefix_basename.endswith('_sbref') and len(suffixes)>len(echo_times): # Check to see if it is magnitude or phase reconstruction: if 'M' in fileinfo.get('ImageType'): mag_or_phase = 'magnitude' From 2f3a7d28afddee7e0be959543dacfcf4962e31ef Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 25 Mar 2020 12:45:08 -0400 Subject: [PATCH 09/12] ENH: issue a warning when number of suffixes isnt twice the number of echoes --- heudiconv/convert.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/heudiconv/convert.py b/heudiconv/convert.py index e19c1796..40921d76 100644 --- a/heudiconv/convert.py +++ b/heudiconv/convert.py @@ -550,7 +550,14 @@ def save_converted_files(res, item_dicoms, bids_options, outtype, prefix, outnam # _sbref sequences reconstructing magnitude and phase generate # two NIfTI files IN THE SAME SERIES, so we cannot just add # the suffix, if we want to be bids compliant: - if bids_file and this_prefix_basename.endswith('_sbref') and len(suffixes)>len(echo_times): + if bids_file and this_prefix_basename.endswith('_sbref') \ + and len(suffixes) > len(echo_times): + if len(suffixes) != len(echo_times)*2: + lgr.warning( + "Got %d suffixes for %d echo times, which isn't " + "multiple of two as if it was magnitude + phase pairs", + len(suffixes), len(echo_times) + ) # Check to see if it is magnitude or phase reconstruction: if 'M' in fileinfo.get('ImageType'): mag_or_phase = 'magnitude' From 6046dc0d80bd53a74362b6a186c10fd5d2bb29ba Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 25 Mar 2020 12:56:31 -0400 Subject: [PATCH 10/12] RF+ENH: avoid loading the same bids json twice, rename fileinfo -> bids_meta for consistency --- heudiconv/convert.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/heudiconv/convert.py b/heudiconv/convert.py index 40921d76..fcc2e52b 100644 --- a/heudiconv/convert.py +++ b/heudiconv/convert.py @@ -516,6 +516,8 @@ def save_converted_files(res, item_dicoms, bids_options, outtype, prefix, outnam bids_files = (sorted(res.outputs.bids) if len(res.outputs.bids) == len(res_files) else [None] * len(res_files)) + # preload since will be used in multiple spots + bids_metas = [load_json(b) for b in bids_files if b] ### Do we have a multi-echo series? ### # Some Siemens sequences (e.g. CMRR's MB-EPI) set the label 'TE1', @@ -529,19 +531,17 @@ def save_converted_files(res, item_dicoms, bids_options, outtype, prefix, outnam # Check for varying echo times echo_times = sorted(list(set( - load_json(b).get('EchoTime', None) - for b in bids_files + b.get('EchoTime', None) + for b in bids_metas if b ))) is_multiecho = len(echo_times) > 1 ### Loop through the bids_files, set the output name and save files - for fl, suffix, bids_file in zip(res_files, suffixes, bids_files): + for fl, suffix, bids_file, bids_meta in zip(res_files, suffixes, bids_files, bids_metas): # TODO: monitor conversion duration - if bids_file: - fileinfo = load_json(bids_file) # set the prefix basename for this specific file (we'll modify it, # and we don't want to modify it for all the bids_files): @@ -550,7 +550,7 @@ def save_converted_files(res, item_dicoms, bids_options, outtype, prefix, outnam # _sbref sequences reconstructing magnitude and phase generate # two NIfTI files IN THE SAME SERIES, so we cannot just add # the suffix, if we want to be bids compliant: - if bids_file and this_prefix_basename.endswith('_sbref') \ + if bids_meta and this_prefix_basename.endswith('_sbref') \ and len(suffixes) > len(echo_times): if len(suffixes) != len(echo_times)*2: lgr.warning( @@ -559,9 +559,9 @@ def save_converted_files(res, item_dicoms, bids_options, outtype, prefix, outnam len(suffixes), len(echo_times) ) # Check to see if it is magnitude or phase reconstruction: - if 'M' in fileinfo.get('ImageType'): + if 'M' in bids_meta.get('ImageType'): mag_or_phase = 'magnitude' - elif 'P' in fileinfo.get('ImageType'): + elif 'P' in bids_meta.get('ImageType'): mag_or_phase = 'phase' else: mag_or_phase = suffix @@ -590,12 +590,12 @@ def save_converted_files(res, item_dicoms, bids_options, outtype, prefix, outnam # (Note: it can be _sbref and multiecho, so don't use "elif"): # For multi-echo sequences, we have to specify the echo number in # the file name: - if bids_file and is_multiecho: + if bids_meta and is_multiecho: # Get the EchoNumber from json file info. If not present, use EchoTime - if 'EchoNumber' in fileinfo.keys(): - echo_number = fileinfo['EchoNumber'] + if 'EchoNumber' in bids_meta: + echo_number = bids_meta['EchoNumber'] else: - echo_number = echo_times.index(fileinfo['EchoTime']) + 1 + echo_number = echo_times.index(bids_meta['EchoTime']) + 1 supported_multiecho = ['_bold', '_phase', '_epi', '_sbref', '_T1w', '_PDT2'] # Now, decide where to insert it. From 2c9e5ec40d52b4a31e4fa157f449993084dd63b7 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 25 Mar 2020 13:04:47 -0400 Subject: [PATCH 11/12] BF: boost minimal datalad version to 0.12.3 since older ones might not be compatible with recent git annex --- heudiconv/external/dlad.py | 4 ++-- heudiconv/info.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/heudiconv/external/dlad.py b/heudiconv/external/dlad.py index 53317338..68a0e758 100644 --- a/heudiconv/external/dlad.py +++ b/heudiconv/external/dlad.py @@ -10,7 +10,7 @@ lgr = logging.getLogger(__name__) -MIN_VERSION = '0.12.2' +MIN_VERSION = '0.12.4' def prepare_datalad(studydir, outdir, sid, session, seqinfo, dicoms, bids): @@ -177,4 +177,4 @@ def mark_sensitive(ds, path_glob): init=dict([('distribution-restrictions', 'sensitive')]), recursive=True) if inspect.isgenerator(res): - res = list(res) \ No newline at end of file + res = list(res) diff --git a/heudiconv/info.py b/heudiconv/info.py index 4f5fc596..ccf36817 100644 --- a/heudiconv/info.py +++ b/heudiconv/info.py @@ -42,7 +42,7 @@ EXTRA_REQUIRES = { 'tests': TESTS_REQUIRES, 'extras': [], # Requires patched version ATM ['dcmstack'], - 'datalad': ['datalad >=0.12.2'] + 'datalad': ['datalad >=0.12.3'] } # Flatten the lists From dce8e23c25a80be04aa2903e81572d66904d3a32 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 25 Mar 2020 17:31:15 -0400 Subject: [PATCH 12/12] ENH: reproin - warning on empty suffix should not be issued if unparsed bids portion has it --- heudiconv/heuristics/reproin.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/heudiconv/heuristics/reproin.py b/heudiconv/heuristics/reproin.py index 4ba74ef9..c5a76914 100644 --- a/heudiconv/heuristics/reproin.py +++ b/heudiconv/heuristics/reproin.py @@ -611,7 +611,14 @@ def infotodict(seqinfo): seqtype_label = 'sbref' if not seqtype_label: - lgr.warning("We ended up with an empty label/suffix for %r", series_spec) + # Might be provided by the bids ending within series_spec, we would + # just want to check if that the last element is not _key-value pair + bids_ending = series_info.get('bids', None) + if not bids_ending \ + or "-" in bids_ending.split('_')[-1]: + lgr.warning( + "We ended up with an empty label/suffix for %r", + series_spec) run = series_info.get('run') if run is not None: