diff --git a/bidscoin/bids.py b/bidscoin/bids.py index 653b7cf5..20fd6618 100644 --- a/bidscoin/bids.py +++ b/bidscoin/bids.py @@ -1722,15 +1722,17 @@ def get_matching_run(datasource: DataSource, bidsmap: dict, runtime=False) -> Tu return run_, False -def get_derivatives(datatype: str) -> list: +def get_derivatives(datatype: str, exceptions: tuple=()) -> list: """ Retrieves a list of suffixes that are stored in the derivatives folder (e.g. the qMRI maps). TODO: Replace with a more systematic/documented method """ if datatype == 'anat': - return [suffix for suffix in datatyperules[datatype]['parametric']['suffixes'] if suffix not in ('UNIT1',)] # The qMRI data (maps) + return [suffix for suffix in datatyperules[datatype]['parametric']['suffixes'] + if suffix not in exceptions + ('UNIT1',)] # The qMRI data (maps) elif datatype == 'fmap': - return [suffix for typegroup in datatyperules[datatype] for suffix in datatyperules[datatype][typegroup]['suffixes'] if typegroup not in ('fieldmaps','pepolar')] # The non-standard fmaps (file collections) + return [suffix for typegroup in datatyperules[datatype] for suffix in datatyperules[datatype][typegroup]['suffixes'] + if suffix not in exceptions and typegroup not in ('fieldmaps','pepolar')] # The non-standard fmaps (file collections) else: return [] diff --git a/bidscoin/bidseditor.py b/bidscoin/bidseditor.py index dff54741..13fb7dc6 100755 --- a/bidscoin/bidseditor.py +++ b/bidscoin/bidseditor.py @@ -46,15 +46,13 @@ # self.ignoredatatypes : HELP_URL_DEFAULT } -TOOLTIP_BIDSCOIN = """BIDScoin -version: Used to check for version conflicts -bidsignore: List of data types that are added to the .bidsignore file, - e.g. extra_data/;myfile.txt;yourfile.csv -subprefix: The subject prefix used in the source data folders (e.g. "Pt" is the subprefix if subject folders are named "Pt018", "Pt019", ...) -sesprefix: The session prefix used in the source data folders (e.g. "M_" is the subprefix if session folders are named "M_pre", "M_post", ...) -unknowntypes: Datatypes that are not part of BIDS but that are converted to a BIDS-like entries in the BIDS folder -ignoretypes: Datatypes that are excluded / not converted -unzip: Wildcard pattern to select tarball/zip-files in the sourcefolders that need to be unzipped (in a tempdir), e.g. '*.tar.gz'""" +TOOLTIP_BIDSCOIN = f"""BIDScoin +version: Used to check for version conflicts +bidsignore: List of data types that are added to the .bidsignore file, + e.g. extra_data/;myfile.txt;yourfile.csv +subprefix: The subject prefix used in the source data folders (e.g. "Pt" is the subprefix if subject folders are named "Pt018", "Pt019", ...) +sesprefix: The session prefix used in the source data folders (e.g. "M_" is the subprefix if session folders are named "M_pre", "M_post", ...) +For more information see: {MAIN_HELP_URL}/options.html""" TOOLTIP_DCM2NIIX = """dcm2niix2bids command: Command to run dcm2niix from the terminal, such as: @@ -467,7 +465,8 @@ def update_subses_samples(self, output_bidsmap, dataformat): subid, sesid = run['datasource'].subid_sesid(subid, sesid or '') bidsname = bids.get_bidsname(subid, sesid, run, not bids.check_ignore(datatype,self.bidsignore) and datatype not in self.ignoredatatypes) ignore = bids.check_ignore(datatype, self.bidsignore) or bids.check_ignore(bidsname+'.json', self.bidsignore, 'file') - if run['datasource'].dynamicvalue(run['bids']['suffix'], True, True) in bids.get_derivatives(datatype): + exceptions = self.output_bidsmap['Options']['bidscoin'].get('notderivative',()) + if run['datasource'].dynamicvalue(run['bids']['suffix'], True, True) in bids.get_derivatives(datatype, exceptions): session = self.bidsfolder/'derivatives'/'[manufacturer]'/subid/sesid else: session = self.bidsfolder/subid/sesid diff --git a/bidscoin/heuristics/bidsmap_dccn.yaml b/bidscoin/heuristics/bidsmap_dccn.yaml index ddcdfcdd..e4fac427 100644 --- a/bidscoin/heuristics/bidsmap_dccn.yaml +++ b/bidscoin/heuristics/bidsmap_dccn.yaml @@ -29,6 +29,7 @@ Options: bidsignore: [mrs/, extra_data/, sub-*_ct.*] # List of entries that are added to the .bidsignore file (for more info, see BIDS specifications), e.g. [extra_data/, pet/, myfile.txt, yourfile.csv] unknowntypes: [mrs, extra_data] # A list of datatypes that are converted to BIDS-like datatype folders ignoretypes: [exclude] # A list of datatypes that are excluded / not converted to BIDS + notderivative: [] # A list of suffixes that should not be considered as a derivative datatype unzip: # Wildcard pattern to select tarball/zip-files in the source folders that need to be unzipped (in a tempdir) to expose the data, e.g. '*.tar.gz' plugins: # List of plugins with plugin-specific key-value pairs (that can be used by the plugin) dcm2niix2bids: # See dcm2niix -h and https://www.nitrc.org/plugins/mwiki/index.php/dcm2nii:MainPage#General_Usage for more info diff --git a/bidscoin/heuristics/bidsmap_sst.yaml b/bidscoin/heuristics/bidsmap_sst.yaml index d6b831e5..16efe0a9 100644 --- a/bidscoin/heuristics/bidsmap_sst.yaml +++ b/bidscoin/heuristics/bidsmap_sst.yaml @@ -28,6 +28,7 @@ Options: bidsignore: [mrs/, extra_data/, sub-*_ct.*] # List of entries that are added to the .bidsignore file (for more info, see BIDS specifications), e.g. [extra_data/, pet/, myfile.txt, yourfile.csv] unknowntypes: [mrs, extra_data] # A list of datatypes that are converted to BIDS-like datatype folders ignoretypes: [exclude] # A list of datatypes that are excluded / not converted to BIDS + notderivative: [] # A list of suffixes that should not be considered as a derivative datatype unzip: # Wildcard pattern to select tarball/zip-files in the source folders that need to be unzipped (in a tempdir) to expose the data, e.g. '*.tar.gz' plugins: # List of plugins with plugin-specific key-value pairs (that can be used by the plugin) dcm2niix2bids: # See dcm2niix -h and https://www.nitrc.org/plugins/mwiki/index.php/dcm2nii:MainPage#General_Usage for more info diff --git a/bidscoin/heuristics/schema.json b/bidscoin/heuristics/schema.json index d9daa603..36649dde 100644 --- a/bidscoin/heuristics/schema.json +++ b/bidscoin/heuristics/schema.json @@ -22,6 +22,7 @@ "sesprefix": { "type": ["string", "null"] }, "unknowntypes": { "type": ["array"] }, "ignoretypes": { "type": ["array"] }, + "notderivative": { "type": ["array"] }, "unzip": { "type": ["string", "null"] } }, "additionalProperties": false diff --git a/bidscoin/plugins/dcm2niix2bids.py b/bidscoin/plugins/dcm2niix2bids.py index 445fe438..729e1039 100644 --- a/bidscoin/plugins/dcm2niix2bids.py +++ b/bidscoin/plugins/dcm2niix2bids.py @@ -122,7 +122,7 @@ def bidsmapper_plugin(session: Path, bidsmap_new: dict, bidsmap_old: dict, templ return # Collect the different DICOM/PAR source files for all runs in the session - sourcefiles = [] + sourcefiles: List[Path] = [] if dataformat == 'DICOM': for sourcedir in lsdirs(session, '**/*'): for n in range(1): # Option: Use range(2) to scan two files and catch e.g. magnitude1/2 fieldmap files that are stored in one Series folder (but bidscoiner sees only the first file anyhow and it makes bidsmapper 2x slower :-() @@ -208,8 +208,8 @@ def bidscoiner_plugin(session: Path, bidsmap: dict, bidsses: Path) -> Union[None return # Make a list of all the data sources / runs + sources: List[Path] = [] manufacturer = 'UNKNOWN' - sources = [] if dataformat == 'DICOM': sources = lsdirs(session, '**/*') manufacturer = datasource.attributes('Manufacturer') @@ -258,8 +258,9 @@ def bidscoiner_plugin(session: Path, bidsmap: dict, bidsses: Path) -> Union[None matched_runs.append(run) # Create the BIDS session/datatype output folder - suffix = datasource.dynamicvalue(run['bids']['suffix'], True, True) - if suffix in bids.get_derivatives(datasource.datatype): + suffix = datasource.dynamicvalue(run['bids']['suffix'], True, True) + exceptions = bidsmap['Options']['bidscoin'].get('notderivative', ()) + if suffix in bids.get_derivatives(datasource.datatype, exceptions): outfolder = bidsfolder/'derivatives'/manufacturer.replace(' ','')/subid/sesid/datasource.datatype else: outfolder = bidsses/datasource.datatype @@ -472,7 +473,7 @@ def bidscoiner_plugin(session: Path, bidsmap: dict, bidsses: Path) -> Union[None outputfile = [file for file in jsonfile.parent.glob(jsonfile.stem + '.*') if file.suffix in ('.nii','.gz')] # Find the corresponding NIfTI/tsv.gz file (there should be only one, let's not make assumptions about the .gz extension) if not outputfile: LOGGER.exception(f"No data-file found with {jsonfile} when updating {scans_tsv}") - elif not bidsignore and not suffix in bids.get_derivatives(datasource.datatype): + elif not bidsignore and not suffix in bids.get_derivatives(datasource.datatype, exceptions): acq_time = '' if dataformat == 'DICOM': acq_time = f"{datasource.attributes('AcquisitionDate')}T{datasource.attributes('AcquisitionTime')}" diff --git a/bidscoin/plugins/spec2nii2bids.py b/bidscoin/plugins/spec2nii2bids.py index 626ea68d..37f0ef19 100644 --- a/bidscoin/plugins/spec2nii2bids.py +++ b/bidscoin/plugins/spec2nii2bids.py @@ -267,8 +267,9 @@ def bidscoiner_plugin(session: Path, bidsmap: dict, bidsses: Path) -> Union[None json.dump(metadata, json_fid, indent=4) # Parse the acquisition time from the source header or else from the json file (NB: assuming the source file represents the first acquisition) - suffix = datasource.dynamicvalue(run['bids']['suffix'], True, True) - if not bidsignore and not suffix in bids.get_derivatives(datasource.datatype): + suffix = datasource.dynamicvalue(run['bids']['suffix'], True, True) + exceptions = bidsmap['Options']['bidscoin'].get('notderivative', ()) + if not bidsignore and not suffix in bids.get_derivatives(datasource.datatype, exceptions): acq_time = '' if dataformat == 'SPAR': acq_time = datasource.attributes('scan_date') diff --git a/docs/options.rst b/docs/options.rst index c3f74442..28bb4e31 100644 --- a/docs/options.rst +++ b/docs/options.rst @@ -23,7 +23,8 @@ These setting can be used by all the BIDScoin tools: - ``datatypes``: Datatypes that are converted to BIDS. This can be useful for ignoring / excluding specific datatypes (without changing their mappings) - ``unknowntypes``: Datatypes that are not part of BIDS but that are converted to a BIDS-like entries in the BIDS folder - ``ignoretypes``: Datatypes that are excluded / not converted""" -- ``zip``: Wildcard pattern to select tarball/zip-files in the source folders that need to be unzipped (in a tempdir) to expose the data. Use for instance '\*.tar.gz' if your source data looks like ``sub-01\01_MPRAGE\dcmfiles.tar.gz``, etc +- ``notderivative``: A list of suffixes that should not be considered as a derivative datatype (i.e. use this to make exceptions for what goes into ``bids/derivatives``) +- ``unzip``: Wildcard pattern to select tarball/zip-files in the source folders that need to be unzipped (in a tempdir) to expose the data. Use for instance '\*.tar.gz' if your source data looks like ``sub-01\01_MPRAGE\dcmfiles.tar.gz``, etc The core working of BIDScoin and its plugins can be tested by clicking the corresponding [Test] button and inspection of the terminal output.