From 48051258a7a465d81b6324d08a4148fb83d921d2 Mon Sep 17 00:00:00 2001 From: Dominika Zemanovicova Date: Wed, 3 Jan 2024 19:53:15 +0100 Subject: [PATCH 1/6] Add only existing jsonfiles to jsonfiles and make it set (fixes adding nonexistent jsonfiles) --- bidscoin/plugins/dcm2niix2bids.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/bidscoin/plugins/dcm2niix2bids.py b/bidscoin/plugins/dcm2niix2bids.py index 8fe30131..098fcb9d 100644 --- a/bidscoin/plugins/dcm2niix2bids.py +++ b/bidscoin/plugins/dcm2niix2bids.py @@ -266,7 +266,7 @@ def bidscoiner_plugin(session: Path, bidsmap: dict, bidsses: Path) -> Union[None bidsname = bids.get_bidsname(subid, sesid, run, not bidsignore, runtime=True) bidsignore = bidsignore or bids.check_ignore(bidsname+'.json', bidsmap['Options']['bidscoin']['bidsignore'], 'file') bidsname = bids.increment_runindex(outfolder, bidsname, run, scans_table) - jsonfiles = [(outfolder/bidsname).with_suffix('.json')] # List -> Collect the associated json-files (for updating them later) -- possibly > 1 + jsonfiles = set() # Set -> Collect the associated json-files (for updating them later) -- possibly > 1 # Check if the bidsname is valid bidstest = (Path('/')/subid/sesid/datasource.datatype/bidsname).with_suffix('.json').as_posix() @@ -308,6 +308,8 @@ def bidscoiner_plugin(session: Path, bidsmap: dict, bidsses: Path) -> Union[None if bcoin.run_command(command): if not list(outfolder.glob(f"{bidsname}.*nii*")): continue + jsonfiles.update(outfolder.glob(f"{bidsname}.json")) # add existing created json files: bidsname.json + # Handle the ABCD GE pepolar sequence extrafile = list(outfolder.glob(f"{bidsname}a.nii*")) if extrafile: @@ -320,7 +322,7 @@ def bidscoiner_plugin(session: Path, bidsmap: dict, bidsses: Path) -> Union[None LOGGER.verbose(f"Renaming GE reversed polarity image: {extrafile[0]} -> {invfile}") extrafile[0].replace(invfile) extrafile[0].with_suffix('').with_suffix('.json').replace(invfile.with_suffix('').with_suffix('.json')) - jsonfiles.append(invfile.with_suffix('').with_suffix('.json')) + jsonfiles.add(invfile.with_suffix('').with_suffix('.json')) else: LOGGER.warning(f"Unexpected variants of {outfolder/bidsname}* were produced by dcm2niix. Possibly this can be remedied by using the dcm2niix -i option (to ignore derived, localizer and 2D images) or by clearing the BIDS folder before running bidscoiner") @@ -335,8 +337,7 @@ def bidscoiner_plugin(session: Path, bidsmap: dict, bidsses: Path) -> Union[None # Rename all files that got additional postfixes from dcm2niix. See: https://github.com/rordenlab/dcm2niix/blob/master/FILENAMING.md dcm2niixpostfixes = ('_c', '_i', '_Eq', '_real', '_imaginary', '_MoCo', '_t', '_Tilt', '_e', '_ph', '_ADC', '_fieldmaphz') #_c%d, _e%d and _ph (and any combination of these in that order) are for multi-coil data, multi-echo data and phase data dcm2niixfiles = sorted(set([dcm2niixfile for dcm2niixpostfix in dcm2niixpostfixes for dcm2niixfile in outfolder.glob(f"{bidsname}*{dcm2niixpostfix}*.nii*")])) - if not jsonfiles[0].is_file() and dcm2niixfiles: # Possibly renamed by dcm2niix, e.g. with multi-echo data (but not always for the first echo) - jsonfiles.pop(0) + for dcm2niixfile in dcm2niixfiles: # Strip each dcm2niix postfix and assign it to bids entities in a newly constructed bidsname @@ -431,12 +432,12 @@ def bidscoiner_plugin(session: Path, bidsmap: dict, bidsses: Path) -> Union[None if oldjsonfile in jsonfiles: jsonfiles.remove(oldjsonfile) if newjsonfile not in jsonfiles: - jsonfiles.append(newjsonfile) + jsonfiles.add(newjsonfile) for oldfile in outfolder.glob(dcm2niixfile.with_suffix('').stem + '.*'): oldfile.replace(newjsonfile.with_suffix(''.join(oldfile.suffixes))) # Loop over all the newly produced json sidecar-files and adapt the data (NB: assumes every NIfTI-file comes with a json-file) - for jsonfile in sorted(set(jsonfiles)): + for jsonfile in sorted(jsonfiles): # Load / copy over the source meta-data metadata = bids.updatemetadata(sourcefile, jsonfile, run['meta'], options['meta'], datasource) From 40792fe1a8e0549735af2445f19c335ff6c01f3c Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 11 Jan 2024 10:52:34 -0500 Subject: [PATCH 2/6] Add github action to codespell master on push and PRs --- .github/workflows/codespell.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 .github/workflows/codespell.yml diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml new file mode 100644 index 00000000..41105888 --- /dev/null +++ b/.github/workflows/codespell.yml @@ -0,0 +1,23 @@ +# Codespell configuration is within pyproject.toml +--- +name: Codespell + +on: + push: + branches: [master] + pull_request: + branches: [master] + +permissions: + contents: read + +jobs: + codespell: + name: Check for spelling errors + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Codespell + uses: codespell-project/actions-codespell@v2 From 601b98472ec900e1d869086e23cc1fb76256b9e7 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 11 Jan 2024 10:52:34 -0500 Subject: [PATCH 3/6] Add rudimentary codespell config --- pyproject.toml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index c0c31449..9568cbc4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -118,3 +118,10 @@ commands = make -C docs html linkchecker docs/_build/html/index.html pyspelling -c docs/.pyspelling.yml """ + +# Ref: https://github.com/codespell-project/codespell#using-a-config-file +[tool.codespell] +skip = '.git,*.pdf,*.svg,go.sum' +check-hidden = true +ignore-regex = '\b(PULS|TE|ALS)\b' +ignore-words-list = 'compliancy,uptodate' From 4722e301894722f2015781a0342edf2bf1774748 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 11 Jan 2024 10:55:01 -0500 Subject: [PATCH 4/6] [DATALAD RUNCMD] run codespell throughout fixing typo automagically === Do not change lines below === { "chain": [], "cmd": "codespell -w || :", "exit": 0, "extra_inputs": [], "inputs": [], "outputs": [], "pwd": "." } ^^^ Do not change lines above ^^^ --- bidscoin/bcoin.py | 2 +- bidscoin/bids.py | 2 +- bidscoin/bidseditor.py | 8 ++++---- bidscoin/cli/_bidsmapper.py | 2 +- bidscoin/plugins/dcm2niix2bids.py | 2 +- bidscoin/plugins/nibabel2bids.py | 2 +- bidscoin/plugins/spec2nii2bids.py | 2 +- docs/bidsmap.rst | 2 +- docs/plugins.rst | 2 +- docs/workflow.rst | 4 ++-- 10 files changed, 14 insertions(+), 14 deletions(-) diff --git a/bidscoin/bcoin.py b/bidscoin/bcoin.py index 3ccee88d..5c0eacbf 100755 --- a/bidscoin/bcoin.py +++ b/bidscoin/bcoin.py @@ -557,7 +557,7 @@ def test_bidscoin(bidsmapfile: Union[Path,dict], options: dict=None, testplugins LOGGER.warning(f"Failed test: {plugin.stem}") if not success: - LOGGER.warning('Not all tests finishd successfully (this may be ok, but check the output above)') + LOGGER.warning('Not all tests finished successfully (this may be ok, but check the output above)') trackusage('bidscoin_error') else: LOGGER.success('All tests finished successfully :-)') diff --git a/bidscoin/bids.py b/bidscoin/bids.py index c455f458..7c13b94f 100644 --- a/bidscoin/bids.py +++ b/bidscoin/bids.py @@ -853,7 +853,7 @@ def load_bidsmap(yamlfile: Path, folder: Path=Path(), plugins:Union[tuple,list]= """ Read the mapping heuristics from the bidsmap yaml-file. If yamlfile is not fullpath, then 'folder' is first searched before the default 'heuristics'. If yamfile is empty, then first 'bidsmap.yaml' is searched for, then 'bidsmap_template'. So fullpath - has precendence over folder and bidsmap.yaml has precedence over the bidsmap_template. + has precedence over folder and bidsmap.yaml has precedence over the bidsmap_template. NB: A run['datasource'] = DataSource object is added to every run-item diff --git a/bidscoin/bidseditor.py b/bidscoin/bidseditor.py index a2f15bc1..dca961c2 100755 --- a/bidscoin/bidseditor.py +++ b/bidscoin/bidseditor.py @@ -349,7 +349,7 @@ def set_tab_bidsmap(self, dataformat): def set_tab_options(self): """Set the options tab""" - # Create the bidscoin tabel + # Create the bidscoin table bidscoin_options = self.output_bidsmap['Options']['bidscoin'] self.options_label['bidscoin'] = bidscoin_label = QLabel('BIDScoin') bidscoin_label.setToolTip(TOOLTIP_BIDSCOIN) @@ -376,7 +376,7 @@ def set_tab_options(self): layout.addWidget(bidscoin_label) layout.addWidget(bidscoin_table) - # Add the plugin tabels + # Add the plugin tables for plugin, options in self.output_bidsmap['Options']['plugins'].items(): plugin_label, plugin_table = self.plugin_table(plugin, options) layout.addWidget(plugin_label) @@ -1182,7 +1182,7 @@ def fill_table(self, table: QTableWidget, data: list): suffix_dropdown.addItems(suffixes) suffix_dropdown.setCurrentIndex(suffix_dropdown.findText(suffix)) suffix_dropdown.currentIndexChanged.connect(self.suffix_dropdown_change) - suffix_dropdown.setToolTip('The suffix that sets the different run types apart. First make sure the "Data type" dropdown-menu is set correctly before chosing the right suffix here') + suffix_dropdown.setToolTip('The suffix that sets the different run types apart. First make sure the "Data type" dropdown-menu is set correctly before choosing the right suffix here') for n, suffix in enumerate(suffixes): suffix_dropdown.setItemData(n, bids.get_suffixhelp(suffix, self.target_datatype), QtCore.Qt.ItemDataRole.ToolTipRole) table.setCellWidget(i, 1, self.spacedwidget(suffix_dropdown)) @@ -1490,7 +1490,7 @@ def accept_run(self): self.done(2) def export_run(self): - """Export the editted run to a (e.g. template) bidsmap on disk""" + """Export the edited run to a (e.g. template) bidsmap on disk""" yamlfile, _ = QFileDialog.getOpenFileName(self, 'Export run item to (template) bidsmap', str(bidsmap_template), 'YAML Files (*.yaml *.yml);;All Files (*)') diff --git a/bidscoin/cli/_bidsmapper.py b/bidscoin/cli/_bidsmapper.py index ceeb2454..2d55907d 100755 --- a/bidscoin/cli/_bidsmapper.py +++ b/bidscoin/cli/_bidsmapper.py @@ -39,7 +39,7 @@ def get_parser() -> argparse.ArgumentParser: parser.add_argument('-n','--subprefix', help="The prefix common for all the source subject-folders (e.g. 'Pt' is the subprefix if subject folders are named 'Pt018', 'Pt019', ...). Use '*' when your subject folders do not have a prefix. Default: the value of the study/template bidsmap, e.g. 'sub-'") parser.add_argument('-m','--sesprefix', help="The prefix common for all the source session-folders (e.g. 'M_' is the subprefix if session folders are named 'M_pre', 'M_post', ..). Use '*' when your session folders do not have a prefix. Default: the value of the study/template bidsmap, e.g. 'ses-'") parser.add_argument('-u','--unzip', help='Wildcard pattern to unpack tarball/zip-files in the sub/ses sourcefolder that need to be unzipped (in a tempdir) to make the data readable. Default: the value of the study/template bidsmap') - parser.add_argument('-s','--store', help='Store provenance data samples in the bidsfolder/code/provenance folder (useful for inspecting e.g. zipped or transfered datasets)', action='store_true') + parser.add_argument('-s','--store', help='Store provenance data samples in the bidsfolder/code/provenance folder (useful for inspecting e.g. zipped or transferred datasets)', action='store_true') parser.add_argument('-a','--automated', help='Save the automatically generated bidsmap to disk and without interactively tweaking it with the bidseditor', action='store_true') parser.add_argument('-f','--force', help='Discard the previously saved bidsmap and logfile', action='store_true') parser.add_argument('--no-update', help="Do not update any sub/sesprefixes in or prepend the sourcefolder name to the <> expression that extracts the subject/session labels. This is normally done to make the extraction more robust, but could cause problems for certain use cases", action='store_true') diff --git a/bidscoin/plugins/dcm2niix2bids.py b/bidscoin/plugins/dcm2niix2bids.py index 8fe30131..429764d4 100644 --- a/bidscoin/plugins/dcm2niix2bids.py +++ b/bidscoin/plugins/dcm2niix2bids.py @@ -272,7 +272,7 @@ def bidscoiner_plugin(session: Path, bidsmap: dict, bidsses: Path) -> Union[None bidstest = (Path('/')/subid/sesid/datasource.datatype/bidsname).with_suffix('.json').as_posix() isbids = BIDSValidator().is_bids(bidstest) if not isbids and not bidsignore: - LOGGER.warning(f"The '{bidstest}' ouput name did not pass the bids-validator test") + LOGGER.warning(f"The '{bidstest}' output name did not pass the bids-validator test") # Check if the output file already exists (-> e.g. when a static runindex is used) if (outfolder/bidsname).with_suffix('.json').is_file(): diff --git a/bidscoin/plugins/nibabel2bids.py b/bidscoin/plugins/nibabel2bids.py index 95c7f13f..a94f4db7 100644 --- a/bidscoin/plugins/nibabel2bids.py +++ b/bidscoin/plugins/nibabel2bids.py @@ -220,7 +220,7 @@ def bidscoiner_plugin(session: Path, bidsmap: dict, bidsses: Path) -> None: bidstest = (Path('/')/subid/sesid/datasource.datatype/bidsname).with_suffix('.json').as_posix() isbids = BIDSValidator().is_bids(bidstest) if not isbids and not bidsignore: - LOGGER.warning(f"The '{bidstest}' ouput name did not pass the bids-validator test") + LOGGER.warning(f"The '{bidstest}' output name did not pass the bids-validator test") # Check if file already exists (-> e.g. when a static runindex is used) if bidsfile.is_file(): diff --git a/bidscoin/plugins/spec2nii2bids.py b/bidscoin/plugins/spec2nii2bids.py index fb0976b2..4b51c5e6 100644 --- a/bidscoin/plugins/spec2nii2bids.py +++ b/bidscoin/plugins/spec2nii2bids.py @@ -230,7 +230,7 @@ def bidscoiner_plugin(session: Path, bidsmap: dict, bidsses: Path) -> Union[None bidstest = (Path('/')/subid/sesid/datasource.datatype/bidsname).with_suffix('.json').as_posix() isbids = BIDSValidator().is_bids(bidstest) if not isbids and not bidsignore: - LOGGER.warning(f"The '{bidstest}' ouput name did not pass the bids-validator test") + LOGGER.warning(f"The '{bidstest}' output name did not pass the bids-validator test") # Check if file already exists (-> e.g. when a static runindex is used) if sidecar.is_file(): diff --git a/docs/bidsmap.rst b/docs/bidsmap.rst index 9cffd482..c31b6acf 100644 --- a/docs/bidsmap.rst +++ b/docs/bidsmap.rst @@ -110,7 +110,7 @@ Editing the template bidsmap filename: # File name, e.g. ".*fmap.*" or ".*(fmap|field.?map|B0.?map).*" filesize: # File size, e.g. "2[4-6]\d MB" for matching files between 240-269 MB nrfiles: # Number of files in the folder that match the above criteria, e.g. "5/d/d" for matching a number between 500-599 - attributes: &anat_dicomattr # An empty / non-matching "reference" dictionary that can be derefenced in other run-items of this data type + attributes: &anat_dicomattr # An empty / non-matching "reference" dictionary that can be dereferenced in other run-items of this data type Modality: ProtocolName: SeriesDescription: diff --git a/docs/plugins.rst b/docs/plugins.rst index 8003e950..8bbbcb1c 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -189,7 +189,7 @@ As can be seen in the API code snippet below (but also see the default plugins f def bidsmapper_plugin(session: Path, bidsmap_new: dict, bidsmap_old: dict, template: dict, store: dict) -> None: """ All the logic to map the Philips PAR/REC fields onto bids labels go into this plugin function. The function is - expecte to update / append new runs to the bidsmap_new data structure. The bidsmap options for this plugin can + expected to update / append new runs to the bidsmap_new data structure. The bidsmap options for this plugin can be found in: bidsmap_new/old['Options']['plugins']['README'] diff --git a/docs/workflow.rst b/docs/workflow.rst index 9f02a2e9..a54240ff 100644 --- a/docs/workflow.rst +++ b/docs/workflow.rst @@ -71,7 +71,7 @@ Step 1a: Running the bidsmapper need to be unzipped (in a tempdir) to make the data readable. Default: the value of the study/template bidsmap -s, --store Store provenance data samples in the bidsfolder/code/provenance folder - (useful for inspecting e.g. zipped or transfered datasets) + (useful for inspecting e.g. zipped or transferred datasets) -a, --automated Save the automatically generated bidsmap to disk and without interactively tweaking it with the bidseditor -f, --force Discard the previously saved bidsmap and logfile @@ -151,7 +151,7 @@ If the preview of the BIDS filename and meta-data both look good, you can store The edit window for customizing a bidsmap run item, featuring the DICOM attributes mapped onto BIDS values and (dynamic) metadata values (e.g. ``Comments``). BIDS values that are restricted to a limited set are presented with a drop-down menu. -Finally, if all BIDS output names in the main window are fine, you can click on the [Save] button and proceed with running the bidscoiner tool (step 2). Note that re-running the bidsmapper or bidseditor is always a safe thing to do since these tools will re-use the existing bidsmap yaml-file and will not delete or write anything to disk except to the bidsmap yaml-file. +Finally, if all BIDS output names in the main window are fine, you can click on the [Save] button and proceed with running the bidscoiner tool (step 2). Note that re-running the bidsmapper or bidseditor is always a safe thing to do since these tools will reuse the existing bidsmap yaml-file and will not delete or write anything to disk except to the bidsmap yaml-file. Field maps `````````` From e434d59811d5138533af17e117333c1923f4e2eb Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 11 Jan 2024 10:56:06 -0500 Subject: [PATCH 5/6] Just ignore "mis" as used in (mis)typed --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9568cbc4..62304851 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -124,4 +124,4 @@ commands = make -C docs html skip = '.git,*.pdf,*.svg,go.sum' check-hidden = true ignore-regex = '\b(PULS|TE|ALS)\b' -ignore-words-list = 'compliancy,uptodate' +ignore-words-list = 'compliancy,uptodate,mis' From 5c3b8de44e0f41aeb6380d44d4fcb4bb01b516b8 Mon Sep 17 00:00:00 2001 From: Marcel Zwiers Date: Thu, 11 Jan 2024 17:51:02 +0100 Subject: [PATCH 6/6] Also collect json-files for physio conversions --- bidscoin/plugins/dcm2niix2bids.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bidscoin/plugins/dcm2niix2bids.py b/bidscoin/plugins/dcm2niix2bids.py index 098fcb9d..18c56db2 100644 --- a/bidscoin/plugins/dcm2niix2bids.py +++ b/bidscoin/plugins/dcm2niix2bids.py @@ -293,6 +293,7 @@ def bidscoiner_plugin(session: Path, bidsmap: dict, bidsses: Path) -> Union[None try: physiodata = physio.readphysio(sourcefile) physio.physio2tsv(physiodata, outfolder/bidsname) + jsonfiles.update(outfolder.glob(f"{bidsname}.json")) # add existing created json files: bidsname.json except Exception as physioerror: LOGGER.error(f"Could not read/convert physiological file: {sourcefile}\n{physioerror}") continue