Merge branch 'master' into jen2

.github/workflows/release.yml

@@ -8,7 +8,7 @@ on:
         -   master
 
 env:
-    AUTO_VERSION: v10.37.6
+    AUTO_VERSION: v11.1.6
 
 jobs:
     auto-release:

.pre-commit-config.yaml

@@ -17,7 +17,7 @@ repos:
     -   id: mixed-line-ending
 
 -   repo: https://github.com/codespell-project/codespell
-    rev: v2.2.6
+    rev: v2.3.0
     hooks:
     -   id: codespell
         additional_dependencies:
@@ -30,7 +30,7 @@ repos:
         args: [--profile, black]
 
 -   repo: https://github.com/psf/black-pre-commit-mirror
-    rev: 24.3.0
+    rev: 24.8.0
     hooks:
     -   id: black
 
@@ -41,20 +41,20 @@ repos:
         args: [--mapping, '4', --sequence, '4', --offset, '0']
 
 -   repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
-    rev: v2.13.0
+    rev: v2.14.0
     hooks:
     -   id: pretty-format-toml
         args: [--autofix, --indent, '4']
 
 -   repo: https://github.com/pycqa/flake8
-    rev: 7.0.0
+    rev: 7.1.1
     hooks:
     -   id: flake8
 
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.9.0
+    rev: v1.11.1
     hooks:
     -   id: mypy
-        additional_dependencies: [types-all]
+        additional_dependencies: [types-PyYAML, types-Jinja2, types-click]
         files: neurodocker
         args: [--config-file, pyproject.toml, --install-types]

CHANGELOG.md

@@ -1,3 +1,112 @@
+# 1.0.1 (Sat Jul 06 2024)
+
+#### 🐛 Bug Fix
+
+- FIX use --transform in freesurfer's template to generalize across freesurfer versions to strip leading `freesurfer/` folder [#626](https://github.com/ReproNim/neurodocker/pull/626) ([@mvdoc](https://github.com/mvdoc))
+- FIX remove comment from freesurfer env line [#622](https://github.com/ReproNim/neurodocker/pull/622) ([@mvdoc](https://github.com/mvdoc))
+- FIX add default header and entrypoint to docker and singularity files [#623](https://github.com/ReproNim/neurodocker/pull/623) ([@mvdoc](https://github.com/mvdoc))
+- [BOT] update pre-commit hooks [#627](https://github.com/ReproNim/neurodocker/pull/627) ([@Remi-Gau](https://github.com/Remi-Gau) [@github-actions[bot]](https://github.com/github-actions[bot]))
+- Fix typo of "software" [#624](https://github.com/ReproNim/neurodocker/pull/624) ([@yarikoptic](https://github.com/yarikoptic))
+- [BOT] update pre-commit hooks [#621](https://github.com/ReproNim/neurodocker/pull/621) ([@Remi-Gau](https://github.com/Remi-Gau) [@github-actions[bot]](https://github.com/github-actions[bot]))
+
+#### Authors: 4
+
+- [@github-actions[bot]](https://github.com/github-actions[bot])
+- Matteo Visconti di Oleggio Castello ([@mvdoc](https://github.com/mvdoc))
+- Remi Gau ([@Remi-Gau](https://github.com/Remi-Gau))
+- Yaroslav Halchenko ([@yarikoptic](https://github.com/yarikoptic))
+
+---
+
+# 1.0.0 (Tue Jun 11 2024)
+
+#### 💥 Breaking Change
+
+- Bump actions/checkout from 3 to 4 [#582](https://github.com/ReproNim/neurodocker/pull/582) ([@dependabot[bot]](https://github.com/dependabot[bot]))
+- Bump actions/checkout from 3 to 4 [#573](https://github.com/ReproNim/neurodocker/pull/573) ([@dependabot[bot]](https://github.com/dependabot[bot]))
+- Bump actions/stale from 5 to 8 [#560](https://github.com/ReproNim/neurodocker/pull/560) ([@dependabot[bot]](https://github.com/dependabot[bot]))
+- Bump actions/checkout from 2 to 3 [#557](https://github.com/ReproNim/neurodocker/pull/557) ([@dependabot[bot]](https://github.com/dependabot[bot]))
+- Bump actions/setup-python from 2 to 4 [#556](https://github.com/ReproNim/neurodocker/pull/556) ([@dependabot[bot]](https://github.com/dependabot[bot]))
+
+#### 🐛 Bug Fix
+
+- [BOT] update pre-commit hooks [#616](https://github.com/ReproNim/neurodocker/pull/616) ([@Remi-Gau](https://github.com/Remi-Gau) [@github-actions[bot]](https://github.com/github-actions[bot]))
+- [BOT] update pre-commit hooks [#614](https://github.com/ReproNim/neurodocker/pull/614) ([@Remi-Gau](https://github.com/Remi-Gau) [@github-actions[bot]](https://github.com/github-actions[bot]))
+- make scripts executable after build [#615](https://github.com/ReproNim/neurodocker/pull/615) ([@stebo85](https://github.com/stebo85))
+- [BOT] update pre-commit hooks [#613](https://github.com/ReproNim/neurodocker/pull/613) ([@Remi-Gau](https://github.com/Remi-Gau) [@github-actions[bot]](https://github.com/github-actions[bot]))
+- Use HTTPS for NeuroDebian APT listing urls [#612](https://github.com/ReproNim/neurodocker/pull/612) ([@TheChymera](https://github.com/TheChymera))
+- [BOT] update pre-commit hooks [#610](https://github.com/ReproNim/neurodocker/pull/610) ([@Remi-Gau](https://github.com/Remi-Gau) [@github-actions[bot]](https://github.com/github-actions[bot]))
+- [FIX] Update branch in update_precommit_hooks.yml [#609](https://github.com/ReproNim/neurodocker/pull/609) ([@Remi-Gau](https://github.com/Remi-Gau))
+- [ENH] package with hatchling and switch to pyproject.toml [#607](https://github.com/ReproNim/neurodocker/pull/607) ([@Remi-Gau](https://github.com/Remi-Gau))
+- [MAINT] update linting config and CI [#604](https://github.com/ReproNim/neurodocker/pull/604) ([@Remi-Gau](https://github.com/Remi-Gau))
+- fixed flake8 errors from #597 [#598](https://github.com/ReproNim/neurodocker/pull/598) ([@Vbitz](https://github.com/Vbitz))
+- Added "--add" command line option to add tar files as docker layers. [#597](https://github.com/ReproNim/neurodocker/pull/597) ([@Vbitz](https://github.com/Vbitz))
+- added versions to Library PATH [#593](https://github.com/ReproNim/neurodocker/pull/593) ([@stebo85](https://github.com/stebo85))
+- update matlabmcr [#592](https://github.com/ReproNim/neurodocker/pull/592) ([@stebo85](https://github.com/stebo85))
+- [FIX] add url bids validator [#591](https://github.com/ReproNim/neurodocker/pull/591) ([@Remi-Gau](https://github.com/Remi-Gau))
+- [DOC] add list of supported software [#587](https://github.com/ReproNim/neurodocker/pull/587) ([@Remi-Gau](https://github.com/Remi-Gau))
+- [ENH] add templates bids_validator [#586](https://github.com/ReproNim/neurodocker/pull/586) ([@Remi-Gau](https://github.com/Remi-Gau))
+- [MAINT] test build jq [#589](https://github.com/ReproNim/neurodocker/pull/589) ([@Remi-Gau](https://github.com/Remi-Gau))
+- [ENH] test on python 3.12 [#588](https://github.com/ReproNim/neurodocker/pull/588) ([@Remi-Gau](https://github.com/Remi-Gau))
+- Update fsl.yaml [#583](https://github.com/ReproNim/neurodocker/pull/583) ([@stebo85](https://github.com/stebo85))
+- [DOC] update doc [#569](https://github.com/ReproNim/neurodocker/pull/569) ([@Remi-Gau](https://github.com/Remi-Gau))
+- [ENH] implement more recent mrtrix version [#579](https://github.com/ReproNim/neurodocker/pull/579) ([@Remi-Gau](https://github.com/Remi-Gau))
+- Update examples.rst [#578](https://github.com/ReproNim/neurodocker/pull/578) ([@stebo85](https://github.com/stebo85))
+- added FSL interactivity note [#576](https://github.com/ReproNim/neurodocker/pull/576) ([@stebo85](https://github.com/stebo85))
+- Include FSL license auto-yes example [#575](https://github.com/ReproNim/neurodocker/pull/575) ([@stebo85](https://github.com/stebo85))
+- [MAINT] Update stale.yml [#577](https://github.com/ReproNim/neurodocker/pull/577) ([@Remi-Gau](https://github.com/Remi-Gau))
+- [FIX] install the proper dependencies for doc build [#574](https://github.com/ReproNim/neurodocker/pull/574) ([@Remi-Gau](https://github.com/Remi-Gau))
+- [DOC] auto doc main CLI [#463](https://github.com/ReproNim/neurodocker/pull/463) ([@Remi-Gau](https://github.com/Remi-Gau))
+- [FIX] use local var for software name in bootstrap workflow [#572](https://github.com/ReproNim/neurodocker/pull/572) ([@Remi-Gau](https://github.com/Remi-Gau))
+- [FIX] use env variable in bootstrap workflow [#571](https://github.com/ReproNim/neurodocker/pull/571) ([@Remi-Gau](https://github.com/Remi-Gau))
+- [INFRA] add possibility to only build a single workflow [#570](https://github.com/ReproNim/neurodocker/pull/570) ([@Remi-Gau](https://github.com/Remi-Gau))
+- [MAINT] test build of conda as part of the bootstrap workflow [#566](https://github.com/ReproNim/neurodocker/pull/566) ([@Remi-Gau](https://github.com/Remi-Gau))
+- [MAINT] minimize file changes that can trigger the bootstrap workflow [#564](https://github.com/ReproNim/neurodocker/pull/564) ([@Remi-Gau](https://github.com/Remi-Gau))
+- set python version for format job [#563](https://github.com/ReproNim/neurodocker/pull/563) ([@stebo85](https://github.com/stebo85))
+- Add mamba option to miniconda [#562](https://github.com/ReproNim/neurodocker/pull/562) ([@Shotgunosine](https://github.com/Shotgunosine))
+- [MAINT] remove oldest distro in automated build testing [#559](https://github.com/ReproNim/neurodocker/pull/559) ([@Remi-Gau](https://github.com/Remi-Gau))
+- [MAINT] split format checking and testing in CI [#554](https://github.com/ReproNim/neurodocker/pull/554) ([@Remi-Gau](https://github.com/Remi-Gau))
+- [MAINT] Simplify type annotations [#551](https://github.com/ReproNim/neurodocker/pull/551) ([@Remi-Gau](https://github.com/Remi-Gau))
+- apply isort and all pre-commit hooks [#549](https://github.com/ReproNim/neurodocker/pull/549) ([@Remi-Gau](https://github.com/Remi-Gau))
+- Mcr bug [#471](https://github.com/ReproNim/neurodocker/pull/471) ([@stebo85](https://github.com/stebo85))
+- Add codespell: config, pre-commit, workflow + 1 typo fixed [#544](https://github.com/ReproNim/neurodocker/pull/544) ([@yarikoptic](https://github.com/yarikoptic) [@stebo85](https://github.com/stebo85))
+- added freesurfer 7.3.2 [#547](https://github.com/ReproNim/neurodocker/pull/547) ([@stebo85](https://github.com/stebo85))
+- add fsl 6.0.7.1 [#543](https://github.com/ReproNim/neurodocker/pull/543) ([@stebo85](https://github.com/stebo85))
+- added freesurfer 7.4.1 [#542](https://github.com/ReproNim/neurodocker/pull/542) ([@hjbockholt](https://github.com/hjbockholt))
+- Fixing my update for mcr2023a [#540](https://github.com/ReproNim/neurodocker/pull/540) ([@dnkennedy](https://github.com/dnkennedy))
+- Update matlabmcr.yaml [#539](https://github.com/ReproNim/neurodocker/pull/539) ([@dnkennedy](https://github.com/dnkennedy))
+- Update years to have full range of years of the project [#534](https://github.com/ReproNim/neurodocker/pull/534) ([@yarikoptic](https://github.com/yarikoptic))
+- Update cat [#533](https://github.com/ReproNim/neurodocker/pull/533) ([@stebo85](https://github.com/stebo85))
+
+#### ⚠️ Pushed to `master`
+
+- Change name of parent directory ([@kaczmarj](https://github.com/kaczmarj))
+
+#### 🏠 Internal
+
+- Upgrade intuit auto to 11.1.6 (most recent) [#617](https://github.com/ReproNim/neurodocker/pull/617) ([@yarikoptic](https://github.com/yarikoptic))
+
+#### 🔩 Dependency Updates
+
+- Bump actions/setup-python from 4 to 5 [#594](https://github.com/ReproNim/neurodocker/pull/594) ([@dependabot[bot]](https://github.com/dependabot[bot]))
+- Bump actions/stale from 8 to 9 [#595](https://github.com/ReproNim/neurodocker/pull/595) ([@dependabot[bot]](https://github.com/dependabot[bot]))
+
+#### Authors: 11
+
+- [@dependabot[bot]](https://github.com/dependabot[bot])
+- [@github-actions[bot]](https://github.com/github-actions[bot])
+- David Kennedy ([@dnkennedy](https://github.com/dnkennedy))
+- Dylan Nielson ([@Shotgunosine](https://github.com/Shotgunosine))
+- H. Jeremy Bockholt ([@hjbockholt](https://github.com/hjbockholt))
+- Horea Christian ([@TheChymera](https://github.com/TheChymera))
+- Jakub Kaczmarzyk ([@kaczmarj](https://github.com/kaczmarj))
+- Joshua Scarsbrook ([@Vbitz](https://github.com/Vbitz))
+- Remi Gau ([@Remi-Gau](https://github.com/Remi-Gau))
+- Steffen Bollmann ([@stebo85](https://github.com/stebo85))
+- Yaroslav Halchenko ([@yarikoptic](https://github.com/yarikoptic))
+
+---
+
 # 0.9.5 (Fri May 12 2023)
 
 #### 🐛 Bug Fix

neurodocker/cli/generate.py

@@ -307,6 +307,8 @@ def _get_params_for_registered_templates() -> list[click.Parameter]:
     names_tmpls = list(registered_templates_items())
     names_tmpls.sort(key=lambda r: r[0])  # sort by name
     for name, tmpl in names_tmpls:
+        if name.startswith("_"):  # Templates starting with _ are private
+            continue
         hlp = _create_help_for_template(Template(tmpl))
         param = OptionEatAll(
             [f"--{name.lower()}"], type=KeyValuePair(), multiple=True, help=hlp
@@ -438,6 +440,18 @@ def _base_generate(
     `generate singularity`. The difference between those two is the renderer used.
     """
     renderer_dict = _params_to_renderer_dict(ctx=ctx, pkg_manager=pkg_manager)
+
+    # Make sure to add default instructions if they are available among the templates
+    if "_default" in registered_templates():
+        # Add header to the instructions, after the base image.
+        renderer_dict["instructions"].insert(1, {"name": "_default", "kwds": {}})
+        # Use default entrypoint if user did not specify one.
+        instruction_names = [instr["name"] for instr in renderer_dict["instructions"]]
+        if "entrypoint" not in instruction_names:
+            renderer_dict["instructions"].append(
+                {"name": "entrypoint", "kwds": {"args": ["/neurodocker/startup.sh"]}}
+            )
+
     r = renderer.from_dict(renderer_dict)
 
     # Print the instructions in JSON if that's what the user wants.

neurodocker/cli/tests/test_cli.py

@@ -177,3 +177,43 @@ def test_render_registered(cmd: str, pkg_manager: str):
     assert result.exit_code == 0, result.output
     assert "jq-1.5/jq-linux64" in result.output
     assert "jq-1.6/jq-linux64" in result.output
+
+
+# Test that we add the default header and default/custom entrypoints
+@pytest.mark.parametrize("cmd", _cmds)
+@pytest.mark.parametrize("pkg_manager", ["apt", "yum"])
+@pytest.mark.parametrize("entrypoint", ["default", "custom"])
+def test_default_header_and_entrypoint(cmd: str, pkg_manager: str, entrypoint: str):
+    runner = CliRunner()
+    cmd_ = [
+        cmd,
+        "--base-image",
+        "debian:buster",
+        "--pkg-manager",
+        pkg_manager,
+    ]
+    if entrypoint == "custom":
+        cmd_.extend(["--entrypoint", "I", "decide"])
+    result = runner.invoke(generate, cmd_)
+    assert result.exit_code == 0, result.output
+
+    if cmd == "docker":
+        assert (
+            'ND_ENTRYPOINT="/neurodocker/startup.sh"\n'
+            'RUN export ND_ENTRYPOINT="/neurodocker/startup.sh"'
+        ) in result.output
+
+        if entrypoint == "default":
+            assert '\nENTRYPOINT ["/neurodocker/startup.sh"]\n' in result.output
+        else:
+            assert '\nENTRYPOINT ["I", "decide"]\n' in result.output
+    else:
+        assert (
+            '\nexport ND_ENTRYPOINT="/neurodocker/startup.sh"\n\n'
+            '%post\nexport ND_ENTRYPOINT="/neurodocker/startup.sh"\n'
+        ) in result.output
+
+        if entrypoint == "default":
+            assert "%runscript\n/neurodocker/startup.sh\n" in result.output
+        else:
+            assert "%runscript\nI decide\n" in result.output

neurodocker/templates/_header.yaml → neurodocker/templates/_default.yaml

@@ -4,7 +4,7 @@
 
 # TODO: read this about locales http://jaredmarkell.com/docker-and-locales/
 
-name: _header
+name: _default
 url: n/a
 # Not actually source, but we do not want to provide URLs.
 source:
@@ -28,14 +28,14 @@ source:
         LC_ALL: en_US.UTF-8
         ND_ENTRYPOINT: /neurodocker/startup.sh
     instructions: |
-        export ND_ENTRYPOINT="{{ _header._env['ND_ENTRYPOINT'] }}"
-        {{ _header.install_dependencies() }}
-        {%- if _header.pkg_manager == "apt" %}
-        sed -i -e 's/# {{ _header._env['LC_ALL'] }} UTF-8/{{ _header._env['LC_ALL'] }} UTF-8/' /etc/locale.gen
+        export ND_ENTRYPOINT="{{ self.env['ND_ENTRYPOINT'] }}"
+        {{ self.install_dependencies() }}
+        {%- if self.pkg_manager == "apt" %}
+        sed -i -e 's/# {{ self.env['LC_ALL'] }} UTF-8/{{ self.env['LC_ALL'] }} UTF-8/' /etc/locale.gen
         dpkg-reconfigure --frontend=noninteractive locales
-        update-locale LANG="{{ _header._env['LANG'] }}"
-        {%- elif _header.pkg_manager == "yum" %}
-        localedef -i {{ _header._env['LC_ALL'].split('.')[0] }} -f {{ _header._env['LC_ALL'].split('.')[1] }} {{ _header._env['LC_ALL'] }}
+        update-locale LANG="{{ self.env['LANG'] }}"
+        {%- elif self.pkg_manager == "yum" %}
+        localedef -i {{ self.env['LC_ALL'].split('.')[0] }} -f {{ self.env['LC_ALL'].split('.')[1] }} {{ self.env['LC_ALL'] }}
         {%- endif %}
         chmod 777 /opt && chmod a+s /opt
         mkdir -p /neurodocker

neurodocker/templates/ants.yaml

@@ -100,3 +100,4 @@ source:
           mv ../Scripts/* {{ self.install_path }} ; \
         fi
         rm -rf /tmp/ants
+        chmod a+rx {{ self.install_path }} -R

neurodocker/templates/freesurfer.yaml

@@ -71,7 +71,7 @@ binaries:
     # default freesurfer options
         FS_OVERRIDE: '0'
         FIX_VERTEX_AREA: ''
-        FSF_OUTPUT_FORMAT: nii.gz# mni env requirements
+        FSF_OUTPUT_FORMAT: nii.gz
     # mni environment requirements
         MINC_BIN_DIR: '{{ self.install_path }}/mni/bin'
         MINC_LIB_DIR: '{{ self.install_path }}/mni/lib'
@@ -84,7 +84,7 @@ binaries:
         echo "Downloading FreeSurfer ..."
         mkdir -p {{ self.install_path }}
         curl -fL {{ self.urls[self.version] }} \
-        | tar -xz -C {{ self.install_path }} --owner root --group root --no-same-owner --strip-components 1 {% if self.exclude_paths -%}\
+        | tar -xz -C {{ self.install_path }} --owner root --group root --no-same-owner --transform='s,freesurfer/,,' {% if self.exclude_paths -%}\
         {%- for exclude_path in self.exclude_paths.split() %}
           {% if not loop.last -%}
           --exclude='{{ exclude_path }}' \

neurodocker/templates/neurodebian.yaml

@@ -2,20 +2,20 @@
 # Instructions to add NeuroDebian repositories.
 
 name: neurodebian
-url: http://neuro.debian.net
+url: https://neuro.debian.net
 binaries:
     urls:
-        australia: http://neuro.debian.net/lists/{{ self.os_codename }}.au.{{ self.full_or_libre }}
-        china-tsinghua: http://neuro.debian.net/lists/{{ self.os_codename }}.cn-bj1.{{ self.full_or_libre }}
-        china-scitech: http://neuro.debian.net/lists/{{ self.os_codename }}.cn-bj2.{{ self.full_or_libre }}
-        china-zhejiang: http://neuro.debian.net/lists/{{ self.os_codename }}.cn-zj.{{ self.full_or_libre }}
-        germany-munich: http://neuro.debian.net/lists/{{ self.os_codename }}.de-m.{{ self.full_or_libre }}
-        germany-magdeburg: http://neuro.debian.net/lists/{{ self.os_codename }}.de-md.{{ self.full_or_libre }}
-        greece: http://neuro.debian.net/lists/{{ self.os_codename }}.gr.{{ self.full_or_libre }}
-        japan: http://neuro.debian.net/lists/{{ self.os_codename }}.jp.{{ self.full_or_libre }}
-        usa-ca: http://neuro.debian.net/lists/{{ self.os_codename }}.us-ca.{{ self.full_or_libre }}
-        usa-nh: http://neuro.debian.net/lists/{{ self.os_codename }}.us-nh.{{ self.full_or_libre }}
-        usa-tn: http://neuro.debian.net/lists/{{ self.os_codename }}.us-tn.{{ self.full_or_libre }}
+        australia: https://neuro.debian.net/lists/{{ self.os_codename }}.au.{{ self.full_or_libre }}
+        china-tsinghua: https://neuro.debian.net/lists/{{ self.os_codename }}.cn-bj1.{{ self.full_or_libre }}
+        china-scitech: https://neuro.debian.net/lists/{{ self.os_codename }}.cn-bj2.{{ self.full_or_libre }}
+        china-zhejiang: https://neuro.debian.net/lists/{{ self.os_codename }}.cn-zj.{{ self.full_or_libre }}
+        germany-munich: https://neuro.debian.net/lists/{{ self.os_codename }}.de-m.{{ self.full_or_libre }}
+        germany-magdeburg: https://neuro.debian.net/lists/{{ self.os_codename }}.de-md.{{ self.full_or_libre }}
+        greece: https://neuro.debian.net/lists/{{ self.os_codename }}.gr.{{ self.full_or_libre }}
+        japan: https://neuro.debian.net/lists/{{ self.os_codename }}.jp.{{ self.full_or_libre }}
+        usa-ca: https://neuro.debian.net/lists/{{ self.os_codename }}.us-ca.{{ self.full_or_libre }}
+        usa-nh: https://neuro.debian.net/lists/{{ self.os_codename }}.us-nh.{{ self.full_or_libre }}
+        usa-tn: https://neuro.debian.net/lists/{{ self.os_codename }}.us-tn.{{ self.full_or_libre }}
     arguments:
         required:
         -   version