From 29beebc64bffedf66475d551dc3e693b9a77dabc Mon Sep 17 00:00:00 2001 From: Remi Gau Date: Sun, 29 Sep 2024 10:19:43 +0200 Subject: [PATCH] reset history with no annex --- .dockerignore | 12 ++ .flake8 | 5 + .gitattributes | 8 + .github/dependabot.yml | 14 ++ .github/workflows/docker.yml | 139 ++++++++++++++ .gitignore | 9 + .pre-commit-config.yaml | 42 +++++ .readthedocs.yml | 26 +++ CHANGELOG.md | 22 +++ Dockerfile | 85 +++++++++ LICENSE | 21 +++ README.md | 7 + code/.gitattributes | 1 + code/README.md | 3 + code/_parsers.py | 188 +++++++++++++++++++ code/_version.py | 1 + code/bids_utils.py | 129 +++++++++++++ code/cat_logging.py | 30 +++ code/data/methods/template.jinja | 8 + code/defaults.py | 46 +++++ code/exit_codes.json | 50 +++++ code/main.py | 311 +++++++++++++++++++++++++++++++ code/methods.py | 38 ++++ code/requirements.txt | 4 + code/utils.py | 45 +++++ docs/.gitattributes | 1 + docs/Makefile | 20 ++ docs/make.bat | 35 ++++ docs/source/conf.py | 35 ++++ docs/source/index.rst | 19 ++ makefile | 40 ++++ pyproject.toml | 36 ++++ 32 files changed, 1430 insertions(+) create mode 100644 .dockerignore create mode 100644 .flake8 create mode 100644 .gitattributes create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/docker.yml create mode 100644 .gitignore create mode 100644 .pre-commit-config.yaml create mode 100644 .readthedocs.yml create mode 100644 CHANGELOG.md create mode 100644 Dockerfile create mode 100644 LICENSE create mode 100644 README.md create mode 100644 code/.gitattributes create mode 100644 code/README.md create mode 100644 code/_parsers.py create mode 100644 code/_version.py create mode 100644 code/bids_utils.py create mode 100644 code/cat_logging.py create mode 100644 code/data/methods/template.jinja create mode 100644 code/defaults.py create mode 100644 code/exit_codes.json create mode 100644 code/main.py create mode 100644 code/methods.py create mode 100644 code/requirements.txt create mode 100644 code/utils.py create mode 100644 docs/.gitattributes create mode 100644 docs/Makefile create mode 100644 docs/make.bat create mode 100644 docs/source/conf.py create mode 100644 docs/source/index.rst create mode 100644 makefile create mode 100644 pyproject.toml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..c8d4e40 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ +# ignoring this might speed up build +# by preventing passing extra content to the docker daemon + +.simg + +tests/data + +env + +tmp + +docs/build diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..890a732 --- /dev/null +++ b/.flake8 @@ -0,0 +1,5 @@ +[flake8] +count = True +show-source = True +statistics = True +max_function_length = 200 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..0770741 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,8 @@ +* annex.backend=MD5E +**/.git* annex.largefiles=nothing +CHANGELOG.md annex.largefiles=nothing +README.md annex.largefiles=nothing +pyproject.toml annex.largefiles=nothing +.readthedocs.yml annex.largefiles=nothing +.github/**/*.yml annex.largefiles=nothing +*.def annex.largefiles=nothing diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..e73f065 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,14 @@ +--- +# Documentation +# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file +version: 2 +updates: +- package-ecosystem: github-actions + directory: / + schedule: + interval: monthly + +- package-ecosystem: gitsubmodule + directory: / + schedule: + interval: monthly diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..a0b4fd0 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,139 @@ +--- +name: docker build + +on: + push: + branches: [main] + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + USER_NAME: bids + REPO_NAME: cat12 + IMAGE: /home/runner/work/Remi-Gau/cat12-container/docker + +jobs: + + docker-build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + # # cache tarred docker image to speed up build on follow up runs + # - uses: actions/cache@v4 + # id: cache + # with: + # path: ${{ env.IMAGE }}/image.tar + # key: data + # - if: ${{ steps.cache.outputs.cache-hit != 'true' }} + # name: Load image + # run: docker load -i ${{ env.IMAGE }}/image.tar + - name: Build the Docker image + run: | + docker build . --tag ${{env.USER_NAME}}/${{env.REPO_NAME}} + - name: Check image size and version + run: | + docker images + docker run --rm ${{env.USER_NAME}}/${{env.REPO_NAME}} --version + - name: Run simple commands + run: | + docker run --rm ${{env.USER_NAME}}/${{env.REPO_NAME}} --help + docker run --rm ${{env.USER_NAME}}/${{env.REPO_NAME}} . /foo participant view segment --verbose 3 + docker run --rm ${{env.USER_NAME}}/${{env.REPO_NAME}} . /foo participant copy segment --verbose 3 + docker run --rm ${{env.USER_NAME}}/${{env.REPO_NAME}} . /foo participant segment --help + - name: Save docker image + run: | + mkdir -p ${{ env.IMAGE }} + docker save "${{env.USER_NAME}}/${{env.REPO_NAME}}" > "${{ env.IMAGE }}/image.tar" + - name: Upload docker artifacts + uses: actions/upload-artifact@v4 + with: + name: docker + path: ${{ env.IMAGE }} + + one-session: + runs-on: ubuntu-latest + strategy: + matrix: + type: [default, simple, enigma] + fail-fast: false + needs: docker-build + steps: + - name: Install dependencies + run: | + sudo apt-get -y -qq update + sudo apt-get -y install tree + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Restore docker image + uses: actions/download-artifact@v4 + with: + name: docker + path: ${{ env.IMAGE }} + - name: Load image + run: docker load -i ${{ env.IMAGE }}/image.tar + - name: Get data + run: make tests/data/MoAEpilot + - name: Segment + run: | + docker run --rm \ + -v ${PWD}/tests/data/MoAEpilot:/data \ + ${{env.USER_NAME}}/${{env.REPO_NAME}} \ + /data /data/derivatives participant \ + segment --verbose 3 --type ${{ matrix.type }} + tree ${PWD}/tests/data/ + - name: Upload output artifact + uses: actions/upload-artifact@v4 + with: + name: output_${{ matrix.type }} + path: /home/runner/work/cat12-container/cat12-container/tests/data/MoAEpilot/derivatives + + two-sessions: + runs-on: ubuntu-latest + strategy: + matrix: + type: [long_0, long_2] + fail-fast: false + needs: docker-build + steps: + - name: Install python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: Install dependencies + run: | + sudo apt-get -y -qq update + sudo apt-get -y install git-annex tree + pip install datalad + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Restore docker image + uses: actions/download-artifact@v4 + with: + name: docker + path: ${{ env.IMAGE }} + - name: Load image + run: docker load -i ${{ env.IMAGE }}/image.tar + - name: Get data + run: make data_ds002799 + - name: Segment + run: | + docker run --rm \ + -v ${PWD}/tests/data/ds002799:/data \ + ${{env.USER_NAME}}/${{env.REPO_NAME}} \ + /data /data/derivatives participant \ + segment --verbose 3 --type ${{ matrix.type }} \ + --participant_label 292 294 \ + --skip_validation + tree ${PWD}/tests/data/ + - name: Upload output artifact + uses: actions/upload-artifact@v4 + with: + name: output_${{ matrix.type }} + path: /home/runner/work/cat12-container/cat12-container/tests/data/ds002799/derivatives diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f9a32a5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +.simg + +tests/data + +env + +tmp + +docs/build diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..6b4d514 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,42 @@ +--- +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: check-yaml + - id: check-json + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-case-conflict + - id: check-merge-conflict + - id: mixed-line-ending + +- repo: https://github.com/jumanjihouse/pre-commit-hook-yamlfmt + rev: 0.2.3 + hooks: + - id: yamlfmt + args: [--mapping, '4', --sequence, '4', --offset, '0'] + +- repo: https://github.com/codespell-project/codespell + rev: v2.3.0 + hooks: + - id: codespell + +- repo: https://github.com/pycqa/isort + rev: 5.13.2 + hooks: + - id: isort + args: [--line-length, '79', --profile, black] + +- repo: https://github.com/psf/black-pre-commit-mirror + rev: 24.8.0 + hooks: + - id: black + args: [--line-length, '79'] + +- repo: https://github.com/pycqa/flake8 + rev: 7.1.1 + hooks: + - id: flake8 + args: [--config, .flake8, --verbose] + additional_dependencies: [flake8-docstrings, flake8-use-fstring, flake8-functions, flake8-bugbear] diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000..1e76c73 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,26 @@ +--- +# .readthedocs.yml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: '3.12' + +# Build documentation in the docs/ directory with Sphinx +sphinx: + configuration: docs/source/conf.py + builder: html + fail_on_warning: false + +# Optionally set the version of Python and requirements required to build your docs +python: + install: + - method: pip + path: . + extra_requirements: + - docs diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..f94d986 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,22 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + + + +## [Unreleased] diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c8e3062 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,85 @@ +FROM ubuntu:22.04@sha256:340d9b015b194dc6e2a13938944e0d016e57b9679963fdeb9ce021daac430221 + +LABEL org.opencontainers.image.authors="fil.spm@ucl.ac.uk, Malgorzata Wierzba (m.wierzba@fz-juelich.de), Felix Hoffstaedter (f.hoffstaedter@fz-juelich.de), Remi Gau (remi.gau@gmail.com)" +LABEL org.opencontainers.image.source="https://gin.g-node.org/felixh/cat12-container" +LABEL org.opencontainers.image.version="v1.1dev" +LABEL org.opencontainers.image.licenses="MIT" +LABEL org.opencontainers.image.title="cat12-container" + +ENV LANG="en_US.UTF-8" \ + LC_ALL="en_US.UTF-8" \ + FORCE_SPMMCR="1" \ + SPM_HTML_BROWSER="0" \ + MCR_INHIBIT_CTF_LOCK="1" \ + MCR_VERSION="2017b" +ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib/x86_64-linux-gnu:/opt/MCR-${MCR_VERSION}/v93/runtime/glnxa64:/opt/MCR-${MCR_VERSION}/v93/bin/glnxa64:/opt/MCR-${MCR_VERSION}/v93/sys/os/glnxa64:/opt/MCR-${MCR_VERSION}/v93/extern/bin/glnxa64" \ + MATLABCMD="/opt/MCR-${MCR_VERSION}/v93/toolbox/matlab" \ + XAPPLRESDIR="/opt//opt/MCR-${MCR_VERSION}/v93/x11/app-defaults" \ + MCRROOT="/opt/MCR-${MCR_VERSION}/v93" \ + CAT_VERSION=".8.1_r2042_R${MCR_VERSION}" \ + DENO_INSTALL="/root/.deno" +ENV SPMROOT="/opt/CAT12${CAT_VERSION}" \ + PATH="$DENO_INSTALL/bin:/opt/CAT12${CAT_VERSION}:$PATH" \ + STANDALONE="/opt/CAT12${CAT_VERSION}/standalone" + +RUN export ND_ENTRYPOINT="/neurodocker/startup.sh" \ + && apt-get update -qq \ + && apt-get install -y -q --no-install-recommends \ + apt-utils \ + bc \ + bzip2 \ + ca-certificates \ + curl \ + dbus-x11 \ + libncurses5 \ + libxext6 \ + libxmu6 \ + libxpm-dev \ + libxt6 \ + locales \ + openjdk-8-jre \ + python3 \ + pip \ + unzip \ + && rm -rf /var/lib/apt/lists/* \ + && sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen \ + && dpkg-reconfigure --frontend=noninteractive locales \ + && update-locale LANG="en_US.UTF-8" \ + && chmod 777 /opt && chmod a+s /opt + +RUN echo "Downloading MATLAB Compiler Runtime ..." \ + && export TMPDIR="$(mktemp -d)" \ + && curl -o "$TMPDIR/mcr.zip" https://ssd.mathworks.com/supportfiles/downloads/R${MCR_VERSION}/deployment_files/R${MCR_VERSION}/installers/glnxa64/MCR_R${MCR_VERSION}_glnxa64_installer.zip \ + && unzip -q "$TMPDIR/mcr.zip" -d "$TMPDIR/mcrtmp" \ + && "$TMPDIR/mcrtmp/install" -destinationFolder /opt/MCR-${MCR_VERSION} -mode silent -agreeToLicense yes \ + && rm -rf "$TMPDIR" \ + && unset TMPDIR + +RUN echo "Downloading standalone CAT12 ..." \ + && curl -fL -o /tmp/cat12.zip http://www.neuro.uni-jena.de/cat12/CAT12${CAT_VERSION}_MCR_Linux.zip \ + && unzip -q /tmp/cat12.zip -d /tmp \ + && rm -rf /tmp/cat12.zip \ + && mkdir -p /opt/CAT12${CAT_VERSION} \ + && mv /tmp/*${CAT_VERSION}*/* /opt/CAT12${CAT_VERSION}/ \ + && chmod -R 777 /opt/CAT12${CAT_VERSION} \ + # Test + && /opt/CAT12${CAT_VERSION}/spm12 function exit + +## Install BIDS validator +RUN curl -fsSL https://deno.land/install.sh | sh && \ + deno install -Agf -n bids-validator jsr:@bids/validator@1.14.12 + +# transfer code and set permission +RUN mkdir -p /code +COPY ./code/requirements.txt /code +RUN pip install -r /code/requirements.txt + +COPY ./code /code +RUN ls /code && find /code -type f -print0 | xargs -0 chmod +r + +# modify enigma script to output content to path defined by an env variable +RUN sed -i -e "s/cat_version/getenv('OUTPUT_DIR')/g" /opt/CAT12${CAT_VERSION}/standalone/cat_standalone_segment_enigma.m + +WORKDIR ${STANDALONE} + +ENTRYPOINT ["python3", "/code/main.py"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d4b07b6 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2021 Małgorzata Wierzba, Felix Hoffstaedter, Michael Hanke + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..2b463e4 --- /dev/null +++ b/README.md @@ -0,0 +1,7 @@ +# CAT-12 container + +Docker and Apptainer image for [CAT12](https://neuro-jena.github.io/cat/). + +CAT12 8.1 r2042 +SPM12, version 7771 (standalone) +MATLAB, version 9.3.0.713579 (R2017b) diff --git a/code/.gitattributes b/code/.gitattributes new file mode 100644 index 0000000..8bb0167 --- /dev/null +++ b/code/.gitattributes @@ -0,0 +1 @@ +* annex.largefiles=nothing diff --git a/code/README.md b/code/README.md new file mode 100644 index 0000000..1ed9d2f --- /dev/null +++ b/code/README.md @@ -0,0 +1,3 @@ +All custom code goes into this directory. All scripts should be written such +that they can be executed from the root of the dataset, and are only using +relative paths for portability. diff --git a/code/_parsers.py b/code/_parsers.py new file mode 100644 index 0000000..44683d3 --- /dev/null +++ b/code/_parsers.py @@ -0,0 +1,188 @@ +from __future__ import annotations + +from argparse import ArgumentParser, HelpFormatter + +from _version import __version__ +from defaults import CAT_VERSION, MCR_VERSION, supported_batches + + +def _base_parser( + formatter_class: type[HelpFormatter] = HelpFormatter, +) -> ArgumentParser: + parser = ArgumentParser( + description=("BIDS app for CAT12."), + formatter_class=formatter_class, + ) + parser.add_argument( + "--version", + action="version", + help="Show program's version number and exit.", + version=f""" + BIDS app: {__version__}; + CAT12: {CAT_VERSION}; + MATLAB MCR: {MCR_VERSION} +""", + ) + parser.add_argument( + "bids_dir", + help=""" +Fullpath to the directory with the input dataset +formatted according to the BIDS standard. + """, + nargs=1, + ) + parser.add_argument( + "output_dir", + help=""" +Fullpath to the directory where the output files will be stored. +""", + nargs=1, + ) + parser.add_argument( + "analysis_level", + help=""" + Level of the analysis that will be performed. + Multiple participant level analyses can be run independently + (in parallel) using the same ``output_dir``. + """, + choices=["participant", "group"], + default="participant", + type=str, + nargs=1, + ) + + return parser + + +def _add_common_arguments(parser: ArgumentParser) -> ArgumentParser: + parser.add_argument( + "--participant_label", + help=""" +The label(s) of the participant(s) that should be analyzed. +The label corresponds to sub- from the BIDS spec +(so it does not include "sub-"). + +If this parameter is not provided, all subjects will be analyzed. +Multiple participants can be specified with a space separated list. + """, + nargs="+", + required=False, + ) + parser = _add_verbose(parser) + parser.add_argument( + "--bids_filter_file", + help=""" +A JSON file describing custom BIDS input filters using PyBIDS. +For further details, please check out TBD. + """, + required=False, + ) + parser.add_argument( + "--skip_validation", + help="Do not run the bids validation.", + action="store_true", + required=False, + ) + return parser + + +def _add_target(parser, with_all=False): + choices = supported_batches() + if with_all: + choices.append("all") + parser.add_argument( + "target", + help="Batch name", + choices=supported_batches(), + type=str, + nargs=1, + ) + return parser + + +def _add_verbose(parser): + parser.add_argument( + "--verbose", + help=""" + Verbosity level. + """, + choices=[0, 1, 2, 3], + default=2, + type=int, + nargs=1, + ) + return parser + + +def common_parser( + formatter_class: type[HelpFormatter] = HelpFormatter, +) -> ArgumentParser: + """Execute the main script.""" + parser = _base_parser(formatter_class=formatter_class) + subparsers = parser.add_subparsers( + dest="command", + help="Choose a sub-command", + required=True, + ) + + subparsers.add_parser( + "help", + help="Show cat12 script help.", + formatter_class=parser.formatter_class, + ) + + view_parser = subparsers.add_parser( + "view", + help="View batch.", + formatter_class=parser.formatter_class, + ) + view_parser = _add_target(view_parser) + view_parser = _add_verbose(view_parser) + + copy_parser = subparsers.add_parser( + "copy", + help="Copy batch to output_dir.", + formatter_class=parser.formatter_class, + ) + copy_parser = _add_target(copy_parser, with_all=True) + copy_parser = _add_verbose(copy_parser) + + segment_parser = subparsers.add_parser( + "segment", + help="segment", + formatter_class=parser.formatter_class, + ) + segment_parser = _add_common_arguments(segment_parser) + segment_parser.add_argument( + "--reset_database", + help="Resets the database of the input dataset.", + action="store_true", + required=False, + ) + segment_parser.add_argument( + "--type", + help="""Type of segmentation. + default: default CAT12 preprocessing batch; + simple: simple processing batch; + long_0 - longitudinal developmental; + long_1 - longitudinal plasticity/learning; + long_2 - longitudinal aging; + long_3 - longitudinal save models 1 and 2; + enigma - enigma segmentation +""", + choices=[ + "default", + "simple", + "long_0", + "long_1", + "long_2", + "long_3", + "enigma", + ], + default="default", + required=False, + type=str, + nargs=1, + ) + + return parser diff --git a/code/_version.py b/code/_version.py new file mode 100644 index 0000000..3dc1f76 --- /dev/null +++ b/code/_version.py @@ -0,0 +1 @@ +__version__ = "0.1.0" diff --git a/code/bids_utils.py b/code/bids_utils.py new file mode 100644 index 0000000..f86ee53 --- /dev/null +++ b/code/bids_utils.py @@ -0,0 +1,129 @@ +"""BIDS utilities.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from _version import __version__ +from bids import BIDSLayout # type: ignore +from cat_logging import cat12_log +from utils import create_dir_if_absent + +logger = cat12_log(name="cat12") + + +def get_dataset_layout( + dataset_path: str | Path, + use_database: bool = False, + reset_database: bool = False, +) -> BIDSLayout: + """Return a BIDSLayout object for the dataset at the given path. + + :param dataset_path: Path to the dataset. + :type dataset_path: Union[str, Path] + + :param use_database: Defaults to False + :type use_database: bool, optional + + :return: _description_ + :rtype: BIDSLayout + """ + if isinstance(dataset_path, str): + dataset_path = Path(dataset_path) + + dataset_path = dataset_path.absolute() + + logger.info(f"indexing {dataset_path}") + + if not use_database: + return BIDSLayout( + dataset_path, + validate=True, + derivatives=False, + ) + + database_path = dataset_path / "pybids_db" + return BIDSLayout( + dataset_path, + validate=False, + derivatives=False, + database_path=database_path, + reset_database=reset_database, + ) + + +def init_derivatives_layout(output_dir: Path) -> BIDSLayout: + """Initialize a derivatives dataset and returns its layout. + + :param output_dir: + :type output_dir: Path + + :return: + :rtype: BIDSLayout + """ + create_dir_if_absent(output_dir) + write_dataset_description(output_dir) + layout_out = get_dataset_layout(output_dir) + return layout_out + + +def list_subjects(layout: BIDSLayout, subjects) -> list[str]: + """List subject in a BIDS dataset for a given Config. + + :param cfg: Configuration object + :type cfg: Config + + :param layout: BIDSLayout of the dataset. + :type layout: BIDSLayout + + :raises RuntimeError: _description_ + + :return: _description_ + :rtype: list + """ + subjects = layout.get(return_type="id", target="subject", subject=subjects) + + if subjects == [] or subjects is None: + raise RuntimeError(f"No subject found in layout:\n\t{layout.root}") + + logger.info(f"processing subjects: {subjects}") + + return subjects + + +def write_dataset_description(output_dir) -> None: + """Add dataset description to a layout. + + :param output_dir: output_dir + :type output_dir: Path + """ + data: dict[str, Any] = { + "Name": "dataset name", + "BIDSVersion": "1.9.0", + "DatasetType": "derivative", + "License": "???", + "ReferencesAndLinks": ["https://doi.org/10.1101/2022.06.11.495736"], + } + data["GeneratedBy"] = [ + { + "Name": "cat12", + "Version": __version__, + "Container": {"Type": "", "Tag": __version__}, + "Description": "", + "CodeURL": "", + }, + ] + data["SourceDatasets"] = [ + { + "DOI": "doi:", + "URL": "", + "Version": "", + } + ] + + output_file = output_dir / "dataset_description.json" + + with open(output_file, "w") as ff: + json.dump(data, ff, indent=4) diff --git a/code/cat_logging.py b/code/cat_logging.py new file mode 100644 index 0000000..ce4fb9a --- /dev/null +++ b/code/cat_logging.py @@ -0,0 +1,30 @@ +"""For logging.""" + +from __future__ import annotations + +import logging + +from rich.logging import RichHandler +from rich.traceback import install + + +def cat12_log(name: str | None = None) -> logging.Logger: + """Create log.""" + # let rich print the traceback + install(show_locals=True) + + FORMAT = "cat12 - %(asctime)s - %(message)s" + + if not name: + name = "rich" + + logging.basicConfig( + level="INFO", + format=FORMAT, + datefmt="[%X]", + handlers=[ + RichHandler(), + ], + ) + + return logging.getLogger(name) diff --git a/code/data/methods/template.jinja b/code/data/methods/template.jinja new file mode 100644 index 0000000..bdf1441 --- /dev/null +++ b/code/data/methods/template.jinja @@ -0,0 +1,8 @@ +These results were generated with the BIDS app for Computational Anatomy Toolbox +(BIDS app: {{version}}; CAT12: {{cat_version}}; MATLAB MCR: {{mcr_version}}) + +The following steps were followed. + +1. ... + +2. ... diff --git a/code/defaults.py b/code/defaults.py new file mode 100644 index 0000000..9e58f1e --- /dev/null +++ b/code/defaults.py @@ -0,0 +1,46 @@ +"""Store defaults.""" + +from __future__ import annotations + +import os + +MCR_VERSION = os.getenv("MCR_VERSION") +CAT_VERSION = " ".join(os.getenv("CAT_VERSION")[1:10].split("_")) + + +def log_levels() -> list[str]: + """Return a list of log levels.""" + return ["ERROR", "WARNING", "INFO", "DEBUG"] + + +def supported_batches() -> list[str]: + """List of batches supported by the app.""" + return [ + "segment", + "simple", + "segment_long", + "segment_enigma", + "resample", + "get_IQR", + "get_TIV", + "get_quality", + "get_ROI_values", + ] + + +# cat_standalone_simple.m +# cat_standalone_segment.m +# cat_standalone_segment_enigma.m +# cat_standalone_segment_long.m + +# cat_standalone_resample.m +# cat_standalone_get_IQR.m +# cat_standalone_get_TIV.m +# cat_standalone_get_quality.m +# cat_standalone_get_ROI_values.m + +# WON'T DO +# cat_standalone_smooth.m +# cat_standalone_tfce.m +# cat_standalone_dicom2nii.m +# cat_standalone_deface.m diff --git a/code/exit_codes.json b/code/exit_codes.json new file mode 100644 index 0000000..9258b78 --- /dev/null +++ b/code/exit_codes.json @@ -0,0 +1,50 @@ +{ + "SUCCESS": { + "Value": 0, + "Description": "The program completed successfully." + }, + "FAILURE": { + "Value": 1, + "Description": "The program failed for unspecified reasons." + }, + "INVALID": { + "Value": 16, + "Description": "An input dataset failed BIDS validation." + }, + "17": { + "Value": 17, + "Description": "Unknown analysis level." + }, + "18": { + "Value": 18, + "Description": "Entity-based filtering options selected no files." + }, + "19": { + "Value": 19, + "Description": "Both command-line arguments and a parameter invocation file were passed to the application." + }, + "USAGE": { + "Value": 64, + "Description": "The command was used incorrectly." + }, + "DATAERR": { + "Value": 65, + "Description": "The input data was incorrect in some way." + }, + "NOINPUT": { + "Value": 66, + "Description": "The input data was missing or unreadable." + }, + "CANTCREAT": { + "Value": 73, + "Description": "An output file/directory cannot be created." + }, + "IOERR": { + "Value": 64, + "Description": "Failure during file reading/writing." + }, + "TEMPFAIL": { + "Value": 75, + "Description": "Temporary failure. Another run is expected to succeed." + } +} diff --git a/code/main.py b/code/main.py new file mode 100644 index 0000000..3136723 --- /dev/null +++ b/code/main.py @@ -0,0 +1,311 @@ +"""Run CAT12 BIDS app.""" + +import json +import os +import shutil +import subprocess +import sys +from datetime import datetime +from pathlib import Path +from subprocess import PIPE, STDOUT, Popen + +import nibabel as nib +from _parsers import common_parser +from _version import __version__ +from bids_utils import ( + get_dataset_layout, + init_derivatives_layout, + list_subjects, +) +from cat_logging import cat12_log +from defaults import log_levels +from methods import generate_method_section +from rich import print +from rich_argparse import RichHelpFormatter +from utils import progress_bar + +env = os.environ +env["PYTHONUNBUFFERED"] = "True" + +argv = sys.argv + +with open(Path(__file__).parent / "exit_codes.json") as f: + EXIT_CODES = json.load(f) + +# Get environment variable +STANDALONE = Path(os.getenv("STANDALONE")) + +logger = cat12_log(name="cat12") + + +def main(): + """Run the app.""" + parser = common_parser(formatter_class=RichHelpFormatter) + + args = parser.parse_args(argv[1:]) + + verbose = args.verbose + if isinstance(verbose, list): + verbose = verbose[0] + + log_level_name = log_levels()[int(verbose)] + logger.setLevel(log_level_name) + + output_dir = Path(args.output_dir[0]) + + command = args.command + + if command == "help": + subprocess.run([STANDALONE / "cat_standalone.sh"]) + sys.exit(EXIT_CODES["SUCCESS"]["Value"]) + + elif command == "copy": + target = args.target[0] + + output_dir.mkdir(exist_ok=True, parents=True) + + if target == "all": + files = STANDALONE.glob("*.m") + else: + files = [STANDALONE / f"cat_standalone_{target}.m"] + + for source_file in files: + logger.info(f"Copying {source_file} to {str(output_dir)}") + shutil.copy(source_file, output_dir) + + sys.exit(EXIT_CODES["SUCCESS"]["Value"]) + + elif command == "view": + target = args.target[0] + source_file = STANDALONE / f"cat_standalone_{target}.m" + with source_file.open("r") as file: + print(f"[green]{file.read()}") + + sys.exit(EXIT_CODES["SUCCESS"]["Value"]) + + bids_dir = Path(args.bids_dir[0]) + if not bids_dir.exists(): + logger.error( + f"The following 'bids_dir' could not be found:\n{bids_dir}" + ) + sys.exit(EXIT_CODES["DATAERR"]["Value"]) + + if not args.skip_validation: + run_validation(bids_dir) + + layout_in = get_dataset_layout(bids_dir) + + subjects = args.participant_label or layout_in.get_subjects() + subjects = list_subjects(layout_in, subjects) + + analysis_level = args.analysis_level[0] + if analysis_level == "group": + logger.error("'group' level analysis not implemented yet.") + sys.exit(EXIT_CODES["FAILURE"]["Value"]) + + if command == "segment": + + segment_type = args.type + if isinstance(segment_type, list): + segment_type = segment_type[0] + + output_dir = output_dir / f"CAT12_{__version__}" + + if segment_type != "enigma": + copy_files(layout_in, output_dir, subjects) + layout_out = init_derivatives_layout(output_dir) + else: + OUTPUT_DIR = os.path.relpath(output_dir, bids_dir) + os.environ["OUTPUT_DIR"] = OUTPUT_DIR + layout_out = layout_in + + batch = define_batch(segment_type=segment_type) + + logger.info(f"{segment_type=} - using batch {batch}.") + + (output_dir / "logs").mkdir(exist_ok=True, parents=True) + shutil.copy2( + src=Path("/opt") + / f'CAT12{os.environ["CAT_VERSION"]}' + / "standalone" + / batch, + dst=output_dir / "logs", + ) + + generate_method_section(output_dir=output_dir, batch=batch) + + text = "processing subjects" + with progress_bar(text=text) as progress: + + subject_loop = progress.add_task( + description="processing subjects", total=len(subjects) + ) + + for subject_label in subjects: + + this_filter = { + "datatype": "anat", + "suffix": "T1w", + "extension": "nii", + "subject": subject_label, + } + + bf = layout_out.get( + **this_filter, + ) + + if not check_input(subject_label, bf, segment_type): + continue + + log_file = log_filename(output_dir, subject_label) + + cmd = [str(STANDALONE / "cat_standalone.sh")] + + with log_file.open("w") as log: + if segment_type in ["default", "simple", "enigma"]: + for file in bf: + cmd.extend([file.path, "-b", batch]) + run_command(cmd, log) + + elif is_longitudinal_segmentation(segment_type): + # TODO do a mean for each time point first + files_to_process = [file.path for file in bf] + cmd.extend(files_to_process) + cmd.extend(["-b", batch, "-a1", segment_type[-1]]) + run_command(cmd, log) + + gunzip_all_niftis( + output_dir=output_dir, subject_label=subject_label + ) + + progress.update(subject_loop, advance=1) + + sys.exit(EXIT_CODES["SUCCESS"]["Value"]) + + +def check_input(subject_label: str, bf: list, segment_type: str): + """Check number of input files.""" + if len(bf) < 1: + logger.warning(f"No data found for subject {subject_label}.") + return False + if is_longitudinal_segmentation(segment_type) and len(bf) < 2: + logger.warning( + ( + "Longitudinal segmentation requested " + f"but subject {subject_label} only has 1 image." + ) + ) + return True + + +def define_batch(segment_type): + """Find batch to run.""" + batch = "cat_standalone_segment.m" + if segment_type == "simple": + batch = "cat_standalone_simple.m" + elif is_longitudinal_segmentation(segment_type): + batch = "cat_standalone_segment_long.m" + elif segment_type == "enigma": + batch = "cat_standalone_segment_enigma.m" + return batch + + +def log_filename(output_dir, subject_label): + """Generate filename for logfile.""" + now = datetime.now().replace(microsecond=0).isoformat() + log_file = ( + output_dir + / f"sub-{subject_label}" + / "log" + / f"{now}_sub-{subject_label}.log".replace(":", "_") + ) + log_file.parent.mkdir(parents=True, exist_ok=True) + return log_file + + +def is_longitudinal_segmentation(segment_type): + """Check if the segmentation requested is longitudinal.""" + return segment_type in [ + "long_0", + "long_1", + "long_2", + "long_3", + ] + + +def run_command(cmd, log): + """Run command and log to STDOUT and log.""" + logger.info(cmd) + with Popen( + cmd, + stdout=PIPE, + stderr=STDOUT, + bufsize=1, + text=True, + encoding="utf-8", + env=env, + ) as proc: + while (_ := proc.poll()) is None: + line = proc.stdout.readline() + sys.stdout.write(str(line)) + log.write(str(line)) + + +def copy_files(layout_in, output_dir, subjects): + """Copy input files to derivatives. + + SPM has the bad habit of dumping derivatives with the raw. + Unzip files as SPM cannot deal with gz files. + """ + text = "copying subjects" + with progress_bar(text=text) as progress: + copy_loop = progress.add_task( + description="copying subjects", total=len(subjects) + ) + for subject_label in subjects: + logger.info(f"Copying {subject_label}") + + this_filter = { + "datatype": "anat", + "suffix": "T1w", + "extension": "nii.*", + "subject": subject_label, + } + bf = layout_in.get( + **this_filter, + regex_search=True, + ) + for file in bf: + output_filename = output_dir / file.relpath + if output_filename.exists(): + continue + + output_filename.parent.mkdir(exist_ok=True, parents=True) + + logger.info(f"Copying {file.path} to {str(output_dir)}") + img = nib.load(file.path) + nib.save(img, output_filename) + + progress.update(copy_loop, advance=1) + + +def run_validation(bids_dir): + """Run bids validator.""" + try: + subprocess.run(f"bids-validator {bids_dir}", shell=True, check=True) + except subprocess.CalledProcessError: + sys.exit(EXIT_CODES["DATAERR"]["Value"]) + + +def gunzip_all_niftis(output_dir: Path, subject_label: str): + """Gunzip all niftis for a subject.""" + logger.info(f"Gunzipping files for {subject_label}") + files = [x for x in (output_dir / f"sub-{subject_label}").glob("**/*.nii")] + for f in files: + nii = nib.load(f) + nii.to_filename(str(f) + ".gz") + f.unlink() + + +if __name__ == "__main__": + main() diff --git a/code/methods.py b/code/methods.py new file mode 100644 index 0000000..51f3bdf --- /dev/null +++ b/code/methods.py @@ -0,0 +1,38 @@ +"""Module responsible for generating method section.""" + +from pathlib import Path + +from _version import __version__ +from defaults import CAT_VERSION, MCR_VERSION +from jinja2 import Environment, FileSystemLoader, select_autoescape + + +def generate_method_section( + output_dir: Path, + version: str = __version__, + cat_version: str = CAT_VERSION, + mcr_version: str = MCR_VERSION, + batch: str = None, +) -> None: + """Add a method section to the output dataset.""" + env = Environment( + loader=FileSystemLoader(Path(__file__).parent), + autoescape=select_autoescape(), + lstrip_blocks=True, + trim_blocks=True, + ) + + template = env.get_template("data/methods/template.jinja") + + output_file = output_dir / "logs" / "CITATION.md" + output_file.parent.mkdir(parents=True, exist_ok=True) + + data = { + "version": version, + "cat_version": cat_version, + "mcr_version": mcr_version, + "batch": batch, + } + + with open(output_file, "w") as f: + print(template.render(data=data), file=f) diff --git a/code/requirements.txt b/code/requirements.txt new file mode 100644 index 0000000..e7f42b7 --- /dev/null +++ b/code/requirements.txt @@ -0,0 +1,4 @@ +rich_argparse +pybids +nibabel +jinja2 diff --git a/code/utils.py b/code/utils.py new file mode 100644 index 0000000..06ad82c --- /dev/null +++ b/code/utils.py @@ -0,0 +1,45 @@ +"""Utility functions.""" + +from __future__ import annotations + +from pathlib import Path + +from cat_logging import cat12_log +from rich.progress import ( + BarColumn, + MofNCompleteColumn, + Progress, + SpinnerColumn, + TaskProgressColumn, + TextColumn, + TimeElapsedColumn, + TimeRemainingColumn, +) + +logger = cat12_log(name="cat12") + + +def progress_bar(text: str, color: str = "green") -> Progress: + """Return a rich progress bar instance.""" + return Progress( + TextColumn(f"[{color}]{text}"), + SpinnerColumn("dots"), + TimeElapsedColumn(), + BarColumn(), + MofNCompleteColumn(), + TaskProgressColumn(), + TimeRemainingColumn(), + ) + + +def create_dir_if_absent(output_path: str | Path) -> None: + """Create a path if it does not exist. + + :param output_path: + :type output_path: Union[str, Path] + """ + if isinstance(output_path, str): + output_path = Path(output_path) + if not output_path.is_dir(): + logger.debug(f"Creating dir: {output_path}") + output_path.mkdir(parents=True, exist_ok=True) diff --git a/docs/.gitattributes b/docs/.gitattributes new file mode 100644 index 0000000..8bb0167 --- /dev/null +++ b/docs/.gitattributes @@ -0,0 +1 @@ +* annex.largefiles=nothing diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d0c3cbf --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000..dc1312a --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..cef871d --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,35 @@ +"""Conf for documentation.""" + +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = "cat12" +copyright = "2024, cat12" +author = "cat12" +release = "0.1.0" + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [] + +templates_path = ["_templates"] +exclude_patterns = [] + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_theme = "pydata_sphinx_theme" +html_static_path = ["_static"] + +extensions = [ + "sphinx_copybutton", + "sphinx_togglebutton", + "myst_parser", + "sphinxarg.ext", +] diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..1324a33 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,19 @@ +Welcome to cat12 BIDS app documentation! +======================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + +Command line API +---------------- + +.. argparse:: + :ref: cat12._parsers.common_parser + :prog: common_parser + + +.. argparse:: + :module: cat12._parsers + :func: common_parser + :prog: common_parser diff --git a/makefile b/makefile new file mode 100644 index 0000000..51a2619 --- /dev/null +++ b/makefile @@ -0,0 +1,40 @@ +tests/data/MoAEpilot: + mkdir -p tests/data + curl -fL -o moae.zip https://www.fil.ion.ucl.ac.uk/spm/download/data/MoAEpilot/MoAEpilot.bids.zip + unzip -q moae.zip -d tests/data + rm -f moae.zip + +build: + docker build . --tag cat12 + +version: + docker run --rm -it cat12 . . participant --version + +view: + docker run --rm -it cat12 . . participant view segment_enigma --verbose 3 + +copy: + docker run --rm -it cat12 . /foo participant copy segment --verbose 3 + +segment: tests/data/MoAEpilot + docker run --rm -it -v $${PWD}/tests/data/MoAEpilot:/data cat12 /data /data/derivatives participant segment --verbose 3 --type default + +enigma: tests/data/MoAEpilot + docker run --rm -it -v $${PWD}/tests/data/MoAEpilot:/data cat12 /data /data/derivatives participant segment --verbose 3 --type enigma + +simple: tests/data/MoAEpilot + docker run --rm -it -v $${PWD}/tests/data/MoAEpilot:/data cat12 /data /data/derivatives participant segment --verbose 3 --type simple + +data_ds002799: + mkdir -p tests/data + cd tests/data && datalad install ///openneuro/ds002799 + cd tests/data/ds002799 && datalad get sub-2*/*/*/*T1w* -J 12 + +ds002799: data_ds002799 + mkdir -p tests/data/outputs + docker run --rm -it \ + -v $${PWD}/tests/data:/data \ + cat12 /data/ds002799 /data/outputs/ds002799 participant \ + segment \ + --type long_2 \ + --participant_label 292 294 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..2bf3137 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,36 @@ +[build-system] +build-backend = "hatchling.build" +requires = ["hatchling"] + +[project] +version = "0.1.0" +dependencies = ["rich_argparse", "pybids", "nibabel"] +license = { text = "MIT" } +name = "cat12" +readme = "README.md" +requires-python = ">=3.9" + +[project.optional-dependencies] +docs = [ + "myst-parser", + "sphinx", + "sphinx-argparse", + "sphinx-copybutton", + "pydata-sphinx-theme", + "sphinx-togglebutton", +] + +[tool.hatch.build.targets.wheel] +packages = ["code"] + +[tool.black] +line-length = 79 + +[tool.codespell] +builtin = "clear,rare" + +[tool.isort] +combine_as_imports = true +line_length = 79 +profile = "black" +skip_gitignore = true