From e184803bd771f5d16c10bea876fce7bb6ea0fa79 Mon Sep 17 00:00:00 2001 From: Jakub Kaczmarzyk Date: Sun, 1 Oct 2017 21:41:27 -0400 Subject: [PATCH] Add/fsl eddy 5011 (#97) * add: fsl eddy v5.0.11 pre-release * fix: list cuda versions with commas * enh: option to install fsl eddy 5.0.11 pre-release over fsl 5.0.10 * remove trailing whitespace * fix: fsl apt deps + fsl tests * remove eddy_openmp from test because it returns err code 1 --- README.md | 2 + neurodocker/generate.py | 4 +- neurodocker/interfaces/fsl.py | 89 ++++++++++++++++++++---- neurodocker/interfaces/tests/test_fsl.py | 10 +-- neurodocker/interfaces/tests/utils.py | 3 + neurodocker/neurodocker.py | 32 +++++---- 6 files changed, 104 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index fada45e8..c14c7a8d 100644 --- a/README.md +++ b/README.md @@ -55,6 +55,8 @@ Valid options for each software package are the keyword arguments for the class | | license_path | Relative path to license file. If provided, this file will be copied into the Docker image. Must be within the build context. | | | min | If true, install a version of FreeSurfer minimized for recon-all. See [freesurfer/freesurfer#70](https://github.com/freesurfer/freesurfer/issues/70). False by default. | | **FSL**** | version* | Any version for which binaries are provided. | +| | eddy_5011 | If true, use pre-release version of FSL eddy v5.0.11 | +| | eddy_5011_cuda | 6.5, 7.0, 7.5, 8.0; only valid if using eddy pre-release | | | use_binaries | If true (default), use pre-compiled binaries. Building from source is not available now but might be added in the future. | | | use_installer | If true, use FSL's Python installer. Only valid on CentOS images. | | **MINC** | version* | 1.9.15 | diff --git a/neurodocker/generate.py b/neurodocker/generate.py index 85dabfbe..1cec9f83 100644 --- a/neurodocker/generate.py +++ b/neurodocker/generate.py @@ -308,8 +308,8 @@ def _add_neurodocker_header(specs): """Return Dockerfile comment that references Neurodocker.""" return ("# Generated by Neurodocker v{}." "\n#" - "\n# Thank you for using Neurodocker. If you discover any issues " - "\n# or ways to improve this software, please submit an issue or " + "\n# Thank you for using Neurodocker. If you discover any issues" + "\n# or ways to improve this software, please submit an issue or" "\n# pull request on our GitHub repository:" "\n# https://github.com/kaczmarj/neurodocker" "\n#" diff --git a/neurodocker/interfaces/fsl.py b/neurodocker/interfaces/fsl.py index 9049b468..e9585767 100644 --- a/neurodocker/interfaces/fsl.py +++ b/neurodocker/interfaces/fsl.py @@ -33,21 +33,29 @@ class FSL(object): use_installer : bool If true, install FSL using FSL's Python installer. Only works on CentOS/RHEL (default false). + eddy_5011 : bool + If true, install pre-release of FSL eddy v5.0.11. + eddy_5011_cuda : {'6.5', '7.0', '7.5', '8.0'} + Version of CUDA for FSL eddy pre-release. Only applies if eddy_5011 is + true. check_urls : bool If true, raise error if a URL used by this class responds with an error code. Notes ----- - Look into ReproNim/simple_workflow to learn how to install specific versions - of FSL on Debian (https://github.com/ReproNim/simple_workflow). + Look into ReproNim/simple_workflow to learn how to install specific + versions of FSL on Debian (https://github.com/ReproNim/simple_workflow). """ def __init__(self, version, pkg_manager, use_binaries=True, - use_installer=False, check_urls=True): + use_installer=False, eddy_5011=False, eddy_5011_cuda=None, + check_urls=True): self.version = LooseVersion(version) self.pkg_manager = pkg_manager self.use_binaries = use_binaries self.use_installer = use_installer + self.eddy_5011 = eddy_5011 + self.eddy_5011_cuda = eddy_5011_cuda self.check_urls = check_urls self._check_args() @@ -60,8 +68,11 @@ def _check_args(self): if self.use_binaries and self.use_installer: raise ValueError("More than one installation method specified.") if self.use_installer and self.pkg_manager != 'yum': - raise ValueError("FSL's Python installer works only on " - "CentOS/RHEL-based systems.") + raise ValueError("FSL's Python installer works only on" + " CentOS/RHEL-based systems.") + if self.version < LooseVersion('5.0.10') and self.eddy_5011: + raise ValueError("Pre-release of FSL eddy can only be installed" + " with FSL v5.0.10.") return True def _create_cmd(self): @@ -71,8 +82,9 @@ def _create_cmd(self): "\n# FSL is non-free. If you are considering commerical use" "\n# of this Docker image, please consult the relevant license:" "\n# https://fsl.fmrib.ox.ac.uk/fsl/fslwiki/Licence" - "\n#-----------------------------------------------------------" - "".format(self.version)) + "\n#-----------------------------------------------------------") + comment = comment.format(self.version) + if self.use_binaries: url = self._get_binaries_url() cmd = self.install_binaries(url) @@ -116,11 +128,14 @@ def _get_binaries_url(self): return url def _install_binaries_deps(self): - """Return command to install FreeSurfer dependencies. Use this for - FreeSurfer binaries, not if attempting to build FreeSurfer from source. - """ - pkgs = {'apt': "bc dc", - 'yum': "bc"} + """Return command to install FSL dependencies.""" + pkgs = {'apt': ("bc dc libfontconfig1 libfreetype6 libgl1-mesa-dev" + " libglu1-mesa-dev libgomp1 libice6 libmng1" + " libxcursor1 libxft2 libxinerama1 libxrandr2" + " libxrender1 libxt6"), + 'yum': ("bc libGL libGLU libgomp libICE libjpeg libmng" + " libpng12 libSM libX11 libXcursor libXext libXft" + " libXinerama libXrandr libXt")} cmd = "{install}\n&& {clean}".format(**manage_pkgs[self.pkg_manager]) return cmd.format(pkgs=pkgs[self.pkg_manager]) @@ -138,13 +153,16 @@ def install_binaries(self, url): if self.version >= LooseVersion('5.0.10'): fsl_python = "/opt/fsl/etc/fslconf/fslpython_install.sh" - cmd += "\n&& /bin/bash {} -q -f /opt/fsl".format(fsl_python) + cmd += "\n&& /bin/bash {} -q -f /opt/fsl".format(fsl_python) + + if self.eddy_5011: + cmd += self._install_eddy_5011() ent_cmds = ["echo Some packages in this Docker container are non-free", ("echo If you are considering commercial use of this" " container, please consult the relevant license:"), - "echo https://fsl.fmrib.ox.ac.uk/fsl/fslwiki/Licence", - "source $FSLDIR/etc/fslconf/fsl.sh",] + "echo https://fsl.fmrib.ox.ac.uk/fsl/fslwiki/Licence", + "source $FSLDIR/etc/fslconf/fsl.sh"] cmd += "\n&& {}".format(_add_to_entrypoint(ent_cmds, with_run=False)) cmd = indent("RUN", cmd) @@ -153,3 +171,44 @@ def install_binaries(self, url): env_cmd = indent("ENV", env_cmd) return "\n".join((cmd, env_cmd)) + + def _get_eddy_5011_url(self): + """Return URL of FSL eddy 5.0.11 pre-release.""" + # This function should probably be removed once FSL v5.0.11 is released + base_url = ("https://fsl.fmrib.ox.ac.uk/fsldownloads/patches/" + "eddy-patch-fsl-5.0.11/centos6/") + cuda_versions = { + '6.5': 'eddy_cuda6.5', + '7.0': 'eddy_cuda7.0', + '7.5': 'eddy_cuda7.5', + '8.0': 'eddy_cuda8.0', + } + if self.eddy_5011_cuda is None: + filename = "eddy_openmp" + else: + filename = cuda_versions.get(self.eddy_5011_cuda, None) + if filename is None: + raise ValueError("Valid CUDA versions are {}" + .format(', '.join(cuda_versions.keys()))) + return urljoin(base_url, filename) + + def _install_eddy_5011(self): + """Return Dockerfile instructions to install FSL eddy v5.0.11 + pre-release. + """ + url = self._get_eddy_5011_url() + + if self.check_urls: + check_url(url) + + cmd = ('\n&& cd /opt/fsl/bin' + '\n&& rm -f eddy_openmp eddy_cuda*' + '\n&& echo "Downloading FSL eddy v5.0.11 pre-release ..."' + '\n&& curl -sSLO {}' + '\n&& chmod +x eddy_*').format(url) + + filename = url.split('/')[-1] + if 'cuda' in filename: + cmd += '\n&& ln -sv {} eddy_cuda'.format(filename) + + return cmd diff --git a/neurodocker/interfaces/tests/test_fsl.py b/neurodocker/interfaces/tests/test_fsl.py index 466fce50..e68ec919 100644 --- a/neurodocker/interfaces/tests/test_fsl.py +++ b/neurodocker/interfaces/tests/test_fsl.py @@ -33,17 +33,19 @@ def test_build_image_fsl_latest_pyinstaller_centos7(self): if push: utils.push_image(image_name) - def test_build_image_fsl_509_binaries_centos7(self): + def test_build_image_fsl_5010_binaries_centos7(self): """Install FSL binaries on CentOS 7.""" specs = {'pkg_manager': 'yum', 'check_urls': True, 'instructions': [ - ('base', 'centos:7'), - ('fsl', {'version': '5.0.9', 'use_binaries': True}) + ('base', 'centos:7'), + ('fsl', {'version': '5.0.10', + 'use_binaries': True, + 'eddy_5011': True}) ]} df = Dockerfile(specs).cmd - dbx_path, image_name = utils.DROPBOX_DOCKERHUB_MAPPING['fsl-5.0.9_centos7'] + dbx_path, image_name = utils.DROPBOX_DOCKERHUB_MAPPING['fsl-5.0.10_centos7'] image, push = utils.get_image_from_memory(df, dbx_path, image_name) cmd = "bash /testscripts/test_fsl.sh" diff --git a/neurodocker/interfaces/tests/utils.py b/neurodocker/interfaces/tests/utils.py index 299813dd..7bdd0323 100644 --- a/neurodocker/interfaces/tests/utils.py +++ b/neurodocker/interfaces/tests/utils.py @@ -28,6 +28,9 @@ 'fsl-5.0.9_centos7': ('/Dockerfile.FSL-5.0.9_centos7', 'kaczmarj/fsl:5.0.9_centos7'), + 'fsl-5.0.10_centos7': ('/Dockerfile.FSL-5.0.10_centos7', + 'kaczmarj/fsl:5.0.10_centos7'), + 'miniconda_centos7': ('/Dockerfile.Miniconda-latest_centos7', 'kaczmarj/miniconda:latest_centos7'), diff --git a/neurodocker/neurodocker.py b/neurodocker/neurodocker.py index 3ccf5608..ed54d963 100644 --- a/neurodocker/neurodocker.py +++ b/neurodocker/neurodocker.py @@ -21,7 +21,7 @@ # https://stackoverflow.com/a/9028031/5666087 class OrderedArgs(Action): def __call__(self, parser, namespace, values, option_string=None): - if not 'ordered_args' in namespace: + if 'ordered_args' not in namespace: setattr(namespace, 'ordered_args', []) previous = namespace.ordered_args previous.append((self.dest, values)) @@ -38,11 +38,11 @@ def list_of_kv(kv): l[1:] = ["=".join(l[1:])] return l - p.add_argument("-b", "--base", #required=True, - help="Base Docker image. Eg, ubuntu:17.04") - p.add_argument("-p", "--pkg-manager", #required=True, - choices=utils.manage_pkgs.keys(), - help="Linux package manager.") + p.add_argument("-b", "--base", # required=True, + help="Base Docker image. Eg, ubuntu:17.04") + p.add_argument("-p", "--pkg-manager", # required=True, + choices=utils.manage_pkgs.keys(), + help="Linux package manager.") # Arguments that should be ordered. p.add_argument('--add', action=OrderedArgs, nargs="+", @@ -93,8 +93,7 @@ def list_of_kv(kv): p.add_argument('--no-print-df', dest='no_print_df', action="store_true", help="Do not print the Dockerfile") p.add_argument("--no-check-urls", action="store_false", dest="check_urls", - help=("Do not verify communication with URLs used in " - "the build.")) + help="Do not verify communication with URLs used in the build.") _ndeb_servers = ", ".join(SUPPORTED_SOFTWARE['neurodebian'].SERVERS.keys()) @@ -145,7 +144,8 @@ def list_of_kv(kv): "Install SPM (and its dependency, Matlab Compiler Runtime). Valid" " keys are version and matlab_version."), "minc": ( - "Install MINC. Valid keys is version (required). Only version 1.9.15 is supported at this time."), + "Install MINC. Valid keys is version (required). Only version" + " 1.9.15 is supported at this time."), "petpvc": ( "Install PETPVC. Valid keys are version (required)."), } @@ -165,11 +165,12 @@ def _add_reprozip_trace_arguments(parser): """Add arguments to `parser` for sub-command `reprozip-trace`.""" p = parser p.add_argument('container', - help="Running container in which to trace commands.") + help="Running container in which to trace commands.") p.add_argument('commands', nargs='+', help="Command(s) to trace.") p.add_argument('--dir', '-d', dest="packfile_save_dir", default=".", - help=("Directory in which to save pack file. Default " - "is current directory.")) + help=("Directory in which to save pack file. Default " + "is current directory.")) + def _add_reprozip_merge_arguments(parser): """Add arguments to `parser` for sub-command `reprozip-merge`.""" @@ -177,9 +178,10 @@ def _add_reprozip_merge_arguments(parser): p.add_argument('outfile', help="Filepath to merged pack file.") p.add_argument('pack_files', nargs='+', help="Pack files to merge.") + def create_parser(): """Return command-line argument parser.""" - parser = ArgumentParser(description=__doc__, #add_help=False, + parser = ArgumentParser(description=__doc__, # add_help=False, formatter_class=RawDescriptionHelpFormatter) verbosity_choices = ('debug', 'info', 'warning', 'error', 'critical') @@ -245,7 +247,7 @@ def reprozip_merge(namespace): def _validate_args(namespace): if (namespace.file is None and - (namespace.base is None or namespace.pkg_manager is None)): + (namespace.base is None or namespace.pkg_manager is None)): raise ValueError("-b/--base and -p/--pkg-manager are required if not" " generating from JSON file.") @@ -267,7 +269,7 @@ def main(args=None): subparser_functions = {'generate': generate, 'reprozip-trace': reprozip_trace, - 'reprozip-merge': reprozip_merge,} + 'reprozip-merge': reprozip_merge} if namespace.subparser_name not in subparser_functions.keys(): print(__doc__)