diff --git a/.gitignore b/.gitignore index 4cf7ecc9..028772ae 100644 --- a/.gitignore +++ b/.gitignore @@ -146,3 +146,4 @@ cython_debug/ # Pycharm .idea/ +venvs diff --git a/neurodocker/cli/tests/test_build_images_with_cli.py b/neurodocker/cli/tests/test_build_images_with_cli.py index f5444535..71ac83fe 100644 --- a/neurodocker/cli/tests/test_build_images_with_cli.py +++ b/neurodocker/cli/tests/test_build_images_with_cli.py @@ -26,7 +26,14 @@ ], ) @pytest.mark.parametrize( - ["pkg_manager", "base_image"], [("apt", "debian:buster-slim"), ("yum", "centos:7")] + ["pkg_manager", "base_image"], + [ + ("apt", "debian:buster-slim"), + ("yum", "centos:7"), + # TODO: make it work -- seems has difficulty installing jq, + # may be curl isn't there or alike + # ("portage", "gentoo"), + ], ) def test_build_image_from_registered( tmp_path: Path, cmd: str, pkg_manager: str, base_image: str @@ -113,3 +120,27 @@ def test_json_roundtrip(cmd: str, inputs: str, tmp_path: Path): stdout, _ = run_fn(img, args=["env"]) assert "CAT=FOO" in stdout assert "DOG=BAR" in stdout + + +def test_gentoo_image(tmp_path: Path): + # TODO: also add singularity like in the test above + + cmd = "docker" + + _TemplateRegistry._reset() + runner = CliRunner() + result = runner.invoke( + generate, + [ + cmd, + "--pkg-manager", + "portage", + "--base-image", + "gentoo", + "--install", + "app-misc/mime-types", + ], + ) + assert result.exit_code == 0, result.output + (tmp_path / "specs.json").write_text(result.output) + # TODO: add more testing here of the result that it is usable diff --git a/neurodocker/cli/tests/test_cli.py b/neurodocker/cli/tests/test_cli.py index 88dcf475..c787b12b 100644 --- a/neurodocker/cli/tests/test_cli.py +++ b/neurodocker/cli/tests/test_cli.py @@ -19,7 +19,7 @@ def test_fail_on_empty_args(cmd: str): @pytest.mark.parametrize("cmd", _cmds) -@pytest.mark.parametrize("pkg_manager", ["apt", "yum"]) +@pytest.mark.parametrize("pkg_manager", ["apt", "portage", "yum"]) def test_fail_on_no_base(cmd: str, pkg_manager: str): runner = CliRunner() result = runner.invoke(generate, [cmd, "--pkg-manager", pkg_manager]) @@ -34,7 +34,7 @@ def test_fail_on_no_pkg_manager(cmd: str): @pytest.mark.parametrize("cmd", _cmds) -@pytest.mark.parametrize("pkg_manager", ["apt", "yum"]) +@pytest.mark.parametrize("pkg_manager", ["apt", "portage", "yum"]) def test_minimal_args(cmd: str, pkg_manager: str): runner = CliRunner() result = runner.invoke( @@ -107,7 +107,7 @@ def test_optioneatall_type_issue_498(): @pytest.mark.parametrize("cmd", _cmds) -@pytest.mark.parametrize("pkg_manager", ["apt", "yum"]) +@pytest.mark.parametrize("pkg_manager", ["apt", "portage", "yum"]) def test_all_args(cmd: str, pkg_manager: str): runner = CliRunner() result = runner.invoke( @@ -156,7 +156,7 @@ def test_all_args(cmd: str, pkg_manager: str): # is what registers the templates. Using the `docker` function # (`reproenv generate docker`) directly does not fire `generate`. @pytest.mark.parametrize("cmd", _cmds) -@pytest.mark.parametrize("pkg_manager", ["apt", "yum"]) +@pytest.mark.parametrize("pkg_manager", ["apt", "portage", "yum"]) def test_render_registered(cmd: str, pkg_manager: str): template_path = Path(__file__).parent runner = CliRunner(env={"REPROENV_TEMPLATE_PATH": str(template_path)}) diff --git a/neurodocker/reproenv/renderers.py b/neurodocker/reproenv/renderers.py index 2df1a4c5..0d4429fe 100644 --- a/neurodocker/reproenv/renderers.py +++ b/neurodocker/reproenv/renderers.py @@ -518,10 +518,22 @@ def entrypoint(self, args: list[str]) -> DockerRenderer: @_log_instruction def from_(self, base_image: str, as_: str = None) -> DockerRenderer: """Add a Dockerfile `FROM` instruction.""" - if as_ is None: - s = "FROM " + base_image + if base_image == "gentoo": + # TODO (if we can, likely not): + # make neurodocker argument "portage_date" for which we + # figure out the corresponding stage3 date and hashes. + s = "FROM docker.io/gentoo/portage:20240529 as portage\n" + s += "FROM docker.io/gentoo/stage3:20240527\n" + s += "COPY --from=portage /var/db/repos/gentoo /var/db/repos/gentoo\n" + # TODO: figure out hashes for the date + # s += "ARG gentoo_hash=2d25617a1d085316761b06c17a93ec972f172fc6\n" + # s += "ARG science_hash=73916dd3680ffd92e5bd3d32b262e5d78c86a448\n" + s += 'ARG FEATURES="-ipc-sandbox -network-sandbox -pid-sandbox"\n' else: - s = f"FROM {base_image} AS {as_}" + if as_ is None: + s = "FROM " + base_image + else: + s = f"FROM {base_image} AS {as_}" self._parts.append(s) return self @@ -753,6 +765,8 @@ def _indent_run_instruction(string: str, indent=4) -> str: def _install(pkgs: list[str], pkg_manager: str, opts: str = None) -> str: if pkg_manager == "apt": return _apt_install(pkgs, opts) + elif pkg_manager == "portage": + return _portage_install(pkgs, opts) elif pkg_manager == "yum": return _yum_install(pkgs, opts) # TODO: add debs here? @@ -802,6 +816,25 @@ def install_one(url: str): return s +def _portage_install(pkgs: list[str], opts: str = None, sort=True) -> str: + """Return command to install packages with `portage` (Gentoo). + + `opts` are options passed to `emerge`. + Default is "--autounmask-continue". + """ + pkgs = sorted(pkgs) if sort else pkgs + opts = "--autounmask-continue" if opts is None else opts + + s = """\ +emerge {opts} \\ + {pkgs} +rm -rf /var/tmp/portage/* +""".format( + opts=opts, pkgs=" \\\n ".join(pkgs) + ) + return s.strip() + + def _yum_install(pkgs: list[str], opts: str = None, sort=True) -> str: """Return command to install packages with `yum` (CentOS, Fedora). diff --git a/neurodocker/reproenv/schemas/renderer.json b/neurodocker/reproenv/schemas/renderer.json index 169fa1f9..6597d7ae 100644 --- a/neurodocker/reproenv/schemas/renderer.json +++ b/neurodocker/reproenv/schemas/renderer.json @@ -11,10 +11,12 @@ "type": "string", "enum": [ "apt", + "portage", "yum" ], "examples": [ "apt", + "portage", "yum" ] }, diff --git a/neurodocker/reproenv/types.py b/neurodocker/reproenv/types.py index 31f61e6d..6cab0f83 100644 --- a/neurodocker/reproenv/types.py +++ b/neurodocker/reproenv/types.py @@ -18,8 +18,8 @@ allowed_installation_methods = {"binaries", "source"} installation_methods_type = Literal["binaries", "source"] -allowed_pkg_managers = {"apt", "yum"} -pkg_managers_type = Literal["apt", "yum"] +allowed_pkg_managers = {"apt", "portage", "yum"} +pkg_managers_type = Literal["apt", "portage", "yum"] # Cross-reference the dictionary types below with the JSON schemas. @@ -32,6 +32,7 @@ class _InstallationDependenciesType(TypedDict, total=False): apt: list[str] debs: list[str] + portage: list[str] yum: list[str] diff --git a/neurodocker/templates/afni.yaml b/neurodocker/templates/afni.yaml index 61247abe..288eabbe 100644 --- a/neurodocker/templates/afni.yaml +++ b/neurodocker/templates/afni.yaml @@ -72,6 +72,8 @@ binaries: - which - unzip - ncurses-compat-libs + # portage: + # - sci-biology/ants debs: - http://mirrors.kernel.org/debian/pool/main/libx/libxp/libxp6_1.0.2-2_amd64.deb - http://snapshot.debian.org/archive/debian-security/20160113T213056Z/pool/updates/main/libp/libpng/libpng12-0_1.2.49-1%2Bdeb7u2_amd64.deb diff --git a/neurodocker/templates/gentoo.yaml b/neurodocker/templates/gentoo.yaml new file mode 100644 index 00000000..62a7426c --- /dev/null +++ b/neurodocker/templates/gentoo.yaml @@ -0,0 +1,65 @@ +--- +# Instructions to add NeuroDebian repositories. + +name: gentoo +url: https://www.gentoo.org/ +binaries: + # ATM template jsonschema demands having urls and instructions. + # In the future we might use urls to point to git repositories + # which are currently under gentoo-portage/ config files. + urls: + something: not-used + arguments: + optional: + gentoo_hash: 282c4b518562e6c194761d9ce0882f30672219c5 + science_hash: e4579f5851734f245b02cc1e98f3b69d9df67e09 + # Below echo commands will be indented by _indent_run_instruction which + # would cause trailing spaces to be added to the files. But it should be ok + # for us, thus we are not bothering to workaround via inlining. + instructions: | + mkdir -p /etc/portage/; \ + echo -e "\ + \nCOMMON_FLAGS=\"-O2 -pipe -march=native\" \ + \nMAKEOPTS=\"--jobs 8 --load-average 9\" \ + \nCFLAGS=\"\${COMMON_FLAGS}\" \ + \nCXXFLAGS=\"\${COMMON_FLAGS}\" \ + \nFCFLAGS=\"\${COMMON_FLAGS}\" \ + \nFFLAGS=\"\${COMMON_FLAGS}\" \ + \nLC_MESSAGES=C \ + \nUSE=\"\${USE} science\" \ + \nACCEPT_LICENSE=\"*\" \ + " > "/etc/portage/make.conf"; \ + mkdir -p "/etc/portage/package.accept_keywords"; \ + echo -e "*/* ~amd64" > "/etc/portage/package.accept_keywords/gen" ; \ + mkdir -p "/etc/portage/package.mask"; \ + touch "/etc/portage/package.mask/bugs"; \ + mkdir -p "/etc/portage/repos.conf" ; \ + echo -e "[gentoo] \ + \nlocation = /var/db/repos/gentoo \ + \nsync-type = git \ + \nsync-uri = https://anongit.gentoo.org/git/repo/gentoo.git \ + \nsync-git-verify-commit-signature = yes" > "/etc/portage/repos.conf/gentoo"; \ + echo -e "[science] \ + \nlocation = /var/db/repos/science \ + \nsync-type = git \ + \nsync-uri = https://anongit.gentoo.org/git/proj/sci.git \ + \npriority = 7777" > "/etc/portage/repos.conf/science"; \ + emerge -v --noreplace dev-vcs/git \ + && emerge -v1u portage \ + && mkdir /outputs \ + && rm /var/db/repos/gentoo -rf \ + && git config --global init.defaultBranch master \ + && \ + set -x && export GIT_TRACE=1 && \ + REPO_URL=$(grep "^sync-uri" /etc/portage/repos.conf/gentoo | sed -e "s/sync-uri *= *//g") && \ + git clone --depth 1 ${REPO_URL} /var/db/repos/gentoo && \ + cd /var/db/repos/gentoo && \ + git fetch --depth 1 origin {{ self.gentoo_hash }} && \ + git reset --hard {{ self.gentoo_hash }} && \ + rm .git -rf && \ + REPO_URL=$(grep "^sync-uri" /etc/portage/repos.conf/science | sed -e "s/sync-uri *= *//g") && \ + git clone --depth 1 ${REPO_URL} /var/db/repos/science && \ + cd /var/db/repos/science && \ + git fetch --depth 1 origin {{ self.science_hash }} && \ + git reset --hard {{ self.science_hash }} && \ + rm .git -rf