[WiP] Draft for Gentoo configuration

.gitignore

@@ -146,3 +146,4 @@ cython_debug/
 
 # Pycharm
 .idea/
+venvs

neurodocker/cli/tests/test_build_images_with_cli.py

@@ -26,7 +26,13 @@
     ],
 )
 @pytest.mark.parametrize(
-    ["pkg_manager", "base_image"], [("apt", "debian:buster-slim"), ("yum", "centos:7")]
+    ["pkg_manager", "base_image"],
+    [
+        ("apt", "debian:buster-slim"),
+        ("yum", "centos:7"),
+        # TODO: make it work -- seems has difficulty installing jq, may be curl isn't there or alike
+        # ("portage", "gentoo"),
+     ],
 )
 def test_build_image_from_registered(
     tmp_path: Path, cmd: str, pkg_manager: str, base_image: str
@@ -113,3 +119,26 @@ def test_json_roundtrip(cmd: str, inputs: str, tmp_path: Path):
         stdout, _ = run_fn(img, args=["env"])
         assert "CAT=FOO" in stdout
         assert "DOG=BAR" in stdout
+
+
+def test_gentoo_image(tmp_path: Path):
+    # TODO: also add singularity like in the test above
+
+    cmd = "docker"
+
+    _TemplateRegistry._reset()
+    runner = CliRunner()
+    result = runner.invoke(
+        generate,
+        [
+            cmd,
+            "--pkg-manager", "portage",
+            "--base-image", "gentoo",
+            # TODO: this flips it out for some reason in the test, works on CLI
+            # "--gentoo", "gentoo_hash=0e9370b45a589867220384ca6c63bc6bcaec3f74",
+            "--install", "afni",  # TODO: replace with some tiny / quick to install pkg
+        ],
+    )
+    assert result.exit_code == 0, result.output
+    (tmp_path / "specs.json").write_text(result.output)
+    # TODO: add more testing here of the result that it is usable

neurodocker/cli/tests/test_cli.py

@@ -19,7 +19,7 @@ def test_fail_on_empty_args(cmd: str):
 
 
 @pytest.mark.parametrize("cmd", _cmds)
-@pytest.mark.parametrize("pkg_manager", ["apt", "yum"])
+@pytest.mark.parametrize("pkg_manager", ["apt", "portage", "yum"])
 def test_fail_on_no_base(cmd: str, pkg_manager: str):
     runner = CliRunner()
     result = runner.invoke(generate, [cmd, "--pkg-manager", pkg_manager])
@@ -34,7 +34,7 @@ def test_fail_on_no_pkg_manager(cmd: str):
 
 
 @pytest.mark.parametrize("cmd", _cmds)
-@pytest.mark.parametrize("pkg_manager", ["apt", "yum"])
+@pytest.mark.parametrize("pkg_manager", ["apt", "portage", "yum"])
 def test_minimal_args(cmd: str, pkg_manager: str):
     runner = CliRunner()
     result = runner.invoke(
@@ -107,7 +107,7 @@ def test_optioneatall_type_issue_498():
 
 
 @pytest.mark.parametrize("cmd", _cmds)
-@pytest.mark.parametrize("pkg_manager", ["apt", "yum"])
+@pytest.mark.parametrize("pkg_manager", ["apt", "portage", "yum"])
 def test_all_args(cmd: str, pkg_manager: str):
     runner = CliRunner()
     result = runner.invoke(
@@ -156,7 +156,7 @@ def test_all_args(cmd: str, pkg_manager: str):
 # is what registers the templates. Using the `docker` function
 # (`reproenv generate docker`) directly does not fire `generate`.
 @pytest.mark.parametrize("cmd", _cmds)
-@pytest.mark.parametrize("pkg_manager", ["apt", "yum"])
+@pytest.mark.parametrize("pkg_manager", ["apt", "portage", "yum"])
 def test_render_registered(cmd: str, pkg_manager: str):
     template_path = Path(__file__).parent
     runner = CliRunner(env={"REPROENV_TEMPLATE_PATH": str(template_path)})

neurodocker/reproenv/renderers.py

@@ -518,10 +518,21 @@ def entrypoint(self, args: list[str]) -> DockerRenderer:
     @_log_instruction
     def from_(self, base_image: str, as_: str = None) -> DockerRenderer:
         """Add a Dockerfile `FROM` instruction."""
-        if as_ is None:
-            s = "FROM " + base_image
+        if base_image == "gentoo":
+            # TODO (if we can, likely not): make neurodocker argument "portage_date" for which we
+            # figure out the corresponding stage3 date and hashes.
+            s = "FROM docker.io/gentoo/portage:20240529 as portage\n"
+            s += "FROM docker.io/gentoo/stage3:20240527\n"
+            s += "COPY --from=portage /var/db/repos/gentoo /var/db/repos/gentoo\n"
+            # TODO: figure out hashes for the date
+            #s += "ARG gentoo_hash=2d25617a1d085316761b06c17a93ec972f172fc6\n"
+            #s += "ARG science_hash=73916dd3680ffd92e5bd3d32b262e5d78c86a448\n"
+            s += "ARG FEATURES=\"-ipc-sandbox -network-sandbox -pid-sandbox\"\n"
         else:
-            s = f"FROM {base_image} AS {as_}"
+            if as_ is None:
+                s = "FROM " + base_image
+            else:
+                s = f"FROM {base_image} AS {as_}"
         self._parts.append(s)
         return self
 
@@ -753,6 +764,8 @@ def _indent_run_instruction(string: str, indent=4) -> str:
 def _install(pkgs: list[str], pkg_manager: str, opts: str = None) -> str:
     if pkg_manager == "apt":
         return _apt_install(pkgs, opts)
+    elif pkg_manager == "portage":
+        return _portage_install(pkgs, opts)
     elif pkg_manager == "yum":
         return _yum_install(pkgs, opts)
     # TODO: add debs here?
@@ -802,6 +815,25 @@ def install_one(url: str):
     return s
 
 
+def _portage_install(pkgs: list[str], opts: str = None, sort=True) -> str:
+    """Return command to install packages with `portage` (Gentoo).
+
+    `opts` are options passed to `emerge`.
+    Default is "--autounmask-continue".
+    """
+    pkgs = sorted(pkgs) if sort else pkgs
+    opts = "--autounmask-continue" if opts is None else opts
+
+    s = """\
+emerge {opts} \\
+    {pkgs}
+rm -rf /var/tmp/portage/*
+""".format(
+        opts=opts, pkgs=" \\\n    ".join(pkgs)
+    )
+    return s.strip()
+
+
 def _yum_install(pkgs: list[str], opts: str = None, sort=True) -> str:
     """Return command to install packages with `yum` (CentOS, Fedora).
 

neurodocker/reproenv/schemas/renderer.json

@@ -11,10 +11,12 @@
       "type": "string",
       "enum": [
         "apt",
+        "portage",
         "yum"
       ],
       "examples": [
         "apt",
+        "portage",
         "yum"
       ]
     },

neurodocker/reproenv/types.py

@@ -18,8 +18,8 @@
 allowed_installation_methods = {"binaries", "source"}
 installation_methods_type = Literal["binaries", "source"]
 
-allowed_pkg_managers = {"apt", "yum"}
-pkg_managers_type = Literal["apt", "yum"]
+allowed_pkg_managers = {"apt", "portage", "yum"}
+pkg_managers_type = Literal["apt", "portage", "yum"]
 
 # Cross-reference the dictionary types below with the JSON schemas.
 
@@ -32,6 +32,7 @@ class _InstallationDependenciesType(TypedDict, total=False):
 
     apt: list[str]
     debs: list[str]
+    portage: list[str]
     yum: list[str]
 
 

neurodocker/templates/afni.yaml

@@ -72,6 +72,8 @@ binaries:
         -   which
         -   unzip
         -   ncurses-compat-libs
+        # portage:
+        # -   sci-biology/ants
         debs:
         -   http://mirrors.kernel.org/debian/pool/main/libx/libxp/libxp6_1.0.2-2_amd64.deb
         -   http://snapshot.debian.org/archive/debian-security/20160113T213056Z/pool/updates/main/libp/libpng/libpng12-0_1.2.49-1%2Bdeb7u2_amd64.deb

neurodocker/templates/gentoo-portage/make.conf

@@ -0,0 +1,21 @@
+### This file contains system-wide build variables, including Gentoo variables such as USE, which enable/disable optional package features.
+
+COMMON_FLAGS="-O2 -pipe -march=native"
+# Comment the following out on systems with less than 8 threads
+MAKEOPTS="--jobs 8 --load-average 9"
+CFLAGS="${COMMON_FLAGS}"
+CXXFLAGS="${COMMON_FLAGS}"
+FCFLAGS="${COMMON_FLAGS}"
+FFLAGS="${COMMON_FLAGS}"
+
+# NOTE: This stage was built with the bindist Use flag enabled
+
+# This sets the language of build output to English.
+# Please keep this setting intact when reporting bugs.
+LC_MESSAGES=C
+
+USE="${USE} science"
+ACCEPT_LICENSE="*"
+
+# Needed in the container environment
+#FEATURES="-ipc-sandbox -network-sandbox -pid-sandbox"

neurodocker/templates/gentoo-portage/package.accept_keywords/gen

@@ -0,0 +1,2 @@
+### This is needed because ::science packages are generally not marked as stable
+*/* ~amd64

neurodocker/templates/gentoo-portage/package.mask/bugs

@@ -0,0 +1,2 @@
+### This is empty, thankfully.
+### If we find bugs in some version of some package we can blacklist the package, version, or feature that causes it here.

neurodocker/templates/gentoo-portage/repos.conf/gentoo

@@ -0,0 +1,6 @@
+[gentoo]
+location = /var/db/repos/gentoo
+# We sync manually, but we need sync-uri to be written down somewhere to do so
+sync-type = git
+sync-uri = https://anongit.gentoo.org/git/repo/gentoo.git
+sync-git-verify-commit-signature = yes

neurodocker/templates/gentoo-portage/repos.conf/science

@@ -0,0 +1,6 @@
+[science]
+location = /var/db/repos/science
+# We sync manually, but we need sync-uri to be written down somewhere to do so
+sync-type = git
+sync-uri = https://anongit.gentoo.org/git/proj/sci.git
+priority = 7777

neurodocker/templates/gentoo.yaml

@@ -0,0 +1,65 @@
+---
+# Instructions to add NeuroDebian repositories.
+
+name: gentoo
+url: https://www.gentoo.org/
+binaries:
+    # ATM template jsonschema demands having urls and instructions.
+    # In the future we might use urls to point to git repositories
+    # which are currently under gentoo-portage/ config files.
+    urls:
+        something: not-used
+    arguments:
+        optional:
+            gentoo_hash: "0e9370b45a589867220384ca6c63bc6bcaec3f74"
+            science_hash: "5307342730267714f7019d62f77b2d9bf7624d8c"
+    # Below echo commands will be indented by _indent_run_instruction which
+    # would cause trailing spaces to be added to the files. But it should be ok
+    # for us, thus we are not bothering to workaround via inlining.
+    instructions: |
+        mkdir -p /etc/portage/; \
+        echo -e "\
+        \nCOMMON_FLAGS=\"-O2 -pipe -march=native\" \
+        \nMAKEOPTS=\"--jobs 8 --load-average 9\" \
+        \nCFLAGS=\"\${COMMON_FLAGS}\" \
+        \nCXXFLAGS=\"\${COMMON_FLAGS}\" \
+        \nFCFLAGS=\"\${COMMON_FLAGS}\" \
+        \nFFLAGS=\"\${COMMON_FLAGS}\" \
+        \nLC_MESSAGES=C \
+        \nUSE=\"\${USE} science\" \
+        \nACCEPT_LICENSE=\"*\" \
+        "  > "/etc/portage/make.conf"; \
+        mkdir -p "/etc/portage/package.accept_keywords"; \
+        echo -e "*/* ~amd64"  > "/etc/portage/package.accept_keywords/gen" ; \
+        mkdir -p "/etc/portage/package.mask"; \
+        touch "/etc/portage/package.mask/bugs"; \
+        mkdir -p "/etc/portage/repos.conf" ; \
+        echo -e "[gentoo] \
+        \nlocation = /var/db/repos/gentoo \
+        \nsync-type = git \
+        \nsync-uri = https://anongit.gentoo.org/git/repo/gentoo.git \
+        \nsync-git-verify-commit-signature = yes"  > "/etc/portage/repos.conf/gentoo"; \
+        echo -e "[science] \
+        \nlocation = /var/db/repos/science \
+        \nsync-type = git \
+        \nsync-uri = https://anongit.gentoo.org/git/proj/sci.git \
+        \npriority = 7777"  > "/etc/portage/repos.conf/science"; \
+        emerge -v --noreplace dev-vcs/git \
+        && emerge -v1u portage \
+        && mkdir /outputs \
+        && rm /var/db/repos/gentoo -rf \
+        && git config --global init.defaultBranch master \
+        && \
+               set -x && export GIT_TRACE=1 && \
+               REPO_URL=$(grep "^sync-uri" /etc/portage/repos.conf/gentoo | sed -e "s/sync-uri *= *//g") && \
+               git clone --depth 1 ${REPO_URL} /var/db/repos/gentoo && \
+               cd /var/db/repos/gentoo && \
+                       git fetch --depth 1 origin {{ self.gentoo_hash }} && \
+                       git reset --hard {{ self.gentoo_hash }} && \
+               rm .git -rf && \
+               REPO_URL=$(grep "^sync-uri" /etc/portage/repos.conf/science | sed -e "s/sync-uri *= *//g") && \
+               git clone --depth 1 ${REPO_URL} /var/db/repos/science && \
+               cd /var/db/repos/science && \
+                       git fetch --depth 1 origin {{ self.science_hash }} && \
+                       git reset --hard {{ self.science_hash }} && \
+               rm .git -rf