Account for PEP668 in pip invocations (#627)

* Account for PEP668 in pip invocations

Set the PIP_BREAK_SYSTEM_PACKAGES environment variable anywhere
pip is in use to account for PEP668 which would change pip to not
allow us to install in the system environment for newer versions of
pip.

* Fix issue 646 bug

docs/definition.rst

@@ -365,6 +365,13 @@ builder runtime functionality. Valid keys for this section are:
       of Ansible and Ansible Runner is performed on the final image. Set this
       value to ``True`` to not perform this check. The default is ``False``.
 
+    ``skip_pip_install``
+      This boolean value controls whether or not we attempt to install pip into
+      the base image. Pip is necessary for Python requirement installation, among
+      other things. You may choose to disable this step and handle installing
+      pip manually if the current method of pip installation does not work for you.
+      The default is ``False``.
+
     ``relax_passwd_permissions``
       This boolean value controls whether the ``root`` group (GID 0) is explicitly granted
       write permission to ``/etc/passwd`` in the final container image. The default entrypoint

src/ansible_builder/_target_scripts/assemble

@@ -31,8 +31,6 @@ PKGMGR_PRESERVE_CACHE="${PKGMGR_PRESERVE_CACHE:-}"
 PYCMD="${PYCMD:=/usr/bin/python3}"
 PIPCMD="${PIPCMD:=$PYCMD -m pip}"
 
-$PYCMD -m ensurepip
-
 if [ -z $PKGMGR ]; then
     # Expect dnf to be installed, however if we find microdnf default to it.
     PKGMGR=/usr/bin/dnf

src/ansible_builder/_target_scripts/install-from-bindep

@@ -25,8 +25,6 @@ PYCMD="${PYCMD:=/usr/bin/python3}"
 PIPCMD="${PIPCMD:=$PYCMD -m pip}"
 PIP_OPTS="${PIP_OPTS-}"
 
-$PYCMD -m ensurepip
-
 if [ -z $PKGMGR ]; then
     # Expect dnf to be installed, however if we find microdnf default to it.
     PKGMGR=/usr/bin/dnf

src/ansible_builder/_target_scripts/pip_install

@@ -0,0 +1,56 @@
+#!/bin/bash
+# Copyright (c) 2024 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#####################################################################
+# Script to encapsulate pip installation.
+#
+# Usage: pip_install <PYCMD>
+#
+# Options:
+#     PYCMD - The path to the python executable to use.
+#####################################################################
+
+set -x
+
+PYCMD=$1
+
+if [ -z "$PYCMD" ]
+then
+    echo "Usage: pip_install <PYCMD>"
+    exit 1
+fi
+
+if [ ! -x "$PYCMD" ]
+then
+    echo "$PYCMD is not an executable"
+    exit 1
+fi
+
+# This is going to be our default functionality for now. This will likely
+# need to change if we add support for non-RHEL distros.
+$PYCMD -m ensurepip --root /
+
+if [ $? -ne 0 ]
+then
+    cat<<EOF
+**********************************************************************
+ERROR - pip installation failed for Python $PYCMD
+**********************************************************************
+EOF
+    exit 1
+fi
+
+exit 0

src/ansible_builder/containerfile.py

@@ -76,23 +76,25 @@ def prepare(self) -> None:
             "# Base build stage",
             "FROM $EE_BASE_IMAGE as base",
             "USER root",
+            "ENV PIP_BREAK_SYSTEM_PACKAGES=1",
         ])
 
         self._insert_global_args()
+        self._create_folder_copy_files()
         self._insert_custom_steps('prepend_base')
 
         if not self.definition.builder_image:
             if self.definition.python_package_system:
                 step = 'RUN $PKGMGR install $PYPKG -y ; if [ -z $PKGMGR_PRESERVE_CACHE ]; then $PKGMGR clean all; fi'
                 self.steps.append(step)
 
-            # We should always make sure pip is available for later stages.
-            self.steps.append('RUN $PYCMD -m ensurepip')
+            # pip needs to be available for later stages.
+            if self.definition.version >= 3 and not self.definition.options['skip_pip_install']:
+                self.steps.append('RUN /output/scripts/pip_install $PYCMD')
 
             if self.definition.ansible_ref_install_list:
                 self.steps.append('RUN $PYCMD -m pip install --no-cache-dir $ANSIBLE_INSTALL_REFS')
 
-        self._create_folder_copy_files()
         self._insert_custom_steps('append_base')
 
         ######################################################################
@@ -124,7 +126,7 @@ def prepare(self) -> None:
         # Second stage (aka, builder): assemble (pip installs, bindep run)
         ######################################################################
 
-        if self.definition.builder_image:
+        if self.definition.builder_image or self.definition.version == 1:
             # Note: A builder image can be specified only in V1 or V2 schema.
             image = "$EE_BUILDER_IMAGE"
         else:
@@ -135,13 +137,19 @@ def prepare(self) -> None:
             "",
             "# Builder build stage",
             f"FROM {image} as builder",
+            "ENV PIP_BREAK_SYSTEM_PACKAGES=1",
             "WORKDIR /build",
         ])
 
         self._insert_global_args()
 
         if image == "base":
             self.steps.append("RUN $PYCMD -m pip install --no-cache-dir bindep pyyaml requirements-parser")
+        else:
+            # For an EE schema earlier than v3 with a custom builder image, we always make sure pip is available.
+            context_dir = Path(self.build_outputs_dir).stem
+            self.steps.append(f'COPY {context_dir}/scripts/pip_install /output/scripts/pip_install')
+            self.steps.append("RUN /output/scripts/pip_install $PYCMD")
 
         self._insert_custom_steps('prepend_builder')
         self._prepare_galaxy_copy_steps()
@@ -156,6 +164,7 @@ def prepare(self) -> None:
             "",
             "# Final build stage",
             "FROM base as final",
+            "ENV PIP_BREAK_SYSTEM_PACKAGES=1",
         ])
 
         self._insert_global_args()
@@ -279,7 +288,7 @@ def _create_folder_copy_files(self) -> None:
         scriptres = importlib.resources.files('ansible_builder._target_scripts')
         script_files = (
             'assemble', 'install-from-bindep', 'introspect.py', 'check_galaxy',
-            'check_ansible', 'entrypoint'
+            'check_ansible', 'pip_install', 'entrypoint'
         )
         for script in script_files:
             with importlib.resources.as_file(scriptres / script) as script_path:

src/ansible_builder/ee_schema.py

@@ -341,6 +341,10 @@
                     "description": "Disables the check for Ansible/Runner in final image",
                     "type": "boolean",
                 },
+                "skip_pip_install": {
+                    "description": "Disables the installation of pip in the base image",
+                    "type": "boolean",
+                },
                 "workdir": {
                     "description": "Default working directory, also often the homedir for ephemeral UIDs",
                     "type": ["string", "null"],
@@ -444,6 +448,7 @@ def _handle_options_defaults(ee_def: dict):
     entrypoint_path = os.path.join(constants.FINAL_IMAGE_BIN_PATH, "entrypoint")
 
     options.setdefault('skip_ansible_check', False)
+    options.setdefault('skip_pip_install', False)
     options.setdefault('relax_passwd_permissions', True)
     options.setdefault('workdir', '/runner')
     options.setdefault('package_manager_path', '/usr/bin/dnf')

test/unit/test_containerfile.py

@@ -207,3 +207,125 @@ def test__handle_additional_build_files(build_dir_and_ee_yml):
     config_dir = tmpdir / '_build' / 'configs'
     assert config_dir.exists()
     assert (config_dir / 'ansible.cfg').exists()
+
+
+def test_pep668_v1(build_dir_and_ee_yml):
+    """
+    Test PEP668 handling with v1 format.
+    """
+    ee_data = """
+    version: 1
+    """
+    tmpdir, ee_path = build_dir_and_ee_yml(ee_data)
+    c = make_containerfile(tmpdir, ee_path, run_validate=True)
+    c.prepare()
+    assert "ENV PIP_BREAK_SYSTEM_PACKAGES=1" in c.steps
+    assert "RUN /output/scripts/pip_install $PYCMD" in c.steps
+
+
+def test_pep668_v2(build_dir_and_ee_yml):
+    """
+    Test PEP668 handling with v2 format.
+    """
+    ee_data = """
+    version: 2
+    images:
+      base_image:
+        name: quay.io/user/mycustombaseimage:latest
+      builder_image:
+        name: quay.io/user/mycustombuilderimage:latest
+    """
+    tmpdir, ee_path = build_dir_and_ee_yml(ee_data)
+    c = make_containerfile(tmpdir, ee_path, run_validate=True)
+    c.prepare()
+    assert "ENV PIP_BREAK_SYSTEM_PACKAGES=1" in c.steps
+    assert "RUN /output/scripts/pip_install $PYCMD" in c.steps
+
+
+def test_pep668_v3(build_dir_and_ee_yml):
+    """
+    Test PEP668 handling with v3 format.
+    """
+    ee_data = """
+    version: 3
+    images:
+      base_image:
+        name: quay.io/user/mycustombaseimage:latest
+    """
+    tmpdir, ee_path = build_dir_and_ee_yml(ee_data)
+    c = make_containerfile(tmpdir, ee_path, run_validate=True)
+    c.prepare()
+    assert "ENV PIP_BREAK_SYSTEM_PACKAGES=1" in c.steps
+    assert "RUN /output/scripts/pip_install $PYCMD" in c.steps
+
+
+def test_pep668_v3_skip_pip_install(build_dir_and_ee_yml):
+    """
+    Test PEP668 handling with v3 format and skipping pip installation.
+    """
+    ee_data = """
+    version: 3
+    images:
+      base_image:
+        name: quay.io/user/mycustombaseimage:latest
+    options:
+      skip_pip_install: True
+    """
+    tmpdir, ee_path = build_dir_and_ee_yml(ee_data)
+    c = make_containerfile(tmpdir, ee_path, run_validate=True)
+    c.prepare()
+    assert "ENV PIP_BREAK_SYSTEM_PACKAGES=1" in c.steps
+    assert "RUN /output/scripts/pip_install $PYCMD" not in c.steps
+
+
+def test_v1_builder_image(build_dir_and_ee_yml):
+    """
+    Test for issue 646 (https://github.com/ansible/ansible-builder/issues/646).
+    Also test that pip_install is installed into the custom builder image.
+    """
+    ee_data = """
+    version: 1
+    """
+    tmpdir, ee_path = build_dir_and_ee_yml(ee_data)
+    c = make_containerfile(tmpdir, ee_path, run_validate=True)
+    c.prepare()
+    assert "FROM $EE_BUILDER_IMAGE as builder" in c.steps
+    assert "COPY _build/scripts/pip_install /output/scripts/pip_install" in c.steps
+    assert "RUN /output/scripts/pip_install $PYCMD" in c.steps
+
+
+def test_v2_builder_image(build_dir_and_ee_yml):
+    """
+    Test that pip_install is installed into the custom v2 builder image.
+    """
+    ee_data = """
+    version: 2
+    images:
+      base_image:
+        name: quay.io/user/mycustombaseimage:latest
+      builder_image:
+        name: quay.io/user/mycustombuilderimage:latest
+    """
+    tmpdir, ee_path = build_dir_and_ee_yml(ee_data)
+    c = make_containerfile(tmpdir, ee_path, run_validate=True)
+    c.prepare()
+    assert "FROM $EE_BUILDER_IMAGE as builder" in c.steps
+    assert "COPY _build/scripts/pip_install /output/scripts/pip_install" in c.steps
+    assert "RUN /output/scripts/pip_install $PYCMD" in c.steps
+
+
+def test_v2_builder_image_default(build_dir_and_ee_yml):
+    """
+    Test that pip_install is NOT installed into the builder image when not defined.
+    """
+    ee_data = """
+    version: 2
+    images:
+      base_image:
+        name: quay.io/user/mycustombaseimage:latest
+    """
+    tmpdir, ee_path = build_dir_and_ee_yml(ee_data)
+    c = make_containerfile(tmpdir, ee_path, run_validate=True)
+    c.prepare()
+    assert "FROM base as builder" in c.steps
+    assert "COPY _build/scripts/pip_install /output/scripts/pip_install" not in c.steps