Update CI files for branch 2.4

.ci/ansible/Containerfile.j2

@@ -28,8 +28,7 @@ RUN pip3 install --upgrade pip setuptools wheel && \
 {{ " " }}-r ./{{ item.name }}/ci_requirements.txt
 {%- endif -%}
 {%- endfor %}
-{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt \
-  pipdeptree && \
+{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt && \
   rm -rf /root/.cache/pip
 
 {% if pulp_env is defined and pulp_env %}

.ci/assets/ci_constraints.txt

@@ -5,8 +5,3 @@ pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*
 
 tablib!=3.6.0
 # 3.6.0: This release introduced a regression removing the "html" optional dependency.
-
-
-
-# Newer version seem to have a conflict around packaging, that pip fails to resolve in time. Remove this when this starts to impose an issue.
-pipdeptree<=3.23.1

.ci/assets/release_requirements.txt

@@ -1,3 +1,3 @@
-bump2version
+bump-my-version
 gitpython
 towncrier

.ci/scripts/calc_constraints.py

@@ -13,6 +13,11 @@
 from packaging.version import Version
 import yaml
 
+try:
+    import tomllib
+except ImportError:
+    import tomli as tomllib
+
 
 CORE_TEMPLATE_URL = "https://raw.githubusercontent.com/pulp/pulpcore/main/template_config.yml"
 
@@ -59,11 +64,11 @@ def to_upper_bound(req):
                 operator = "~="
                 version = Version(spec.version)
                 if version.micro != 0:
-                    max_version = f"{version.major}.{version.minor}.{version.micro-1}"
+                    max_version = f"{version.major}.{version.minor}.{version.micro - 1}"
                 elif version.minor != 0:
-                    max_version = f"{version.major}.{version.minor-1}"
+                    max_version = f"{version.major}.{version.minor - 1}"
                 elif version.major != 0:
-                    max_version = f"{version.major-1}.0"
+                    max_version = f"{version.major - 1}.0"
                 else:
                     return f"# NO BETTER CONSTRAINT: {req}"
                 return f"{requirement.name}{operator}{max_version}"
@@ -100,18 +105,32 @@ def main():
     parser.add_argument("filename", nargs="*")
     args = parser.parse_args()
 
-    with fileinput.input(files=args.filename) as req_file:
-        for line in req_file:
-            if line.strip().startswith("#"):
-                # Shortcut comment only lines
-                print(line.strip())
-            else:
-                req, comment = split_comment(line)
-                if args.upper:
-                    new_req = to_upper_bound(req)
+    modifier = to_upper_bound if args.upper else to_lower_bound
+
+    req_files = [filename for filename in args.filename if not filename.endswith("pyproject.toml")]
+    pyp_files = [filename for filename in args.filename if filename.endswith("pyproject.toml")]
+    if req_files:
+        with fileinput.input(files=req_files) as req_file:
+            for line in req_file:
+                if line.strip().startswith("#"):
+                    # Shortcut comment only lines
+                    print(line.strip())
                 else:
-                    new_req = to_lower_bound(req)
-                print(new_req + comment)
+                    req, comment = split_comment(line)
+                    new_req = modifier(req)
+                    print(new_req + comment)
+    for filename in pyp_files:
+        with open(filename, "rb") as fp:
+            pyproject = tomllib.load(fp)
+            for req in pyproject["project"]["dependencies"]:
+                new_req = modifier(req)
+                print(new_req)
+            optional_dependencies = pyproject["project"].get("optional-dependencies")
+            if optional_dependencies:
+                for opt in optional_dependencies.values():
+                    for req in opt:
+                        new_req = modifier(req)
+                        print(new_req)
 
 
 if __name__ == "__main__":

.ci/scripts/check_release.py

@@ -86,6 +86,13 @@ def main():
                 )
                 if req_txt_diff:
                     reasons.append("requirements.txt")
+                pyproject_diff = repo.git.diff(
+                    f"{last_tag}", f"origin/{branch}", "--name-only", "--", "pyproject.toml"
+                )
+                if pyproject_diff:
+                    reasons.append(
+                        "pyproject.toml changed (PLEASE check if dependencies are affected."
+                    )
 
                 if reasons:
                     curr_version = Version(last_tag)

.ci/scripts/check_requirements.py

@@ -5,11 +5,13 @@
 #
 # For more info visit https://github.com/pulp/plugin_template
 
+import tomllib
 import warnings
-from pkg_resources import Requirement
+from packaging.requirements import Requirement
 
 
 CHECK_MATRIX = [
+    ("pyproject.toml", True, True, True),
     ("requirements.txt", True, True, True),
     ("dev_requirements.txt", False, True, False),
     ("ci_requirements.txt", False, True, True),
@@ -20,17 +22,33 @@
     ("clitest_requirements.txt", False, True, True),
 ]
 
-errors = []
 
-for filename, check_upperbound, check_prereleases, check_r in CHECK_MATRIX:
-    try:
+def iterate_file(filename):
+    if filename == "pyproject.toml":
+        with open(filename, "rb") as fd:
+            pyproject_toml = tomllib.load(fd)
+            if "project" in pyproject_toml:
+                for nr, line in enumerate(pyproject_toml["project"]["dependencies"]):
+                    yield nr, line
+    else:
         with open(filename, "r") as fd:
             for nr, line in enumerate(fd.readlines()):
                 line = line.strip()
                 if not line or line.startswith("#"):
                     continue
+                if "#" in line:
+                    line = line.split("#", maxsplit=1)[0]
+                yield nr, line.strip()
+
+
+def main():
+    errors = []
+
+    for filename, check_upperbound, check_prereleases, check_r in CHECK_MATRIX:
+        try:
+            for nr, line in iterate_file(filename):
                 try:
-                    req = Requirement.parse(line)
+                    req = Requirement(line)
                 except ValueError:
                     if line.startswith("git+"):
                         # The single exception...
@@ -49,18 +67,21 @@
                             and req.name != "pulp-ostree-client"
                         ):
                             errors.append(f"{filename}:{nr}: Prerelease versions found in {line}.")
-                    ops = [op for op, ver in req.specs]
-                    spec = str(req.specs)
+                    ops = [spec.operator for spec in req.specifier]
                     if "~=" in ops:
                         warnings.warn(f"{filename}:{nr}: Please avoid using ~= on {req.name}!")
                     elif "<" not in ops and "<=" not in ops and "==" not in ops:
                         if check_upperbound:
                             errors.append(f"{filename}:{nr}: Upper bound missing in {line}.")
-    except FileNotFoundError:
-        # skip this test for plugins that don't use this requirements.txt
-        pass
+        except FileNotFoundError:
+            # skip this test for plugins that don't use this requirements.txt
+            pass
+
+    if errors:
+        print("Dependency issues found:")
+        print("\n".join(errors))
+        exit(1)
+
 
-if errors:
-    print("Dependency issues found:")
-    print("\n".join(errors))
-    exit(1)
+if __name__ == "__main__":
+    main()

.github/template_gitref

@@ -1 +1 @@
-2021.08.26-387-ge627e91
+2021.08.26-393-g0e700c1

.github/workflows/build.yml

@@ -34,14 +34,14 @@ jobs:
       - name: "Install python dependencies"
         run: |
           echo ::group::PYDEPS
-          pip install packaging twine wheel mkdocs jq
+          pip install build packaging twine wheel mkdocs jq
           echo ::endgroup::
       - name: "Install OS packages"
         run: |
           sudo apt-get install -y libgirepository1.0-dev libostree-dev libcairo2-dev
       - name: "Build package"
         run: |
-          python3 setup.py sdist bdist_wheel --python-tag py3
+          python3 -m build
           twine check dist/*
       - name: "Install built packages"
         run: |

.github/workflows/ci.yml

@@ -42,9 +42,6 @@ jobs:
           GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}"
         run: |
           .github/workflows/scripts/check_commit.sh
-      - name: "Verify requirements files"
-        run: |
-          python .ci/scripts/check_requirements.py
 
   docs:
     uses: "./.github/workflows/docs.yml"

.github/workflows/create-branch.yml

@@ -39,7 +39,7 @@ jobs:
       - name: "Install python dependencies"
         run: |
           echo ::group::PYDEPS
-          pip install bump2version packaging -r plugin_template/requirements.txt
+          pip install bump-my-version packaging -r plugin_template/requirements.txt
           echo ::endgroup::
 
       - name: "Setting secrets"
@@ -54,7 +54,7 @@ jobs:
         run: |
           # Just to be sure...
           git checkout main
-          NEW_BRANCH="$(bump2version --dry-run --list release | sed -Ene 's/^new_version=([[:digit:]]+\.[[:digit:]]+)\..*$/\1/p')"
+          NEW_BRANCH="$(bump-my-version show new_version --increment release | sed -Ene 's/^([[:digit:]]+\.[[:digit:]]+)\.[[:digit:]]+$/\1/p')"
           if [ -z "$NEW_BRANCH" ]
           then
             echo Could not determine the new branch name.
@@ -70,7 +70,7 @@ jobs:
       - name: Bump version on main branch
         working-directory: pulp_ostree
         run: |
-          bump2version --no-commit minor
+          bump-my-version bump --no-commit minor
 
       - name: Remove entries from CHANGES directory
         working-directory: pulp_ostree

.github/workflows/lint.yml

@@ -16,7 +16,7 @@ defaults:
 
 jobs:
   lint:
-    runs-on: ubuntu-latest
+    runs-on: "ubuntu-latest"
 
     steps:
       - uses: "actions/checkout@v4"
@@ -34,29 +34,42 @@ jobs:
           pip install -r lint_requirements.txt
           echo ::endgroup::
 
-      - name: Lint workflow files
+      - name: "Lint workflow files"
         run: |
           yamllint -s -d '{extends: relaxed, rules: {line-length: disable}}' .github/workflows
 
+      - name: "Verify bump version config"
+        run: |
+          bump-my-version bump --dry-run release
+          bump-my-version show-bump
+
       # run black separately from flake8 to get a diff
-      - name: Run black
+      - name: "Run black"
         run: |
           black --version
           black --check --diff .
 
       # Lint code.
-      - name: Run flake8
-        run: flake8
+      - name: "Run flake8"
+        run: |
+          flake8
 
-      - name: Run extra lint checks
-        run: "[ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh"
+      - name: "Run extra lint checks"
+        run: |
+          [ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh
 
-      # check for any files unintentionally left out of MANIFEST.in
-      - name: Check manifest
-        run: check-manifest
+      - name: "Check for any files unintentionally left out of MANIFEST.in"
+        run: |
+          check-manifest
 
-      - name: Check for pulpcore imports outside of pulpcore.plugin
-        run: sh .ci/scripts/check_pulpcore_imports.sh
+      - name: "Verify requirements files"
+        run: |
+          python .ci/scripts/check_requirements.py
+
+      - name: "Check for pulpcore imports outside of pulpcore.plugin"
+        run: |
+          sh .ci/scripts/check_pulpcore_imports.sh
 
-      - name: Check for gettext problems
-        run: sh .ci/scripts/check_gettext.sh
+      - name: "Check for common gettext problems"
+        run: |
+          sh .ci/scripts/check_gettext.sh

.github/workflows/release.yml

@@ -35,7 +35,7 @@ jobs:
       - name: "Install python dependencies"
         run: |
           echo ::group::PYDEPS
-          pip install bump2version towncrier
+          pip install bump-my-version towncrier
           echo ::endgroup::
 
       - name: "Configure Git with pulpbot name and email"

.github/workflows/scripts/before_install.sh

@@ -45,11 +45,6 @@ if [ -f $PRE_BEFORE_INSTALL ]; then
   source $PRE_BEFORE_INSTALL
 fi
 
-if [[ -n $(echo -e $COMMIT_MSG | grep -P "Required PR:.*") ]]; then
-  echo "The Required PR mechanism has been removed. Consider adding a scm requirement to requirements.txt."
-  exit 1
-fi
-
 if [ "$GITHUB_EVENT_NAME" = "pull_request" ] || [ "${BRANCH_BUILD}" = "1" -a "${BRANCH}" != "main" ]
 then
   echo $COMMIT_MSG | sed -n -e 's/.*CI Base Image:\s*\([-_/[:alnum:]]*:[-_[:alnum:]]*\).*/ci_base: "\1"/p' >> .ci/ansible/vars/main.yaml

.github/workflows/scripts/install.sh

@@ -156,5 +156,5 @@ if [[ "$TEST" = "azure" ]]; then
 fi
 
 echo ::group::PIP_LIST
-cmd_prefix bash -c "pip3 list && pipdeptree"
+cmd_prefix bash -c "pip3 list"
 echo ::endgroup::

.github/workflows/scripts/release.sh

@@ -10,7 +10,8 @@ then
   exit 1
 fi
 
-NEW_VERSION="$(bump2version --dry-run --list release | sed -ne 's/^new_version=//p')"
+# The tail is a necessary workaround to remove the warning from the output.
+NEW_VERSION="$(bump-my-version show new_version --increment release | tail -n -1)"
 echo "Release ${NEW_VERSION}"
 
 if ! [[ "${NEW_VERSION}" == "${BRANCH}"* ]]
@@ -20,7 +21,7 @@ then
 fi
 
 towncrier build --yes --version "${NEW_VERSION}"
-bump2version release --commit --message "Release {new_version}" --tag --tag-name "{new_version}" --tag-message "Release {new_version}" --allow-dirty
-bump2version patch --commit
+bump-my-version bump release --commit --message "Release {new_version}" --tag --tag-name "{new_version}" --tag-message "Release {new_version}" --allow-dirty
+bump-my-version bump patch --commit
 
 git push origin "${BRANCH}" "${NEW_VERSION}"

.github/workflows/test.yml

@@ -147,5 +147,5 @@ jobs:
           docker logs pulp || true
           docker exec pulp ls -latr /etc/yum.repos.d/ || true
           docker exec pulp cat /etc/yum.repos.d/* || true
-          docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree"
+          docker exec pulp bash -c "pip3 list" || true
 ...

lint_requirements.txt

@@ -5,9 +5,10 @@
 #
 # For more info visit https://github.com/pulp/plugin_template
 
-# python packages handy for developers, but not required by pulp
 black==24.3.0
+bump-my-version
 check-manifest
 flake8
 flake8-black
+packaging
 yamllint