diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index e4b8633..d57b44e 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -28,8 +28,7 @@ RUN pip3 install --upgrade pip setuptools wheel && \ {{ " " }}-r ./{{ item.name }}/ci_requirements.txt {%- endif -%} {%- endfor %} -{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt \ - pipdeptree && \ +{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt && \ rm -rf /root/.cache/pip {% if pulp_env is defined and pulp_env %} diff --git a/.ci/assets/ci_constraints.txt b/.ci/assets/ci_constraints.txt index 14b5159..2617a40 100644 --- a/.ci/assets/ci_constraints.txt +++ b/.ci/assets/ci_constraints.txt @@ -5,8 +5,3 @@ pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.* tablib!=3.6.0 # 3.6.0: This release introduced a regression removing the "html" optional dependency. - - - -# Newer version seem to have a conflict around packaging, that pip fails to resolve in time. Remove this when this starts to impose an issue. -pipdeptree<=3.23.1 diff --git a/.ci/assets/release_requirements.txt b/.ci/assets/release_requirements.txt index c064e94..6635a87 100644 --- a/.ci/assets/release_requirements.txt +++ b/.ci/assets/release_requirements.txt @@ -1,3 +1,3 @@ -bump2version +bump-my-version gitpython towncrier diff --git a/.ci/scripts/calc_constraints.py b/.ci/scripts/calc_constraints.py index 3372754..c27840e 100755 --- a/.ci/scripts/calc_constraints.py +++ b/.ci/scripts/calc_constraints.py @@ -13,6 +13,11 @@ from packaging.version import Version import yaml +try: + import tomllib +except ImportError: + import tomli as tomllib + CORE_TEMPLATE_URL = "https://raw.githubusercontent.com/pulp/pulpcore/main/template_config.yml" @@ -59,11 +64,11 @@ def to_upper_bound(req): operator = "~=" version = Version(spec.version) if version.micro != 0: - max_version = f"{version.major}.{version.minor}.{version.micro-1}" + max_version = f"{version.major}.{version.minor}.{version.micro - 1}" elif version.minor != 0: - max_version = f"{version.major}.{version.minor-1}" + max_version = f"{version.major}.{version.minor - 1}" elif version.major != 0: - max_version = f"{version.major-1}.0" + max_version = f"{version.major - 1}.0" else: return f"# NO BETTER CONSTRAINT: {req}" return f"{requirement.name}{operator}{max_version}" @@ -100,18 +105,32 @@ def main(): parser.add_argument("filename", nargs="*") args = parser.parse_args() - with fileinput.input(files=args.filename) as req_file: - for line in req_file: - if line.strip().startswith("#"): - # Shortcut comment only lines - print(line.strip()) - else: - req, comment = split_comment(line) - if args.upper: - new_req = to_upper_bound(req) + modifier = to_upper_bound if args.upper else to_lower_bound + + req_files = [filename for filename in args.filename if not filename.endswith("pyproject.toml")] + pyp_files = [filename for filename in args.filename if filename.endswith("pyproject.toml")] + if req_files: + with fileinput.input(files=req_files) as req_file: + for line in req_file: + if line.strip().startswith("#"): + # Shortcut comment only lines + print(line.strip()) else: - new_req = to_lower_bound(req) - print(new_req + comment) + req, comment = split_comment(line) + new_req = modifier(req) + print(new_req + comment) + for filename in pyp_files: + with open(filename, "rb") as fp: + pyproject = tomllib.load(fp) + for req in pyproject["project"]["dependencies"]: + new_req = modifier(req) + print(new_req) + optional_dependencies = pyproject["project"].get("optional-dependencies") + if optional_dependencies: + for opt in optional_dependencies.values(): + for req in opt: + new_req = modifier(req) + print(new_req) if __name__ == "__main__": diff --git a/.ci/scripts/check_release.py b/.ci/scripts/check_release.py index 9d67bc6..f0a6cc9 100755 --- a/.ci/scripts/check_release.py +++ b/.ci/scripts/check_release.py @@ -86,6 +86,13 @@ def main(): ) if req_txt_diff: reasons.append("requirements.txt") + pyproject_diff = repo.git.diff( + f"{last_tag}", f"origin/{branch}", "--name-only", "--", "pyproject.toml" + ) + if pyproject_diff: + reasons.append( + "pyproject.toml changed (PLEASE check if dependencies are affected." + ) if reasons: curr_version = Version(last_tag) diff --git a/.ci/scripts/check_requirements.py b/.ci/scripts/check_requirements.py index 33d7cc0..a2c713d 100755 --- a/.ci/scripts/check_requirements.py +++ b/.ci/scripts/check_requirements.py @@ -5,11 +5,13 @@ # # For more info visit https://github.com/pulp/plugin_template +import tomllib import warnings -from pkg_resources import Requirement +from packaging.requirements import Requirement CHECK_MATRIX = [ + ("pyproject.toml", True, True, True), ("requirements.txt", True, True, True), ("dev_requirements.txt", False, True, False), ("ci_requirements.txt", False, True, True), @@ -20,17 +22,33 @@ ("clitest_requirements.txt", False, True, True), ] -errors = [] -for filename, check_upperbound, check_prereleases, check_r in CHECK_MATRIX: - try: +def iterate_file(filename): + if filename == "pyproject.toml": + with open(filename, "rb") as fd: + pyproject_toml = tomllib.load(fd) + if "project" in pyproject_toml: + for nr, line in enumerate(pyproject_toml["project"]["dependencies"]): + yield nr, line + else: with open(filename, "r") as fd: for nr, line in enumerate(fd.readlines()): line = line.strip() if not line or line.startswith("#"): continue + if "#" in line: + line = line.split("#", maxsplit=1)[0] + yield nr, line.strip() + + +def main(): + errors = [] + + for filename, check_upperbound, check_prereleases, check_r in CHECK_MATRIX: + try: + for nr, line in iterate_file(filename): try: - req = Requirement.parse(line) + req = Requirement(line) except ValueError: if line.startswith("git+"): # The single exception... @@ -49,18 +67,21 @@ and req.name != "pulp-ostree-client" ): errors.append(f"{filename}:{nr}: Prerelease versions found in {line}.") - ops = [op for op, ver in req.specs] - spec = str(req.specs) + ops = [spec.operator for spec in req.specifier] if "~=" in ops: warnings.warn(f"{filename}:{nr}: Please avoid using ~= on {req.name}!") elif "<" not in ops and "<=" not in ops and "==" not in ops: if check_upperbound: errors.append(f"{filename}:{nr}: Upper bound missing in {line}.") - except FileNotFoundError: - # skip this test for plugins that don't use this requirements.txt - pass + except FileNotFoundError: + # skip this test for plugins that don't use this requirements.txt + pass + + if errors: + print("Dependency issues found:") + print("\n".join(errors)) + exit(1) + -if errors: - print("Dependency issues found:") - print("\n".join(errors)) - exit(1) +if __name__ == "__main__": + main() diff --git a/.github/template_gitref b/.github/template_gitref index 4fb36e9..d37a43c 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-387-ge627e91 +2021.08.26-393-g0e700c1 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2570555..f068775 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,14 +34,14 @@ jobs: - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install packaging twine wheel mkdocs jq + pip install build packaging twine wheel mkdocs jq echo ::endgroup:: - name: "Install OS packages" run: | sudo apt-get install -y libgirepository1.0-dev libostree-dev libcairo2-dev - name: "Build package" run: | - python3 setup.py sdist bdist_wheel --python-tag py3 + python3 -m build twine check dist/* - name: "Install built packages" run: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8bc4536..d34e5d6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,9 +42,6 @@ jobs: GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" run: | .github/workflows/scripts/check_commit.sh - - name: "Verify requirements files" - run: | - python .ci/scripts/check_requirements.py docs: uses: "./.github/workflows/docs.yml" diff --git a/.github/workflows/create-branch.yml b/.github/workflows/create-branch.yml index 250203b..7099cba 100644 --- a/.github/workflows/create-branch.yml +++ b/.github/workflows/create-branch.yml @@ -39,7 +39,7 @@ jobs: - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install bump2version packaging -r plugin_template/requirements.txt + pip install bump-my-version packaging -r plugin_template/requirements.txt echo ::endgroup:: - name: "Setting secrets" @@ -54,7 +54,7 @@ jobs: run: | # Just to be sure... git checkout main - NEW_BRANCH="$(bump2version --dry-run --list release | sed -Ene 's/^new_version=([[:digit:]]+\.[[:digit:]]+)\..*$/\1/p')" + NEW_BRANCH="$(bump-my-version show new_version --increment release | sed -Ene 's/^([[:digit:]]+\.[[:digit:]]+)\.[[:digit:]]+$/\1/p')" if [ -z "$NEW_BRANCH" ] then echo Could not determine the new branch name. @@ -70,7 +70,7 @@ jobs: - name: Bump version on main branch working-directory: pulp_ostree run: | - bump2version --no-commit minor + bump-my-version bump --no-commit minor - name: Remove entries from CHANGES directory working-directory: pulp_ostree diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index b00b398..1fb522d 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -16,7 +16,7 @@ defaults: jobs: lint: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" steps: - uses: "actions/checkout@v4" @@ -34,29 +34,42 @@ jobs: pip install -r lint_requirements.txt echo ::endgroup:: - - name: Lint workflow files + - name: "Lint workflow files" run: | yamllint -s -d '{extends: relaxed, rules: {line-length: disable}}' .github/workflows + - name: "Verify bump version config" + run: | + bump-my-version bump --dry-run release + bump-my-version show-bump + # run black separately from flake8 to get a diff - - name: Run black + - name: "Run black" run: | black --version black --check --diff . # Lint code. - - name: Run flake8 - run: flake8 + - name: "Run flake8" + run: | + flake8 - - name: Run extra lint checks - run: "[ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh" + - name: "Run extra lint checks" + run: | + [ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh - # check for any files unintentionally left out of MANIFEST.in - - name: Check manifest - run: check-manifest + - name: "Check for any files unintentionally left out of MANIFEST.in" + run: | + check-manifest - - name: Check for pulpcore imports outside of pulpcore.plugin - run: sh .ci/scripts/check_pulpcore_imports.sh + - name: "Verify requirements files" + run: | + python .ci/scripts/check_requirements.py + + - name: "Check for pulpcore imports outside of pulpcore.plugin" + run: | + sh .ci/scripts/check_pulpcore_imports.sh - - name: Check for gettext problems - run: sh .ci/scripts/check_gettext.sh + - name: "Check for common gettext problems" + run: | + sh .ci/scripts/check_gettext.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 13cf26e..6230d4a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -35,7 +35,7 @@ jobs: - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install bump2version towncrier + pip install bump-my-version towncrier echo ::endgroup:: - name: "Configure Git with pulpbot name and email" diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index 825782d..10432e6 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -45,11 +45,6 @@ if [ -f $PRE_BEFORE_INSTALL ]; then source $PRE_BEFORE_INSTALL fi -if [[ -n $(echo -e $COMMIT_MSG | grep -P "Required PR:.*") ]]; then - echo "The Required PR mechanism has been removed. Consider adding a scm requirement to requirements.txt." - exit 1 -fi - if [ "$GITHUB_EVENT_NAME" = "pull_request" ] || [ "${BRANCH_BUILD}" = "1" -a "${BRANCH}" != "main" ] then echo $COMMIT_MSG | sed -n -e 's/.*CI Base Image:\s*\([-_/[:alnum:]]*:[-_[:alnum:]]*\).*/ci_base: "\1"/p' >> .ci/ansible/vars/main.yaml diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index a4354dd..e7f3c74 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -156,5 +156,5 @@ if [[ "$TEST" = "azure" ]]; then fi echo ::group::PIP_LIST -cmd_prefix bash -c "pip3 list && pipdeptree" +cmd_prefix bash -c "pip3 list" echo ::endgroup:: diff --git a/.github/workflows/scripts/release.sh b/.github/workflows/scripts/release.sh index 9525f22..a08353c 100755 --- a/.github/workflows/scripts/release.sh +++ b/.github/workflows/scripts/release.sh @@ -10,7 +10,8 @@ then exit 1 fi -NEW_VERSION="$(bump2version --dry-run --list release | sed -ne 's/^new_version=//p')" +# The tail is a necessary workaround to remove the warning from the output. +NEW_VERSION="$(bump-my-version show new_version --increment release | tail -n -1)" echo "Release ${NEW_VERSION}" if ! [[ "${NEW_VERSION}" == "${BRANCH}"* ]] @@ -20,7 +21,7 @@ then fi towncrier build --yes --version "${NEW_VERSION}" -bump2version release --commit --message "Release {new_version}" --tag --tag-name "{new_version}" --tag-message "Release {new_version}" --allow-dirty -bump2version patch --commit +bump-my-version bump release --commit --message "Release {new_version}" --tag --tag-name "{new_version}" --tag-message "Release {new_version}" --allow-dirty +bump-my-version bump patch --commit git push origin "${BRANCH}" "${NEW_VERSION}" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 836f9e6..461e7db 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -147,5 +147,5 @@ jobs: docker logs pulp || true docker exec pulp ls -latr /etc/yum.repos.d/ || true docker exec pulp cat /etc/yum.repos.d/* || true - docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" + docker exec pulp bash -c "pip3 list" || true ... diff --git a/lint_requirements.txt b/lint_requirements.txt index c579bcb..eac20d3 100644 --- a/lint_requirements.txt +++ b/lint_requirements.txt @@ -5,9 +5,10 @@ # # For more info visit https://github.com/pulp/plugin_template -# python packages handy for developers, but not required by pulp black==24.3.0 +bump-my-version check-manifest flake8 flake8-black +packaging yamllint diff --git a/releasing.md b/releasing.md index b2c1d91..2102cfb 100644 --- a/releasing.md +++ b/releasing.md @@ -9,19 +9,21 @@ This document outlines the steps to perform a release. ### Determine if a Release is Required +- Make sure to have GitPython python package installed - Run the release checker script: ``` python3 .ci/scripts/check_release.py ``` -### Create a New Y-branch (e.g., 3.23) -- If a new minor version (Y) is needed, trigger a [Create New Release Branch](https://github.com/pulp/pulp_ostree/actions/workflows/create-branch.yml) job via the GitHub Actions. +### Release a New Y-version (e.g., 3.23.0) +- If a new minor version (Y) is needed, trigger a [Create New Release Branch](https://github.com/pulp/pulp_ostree/actions/workflows/create-branch.yml) job from the main branch via the GitHub Actions. - Look for the "Bump minor version" pull request and merge it. +- Trigger a [Release Pipeline](https://github.com/pulp/pulp_ostree/actions/workflows/release.yml) job by specifying the new release branch (X.**Y**) via the GitHub Actions. ### Release a New Z-version (Patch Release) (e.g., 3.23.1, 3.22.12) - Trigger a [Release Pipeline](https://github.com/pulp/pulp_ostree/actions/workflows/release.yml) job by specifying the release branch (X.Y) via the GitHub Actions. -### Final Steps (Optional but Recommended) -- Ensure the new version appears on PyPI. -- Verify that the changelog has been updated by looking for the "Update Changelog" pull request. -- Post a brief announcement about the new release on the [Pulp Discourse](https://discourse.pulpproject.org/). +## Final Steps +- Ensure the new version appears on PyPI (it should appear after [Publish Release](https://github.com/pulp/pulp_ostree/actions/workflows/publish.yml) workflow succeeds). +- Verify that the changelog has been updated by looking for the "Update Changelog" pull request (A new PR should be available on the next day). +- [optional] Post a brief announcement about the new release on the [Pulp Discourse](https://discourse.pulpproject.org/).