diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index bad75f3b..afe0b118 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -9,12 +9,17 @@ ADD ./{{ item.name }} ./{{ item.name }} # S3 botocore needs to be patched to handle responses from minio during 0-byte uploads # Hacking botocore (https://github.com/boto/botocore/pull/1990) -RUN pip3 install +RUN pip3 install --upgrade pip setuptools wheel && \ + rm -rf /root/.cache/pip && \ + pip3 install {%- if s3_test | default(false) -%} {{ " " }}git+https://github.com/gerrod3/botocore.git@fix-100-continue {%- endif -%} {%- for item in plugins -%} {{ " " }}{{ item.source }} +{%- if item.upperbounds | default(false) -%} +{{ " " }}-c ./{{ item.name }}/upperbounds_constraints.txt +{%- endif -%} {%- if item.lowerbounds | default(false) -%} {{ " " }}-c ./{{ item.name }}/lowerbounds_constraints.txt {%- endif -%} @@ -22,7 +27,8 @@ RUN pip3 install {{ " " }}-r ./{{ item.name }}/ci_requirements.txt {%- endif -%} {%- endfor %} -{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt +{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt && \ + rm -rf /root/.cache/pip {% if pulp_env is defined and pulp_env %} {% for key, value in pulp_env.items() %} diff --git a/.ci/scripts/calc_constraints.py b/.ci/scripts/calc_constraints.py new file mode 100755 index 00000000..3372754e --- /dev/null +++ b/.ci/scripts/calc_constraints.py @@ -0,0 +1,118 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_ostree' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +import argparse +import fileinput +import urllib.request +import sys +from packaging.requirements import Requirement +from packaging.version import Version +import yaml + + +CORE_TEMPLATE_URL = "https://raw.githubusercontent.com/pulp/pulpcore/main/template_config.yml" + + +def fetch_pulpcore_upper_bound(requirement): + with urllib.request.urlopen(CORE_TEMPLATE_URL) as f: + template = yaml.safe_load(f.read()) + supported_versions = template["supported_release_branches"] + supported_versions.append(template["latest_release_branch"]) + applicable_versions = sorted( + requirement.specifier.filter((Version(v) for v in supported_versions)) + ) + if len(applicable_versions) == 0: + raise Exception("No supported pulpcore version in required range.") + return f"{requirement.name}~={applicable_versions[-1]}" + + +def split_comment(line): + split_line = line.split("#", maxsplit=1) + try: + comment = " # " + split_line[1].strip() + except IndexError: + comment = "" + return split_line[0].strip(), comment + + +def to_upper_bound(req): + try: + requirement = Requirement(req) + except ValueError: + return f"# UNPARSABLE: {req}" + else: + if requirement.name == "pulpcore": + # An exception to allow for pulpcore deprecation policy. + return fetch_pulpcore_upper_bound(requirement) + for spec in requirement.specifier: + if spec.operator == "~=": + return f"# NO BETTER CONSTRAINT: {req}" + if spec.operator == "<=": + operator = "==" + max_version = spec.version + return f"{requirement.name}{operator}{max_version}" + if spec.operator == "<": + operator = "~=" + version = Version(spec.version) + if version.micro != 0: + max_version = f"{version.major}.{version.minor}.{version.micro-1}" + elif version.minor != 0: + max_version = f"{version.major}.{version.minor-1}" + elif version.major != 0: + max_version = f"{version.major-1}.0" + else: + return f"# NO BETTER CONSTRAINT: {req}" + return f"{requirement.name}{operator}{max_version}" + return f"# NO UPPER BOUND: {req}" + + +def to_lower_bound(req): + try: + requirement = Requirement(req) + except ValueError: + return f"# UNPARSABLE: {req}" + else: + for spec in requirement.specifier: + if spec.operator == ">=": + if requirement.name == "pulpcore": + # Currently an exception to allow for pulpcore bugfix releases. + # TODO Semver libraries should be allowed too. + operator = "~=" + else: + operator = "==" + min_version = spec.version + return f"{requirement.name}{operator}{min_version}" + return f"# NO LOWER BOUND: {req}" + + +def main(): + """Calculate constraints for the lower bound of dependencies where possible.""" + parser = argparse.ArgumentParser( + prog=sys.argv[0], + description="Calculate constraints for the lower or upper bound of dependencies where " + "possible.", + ) + parser.add_argument("-u", "--upper", action="store_true") + parser.add_argument("filename", nargs="*") + args = parser.parse_args() + + with fileinput.input(files=args.filename) as req_file: + for line in req_file: + if line.strip().startswith("#"): + # Shortcut comment only lines + print(line.strip()) + else: + req, comment = split_comment(line) + if args.upper: + new_req = to_upper_bound(req) + else: + new_req = to_lower_bound(req) + print(new_req + comment) + + +if __name__ == "__main__": + main() diff --git a/.ci/scripts/calc_deps_lowerbounds.py b/.ci/scripts/calc_deps_lowerbounds.py deleted file mode 100755 index 599006ee..00000000 --- a/.ci/scripts/calc_deps_lowerbounds.py +++ /dev/null @@ -1,34 +0,0 @@ -# WARNING: DO NOT EDIT! -# -# This file was generated by plugin_template, and is managed by it. Please use -# './plugin-template --github pulp_ostree' to update this file. -# -# For more info visit https://github.com/pulp/plugin_template - -from packaging.requirements import Requirement - - -def main(): - """Calculate the lower bound of dependencies where possible.""" - with open("requirements.txt") as req_file: - for line in req_file: - try: - requirement = Requirement(line) - except ValueError: - print(line.strip()) - else: - for spec in requirement.specifier: - if spec.operator == ">=": - if requirement.name == "pulpcore": - operator = "~=" - else: - operator = "==" - min_version = str(spec)[2:] - print(f"{requirement.name}{operator}{min_version}") - break - else: - print(line.strip()) - - -if __name__ == "__main__": - main() diff --git a/.github/template_gitref b/.github/template_gitref index b222a637..3d9674dc 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-351-g4e8a1a7 +2021.08.26-361-gcd6f9f0 diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index 17715ef4..936ea91a 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -65,9 +65,11 @@ then exit $s fi +if [[ "$TEST" = "pulp" ]]; then + python3 .ci/scripts/calc_constraints.py -u requirements.txt > upperbounds_constraints.txt +fi if [[ "$TEST" = "lowerbounds" ]]; then - python3 .ci/scripts/calc_deps_lowerbounds.py > lowerbounds_constraints.txt - sed -i 's/\[.*\]//g' lowerbounds_constraints.txt + python3 .ci/scripts/calc_constraints.py requirements.txt > lowerbounds_constraints.txt fi if [ -f $POST_BEFORE_INSTALL ]; then diff --git a/.github/workflows/scripts/build_python_client.sh b/.github/workflows/scripts/build_python_client.sh index 9fcf2ffb..72c616f4 100755 --- a/.github/workflows/scripts/build_python_client.sh +++ b/.github/workflows/scripts/build_python_client.sh @@ -18,9 +18,7 @@ cd "$(dirname "$(realpath -e "$0")")"/../../.. pushd ../pulp-openapi-generator rm -rf "pulp_ostree-client" -# We need to copy that over to be visible in the container. -cp "../pulp_ostree/ostree-api.json" . -./gen-client.sh "ostree-api.json" "ostree" python "pulp_ostree" +./gen-client.sh "../pulp_ostree/ostree-api.json" "ostree" python "pulp_ostree" pushd pulp_ostree-client python setup.py sdist bdist_wheel --python-tag py3 diff --git a/.github/workflows/scripts/build_ruby_client.sh b/.github/workflows/scripts/build_ruby_client.sh index 18f7dd72..a5856333 100755 --- a/.github/workflows/scripts/build_ruby_client.sh +++ b/.github/workflows/scripts/build_ruby_client.sh @@ -18,15 +18,7 @@ cd "$(dirname "$(realpath -e "$0")")"/../../.. pushd ../pulp-openapi-generator rm -rf "pulp_ostree-client" -# We need to copy that over to be visible in the container. -#cp "../pulp_ostree/ostree-api.json" . -#./gen-client.sh "ostree-api.json" "ostree" ruby "pulp_ostree" - -# ------------- -# The generator still needs to have it called api.json at this time... -cp "../pulp_ostree/api.json" . -./gen-client.sh "api.json" "ostree" ruby "pulp_ostree" -# ------------- +./gen-client.sh "../pulp_ostree/ostree-api.json" "ostree" ruby "pulp_ostree" pushd pulp_ostree-client gem build pulp_ostree_client diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index b99832a0..329a40a8 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -55,6 +55,11 @@ if [[ -f ../../ci_requirements.txt ]]; then ci_requirements: true VARSYAML fi +if [ "$TEST" = "pulp" ]; then + cat >> vars/main.yaml << VARSYAML + upperbounds: true +VARSYAML +fi if [ "$TEST" = "lowerbounds" ]; then cat >> vars/main.yaml << VARSYAML lowerbounds: true diff --git a/.github/workflows/update_ci.yml b/.github/workflows/update_ci.yml index 48e72129..7a8cec3c 100644 --- a/.github/workflows/update_ci.yml +++ b/.github/workflows/update_ci.yml @@ -79,7 +79,7 @@ jobs: - name: "Run update" working-directory: "pulp_ostree" run: | - ../plugin_template/scripts/update_ci.sh + ../plugin_template/scripts/update_ci.sh --release - name: "Create Pull Request for CI files" uses: "peter-evans/create-pull-request@v6" diff --git a/docs/template_gitref b/docs/template_gitref index b222a637..3d9674dc 100644 --- a/docs/template_gitref +++ b/docs/template_gitref @@ -1 +1 @@ -2021.08.26-351-g4e8a1a7 +2021.08.26-361-gcd6f9f0