From 9d7482c0b13eb14583d5fcff2720778986a8ff64 Mon Sep 17 00:00:00 2001 From: edX requirements bot <49161187+edx-requirements-bot@users.noreply.github.com> Date: Sat, 23 Jul 2022 11:06:59 -0400 Subject: [PATCH] fix: Install pip and pip-tools in upgrade script (#6770) * fix: Install pip and pip-tools in upgrade script * fix: upgrade Co-authored-by: Jawayria --- Makefile | 11 +++++++- .../roles/aws/templates/requirements.txt.j2 | 10 ++++---- requirements.txt | 12 ++++----- requirements/common_constraints.txt | 25 +++++++++++++++++++ requirements/constraints.txt | 2 ++ requirements/jenkins.in | 2 +- requirements/pip-tools.in | 1 + requirements/pip-tools.txt | 14 ++++++++--- requirements/pip.in | 7 ++++++ requirements/pip.txt | 16 ++++++++++++ util/jenkins/requirements-cloudflare.txt | 4 +-- util/pingdom/requirements.txt | 4 +-- util/vpc-tools/requirements.txt | 4 +-- 13 files changed, 90 insertions(+), 22 deletions(-) create mode 100644 requirements/common_constraints.txt create mode 100644 requirements/pip.in create mode 100644 requirements/pip.txt diff --git a/Makefile b/Makefile index c642ea22cd8..f3918dc8869 100755 --- a/Makefile +++ b/Makefile @@ -20,12 +20,21 @@ requirements: pip install -qr pre-requirements.txt --exists-action w pip install -qr requirements.txt --exists-action w +COMMON_CONSTRAINTS_TXT=requirements/common_constraints.txt +.PHONY: $(COMMON_CONSTRAINTS_TXT) +$(COMMON_CONSTRAINTS_TXT): + wget -O "$(@)" https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt || touch "$(@)" + upgrade: export CUSTOM_COMPILE_COMMAND=make upgrade -upgrade: ## update the pip requirements files to use the latest releases satisfying our constraints +upgrade: $(COMMON_CONSTRAINTS_TXT) + ## update the pip requirements files to use the latest releases satisfying our constraints pip install -qr pre-requirements.txt --exists-action w pip install -qr requirements/pip-tools.txt # Make sure to compile files after any other files they include! + pip-compile --allow-unsafe --rebuild --upgrade -o requirements/pip.txt requirements/pip.in pip-compile --upgrade -o requirements/pip-tools.txt requirements/pip-tools.in + pip install -qr requirements/pip.txt + pip install -qr requirements/pip-tools.txt pip-compile --upgrade -o requirements.txt requirements/base.in pip-compile --upgrade -o playbooks/roles/aws/templates/requirements.txt.j2 requirements/aws.in pip-compile --upgrade -o util/elasticsearch/requirements.txt requirements/elasticsearch.in diff --git a/playbooks/roles/aws/templates/requirements.txt.j2 b/playbooks/roles/aws/templates/requirements.txt.j2 index 5fb598be16f..9249c77305c 100644 --- a/playbooks/roles/aws/templates/requirements.txt.j2 +++ b/playbooks/roles/aws/templates/requirements.txt.j2 @@ -4,13 +4,13 @@ # # make upgrade # -awscli==1.25.0 +awscli==1.25.18 # via -r requirements/aws.in boto==2.49.0 # via -r requirements/aws.in -boto3==1.24.0 +boto3==1.24.18 # via -r requirements/aws.in -botocore==1.27.0 +botocore==1.27.18 # via # awscli # boto3 @@ -19,7 +19,7 @@ colorama==0.4.4 # via awscli docutils==0.16 # via awscli -jmespath==1.0.0 +jmespath==1.0.1 # via # boto3 # botocore @@ -29,7 +29,7 @@ python-dateutil==2.8.2 # via # botocore # s3cmd -python-magic==0.4.26 +python-magic==0.4.27 # via s3cmd pyyaml==5.3.1 # via diff --git a/requirements.txt b/requirements.txt index ec54e8f6c72..3cbed8b3238 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ # ansible==2.9.0 # via -r requirements/base.in -awscli==1.25.0 +awscli==1.25.18 # via -r requirements/base.in bcrypt==3.1.7 # via @@ -14,14 +14,14 @@ bcrypt==3.1.7 # paramiko boto==2.49.0 # via -r requirements/base.in -boto3==1.24.0 +boto3==1.24.18 # via -r requirements/base.in -botocore==1.27.0 +botocore==1.27.18 # via # awscli # boto3 # s3transfer -certifi==2022.5.18.1 +certifi==2022.6.15 # via requests cffi==1.15.0 # via @@ -54,7 +54,7 @@ jinja2==2.8 # via # -r requirements/base.in # ansible -jmespath==1.0.0 +jmespath==1.0.1 # via # boto3 # botocore @@ -91,7 +91,7 @@ pyyaml==5.4.1 # -r requirements/base.in # ansible # awscli -requests==2.27.1 +requests==2.28.0 # via # -r requirements/base.in # datadog diff --git a/requirements/common_constraints.txt b/requirements/common_constraints.txt new file mode 100644 index 00000000000..cca3ccd308c --- /dev/null +++ b/requirements/common_constraints.txt @@ -0,0 +1,25 @@ +# A central location for most common version constraints +# (across edx repos) for pip-installation. +# +# Similar to other constraint files this file doesn't install any packages. +# It specifies version constraints that will be applied if a package is needed. +# When pinning something here, please provide an explanation of why it is a good +# idea to pin this package across all edx repos, Ideally, link to other information +# that will help people in the future to remove the pin when possible. +# Writing an issue against the offending project and linking to it here is good. +# +# Note: Changes to this file will automatically be used by other repos, referencing +# this file from Github directly. It does not require packaging in edx-lint. + + +# using LTS django version +Django<4.0 + +# elasticsearch>=7.14.0 includes breaking changes in it which caused issues in discovery upgrade process. +# elastic search changelog: https://www.elastic.co/guide/en/enterprise-search/master/release-notes-7.14.0.html +elasticsearch<7.14.0 + +setuptools<60 + +# django-simple-history>3.0.0 adds indexing and causes a lot of migrations to be affected +django-simple-history==3.0.0 diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 340665643fe..aaa989ce567 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -1 +1,3 @@ +-c common_constraints.txt + bcrypt<3.2.0 # 3.2.0 dropped support for python 2.7 diff --git a/requirements/jenkins.in b/requirements/jenkins.in index c155d37fa1a..f69c8fe2cea 100644 --- a/requirements/jenkins.in +++ b/requirements/jenkins.in @@ -13,4 +13,4 @@ opsgenie-sdk==0.3.1 PyMySQL==0.9.3 python-gnupg redis==2.10.6 -splunk-sdk==1.6.6 +splunk-sdk==1.6.16 # older versions have been yanked diff --git a/requirements/pip-tools.in b/requirements/pip-tools.in index c17d0b41e75..3f1b64ae937 100644 --- a/requirements/pip-tools.in +++ b/requirements/pip-tools.in @@ -1,3 +1,4 @@ # Just the dependencies to run pip-tools, mainly for the "upgrade" make target +-c constraints.txt pip-tools # Contains pip-compile, used to generate pip requirements files diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index ce56f4c7f38..97f4ed93dc9 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -4,14 +4,22 @@ # # make upgrade # +build==0.8.0 + # via pip-tools click==8.1.3 # via pip-tools +packaging==21.3 + # via build pep517==0.12.0 - # via pip-tools -pip-tools==6.6.2 + # via build +pip-tools==6.7.0 # via -r requirements/pip-tools.in +pyparsing==3.0.9 + # via packaging tomli==2.0.1 - # via pep517 + # via + # build + # pep517 wheel==0.37.1 # via pip-tools diff --git a/requirements/pip.in b/requirements/pip.in new file mode 100644 index 00000000000..715478cdc0c --- /dev/null +++ b/requirements/pip.in @@ -0,0 +1,7 @@ +-c constraints.txt +# Core dependencies for installing other packages + +pip +setuptools +wheel + diff --git a/requirements/pip.txt b/requirements/pip.txt new file mode 100644 index 00000000000..8a667c4a29e --- /dev/null +++ b/requirements/pip.txt @@ -0,0 +1,16 @@ +# +# This file is autogenerated by pip-compile with python 3.8 +# To update, run: +# +# make upgrade +# +wheel==0.37.1 + # via -r requirements/pip.in + +# The following packages are considered to be unsafe in a requirements file: +pip==22.1.2 + # via -r requirements/pip.in +setuptools==59.8.0 + # via + # -c requirements/common_constraints.txt + # -r requirements/pip.in diff --git a/util/jenkins/requirements-cloudflare.txt b/util/jenkins/requirements-cloudflare.txt index bbfeaaa1cb3..a3ee4e40405 100644 --- a/util/jenkins/requirements-cloudflare.txt +++ b/util/jenkins/requirements-cloudflare.txt @@ -4,7 +4,7 @@ # # make upgrade # -certifi==2022.5.18.1 +certifi==2022.6.15 # via requests charset-normalizer==2.0.12 # via requests @@ -12,7 +12,7 @@ click==8.1.3 # via -r requirements/cloudflare.in idna==3.3 # via requests -requests==2.27.1 +requests==2.28.0 # via -r requirements/cloudflare.in urllib3==1.26.9 # via requests diff --git a/util/pingdom/requirements.txt b/util/pingdom/requirements.txt index 54560ad3e33..f2befe795ab 100644 --- a/util/pingdom/requirements.txt +++ b/util/pingdom/requirements.txt @@ -4,7 +4,7 @@ # # make upgrade # -certifi==2022.5.18.1 +certifi==2022.6.15 # via requests charset-normalizer==2.0.12 # via requests @@ -14,7 +14,7 @@ idna==3.3 # via requests pyyaml==6.0 # via -r requirements/pingdom.in -requests==2.27.1 +requests==2.28.0 # via -r requirements/pingdom.in six==1.14.0 # via -r requirements/pingdom.in diff --git a/util/vpc-tools/requirements.txt b/util/vpc-tools/requirements.txt index 6692bd4ad7a..e3fca741449 100644 --- a/util/vpc-tools/requirements.txt +++ b/util/vpc-tools/requirements.txt @@ -6,7 +6,7 @@ # boto==2.49.0 # via -r requirements/vpc-tools.in -certifi==2022.5.18.1 +certifi==2022.6.15 # via requests charset-normalizer==2.0.12 # via requests @@ -14,7 +14,7 @@ docopt==0.6.2 # via -r requirements/vpc-tools.in idna==3.3 # via requests -requests==2.27.1 +requests==2.28.0 # via -r requirements/vpc-tools.in urllib3==1.26.9 # via requests