fix: Install pip and pip-tools in upgrade script (#6770)

* fix: Install pip and pip-tools in upgrade script * fix: upgrade Co-authored-by: Jawayria <[email protected]>
openedx-unsupported · Jul 23, 2022 · 9d7482c · 9d7482c
1 parent ffd68cd
commit 9d7482c
Show file tree

Hide file tree

Showing 13 changed files with 90 additions and 22 deletions.
diff --git a/Makefile b/Makefile
@@ -20,12 +20,21 @@ requirements:
 	pip install -qr pre-requirements.txt --exists-action w
 	pip install -qr requirements.txt --exists-action w
 
+COMMON_CONSTRAINTS_TXT=requirements/common_constraints.txt
+.PHONY: $(COMMON_CONSTRAINTS_TXT)
+$(COMMON_CONSTRAINTS_TXT):
+	wget -O "$(@)" https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt || touch "$(@)"
+
 upgrade: export CUSTOM_COMPILE_COMMAND=make upgrade
-upgrade: ## update the pip requirements files to use the latest releases satisfying our constraints
+upgrade: $(COMMON_CONSTRAINTS_TXT)
+	## update the pip requirements files to use the latest releases satisfying our constraints
 	pip install -qr pre-requirements.txt --exists-action w
 	pip install -qr requirements/pip-tools.txt
 	# Make sure to compile files after any other files they include!
+	pip-compile --allow-unsafe --rebuild --upgrade -o requirements/pip.txt requirements/pip.in
 	pip-compile --upgrade -o requirements/pip-tools.txt requirements/pip-tools.in
+	pip install -qr requirements/pip.txt
+	pip install -qr requirements/pip-tools.txt
 	pip-compile --upgrade -o requirements.txt requirements/base.in
 	pip-compile --upgrade -o playbooks/roles/aws/templates/requirements.txt.j2 requirements/aws.in
 	pip-compile --upgrade -o util/elasticsearch/requirements.txt requirements/elasticsearch.in

diff --git a/playbooks/roles/aws/templates/requirements.txt.j2 b/playbooks/roles/aws/templates/requirements.txt.j2
@@ -4,13 +4,13 @@
 #
 #    make upgrade
 #
-awscli==1.25.0
+awscli==1.25.18
     # via -r requirements/aws.in
 boto==2.49.0
     # via -r requirements/aws.in
-boto3==1.24.0
+boto3==1.24.18
     # via -r requirements/aws.in
-botocore==1.27.0
+botocore==1.27.18
     # via
     #   awscli
     #   boto3
@@ -19,7 +19,7 @@ colorama==0.4.4
     # via awscli
 docutils==0.16
     # via awscli
-jmespath==1.0.0
+jmespath==1.0.1
     # via
     #   boto3
     #   botocore
@@ -29,7 +29,7 @@ python-dateutil==2.8.2
     # via
     #   botocore
     #   s3cmd
-python-magic==0.4.26
+python-magic==0.4.27
     # via s3cmd
 pyyaml==5.3.1
     # via

diff --git a/requirements.txt b/requirements.txt
@@ -6,22 +6,22 @@
 #
 ansible==2.9.0
     # via -r requirements/base.in
-awscli==1.25.0
+awscli==1.25.18
     # via -r requirements/base.in
 bcrypt==3.1.7
     # via
     #   -c requirements/constraints.txt
     #   paramiko
 boto==2.49.0
     # via -r requirements/base.in
-boto3==1.24.0
+boto3==1.24.18
     # via -r requirements/base.in
-botocore==1.27.0
+botocore==1.27.18
     # via
     #   awscli
     #   boto3
     #   s3transfer
-certifi==2022.5.18.1
+certifi==2022.6.15
     # via requests
 cffi==1.15.0
     # via
@@ -54,7 +54,7 @@ jinja2==2.8
     # via
     #   -r requirements/base.in
     #   ansible
-jmespath==1.0.0
+jmespath==1.0.1
     # via
     #   boto3
     #   botocore
@@ -91,7 +91,7 @@ pyyaml==5.4.1
     #   -r requirements/base.in
     #   ansible
     #   awscli
-requests==2.27.1
+requests==2.28.0
     # via
     #   -r requirements/base.in
     #   datadog

diff --git a/requirements/common_constraints.txt b/requirements/common_constraints.txt
@@ -0,0 +1,25 @@
+# A central location for most common version constraints
+# (across edx repos) for pip-installation.
+#
+# Similar to other constraint files this file doesn't install any packages.
+# It specifies version constraints that will be applied if a package is needed.
+# When pinning something here, please provide an explanation of why it is a good
+# idea to pin this package across all edx repos, Ideally, link to other information
+# that will help people in the future to remove the pin when possible.
+# Writing an issue against the offending project and linking to it here is good.
+#
+# Note: Changes to this file will automatically be used by other repos, referencing
+#  this file from Github directly. It does not require packaging in edx-lint.
+
+
+# using LTS django version
+Django<4.0
+
+# elasticsearch>=7.14.0 includes breaking changes in it which caused issues in discovery upgrade process.
+# elastic search changelog: https://www.elastic.co/guide/en/enterprise-search/master/release-notes-7.14.0.html
+elasticsearch<7.14.0
+
+setuptools<60
+
+# django-simple-history>3.0.0 adds indexing and causes a lot of migrations to be affected
+django-simple-history==3.0.0
diff --git a/requirements/constraints.txt b/requirements/constraints.txt
@@ -1 +1,3 @@
+-c common_constraints.txt
+
 bcrypt<3.2.0 # 3.2.0 dropped support for python 2.7
diff --git a/requirements/jenkins.in b/requirements/jenkins.in
@@ -13,4 +13,4 @@ opsgenie-sdk==0.3.1
 PyMySQL==0.9.3
 python-gnupg
 redis==2.10.6
-splunk-sdk==1.6.6
+splunk-sdk==1.6.16      # older versions have been yanked
diff --git a/requirements/pip-tools.in b/requirements/pip-tools.in
@@ -1,3 +1,4 @@
 # Just the dependencies to run pip-tools, mainly for the "upgrade" make target
+-c constraints.txt
 
 pip-tools                 # Contains pip-compile, used to generate pip requirements files
diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt
@@ -4,14 +4,22 @@
 #
 #    make upgrade
 #
+build==0.8.0
+    # via pip-tools
 click==8.1.3
     # via pip-tools
+packaging==21.3
+    # via build
 pep517==0.12.0
-    # via pip-tools
-pip-tools==6.6.2
+    # via build
+pip-tools==6.7.0
     # via -r requirements/pip-tools.in
+pyparsing==3.0.9
+    # via packaging
 tomli==2.0.1
-    # via pep517
+    # via
+    #   build
+    #   pep517
 wheel==0.37.1
     # via pip-tools
 

diff --git a/requirements/pip.in b/requirements/pip.in
@@ -0,0 +1,7 @@
+-c constraints.txt
+# Core dependencies for installing other packages
+
+pip
+setuptools
+wheel
+
diff --git a/requirements/pip.txt b/requirements/pip.txt
@@ -0,0 +1,16 @@
+#
+# This file is autogenerated by pip-compile with python 3.8
+# To update, run:
+#
+#    make upgrade
+#
+wheel==0.37.1
+    # via -r requirements/pip.in
+
+# The following packages are considered to be unsafe in a requirements file:
+pip==22.1.2
+    # via -r requirements/pip.in
+setuptools==59.8.0
+    # via
+    #   -c requirements/common_constraints.txt
+    #   -r requirements/pip.in
diff --git a/util/jenkins/requirements-cloudflare.txt b/util/jenkins/requirements-cloudflare.txt
@@ -4,15 +4,15 @@
 #
 #    make upgrade
 #
-certifi==2022.5.18.1
+certifi==2022.6.15
     # via requests
 charset-normalizer==2.0.12
     # via requests
 click==8.1.3
     # via -r requirements/cloudflare.in
 idna==3.3
     # via requests
-requests==2.27.1
+requests==2.28.0
     # via -r requirements/cloudflare.in
 urllib3==1.26.9
     # via requests
diff --git a/util/pingdom/requirements.txt b/util/pingdom/requirements.txt
@@ -4,7 +4,7 @@
 #
 #    make upgrade
 #
-certifi==2022.5.18.1
+certifi==2022.6.15
     # via requests
 charset-normalizer==2.0.12
     # via requests
@@ -14,7 +14,7 @@ idna==3.3
     # via requests
 pyyaml==6.0
     # via -r requirements/pingdom.in
-requests==2.27.1
+requests==2.28.0
     # via -r requirements/pingdom.in
 six==1.14.0
     # via -r requirements/pingdom.in

diff --git a/util/vpc-tools/requirements.txt b/util/vpc-tools/requirements.txt
@@ -6,15 +6,15 @@
 #
 boto==2.49.0
     # via -r requirements/vpc-tools.in
-certifi==2022.5.18.1
+certifi==2022.6.15
     # via requests
 charset-normalizer==2.0.12
     # via requests
 docopt==0.6.2
     # via -r requirements/vpc-tools.in
 idna==3.3
     # via requests
-requests==2.27.1
+requests==2.28.0
     # via -r requirements/vpc-tools.in
 urllib3==1.26.9
     # via requests