-
Notifications
You must be signed in to change notification settings - Fork 57
/
requirements_txt_checker.sh
executable file
·114 lines (100 loc) · 4.18 KB
/
requirements_txt_checker.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
#!/bin/bash
# Checks that the requirements.txt file is in sync with
# requirements.in and that there are no known vulnerabilities,
# and that there are no updated packages.
#
# Usage
# $ ./requirements_txt_checker.sh
#
set -euf -o pipefail # abort script on error
# You will need the latest pip-tools and pyup.io's safety tool:
# pip3 install -U pip-tools safety
function run_checks() {
FILE_BASE=$1
# Flatten out all of the dependencies of our dependencies to
# a temporary file.
FN=$(mktemp)
echo "Flattening transitive dependencies of '$FILE_BASE.txt' to a temporary file '$FN'"
pip-compile -v --generate-hashes --resolver=backtracking --allow-unsafe --upgrade --output-file $FN --no-header --no-annotate ${FILE_BASE}.in > /dev/null
# like requirements_txt_updater.sh
# except added --no-annotate (see below)
# The reverse-dependency metadata doesn't seem to be entirely
# accurate and changes nondeterministically? We omit it above
# with --no-annotate and remove it from a copy of our requirements.txt
# file before comparing.
FN2=$(mktemp)
echo "Clean up of temporary requirements file for comparisons"
cat ${FILE_BASE}.txt \
| python3 -c "import sys, re; print(re.sub(r'\n[ \t]+\# .*', '', sys.stdin.read()));" \
> $FN2
# e.g.,
# Reduce
#
# > zipp==3.4.0 \
# > --hash=sha256:102c24ef8f171fd729d46599845e95c7ab894a4cf45f5de11a44cc7444fb1108 \
# > --hash=sha256:ed5eee1974372595f9e416cc7bbeeb12335201d8081ca8a0743c954d4446e5cb
# > # via
# > # importlib-metadata
# > # importlib-resources
# >
# > # The following packages are considered to be unsafe in a requirements file:
# > setuptools==53.0.0 \
# > --hash=sha256:0e86620d658c5ca87a71a283bd308fcaeb4c33e17792ef6f081aec17c171347f \
# > --hash=sha256:1b18ef17d74ba97ac9c0e4b4265f123f07a8ae85d9cd093949fa056d3eeeead5
# > # via
# > # fs
#
# to
#
# > zipp==3.4.0 \
# > --hash=sha256:102c24ef8f171fd729d46599845e95c7ab894a4cf45f5de11a44cc7444fb1108 \
# > --hash=sha256:ed5eee1974372595f9e416cc7bbeeb12335201d8081ca8a0743c954d4446e5cb
# >
# > # The following packages are considered to be unsafe in a requirements file:
# > setuptools==53.0.0 \
# > --hash=sha256:0e86620d658c5ca87a71a283bd308fcaeb4c33e17792ef6f081aec17c171347f \
# > --hash=sha256:1b18ef17d74ba97ac9c0e4b4265f123f07a8ae85d9cd093949fa056d3eeeead5
# Note: sys.stdin.read() appends a '\n', but leaving since ignoring whitespace below
# Compare the requirements.txt in the repository to the one found by
# generating it from requirements.in.
echo "Comparing '$FILE_BASE.txt' to temporary generated version from '$FILE_BASE.in'"
if ! diff -B -u $FN $FN2; then
echo
echo "'${FILE_BASE}.txt' is not in sync with '${FILE_BASE}.in'. Some packages may have updates available. Run requirements_txt_updater.sh."
else
echo "'${FILE_BASE}.txt' is in sync with '${FILE_BASE}.in'."
fi
rm $FN $FN2
echo
}
run_checks requirements
run_checks requirements_mysql
run_checks requirements_util
# Check installed packages for anything outdated. Unfortunately
# this scans *installed* packages, so it assumes you are working
# in a development environment that's been set up and that you
# have no other installed packages because it will report updates
# on those too.
echo "Check installed packages for anything outdated"
FN=$(mktemp)
pip3 list --outdated --format=columns > $FN
if [ -f requirements_txt_checker_ignoreupdates.txt ]; then
# Some updates we ignore. Those are listed in requirements_txt_checker_ignoreupdates.txt
# in a format similar to a requirements.txt, each line like:
# pagename==version
# For each line, remove that line from the `pip list --outdated` output.
for PKG_EQ_VER in $(cat requirements_txt_checker_ignoreupdates.txt); do
PKG_VER=(${PKG_EQ_VER//==/ }) # split on ==, make a bash array
echo "Ignoring new ${PKG_VER[0]} version ${PKG_VER[1]}."
grep "^${PKG_VER[0]} *[^ ][^ ]* *${PKG_VER[1]} " $FN || /bin/true
sed -i "/^${PKG_VER[0]} *[^ ][^ ]* *${PKG_VER[1]} /d" $FN
done
fi
if [ $(cat $FN | wc -l) -gt 2 ]; then
echo "Some packages are out of date:"
echo
cat $FN
else
echo "All packages are up to date with latest upstream versions."
fi
rm $FN