forked from openstack/requirements
-
Notifications
You must be signed in to change notification settings - Fork 1
/
update.py
208 lines (164 loc) · 6.55 KB
/
update.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
# Copyright 2012 OpenStack Foundation
# Copyright 2013 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
r"""
A simple script to update the requirements files from a global set of
allowable requirements.
The script can be called like this:
$> python update.py ../myproj
Any requirements listed in the target files will have their versions
updated to match the global requirements. Requirements not in the global
files will be dropped.
"""
import optparse
import os
import os.path
import sys
from pip import req
_setup_py_text = """#!/usr/bin/env python
# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT
import setuptools
# In python < 2.7.4, a lazy loading of package `pbr` will break
# setuptools if some other modules registered functions in `atexit`.
# solution from: http://bugs.python.org/issue15881#msg170215
try:
import multiprocessing # noqa
except ImportError:
pass
setuptools.setup(
setup_requires=['pbr'],
pbr=True)
"""
def _parse_pip(pip):
install_require = req.InstallRequirement.from_line(pip)
if install_require.editable:
return pip
elif install_require.url:
return pip
else:
return install_require.req.key
def _pass_through(pip):
return (not pip or
pip.startswith('#') or
pip.startswith('http://tarballs.openstack.org/') or
pip.startswith('-e') or
pip.startswith('-f'))
def _read(filename):
with open(filename, 'r') as f:
return f.read()
def _readlines(filename):
with open(filename, 'r') as f:
return f.readlines()
def _parse_reqs(filename):
reqs = dict()
pip_requires = _readlines(filename)
for pip in pip_requires:
pip = pip.strip()
if _pass_through(pip):
continue
reqs[_parse_pip(pip)] = pip
return reqs
def _sync_requirements_file(source_reqs, dev_reqs, dest_path, suffix):
dest_reqs = _readlines(dest_path)
# this is specifically for global-requirements gate jobs so we don't
# modify the git tree
if suffix:
dest_path = "%s.%s" % (dest_path, suffix)
print("Syncing %s" % dest_path)
with open(dest_path, 'w') as new_reqs:
for old_line in dest_reqs:
old_require = old_line.strip()
if _pass_through(old_require):
new_reqs.write(old_line)
continue
old_pip = _parse_pip(old_require.lower())
# Special cases:
# projects need to align pep8 version on their own time
if "pep8" in old_pip:
new_reqs.write(old_line)
continue
if old_pip in source_reqs:
# allow it to be in dev-requirements
if ((old_pip in dev_reqs) and (old_require.lower() ==
dev_reqs[old_pip])):
new_reqs.write("%s\n" % dev_reqs[old_pip])
else:
new_reqs.write("%s\n" % source_reqs[old_pip])
else:
# What do we do if we find something unexpected?
#
# In the default cause we should die horribly, because
# the point of global requirements was a single lever
# to control all the pip installs in the gate.
#
# However, we do have other projects using
# devstack jobs that might have legitimate reasons to
# override. For those we support NON_STANDARD_REQS=1
# environment variable to turn this into a warning only.
print("'%s' is not in global-requirements.txt" % old_pip)
if os.getenv('NON_STANDARD_REQS', '0') != '1':
sys.exit(1)
def _copy_requires(suffix, dest_dir):
"""Copy requirements files."""
source_reqs = _parse_reqs('global-requirements.txt')
dev_reqs = _parse_reqs('dev-requirements.txt')
target_files = (
'requirements.txt', 'tools/pip-requires',
'test-requirements.txt', 'tools/test-requires',
'requirements-py3.txt', 'test-requirements-py3.txt')
for dest in target_files:
dest_path = os.path.join(dest_dir, dest)
if os.path.exists(dest_path):
print("_sync_requirements_file(%s, %s, %s)" %
(source_reqs, dev_reqs, dest_path))
_sync_requirements_file(source_reqs, dev_reqs, dest_path, suffix)
def _write_setup_py(dest_path):
target_setup_py = os.path.join(dest_path, 'setup.py')
setup_cfg = os.path.join(dest_path, 'setup.cfg')
# If it doesn't have a setup.py, then we don't want to update it
if not os.path.exists(target_setup_py):
return
has_pbr = 'pbr' in _read(target_setup_py)
is_pbr = 'name = pbr' in _read(setup_cfg)
if has_pbr and not is_pbr:
print("Syncing setup.py")
# We only want to sync things that are up to date with pbr mechanics
with open(target_setup_py, 'w') as setup_file:
setup_file.write(_setup_py_text)
def main(options, args):
if len(args) != 1:
print("Must specify directory to update")
sys.exit(1)
_copy_requires(options.suffix, args[0])
_write_setup_py(args[0])
if __name__ == "__main__":
parser = optparse.OptionParser()
parser.add_option("-o", "--output-suffix", dest="suffix", default="",
help="output suffix for updated files (i.e. .global)")
(options, args) = parser.parse_args()
main(options, args)