-
Notifications
You must be signed in to change notification settings - Fork 1
/
generate_packages.py
122 lines (96 loc) · 3.74 KB
/
generate_packages.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
import sys
import os
import random
import string
import tarfile
import shutil
import numpy
from helper_modules.distribution import distribution_factory
import json
from helper_modules.package import Package
import argparse
from subprocess import check_output
import multiprocessing
CORES = 40
RANDOM = None
CWD = os.path.dirname(os.path.realpath(__file__))
def worker(packages):
for p in packages:
write_package('%s/web' % CWD, p)
def write_packages(pkgs_dir, packages):
pkg_shards = [[] for i in range(CORES)]
for i in range(len(packages)):
pkg_shards[i%CORES].append(packages[i])
pool = multiprocessing.Pool(CORES)
pool.map(worker, pkg_shards)
def create_tar(pkg_dir, pkg):
data_dir = '%s/%s/data' % (pkg_dir, pkg.name)
os.makedirs(data_dir)
if pkg.uncompressed > 3300:
with open('%s/data.dat' % data_dir, 'w') as data:
random_char = chr(random.randint(0, 255))
data.write(random_char * (pkg.uncompressed - 3300))
with tarfile.open(pkg_dir + '-0.1.tar.gz', 'w:gz') as tar:
tar.add(pkg_dir, arcname=pkg.name)
size = os.stat(pkg_dir + '-0.1.tar.gz').st_size
if size < pkg.compressed:
with open('%s/ballast.dat' % pkg_dir, 'wb') as ballast:
rand_len = pkg.compressed - size
ballast.write(RANDOM[:rand_len])
with tarfile.open(pkg_dir + '-0.1.tar.gz', 'w:gz') as tar:
tar.add(pkg_dir, arcname=pkg.name)
def write_pkg_simple(simple_dir, pkg):
os.makedirs('%s/%s' % (simple_dir, pkg.name))
with open('%s/%s/index.html' % (simple_dir, pkg.name), 'w') as index:
index.write('''
<body><a href="../../packages/{pkg_dir}-0.1.tar.gz">{pkg_tar}</a></body>
'''.format(pkg_dir=pkg.get_dir(), pkg_tar=pkg.name + '-0.1.tar.gz'))
def write_package(mirror_dir, pkg):
pkg_dir = '%s/packages/%s' % (mirror_dir, pkg.get_dir())
# create package directories
os.makedirs('%s/%s' % (pkg_dir, pkg.name))
# create contents
with open('%s/setup.py' % pkg_dir, 'w') as f:
f.write(pkg.setup_code())
with open('%s/%s/__init__.py' % (pkg_dir, pkg.name), 'w') as f:
f.write(pkg.init_code())
shutil.copyfile('load_simulator.so', '%s/load_simulator.so' % (pkg_dir))
shutil.copyfile('load_simulator.so', '%s/%s/load_simulator.so' % (pkg_dir, pkg.name))
create_tar(pkg_dir, pkg)
shutil.rmtree(pkg_dir)
write_pkg_simple('%s/simple' % mirror_dir, pkg)
def write_simple(mirror_dir, pkgs):
with open('%s/simple/index.html' % mirror_dir, 'w') as index:
index.write('<body>')
for pkg in pkgs:
index.write('<a href="{pkg_name}">{pkg_name}</a>'.format(pkg_name=pkg.name))
index.write('</body>')
def main():
if len(sys.argv) != 3:
print('usage: python %s <spec_file> <random_file>')
return
with open(sys.argv[1]) as spec_file:
spec = json.load(spec_file)
max_size = max(pkg['compressed'] for pkg in spec)
global RANDOM
if len(sys.argv) >= 3:
with open(sys.argv[2], 'rb') as f:
RANDOM = f.read()
if len(RANDOM) < max_size:
print('random file is not large enough: %d < %d' % (len(RANDOM), max_size))
return
mirror_dir = '%s/web' % CWD
# create mirror dir if not found
if not os.path.exists(mirror_dir):
os.makedirs(mirror_dir)
# ensure we have the load simulator binary
os.system('gcc -fPIC -shared -I/usr/include/python2.7 -lpython2.7 load_simulator.c -o load_simulator.so')
print('Creating packages...')
packages = [Package(**entry) for entry in spec]
print('Writing out packages....')
write_packages(mirror_dir, packages)
print('Writing out index.html')
write_simple(mirror_dir, packages)
print('Done')
if __name__ == '__main__':
main()