Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add windows support #180

Open
wants to merge 5 commits into
base: dev
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions build.bat
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
@echo off
call setup.bat || exit /b
call .venv/Scripts/activate || exit /b
pip install pyinstaller || exit /b
python win-package.py || exit /b
2 changes: 1 addition & 1 deletion build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ package_scripts() {
continue
fi
tmp_name=$(mktemp).spec
cat "precise.template.spec" | replace "%%SCRIPT%%" "$script" | replace "%%TRAIN_LIBS%%" "$train_libs" > "$tmp_name"
cat "precise.template.spec" | replace "%%SCRIPT%%" "$script" | replace "%%TRAIN_LIBS%%" "$train_libs" | replace "%%STRIP%%" "True" > "$tmp_name"
pyinstaller -y "$tmp_name"
if [ "$exe" != "$combined_folder" ]; then
cp -R dist/$exe/* "dist/$combined_folder"
Expand Down
8 changes: 6 additions & 2 deletions precise.template.spec
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,11 @@ from glob import iglob
from os.path import basename, dirname, abspath
import os
import fnmatch
from PyInstaller.utils.hooks import collect_data_files

script_name = '%%SCRIPT%%'
train_libs = %%TRAIN_LIBS%%
strip = True
strip = %%STRIP%%
site_packages = '.venv/lib/python3.6/site-packages/'
hidden_imports = ['prettyparse', 'speechpy']
binaries = []
Expand All @@ -29,11 +30,13 @@ if train_libs:
]
hidden_imports += ['h5py']

datas = collect_data_files('astor', subdir=None, include_py_files=False)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What does this return? A list of data files?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The astor lib uses an internal file called VERSION.l, which pyinstaller does not include by default.

Pyinstaller does allow you to specify data files which should be bundled along.

See
https://pyinstaller.readthedocs.io/en/stable/spec-files.html#adding-data-files

The function returns a list of tuples where the first is a file name and the second is the folder to insert it to.

Here it will collect all yhe non python files in the root directory of astor.

If that's preferable I think I can specify the exact file instead as shown here:
https://pyinstaller.readthedocs.io/en/stable/spec-files.html#using-data-files-from-a-module


a = Analysis(
[abspath('precise/scripts/{}.py'.format(script_name))],
pathex=['.'],
binaries=binaries,
datas=[],
datas=datas,
hiddenimports=hidden_imports,
hookspath=[],
runtime_hooks=[],
Expand All @@ -43,6 +46,7 @@ a = Analysis(
cipher=block_cipher
)


for i in range(len(a.binaries)):
dest, origin, kind = a.binaries[i]
if '_pywrap_tensorflow_internal' in dest:
Expand Down
9 changes: 9 additions & 0 deletions setup.bat
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
@echo off
if not exist ".venv" (
py -3.6 -m venv .venv || exit /b
)
call .venv/Scripts/activate || exit /b
pip install -e runner/ || exit /b
pip install -e . || exit /b
rem Optional, for comparison
pip install pocketsphinx || exit /b
4 changes: 2 additions & 2 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,8 @@
},
include_package_data=True,
install_requires=[
'numpy==1.16',
'tensorflow>=1.13,<1.14', # Must be on piwheels
'numpy==1.16.2',
'tensorflow==1.13.1', # Must be on piwheels
'sonopy',
'pyaudio',
'keras<=2.1.5',
Expand Down
90 changes: 90 additions & 0 deletions win-package.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
# This file should not be used directly but using build.bat

import os
import os.path
from os.path import join, normpath, basename
import platform
import re
import tempfile
import shutil
import glob
import tarfile
from distutils import dir_util

from precise import __version__


import hashlib
def filemd5(fname):
hash_md5 = hashlib.md5()
with open(fname, "rb") as f:
for chunk in iter(lambda: f.read(4096), b""):
hash_md5.update(chunk)
return hash_md5.hexdigest()

def make_tarfile(output_filename, source_dir):
with tarfile.open(output_filename, "w:gz") as tar:
tar.add(source_dir, arcname=basename(source_dir))

def tar_name(tar_prefix):
arch = platform.machine()
return "{tar_prefix}_{version}_win_{archname}.tar.gz".format(
tar_prefix=tar_prefix,
version=__version__,
archname = arch)

def filecontains(path, string):
try:
f = open(path)
content = f.read()
f.close()
return string in content
except FileNotFoundError:
return False

def package_scripts(tar_prefix, combined_folder, scripts, train_libs):
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it looks like it could be useful to split this function up a little bit for readability

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, can do.
Basically it's a port of build.sh, but why not :)

completed_file=join("dist", "completed_{}.txt".format(combined_folder))
if not os.path.isfile(completed_file):
try:
shutil.rmtree(join("dist",combined_folder))
except FileNotFoundError:
pass

for script in scripts:
exe = "precise-{}".format(script.replace('_', '-'))
if filecontains(completed_file, exe):
continue
with tempfile.NamedTemporaryFile(mode = 'w',suffix='.spec', delete=False) as temp_file:
temp_path = temp_file.name
with open("precise.template.spec") as template_spec:
spec = template_spec.read() \
.replace("%%SCRIPT%%",script) \
.replace("%%TRAIN_LIBS%%", train_libs) \
.replace("%%STRIP%%", "False")
temp_file.write(spec + '\n')
if os.system("pyinstaller -y {} --workpath=build/{}".format(temp_path, exe)) != 0:
raise Exception("pyinstaller error")
print(temp_path)
if exe != combined_folder:
dir_util.copy_tree(join("dist",exe), join("dist",combined_folder), update=1)
shutil.rmtree(join("dist",exe))
shutil.rmtree(join("build",exe))
with open(completed_file, 'a') as f:
f.write(exe + '\n')

out_name = tar_name(tar_prefix)
make_tarfile(join('dist', out_name), join('dist', combined_folder))
with open(join('dist', "{}.md5".format(out_name)), 'w') as md5file:
md5file.write(filemd5(join('dist', out_name)))

def main():
all_scripts=re.findall('(?<=precise.scripts.)[a-z_]+', open('setup.py').read())
package_scripts("precise-all", "precise", all_scripts, "True")
package_scripts("precise-engine", "precise-engine", ["engine"], "False")

tar_1 = join("dist",tar_name("precise-all"))
tar_2 = join("dist",tar_name("precise-engine"))
print("Wrote to {} and {}".format(tar_1, tar_2))

if __name__ == '__main__':
main()