From ba1c51524a087adff5aafd672a44ab7da1314ae8 Mon Sep 17 00:00:00 2001 From: sinoroc Date: Tue, 14 May 2019 17:38:47 +0200 Subject: [PATCH 01/15] Use 'python3' explicitly in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ff11480..8dd31c5 100755 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ Setup script """ From 97978b9b5677acd0719e01f9c5bc54695852b215 Mon Sep 17 00:00:00 2001 From: sinoroc Date: Tue, 14 May 2019 17:39:30 +0200 Subject: [PATCH 02/15] Remove unused 'toxenvname' --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 74f1676..fc9ab33 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ requires = [testenv] commands = - make toxenvname={envname} review + make review extras = test whitelist_externals = From 8902c5f1f46b492117eb2fbdcce1026c420fe9c4 Mon Sep 17 00:00:00 2001 From: sinoroc Date: Tue, 14 May 2019 17:40:02 +0200 Subject: [PATCH 03/15] Clear up confusion in targets check review package --- Makefile | 11 ++++++----- setup.cfg | 5 +++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 4c6691d..42086f8 100644 --- a/Makefile +++ b/Makefile @@ -14,17 +14,19 @@ develop: .PHONY: package -package: sdist wheel check zapp +package: sdist wheel zapp .PHONY: sdist sdist: python3 setup.py sdist + python3 -m twine check dist/*.tar.gz .PHONY: wheel wheel: python3 setup.py bdist_wheel + python3 -m twine check dist/*.whl .PHONY: zapp @@ -33,9 +35,8 @@ zapp: .PHONY: check -check: sdist wheel - python3 -m twine check dist/*.tar.gz - python3 -m twine check dist/*.whl +check: + python3 setup.py check .PHONY: lint @@ -63,7 +64,7 @@ pytest: .PHONY: review -review: +review: check python3 -m pytest --pep8 --pylint diff --git a/setup.cfg b/setup.cfg index 39a458c..ad10adc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,6 +5,11 @@ entry_point = toolmaker.cli:main +[check] +metadata = 1 +strict = 1 + + [metadata] name = toolmaker author = sinoroc From e678ece3c2ba9285473a57728f45840c022ca51f Mon Sep 17 00:00:00 2001 From: sinoroc Date: Mon, 27 May 2019 16:52:59 +0200 Subject: [PATCH 04/15] Set version '0.0.1.dev0' --- CHANGELOG.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index dd2806c..6b29bba 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,8 +2,14 @@ .. Keep the current version number on line number 5 +0.0.1.dev0 +========== + + 0.0.0 ===== +2019-05-13 + .. EOF From 7b941d548d30729b66080b0b28eced4188baa60e Mon Sep 17 00:00:00 2001 From: sinoroc Date: Mon, 27 May 2019 16:54:36 +0200 Subject: [PATCH 05/15] Add 'long_description_content_type' in 'setup.cfg' --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index ad10adc..2334add 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,6 +17,7 @@ author_email = sinoroc.code+python@gmail.com description = toolmaker application license = Apache-2.0 long_description = file: README.rst +long_description_content_type = text/x-rst url = https://pypi.org/project/toolmaker From 45281b7980efc4e44c658a84455d1a7330f3e502 Mon Sep 17 00:00:00 2001 From: sinoroc Date: Sun, 9 Jun 2019 14:18:32 +0200 Subject: [PATCH 06/15] Use importlib_metadata instead of pkg_resources Starting with Python version 3.8 importlib.metadata will be part of the standard library. For earlier Python versions there is the backported library 'importlib_metadata'. This library replaces parts of 'pkg_resources'. --- setup.cfg | 2 +- src/toolmaker/meta.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 2334add..6b31123 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,7 +23,7 @@ url = https://pypi.org/project/toolmaker [options] install_requires = - setuptools + importlib_metadata package_dir = = src packages = find: diff --git a/src/toolmaker/meta.py b/src/toolmaker/meta.py index 29aae1c..0a15be7 100644 --- a/src/toolmaker/meta.py +++ b/src/toolmaker/meta.py @@ -4,10 +4,10 @@ """ Meta information """ -import pkg_resources +import importlib_metadata -VERSION = pkg_resources.get_distribution('toolmaker').version +VERSION = importlib_metadata.version('toolmaker') # EOF From 25322262a8e7cf2a4f558028513fb027d1b97f8a Mon Sep 17 00:00:00 2001 From: sinoroc Date: Sun, 9 Jun 2019 16:46:17 +0200 Subject: [PATCH 07/15] Change default behaviour to not rebuild tools The default behaviour is now to not build a tool if it already exists. Additionnaly there is now an optional flag ('--force', '-f') to force the rebuild of an already existing tool. --- src/toolmaker/cli.py | 28 +++++++++++++++++++++++++--- src/toolmaker/core.py | 15 +++++++++------ 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/src/toolmaker/cli.py b/src/toolmaker/cli.py index 70f0c7f..57f0e3a 100644 --- a/src/toolmaker/cli.py +++ b/src/toolmaker/cli.py @@ -27,6 +27,10 @@ def _create_args_parser(default_config_path, tools_names=None): default=str(default_config_path), type=argparse.FileType('r'), ) + args_parser.add_argument( + '--force', '-f', + action='store_true', + ) group = args_parser.add_mutually_exclusive_group(required=True) group.add_argument( '--all', '-a', @@ -81,13 +85,31 @@ def main(): for tool_name in tools_names: if tool_name.endswith('.pex'): logger.info("Building pex tool '%s'", tool_name) - core.build_pex(cwd_path, venv_context, config, tool_name) + core.build_pex( + cwd_path, + venv_context, + config, + tool_name, + args.force, + ) if tool_name.endswith('.shiv'): logger.info("Building shiv tool '%s'", tool_name) - core.build_shiv(cwd_path, venv_context, config, tool_name) + core.build_shiv( + cwd_path, + venv_context, + config, + tool_name, + args.force, + ) if tool_name.endswith('.zapp'): logger.info("Building zapp tool '%s'", tool_name) - core.build_zapp(cwd_path, venv_context, config, tool_name) + core.build_zapp( + cwd_path, + venv_context, + config, + tool_name, + args.force, + ) # EOF diff --git a/src/toolmaker/core.py b/src/toolmaker/core.py index 543c029..65244ba 100644 --- a/src/toolmaker/core.py +++ b/src/toolmaker/core.py @@ -62,7 +62,7 @@ def _zapp(venv_context, output_file_path, entry_point, requirements): ) -def build_pex(cwd_path, venv_context, config, section_name): +def build_pex(cwd_path, venv_context, config, section_name, force): """ Build pex """ section = config[section_name] @@ -74,10 +74,11 @@ def build_pex(cwd_path, venv_context, config, section_name): req for req in section['requirements'].splitlines() if req ] entry_point = section['entry_point'] - _pex(venv_context, entry_point, output_file_path, requirements) + if force or not output_file_path.exists(): + _pex(venv_context, entry_point, output_file_path, requirements) -def build_shiv(cwd_path, venv_context, config, section_name): +def build_shiv(cwd_path, venv_context, config, section_name, force): """ Build shiv """ section = config[section_name] @@ -89,10 +90,11 @@ def build_shiv(cwd_path, venv_context, config, section_name): req for req in section['requirements'].splitlines() if req ] console_script = section['console_script'] - _shiv(venv_context, console_script, output_file_path, requirements) + if force or not output_file_path.exists(): + _shiv(venv_context, console_script, output_file_path, requirements) -def build_zapp(cwd_path, venv_context, config, section_name): +def build_zapp(cwd_path, venv_context, config, section_name, force): """ Build zapp """ section = config[section_name] @@ -104,7 +106,8 @@ def build_zapp(cwd_path, venv_context, config, section_name): req for req in section['requirements'].splitlines() if req ] entry_point = section['entry_point'] - _zapp(venv_context, output_file_path, entry_point, requirements) + if force or not output_file_path.exists(): + _zapp(venv_context, output_file_path, entry_point, requirements) class _EnvBuilder(venv.EnvBuilder): From 1e8cdcc98e18bdfc9c156cf128f6a9174f7d1f3d Mon Sep 17 00:00:00 2001 From: sinoroc Date: Sun, 9 Jun 2019 17:15:28 +0200 Subject: [PATCH 08/15] Refactor extraction of requirements --- src/toolmaker/core.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/toolmaker/core.py b/src/toolmaker/core.py index 65244ba..d751d34 100644 --- a/src/toolmaker/core.py +++ b/src/toolmaker/core.py @@ -62,6 +62,15 @@ def _zapp(venv_context, output_file_path, entry_point, requirements): ) +def _get_requirements(section): + requirements = [ + req.strip() + for req in section['requirements'].splitlines() + if req.strip() + ] + return requirements + + def build_pex(cwd_path, venv_context, config, section_name, force): """ Build pex """ @@ -70,9 +79,7 @@ def build_pex(cwd_path, venv_context, config, section_name, force): output_dir_path.mkdir(exist_ok=True) output_file_name = section['output_file'] output_file_path = output_dir_path.joinpath(output_file_name) - requirements = [ - req for req in section['requirements'].splitlines() if req - ] + requirements = _get_requirements(section) entry_point = section['entry_point'] if force or not output_file_path.exists(): _pex(venv_context, entry_point, output_file_path, requirements) @@ -86,9 +93,7 @@ def build_shiv(cwd_path, venv_context, config, section_name, force): output_dir_path.mkdir(exist_ok=True) output_file_name = section['output_file'] output_file_path = output_dir_path.joinpath(output_file_name) - requirements = [ - req for req in section['requirements'].splitlines() if req - ] + requirements = _get_requirements(section) console_script = section['console_script'] if force or not output_file_path.exists(): _shiv(venv_context, console_script, output_file_path, requirements) @@ -102,9 +107,7 @@ def build_zapp(cwd_path, venv_context, config, section_name, force): output_dir_path.mkdir(exist_ok=True) output_file_name = section['output_file'] output_file_path = output_dir_path.joinpath(output_file_name) - requirements = [ - req for req in section['requirements'].splitlines() if req - ] + requirements = _get_requirements(section) entry_point = section['entry_point'] if force or not output_file_path.exists(): _zapp(venv_context, output_file_path, entry_point, requirements) From abd7b52c2de0ae39744be30f4dd26bdae9ad41e1 Mon Sep 17 00:00:00 2001 From: sinoroc Date: Mon, 10 Jun 2019 18:17:50 +0200 Subject: [PATCH 09/15] Refactor code Pass config section instead of whole config where possible. Reorder function arguments. Move duplicate code to functions. --- src/toolmaker/cli.py | 7 +++--- src/toolmaker/core.py | 50 +++++++++++++++++++++---------------------- 2 files changed, 28 insertions(+), 29 deletions(-) diff --git a/src/toolmaker/cli.py b/src/toolmaker/cli.py index 57f0e3a..33ec940 100644 --- a/src/toolmaker/cli.py +++ b/src/toolmaker/cli.py @@ -83,13 +83,14 @@ def main(): core.venv_update(venv_context) for tool_name in tools_names: + tool_config = config[tool_name] if tool_name.endswith('.pex'): logger.info("Building pex tool '%s'", tool_name) core.build_pex( cwd_path, venv_context, - config, tool_name, + tool_config, args.force, ) if tool_name.endswith('.shiv'): @@ -97,8 +98,8 @@ def main(): core.build_shiv( cwd_path, venv_context, - config, tool_name, + tool_config, args.force, ) if tool_name.endswith('.zapp'): @@ -106,8 +107,8 @@ def main(): core.build_zapp( cwd_path, venv_context, - config, tool_name, + tool_config, args.force, ) diff --git a/src/toolmaker/core.py b/src/toolmaker/core.py index d751d34..6689b09 100644 --- a/src/toolmaker/core.py +++ b/src/toolmaker/core.py @@ -62,54 +62,52 @@ def _zapp(venv_context, output_file_path, entry_point, requirements): ) -def _get_requirements(section): +def _get_requirements(config): requirements = [ req.strip() - for req in section['requirements'].splitlines() + for req in config['requirements'].splitlines() if req.strip() ] return requirements -def build_pex(cwd_path, venv_context, config, section_name, force): +def _get_output_file_path(cwd_path, tool_name, config): + output_dir_path = cwd_path.joinpath(tool_name) + output_file_name = config['output_file'] + output_file_path = output_dir_path.joinpath(output_file_name) + return output_file_path + + +def build_pex(cwd_path, venv_context, tool_name, config, force): """ Build pex """ - section = config[section_name] - output_dir_path = cwd_path.joinpath(section_name) - output_dir_path.mkdir(exist_ok=True) - output_file_name = section['output_file'] - output_file_path = output_dir_path.joinpath(output_file_name) - requirements = _get_requirements(section) - entry_point = section['entry_point'] + output_file_path = _get_output_file_path(cwd_path, tool_name, config) if force or not output_file_path.exists(): + requirements = _get_requirements(config) + entry_point = config['entry_point'] + output_file_path.parent.mkdir(exist_ok=True) _pex(venv_context, entry_point, output_file_path, requirements) -def build_shiv(cwd_path, venv_context, config, section_name, force): +def build_shiv(cwd_path, venv_context, tool_name, config, force): """ Build shiv """ - section = config[section_name] - output_dir_path = cwd_path.joinpath(section_name) - output_dir_path.mkdir(exist_ok=True) - output_file_name = section['output_file'] - output_file_path = output_dir_path.joinpath(output_file_name) - requirements = _get_requirements(section) - console_script = section['console_script'] + output_file_path = _get_output_file_path(cwd_path, tool_name, config) if force or not output_file_path.exists(): + requirements = _get_requirements(config) + console_script = config['console_script'] + output_file_path.parent.mkdir(exist_ok=True) _shiv(venv_context, console_script, output_file_path, requirements) -def build_zapp(cwd_path, venv_context, config, section_name, force): +def build_zapp(cwd_path, venv_context, tool_name, config, force): """ Build zapp """ - section = config[section_name] - output_dir_path = cwd_path.joinpath(section_name) - output_dir_path.mkdir(exist_ok=True) - output_file_name = section['output_file'] - output_file_path = output_dir_path.joinpath(output_file_name) - requirements = _get_requirements(section) - entry_point = section['entry_point'] + output_file_path = _get_output_file_path(cwd_path, tool_name, config) if force or not output_file_path.exists(): + requirements = _get_requirements(config) + entry_point = config['entry_point'] + output_file_path.parent.mkdir(exist_ok=True) _zapp(venv_context, output_file_path, entry_point, requirements) From 7989dfde148ac467523f1da1520de0d3c91f781f Mon Sep 17 00:00:00 2001 From: sinoroc Date: Mon, 10 Jun 2019 18:59:59 +0200 Subject: [PATCH 10/15] Move some code from 'cli' to 'core' --- src/toolmaker/cli.py | 42 ++---------------------- src/toolmaker/core.py | 74 +++++++++++++++++++++++++++++++++++++------ 2 files changed, 67 insertions(+), 49 deletions(-) diff --git a/src/toolmaker/cli.py b/src/toolmaker/cli.py index 33ec940..1e23b8f 100644 --- a/src/toolmaker/cli.py +++ b/src/toolmaker/cli.py @@ -50,6 +50,7 @@ def main(): """ logger = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) + cwd_path = pathlib.Path.cwd() default_config_path = cwd_path.joinpath('toolmaker.cfg') @@ -58,6 +59,7 @@ def main(): config = None if args.config: + logger.info("Reading configuration from file '%s'", args.config) config = configparser.ConfigParser() try: config.read_file(args.config) @@ -72,45 +74,7 @@ def main(): if args.tools: tools_names = args.tools - logger.info("Preparing to build tools %s", tools_names) - - venv_path = cwd_path.joinpath('venv') - - logger.info("Creating virtual environment '%s'...", venv_path) - venv_context = core.venv_create(venv_path) - - logger.info("Updating virtual environment") - core.venv_update(venv_context) - - for tool_name in tools_names: - tool_config = config[tool_name] - if tool_name.endswith('.pex'): - logger.info("Building pex tool '%s'", tool_name) - core.build_pex( - cwd_path, - venv_context, - tool_name, - tool_config, - args.force, - ) - if tool_name.endswith('.shiv'): - logger.info("Building shiv tool '%s'", tool_name) - core.build_shiv( - cwd_path, - venv_context, - tool_name, - tool_config, - args.force, - ) - if tool_name.endswith('.zapp'): - logger.info("Building zapp tool '%s'", tool_name) - core.build_zapp( - cwd_path, - venv_context, - tool_name, - tool_config, - args.force, - ) + core.build(cwd_path, config, tools_names, args.force) # EOF diff --git a/src/toolmaker/core.py b/src/toolmaker/core.py index 6689b09..945f835 100644 --- a/src/toolmaker/core.py +++ b/src/toolmaker/core.py @@ -5,11 +5,16 @@ """ +import logging import pathlib import subprocess import venv +LOGGER = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) + + def _run_in_venv(venv_context, command): venv_bin_path = pathlib.Path(venv_context.bin_path) command_in_venv = [ @@ -71,44 +76,93 @@ def _get_requirements(config): return requirements -def _get_output_file_path(cwd_path, tool_name, config): - output_dir_path = cwd_path.joinpath(tool_name) +def _get_output_file_path(work_dir_path, tool_name, config): + output_dir_path = work_dir_path.joinpath(tool_name) output_file_name = config['output_file'] output_file_path = output_dir_path.joinpath(output_file_name) return output_file_path -def build_pex(cwd_path, venv_context, tool_name, config, force): +def build_pex(work_dir_path, venv_context, tool_name, config, force): """ Build pex """ - output_file_path = _get_output_file_path(cwd_path, tool_name, config) + output_file_path = _get_output_file_path(work_dir_path, tool_name, config) if force or not output_file_path.exists(): requirements = _get_requirements(config) entry_point = config['entry_point'] output_file_path.parent.mkdir(exist_ok=True) _pex(venv_context, entry_point, output_file_path, requirements) + else: + LOGGER.info("Tool '%s' already exists, build skipped", tool_name) -def build_shiv(cwd_path, venv_context, tool_name, config, force): +def build_shiv(work_dir_path, venv_context, tool_name, config, force): """ Build shiv """ - output_file_path = _get_output_file_path(cwd_path, tool_name, config) + output_file_path = _get_output_file_path(work_dir_path, tool_name, config) if force or not output_file_path.exists(): requirements = _get_requirements(config) console_script = config['console_script'] output_file_path.parent.mkdir(exist_ok=True) _shiv(venv_context, console_script, output_file_path, requirements) + else: + LOGGER.info("Tool '%s' already exists, build skipped", tool_name) -def build_zapp(cwd_path, venv_context, tool_name, config, force): +def build_zapp(work_dir_path, venv_context, tool_name, config, force): """ Build zapp """ - output_file_path = _get_output_file_path(cwd_path, tool_name, config) + output_file_path = _get_output_file_path(work_dir_path, tool_name, config) if force or not output_file_path.exists(): requirements = _get_requirements(config) entry_point = config['entry_point'] output_file_path.parent.mkdir(exist_ok=True) _zapp(venv_context, output_file_path, entry_point, requirements) + else: + LOGGER.info("Tool '%s' already exists, build skipped", tool_name) + + +def build(work_dir_path, config, tools_names, force): + """ Build tools + """ + LOGGER.info("Preparing to build tools %s", tools_names) + venv_path = work_dir_path.joinpath('venv') + + LOGGER.info("Creating virtual environment '%s'...", venv_path) + venv_context = _venv_create(venv_path) + + LOGGER.info("Updating virtual environment") + _venv_update(venv_context) + + for tool_name in tools_names: + tool_config = config[tool_name] + if tool_name.endswith('.pex'): + LOGGER.info("Building pex tool '%s'", tool_name) + build_pex( + work_dir_path, + venv_context, + tool_name, + tool_config, + force, + ) + if tool_name.endswith('.shiv'): + LOGGER.info("Building shiv tool '%s'", tool_name) + build_shiv( + work_dir_path, + venv_context, + tool_name, + tool_config, + force, + ) + if tool_name.endswith('.zapp'): + LOGGER.info("Building zapp tool '%s'", tool_name) + build_zapp( + work_dir_path, + venv_context, + tool_name, + tool_config, + force, + ) class _EnvBuilder(venv.EnvBuilder): @@ -122,7 +176,7 @@ def post_setup(self, context): self.context = context -def venv_create(venv_path): +def _venv_create(venv_path): """ Create virtual environment """ venv_builder = _EnvBuilder(with_pip=True) @@ -130,7 +184,7 @@ def venv_create(venv_path): return venv_builder.context -def venv_update(venv_context): +def _venv_update(venv_context): """ Update virtual environment """ _pip_install(venv_context, ['pip']) From b888be743f230b2e14e345e5a8e464ea5278c456 Mon Sep 17 00:00:00 2001 From: sinoroc Date: Tue, 11 Jun 2019 13:31:07 +0200 Subject: [PATCH 11/15] Add action as CLI optional argument The action has to be specified on the command line. Either one of: * '--build', '-b' for build (existing tools are skipped); * '--rebuild', '-r' for rebuild (existing tools are rebuilt, this replaces the CLI flag '--force', '-f'); * '--delete', '-d' for delete (tool target file is deleted if it exists, then its parent directory is deleted if it is empty) --- src/toolmaker/cli.py | 26 ++++++++++++++++++++------ src/toolmaker/core.py | 29 ++++++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 7 deletions(-) diff --git a/src/toolmaker/cli.py b/src/toolmaker/cli.py index 1e23b8f..bf47826 100644 --- a/src/toolmaker/cli.py +++ b/src/toolmaker/cli.py @@ -27,16 +27,25 @@ def _create_args_parser(default_config_path, tools_names=None): default=str(default_config_path), type=argparse.FileType('r'), ) - args_parser.add_argument( - '--force', '-f', + action_group = args_parser.add_mutually_exclusive_group() + action_group.add_argument( + '--build', '-b', + action='store_true', + ) + action_group.add_argument( + '--rebuild', '-r', + action='store_true', + ) + action_group.add_argument( + '--delete', '-d', action='store_true', ) - group = args_parser.add_mutually_exclusive_group(required=True) - group.add_argument( + tools_group = args_parser.add_mutually_exclusive_group(required=True) + tools_group.add_argument( '--all', '-a', action='store_true', ) - group.add_argument( + tools_group.add_argument( '--tools', '-t', choices=tools_names, metavar='tool', @@ -74,7 +83,12 @@ def main(): if args.tools: tools_names = args.tools - core.build(cwd_path, config, tools_names, args.force) + if args.delete: + core.delete(cwd_path, config, tools_names) + elif args.rebuild: + core.build(cwd_path, config, tools_names, force=True) + else: + core.build(cwd_path, config, tools_names) # EOF diff --git a/src/toolmaker/core.py b/src/toolmaker/core.py index 945f835..6d7e9bb 100644 --- a/src/toolmaker/core.py +++ b/src/toolmaker/core.py @@ -122,7 +122,7 @@ def build_zapp(work_dir_path, venv_context, tool_name, config, force): LOGGER.info("Tool '%s' already exists, build skipped", tool_name) -def build(work_dir_path, config, tools_names, force): +def build(work_dir_path, config, tools_names, force=False): """ Build tools """ LOGGER.info("Preparing to build tools %s", tools_names) @@ -165,6 +165,33 @@ def build(work_dir_path, config, tools_names, force): ) +def delete(work_dir_path, config, tools_names): + """ Delete tools + """ + LOGGER.info("Deleting tools %s", tools_names) + + for tool_name in tools_names: + output_file_path = _get_output_file_path( + work_dir_path, + tool_name, + config[tool_name], + ) + if output_file_path.exists() and output_file_path.is_file(): + LOGGER.info("Deleting file '%s'", output_file_path) + output_file_path.unlink() + output_dir_path = output_file_path.parent + if output_dir_path.exists() and output_dir_path.is_dir(): + LOGGER.info("Deleting directory '%s'", output_dir_path) + try: + output_dir_path.rmdir() + except OSError as ose: + import errno + if ose.errno == errno.ENOTEMPTY: + LOGGER.warning("Directory '%s' not empty", output_dir_path) + else: + raise + + class _EnvBuilder(venv.EnvBuilder): def __init__(self, *args, **kwargs): From 1f05382a7d4deb9c67a9efb47f928029eb9a250a Mon Sep 17 00:00:00 2001 From: sinoroc Date: Wed, 12 Jun 2019 16:06:34 +0200 Subject: [PATCH 12/15] Replace CLI calls in venv with API calls The first version of this tool was meant to be a single file script without external dependencies. This was achieved by creating a virtual environment with 'venv' and populate it with the necessary tools. This approach works well and is still valid. The tool now outgrew the one-single-file phase and thus the independence from external dependencies is not a constraint anymore. --- setup.cfg | 3 + src/toolmaker/core.py | 147 +++++++++++++----------------------------- 2 files changed, 48 insertions(+), 102 deletions(-) diff --git a/setup.cfg b/setup.cfg index 6b31123..fcd745b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,6 +24,9 @@ url = https://pypi.org/project/toolmaker [options] install_requires = importlib_metadata + pex + shiv + zapp package_dir = = src packages = find: diff --git a/src/toolmaker/core.py b/src/toolmaker/core.py index 6d7e9bb..a3bb24d 100644 --- a/src/toolmaker/core.py +++ b/src/toolmaker/core.py @@ -6,65 +6,45 @@ import logging -import pathlib -import subprocess -import venv + +import pex +import pex.bin.pex +import shiv +import shiv.cli +import zapp LOGGER = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) -def _run_in_venv(venv_context, command): - venv_bin_path = pathlib.Path(venv_context.bin_path) - command_in_venv = [ - str(venv_bin_path.joinpath(command[0])), - ] + command[1:] - subprocess.run(command_in_venv) - - -def _pip_install(venv_context, packages): - command = [ - 'pip', - 'install', - '--upgrade', - ] + packages - _run_in_venv(venv_context, command) - - -def _pex(venv_context, entry_point, output_file_path, requirements): - _run_in_venv( - venv_context, - [ - 'pex', - '--entry-point={}'.format(entry_point), - '--output-file={}'.format(str(output_file_path)), - ] + requirements, +def _pex(requirements, entry_point, output_file_path): + cmd = [ + '--entry-point={}'.format(entry_point), + '--output-file={}'.format(str(output_file_path)), + ] + requirements + pex.bin.pex.main(cmd) + + +def _shiv(requirements, entry_point, output_file_path): + # Since it is decorated by 'click', the 'main' function is not callable + # with its original arguments. The original function is "hidden" under + # 'callback'. + shiv.cli.main.callback( + str(output_file_path), # output_file + entry_point, # entry_point + None, # console_script + '/usr/bin/env python3', # python + None, # site_packages + True, # compressed + False, # compile_pyc + False, # extend_pythonpath + requirements, # pip_args ) -def _shiv(venv_context, console_script, output_file_path, requirements): - _run_in_venv( - venv_context, - [ - 'shiv', - '--console-script', - console_script, - '--output-file', - str(output_file_path), - ] + requirements, - ) - - -def _zapp(venv_context, output_file_path, entry_point, requirements): - _run_in_venv( - venv_context, - [ - 'zapp', - output_file_path, - entry_point, - ] + requirements, - ) +def _zapp(requirements, entry_point, output_file_path): + zapp.core.build_zapp(requirements, entry_point, output_file_path) def _get_requirements(config): @@ -83,41 +63,44 @@ def _get_output_file_path(work_dir_path, tool_name, config): return output_file_path -def build_pex(work_dir_path, venv_context, tool_name, config, force): +def _build_pex(work_dir_path, tool_name, config, force): """ Build pex """ output_file_path = _get_output_file_path(work_dir_path, tool_name, config) if force or not output_file_path.exists(): + LOGGER.info("Building pex tool '%s'...", tool_name) requirements = _get_requirements(config) entry_point = config['entry_point'] output_file_path.parent.mkdir(exist_ok=True) - _pex(venv_context, entry_point, output_file_path, requirements) + _pex(requirements, entry_point, output_file_path) else: LOGGER.info("Tool '%s' already exists, build skipped", tool_name) -def build_shiv(work_dir_path, venv_context, tool_name, config, force): +def _build_shiv(work_dir_path, tool_name, config, force): """ Build shiv """ output_file_path = _get_output_file_path(work_dir_path, tool_name, config) if force or not output_file_path.exists(): + LOGGER.info("Building shiv tool '%s'...", tool_name) requirements = _get_requirements(config) - console_script = config['console_script'] + entry_point = config['entry_point'] output_file_path.parent.mkdir(exist_ok=True) - _shiv(venv_context, console_script, output_file_path, requirements) + _shiv(requirements, entry_point, output_file_path) else: LOGGER.info("Tool '%s' already exists, build skipped", tool_name) -def build_zapp(work_dir_path, venv_context, tool_name, config, force): +def _build_zapp(work_dir_path, tool_name, config, force): """ Build zapp """ output_file_path = _get_output_file_path(work_dir_path, tool_name, config) if force or not output_file_path.exists(): + LOGGER.info("Building zapp tool '%s'...", tool_name) requirements = _get_requirements(config) entry_point = config['entry_point'] output_file_path.parent.mkdir(exist_ok=True) - _zapp(venv_context, output_file_path, entry_point, requirements) + _zapp(requirements, entry_point, output_file_path) else: LOGGER.info("Tool '%s' already exists, build skipped", tool_name) @@ -125,40 +108,27 @@ def build_zapp(work_dir_path, venv_context, tool_name, config, force): def build(work_dir_path, config, tools_names, force=False): """ Build tools """ - LOGGER.info("Preparing to build tools %s", tools_names) - venv_path = work_dir_path.joinpath('venv') - - LOGGER.info("Creating virtual environment '%s'...", venv_path) - venv_context = _venv_create(venv_path) - - LOGGER.info("Updating virtual environment") - _venv_update(venv_context) + LOGGER.info("Building tools %s...", tools_names) for tool_name in tools_names: tool_config = config[tool_name] if tool_name.endswith('.pex'): - LOGGER.info("Building pex tool '%s'", tool_name) - build_pex( + _build_pex( work_dir_path, - venv_context, tool_name, tool_config, force, ) if tool_name.endswith('.shiv'): - LOGGER.info("Building shiv tool '%s'", tool_name) - build_shiv( + _build_shiv( work_dir_path, - venv_context, tool_name, tool_config, force, ) if tool_name.endswith('.zapp'): - LOGGER.info("Building zapp tool '%s'", tool_name) - build_zapp( + _build_zapp( work_dir_path, - venv_context, tool_name, tool_config, force, @@ -168,7 +138,7 @@ def build(work_dir_path, config, tools_names, force=False): def delete(work_dir_path, config, tools_names): """ Delete tools """ - LOGGER.info("Deleting tools %s", tools_names) + LOGGER.info("Deleting tools %s...", tools_names) for tool_name in tools_names: output_file_path = _get_output_file_path( @@ -192,31 +162,4 @@ def delete(work_dir_path, config, tools_names): raise -class _EnvBuilder(venv.EnvBuilder): - - def __init__(self, *args, **kwargs): - self.context = None - super().__init__(*args, **kwargs) - - def post_setup(self, context): - """ Override """ - self.context = context - - -def _venv_create(venv_path): - """ Create virtual environment - """ - venv_builder = _EnvBuilder(with_pip=True) - venv_builder.create(venv_path) - return venv_builder.context - - -def _venv_update(venv_context): - """ Update virtual environment - """ - _pip_install(venv_context, ['pip']) - _pip_install(venv_context, ['setuptools']) - _pip_install(venv_context, ['pex[cachecontrol,requests]', 'shiv', 'zapp']) - - # EOF From f6af3371c19da15cbec1a3e1f65e96e43aaaf949 Mon Sep 17 00:00:00 2001 From: sinoroc Date: Wed, 12 Jun 2019 18:24:22 +0200 Subject: [PATCH 13/15] Improve documentation Improve 'README.rst'. Improve 'toolmaker.cfg' and move to 'example.cfg' --- README.rst | 74 ++++++++++++++++++++++++++++++++++++++++++++++++--- example.cfg | 25 +++++++++++++++++ toolmaker.cfg | 60 ----------------------------------------- 3 files changed, 96 insertions(+), 63 deletions(-) create mode 100644 example.cfg delete mode 100644 toolmaker.cfg diff --git a/README.rst b/README.rst index c90cf0c..b752346 100644 --- a/README.rst +++ b/README.rst @@ -1,11 +1,74 @@ .. +Introduction +============ + +This tool automates the build of Python tools according to a configuration +file. The tools can be built with `zapp`_, `shiv`_, or `pex`_. + + +Repositories +------------ + +Binary distributions: + +* http://pypi.org/project/toolmaker/ + +Source code: + +* https://gitlab.com/sinoroc/toolmaker +* https://github.com/sinoroc/toolmaker + + Usage ===== -Place in a subdirectory of your ``bin`` directory and use in combination with -*GNU stow*. +Configuration +------------- + +By default this tool looks for a configuration file ``toolmaker.cfg`` in the +current working directory. + +.. code:: + + [http.pex] + entry_point = http.server + output_file = http + requirements = + + [pipdeptree.zapp] + entry_point = pipdeptree:main + output_file = pipdeptree + requirements = + pipdeptree + setuptools + + [shiv.shiv] + console_script = shiv + output_file = shiv + requirements = + shiv + + +Action +------ + +The action can be specified on the command line. Either one of: + +* ``--build``, ``-b`` to build (already existing tools are skipped); +* ``--rebuild``, ``-r`` to rebuild (already existing tools are rebuilt); +* ``--delete``, ``-d`` to delete (tool target file is deleted if it exists, then + its parent directory is deleted if it is empty). + +The default action when no flag is specified is to build the tools. + + +Tips +---- + +Place in a subdirectory of a directory that is available on your ``PATH`` +(typically your ``~/bin`` directory) and use in combination with `GNU Stow`_. Details @@ -14,7 +77,7 @@ Details Similar projects ---------------- -* https://github.com/Valassis-Digital-Media/Zapper +* `Zapper`_ Hacking @@ -64,8 +127,13 @@ Outside of a Python virtual environment run the following command:: .. Links .. _`GNU Make`: https://www.gnu.org/software/make/ +.. _`GNU Stow`: https://www.gnu.org/software/stow/ +.. _`pex`: https://pypi.org/project/pex/ .. _`pytest`: https://pytest.org/ +.. _`shiv`: https://pypi.org/project/shiv/ .. _`tox`: https://tox.readthedocs.io/ +.. _`zapp`: https://pypi.org/project/zapp/ +.. _`Zapper`: https://github.com/Valassis-Digital-Media/Zapper .. EOF diff --git a/example.cfg b/example.cfg new file mode 100644 index 0000000..31a446b --- /dev/null +++ b/example.cfg @@ -0,0 +1,25 @@ +# + + +[http.pex] +entry_point = http.server +output_file = http +requirements = + + +[pipdeptree.zapp] +entry_point = pipdeptree:main +output_file = pipdeptree +requirements = + pipdeptree + setuptools + + +[shiv.shiv] +entry_point = shiv.cli:main +output_file = shiv +requirements = + shiv + + +# EOF diff --git a/toolmaker.cfg b/toolmaker.cfg deleted file mode 100644 index aa5ebae..0000000 --- a/toolmaker.cfg +++ /dev/null @@ -1,60 +0,0 @@ -# - - -[cookiecutter.shiv] -console_script = cookiecutter -output_file = cookiecutter -requirements = - cookiecutter - - -[http.pex] -entry_point = http.server -output_file = http -requirements = - - -[pex.pex] -entry_point = pex.bin.pex:main -output_file = pex -requirements = - pex[cachecontrol,requests] - - -[pipdeptree.zapp] -entry_point = pipdeptree:main -output_file = pipdeptree -requirements = - pipdeptree - - -[shiv.shiv] -console_script = shiv -output_file = shiv -requirements = - shiv - - -[tox.shiv] -console_script = tox -output_file = tox -requirements = - tox - tox-venv - - -[twine.shiv] -console_script = twine -output_file = twine -requirements = - twine - - -[zapp.zapp] -entry_point = zapp.cli:main -output_file = zapp -requirements = - zapp - - -# EOF From 8714d0e90b90612c11e879813c510e303a8e23e2 Mon Sep 17 00:00:00 2001 From: sinoroc Date: Wed, 12 Jun 2019 18:30:29 +0200 Subject: [PATCH 14/15] Allow failures for Python dev version on GitLab CI --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f4e4638..96fbae3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,6 +21,7 @@ 'test py38 dev': <<: *job_test_common + allow_failure: true image: 'python:3.8-rc' variables: 'TOXENV': 'py38' From 204cd04e507e7c08e1bed0073edca7075addd71b Mon Sep 17 00:00:00 2001 From: sinoroc Date: Wed, 12 Jun 2019 18:36:05 +0200 Subject: [PATCH 15/15] Prepare release version 0.0.1 --- CHANGELOG.rst | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 6b29bba..3a6aae9 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,8 +2,21 @@ .. Keep the current version number on line number 5 -0.0.1.dev0 -========== +0.0.1 +===== + +2019-06-12 + +* Add action as CLI optional argument. The action has to be specified on the + command line. Either one of: + + * ``--build``, ``-b`` to build (existing tools are skipped); + * ``--rebuild``, ``-r`` to rebuild (existing tools are rebuilt); + * ``--delete``, ``-d`` to delete (tool target file is deleted if it exists, + its parent directory is deleted if it is empty). + +* Replace CLI calls to external tools in virtual environment with API calls to + external libraries. 0.0.0 @@ -11,5 +24,7 @@ 2019-05-13 +Release initial version. + .. EOF