From 08e86ba9b53e31ed9a12f7f93b021c7998f06a30 Mon Sep 17 00:00:00 2001 From: sinoroc Date: Wed, 12 Jun 2019 18:59:04 +0200 Subject: [PATCH 01/12] Set version 0.0.2.dev0 --- CHANGELOG.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 3a6aae9..d2eb59c 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,10 @@ .. Keep the current version number on line number 5 +0.0.2.dev0 +========== + + 0.0.1 ===== From 94134fbbb6703ca83d6d714152d81ec5ea0f6f2f Mon Sep 17 00:00:00 2001 From: sinoroc Date: Fri, 6 Sep 2019 00:16:45 +0200 Subject: [PATCH 02/12] Use the secure URL (https) for PyPI in README.rst --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index b752346..38672c1 100644 --- a/README.rst +++ b/README.rst @@ -13,7 +13,7 @@ Repositories Binary distributions: -* http://pypi.org/project/toolmaker/ +* https://pypi.org/project/toolmaker/ Source code: From 94d3ad1f68fe8b6f3b76b506d15322e32996c31f Mon Sep 17 00:00:00 2001 From: sinoroc Date: Fri, 6 Sep 2019 00:20:06 +0200 Subject: [PATCH 03/12] Add 'pipx' to list of similar projects --- README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.rst b/README.rst index 38672c1..0a5c835 100644 --- a/README.rst +++ b/README.rst @@ -77,6 +77,7 @@ Details Similar projects ---------------- +* `pipx`_ * `Zapper`_ @@ -129,6 +130,7 @@ Outside of a Python virtual environment run the following command:: .. _`GNU Make`: https://www.gnu.org/software/make/ .. _`GNU Stow`: https://www.gnu.org/software/stow/ .. _`pex`: https://pypi.org/project/pex/ +.. _`pipx`: https://pipxproject.github.io/pipx/ .. _`pytest`: https://pytest.org/ .. _`shiv`: https://pypi.org/project/shiv/ .. _`tox`: https://tox.readthedocs.io/ From 380b77144e7da78a73f127c8c51a3d06f99e48f5 Mon Sep 17 00:00:00 2001 From: sinoroc Date: Fri, 6 Sep 2019 00:39:03 +0200 Subject: [PATCH 04/12] Remove CI against development versions of Python --- .gitlab-ci.yml | 7 ------- .travis.yml | 10 ---------- 2 files changed, 17 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 96fbae3..c8cb61d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -19,12 +19,5 @@ variables: 'TOXENV': 'py37' -'test py38 dev': - <<: *job_test_common - allow_failure: true - image: 'python:3.8-rc' - variables: - 'TOXENV': 'py38' - ... EOF diff --git a/.travis.yml b/.travis.yml index 2e713c2..f9ee8fc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,17 +7,7 @@ language: 'python' python: - '3.6' - - '3.6-dev' - '3.7' - - '3.7-dev' - - '3.8-dev' - -matrix: - allow_failures: - - python: '3.6-dev' - - python: '3.7-dev' - - python: '3.8-dev' - fast_finish: true install: - 'python3 -m pip install tox tox-travis tox-venv' From 50027174811a2a3eaab11f3efded0f2166d4d3f8 Mon Sep 17 00:00:00 2001 From: sinoroc Date: Sun, 8 Sep 2019 20:18:41 +0200 Subject: [PATCH 05/12] Package as 'pex' instead of 'zapp' The 'zapp' version of this tool does not work. This is due to 'pex' then not being able to extract its vendored elements directly from the zip archive. On the other hand 'pex' works differently and extracts the whole application to disk first. --- Makefile | 8 ++++---- setup.cfg | 4 ---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 42086f8..649b987 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ develop: .PHONY: package -package: sdist wheel zapp +package: sdist wheel pex .PHONY: sdist @@ -29,9 +29,9 @@ wheel: python3 -m twine check dist/*.whl -.PHONY: zapp -zapp: - python3 setup.py bdist_zapp +.PHONY: pex +pex: + python3 setup.py bdist_pex .PHONY: check diff --git a/setup.cfg b/setup.cfg index fcd745b..339dba4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,10 +1,6 @@ # -[bdist_zapp] -entry_point = toolmaker.cli:main - - [check] metadata = 1 strict = 1 From 37e7e2025a753d65589e88879ef72a8dbe879f3d Mon Sep 17 00:00:00 2001 From: sinoroc Date: Sun, 8 Sep 2019 20:22:46 +0200 Subject: [PATCH 06/12] Fix project URL in 'setup.cfg' --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 339dba4..fc87a0a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,7 +14,7 @@ description = toolmaker application license = Apache-2.0 long_description = file: README.rst long_description_content_type = text/x-rst -url = https://pypi.org/project/toolmaker +url = https://pypi.org/project/toolmaker/ [options] From 4ff5e0cfa4d4637ed3ddf4d4e9f95268f6717f03 Mon Sep 17 00:00:00 2001 From: sinoroc Date: Sun, 8 Sep 2019 20:23:09 +0200 Subject: [PATCH 07/12] Add the extras 'pex[cachecontrol, requests]' --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index fc87a0a..0e302d0 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,7 +20,7 @@ url = https://pypi.org/project/toolmaker/ [options] install_requires = importlib_metadata - pex + pex[cachecontrol, requests] shiv zapp package_dir = From fcba1e937673d822fe15deb4f43c7ee91f9661db Mon Sep 17 00:00:00 2001 From: sinoroc Date: Thu, 12 Sep 2019 17:48:45 +0200 Subject: [PATCH 08/12] Add namespaces in configuration file Maybe at some point the configuration could be placed in a file shared with other tools. --- README.rst | 8 +-- example.cfg | 8 +-- src/toolmaker/cli.py | 9 +-- src/toolmaker/core.py | 135 +++++++++++++++++++++++++++--------------- 4 files changed, 98 insertions(+), 62 deletions(-) diff --git a/README.rst b/README.rst index 0a5c835..363ac18 100644 --- a/README.rst +++ b/README.rst @@ -32,20 +32,20 @@ current working directory. .. code:: - [http.pex] + [toolmaker.tool.pex:http.pex] entry_point = http.server output_file = http requirements = - [pipdeptree.zapp] + [toolmaker.tool.zapp:pipdeptree.zapp] entry_point = pipdeptree:main output_file = pipdeptree requirements = pipdeptree setuptools - [shiv.shiv] - console_script = shiv + [toolmaker.tool.shiv:shiv.shiv] + entry_point = shiv.cli:main output_file = shiv requirements = shiv diff --git a/example.cfg b/example.cfg index 31a446b..0215d81 100644 --- a/example.cfg +++ b/example.cfg @@ -1,21 +1,19 @@ # -[http.pex] +[toolmaker.tool.pex:http.pex] entry_point = http.server output_file = http requirements = - -[pipdeptree.zapp] +[toolmaker.tool.zapp:pipdeptree.zapp] entry_point = pipdeptree:main output_file = pipdeptree requirements = pipdeptree setuptools - -[shiv.shiv] +[toolmaker.tool.shiv:shiv.shiv] entry_point = shiv.cli:main output_file = shiv requirements = diff --git a/src/toolmaker/cli.py b/src/toolmaker/cli.py index bf47826..6079f4d 100644 --- a/src/toolmaker/cli.py +++ b/src/toolmaker/cli.py @@ -66,16 +66,17 @@ def main(): args_parser = _create_args_parser(default_config_path) args = args_parser.parse_args() - config = None + raw_config = None if args.config: logger.info("Reading configuration from file '%s'", args.config) - config = configparser.ConfigParser() + raw_config = configparser.ConfigParser() try: - config.read_file(args.config) + raw_config.read_file(args.config) except configparser.Error as config_error: args_parser.error(config_error) - tools_names = config.sections() + config = core.parse_config(raw_config) + tools_names = list(config['tools'].keys()) if not args.all: args_parser = _create_args_parser(default_config_path, tools_names) args = args_parser.parse_args() diff --git a/src/toolmaker/core.py b/src/toolmaker/core.py index a3bb24d..7138c08 100644 --- a/src/toolmaker/core.py +++ b/src/toolmaker/core.py @@ -56,17 +56,18 @@ def _get_requirements(config): return requirements -def _get_output_file_path(work_dir_path, tool_name, config): - output_dir_path = work_dir_path.joinpath(tool_name) +def _get_output_file_path(work_dir_path, config): + output_dir_path = work_dir_path.joinpath(config['name']) output_file_name = config['output_file'] output_file_path = output_dir_path.joinpath(output_file_name) return output_file_path -def _build_pex(work_dir_path, tool_name, config, force): +def _build_pex(work_dir_path, config, force): """ Build pex """ - output_file_path = _get_output_file_path(work_dir_path, tool_name, config) + tool_name = config['name'] + output_file_path = _get_output_file_path(work_dir_path, config) if force or not output_file_path.exists(): LOGGER.info("Building pex tool '%s'...", tool_name) requirements = _get_requirements(config) @@ -77,10 +78,11 @@ def _build_pex(work_dir_path, tool_name, config, force): LOGGER.info("Tool '%s' already exists, build skipped", tool_name) -def _build_shiv(work_dir_path, tool_name, config, force): +def _build_shiv(work_dir_path, config, force): """ Build shiv """ - output_file_path = _get_output_file_path(work_dir_path, tool_name, config) + tool_name = config['name'] + output_file_path = _get_output_file_path(work_dir_path, config) if force or not output_file_path.exists(): LOGGER.info("Building shiv tool '%s'...", tool_name) requirements = _get_requirements(config) @@ -91,10 +93,11 @@ def _build_shiv(work_dir_path, tool_name, config, force): LOGGER.info("Tool '%s' already exists, build skipped", tool_name) -def _build_zapp(work_dir_path, tool_name, config, force): +def _build_zapp(work_dir_path, config, force): """ Build zapp """ - output_file_path = _get_output_file_path(work_dir_path, tool_name, config) + tool_name = config['name'] + output_file_path = _get_output_file_path(work_dir_path, config) if force or not output_file_path.exists(): LOGGER.info("Building zapp tool '%s'...", tool_name) requirements = _get_requirements(config) @@ -105,34 +108,64 @@ def _build_zapp(work_dir_path, tool_name, config, force): LOGGER.info("Tool '%s' already exists, build skipped", tool_name) +TOOL_SECTION_NAMES = ( + 'toolmaker.tool.pex', + 'toolmaker.tool.shiv', + 'toolmaker.tool.zapp', +) + + +def _parse_tool_config(raw_config, section_name): + tool_config = None + tokens = section_name.split(':', maxsplit=1) + if len(tokens) == 2 and tokens[0] in TOOL_SECTION_NAMES: + tool_config = { + 'name': tokens[1], + 'type': tokens[0].split('.')[2], + } + tool_config.update(raw_config[section_name]) + return tool_config + + +def parse_config(raw_config): + """ Parse the raw configration file to build a configuration dictionary. + """ + config = { + 'tools': {}, + } + for section_name in raw_config.sections(): + tool_config = _parse_tool_config(raw_config, section_name) + if tool_config: + config['tools'][tool_config['name']] = tool_config + return config + + def build(work_dir_path, config, tools_names, force=False): """ Build tools """ LOGGER.info("Building tools %s...", tools_names) for tool_name in tools_names: - tool_config = config[tool_name] - if tool_name.endswith('.pex'): - _build_pex( - work_dir_path, - tool_name, - tool_config, - force, - ) - if tool_name.endswith('.shiv'): - _build_shiv( - work_dir_path, - tool_name, - tool_config, - force, - ) - if tool_name.endswith('.zapp'): - _build_zapp( - work_dir_path, - tool_name, - tool_config, - force, - ) + if tool_name in config['tools']: + tool_config = config['tools'][tool_name] + if tool_config['type'] == 'pex': + _build_pex( + work_dir_path, + tool_config, + force, + ) + if tool_config['type'] == 'shiv': + _build_shiv( + work_dir_path, + tool_config, + force, + ) + if tool_config['type'] == 'zapp': + _build_zapp( + work_dir_path, + tool_config, + force, + ) def delete(work_dir_path, config, tools_names): @@ -141,25 +174,29 @@ def delete(work_dir_path, config, tools_names): LOGGER.info("Deleting tools %s...", tools_names) for tool_name in tools_names: - output_file_path = _get_output_file_path( - work_dir_path, - tool_name, - config[tool_name], - ) - if output_file_path.exists() and output_file_path.is_file(): - LOGGER.info("Deleting file '%s'", output_file_path) - output_file_path.unlink() - output_dir_path = output_file_path.parent - if output_dir_path.exists() and output_dir_path.is_dir(): - LOGGER.info("Deleting directory '%s'", output_dir_path) - try: - output_dir_path.rmdir() - except OSError as ose: - import errno - if ose.errno == errno.ENOTEMPTY: - LOGGER.warning("Directory '%s' not empty", output_dir_path) - else: - raise + if tool_name in config['tools']: + tool_config = config['tools'][tool_name] + output_file_path = _get_output_file_path( + work_dir_path, + tool_config, + ) + if output_file_path.exists() and output_file_path.is_file(): + LOGGER.info("Deleting file '%s'", output_file_path) + output_file_path.unlink() + output_dir_path = output_file_path.parent + if output_dir_path.exists() and output_dir_path.is_dir(): + LOGGER.info("Deleting directory '%s'", output_dir_path) + try: + output_dir_path.rmdir() + except OSError as ose: + import errno + if ose.errno == errno.ENOTEMPTY: + LOGGER.warning( + "Directory '%s' not empty", + output_dir_path, + ) + else: + raise # EOF From 8356ed0d4768afeba158d318ee313b15ce677c6f Mon Sep 17 00:00:00 2001 From: sinoroc Date: Tue, 17 Sep 2019 17:15:56 +0200 Subject: [PATCH 09/12] Add support for default settings for all tools Take advantage of the 'default_section' feature of 'configparser' to add support for default settings for the tools. Settings that are placed in the section 'toolmaker.tool.defaults' are applied to all the tools. --- README.rst | 2 ++ example.cfg | 2 ++ src/toolmaker/cli.py | 4 +++- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 363ac18..729e9b5 100644 --- a/README.rst +++ b/README.rst @@ -32,6 +32,8 @@ current working directory. .. code:: + [toolmaker.tool.defaults] + [toolmaker.tool.pex:http.pex] entry_point = http.server output_file = http diff --git a/example.cfg b/example.cfg index 0215d81..a01456b 100644 --- a/example.cfg +++ b/example.cfg @@ -1,6 +1,8 @@ # +[toolmaker.tool.defaults] + [toolmaker.tool.pex:http.pex] entry_point = http.server output_file = http diff --git a/src/toolmaker/cli.py b/src/toolmaker/cli.py index 6079f4d..e205043 100644 --- a/src/toolmaker/cli.py +++ b/src/toolmaker/cli.py @@ -69,7 +69,9 @@ def main(): raw_config = None if args.config: logger.info("Reading configuration from file '%s'", args.config) - raw_config = configparser.ConfigParser() + raw_config = configparser.ConfigParser( + default_section='toolmaker.tool.defaults', + ) try: raw_config.read_file(args.config) except configparser.Error as config_error: From 354d554440b1aa0d7539e977ce5a4f7f4f18d03f Mon Sep 17 00:00:00 2001 From: sinoroc Date: Tue, 17 Sep 2019 17:21:21 +0200 Subject: [PATCH 10/12] Set the interpolation to 'ExtendedInterpolation' Change the interpolation for the configuration parser from configparser.BasicInterpolation to configparser.ExtendedInterpolation --- src/toolmaker/cli.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/toolmaker/cli.py b/src/toolmaker/cli.py index e205043..2137d1f 100644 --- a/src/toolmaker/cli.py +++ b/src/toolmaker/cli.py @@ -71,6 +71,7 @@ def main(): logger.info("Reading configuration from file '%s'", args.config) raw_config = configparser.ConfigParser( default_section='toolmaker.tool.defaults', + interpolation=configparser.ExtendedInterpolation(), ) try: raw_config.read_file(args.config) From 2d0f7e78c3ba25522be2aef97bcd1a23b2b0f88a Mon Sep 17 00:00:00 2001 From: sinoroc Date: Tue, 17 Sep 2019 17:25:06 +0200 Subject: [PATCH 11/12] Add support for 'output_file_win' setting On Windows ('nt' platform) it is convenient to add the prefix '.pyz' to the file names of the resulting zipapp archives, since on Windows the 'pyz' extension is commonly added to the 'PATHEXT' environment variable. Files with such an extension are then recognized as executables and it is then possible for a zipapp 'something.pyz' to be called by leaving out the extension and simply typing 'something'. On 'posix' platforms the 'executable' file mode bit is used to indicate which files are executable and thus the file name suffix is not relevant and often it is preferably left out entirely. --- README.rst | 1 + example.cfg | 1 + src/toolmaker/core.py | 3 +++ 3 files changed, 5 insertions(+) diff --git a/README.rst b/README.rst index 729e9b5..ac46a98 100644 --- a/README.rst +++ b/README.rst @@ -33,6 +33,7 @@ current working directory. .. code:: [toolmaker.tool.defaults] + output_file_win = ${output_file}.pyz [toolmaker.tool.pex:http.pex] entry_point = http.server diff --git a/example.cfg b/example.cfg index a01456b..409927e 100644 --- a/example.cfg +++ b/example.cfg @@ -2,6 +2,7 @@ [toolmaker.tool.defaults] +output_file_win = ${output_file}.pyz [toolmaker.tool.pex:http.pex] entry_point = http.server diff --git a/src/toolmaker/core.py b/src/toolmaker/core.py index 7138c08..1a22ad4 100644 --- a/src/toolmaker/core.py +++ b/src/toolmaker/core.py @@ -6,6 +6,7 @@ import logging +import os import pex import pex.bin.pex @@ -59,6 +60,8 @@ def _get_requirements(config): def _get_output_file_path(work_dir_path, config): output_dir_path = work_dir_path.joinpath(config['name']) output_file_name = config['output_file'] + if os.name == 'nt' and 'output_file_win' in config: + output_file_name = config['output_file_win'] output_file_path = output_dir_path.joinpath(output_file_name) return output_file_path From 841f30f9d8c90bf9adb143af60686f7318d0e37e Mon Sep 17 00:00:00 2001 From: sinoroc Date: Wed, 18 Sep 2019 11:18:17 +0200 Subject: [PATCH 12/12] Release version 0.0.2 --- CHANGELOG.rst | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d2eb59c..e4121e8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,8 +2,18 @@ .. Keep the current version number on line number 5 -0.0.2.dev0 -========== +0.0.2 +===== + +2019-09-18 + +* Add namespaces in configuration file + +* Add support for default settings + +* Change interpolation in configuration files + +* Add support for Windows specific output file name 0.0.1