diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f4e4638..96fbae3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -21,6 +21,7 @@ 'test py38 dev': <<: *job_test_common + allow_failure: true image: 'python:3.8-rc' variables: 'TOXENV': 'py38' diff --git a/CHANGELOG.rst b/CHANGELOG.rst index dd2806c..3a6aae9 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,8 +2,29 @@ .. Keep the current version number on line number 5 +0.0.1 +===== + +2019-06-12 + +* Add action as CLI optional argument. The action has to be specified on the + command line. Either one of: + + * ``--build``, ``-b`` to build (existing tools are skipped); + * ``--rebuild``, ``-r`` to rebuild (existing tools are rebuilt); + * ``--delete``, ``-d`` to delete (tool target file is deleted if it exists, + its parent directory is deleted if it is empty). + +* Replace CLI calls to external tools in virtual environment with API calls to + external libraries. + + 0.0.0 ===== +2019-05-13 + +Release initial version. + .. EOF diff --git a/Makefile b/Makefile index 4c6691d..42086f8 100644 --- a/Makefile +++ b/Makefile @@ -14,17 +14,19 @@ develop: .PHONY: package -package: sdist wheel check zapp +package: sdist wheel zapp .PHONY: sdist sdist: python3 setup.py sdist + python3 -m twine check dist/*.tar.gz .PHONY: wheel wheel: python3 setup.py bdist_wheel + python3 -m twine check dist/*.whl .PHONY: zapp @@ -33,9 +35,8 @@ zapp: .PHONY: check -check: sdist wheel - python3 -m twine check dist/*.tar.gz - python3 -m twine check dist/*.whl +check: + python3 setup.py check .PHONY: lint @@ -63,7 +64,7 @@ pytest: .PHONY: review -review: +review: check python3 -m pytest --pep8 --pylint diff --git a/README.rst b/README.rst index c90cf0c..b752346 100644 --- a/README.rst +++ b/README.rst @@ -1,11 +1,74 @@ .. +Introduction +============ + +This tool automates the build of Python tools according to a configuration +file. The tools can be built with `zapp`_, `shiv`_, or `pex`_. + + +Repositories +------------ + +Binary distributions: + +* http://pypi.org/project/toolmaker/ + +Source code: + +* https://gitlab.com/sinoroc/toolmaker +* https://github.com/sinoroc/toolmaker + + Usage ===== -Place in a subdirectory of your ``bin`` directory and use in combination with -*GNU stow*. +Configuration +------------- + +By default this tool looks for a configuration file ``toolmaker.cfg`` in the +current working directory. + +.. code:: + + [http.pex] + entry_point = http.server + output_file = http + requirements = + + [pipdeptree.zapp] + entry_point = pipdeptree:main + output_file = pipdeptree + requirements = + pipdeptree + setuptools + + [shiv.shiv] + console_script = shiv + output_file = shiv + requirements = + shiv + + +Action +------ + +The action can be specified on the command line. Either one of: + +* ``--build``, ``-b`` to build (already existing tools are skipped); +* ``--rebuild``, ``-r`` to rebuild (already existing tools are rebuilt); +* ``--delete``, ``-d`` to delete (tool target file is deleted if it exists, then + its parent directory is deleted if it is empty). + +The default action when no flag is specified is to build the tools. + + +Tips +---- + +Place in a subdirectory of a directory that is available on your ``PATH`` +(typically your ``~/bin`` directory) and use in combination with `GNU Stow`_. Details @@ -14,7 +77,7 @@ Details Similar projects ---------------- -* https://github.com/Valassis-Digital-Media/Zapper +* `Zapper`_ Hacking @@ -64,8 +127,13 @@ Outside of a Python virtual environment run the following command:: .. Links .. _`GNU Make`: https://www.gnu.org/software/make/ +.. _`GNU Stow`: https://www.gnu.org/software/stow/ +.. _`pex`: https://pypi.org/project/pex/ .. _`pytest`: https://pytest.org/ +.. _`shiv`: https://pypi.org/project/shiv/ .. _`tox`: https://tox.readthedocs.io/ +.. _`zapp`: https://pypi.org/project/zapp/ +.. _`Zapper`: https://github.com/Valassis-Digital-Media/Zapper .. EOF diff --git a/example.cfg b/example.cfg new file mode 100644 index 0000000..31a446b --- /dev/null +++ b/example.cfg @@ -0,0 +1,25 @@ +# + + +[http.pex] +entry_point = http.server +output_file = http +requirements = + + +[pipdeptree.zapp] +entry_point = pipdeptree:main +output_file = pipdeptree +requirements = + pipdeptree + setuptools + + +[shiv.shiv] +entry_point = shiv.cli:main +output_file = shiv +requirements = + shiv + + +# EOF diff --git a/setup.cfg b/setup.cfg index 39a458c..fcd745b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,6 +5,11 @@ entry_point = toolmaker.cli:main +[check] +metadata = 1 +strict = 1 + + [metadata] name = toolmaker author = sinoroc @@ -12,12 +17,16 @@ author_email = sinoroc.code+python@gmail.com description = toolmaker application license = Apache-2.0 long_description = file: README.rst +long_description_content_type = text/x-rst url = https://pypi.org/project/toolmaker [options] install_requires = - setuptools + importlib_metadata + pex + shiv + zapp package_dir = = src packages = find: diff --git a/setup.py b/setup.py index ff11480..8dd31c5 100755 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ Setup script """ diff --git a/src/toolmaker/cli.py b/src/toolmaker/cli.py index 70f0c7f..bf47826 100644 --- a/src/toolmaker/cli.py +++ b/src/toolmaker/cli.py @@ -27,12 +27,25 @@ def _create_args_parser(default_config_path, tools_names=None): default=str(default_config_path), type=argparse.FileType('r'), ) - group = args_parser.add_mutually_exclusive_group(required=True) - group.add_argument( + action_group = args_parser.add_mutually_exclusive_group() + action_group.add_argument( + '--build', '-b', + action='store_true', + ) + action_group.add_argument( + '--rebuild', '-r', + action='store_true', + ) + action_group.add_argument( + '--delete', '-d', + action='store_true', + ) + tools_group = args_parser.add_mutually_exclusive_group(required=True) + tools_group.add_argument( '--all', '-a', action='store_true', ) - group.add_argument( + tools_group.add_argument( '--tools', '-t', choices=tools_names, metavar='tool', @@ -46,6 +59,7 @@ def main(): """ logger = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) + cwd_path = pathlib.Path.cwd() default_config_path = cwd_path.joinpath('toolmaker.cfg') @@ -54,6 +68,7 @@ def main(): config = None if args.config: + logger.info("Reading configuration from file '%s'", args.config) config = configparser.ConfigParser() try: config.read_file(args.config) @@ -68,26 +83,12 @@ def main(): if args.tools: tools_names = args.tools - logger.info("Preparing to build tools %s", tools_names) - - venv_path = cwd_path.joinpath('venv') - - logger.info("Creating virtual environment '%s'...", venv_path) - venv_context = core.venv_create(venv_path) - - logger.info("Updating virtual environment") - core.venv_update(venv_context) - - for tool_name in tools_names: - if tool_name.endswith('.pex'): - logger.info("Building pex tool '%s'", tool_name) - core.build_pex(cwd_path, venv_context, config, tool_name) - if tool_name.endswith('.shiv'): - logger.info("Building shiv tool '%s'", tool_name) - core.build_shiv(cwd_path, venv_context, config, tool_name) - if tool_name.endswith('.zapp'): - logger.info("Building zapp tool '%s'", tool_name) - core.build_zapp(cwd_path, venv_context, config, tool_name) + if args.delete: + core.delete(cwd_path, config, tools_names) + elif args.rebuild: + core.build(cwd_path, config, tools_names, force=True) + else: + core.build(cwd_path, config, tools_names) # EOF diff --git a/src/toolmaker/core.py b/src/toolmaker/core.py index 543c029..a3bb24d 100644 --- a/src/toolmaker/core.py +++ b/src/toolmaker/core.py @@ -5,133 +5,161 @@ """ -import pathlib -import subprocess -import venv - - -def _run_in_venv(venv_context, command): - venv_bin_path = pathlib.Path(venv_context.bin_path) - command_in_venv = [ - str(venv_bin_path.joinpath(command[0])), - ] + command[1:] - subprocess.run(command_in_venv) - - -def _pip_install(venv_context, packages): - command = [ - 'pip', - 'install', - '--upgrade', - ] + packages - _run_in_venv(venv_context, command) - - -def _pex(venv_context, entry_point, output_file_path, requirements): - _run_in_venv( - venv_context, - [ - 'pex', - '--entry-point={}'.format(entry_point), - '--output-file={}'.format(str(output_file_path)), - ] + requirements, +import logging + +import pex +import pex.bin.pex +import shiv +import shiv.cli +import zapp + + +LOGGER = logging.getLogger(__name__) +logging.basicConfig(level=logging.INFO) + + +def _pex(requirements, entry_point, output_file_path): + cmd = [ + '--entry-point={}'.format(entry_point), + '--output-file={}'.format(str(output_file_path)), + ] + requirements + pex.bin.pex.main(cmd) + + +def _shiv(requirements, entry_point, output_file_path): + # Since it is decorated by 'click', the 'main' function is not callable + # with its original arguments. The original function is "hidden" under + # 'callback'. + shiv.cli.main.callback( + str(output_file_path), # output_file + entry_point, # entry_point + None, # console_script + '/usr/bin/env python3', # python + None, # site_packages + True, # compressed + False, # compile_pyc + False, # extend_pythonpath + requirements, # pip_args ) -def _shiv(venv_context, console_script, output_file_path, requirements): - _run_in_venv( - venv_context, - [ - 'shiv', - '--console-script', - console_script, - '--output-file', - str(output_file_path), - ] + requirements, - ) - - -def _zapp(venv_context, output_file_path, entry_point, requirements): - _run_in_venv( - venv_context, - [ - 'zapp', - output_file_path, - entry_point, - ] + requirements, - ) +def _zapp(requirements, entry_point, output_file_path): + zapp.core.build_zapp(requirements, entry_point, output_file_path) -def build_pex(cwd_path, venv_context, config, section_name): - """ Build pex - """ - section = config[section_name] - output_dir_path = cwd_path.joinpath(section_name) - output_dir_path.mkdir(exist_ok=True) - output_file_name = section['output_file'] - output_file_path = output_dir_path.joinpath(output_file_name) +def _get_requirements(config): requirements = [ - req for req in section['requirements'].splitlines() if req + req.strip() + for req in config['requirements'].splitlines() + if req.strip() ] - entry_point = section['entry_point'] - _pex(venv_context, entry_point, output_file_path, requirements) + return requirements -def build_shiv(cwd_path, venv_context, config, section_name): - """ Build shiv - """ - section = config[section_name] - output_dir_path = cwd_path.joinpath(section_name) - output_dir_path.mkdir(exist_ok=True) - output_file_name = section['output_file'] +def _get_output_file_path(work_dir_path, tool_name, config): + output_dir_path = work_dir_path.joinpath(tool_name) + output_file_name = config['output_file'] output_file_path = output_dir_path.joinpath(output_file_name) - requirements = [ - req for req in section['requirements'].splitlines() if req - ] - console_script = section['console_script'] - _shiv(venv_context, console_script, output_file_path, requirements) + return output_file_path -def build_zapp(cwd_path, venv_context, config, section_name): +def _build_pex(work_dir_path, tool_name, config, force): + """ Build pex + """ + output_file_path = _get_output_file_path(work_dir_path, tool_name, config) + if force or not output_file_path.exists(): + LOGGER.info("Building pex tool '%s'...", tool_name) + requirements = _get_requirements(config) + entry_point = config['entry_point'] + output_file_path.parent.mkdir(exist_ok=True) + _pex(requirements, entry_point, output_file_path) + else: + LOGGER.info("Tool '%s' already exists, build skipped", tool_name) + + +def _build_shiv(work_dir_path, tool_name, config, force): + """ Build shiv + """ + output_file_path = _get_output_file_path(work_dir_path, tool_name, config) + if force or not output_file_path.exists(): + LOGGER.info("Building shiv tool '%s'...", tool_name) + requirements = _get_requirements(config) + entry_point = config['entry_point'] + output_file_path.parent.mkdir(exist_ok=True) + _shiv(requirements, entry_point, output_file_path) + else: + LOGGER.info("Tool '%s' already exists, build skipped", tool_name) + + +def _build_zapp(work_dir_path, tool_name, config, force): """ Build zapp """ - section = config[section_name] - output_dir_path = cwd_path.joinpath(section_name) - output_dir_path.mkdir(exist_ok=True) - output_file_name = section['output_file'] - output_file_path = output_dir_path.joinpath(output_file_name) - requirements = [ - req for req in section['requirements'].splitlines() if req - ] - entry_point = section['entry_point'] - _zapp(venv_context, output_file_path, entry_point, requirements) - - -class _EnvBuilder(venv.EnvBuilder): - - def __init__(self, *args, **kwargs): - self.context = None - super().__init__(*args, **kwargs) - - def post_setup(self, context): - """ Override """ - self.context = context - - -def venv_create(venv_path): - """ Create virtual environment + output_file_path = _get_output_file_path(work_dir_path, tool_name, config) + if force or not output_file_path.exists(): + LOGGER.info("Building zapp tool '%s'...", tool_name) + requirements = _get_requirements(config) + entry_point = config['entry_point'] + output_file_path.parent.mkdir(exist_ok=True) + _zapp(requirements, entry_point, output_file_path) + else: + LOGGER.info("Tool '%s' already exists, build skipped", tool_name) + + +def build(work_dir_path, config, tools_names, force=False): + """ Build tools """ - venv_builder = _EnvBuilder(with_pip=True) - venv_builder.create(venv_path) - return venv_builder.context - - -def venv_update(venv_context): - """ Update virtual environment + LOGGER.info("Building tools %s...", tools_names) + + for tool_name in tools_names: + tool_config = config[tool_name] + if tool_name.endswith('.pex'): + _build_pex( + work_dir_path, + tool_name, + tool_config, + force, + ) + if tool_name.endswith('.shiv'): + _build_shiv( + work_dir_path, + tool_name, + tool_config, + force, + ) + if tool_name.endswith('.zapp'): + _build_zapp( + work_dir_path, + tool_name, + tool_config, + force, + ) + + +def delete(work_dir_path, config, tools_names): + """ Delete tools """ - _pip_install(venv_context, ['pip']) - _pip_install(venv_context, ['setuptools']) - _pip_install(venv_context, ['pex[cachecontrol,requests]', 'shiv', 'zapp']) + LOGGER.info("Deleting tools %s...", tools_names) + + for tool_name in tools_names: + output_file_path = _get_output_file_path( + work_dir_path, + tool_name, + config[tool_name], + ) + if output_file_path.exists() and output_file_path.is_file(): + LOGGER.info("Deleting file '%s'", output_file_path) + output_file_path.unlink() + output_dir_path = output_file_path.parent + if output_dir_path.exists() and output_dir_path.is_dir(): + LOGGER.info("Deleting directory '%s'", output_dir_path) + try: + output_dir_path.rmdir() + except OSError as ose: + import errno + if ose.errno == errno.ENOTEMPTY: + LOGGER.warning("Directory '%s' not empty", output_dir_path) + else: + raise # EOF diff --git a/src/toolmaker/meta.py b/src/toolmaker/meta.py index 29aae1c..0a15be7 100644 --- a/src/toolmaker/meta.py +++ b/src/toolmaker/meta.py @@ -4,10 +4,10 @@ """ Meta information """ -import pkg_resources +import importlib_metadata -VERSION = pkg_resources.get_distribution('toolmaker').version +VERSION = importlib_metadata.version('toolmaker') # EOF diff --git a/toolmaker.cfg b/toolmaker.cfg deleted file mode 100644 index aa5ebae..0000000 --- a/toolmaker.cfg +++ /dev/null @@ -1,60 +0,0 @@ -# - - -[cookiecutter.shiv] -console_script = cookiecutter -output_file = cookiecutter -requirements = - cookiecutter - - -[http.pex] -entry_point = http.server -output_file = http -requirements = - - -[pex.pex] -entry_point = pex.bin.pex:main -output_file = pex -requirements = - pex[cachecontrol,requests] - - -[pipdeptree.zapp] -entry_point = pipdeptree:main -output_file = pipdeptree -requirements = - pipdeptree - - -[shiv.shiv] -console_script = shiv -output_file = shiv -requirements = - shiv - - -[tox.shiv] -console_script = tox -output_file = tox -requirements = - tox - tox-venv - - -[twine.shiv] -console_script = twine -output_file = twine -requirements = - twine - - -[zapp.zapp] -entry_point = zapp.cli:main -output_file = zapp -requirements = - zapp - - -# EOF diff --git a/tox.ini b/tox.ini index 74f1676..fc9ab33 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ requires = [testenv] commands = - make toxenvname={envname} review + make review extras = test whitelist_externals =