diff --git a/README.md b/README.md index 1dfc8dba6..c741221d0 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,7 @@ All extensions have been vetted and approved by the Tilt team. - [`print_tiltfile_dir`](/print_tiltfile_dir): Print all files in the Tiltfile directory. If recursive is set to True, also prints files in all recursive subdirectories. - [`procfile`](/procfile): Create Tilt resources from a foreman Procfile. - [`pulumi`](/pulumi): Install Kubernetes resources with [Pulumi](https://www.pulumi.com/). +- [`pypiserver`](/pypiserver): Run [pypiserver](https://pypi.org/project/pypiserver/) local container. - [`restart_process`](/restart_process): Wrap a `docker_build` or `custom_build` to restart the given entrypoint after a Live Update (replaces `restart_container()`) - [`secret`](/secret): Functions for creating secrets. - [`snyk`](/snyk): Use [Snyk](https://snyk.io) to test your containers, configuration files, and open source dependencies. diff --git a/pypiserver/README.md b/pypiserver/README.md new file mode 100644 index 000000000..c610d3db9 --- /dev/null +++ b/pypiserver/README.md @@ -0,0 +1,58 @@ +# Pypiserver + +This extension runs docker container with [pypiserver](https://pypi.org/project/pypiserver/). + +Author: [Jakub Stepien](https://github.com/korbajan) + +## Requirements +- bash +- python +- docker (with compose) + +## Usage + +``` +load( + 'ext://pypiserver', + 'run_pypiserver_container', + 'build_package', +) + +pypiserver = run_pypiserver_container() +build_package('./test/foo') +``` + +This will: +- run pypiserver (with disabled authentication) docker container, +- build package from ***./test/foo*** and upload it to this pypiserver. + +Then if your Dockerfile contains ARG PIP_INDEX_URL you can use it with docker_build: + +``` +docker_build( + ref='python_services_which_depends_on_foo_package', + context=path_to_service_context_dir, + dockerfile=path_to_service_dockerfile, + build_args={'PIP_INDEX_URL': pypiserver} +) +``` + +or, depend on your needs, you can also export it directly to the environment tilt is runing under: + +``` +os.putenv('PIP_INDEX_URL', pypiserv) +``` + +## Functions + +### `run_pypiserver()` +starts 'tilt-pypiserver' container with exposed port (by default 8222 if it is not already opened) and returns string with its address in form of ***http//localhost:{port}*** + +### `build_package(package_dir_path, name=None, upload=True, labels=[])` +builds a package from ***package_dir_path*** path - there has to be ***setup.py*** under this directory - and if upload is True then upload it to local pypiserver. + +## Future work + +Add support for authentication (mount user or default apache httpasswd file into pypiserver container) +Add custom build cmd for building package +Implement building based on pyproject.toml diff --git a/pypiserver/Tiltfile b/pypiserver/Tiltfile new file mode 100644 index 000000000..d7f2f676e --- /dev/null +++ b/pypiserver/Tiltfile @@ -0,0 +1,66 @@ +EXT_PATH = os.getcwd() +PYPI_PORT_VAR_NAME = 'PYPISERVER_PORT' # docker compose purposes +DEFAULT_PYPI_PORT_VALUE = '8222' +DEFAULT_CACHE_DIR = '.tilt-cache' +DEFAULT_PYPI_CONTAINER_NAME = 'tilt-pypiserver' +PYPI_CONTAINER_VAR_NAME = 'PYPI_CONTAINER_NAME' # docker compose purpose +PYPISERVER_DATA_CACHE_DIR_VAR_NAME = 'PYPISERVER_PACKAGES_DATA_CACHE' # docker compose purposes +PYPISERVER_DATA_CACHE_DIR = 'pypiserver' + +PYTHON_CMD = 'python3' + +def _find_root_tiltfile_dir(): + # Find top-level Tilt path + current = os.path.abspath('./') + while current != '/': + if os.path.exists(os.path.join(current, 'Tiltfile')): + return current + current = os.path.dirname(current) + fail('Could not find root Tiltfile') + +def _cache_dir(): + cachedir = os.getenv('TILT_CACHE_DIR', '') + if cachedir == '': + cachedir = os.path.join(_find_root_tiltfile_dir(), DEFAULT_CACHE_DIR) + if not os.path.exists(cachedir): + local('mkdir -p %s' % shlex.quote(cachedir), echo_off=True) + os.putenv('TILT_CACHE_DIR', cachedir) + return cachedir + +def run_pypiserver_container(): + if str(local([PYTHON_CMD, '%s/src/is_port_used.py' % EXT_PATH, DEFAULT_PYPI_PORT_VALUE], quiet=True, echo_off=True)).rstrip('\n') == 'false': + port = DEFAULT_PYPI_PORT_VALUE + else: + port = str(local([PYTHON_CMD, '%s/src/find_free_port.py' % EXT_PATH], quiet=True, echo_off=True)).rstrip('\n') + os.putenv(PYPI_PORT_VAR_NAME, port) + packages_dir = os.path.join(_cache_dir(), PYPISERVER_DATA_CACHE_DIR, 'packages') + if not os.path.exists(packages_dir): + local('mkdir -p %s' % shlex.quote(packages_dir), echo_off=True) + os.putenv(PYPI_CONTAINER_VAR_NAME, DEFAULT_PYPI_CONTAINER_NAME) + os.putenv(PYPISERVER_DATA_CACHE_DIR_VAR_NAME, packages_dir) + docker_compose(os.path.join(EXT_PATH, './compose.yaml')) + return 'http://localhost:%s' % port + +def build_package(path, name=None, upload=True, labels=None): + cachedir = _cache_dir() + packages_dir = os.path.join(cachedir, 'python_packages') + package_name = str(local([PYTHON_CMD, 'setup.py', '--name'], dir=path, quiet=True, echo_off=True)).rstrip('\n') + package_fullname = str(local([PYTHON_CMD, 'setup.py', '--fullname'], dir=path, quiet=True, echo_off=True)).rstrip('\n') + + #cmd = [PYTHON_CMD, 'setup.py', 'sdist', '--dist-dir=%s' % packages_dir] + cmd = ['bash', '%s/src/build.sh' % EXT_PATH, packages_dir] + if upload: + #cmd.extend(['upload', '-r', 'http://localhost:%s' % os.getenv(PYPI_PORT_VAR_NAME)]) + cmd.append('http://localhost:%s' % os.getenv(PYPI_PORT_VAR_NAME)) + local_resource( + labels=labels or [], + name='%s-package' % package_name, + cmd=cmd, + deps=[path], + ignore=[ + os.path.join(path, package_fullname), + os.path.join(path, '*egg-info') + ], + dir=path, + resource_deps=['pypiserver'] if upload else [] + ) diff --git a/pypiserver/compose.yaml b/pypiserver/compose.yaml new file mode 100644 index 000000000..bba25949f --- /dev/null +++ b/pypiserver/compose.yaml @@ -0,0 +1,11 @@ +services: + pypiserver: + container_name: ${PYPI_CONTAINER_NAME} + image: docker.io/pypiserver/pypiserver:latest + environment: + - PYTHONUNBUFFERED=1 + ports: + - "${PYPISERVER_PORT}:8080" + volumes: + - ${PYPISERVER_PACKAGES_DATA_CACHE}:/data/packages + command: run --overwrite --authenticate . --passwords . /data/packages diff --git a/pypiserver/src/build.sh b/pypiserver/src/build.sh new file mode 100644 index 000000000..2b44243ed --- /dev/null +++ b/pypiserver/src/build.sh @@ -0,0 +1,15 @@ +if [[ $# -eq 1 ]]; then + printf "Specify path to the package" + exit 1 +fi +if [[ $# -gt 2 ]]; then + printf "To many arguments" + exit 1 +fi +cmd="setup.py sdist --dist-dir=${1}" +if [[ $# -eq 2 ]]; then + sleep 0.5 + cmd="${cmd} upload -r ${2}" +fi + +python3 ${cmd} diff --git a/pypiserver/src/find_free_port.py b/pypiserver/src/find_free_port.py new file mode 100644 index 000000000..008346e5d --- /dev/null +++ b/pypiserver/src/find_free_port.py @@ -0,0 +1,16 @@ +import socket + + +def _get_free_tcp_port(): + tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + tcp.bind(("", 0)) + _, port = tcp.getsockname() + tcp.close() + return int(port) + + +def _get_local_port(): + return str(_get_free_tcp_port()) + + +print(_get_local_port()) diff --git a/pypiserver/src/is_port_used.py b/pypiserver/src/is_port_used.py new file mode 100644 index 000000000..8e70c403d --- /dev/null +++ b/pypiserver/src/is_port_used.py @@ -0,0 +1,16 @@ +import sys +import socket + + +def _is_port_in_use(port): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + in_use = s.connect_ex(("127.0.0.1", port)) == 0 + s.close() + return in_use + + +port = sys.argv[1] +if not port: + print("ERROR: You must specify port number!!!") + sys.exit(1) +print("true" if _is_port_in_use(int(port)) else "false") diff --git a/pypiserver/test/Tiltfile b/pypiserver/test/Tiltfile new file mode 100644 index 000000000..afe97bf00 --- /dev/null +++ b/pypiserver/test/Tiltfile @@ -0,0 +1,3 @@ +load('../Tiltfile', 'run_pypiserver_container', 'build_package') +run_pypiserver_container() + diff --git a/pypiserver/test/foo/README.txt b/pypiserver/test/foo/README.txt new file mode 100644 index 000000000..ce48339a7 --- /dev/null +++ b/pypiserver/test/foo/README.txt @@ -0,0 +1 @@ +Example package called 'foo'. diff --git a/pypiserver/test/foo/foo/__init__.py b/pypiserver/test/foo/foo/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/pypiserver/test/foo/foo/foo.py b/pypiserver/test/foo/foo/foo.py new file mode 100644 index 000000000..9f51528b7 --- /dev/null +++ b/pypiserver/test/foo/foo/foo.py @@ -0,0 +1,6 @@ +def hello(): + return 'hello foo' + + +def bye(): + return 'bye foo' diff --git a/pypiserver/test/foo/setup.py b/pypiserver/test/foo/setup.py new file mode 100644 index 000000000..b6a7e0f1a --- /dev/null +++ b/pypiserver/test/foo/setup.py @@ -0,0 +1,10 @@ +from distutils.core import setup + +setup( + name='foo', + version='0.1.0', + author='foo', + author_email='foo@example.com', + packages=['foo'], + description='package example.', +) diff --git a/pypiserver/test/test.sh b/pypiserver/test/test.sh new file mode 100755 index 000000000..4f8759440 --- /dev/null +++ b/pypiserver/test/test.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +cd "$(dirname "$0")" + +set -ex +tilt ci +tilt down +