Add pypiserver extension (tilt-dev#552)

Signed-off-by: korbajan <[email protected]>

README.md

@@ -50,6 +50,7 @@ All extensions have been vetted and approved by the Tilt team.
 - [`print_tiltfile_dir`](/print_tiltfile_dir): Print all files in the Tiltfile directory. If recursive is set to True, also prints files in all recursive subdirectories.
 - [`procfile`](/procfile): Create Tilt resources from a foreman Procfile.
 - [`pulumi`](/pulumi): Install Kubernetes resources with [Pulumi](https://www.pulumi.com/).
+- [`pypiserver`](/pypiserver): Run [pypiserver](https://pypi.org/project/pypiserver/) local container.
 - [`restart_process`](/restart_process): Wrap a `docker_build` or `custom_build` to restart the given entrypoint after a Live Update (replaces `restart_container()`)
 - [`secret`](/secret): Functions for creating secrets.
 - [`snyk`](/snyk): Use [Snyk](https://snyk.io) to test your containers, configuration files, and open source dependencies.

pypiserver/README.md

@@ -0,0 +1,58 @@
+# Pypiserver
+
+This extension runs docker container with [pypiserver](https://pypi.org/project/pypiserver/). 
+
+Author: [Jakub Stepien](https://github.com/korbajan)
+
+## Requirements
+- bash
+- python
+- docker (with compose)
+
+## Usage
+
+```
+load(
+    'ext://pypiserver',
+    'run_pypiserver_container',
+    'build_package',
+)
+
+pypiserver = run_pypiserver_container()
+build_package('./test/foo')
+```
+
+This will:
+- run pypiserver (with disabled authentication) docker container,
+- build package from ***./test/foo*** and upload it to this pypiserver. 
+
+Then if your Dockerfile contains ARG PIP_INDEX_URL you can use it with docker_build:
+
+```
+docker_build(
+  ref='python_services_which_depends_on_foo_package',
+  context=path_to_service_context_dir,
+  dockerfile=path_to_service_dockerfile,
+  build_args={'PIP_INDEX_URL': pypiserver}
+)
+```
+
+or, depend on your needs, you can also export it directly to the environment tilt is runing under:
+
+```
+os.putenv('PIP_INDEX_URL', pypiserv)
+```
+
+## Functions
+
+### `run_pypiserver()`
+starts 'tilt-pypiserver' container with exposed port (by default 8222 if it is not already opened) and returns string with its address in form of ***http//localhost:{port}***
+
+### `build_package(package_dir_path, name=None, upload=True, labels=[])`
+builds a package from ***package_dir_path*** path - there has to be ***setup.py*** under this directory - and if upload is True then upload it to local pypiserver.
+
+## Future work
+
+Add support for authentication (mount user or default apache httpasswd file into pypiserver container)
+Add custom build cmd for building package
+Implement building based on pyproject.toml

pypiserver/Tiltfile

@@ -0,0 +1,66 @@
+EXT_PATH = os.getcwd()
+PYPI_PORT_VAR_NAME = 'PYPISERVER_PORT' # docker compose purposes
+DEFAULT_PYPI_PORT_VALUE = '8222'
+DEFAULT_CACHE_DIR = '.tilt-cache'
+DEFAULT_PYPI_CONTAINER_NAME = 'tilt-pypiserver'
+PYPI_CONTAINER_VAR_NAME = 'PYPI_CONTAINER_NAME' # docker compose purpose
+PYPISERVER_DATA_CACHE_DIR_VAR_NAME = 'PYPISERVER_PACKAGES_DATA_CACHE' # docker compose purposes
+PYPISERVER_DATA_CACHE_DIR = 'pypiserver'
+
+PYTHON_CMD = 'python3'
+
+def _find_root_tiltfile_dir():
+    # Find top-level Tilt path
+    current = os.path.abspath('./')
+    while current != '/':
+        if os.path.exists(os.path.join(current, 'Tiltfile')):
+            return current
+        current = os.path.dirname(current)
+    fail('Could not find root Tiltfile')
+
+def _cache_dir():
+    cachedir = os.getenv('TILT_CACHE_DIR', '')
+    if cachedir == '':
+        cachedir = os.path.join(_find_root_tiltfile_dir(), DEFAULT_CACHE_DIR)
+    if not os.path.exists(cachedir):
+        local('mkdir -p %s' % shlex.quote(cachedir), echo_off=True)
+    os.putenv('TILT_CACHE_DIR', cachedir)
+    return cachedir
+
+def run_pypiserver_container():
+    if str(local([PYTHON_CMD, '%s/src/is_port_used.py' % EXT_PATH, DEFAULT_PYPI_PORT_VALUE], quiet=True, echo_off=True)).rstrip('\n') == 'false':
+        port = DEFAULT_PYPI_PORT_VALUE
+    else:
+        port = str(local([PYTHON_CMD, '%s/src/find_free_port.py' % EXT_PATH], quiet=True, echo_off=True)).rstrip('\n')
+    os.putenv(PYPI_PORT_VAR_NAME, port)
+    packages_dir = os.path.join(_cache_dir(), PYPISERVER_DATA_CACHE_DIR, 'packages')
+    if not os.path.exists(packages_dir):
+        local('mkdir -p %s' % shlex.quote(packages_dir), echo_off=True)
+    os.putenv(PYPI_CONTAINER_VAR_NAME, DEFAULT_PYPI_CONTAINER_NAME)
+    os.putenv(PYPISERVER_DATA_CACHE_DIR_VAR_NAME, packages_dir)
+    docker_compose(os.path.join(EXT_PATH, './compose.yaml'))
+    return 'http://localhost:%s' % port
+
+def build_package(path, name=None, upload=True, labels=None):
+    cachedir = _cache_dir()
+    packages_dir = os.path.join(cachedir, 'python_packages')
+    package_name = str(local([PYTHON_CMD, 'setup.py', '--name'], dir=path, quiet=True, echo_off=True)).rstrip('\n')
+    package_fullname = str(local([PYTHON_CMD, 'setup.py', '--fullname'], dir=path, quiet=True, echo_off=True)).rstrip('\n')
+
+    #cmd = [PYTHON_CMD, 'setup.py', 'sdist', '--dist-dir=%s' % packages_dir]
+    cmd = ['bash', '%s/src/build.sh' % EXT_PATH, packages_dir]
+    if upload:
+        #cmd.extend(['upload', '-r', 'http://localhost:%s' % os.getenv(PYPI_PORT_VAR_NAME)])
+        cmd.append('http://localhost:%s' % os.getenv(PYPI_PORT_VAR_NAME))
+    local_resource(
+        labels=labels or [],
+        name='%s-package' % package_name,
+        cmd=cmd,
+        deps=[path],
+        ignore=[
+          os.path.join(path, package_fullname),
+          os.path.join(path, '*egg-info')
+        ],
+        dir=path,
+        resource_deps=['pypiserver'] if upload else []
+    )

pypiserver/compose.yaml

@@ -0,0 +1,11 @@
+services:
+  pypiserver:
+    container_name: ${PYPI_CONTAINER_NAME}
+    image: docker.io/pypiserver/pypiserver:latest
+    environment:
+      - PYTHONUNBUFFERED=1
+    ports:
+      - "${PYPISERVER_PORT}:8080"
+    volumes:
+      - ${PYPISERVER_PACKAGES_DATA_CACHE}:/data/packages
+    command: run --overwrite --authenticate . --passwords . /data/packages

pypiserver/src/build.sh

@@ -0,0 +1,15 @@
+if [[ $# -eq 1 ]]; then
+  printf "Specify path to the package"
+  exit 1
+fi
+if [[ $# -gt 2 ]]; then
+  printf "To many arguments"
+  exit 1
+fi
+cmd="setup.py sdist --dist-dir=${1}"
+if [[ $# -eq 2 ]]; then
+  sleep 0.5
+  cmd="${cmd} upload -r ${2}"
+fi
+
+python3 ${cmd}

pypiserver/src/find_free_port.py

@@ -0,0 +1,16 @@
+import socket
+
+
+def _get_free_tcp_port():
+    tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    tcp.bind(("", 0))
+    _, port = tcp.getsockname()
+    tcp.close()
+    return int(port)
+
+
+def _get_local_port():
+    return str(_get_free_tcp_port())
+
+
+print(_get_local_port())

pypiserver/src/is_port_used.py

@@ -0,0 +1,16 @@
+import sys
+import socket
+
+
+def _is_port_in_use(port):
+    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    in_use = s.connect_ex(("127.0.0.1", port)) == 0
+    s.close()
+    return in_use
+
+
+port = sys.argv[1]
+if not port:
+    print("ERROR: You must specify port number!!!") 
+    sys.exit(1)
+print("true" if _is_port_in_use(int(port)) else "false")

pypiserver/test/Tiltfile

@@ -0,0 +1,3 @@
+load('../Tiltfile', 'run_pypiserver_container', 'build_package')
+run_pypiserver_container()
+

pypiserver/test/foo/README.txt

@@ -0,0 +1 @@
+Example package called 'foo'.

pypiserver/test/foo/foo/foo.py

@@ -0,0 +1,6 @@
+def hello():
+    return 'hello foo'
+
+
+def bye():
+    return 'bye foo'

pypiserver/test/foo/setup.py

@@ -0,0 +1,10 @@
+from distutils.core import setup
+
+setup(
+    name='foo',
+    version='0.1.0',
+    author='foo',
+    author_email='[email protected]',
+    packages=['foo'],
+    description='package example.',
+)

pypiserver/test/test.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+cd "$(dirname "$0")"
+
+set -ex
+tilt ci
+tilt down
+