diff --git a/neuro-shortcuts/.gitignore b/neuro-shortcuts/.gitignore new file mode 100644 index 00000000..49a35173 --- /dev/null +++ b/neuro-shortcuts/.gitignore @@ -0,0 +1,11 @@ +/.env/ +/venv/ +/build/ +.mypy_cache/ +pip-wheel-metadata +*.swo +*.egg-info +**/__pycache__ +.eggs +.tmontmp +.testmondata diff --git a/neuro-shortcuts/Makefile b/neuro-shortcuts/Makefile new file mode 100644 index 00000000..a31ae1a3 --- /dev/null +++ b/neuro-shortcuts/Makefile @@ -0,0 +1,22 @@ +ISORT_DIRS := tasks.py setup.py +BLACK_DIRS := $(ISORT_DIRS) +MYPY_DIRS := tests + +setup: + pip install --disable-pip-version-check -r requirements-dev.txt + +.PHONY: lint +lint: + isort -c -rc ${ISORT_DIRS} + black --check $(BLACK_DIRS) + mypy $(MYPY_DIRS) + flake8 $(FLAKE8_DIRS) + +.PHONY: format +format: + isort -rc $(ISORT_DIRS) + black $(BLACK_DIRS) + +.PHONY: test +test: + pytest -v tests/ diff --git a/neuro-shortcuts/invoke.yaml b/neuro-shortcuts/invoke.yaml new file mode 100644 index 00000000..9944a540 --- /dev/null +++ b/neuro-shortcuts/invoke.yaml @@ -0,0 +1,4 @@ +debug: false +run: + echo: true + color: true \ No newline at end of file diff --git a/neuro-shortcuts/requirements-dev.txt b/neuro-shortcuts/requirements-dev.txt new file mode 100644 index 00000000..7d613ea3 --- /dev/null +++ b/neuro-shortcuts/requirements-dev.txt @@ -0,0 +1,5 @@ +#neuromation==19.9.2 +mypy==0.711 +flake8==3.7.8 +isort==4.3.21 +black==19.3b0 diff --git a/neuro-shortcuts/setup.cfg b/neuro-shortcuts/setup.cfg new file mode 100644 index 00000000..49c28cb5 --- /dev/null +++ b/neuro-shortcuts/setup.cfg @@ -0,0 +1,25 @@ +[flake8] +exclude = .git,.env,venv,__pycache__,.eggs +max-line-length = 88 +ignore = N801,N802,N803,E252,W503,E133,E203 + +[isort] +line_length=88 +include_trailing_comma=True +multi_line_output=3 +force_grid_wrap=0 +combine_as_imports=True +lines_after_imports=2 +known_standard_library=dataclasses +known_third_party=aiohttp,async_timeout,pytest + +[mypy] +check_untyped_defs = True +disallow_any_generics = True +disallow_untyped_defs = True +follow_imports = silent +strict_optional = True +warn_redundant_casts = True +warn_unused_ignores = True +warn_unused_configs = True +incremental = False diff --git a/neuro-shortcuts/setup.py b/neuro-shortcuts/setup.py new file mode 100644 index 00000000..d19366f8 --- /dev/null +++ b/neuro-shortcuts/setup.py @@ -0,0 +1,12 @@ +from setuptools import find_packages, setup + + +setup( + name="neuro-shortcuts", + version="0.0.1b1", + packages=find_packages(), + python_requires=">=3.7.0", + install_requires=(), + # TODO: console_scripts for running from python console + scripts=["neu.py"], +) diff --git a/neuro-shortcuts/tasks.py b/neuro-shortcuts/tasks.py new file mode 100755 index 00000000..0d3987a2 --- /dev/null +++ b/neuro-shortcuts/tasks.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python + +from invoke import task as task + + +PROJECT_NAME = "{{cookiecutter.project_slug}}" + +CODE_PATH = PROJECT_NAME +DATA_PATH = "data" +NOTEBOOKS_PATH = "notebooks" +REQUIREMENTS_PATH = "requirements" +RESULTS_PATH = "results" +PROJECT_PATH_STORAGE = f"storage:{PROJECT_NAME}" +CODE_PATH_STORAGE = f"{PROJECT_PATH_STORAGE}/{CODE_PATH}" +DATA_PATH_STORAGE = f"{PROJECT_PATH_STORAGE}/{DATA_PATH}" +NOTEBOOKS_PATH_STORAGE = f"{PROJECT_PATH_STORAGE}/{NOTEBOOKS_PATH}" +REQUIREMENTS_PATH_STORAGE = f"{PROJECT_PATH_STORAGE}/{REQUIREMENTS_PATH}" +RESULTS_PATH_STORAGE = f"{PROJECT_PATH_STORAGE}/{RESULTS_PATH}" + +PROJECT_PATH_ENV = "/project" +CODE_PATH_ENV = f"{PROJECT_PATH_ENV}/{CODE_PATH}" +DATA_PATH_ENV = f"{PROJECT_PATH_ENV}/{DATA_PATH}" +NOTEBOOKS_PATH_ENV = f"{PROJECT_PATH_ENV}/{NOTEBOOKS_PATH}" +REQUIREMENTS_PATH_ENV = f"{PROJECT_PATH_ENV}/{REQUIREMENTS_PATH}" +RESULTS_PATH_ENV = f"{PROJECT_PATH_ENV}/{RESULTS_PATH}" + +SETUP_NAME = "setup" +TRAINING_NAME = "training" +JUPYTER_NAME = "jupyter" +TENSORBOARD_NAME = "tensorboard" +FILEBROWSER_NAME = "filebrowser" + +BASE_ENV_NAME = "image:neuro/base" +CUSTOM_ENV_NAME = "image:neuro/custom" + + +# ##### SETUP ##### + + +@task +def help(context): + context.run("invoke --list") + + +@task +def setup(context): + """ This is documentation for setup + """ + context.run(f"neuro kill {SETUP_NAME}") + command = "sleep 1h" + context.run( + f"neuro run --name {SETUP_NAME} --preset cpu-small --detach " + f"--volume {PROJECT_PATH_STORAGE}:{PROJECT_PATH_ENV}:ro " + f"{BASE_ENV_NAME} '{command}'" + ) + context.run(f"neuro cp -r {REQUIREMENTS_PATH} {REQUIREMENTS_PATH_STORAGE}") + # TODO: fix commands below + # For some reason the second command fail + # neuro exec {SETUP_NAME} 'apt-get update' + # neuro exec {SETUP_NAME} 'cat {REQUIREMENTS_PATH_ENV}/apt.txt | xargs apt-get install -y' # noqa + context.run( + f"neuro exec {SETUP_NAME} 'pip install -r {REQUIREMENTS_PATH_ENV}/pip.txt'" + ) + context.run(f"neuro job save {SETUP_NAME} {CUSTOM_ENV_NAME}") + context.run(f"neuro kill {SETUP_NAME}") + + +# ##### STORAGE ##### + + +@task +def upload_code(context): + context.run(f"neuro cp -r -T {CODE_PATH} {CODE_PATH_STORAGE}") + + +@task +def clean_code(context): + context.run(f"neuro rm -r {CODE_PATH_STORAGE}") + + +@task +def upload_data(context): + context.run(f"neuro storage load -p -u -T {DATA_PATH} {DATA_PATH_STORAGE}") + + +@task +def clean_data(context): + context.run(f"neuro rm -r {DATA_PATH_STORAGE}") + + +@task +def upload_notebooks(context): + context.run(f"neuro cp -r -T {NOTEBOOKS_PATH} {NOTEBOOKS_PATH_STORAGE}") + + +@task +def download_notebooks(context): + context.run(f"neuro cp -r {NOTEBOOKS_PATH_STORAGE} {NOTEBOOKS_PATH}") + + +@task +def clean_notebooks(context): + context.run(f"neuro rm -r {NOTEBOOKS_PATH_STORAGE}") + + +@task +def upload(context): + upload_code() + upload_data() + upload_notebooks() + + +@task +def clean(context): + clean_code() + clean_data() + clean_notebooks() + + +# ##### JOBS ##### + + +@task +def training(context): + cmd = ( + f"python {CODE_PATH_ENV}/train.py --log_dir " + f"{RESULTS_PATH_ENV} --data_root {DATA_PATH_ENV}/cifar10" + ) + context.run( + f"neuro context.run --name {TRAINING_NAME} --preset gpu-small " + f"--volume {DATA_PATH_STORAGE}:{DATA_PATH_ENV}:ro " + f"--volume {CODE_PATH_STORAGE}:{CODE_PATH_ENV}:ro " + f"--volume {RESULTS_PATH_STORAGE}:{RESULTS_PATH_ENV}:rw " + f"{CUSTOM_ENV_NAME} " + f"'{cmd}'" + ) + + +@task +def kill_training(context): + context.run(f"neuro kill {TRAINING_NAME}") + + +@task +def connect_training(context): + context.run(f"neuro exec {TRAINING_NAME} bash") + + +@task +def jupyter(context): + cmd = ( + f"jupyter notebook --no-browser --ip=0.0.0.0 --allow-root " + f"--NotebookApp.token= --notebook-dir={NOTEBOOKS_PATH_ENV}" + ) + context.run( + f"neuro context.run " + f"--name {JUPYTER_NAME} " + f"--preset gpu-small " + f"--http 8888 --no-http-auth --detach " + f"--volume {DATA_PATH_STORAGE}:{DATA_PATH_ENV}:ro " + f"--volume {CODE_PATH_STORAGE}:{CODE_PATH_ENV}:rw " + f"--volume {NOTEBOOKS_PATH_STORAGE}:{NOTEBOOKS_PATH_ENV}:rw " + f"--volume {RESULTS_PATH_STORAGE}:{RESULTS_PATH_ENV}:rw " + f"{CUSTOM_ENV_NAME} " + f"'{cmd}'" + ) + context.run(f"neuro job browse {JUPYTER_NAME}") + + +@task +def kill_jupyter(context): + context.run(f"neuro kill {JUPYTER_NAME}") + + +@task +def tensorboard(context): + cmd = f"tensorboard --logdir={RESULTS_PATH_ENV}" + context.run( + f"neuro context.run " + f"--name {TENSORBOARD_NAME} " + f"--preset cpu-small " + f"--http 6006 --no-http-auth --detach " + f"--volume {RESULTS_PATH_STORAGE}:{RESULTS_PATH_ENV}:ro " + f"{CUSTOM_ENV_NAME} " + f"'{cmd}'" + ) + context.run(f"neuro job browse {TENSORBOARD_NAME}") + + +@task +def kill_tensorboard(context): + context.run(f"neuro kill {TENSORBOARD_NAME}") + + +@task +def filebrowser(context): + context.run( + f"neuro context.run " + f"--name {FILEBROWSER_NAME} " + f"--preset cpu-small " + f"--http 80 --no-http-auth --detach " + f"--volume {PROJECT_PATH_STORAGE}:/srv:rw " + f"filebrowser/filebrowser" + ) + context.run(f"neuro job browse {FILEBROWSER_NAME}") + + +@task +def kill_filebrowser(context): + context.run(f"neuro kill {FILEBROWSER_NAME}") + + +@task(pre=[kill_training, kill_jupyter, kill_tensorboard, kill_filebrowser]) +def kill(context): + pass + + +# ##### LOCAL ##### + + +@task +def setup_local(context): + context.run("pip install -r requirements/pip.txt") + + +@task +def lint(context): + context.run("flake8 .") + context.run("mypy .") + + +@task +def install(context): + context.run("python setup.py install --user") + + +# ##### MISC ##### + + +@task +def ps(context): + context.run(f"neuro ps") diff --git a/neuro-shortcuts/tests/__init__.py b/neuro-shortcuts/tests/__init__.py new file mode 100644 index 00000000..e69de29b