diff --git a/.env.demo b/.env.demo deleted file mode 100644 index ce370052..00000000 --- a/.env.demo +++ /dev/null @@ -1 +0,0 @@ -AIIDALAB_DEFAULT_APPS=aiidalab-widgets-base quantum-espresso aiidalab-optimade diff --git a/Dockerfile b/Dockerfile index 0c97567f..861a722f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -107,7 +107,7 @@ COPY opt/prepare-aiidalab.sh /opt/ COPY my_init.d/prepare-aiidalab.sh /etc/my_init.d/80_prepare-aiidalab.sh # Install the aiidalab-home app. -ARG aiidalab_home_version=v21.10.0 +ARG aiidalab_home_version=v22.01.0 RUN git clone https://github.com/aiidalab/aiidalab-home && cd aiidalab-home && git checkout $aiidalab_home_version RUN chmod 774 aiidalab-home diff --git a/README.md b/README.md index dd1cbeaa..ec438c65 100644 --- a/README.md +++ b/README.md @@ -1,58 +1,37 @@ # Docker Stack for AiiDAlab -This repo contains the Docker file used in the [AiiDAlab](https://www.materialscloud.org/aiidalab). +This repository contains the Dockerfile for the official AiiDAlab docker images. -Docker images are available from Dockerhub via `docker pull aiidalab/aiidalab-docker-stack:latest`. -See [aiidalab/aiidalab-docker-stack](https://hub.docker.com/repository/docker/aiidalab/aiidalab-docker-stack) for a list of available tags. +Docker images are automatically built and pushed to Docker Hub at https://hub.docker.com/r/aiidalab/aiidalab-docker-stack with the following tags: -# Documentation +- `latest` – the latest tagged release. +- `` – a specific tagged release, example: `21.12.0`. +- `master`/`develop` – the latest commit on the corresponding branches with the same name. -## Local deployment +## Get started -To launch a local instance of AiiDAlab, first clone this repository, e.g., with -```console -git clone https://github.com/aiidalab/aiidalab-docker-stack.git -cd aiidalab-docker-stack -``` -and then install the Python requirements needed to run the manage script: -``` -pip install -r requirements-manage.txt -``` +### Local deployment -Before starting AiiDAlab, it is recommended to configure it for your needs. -For example, to mount the AiiDAlab home directory on your local host at `~/aiidalab` instead of using a Docker volume, execute: -```console -./manage.py configure --home-dir=~/aiidalab -``` -This creates a `.env` file in the local directory that stores the provided settings. +To run AiiDAlab on your own workstation or laptop you can either +- run the image directly with: `docker run aiidalab-docker-stack -p 8888:8888`, or +- _(recommended)_ use the `aiidalab-launch` tool which is a thin docker wrapper. -You can then launch your local AiiDAlab deployment with: +The `aiidalab-launch` tool provides a convenient and robust method of both launching and managing one or multiple AiiDAlab instances on your computer. +To use it, simply install it via pip ```console -$ ./manage.py up +pip install aiidalab-launch ``` -You should see output similar to this: -``` -Starting AiiDAlab (this can take multiple minutes) ... -Open this link in the browser to enter AiiDAlab: -http://localhost:8888/?token=be20d9872d... -``` - -Note: AiiDAlab will keep running until you shutdown or restart the host computer, in which case, you will have to run the `up` command again to restart AiiDAlab. - -Please see `./manage.py --help` for a full list of available commands. - -## Development deployment - -For a local development deployment, run +and then start AiiDAlab with ```console -./manage.py --develop up +aiidalab-launch start ``` +Note: AiiDAlab will keep running until you explicitly stop it or shutdown/restart your computer. +In that case, you will have to run the `aiidalab-launch start` command again to restart AiiDAlab. -This will build and start an image where the `aiidalab` package, the `aiidalab-widgets-base` library, and the `aiidalab-home` app are installed with their latest development versions instead of their latest release versions. +Please see `aiidalab-launch --help` for a full list of available commands and options. -## Other deployments +### Cloud and other deployments -The `manage.py` script uses docker-compose to manage the local AiiDAlab deployment. Please see the [AiiDAlab documentation](https://aiidalab.readthedocs.io/) for information on how to use and deploy AiiDAlab docker images in alternative ways. ## Citation @@ -62,7 +41,7 @@ Users of AiiDAlab are kindly asked to cite the following publication in their ow A. V. Yakutovich et al., Comp. Mat. Sci. 188, 110165 (2021). [DOI:10.1016/j.commatsci.2020.110165](https://doi.org/10.1016/j.commatsci.2020.110165) -# Acknowledgements +## Acknowledgements This work is supported by the [MARVEL National Centre for Competency in Research]() funded by the [Swiss National Science Foundation](), as well as by the [MaX diff --git a/docker-compose.develop.yml b/docker-compose.develop.yml deleted file mode 100644 index 5188c079..00000000 --- a/docker-compose.develop.yml +++ /dev/null @@ -1,12 +0,0 @@ -version: "3.8" -services: - aiidalab: - build: - args: - - aiidalab_home_version=develop - - > - extra_requirements= - aiidalab@git+https://github.com/aiidalab/aiidalab@develop - aiidalab-widgets-base@git+https://github.com/aiidalab/aiidalab-widgets-base@develop - black - pre-commit diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 996181ad..00000000 --- a/docker-compose.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: "3.8" -services: - aiidalab: - build: - context: . - args: - - aiidalab_home_version=v21.10.0 - ports: - - ${AIIDALAB_PORT:-8888}:8888 - environment: - AIIDALAB_DEFAULT_APPS: ${AIIDALAB_DEFAULT_APPS:-aiidalab-widgets-base} - JUPYTER_TOKEN: ${JUPYTER_TOKEN:-} - SYSTEM_USER: ${SYSTEM_USER:-aiida} - volumes: - - "${AIIDALAB_HOME_VOLUME:-aiidalab-home}:/home/aiida" -volumes: - aiidalab-home: diff --git a/manage.py b/manage.py deleted file mode 100755 index ce43c145..00000000 --- a/manage.py +++ /dev/null @@ -1,277 +0,0 @@ -#!/usr/bin/env python -"""Convenience script wrapper to start and stop AiiDAlab via docker-compose. - -Authors: - * Carl Simon Adorf -""" -import json -import os -import re -from collections import OrderedDict -from pathlib import Path -from secrets import token_hex -from subprocess import SubprocessError, run -from textwrap import wrap -from time import sleep - -import click -from dotenv import dotenv_values - - -def _get_service_container_id(docker_compose, service): - return ( - docker_compose(["ps", "-q", service], capture_output=True) - .stdout.decode() - .strip() - ) - - -def _service_is_up(docker_compose, service): - service_container_id = _get_service_container_id(docker_compose, "aiidalab") - if service_container_id: - running_containers = ( - run(["docker", "ps", "-q", "--no-trunc"], capture_output=True) - .stdout.decode() - .strip() - .splitlines() - ) - return service_container_id in running_containers - else: - return False - - -@click.group() -@click.option( - "--develop", - is_flag=True, - help="Use this option to build AiiDAlab with development versions.", -) -@click.option("-p", "--project-name", help="Specify an alternative project name.") -@click.option( - "--env-file", - type=click.Path(dir_okay=False, writable=True, path_type=Path), - default=".env", - help="The path of the env-file to use for configuration.", - show_default=True, -) -@click.option( - "-v", - "--verbose", - count=True, - help="Provide this option to increase the output verbosity of the launcher.", -) -@click.option( - "--yes", - is_flag=True, - help="Automatically respond with yes to any prompts.", -) -@click.pass_context -def cli(ctx, develop, project_name, env_file, verbose, yes): - - # Specify the compose-files that will be merged to generate the final config. - compose_file_args = ["-f", "docker-compose.yml"] - if develop: - compose_file_args.extend(["-f", "docker-compose.develop.yml"]) - - # This command is to be used by all sub-commands. - def _compose_cmd(args, **kwargs): - args.insert(0, f"--env-file={env_file}") - if project_name: - args.insert(0, f"--project-name={project_name}") - kwargs.setdefault("capture_output", not verbose) - kwargs.setdefault("check", True) - kwargs.setdefault("env", {}) - kwargs["env"].setdefault("PATH", os.environ["PATH"]) - return run(["docker-compose", *compose_file_args, *args], **kwargs) - - ctx.obj = dict() - ctx.obj["compose_cmd"] = _compose_cmd - ctx.obj["env_file"] = env_file - ctx.obj["verbose"] = verbose - ctx.obj["yes"] = yes - - -@cli.command() -@click.pass_context -def show_config(ctx): - """Show the merged docker-compose config.""" - _docker_compose = ctx.obj["compose_cmd"] - click.echo(_docker_compose(["config"], capture_output=True).stdout) - - -@cli.command() -@click.option( - "--home-dir", - type=click.Path(), - help="Specify a path to a directory on a host system that is to be mounted " - "as the home directory on the AiiDAlab service. Uses docker volume if not provided.", -) -@click.option( - "--port", - help="Port on which AiiDAlab can be accessed.", -) -@click.option( - "--username", help="Specify the username to be used within the container." -) -@click.option( - "--jupyter-token", - help="A secret token that is needed to access AiiDAlab for the first time. " - "Defaults to a random string if not provided (recommended).", -) -@click.option( - "--app", - multiple=True, - help="Specify app to install on first server start, using the same syntax as " - "`aiidalab install`. This option can be used multiple times to specify " - "multiple default apps.", -) -@click.pass_context -def configure(ctx, home_dir, port, username, jupyter_token, app): - """Configure the local AiiDAlab environment.""" - env_file = ctx.obj["env_file"] - - # First, specify the defaults. - env = { - "AIIDALAB_HOME_VOLUME": "aiidalab-home", - "AIIDALAB_PORT": "8888", - "AIIDALAB_DEFAULT_APPS": "aiidalab-widgets-base", - "JUPYTER_TOKEN": token_hex(32), - "SYSTEM_USER": "aiida", - } - - # Next, update them with the currently stored values. - env.update(dotenv_values(env_file)) - - # Finally, update them with any of the values provided as arguments. - provided = { - "AIIDALAB_HOME_VOLUME": str(home_dir) if home_dir else None, - "AIIDALAB_PORT": str(port) if port else None, - "AIIDALAB_DEFAULT_APPS": " ".join(app), - "JUPYTER_TOKEN": str(jupyter_token) if jupyter_token else None, - "SYSTEM_USER": username or None, - } - env.update({key: value for key, value in provided.items() if value}) - - # Write environment to the env_file. - env_file.write_text( - "\n".join(f"{key}={value}" for key, value in env.items()) + "\n" - ) - click.echo(f"Written configuration to '{env_file}'.") - - -@cli.command() -@click.option( - "--restart", is_flag=True, help="Restart AiiDAlab in case that it is already up." -) -@click.pass_context -def up(ctx, restart): - """Start AiiDAlab on this host.""" - - # Check for an '.env' file. The file can be automatically created via the - # `configure` command. - msg_warn_up_without_env_file = "\n".join( - wrap( - "Warning: Did not find an '.env' file in the current working directory. It " - "is recommended to run the 'configure' command prior to first start. " - "Continue anyways?" - ) - ) - env_file = Path.cwd().joinpath(".env") - if not env_file.exists() and not ctx.obj["yes"]: - click.confirm(msg_warn_up_without_env_file, abort=True) - - # Get the `docker-compose` proxy command from the global context. - _docker_compose = ctx.obj["compose_cmd"] - - # Check if server is already started. - if not restart and _service_is_up(_docker_compose, "aiidalab"): - click.echo( - "Service is already running. Use the `--restart` option to force a restart." - ) - - # Actually run the `docker-compose up` command. - click.echo("Starting AiiDAlab (this can take multiple minutes) ...") - _docker_compose( - ["up", "--detach", "--build"] + (["--force-recreate"] if restart else []) - ) - - # We display the entry point to the user by invoking the `status()` - # function, which the user can also invoke directly via the `status` - # sub-command. We sleep briefly, as trying to determine the entry point - # immediately after "upping" the service is prone to fail. - sleep(0.5) - ctx.invoke(status) - - -@cli.command() -@click.option( - "-v", - "--volumes", - is_flag=True, - help="In addition to stopping the service, also remove any volumes. " - "Warning: This can lead to irreversible data loss!", -) -@click.pass_context -def down(ctx, volumes): - """Stop AiiDAlab on this host. - - This is a thin wrapper around `docker-compose down`. - """ - _docker_compose = ctx.obj["compose_cmd"] - if volumes and not ctx.obj["yes"]: - click.confirm( - "Are you sure you want to remove all volumes? " - "This can lead to irreversible data loss!", - abort=True, - ) - - _docker_compose(["down"] + (["--volumes"] if volumes else [])) - click.echo("AiiDAlab stopped.") - - -@cli.command("status") -@click.pass_context -def status(ctx): - """Show status of the AiiDAlab instance. - - Shows the entrypoint for running instances. - """ - _docker_compose = ctx.obj["compose_cmd"] - - try: - _docker_compose(["exec", "aiidalab", "wait-for-services"]) - aiidalab_container_id = _get_service_container_id(_docker_compose, "aiidalab") - config = json.loads( - run( - ["docker", "inspect", aiidalab_container_id], - check=True, - capture_output=True, - ).stdout - )[0] - - host_port = config["HostConfig"]["PortBindings"]["8888/tcp"][0]["HostPort"] - for env in config["Config"]["Env"]: - if "JUPYTER_TOKEN" in env: - jupyter_token = env.split("=")[1] - break - else: - raise RuntimeError("Failed to determine jupyter token.") - except SubprocessError as error: - click.echo( - "Unable to communicate with the AiiDAlab container. Is it running? " - "Use `up` to start it." - ) - except (KeyError, IndexError) as error: - raise click.ClickException( - f"Failed to determine entry point due to error: '{error}'" - ) - else: - click.secho( - f"Open this link in the browser to enter AiiDAlab:\n" - f"http://localhost:{host_port}/?token={jupyter_token}", - fg="green", - ) - - -if __name__ == "__main__": - cli() diff --git a/my_init.d/prepare-aiidalab.sh b/my_init.d/prepare-aiidalab.sh index abc0b49f..eb40bbff 100755 --- a/my_init.d/prepare-aiidalab.sh +++ b/my_init.d/prepare-aiidalab.sh @@ -1,9 +1,6 @@ #!/bin/bash set -em -# For backwards compatibility -ln -sf /home/${SYSTEM_USER} /project - # Change group of the aiidalab-home folder chown root:${SYSTEM_USER} -R /opt/aiidalab-home diff --git a/opt/prepare-aiidalab.sh b/opt/prepare-aiidalab.sh index 7df217b6..2a3e086b 100755 --- a/opt/prepare-aiidalab.sh +++ b/opt/prepare-aiidalab.sh @@ -6,6 +6,10 @@ set -x # Environment. export SHELL=/bin/bash +# Fix https://github.com/aiidalab/aiidalab-docker-stack/issues/225 +if [ -L /home/${SYSTEM_USER}/${SYSTEM_USER} ]; then + rm /home/${SYSTEM_USER}/${SYSTEM_USER} +fi # Setup AiiDA jupyter extension. # Don't forget to copy this file to .ipython/profile_default/startup/ @@ -74,7 +78,7 @@ reentry scan # Clear user trash directory. if [ -e /home/${SYSTEM_USER}/.trash ]; then - find /home/${SYSTEM_USER}/.trash/ -mindepth 1 -writable -delete + rm -rf /home/${SYSTEM_USER}/.trash/* fi # Remove old apps_meta.sqlite requests cache files. diff --git a/requirements-manage.txt b/requirements-manage.txt deleted file mode 100644 index a92b689a..00000000 --- a/requirements-manage.txt +++ /dev/null @@ -1,3 +0,0 @@ -click==8.0.3 -docker-compose==1.29.2 -python-dotenv==0.19.1 diff --git a/requirements-server.txt b/requirements-server.txt index 9ab93422..af820181 100644 --- a/requirements-server.txt +++ b/requirements-server.txt @@ -2,6 +2,8 @@ jupyterhub==1.5.0 jupyterlab==3.0.17 notebook==6.4.1 +# Detect notebooks written in MyST Markdown +jupytext==1.13.4 # Dependencies for jupyter widgets bqplot==0.12.25 ipytree==0.1.8