diff --git a/docker-bake.hcl b/docker-bake.hcl index b9a204de..81fa26a7 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -83,6 +83,9 @@ target "base-with-services" { "PGSQL_VERSION" = "${PGSQL_VERSION}" } } +# PYTHON_MINOR_VERSION is a Python version string +# without the patch version (e.g. "3.9") +# Used to construct paths to Python site-packages folder. target "lab" { inherits = ["lab-meta"] context = "stack/lab" @@ -93,6 +96,7 @@ target "lab" { args = { "AIIDALAB_VERSION" = "${AIIDALAB_VERSION}" "AIIDALAB_HOME_VERSION" = "${AIIDALAB_HOME_VERSION}" + "PYTHON_MINOR_VERSION" = join(".", slice(split(".", "${PYTHON_VERSION}"), 0, 2)) } } target "full-stack" { diff --git a/stack/base/Dockerfile b/stack/base/Dockerfile index d0ee2b27..b74691aa 100644 --- a/stack/base/Dockerfile +++ b/stack/base/Dockerfile @@ -20,8 +20,7 @@ ARG AIIDA_VERSION # We pin aiida-core to the exact installed version, # to prevent accidental upgrade or downgrade, that might # induce DB migration or break shared dependencies of AiiDAlab Apps. -RUN echo "pip==23.3.1" > /opt/requirements.txt && \ - echo "aiida-core==${AIIDA_VERSION}" >> /opt/requirements.txt +RUN echo "aiida-core==${AIIDA_VERSION}" > /opt/requirements.txt # Install the shared requirements. RUN mamba install --yes \ @@ -32,11 +31,15 @@ RUN mamba install --yes \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" -# Pin shared requirements in the base environemnt. +# Pin shared requirements in the conda base environment. RUN cat /opt/requirements.txt | xargs -I{} conda config --system --add pinned_packages {} +# Upgrade pip to latest +RUN pip install --upgrade --no-cache-dir pip # Configure pip to use requirements file as constraints file. -ENV PIP_CONSTRAINT=/opt/requirements.txt +ENV PIP_CONSTRAINT /opt/requirements.txt +# Ensure that pip installs packages to ~/.local by default +ENV PIP_USER 1 # Enable verdi autocompletion. RUN mkdir -p "${CONDA_DIR}/etc/conda/activate.d" && \ diff --git a/stack/lab/Dockerfile b/stack/lab/Dockerfile index 9b36cc98..565c74e8 100644 --- a/stack/lab/Dockerfile +++ b/stack/lab/Dockerfile @@ -41,20 +41,20 @@ ARG AIIDALAB_HOME_VERSION RUN git clone https://github.com/aiidalab/aiidalab-home && \ cd aiidalab-home && \ git checkout v"${AIIDALAB_HOME_VERSION}" && \ - pip install --quiet --no-cache-dir "./" && \ + pip install --no-user --quiet --no-cache-dir "./" && \ fix-permissions "./" && \ fix-permissions "${CONDA_DIR}" && \ fix-permissions "/home/${NB_USER}" -# Install and enable appmode. -RUN git clone https://github.com/oschuett/appmode.git && \ - cd appmode && \ - git checkout v0.8.0 -COPY gears.svg ./appmode/appmode/static/gears.svg -RUN pip install ./appmode --no-cache-dir && \ - jupyter nbextension enable --py --sys-prefix appmode && \ +# Install and enable appmode, turning Jupyter notebooks to Apps +RUN pip install appmode==0.8.0 --no-cache-dir --no-user +# Enable appmode extension +RUN jupyter nbextension enable --py --sys-prefix appmode && \ jupyter serverextension enable --py --sys-prefix appmode +# Swap appmode icon for AiiDAlab gears icon, shown during app load +COPY --chown=${NB_UID}:${NB_GID} gears.svg ${CONDA_DIR}/share/jupyter/nbextensions/appmode/gears.svg + # Copy start-up scripts for AiiDAlab. COPY before-notebook.d/* /usr/local/bin/before-notebook.d/ @@ -107,8 +107,5 @@ ENV NOTEBOOK_ARGS \ "--TerminalManager.cull_interval=60" # Set up the logo of notebook interface -COPY --chown=${NB_UID}:${NB_GID} aiidalab-wide-logo.png /tmp/notebook-logo.png - -# The directory location of logo.png is in the `${CONDA_DIR}/lib/python3.9/site-packages/notebook/static/base/images/logo.png`, -# but the python version may change in the future, thus we use the wildcard to match the python version. -RUN mv /tmp/notebook-logo.png ${CONDA_DIR}/lib/python3*/site-packages/notebook/static/base/images/logo.png +ARG PYTHON_MINOR_VERSION +COPY --chown=${NB_UID}:${NB_GID} aiidalab-wide-logo.png ${CONDA_DIR}/lib/python${PYTHON_MINOR_VERSION}/site-packages/notebook/static/base/images/logo.png diff --git a/tests/conftest.py b/tests/conftest.py index 5524cc44..860167e1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,6 +6,8 @@ from requests.exceptions import ConnectionError +VARIANTS = ("base", "lab", "base-with-services", "full-stack") + def is_responsive(url): try: @@ -16,12 +18,20 @@ def is_responsive(url): return False +def variant_checker(value): + msg = f"Invalid image variant '{value}', must be one of: {VARIANTS}" + if value not in VARIANTS: + raise pytest.UsageError(msg) + return value + + def pytest_addoption(parser): parser.addoption( "--variant", action="store", - default="base", + required=True, help="Variant (image name) of the docker-compose file to use.", + type=variant_checker, ) @@ -54,14 +64,30 @@ def execute(command, user=None, **kwargs): command = f"exec -T --user={user} aiidalab {command}" else: command = f"exec -T aiidalab {command}" - return docker_compose.execute(command, **kwargs) + out = docker_compose.execute(command, **kwargs) + return out.decode() return execute @pytest.fixture def nb_user(aiidalab_exec): - return aiidalab_exec("bash -c 'echo \"${NB_USER}\"'").decode().strip() + return aiidalab_exec("bash -c 'echo \"${NB_USER}\"'").strip() + + +@pytest.fixture +def pip_install(aiidalab_exec, nb_user): + """Temporarily install package via pip""" + package = None + + def _pip_install(pkg, **args): + nonlocal package + package = pkg + return aiidalab_exec(f"pip install {pkg}", **args) + + yield _pip_install + if package: + aiidalab_exec(f"pip uninstall --yes {package}") @pytest.fixture(scope="session") diff --git a/tests/test-base-with-services.py b/tests/test-base-with-services.py index ddcfcea1..235fb712 100644 --- a/tests/test-base-with-services.py +++ b/tests/test-base-with-services.py @@ -5,11 +5,8 @@ def test_correct_pgsql_version_installed(aiidalab_exec, pgsql_version): - info = json.loads( - aiidalab_exec( - "mamba list -n aiida-core-services --json --full-name postgresql" - ).decode() - )[0] + cmd = "mamba list -n aiida-core-services --json --full-name postgresql" + info = json.loads(aiidalab_exec(cmd))[0] assert info["name"] == "postgresql" assert parse(info["version"]).major == parse(pgsql_version).major @@ -18,5 +15,4 @@ def test_rabbitmq_can_start(aiidalab_exec): """Test the rabbitmq-server can start, the output should be empty if the command is successful.""" output = aiidalab_exec("mamba run -n aiida-core-services rabbitmq-server -detached") - - assert output == b"" + assert output == "" diff --git a/tests/test-base.py b/tests/test-base.py index 3a7a4b56..25143233 100644 --- a/tests/test-base.py +++ b/tests/test-base.py @@ -5,50 +5,61 @@ from packaging.version import parse -@pytest.mark.parametrize("incompatible_version", ["2.3.0"]) -def test_prevent_pip_install_of_incompatible_aiida_version( - aiidalab_exec, nb_user, aiida_version, incompatible_version +@pytest.mark.parametrize("pkg_manager", ["pip", "mamba"]) +def test_prevent_installation_of_aiida( + aiidalab_exec, nb_user, aiida_version, pkg_manager ): - package_manager = "pip" + """aiida-core is pinned to the exact version in the container, + test that both pip and mamba refuse to install a different version""" + + incompatible_version = "2.3.0" assert parse(aiida_version) != parse(incompatible_version) + # Expected to succeed, although should be a no-op. - aiidalab_exec( - f"{package_manager} install aiida-core=={aiida_version}", user=nb_user - ) + aiidalab_exec(f"{pkg_manager} install aiida-core=={aiida_version}", user=nb_user) with pytest.raises(Exception): aiidalab_exec( - f"{package_manager} install aiida-core=={incompatible_version}", + f"{pkg_manager} install aiida-core=={incompatible_version}", user=nb_user, ) def test_correct_python_version_installed(aiidalab_exec, python_version): - info = json.loads(aiidalab_exec("mamba list --json --full-name python").decode())[0] + info = json.loads(aiidalab_exec("mamba list --json --full-name python"))[0] assert info["name"] == "python" assert parse(info["version"]) == parse(python_version) def test_create_conda_environment(aiidalab_exec, nb_user): - output = aiidalab_exec("conda create -y -n tmp", user=nb_user).decode().strip() + output = aiidalab_exec("conda create -y -n tmp", user=nb_user).strip() assert "conda activate tmp" in output # New conda environments should be created in ~/.conda/envs/ - output = aiidalab_exec("conda env list", user=nb_user).decode().strip() + output = aiidalab_exec("conda env list", user=nb_user).strip() assert f"/home/{nb_user}/.conda/envs/tmp" in output -def test_pip_check(aiidalab_exec): - aiidalab_exec("pip check") - - def test_correct_aiida_version_installed(aiidalab_exec, aiida_version): - info = json.loads( - aiidalab_exec("mamba list --json --full-name aiida-core").decode() - )[0] + cmd = "mamba list --json --full-name aiida-core" + info = json.loads(aiidalab_exec(cmd))[0] assert info["name"] == "aiida-core" assert parse(info["version"]) == parse(aiida_version) def test_path_local_pip(aiidalab_exec, nb_user): """test that the pip local bin path ~/.local/bin is added to PATH""" - output = aiidalab_exec("bash -c 'echo \"${PATH}\"'", user=nb_user).decode() + output = aiidalab_exec("bash -c 'echo \"${PATH}\"'", user=nb_user) assert f"/home/{nb_user}/.local/bin" in output + + +def test_pip_user_install(aiidalab_exec, pip_install, nb_user): + """Test that pip installs packages to ~/.local/ by default""" + import email + + # We use 'tuna' as an example of python-only package without dependencies + pkg = "tuna" + pip_install(pkg) + output = aiidalab_exec(f"pip show {pkg}") + + # `pip show` output is in the RFC-compliant email header format + msg = email.message_from_string(output) + assert msg.get("Location").startswith(f"/home/{nb_user}/.local/") diff --git a/tests/test-common.py b/tests/test-common.py index 01ac9f57..fad551ee 100644 --- a/tests/test-common.py +++ b/tests/test-common.py @@ -9,14 +9,18 @@ def test_notebook_service_available(notebook_service): def test_verdi_status(aiidalab_exec, nb_user): - output = aiidalab_exec("verdi status", user=nb_user).decode().strip() + output = aiidalab_exec("verdi status", user=nb_user).strip() assert "Connected to RabbitMQ" in output assert "Daemon is running" in output def test_ssh_agent_is_running(aiidalab_exec, nb_user): - output = aiidalab_exec("ps aux | grep ssh-agent", user=nb_user).decode().strip() + output = aiidalab_exec("ps aux | grep ssh-agent", user=nb_user).strip() assert "ssh-agent" in output # also check only one ssh-agent process is running assert len(output.splitlines()) == 1 + + +def test_pip_check(aiidalab_exec): + aiidalab_exec("pip check") diff --git a/tests/test-full-stack.py b/tests/test-full-stack.py index 55ad4a9c..bcd7ba6b 100644 --- a/tests/test-full-stack.py +++ b/tests/test-full-stack.py @@ -4,13 +4,10 @@ @pytest.fixture(scope="function") def generate_aiidalab_install_output(aiidalab_exec, nb_user): def _generate_aiidalab_install_output(package_name): - output = ( - aiidalab_exec(f"aiidalab install --yes --pre {package_name}", user=nb_user) - .decode() - .strip() - ) + cmd = f"aiidalab install --yes --pre {package_name}" + output = aiidalab_exec(cmd, user=nb_user).strip() - output += aiidalab_exec(f"pip check", user=nb_user).decode().strip() + output += aiidalab_exec("pip check", user=nb_user).strip() # Uninstall the package to make sure the test is repeatable app_name = package_name.split("@")[0] diff --git a/tests/test-lab.py b/tests/test-lab.py index df4775dc..9e350af5 100644 --- a/tests/test-lab.py +++ b/tests/test-lab.py @@ -4,21 +4,30 @@ def test_correct_aiidalab_version_installed(aiidalab_exec, aiidalab_version): - info = json.loads(aiidalab_exec("mamba list --json --full-name aiidalab").decode())[ - 0 - ] + cmd = "mamba list --json --full-name aiidalab" + info = json.loads(aiidalab_exec(cmd))[0] assert info["name"] == "aiidalab" assert parse(info["version"]) == parse(aiidalab_version) def test_correct_aiidalab_home_version_installed(aiidalab_exec, aiidalab_home_version): - info = json.loads( - aiidalab_exec("mamba list --json --full-name aiidalab-home").decode() - )[0] + cmd = "mamba list --json --full-name aiidalab-home" + info = json.loads(aiidalab_exec(cmd))[0] assert info["name"] == "aiidalab-home" assert parse(info["version"]) == parse(aiidalab_home_version) +def test_appmode_installed(aiidalab_exec): + """Test that appmode pip package is installed in correct location""" + import email + + output = aiidalab_exec("pip show appmode") + + # `pip show` output is in the RFC-compliant email header format + msg = email.message_from_string(output) + assert msg.get("Location").startswith("/opt/conda/lib/python") + + @pytest.mark.parametrize("incompatible_version", ["22.7.1"]) def test_prevent_pip_install_of_incompatible_aiidalab_version( aiidalab_exec,