From bf398a34778298da02fa4a12c86791c6ef52bd03 Mon Sep 17 00:00:00 2001 From: Lea Vauchier Date: Thu, 11 Jan 2024 16:06:15 +0100 Subject: [PATCH] Update docker image to use mamba based image + use conda packages as much as possible --- .dockerignore | 14 +++++++++ Dockerfile | 56 ++++++++++------------------------- environment.yml | 78 ++++++++++++++++++++++++------------------------- 3 files changed, 69 insertions(+), 79 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..016b68f0 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,14 @@ +.hydra +.vscode +htmlcov + +# Caches +**/__pycache__ +.pytest_cache + + +# Distribution / packaging +build/ +dist/ +*.egg-info +*.egg \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 7fc19daf..f72d8ced 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,49 +1,25 @@ -FROM nvidia/cuda:11.3.1-base-ubuntu20.04 +FROM mambaorg/micromamba:focal-cuda-11.3.1 +# focal is Ubuntu 20.04 -# set the IGN proxy, otherwise apt-get and other applications don't work -# Should be commented out outside of IGN -ENV http_proxy 'http://192.168.4.9:3128/' -ENV https_proxy 'http://192.168.4.9:3128/' - -# Remove any third-party apt sources to avoid issues with expiring keys. -RUN rm -f /etc/apt/sources.list.d/*.list - -# Install some basic utilities -RUN apt-get update && apt-get install -y \ - nano \ - curl \ - ca-certificates \ - sudo \ - git \ - bzip2 \ - libx11-6 \ - && rm -rf /var/lib/apt/lists/* - -# Create a working directory -RUN mkdir /app +WORKDIR /app +COPY environment.yml environment.yml -# Set up the Conda environment and make python accessible via PATH. -ENV CONDA_AUTO_UPDATE_CONDA=false -ENV PATH=/miniconda:/miniconda/bin:$PATH -COPY environment.yml /app/environment.yml -RUN curl -sLo /miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-py39_4.10.3-Linux-x86_64.sh \ - && chmod +x /miniconda.sh \ - && /miniconda.sh -b -p /miniconda \ - && rm /miniconda.sh \ - && /miniconda/bin/conda env update -n base -f /app/environment.yml \ - && rm /app/environment.yml \ - && /miniconda/bin/conda clean -ya +RUN micromamba env create -f /app/environment.yml +ENV PATH=$PATH:/opt/conda/envs/myria3d/bin/ # Need to export this for torch_geometric to find where cuda is. # See https://github.com/pyg-team/pytorch_geometric/issues/2040#issuecomment-766610625 -ENV LD_LIBRARY_PATH="/miniconda/lib/:$LD_LIBRARY_PATH" +ENV LD_LIBRARY_PATH="/opt/conda/envs/myria3d/lib/:$LD_LIBRARY_PATH" -# Check succes of environment creation. +# Check success of environment creation. RUN python -c "import torch_geometric;" -# Copy the repository content in /app -WORKDIR /app -COPY . . +# use chown to prevent permission issues +COPY --chown=$MAMBA_USER:MAMBA_USER . . +# locate proj +ENV PROJ_LIB=/opt/conda/envs/myria3d/share/proj/ +# Check taht myria3d can run +RUN python -m myria3d.predict -h -# Set the default command to bash for image inspection. -CMD ["bash"] +# # Set the default command to bash for image inspection. +# CMD ["bash"] diff --git a/environment.yml b/environment.yml index d88c7fd6..e9b02fca 100644 --- a/environment.yml +++ b/environment.yml @@ -4,19 +4,41 @@ name: myria3d channels: - conda-forge - anaconda + - pytorch + - comet_ml + - pyg dependencies: - python==3.9.* - pip # cudatoolkit to specify the cuda driver in the conda env - - conda-forge::cudatoolkit=11.3.1 # single equal sign there, not a typo - - numba==0.55.1 + # - conda-forge::cudatoolkit=11.3.1 # single equal sign there, not a typo + # - numba==0.55.1 # --------- data formats --------- # - - numpy==1.20 + - numpy - h5py + # --------- Deep Learning --------- # + - pytorch::pytorch==1.11.0 + - pytorch::torchvision==0.12.0 + - conda-forge::pytorch-lightning==1.5.9 + - conda-forge::torchmetrics==0.7.* + - comet_ml::comet_ml==3.31.* + - conda-forge::urllib3<2 # To solve for https://github.com/GeneralMills/pytrends/issues/591 + - pyg::pytorch-cluster + - pyg::pytorch-scatter + - pyg::pytorch-sparse + - pyg::pyg==2.1.0 + # Nota: if libcusparse.so.11. errors occur, run + # export LD_LIBRARY_PATH="/home/${USER}/miniconda/envs/lib:$LD_LIBRARY_PATH" + # ou + # export LD_LIBRARY_PATH="/home/${USER}/anaconda3/envs/lib:$LD_LIBRARY_PATH" + # see https://github.com/pyg-team/pytorch_geometric/issues/2040#issuecomment-766610625 # --------- geo --------- # - - pygeos - - shapely + - pdal - python-pdal + - pyproj + # --------- Visualization --------- # + - pandas + - matplotlib # --------- loggers --------- # # --------- linters --------- # - pre-commit # hooks for applying linters on commit @@ -35,39 +57,17 @@ dependencies: - seaborn # used in some callbacks # - jupyterlab # better jupyter notebooks - pudb # debugger + # # --------- Documentation --------- # + - sphinx==4.5.* + - recommonmark==0.7.* + - sphinx_rtd_theme==1.0.* + - docutils==0.17 + - rstcheck==3.3.* # RST Linter - pip: - # --------- Deep Learning --------- # - # Extra index may need to be on first line - - --extra-index-url https://download.pytorch.org/whl/cu113 - - torch==1.11.* - - torchvision - - pytorch-lightning==1.5.9 - - torchmetrics==0.7.* # Else, pytorch-lightning will install the latest - - comet_ml==3.31.* - - urllib3<2 # To solve for https://github.com/GeneralMills/pytrends/issues/591 - # Wheels for torch-geometric optionnal dependencies - - https://data.pyg.org/whl/torch-1.11.0%2Bcu113/torch_cluster-1.6.0-cp39-cp39-linux_x86_64.whl - - https://data.pyg.org/whl/torch-1.11.0%2Bcu113/torch_scatter-2.0.9-cp39-cp39-linux_x86_64.whl - - https://data.pyg.org/whl/torch-1.11.0%2Bcu113/torch_sparse-0.6.14-cp39-cp39-linux_x86_64.whl - - git+https://github.com/pyg-team/pytorch_geometric.git@2.1.0 - # Nota: if libcusparse.so.11. errors occur, run - # export LD_LIBRARY_PATH="/home/${USER}/miniconda/envs/lib:$LD_LIBRARY_PATH" - # ou - # export LD_LIBRARY_PATH="/home/${USER}/anaconda3/envs/lib:$LD_LIBRARY_PATH" - # see https://github.com/pyg-team/pytorch_geometric/issues/2040#issuecomment-766610625 - # --------- Visualization --------- # - - pandas==1.4.* - - matplotlib==3.5.* # --------- hydra configs --------- # - - hydra-core==1.1.* - - hydra-colorlog==1.1.* - # # --------- Documentation --------- # - - sphinx==4.5.* - - sphinx_rtd_theme==1.0.* - - myst_parser==0.17.* - - sphinx_paramlinks==0.5.* - - recommonmark==0.7.* - - sphinxnotes-mock==1.0.0b0 # still a beta - - docutils==0.17 - - rstcheck==3.3.* # RST Linter - - pyproj==3.6.* + - hydra-core==1.1.* + - hydra-colorlog==1.1.* + # --------- Documentation --------- # + - myst_parser==0.17.* + - sphinxnotes-mock==1.0.0b0 # still a beta + - sphinx_paramlinks==0.5.* \ No newline at end of file