Merge pull request #14 from IGNF/dev

Tests & Refactor (incl. dependencies, CICD workflow, Documentation workflow) & Doc.

.env_example

@@ -1,10 +1,7 @@
 ## LOG PATH
 LOGS_DIR="path/to/dir/"
 
-# Prepared data: splitted LAS files in LAS_NAME subdirs in train/val/test subdirs : 
-# - /PREPARED_DATA_DIR/val/name/000.data
-# - /PREPARED_DATA_DIR/val/name/001.data
-# ...
+## DATA
 PREPARED_DATA_DIR="path/to/prepared/data/dir"
 
 ## LOGGER

.github/workflows/cicd.yaml

@@ -1,52 +1,99 @@
 name: CICD
 
-# on:
-#   pull_request:
-#     branches: [main]
-
-on: push
+on:
+  # Run CICD for non-draft pull request
+  pull_request:
+    branches:
+      - dev
+      - main
+  # Also run when the pull request merges (which generates a push)
+  # So that we can tag the docker image appropriately.
+  push:
+    branches:
+      - dev
+      - prod
+      - main
 
 jobs:
-  lidar_on_docker:
+  CICD:
     runs-on: self-hosted
 
     steps:
 
-    - name: Checkout branch       
+    - name: Checkout branch
       uses: actions/checkout@v2
 
-    - name: build docker image       
-      run: docker build -t lidar_deep_im .
+    - name: Build docker image
+      run: docker build -t myria3d .
 
     - name: Check code neatness (linter)
-      run: docker run lidar_deep_im flake8
+      run: docker run myria3d flake8
 
-    # - name: unit testing
-    #   run: docker run lidar_deep_im pytest --ignore=actions-runner --ignore="notebooks"
+    - name: Fast unit tests first
+      run: >
+        docker run
+        -v /var/data/CICD_github_assets/myria3d_B2V0.5/inputs/:/myria3d/tests/files/large/
+        --ipc=host
+        myria3d
+        python -m
+        pytest -rA -v -m "not slow"
+        --ignore=actions-runner
 
     - name: Full module run on LAS subset
-      run: docker run -v /var/data/CICD_github_assets:/CICD_github_assets lidar_deep_im 
-      # sudo mount -v -t cifs -o user=mdaab,domain=IGN,uid=24213,gid=10550 //store.ign.fr/store-lidarhd/projet-LHD/IA/Validation_Module/CICD_github_assets/B2V0.5 /var/data/CICD_github_assets
-
-    # - name: Evaluate decisions using optimization code on a single, corrected LAS 
-    #   run: >
-    #     docker run -v /var/data/cicd/CICD_github_assets:/CICD_github_assets lidar_deep_im
-    #     python lidar_prod/run.py print_config=true +task='optimize'
-    #     +building_validation.optimization.debug=true
-    #     building_validation.optimization.todo='prepare+evaluate+update'
-    #     building_validation.optimization.paths.input_las_dir=/CICD_github_assets/M8.0/20220204_building_val_V0.0_model/20211001_buiding_val_val/
-    #     building_validation.optimization.paths.results_output_dir=/CICD_github_assets/opti/
-    #     building_validation.optimization.paths.building_validation_thresholds_pickle=/CICD_github_assets/M8.3B2V0.0/optimized_thresholds.pickle
-
-    - name: chekc the user
-      run: whoami
-
-    - name: save the docker image because everything worked
-      run: docker save lidar_deep_im > /var/data/CICD_github_assets/lidar_deep_im.tar # need writing right
-
-    - name: clean the server for further uses
+      run: >
+        docker run
+        -v /var/data/cicd/CICD_github_assets/myria3d_B2V0.5/inputs/:/inputs/
+        -v /var/data/cicd/CICD_github_assets/myria3d_B2V0.5/outputs/:/outputs/
+        --ipc=host
+        myria3d
+        python -m 
+        myria3d.predict
+        --config-path /inputs/
+        --config-name predict_config_V2.0.0.yaml
+        predict.src_las=/inputs/792000_6272000_subset_buildings.las
+        predict.output_dir=/outputs/
+        predict.ckpt_path=/inputs/RandLaNet_Buildings_B2V0.5_epoch_033.ckpt
+        datamodule.batch_size=25
+        hydra.run.dir=/myria3d
+
+    # Always run with --ipc=host to enable sufficient shared memory when predicting on large data
+    - name: Slow unit tests last, using large files stored in self-hosted action-runner.
+      run: >
+        docker run
+        -v /var/data/cicd/CICD_github_assets/myria3d_B2V0.5/inputs/:/myria3d/tests/data/large/
+        --ipc=host
+        myria3d
+        python -m
+        pytest -rA -v -m "slow" 
+        --ignore=actions-runner
+
+    # Everything ran so we tag the valid docker image to keep it
+    # This happens for push events, which are in particular 
+    # triggered when a pull request is merged.
+    - name: Get the branch name into an environment variable.
+      if: github.event_name == 'push'
+      uses: nelonoel/[email protected]
+
+    - name: Print the branch name
+      if: github.event_name == 'push'
+      run: echo ${BRANCH_NAME}
+
+    - name: Tag the docker image with branch name
+      if: github.event_name == 'push'
+      run: docker tag myria3d:latest myria3d:${BRANCH_NAME}
+
+    - name: Run the new, tagged image at least once so that is it not prunned by mistake when using docker system prune
+      if: github.event_name == 'push'
+      run: docker run myria3d:${BRANCH_NAME} bash
+
+    # This needs writing rights to the mounted path
+    - name: Save the docker image as myria3d_${BRANCH_NAME}.tar
+      if: github.event_name == 'push'
+      run: docker save myria3d:${BRANCH_NAME} -o /var/data/cicd/CICD_github_assets/CICD_docker_images/myria3d_${BRANCH_NAME}.tar
+
+    - name: Clean dangling docker images
       if: always()  # always do it, even if something failed
-      run: docker system prune  # remove obsolete docker images (take a HUGE amount of space)
+      run: docker system prune --force # remove dangling docker images, without asking user for confirmation
 
 
 

.github/workflows/gh-pages.yml

@@ -6,13 +6,15 @@ on:
   push:         # <- trigger when we call push
     branches:
       - main  # <- but only on main branch
-      - FixDocAPI  # <- also on this branch until documentation is up and running.
 
 jobs:
 
   build-and-deploy:
     runs-on: ubuntu-latest
-
+    # Use bash instead of sh for conda activation
+    defaults:
+      run:
+        shell: bash -l {0}
     # Task to do when launching the workflow
     steps:
 
@@ -22,29 +24,49 @@ jobs:
         uses: actions/checkout@v2
 
       # 2. Sphinx part : install tool and dependencies
+      # See https://github.com/conda-incubator/setup-miniconda#caching-environments
 
-      - name: "Set up Python"
-        uses: actions/setup-python@v1
+      - name: Setup a conda-incubator with an empty conda env
+        uses: conda-incubator/setup-miniconda@v2
         with:
-          python-version: 3.9.12
+            python-version: 3.9.12
+            miniforge-variant: Mambaforge
+            miniforge-version: latest
+            use-mamba: true
+            # For caching to work:
+            use-only-tar-bz2: true
+            # Environment to create and activate for next steps
+            activate-environment: myria3d
 
-      # Packages that depend on torch need need to be installed afterwards, 
-      # hence the "requirements_torch_deps.txt file.
-      - name: "Install Python dependencies"
-        working-directory: ./docs/
-        run: |
-          python3 -m pip install --upgrade pip
-          pip3 install -r requirements.txt
-          pip3 install -r requirements_torch_deps.txt
+      # Cache the conda environment
+      # See https://github.com/conda-incubator/setup-miniconda#caching-environments
+      - name: Get Date
+        id: get-date
+        run: echo "::set-output name=today::$(/bin/date -u '+%Y%m%d')"
+        shell: bash
+
+      - name: Cache conda environment
+        uses: actions/cache@v2
+        with:
+          path: ${{ env.CONDA }}/envs
+          key: conda-${{ runner.os }}--${{ runner.arch }}--${{ steps.get-date.outputs.today }}-${{ hashFiles('setup_env/requirements.yml') }}-${{ hashFiles('setup_env/requirements.txt') }}
+        env:
+          # Increase this value to reset cache if etc/example-environment.yml has not changed
+          CACHE_NUMBER: 0
+        id: cache
+
+      - name: Update environment if there was no cached env.
+        run: mamba env update -n myria3d -f setup_env/requirements.yml
+        if: steps.cache.outputs.cache-hit != 'true'
 
+      - name: List installed packages
+        run: mamba list
 
       - name: "Build Sphinx Doc"
         working-directory: ./docs/
-        run: |
-          make html
+        run: make html
 
       # 3. Déploiement sur les Github Pages
-
       - name: "Deploy Github Pages"
         uses: JamesIves/[email protected]
         with:

.gitignore

@@ -1,7 +1,12 @@
-# Usr defined
+# User defined
 nohup.out
+inputs/
 outputs/
-bash/schedule/*
+checkpoints/
+runs/
+# large files that are not tracked
+tests/data/large
+
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

Dockerfile

@@ -0,0 +1,73 @@
+FROM nvidia/cuda:10.2-devel-ubuntu18.04
+# An nvidia image seems to be necessary for torch-points-kernel. 
+# Also, a "devel" image seems required for the same library
+
+# set the IGN proxy, otherwise apt-get and other applications don't work
+# Should be commented out outside of IGN
+ENV http_proxy 'http://192.168.4.9:3128/'
+ENV https_proxy 'http://192.168.4.9:3128/'
+
+# set the timezone, otherwise it asks for it... and freezes
+ENV TZ=Europe/Paris
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+# Needed to use apt-get afterwards due to CUDA changes described here since April 27, 2022:
+# https://forums.developer.nvidia.com/t/notice-cuda-linux-repository-key-rotation/212772
+# Not the recommpended method, but else we need wget installed afterwards.
+# We changed to 10.2-devel-ubuntu18.04 so that might not be needed. 
+RUN apt-get update && apt-get upgrade -y && apt-get install -y wget
+RUN apt-key del 7fa2af80
+RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub
+
+# all the apt-get installs
+RUN apt-get update && apt-get upgrade -y && apt-get install -y \
+        software-properties-common  \
+        wget                        \
+        git                         \
+        libgl1-mesa-glx libegl1-mesa libxrandr2 libxrandr2 libxss1 libxcursor1 libxcomposite1 libasound2 libxi6 libxtst6   # package needed for anaconda
+
+RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh \
+        && /bin/bash ~/miniconda.sh -b -p /opt/conda \
+        && rm ~/miniconda.sh
+
+ENV PATH /opt/conda/bin:$PATH
+
+# Only copy necessary files to set up the environment, 
+# to use docker caching if requirements files were not updated.
+WORKDIR /setup_env
+COPY ./setup_env/ .
+
+# install mamba to setup the env faster
+RUN conda install -y mamba -n base -c conda-forge
+# Build the environment
+RUN mamba env create -f requirements.yml
+
+# Copy the repository content in /myria3d 
+WORKDIR /myria3d
+COPY . .
+
+# Make RUN commands use the new environment:
+SHELL ["conda", "run", "-n", "myria3d", "/bin/bash", "-c"]
+
+# the entrypoint garanty that all command will be runned in the conda environment
+ENTRYPOINT ["conda",  \   
+        "run", \
+        "-n", \
+        "myria3d"]
+
+# Example usage
+CMD         ["python", \
+        "-m", \
+        "myria3d.predict", \
+        "--config-path", \
+        "/CICD_github_assets/parametres_etape1/.hydra", \ 
+        "--config-name", \
+        "predict_config_V1.6.3.yaml", \
+        "predict.src_las=/CICD_github_assets/parametres_etape1/test/792000_6272000_subset_buildings.las", \
+        "predict.output_dir=/CICD_github_assets/output_etape1", \
+        "predict.ckpt_path=/CICD_github_assets/parametres_etape1/checkpoints/epoch_033.ckpt", \
+        "predict.gpus=0", \
+        "datamodule.batch_size=10", \ 
+        "datamodule.subtile_overlap=0", \ 
+        "hydra.run.dir=/myria3d"]
+