Merge pull request #102 from agimus-project/next-pack

Update packaging, CI, and tools

.git-blame-ignore-revs

@@ -0,0 +1 @@
+0e80a5fc7c514dacd1d8b957adf42ad636b4b1e3 pre-commit run -a

.github/workflows/test.yml → .github/workflows/conda-test.yml

@@ -1,4 +1,4 @@
-name: Tests & Coverage
+name: Tests with conda
 
 on: [ push, pull_request ]
 
@@ -10,7 +10,7 @@ jobs:
         shell: bash -el {0}
     strategy:
       matrix:
-        python-version: [ "3.9" ]
+        python-version: ["3.9", "3.10"]
     steps:
       - uses: actions/checkout@v4
         with:
@@ -34,10 +34,12 @@ jobs:
         run: echo "::set-output name=today::$(/bin/date -u '+%Y%m%d')"
         shell: bash
 
-      - name: Caching of the happypose installation
+      - name: Caching of the happypose installation and data
         uses: actions/cache@v3
         with:
-          path: ${{ env.CONDA }}/envs
+          path: |
+            ${{ env.CONDA }}/envs
+            local_data
           key:
             conda-${{ runner.os }}--${{ runner.arch }}--${{steps.get-date.outputs.today }}-${{hashFiles('environment.yml') }}-${{env.CACHE_NUMBER }}
         env:
@@ -46,41 +48,23 @@ jobs:
         id: cache
 
       - name: Update conda environment with happypose dependencies
-        run:
-          mamba env update -n happypose -f environment.yml
-        if: steps.cache.outputs.cache-hit != 'true'
-
-      - name: Install bop toolkit (temporal fix)
-        run: |
-          cd deps/bop_toolkit_challenge/
-          sed 's/==.*$//' requirements.txt > req_nover.txt
-          pip install -r req_nover.txt -e .
+        run: mamba env update -n happypose -f environment.yml
         if: steps.cache.outputs.cache-hit != 'true'
 
       - name: Install happypose
-        run: |
-          cd happypose/pose_estimators/cosypose
-          pip install .
-          cd ../../..
-          pip install -e .
+        run: pip install -e ".[render,evaluation]"
 
       - name: Download pre-trained models required for tests
         run: |
-          mkdir local_data
-          python -m happypose.toolbox.utils.download --cosypose_model=detector-bop-ycbv-pbr--970850
-          python -m happypose.toolbox.utils.download --cosypose_model=coarse-bop-ycbv-pbr--724183
-          python -m happypose.toolbox.utils.download --cosypose_model=refiner-bop-ycbv-pbr--604090
-          
-          python -m happypose.toolbox.utils.download --megapose_models
-          
-          cd tests/data
-          git clone https://github.com/petrikvladimir/happypose_test_data.git crackers_example
+          mkdir -p local_data
+          python -m happypose.toolbox.utils.download \
+            --megapose_models \
+            --examples \
+              crackers_example \
+            --cosypose_models  \
+              detector-bop-ycbv-pbr--970850 \
+              coarse-bop-ycbv-pbr--724183 \
+              refiner-bop-ycbv-pbr--604090
 
       - name: Run tests
-        run: |
-          pip install pytest coverage
-          coverage run --source=happypose -m pytest tests
-          coverage xml
-      - uses: codecov/codecov-action@v3
-        env:
-          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+        run: python -m unittest

.github/workflows/pip-test.yml

@@ -0,0 +1,48 @@
+name: Tests with pip
+
+on: [ push, pull_request ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10"]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+
+      - name: Install EGL mesa - required for Panda3D renderer
+        run: sudo apt-get update && sudo apt-get install -qqy libegl1-mesa libegl1-mesa-dev rclone
+
+      - name: Caching of the happypose installation and data
+        uses: actions/cache@v3
+        with:
+          path: local_data
+          key: data
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Update pip
+        run: pip install -U pip
+
+      - name: Install happypose
+        run: pip install ".[render,cpu,evaluation]" --extra-index-url https://download.pytorch.org/whl/cpu
+
+      - name: Download pre-trained models required for tests
+        run: |
+          mkdir -p local_data
+          python -m happypose.toolbox.utils.download \
+            --megapose_models \
+            --examples \
+              crackers_example \
+            --cosypose_models  \
+              detector-bop-ycbv-pbr--970850 \
+              coarse-bop-ycbv-pbr--724183 \
+              refiner-bop-ycbv-pbr--604090
+
+      - name: Run tests
+        run: python -m unittest

.github/workflows/poetry-test.yml

@@ -0,0 +1,56 @@
+name: Tests with poetry + Coverage
+
+on: [ push, pull_request ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10"]
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+
+      - name: Install EGL mesa - required for Panda3D renderer
+        run: sudo apt-get update && sudo apt-get install -qqy libegl1-mesa libegl1-mesa-dev rclone
+
+      - name: Setup poetry
+        run: pipx install poetry
+
+      - name: Caching of the happypose installation and data
+        uses: actions/cache@v3
+        with:
+          path: local_data
+          key: data
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: poetry
+
+      - name: Install happypose
+        run: poetry install --with dev -E render -E cpu -E evaluation
+
+      - name: Download pre-trained models required for tests
+        run: |
+          mkdir -p local_data
+          poetry run python -m happypose.toolbox.utils.download \
+            --megapose_models \
+            --examples \
+              crackers_example \
+            --cosypose_models  \
+              detector-bop-ycbv-pbr--970850 \
+              coarse-bop-ycbv-pbr--724183 \
+              refiner-bop-ycbv-pbr--604090
+
+      - name: Run tests
+        run: poetry run coverage run --source=happypose -m unittest
+
+      - name: Process coverage
+        run: poetry run coverage xml
+
+      - uses: codecov/codecov-action@v3
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.gitignore

@@ -14,3 +14,5 @@ __pycache__
 *.cpython
 experiments/outputs
 /book
+happypose/pose_estimators/cosypose/build/
+local_data

.gitmodules

@@ -1,15 +1,3 @@
-[submodule "happypose/pose_estimators/cosypose/deps/bop_toolkit_challenge"]
-	path = happypose/pose_estimators/cosypose/deps/bop_toolkit_challenge
-	url = https://github.com/ylabbe/bop_toolkit_challenge20.git
-[submodule "happypose/pose_estimators/cosypose/deps/bop_toolkit_cosypose"]
-	path = happypose/pose_estimators/cosypose/deps/bop_toolkit_cosypose
-	url = https://github.com/ylabbe/bop_toolkit_cosypose.git
-[submodule "happypose/pose_estimators/cosypose/deps/job-runner"]
-	path = happypose/pose_estimators/cosypose/deps/job-runner
-	url = https://github.com/ylabbe/job-runner.git
-[submodule "deps/bop_toolkit_challenge"]
-	path = deps/bop_toolkit_challenge
-	url = https://github.com/thodan/bop_toolkit
 [submodule "deps/bop_renderer"]
 	path = deps/bop_renderer
 	url = https://github.com/thodan/bop_renderer/

.pre-commit-config.yaml

@@ -1,38 +1,39 @@
-exclude: 'cosypose|megapose6d|happypose|experiments'
 repos:
-#-   repo: https://github.com/pre-commit/pre-commit-hooks
-    #rev: v4.4.0
-    #hooks:
-    #-   id: check-added-large-files
-    #-   id: check-ast
-    #-   id: check-executables-have-shebangs
-    #-   id: check-json
-    #-   id: check-merge-conflict
-    #-   id: check-symlinks
-    #-   id: check-toml
-    #-   id: check-yaml
-    #-   id: debug-statements
-    #-   id: destroyed-symlinks
-    #-   id: detect-private-key
-    #-   id: end-of-file-fixer
-    #-   id: fix-byte-order-marker
-    #-   id: mixed-line-ending
-    #-   id: trailing-whitespace
-#-   repo: https://github.com/PyCQA/isort
-    #rev: 5.12.0
-    #hooks:
-    #-   id: isort
-#-   repo: https://github.com/psf/black
-    #rev: 23.3.0
-    #hooks:
-    #-   id: black
--   repo: https://github.com/pappasam/toml-sort
-    rev: v0.23.1
-    hooks:
-    - id: toml-sort-fix
-      exclude: 'poetry.lock'
-#-   repo: https://github.com/charliermarsh/ruff-pre-commit
-    #rev: v0.0.267
-    #hooks:
-      #- id: ruff
-        #args: [--fix, --exit-non-zero-on-fix]
+- repo: https://github.com/PyCQA/isort
+  rev: 5.12.0
+  hooks:
+  - id: isort
+- repo: https://github.com/charliermarsh/ruff-pre-commit
+  rev: v0.0.267
+  hooks:
+  - id: ruff
+    args:
+    - --fix
+    - --exit-non-zero-on-fix
+- repo: https://github.com/pappasam/toml-sort
+  rev: v0.23.1
+  hooks:
+  - id: toml-sort-fix
+    exclude: poetry.lock
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.4.0
+  hooks:
+  - id: check-added-large-files
+  - id: check-ast
+  - id: check-executables-have-shebangs
+  - id: check-json
+  - id: check-merge-conflict
+  - id: check-symlinks
+  - id: check-toml
+  - id: check-yaml
+  - id: debug-statements
+  - id: destroyed-symlinks
+  - id: detect-private-key
+  - id: end-of-file-fixer
+  - id: fix-byte-order-marker
+  - id: mixed-line-ending
+  - id: trailing-whitespace
+- repo: https://github.com/psf/black
+  rev: 23.3.0
+  hooks:
+  - id: black

README.md

@@ -84,7 +84,7 @@ conda activate happypose
 mamba install compilers -c conda-forge
 pip install open3d
 mkdir /build && cd /build && git clone https://github.com/MIT-SPARK/TEASER-plusplus.git
-cd TEASER-plusplus && mkdir build && cd build 
+cd TEASER-plusplus && mkdir build && cd build
 cmake -DTEASERPP_PYTHON_VERSION=3.9 .. && make teaserpp_python
 cd python && pip install .
-```
+```

configs/code_snapshot/dsgen_deps.yaml

@@ -5,4 +5,3 @@ python_packages_dir:
   - ${oc.env:PROJECTS_DIR}/happypose
   - ${oc.env:PROJECTS_DIR}/blenderproc
   - ${oc.env:PROJECTS_DIR}/bop_toolkit_lib
-

configs/code_snapshot/happypose.yaml

@@ -5,5 +5,5 @@ python_packages_dir:
   - ${oc.env:PROJECTS_DIR}/happypose
 
 snapshot_dir: ${oc.env:HP_DATA_DIR}/code_snapshots/${hydra:job.id}
-  
-exclude_path: ${oc.env:HP_ROOT_DIR}/configs/snapshot_ignore.txt
+
+exclude_path: ${oc.env:HP_ROOT_DIR}/configs/snapshot_ignore.txt

configs/dsgen/default.yaml

@@ -1,2 +1,2 @@
 defaults:
-  - base_dsgen
+  - base_dsgen

configs/dsgen/fastrun.yaml

@@ -2,4 +2,4 @@ defaults:
   - default
 few: True
 verbose: True
-debug: True
+debug: True

configs/dsgen/gso_1M.yaml

@@ -1,3 +1,3 @@
 defaults:
   - default
-dataset_id: gso_1M
+dataset_id: gso_1M

configs/job_env/happypose.yaml

@@ -1,3 +1,3 @@
 defaults:
   - base_job_env
-conda_env: 'happypose'
+conda_env: 'happypose'

configs/job_env/jz_yann.yaml

@@ -5,4 +5,4 @@ env:
   HP_DATA_DIR: $WORK/data/happypose
   HAPPYPOSE_DATA_DIR: $WORK/data/megapose
   BLENDER_INSTALL_DIR: $WORK/blender/blender-2.93.0-linux-x64
-  BLENDERPROC_DIR: $WORK/projects/blenderproc
+  BLENDERPROC_DIR: $WORK/projects/blenderproc

configs/job_env/lda.yaml

@@ -5,4 +5,4 @@ env:
   HP_DATA_DIR: /home/ylabbe/data/happypose
   HAPPYPOSE_DATA_DIR: /home/ylabbe/data/megapose-private
   BLENDER_INSTALL_DIR: $HOME/blenderproc/blender-2.93.88-linux-x64
-  BLENDER_PROC_DIR: /home/ylabbe/projects/blenderproc
+  BLENDER_PROC_DIR: /home/ylabbe/projects/blenderproc

configs/local_job/single_gpu.yaml

@@ -2,4 +2,4 @@ defaults:
   - base_local_job
 nodes: 1
 tasks_per_node: 1
-gpus_per_node: 1
+gpus_per_node: 1

configs/local_node/lda.yaml

@@ -3,4 +3,4 @@ defaults:
 gpus_per_node: 1
 mem_per_gpu: '8GB'
 cpus_per_gpu: 12
-mem_per_cpu: '2.7GB'
+mem_per_cpu: '2.7GB'

configs/run_ds_postproc/default.yaml

@@ -16,8 +16,8 @@ runner:
     cpus_per_task: 1
 
 hydra:
-  run: 
+  run:
     dir: ${oc.env:HP_DATA_DIR}/hydra_outputs/${now:%Y-%m-%d}/${now:%H-%M-%S}
 
 n_jobs: 128
-ds_dir: ${oc.env:HP_DATA_DIR}/blender_pbr_datasets/gso_1M
+ds_dir: ${oc.env:HP_DATA_DIR}/blender_pbr_datasets/gso_1M