add back testing, add osx conda build, add ubuntu conda deployment

facebookresearch · Nov 21, 2024 · e255dc1 · e255dc1
1 parent 1a1da75
commit e255dc1
Showing 1 changed file with 190 additions and 120 deletions.
diff --git a/.github/workflows/install_and_test.yml b/.github/workflows/install_and_test.yml
@@ -71,127 +71,127 @@ jobs:
     - name: run mypy
       run: mypy
 
-  # install_and_test_ubuntu:
-  #   runs-on: 4-core-ubuntu-gpu-t4
-  #   env:
-  #     FPS_THRESHOLD: 900
-  #   defaults:
-  #     run:
-  #       shell: bash -el {0}
-  #   steps:
-  #   - uses: actions/[email protected]
-  #     with:
-  #       path: "./habitat-sim"
-  #   - name: CPU info
-  #     run: cat /proc/cpuinfo
-  #   - uses: "./habitat-sim/.github/actions/install_ubuntu_deps"
-  #   - uses: "./habitat-sim/.github/actions/install_ubuntu_gpu_deps"
-  #   - name: Build, install habitat-sim
-  #     run: |-
-  #       #give cmake ownership to the runner for installation
-  #       sudo chown runner -R /opt/cmake312/
-  #       #activate conda env
-  #       export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH
-  #       conda init
-  #       source ~/.bashrc
-  #       conda activate habitat
-  #       #install habitat-sim
-  #       cd habitat-sim
-  #       pip install -r requirements.txt --progress-bar off
-  #       git submodule update --init --recursive --jobs 8
-  #       python -u setup.py install --build-type "Release" --lto --headless --bullet
-  #   - name: Download test data
-  #     run: |-
-  #       # Disable clone protection for git lfs
-  #       export GIT_CLONE_PROTECTION_ACTIVE=false
+  install_and_test_ubuntu:
+    runs-on: 4-core-ubuntu-gpu-t4
+    env:
+      FPS_THRESHOLD: 900
+    defaults:
+      run:
+        shell: bash -el {0}
+    steps:
+    - uses: actions/[email protected]
+      with:
+        path: "./habitat-sim"
+    - name: CPU info
+      run: cat /proc/cpuinfo
+    - uses: "./habitat-sim/.github/actions/install_ubuntu_deps"
+    - uses: "./habitat-sim/.github/actions/install_ubuntu_gpu_deps"
+    - name: Build, install habitat-sim
+      run: |-
+        #give cmake ownership to the runner for installation
+        sudo chown runner -R /opt/cmake312/
+        #activate conda env
+        export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH
+        conda init
+        source ~/.bashrc
+        conda activate habitat
+        #install habitat-sim
+        cd habitat-sim
+        pip install -r requirements.txt --progress-bar off
+        git submodule update --init --recursive --jobs 8
+        python -u setup.py install --build-type "Release" --lto --headless --bullet
+    - name: Download test data
+      run: |-
+        # Disable clone protection for git lfs
+        export GIT_CLONE_PROTECTION_ACTIVE=false
 
-  #       git --version
-  #       git-lfs --version
-  #       export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH
-  #       conda init
-  #       source ~/.bashrc
-  #       conda activate habitat
-  #       conda install -y gitpython git-lfs
-  #       cd habitat-sim
-  #       git lfs install
-  #       python src_python/habitat_sim/utils/datasets_download.py --uids ci_test_assets ycb rearrange_dataset_v2 --replace --data-path data/ --no-prune
-  #       ls -la data/scene_datasets/habitat-test-scenes/
-  #   - name: Run sim benchmark
-  #     run: |-
-  #       export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH
-  #       conda init
-  #       source ~/.bashrc
-  #       conda activate habitat
-  #       cd habitat-sim
-  #       python examples/example.py --scene data/scene_datasets/habitat-test-scenes/van-gogh-room.glb --silent --test_fps_regression $FPS_THRESHOLD
-  #   - name: Run sim tests
-  #     run: |-
-  #       #give cmake ownership to the runner for installation
-  #       sudo chown runner -R /opt/cmake312/
-  #       #activate conda env
-  #       export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH
-  #       conda init
-  #       source ~/.bashrc
-  #       conda activate habitat
-  #       cd habitat-sim
-  #       export PYTHONPATH=$(pwd):$PYTHONPATH
+        git --version
+        git-lfs --version
+        export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH
+        conda init
+        source ~/.bashrc
+        conda activate habitat
+        conda install -y gitpython git-lfs
+        cd habitat-sim
+        git lfs install
+        python src_python/habitat_sim/utils/datasets_download.py --uids ci_test_assets ycb rearrange_dataset_v2 --replace --data-path data/ --no-prune
+        ls -la data/scene_datasets/habitat-test-scenes/
+    - name: Run sim benchmark
+      run: |-
+        export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH
+        conda init
+        source ~/.bashrc
+        conda activate habitat
+        cd habitat-sim
+        python examples/example.py --scene data/scene_datasets/habitat-test-scenes/van-gogh-room.glb --silent --test_fps_regression $FPS_THRESHOLD
+    - name: Run sim tests
+      run: |-
+        #give cmake ownership to the runner for installation
+        sudo chown runner -R /opt/cmake312/
+        #activate conda env
+        export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH
+        conda init
+        source ~/.bashrc
+        conda activate habitat
+        cd habitat-sim
+        export PYTHONPATH=$(pwd):$PYTHONPATH
 
-  #       #set pytest to show partial progress
-  #       export PYTHONUNBUFFERED=1
+        #set pytest to show partial progress
+        export PYTHONUNBUFFERED=1
 
-  #       # run tests with code coverage
-  #       CORRADE_TEST_COLOR=ON GTEST_COLOR=yes ./build.sh --headless \
-  #         --bullet \
-  #         --with-cuda \
-  #         --build-datatool \
-  #         --run-tests \
-  #         --no-lto \
-  #         --cmake-args='-DCMAKE_CXX_FLAGS="--coverage"'
-  #       PYTHONPATH=src_python pytest -n 4 --durations=10 --cov-report=xml --cov=./
+        # run tests with code coverage
+        CORRADE_TEST_COLOR=ON GTEST_COLOR=yes ./build.sh --headless \
+          --bullet \
+          --with-cuda \
+          --build-datatool \
+          --run-tests \
+          --no-lto \
+          --cmake-args='-DCMAKE_CXX_FLAGS="--coverage"'
+        PYTHONPATH=src_python pytest -n 4 --durations=10 --cov-report=xml --cov=./
 
-  #       #run the marked pytest-benchmark tests and print the results
-  #       PYTHONPATH=src_python pytest -m sim_benchmarks
+        #run the marked pytest-benchmark tests and print the results
+        PYTHONPATH=src_python pytest -m sim_benchmarks
 
-  #       #re-build without bullet and cuda and run physics tests again
-  #       #TODO: instead of reinstall, do this with configuration
-  #       ./build.sh --headless --cmake-args='-DCMAKE_CXX_FLAGS="--coverage"'
-  #       PYTHONPATH=src_python pytest -n 4 --durations=10 --cov-report=xml --cov=./ --cov-append tests/test_physics.py tests/test_sensors.py
-  #   - name: upload test coverage
-  #     run: |-
-  #       export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH
-  #       conda init
-  #       source ~/.bashrc
-  #       conda activate habitat
-  #       cd habitat-sim
+        #re-build without bullet and cuda and run physics tests again
+        #TODO: instead of reinstall, do this with configuration
+        ./build.sh --headless --cmake-args='-DCMAKE_CXX_FLAGS="--coverage"'
+        PYTHONPATH=src_python pytest -n 4 --durations=10 --cov-report=xml --cov=./ --cov-append tests/test_physics.py tests/test_sensors.py
+    - name: upload test coverage
+      run: |-
+        export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH
+        conda init
+        source ~/.bashrc
+        conda activate habitat
+        cd habitat-sim
 
-  #       curl -Os https://uploader.codecov.io/latest/linux/codecov
-  #       chmod +x codecov
-  #       #Uploading test coverage for Python code
-  #       ./codecov -f coverage.xml -cF Python
+        curl -Os https://uploader.codecov.io/latest/linux/codecov
+        chmod +x codecov
+        #Uploading test coverage for Python code
+        ./codecov -f coverage.xml -cF Python
 
-  #       #Uploading test coverage for C++ code
-  #       lcov --directory . --capture --output-file coverage.info
-  #       # Replaces -1 linecount with zero to prevent lcov from crashing:
-  #       # https://github.com/psycofdj/coverxygen/issues/6
-  #       sed -i -e 's/,-1$/,0/g' coverage.info
-  #       #lcov --remove coverage.info "*/deps/*" --output-file coverage.info > /dev/null
-  #       #lcov --remove coverage.info "*/test/*" --output-file coverage.info > /dev/null
-  #       #lcov --remove coverage.info "*/tests/*" --output-file coverage.info > /dev/null
-  #       ./codecov -f coverage.info -cF CPP
-  #   - name: install habitat-sim with audio and run audio_agent script
-  #     run: |-
-  #       #give cmake ownership to the runner for installation
-  #       sudo chown runner -R /opt/cmake312/
-  #       export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH
-  #       conda init
-  #       source ~/.bashrc
-  #       conda activate habitat
-  #       cd habitat-sim
-  #       pip install -r requirements.txt --progress-bar off
-  #       pip install imageio imageio-ffmpeg
-  #       git submodule update --init --recursive --jobs 8
-  #       python -u setup.py install --build-type "Release" --lto --headless --audio
-  #       python examples/tutorials/audio_agent.py
+        #Uploading test coverage for C++ code
+        lcov --directory . --capture --output-file coverage.info
+        # Replaces -1 linecount with zero to prevent lcov from crashing:
+        # https://github.com/psycofdj/coverxygen/issues/6
+        sed -i -e 's/,-1$/,0/g' coverage.info
+        #lcov --remove coverage.info "*/deps/*" --output-file coverage.info > /dev/null
+        #lcov --remove coverage.info "*/test/*" --output-file coverage.info > /dev/null
+        #lcov --remove coverage.info "*/tests/*" --output-file coverage.info > /dev/null
+        ./codecov -f coverage.info -cF CPP
+    - name: install habitat-sim with audio and run audio_agent script
+      run: |-
+        #give cmake ownership to the runner for installation
+        sudo chown runner -R /opt/cmake312/
+        export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH
+        conda init
+        source ~/.bashrc
+        conda activate habitat
+        cd habitat-sim
+        pip install -r requirements.txt --progress-bar off
+        pip install imageio imageio-ffmpeg
+        git submodule update --init --recursive --jobs 8
+        python -u setup.py install --build-type "Release" --lto --headless --audio
+        python examples/tutorials/audio_agent.py
     #NOTE: use the below to debug with ssh: simply move this "job" just before the crashing job to intercept the workflow
     #- name: Debugging with tmate
     #  uses: mxschmitt/[email protected]
@@ -212,7 +212,7 @@ jobs:
           path: "./habitat-sim"
       - name: Set CI_TEST variable based on "main merge" vs. "pull_request" workflow
         run: |-
-          echo "NIGHTLY='false'" >> $GITHUB_ENV
+          echo "NIGHTLY=''" >> $GITHUB_ENV
           echo "CI_TEST='true'" >> $GITHUB_ENV
           if [[ ${{ github.event_name }} == 'push' ]]; then
             if [[ ${GITHUB_REF} == refs/heads/* ]]; then
@@ -226,14 +226,12 @@ jobs:
           elif [[ ${{ github.event_name }} == 'schedule' ]]; then
             echo "trigger: a nightly build"
             echo "CI_TEST='false'" >> $GITHUB_ENV
-            echo "NIGHTLY='true'" >> $GITHUB_ENV
+            echo "NIGHTLY='--nightly'" >> $GITHUB_ENV
             echo "CONDA_USERNAME=$AIHABITAT_NIGHTLY_CONDA_USERNAME" >> $GITHUB_ENV
             echo "CONDA_PWD=$AIHABITAT_NIGHTLY_CONDA_PWD" >> $GITHUB_ENV
           elif [[ ${{ github.event_name }} == 'pull_request' ]]; then
             echo "trigger: a pull request"
           fi
-      #- name: Debugging with tmate
-      #  uses: mxschmitt/[email protected]
       - name: Build conda Linux packages
         run: |-
           cd habitat-sim/conda-build
@@ -252,7 +250,8 @@ jobs:
             export PATH=$HOME/anaconda/bin:$PATH
 
             # Delete old nightly builds
-            #python common/delete_old_night_packages.py --username $CONDA_USERNAME --password $CONDA_PWD
+            #NOTE: The ${{ env.NIGHTLY }} == "--nightly" variable is necessary to trigger this
+            #python common/delete_old_night_packages.py --username ${{ env.CONDA_USERNAME }} --password ${{ env.CONDA_PWD }} ${{ env.NIGHTLY }}
           fi
 
           #install Docker
@@ -268,5 +267,76 @@ jobs:
           if [[ ${{ env.CI_TEST }} ]]; then
             docker run --ipc=host --rm -v $(pwd)/../../:/remote hsim_condabuild_dcontainer /bin/bash -c "source ~/.bashrc && conda activate py39 && cd /remote/habitat-sim/ && git config --global --add safe.directory '*' && cd /remote/habitat-sim/conda-build && python matrix_builder.py --ci_test"
           else
-            echo "TODO: the deployment command"
+            docker run --ipc=host --rm -v $(pwd)/../../:/remote --env ${{ env.CONDA_PWD }} hsim_condabuild_dcontainer /bin/bash -c "source ~/.bashrc && conda activate py39 && cd /remote/habitat-sim/ && git config --global --add safe.directory '*' && cd /remote/habitat-sim/conda-build && yes | anaconda login --username ${{ env.CONDA_USERNAME }} --password ${{ env.CONDA_PWD }} --hostname "aihabitat-conda-ci-builder-linux" && python matrix_builder.py --conda_upload ${{ env.NIGHTLY }}"
+          fi
+
+  build_conda_osx:
+    runs-on: macos-14
+    env:
+      AIHABITAT_CONDA_USERNAME: aihabitat
+      AIHABITAT_CONDA_PWD: ${{ secrets.AIHABITAT_CONDA_PWD }}
+      AIHABITAT_NIGHTLY_CONDA_USERNAME: aihabitat-nightly
+      AIHABITAT_NIGHTLY_CONDA_PWD: ${{ secrets.AIHABITAT_NIGHTLY_CONDA_PWD }}
+    defaults:
+      run:
+        shell: bash -el {0}
+    steps:
+      - uses: actions/[email protected]
+        with:
+          path: "./habitat-sim"
+      - name: Conda Install OSX
+        run: |-
+          curl -o ~/miniconda.sh -O https://repo.anaconda.com/miniconda/Miniconda3-py310_24.1.2-0-MacOSX-arm64.sh
+          chmod +x ~/miniconda.sh
+          ~/miniconda.sh -b -p $HOME/miniconda
+          rm ~/miniconda.sh
+          export PATH=$HOME/miniconda/bin:$PATH
+          conda init "$(basename "${SHELL}")"
+      - name: Set CI_TEST variable based on "main merge" vs. "pull_request" workflow
+        run: |-
+          echo "NIGHTLY=''" >> $GITHUB_ENV
+          echo "CI_TEST='true'" >> $GITHUB_ENV
+          if [[ ${{ github.event_name }} == 'push' ]]; then
+            if [[ ${GITHUB_REF} == refs/heads/* ]]; then
+              echo "trigger: a push to main branch"
+            elif [[ ${GITHUB_REF} == refs/tags/* ]]; then
+              echo "trigger: a version tag push"
+              echo "CI_TEST='false'" >> $GITHUB_ENV
+              echo "CONDA_USERNAME=$AIHABITAT_CONDA_USERNAME" >> $GITHUB_ENV
+              echo "CONDA_PWD=$AIHABITAT_CONDA_PWD" >> $GITHUB_ENV
+            fi
+          elif [[ ${{ github.event_name }} == 'schedule' ]]; then
+            echo "trigger: a nightly build"
+            echo "CI_TEST='false'" >> $GITHUB_ENV
+            echo "NIGHTLY='--nightly'" >> $GITHUB_ENV
+            echo "CONDA_USERNAME=$AIHABITAT_NIGHTLY_CONDA_USERNAME" >> $GITHUB_ENV
+            echo "CONDA_PWD=$AIHABITAT_NIGHTLY_CONDA_PWD" >> $GITHUB_ENV
+          elif [[ ${{ github.event_name }} == 'pull_request' ]]; then
+            echo "trigger: a pull request"
+          fi
+      - name: Build conda OSX packages
+        run: |-
+          export PATH=$HOME/miniconda/bin:$PATH
+          source ~/.bash_profile
+          conda activate
+          conda install -y anaconda-client ccache cmake git ninja conda-build pip
+          echo $(which -a python)
+          pip install gitpython
+          cd habitat-sim
+          git submodule update --init --recursive --jobs 8
+          pip install .
+          pip uninstall -y habitat-sim
+          rm -rf build
+          (yes || true) | anaconda login --username ${{ env.CONDA_USERNAME }} --password ${{ env.CONDA_PWD }} --hostname "aihabitat-conda-ci-builder-macos"
+          #pivot on the build type
+          if [[ ${{ env.CI_TEST }} ]]; then
+            echo "This is a CI test."
+            cd conda-build
+            export PYTHONIOENCODING="utf-8"
+            python matrix_builder.py --ci_test
+          else
+            conda config --set anaconda_upload yes
+            cd conda-build
+            export PYTHONIOENCODING="utf-8"
+            python matrix_builder.py ${{ env.NIGHTLY }}
           fi