From e255dc19acc965c9da474ad582a8a47f7330ed75 Mon Sep 17 00:00:00 2001 From: Alexander Clegg Date: Thu, 21 Nov 2024 11:00:43 -0800 Subject: [PATCH] add back testing, add osx conda build, add ubuntu conda deployment --- .github/workflows/install_and_test.yml | 310 +++++++++++++++---------- 1 file changed, 190 insertions(+), 120 deletions(-) diff --git a/.github/workflows/install_and_test.yml b/.github/workflows/install_and_test.yml index e94b9ca01c..a0add34ea1 100644 --- a/.github/workflows/install_and_test.yml +++ b/.github/workflows/install_and_test.yml @@ -71,127 +71,127 @@ jobs: - name: run mypy run: mypy - # install_and_test_ubuntu: - # runs-on: 4-core-ubuntu-gpu-t4 - # env: - # FPS_THRESHOLD: 900 - # defaults: - # run: - # shell: bash -el {0} - # steps: - # - uses: actions/checkout@v4.1.1 - # with: - # path: "./habitat-sim" - # - name: CPU info - # run: cat /proc/cpuinfo - # - uses: "./habitat-sim/.github/actions/install_ubuntu_deps" - # - uses: "./habitat-sim/.github/actions/install_ubuntu_gpu_deps" - # - name: Build, install habitat-sim - # run: |- - # #give cmake ownership to the runner for installation - # sudo chown runner -R /opt/cmake312/ - # #activate conda env - # export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH - # conda init - # source ~/.bashrc - # conda activate habitat - # #install habitat-sim - # cd habitat-sim - # pip install -r requirements.txt --progress-bar off - # git submodule update --init --recursive --jobs 8 - # python -u setup.py install --build-type "Release" --lto --headless --bullet - # - name: Download test data - # run: |- - # # Disable clone protection for git lfs - # export GIT_CLONE_PROTECTION_ACTIVE=false + install_and_test_ubuntu: + runs-on: 4-core-ubuntu-gpu-t4 + env: + FPS_THRESHOLD: 900 + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@v4.1.1 + with: + path: "./habitat-sim" + - name: CPU info + run: cat /proc/cpuinfo + - uses: "./habitat-sim/.github/actions/install_ubuntu_deps" + - uses: "./habitat-sim/.github/actions/install_ubuntu_gpu_deps" + - name: Build, install habitat-sim + run: |- + #give cmake ownership to the runner for installation + sudo chown runner -R /opt/cmake312/ + #activate conda env + export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH + conda init + source ~/.bashrc + conda activate habitat + #install habitat-sim + cd habitat-sim + pip install -r requirements.txt --progress-bar off + git submodule update --init --recursive --jobs 8 + python -u setup.py install --build-type "Release" --lto --headless --bullet + - name: Download test data + run: |- + # Disable clone protection for git lfs + export GIT_CLONE_PROTECTION_ACTIVE=false - # git --version - # git-lfs --version - # export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH - # conda init - # source ~/.bashrc - # conda activate habitat - # conda install -y gitpython git-lfs - # cd habitat-sim - # git lfs install - # python src_python/habitat_sim/utils/datasets_download.py --uids ci_test_assets ycb rearrange_dataset_v2 --replace --data-path data/ --no-prune - # ls -la data/scene_datasets/habitat-test-scenes/ - # - name: Run sim benchmark - # run: |- - # export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH - # conda init - # source ~/.bashrc - # conda activate habitat - # cd habitat-sim - # python examples/example.py --scene data/scene_datasets/habitat-test-scenes/van-gogh-room.glb --silent --test_fps_regression $FPS_THRESHOLD - # - name: Run sim tests - # run: |- - # #give cmake ownership to the runner for installation - # sudo chown runner -R /opt/cmake312/ - # #activate conda env - # export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH - # conda init - # source ~/.bashrc - # conda activate habitat - # cd habitat-sim - # export PYTHONPATH=$(pwd):$PYTHONPATH + git --version + git-lfs --version + export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH + conda init + source ~/.bashrc + conda activate habitat + conda install -y gitpython git-lfs + cd habitat-sim + git lfs install + python src_python/habitat_sim/utils/datasets_download.py --uids ci_test_assets ycb rearrange_dataset_v2 --replace --data-path data/ --no-prune + ls -la data/scene_datasets/habitat-test-scenes/ + - name: Run sim benchmark + run: |- + export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH + conda init + source ~/.bashrc + conda activate habitat + cd habitat-sim + python examples/example.py --scene data/scene_datasets/habitat-test-scenes/van-gogh-room.glb --silent --test_fps_regression $FPS_THRESHOLD + - name: Run sim tests + run: |- + #give cmake ownership to the runner for installation + sudo chown runner -R /opt/cmake312/ + #activate conda env + export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH + conda init + source ~/.bashrc + conda activate habitat + cd habitat-sim + export PYTHONPATH=$(pwd):$PYTHONPATH - # #set pytest to show partial progress - # export PYTHONUNBUFFERED=1 + #set pytest to show partial progress + export PYTHONUNBUFFERED=1 - # # run tests with code coverage - # CORRADE_TEST_COLOR=ON GTEST_COLOR=yes ./build.sh --headless \ - # --bullet \ - # --with-cuda \ - # --build-datatool \ - # --run-tests \ - # --no-lto \ - # --cmake-args='-DCMAKE_CXX_FLAGS="--coverage"' - # PYTHONPATH=src_python pytest -n 4 --durations=10 --cov-report=xml --cov=./ + # run tests with code coverage + CORRADE_TEST_COLOR=ON GTEST_COLOR=yes ./build.sh --headless \ + --bullet \ + --with-cuda \ + --build-datatool \ + --run-tests \ + --no-lto \ + --cmake-args='-DCMAKE_CXX_FLAGS="--coverage"' + PYTHONPATH=src_python pytest -n 4 --durations=10 --cov-report=xml --cov=./ - # #run the marked pytest-benchmark tests and print the results - # PYTHONPATH=src_python pytest -m sim_benchmarks + #run the marked pytest-benchmark tests and print the results + PYTHONPATH=src_python pytest -m sim_benchmarks - # #re-build without bullet and cuda and run physics tests again - # #TODO: instead of reinstall, do this with configuration - # ./build.sh --headless --cmake-args='-DCMAKE_CXX_FLAGS="--coverage"' - # PYTHONPATH=src_python pytest -n 4 --durations=10 --cov-report=xml --cov=./ --cov-append tests/test_physics.py tests/test_sensors.py - # - name: upload test coverage - # run: |- - # export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH - # conda init - # source ~/.bashrc - # conda activate habitat - # cd habitat-sim + #re-build without bullet and cuda and run physics tests again + #TODO: instead of reinstall, do this with configuration + ./build.sh --headless --cmake-args='-DCMAKE_CXX_FLAGS="--coverage"' + PYTHONPATH=src_python pytest -n 4 --durations=10 --cov-report=xml --cov=./ --cov-append tests/test_physics.py tests/test_sensors.py + - name: upload test coverage + run: |- + export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH + conda init + source ~/.bashrc + conda activate habitat + cd habitat-sim - # curl -Os https://uploader.codecov.io/latest/linux/codecov - # chmod +x codecov - # #Uploading test coverage for Python code - # ./codecov -f coverage.xml -cF Python + curl -Os https://uploader.codecov.io/latest/linux/codecov + chmod +x codecov + #Uploading test coverage for Python code + ./codecov -f coverage.xml -cF Python - # #Uploading test coverage for C++ code - # lcov --directory . --capture --output-file coverage.info - # # Replaces -1 linecount with zero to prevent lcov from crashing: - # # https://github.com/psycofdj/coverxygen/issues/6 - # sed -i -e 's/,-1$/,0/g' coverage.info - # #lcov --remove coverage.info "*/deps/*" --output-file coverage.info > /dev/null - # #lcov --remove coverage.info "*/test/*" --output-file coverage.info > /dev/null - # #lcov --remove coverage.info "*/tests/*" --output-file coverage.info > /dev/null - # ./codecov -f coverage.info -cF CPP - # - name: install habitat-sim with audio and run audio_agent script - # run: |- - # #give cmake ownership to the runner for installation - # sudo chown runner -R /opt/cmake312/ - # export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH - # conda init - # source ~/.bashrc - # conda activate habitat - # cd habitat-sim - # pip install -r requirements.txt --progress-bar off - # pip install imageio imageio-ffmpeg - # git submodule update --init --recursive --jobs 8 - # python -u setup.py install --build-type "Release" --lto --headless --audio - # python examples/tutorials/audio_agent.py + #Uploading test coverage for C++ code + lcov --directory . --capture --output-file coverage.info + # Replaces -1 linecount with zero to prevent lcov from crashing: + # https://github.com/psycofdj/coverxygen/issues/6 + sed -i -e 's/,-1$/,0/g' coverage.info + #lcov --remove coverage.info "*/deps/*" --output-file coverage.info > /dev/null + #lcov --remove coverage.info "*/test/*" --output-file coverage.info > /dev/null + #lcov --remove coverage.info "*/tests/*" --output-file coverage.info > /dev/null + ./codecov -f coverage.info -cF CPP + - name: install habitat-sim with audio and run audio_agent script + run: |- + #give cmake ownership to the runner for installation + sudo chown runner -R /opt/cmake312/ + export PATH=$HOME/miniconda/bin:/usr/local/cuda/bin:$PATH + conda init + source ~/.bashrc + conda activate habitat + cd habitat-sim + pip install -r requirements.txt --progress-bar off + pip install imageio imageio-ffmpeg + git submodule update --init --recursive --jobs 8 + python -u setup.py install --build-type "Release" --lto --headless --audio + python examples/tutorials/audio_agent.py #NOTE: use the below to debug with ssh: simply move this "job" just before the crashing job to intercept the workflow #- name: Debugging with tmate # uses: mxschmitt/action-tmate@v3.18 @@ -212,7 +212,7 @@ jobs: path: "./habitat-sim" - name: Set CI_TEST variable based on "main merge" vs. "pull_request" workflow run: |- - echo "NIGHTLY='false'" >> $GITHUB_ENV + echo "NIGHTLY=''" >> $GITHUB_ENV echo "CI_TEST='true'" >> $GITHUB_ENV if [[ ${{ github.event_name }} == 'push' ]]; then if [[ ${GITHUB_REF} == refs/heads/* ]]; then @@ -226,14 +226,12 @@ jobs: elif [[ ${{ github.event_name }} == 'schedule' ]]; then echo "trigger: a nightly build" echo "CI_TEST='false'" >> $GITHUB_ENV - echo "NIGHTLY='true'" >> $GITHUB_ENV + echo "NIGHTLY='--nightly'" >> $GITHUB_ENV echo "CONDA_USERNAME=$AIHABITAT_NIGHTLY_CONDA_USERNAME" >> $GITHUB_ENV echo "CONDA_PWD=$AIHABITAT_NIGHTLY_CONDA_PWD" >> $GITHUB_ENV elif [[ ${{ github.event_name }} == 'pull_request' ]]; then echo "trigger: a pull request" fi - #- name: Debugging with tmate - # uses: mxschmitt/action-tmate@v3.18 - name: Build conda Linux packages run: |- cd habitat-sim/conda-build @@ -252,7 +250,8 @@ jobs: export PATH=$HOME/anaconda/bin:$PATH # Delete old nightly builds - #python common/delete_old_night_packages.py --username $CONDA_USERNAME --password $CONDA_PWD + #NOTE: The ${{ env.NIGHTLY }} == "--nightly" variable is necessary to trigger this + #python common/delete_old_night_packages.py --username ${{ env.CONDA_USERNAME }} --password ${{ env.CONDA_PWD }} ${{ env.NIGHTLY }} fi #install Docker @@ -268,5 +267,76 @@ jobs: if [[ ${{ env.CI_TEST }} ]]; then docker run --ipc=host --rm -v $(pwd)/../../:/remote hsim_condabuild_dcontainer /bin/bash -c "source ~/.bashrc && conda activate py39 && cd /remote/habitat-sim/ && git config --global --add safe.directory '*' && cd /remote/habitat-sim/conda-build && python matrix_builder.py --ci_test" else - echo "TODO: the deployment command" + docker run --ipc=host --rm -v $(pwd)/../../:/remote --env ${{ env.CONDA_PWD }} hsim_condabuild_dcontainer /bin/bash -c "source ~/.bashrc && conda activate py39 && cd /remote/habitat-sim/ && git config --global --add safe.directory '*' && cd /remote/habitat-sim/conda-build && yes | anaconda login --username ${{ env.CONDA_USERNAME }} --password ${{ env.CONDA_PWD }} --hostname "aihabitat-conda-ci-builder-linux" && python matrix_builder.py --conda_upload ${{ env.NIGHTLY }}" + fi + + build_conda_osx: + runs-on: macos-14 + env: + AIHABITAT_CONDA_USERNAME: aihabitat + AIHABITAT_CONDA_PWD: ${{ secrets.AIHABITAT_CONDA_PWD }} + AIHABITAT_NIGHTLY_CONDA_USERNAME: aihabitat-nightly + AIHABITAT_NIGHTLY_CONDA_PWD: ${{ secrets.AIHABITAT_NIGHTLY_CONDA_PWD }} + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@v4.1.1 + with: + path: "./habitat-sim" + - name: Conda Install OSX + run: |- + curl -o ~/miniconda.sh -O https://repo.anaconda.com/miniconda/Miniconda3-py310_24.1.2-0-MacOSX-arm64.sh + chmod +x ~/miniconda.sh + ~/miniconda.sh -b -p $HOME/miniconda + rm ~/miniconda.sh + export PATH=$HOME/miniconda/bin:$PATH + conda init "$(basename "${SHELL}")" + - name: Set CI_TEST variable based on "main merge" vs. "pull_request" workflow + run: |- + echo "NIGHTLY=''" >> $GITHUB_ENV + echo "CI_TEST='true'" >> $GITHUB_ENV + if [[ ${{ github.event_name }} == 'push' ]]; then + if [[ ${GITHUB_REF} == refs/heads/* ]]; then + echo "trigger: a push to main branch" + elif [[ ${GITHUB_REF} == refs/tags/* ]]; then + echo "trigger: a version tag push" + echo "CI_TEST='false'" >> $GITHUB_ENV + echo "CONDA_USERNAME=$AIHABITAT_CONDA_USERNAME" >> $GITHUB_ENV + echo "CONDA_PWD=$AIHABITAT_CONDA_PWD" >> $GITHUB_ENV + fi + elif [[ ${{ github.event_name }} == 'schedule' ]]; then + echo "trigger: a nightly build" + echo "CI_TEST='false'" >> $GITHUB_ENV + echo "NIGHTLY='--nightly'" >> $GITHUB_ENV + echo "CONDA_USERNAME=$AIHABITAT_NIGHTLY_CONDA_USERNAME" >> $GITHUB_ENV + echo "CONDA_PWD=$AIHABITAT_NIGHTLY_CONDA_PWD" >> $GITHUB_ENV + elif [[ ${{ github.event_name }} == 'pull_request' ]]; then + echo "trigger: a pull request" + fi + - name: Build conda OSX packages + run: |- + export PATH=$HOME/miniconda/bin:$PATH + source ~/.bash_profile + conda activate + conda install -y anaconda-client ccache cmake git ninja conda-build pip + echo $(which -a python) + pip install gitpython + cd habitat-sim + git submodule update --init --recursive --jobs 8 + pip install . + pip uninstall -y habitat-sim + rm -rf build + (yes || true) | anaconda login --username ${{ env.CONDA_USERNAME }} --password ${{ env.CONDA_PWD }} --hostname "aihabitat-conda-ci-builder-macos" + #pivot on the build type + if [[ ${{ env.CI_TEST }} ]]; then + echo "This is a CI test." + cd conda-build + export PYTHONIOENCODING="utf-8" + python matrix_builder.py --ci_test + else + conda config --set anaconda_upload yes + cd conda-build + export PYTHONIOENCODING="utf-8" + python matrix_builder.py ${{ env.NIGHTLY }} fi