From d6219152ef79fa329028654cc07bbd0d3b93361d Mon Sep 17 00:00:00 2001 From: Simon Alibert Date: Thu, 25 Apr 2024 12:33:59 +0200 Subject: [PATCH] Cleanup --- ...ker_images.yml => build-docker-images.yml} | 0 .github/workflows/test-pip.yml | 59 --------- .github/workflows/test-poetry.yml | 120 +++++++++++++++++ .github/workflows/test.yml | 122 +++++------------- docker/lerobot-gpu/Dockerfile_OLD | 40 ------ 5 files changed, 150 insertions(+), 191 deletions(-) rename .github/workflows/{build_docker_images.yml => build-docker-images.yml} (100%) delete mode 100644 .github/workflows/test-pip.yml create mode 100644 .github/workflows/test-poetry.yml delete mode 100644 docker/lerobot-gpu/Dockerfile_OLD diff --git a/.github/workflows/build_docker_images.yml b/.github/workflows/build-docker-images.yml similarity index 100% rename from .github/workflows/build_docker_images.yml rename to .github/workflows/build-docker-images.yml diff --git a/.github/workflows/test-pip.yml b/.github/workflows/test-pip.yml deleted file mode 100644 index 2d0c28526d..0000000000 --- a/.github/workflows/test-pip.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Tests - -on: - pull_request: - branches: - - main - push: - branches: - - main - -jobs: - tests: - runs-on: ubuntu-latest - env: - DATA_DIR: tests/data - MUJOCO_GL: egl - steps: - - uses: actions/checkout@v4 - with: - lfs: true - - name: Checkout gym-aloha - uses: actions/checkout@v4 - with: - repository: huggingface/gym-aloha - path: envs/gym-aloha - ssh-key: ${{ secrets.SSH_PRIVATE_KEY }} - - name: Checkout gym-xarm - uses: actions/checkout@v4 - with: - repository: huggingface/gym-xarm - path: envs/gym-xarm - ssh-key: ${{ secrets.SSH_PRIVATE_KEY }} - - name: Checkout gym-pusht - uses: actions/checkout@v4 - with: - repository: huggingface/gym-pusht - path: envs/gym-pusht - ssh-key: ${{ secrets.SSH_PRIVATE_KEY }} - - - name: Set up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: "3.10" - cache: ${{ (github.ref == 'refs/heads/main') && 'pip' || '' }} - cache-dependency-path: pyproject.toml - - - name: Install EGL - run: sudo apt-get update && sudo apt-get install -y libegl1-mesa-dev - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -e ".[test, aloha, xarm, pusht]" - - - name: Test with pytest - run: pytest -v --cov=./lerobot tests - - - name: Test end-to-end - run: make test-ete diff --git a/.github/workflows/test-poetry.yml b/.github/workflows/test-poetry.yml new file mode 100644 index 0000000000..5fe4d703ae --- /dev/null +++ b/.github/workflows/test-poetry.yml @@ -0,0 +1,120 @@ +# name: Tests poetry + +# on: +# pull_request: +# branches: +# - main +# push: +# branches: +# - main + +# jobs: +# tests: +# runs-on: ubuntu-latest +# env: +# POETRY_VERSION: 1.8.2 +# DATA_DIR: tests/data +# MUJOCO_GL: egl +# steps: +# #---------------------------------------------- +# # check-out repo and set-up python +# #---------------------------------------------- +# - name: Check out repository +# uses: actions/checkout@v4 +# with: +# lfs: true + +# - name: Set up python +# id: setup-python +# uses: actions/setup-python@v5 +# with: +# python-version: '3.10' + +# - name: Add SSH key for installing envs +# uses: webfactory/ssh-agent@v0.9.0 +# with: +# ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} + +# #---------------------------------------------- +# # install & configure poetry +# #---------------------------------------------- +# - name: Load cached Poetry installation +# id: restore-poetry-cache +# uses: actions/cache/restore@v3 +# with: +# path: ~/.local +# key: poetry-${{ env.POETRY_VERSION }} + +# - name: Install Poetry +# if: steps.restore-poetry-cache.outputs.cache-hit != 'true' +# uses: snok/install-poetry@v1 +# with: +# version: ${{ env.POETRY_VERSION }} +# virtualenvs-create: true +# installer-parallel: true + +# - name: Save cached Poetry installation +# if: | +# steps.restore-poetry-cache.outputs.cache-hit != 'true' && +# github.ref_name == 'main' +# id: save-poetry-cache +# uses: actions/cache/save@v3 +# with: +# path: ~/.local +# key: poetry-${{ env.POETRY_VERSION }} + +# - name: Configure Poetry +# run: poetry config virtualenvs.in-project true + +# #---------------------------------------------- +# # install dependencies +# #---------------------------------------------- +# # TODO(aliberts): move to gpu runners +# - name: Select cpu dependencies # HACK +# run: cp -t . .github/poetry/cpu/pyproject.toml .github/poetry/cpu/poetry.lock + +# - name: Load cached venv +# id: restore-dependencies-cache +# uses: actions/cache/restore@v3 +# with: +# path: .venv +# key: venv-${{ steps.setup-python.outputs.python-version }}-${{ env.POETRY_VERSION }}-${{ hashFiles('**/poetry.lock') }} + +# - name: Install dependencies +# if: steps.restore-dependencies-cache.outputs.cache-hit != 'true' +# run: poetry install --no-interaction --no-root --all-extras + +# - name: Save cached venv +# if: | +# steps.restore-dependencies-cache.outputs.cache-hit != 'true' && +# github.ref_name == 'main' +# id: save-dependencies-cache +# uses: actions/cache/save@v3 +# with: +# path: .venv +# key: venv-${{ steps.setup-python.outputs.python-version }}-${{ env.POETRY_VERSION }}-${{ hashFiles('**/poetry.lock') }} + +# - name: Install EGL +# run: sudo apt-get update && sudo apt-get install -y libegl1-mesa-dev + +# #---------------------------------------------- +# # install project +# #---------------------------------------------- +# - name: Install project +# run: poetry install --no-interaction --all-extras + +# #---------------------------------------------- +# # run tests & coverage +# #---------------------------------------------- +# - name: Run tests +# run: | +# source .venv/bin/activate +# pytest -v --cov=./lerobot tests + +# #---------------------------------------------- +# # run end-to-end tests +# #---------------------------------------------- +# - name: Tests end-to-end +# run: | +# source .venv/bin/activate +# make test-ete diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 10346ab4d8..854db75f20 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,10 +1,9 @@ -name: Tests poetry +name: Test on: pull_request: branches: - main - types: [opened, synchronize, reopened, labeled] push: branches: - main @@ -13,109 +12,48 @@ jobs: tests: runs-on: ubuntu-latest env: - POETRY_VERSION: 1.8.2 DATA_DIR: tests/data MUJOCO_GL: egl steps: - #---------------------------------------------- - # check-out repo and set-up python - #---------------------------------------------- - - name: Check out repository - uses: actions/checkout@v4 + - uses: actions/checkout@v4 with: lfs: true - - - name: Set up python - id: setup-python - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - - name: Add SSH key for installing envs - uses: webfactory/ssh-agent@v0.9.0 - with: - ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} - - #---------------------------------------------- - # install & configure poetry - #---------------------------------------------- - - name: Load cached Poetry installation - id: restore-poetry-cache - uses: actions/cache/restore@v3 - with: - path: ~/.local - key: poetry-${{ env.POETRY_VERSION }} - - - name: Install Poetry - if: steps.restore-poetry-cache.outputs.cache-hit != 'true' - uses: snok/install-poetry@v1 + - name: Checkout gym-aloha + uses: actions/checkout@v4 with: - version: ${{ env.POETRY_VERSION }} - virtualenvs-create: true - installer-parallel: true - - - name: Save cached Poetry installation - if: | - steps.restore-poetry-cache.outputs.cache-hit != 'true' && - github.ref_name == 'main' - id: save-poetry-cache - uses: actions/cache/save@v3 + repository: huggingface/gym-aloha + path: envs/gym-aloha + ssh-key: ${{ secrets.SSH_PRIVATE_KEY }} + - name: Checkout gym-xarm + uses: actions/checkout@v4 with: - path: ~/.local - key: poetry-${{ env.POETRY_VERSION }} - - - name: Configure Poetry - run: poetry config virtualenvs.in-project true - - #---------------------------------------------- - # install dependencies - #---------------------------------------------- - # TODO(aliberts): move to gpu runners - - name: Select cpu dependencies # HACK - run: cp -t . .github/poetry/cpu/pyproject.toml .github/poetry/cpu/poetry.lock - - - name: Load cached venv - id: restore-dependencies-cache - uses: actions/cache/restore@v3 + repository: huggingface/gym-xarm + path: envs/gym-xarm + ssh-key: ${{ secrets.SSH_PRIVATE_KEY }} + - name: Checkout gym-pusht + uses: actions/checkout@v4 with: - path: .venv - key: venv-${{ steps.setup-python.outputs.python-version }}-${{ env.POETRY_VERSION }}-${{ hashFiles('**/poetry.lock') }} - - - name: Install dependencies - if: steps.restore-dependencies-cache.outputs.cache-hit != 'true' - run: poetry install --no-interaction --no-root --all-extras + repository: huggingface/gym-pusht + path: envs/gym-pusht + ssh-key: ${{ secrets.SSH_PRIVATE_KEY }} - - name: Save cached venv - if: | - steps.restore-dependencies-cache.outputs.cache-hit != 'true' && - github.ref_name == 'main' - id: save-dependencies-cache - uses: actions/cache/save@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v5 with: - path: .venv - key: venv-${{ steps.setup-python.outputs.python-version }}-${{ env.POETRY_VERSION }}-${{ hashFiles('**/poetry.lock') }} + python-version: "3.10" + cache: ${{ (github.ref == 'refs/heads/main') && 'pip' || '' }} + cache-dependency-path: pyproject.toml - name: Install EGL run: sudo apt-get update && sudo apt-get install -y libegl1-mesa-dev - #---------------------------------------------- - # install project - #---------------------------------------------- - - name: Install project - run: poetry install --no-interaction --all-extras - - #---------------------------------------------- - # run tests & coverage - #---------------------------------------------- - - name: Run tests + - name: Install dependencies run: | - source .venv/bin/activate - pytest -v --cov=./lerobot tests + python -m pip install --upgrade pip + pip install -e ".[test, aloha, xarm, pusht]" - #---------------------------------------------- - # run end-to-end tests - #---------------------------------------------- - - name: Tests end-to-end - run: | - source .venv/bin/activate - make test-ete + - name: Test with pytest + run: pytest -v --cov=./lerobot tests + + - name: Test end-to-end + run: make test-ete diff --git a/docker/lerobot-gpu/Dockerfile_OLD b/docker/lerobot-gpu/Dockerfile_OLD deleted file mode 100644 index 985382cade..0000000000 --- a/docker/lerobot-gpu/Dockerfile_OLD +++ /dev/null @@ -1,40 +0,0 @@ -# Builds GPU Docker image of PyTorch -# Stage 1 -FROM continuumio/miniconda3:latest AS venv-image -ENV PYTHON_VERSION=3.10 - -# Setup virtual environment -RUN conda create -y -n lerobot python=${PYTHON_VERSION} -ENV PATH=/opt/conda/envs/lerobot/bin:$PATH - -# Stage 2 -# FROM nvidia/cudagl:11.3.0-devel-ubuntu20.04 AS build-image -# FROM nvidia/opengl:1.0-glvnd-devel-ubuntu22.04 AS build-image -FROM nvidia/cuda:12.4.1-base-ubuntu22.04 AS build-image -ARG DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - cmake \ - pkg-config \ - libhdf5-dev \ - libegl1-mesa-dev \ - libglu1-mesa-dev libosmesa6-dev xvfb patchelf ffmpeg swig \ - && apt-get clean && rm -rf /var/lib/apt/lists/* - - -COPY --from=venv-image /opt/conda/envs/lerobot /opt/conda/envs/lerobot -COPY --from=venv-image /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh - -RUN chsh -s /bin/bash -SHELL ["/bin/bash", "-c"] -ENV PATH=/opt/conda/envs/lerobot/bin:$PATH - -COPY . /lerobot -WORKDIR /lerobot -RUN pip install --upgrade --no-cache-dir pip -RUN pip install --no-cache-dir ".[test, aloha, xarm, pusht]" - -# Make sure we use the virtualenv -RUN echo ". /etc/profile.d/conda.sh" >> ~/.profile -RUN echo "source activate lerobot" >> ~/.profile -CMD ["/lerobot/docker/bin/docker_entrypoint"]