Merge pull request #8191 from madhavajay/madhava/debug_k8s

force torch cpu
OpenMined · Oct 26, 2023 · a8d7228 · a8d7228
2 parents cd9e5ef + f0d4a6f
commit a8d7228
Show file tree

Hide file tree

Showing 8 changed files with 60 additions and 3 deletions.
diff --git a/.github/workflows/container-scan.yml b/.github/workflows/container-scan.yml
@@ -21,9 +21,13 @@ jobs:
 
       # free 10GB of space
       - name: Remove unnecessary files
+        if: matrix.os == 'ubuntu-latest'
         run: |
           sudo rm -rf /usr/share/dotnet
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          docker image prune --all --force
+          docker builder prune --all --force
+          docker system prune --all --force
 
       # Build the docker image for testing
       - name: Build a Docker image
@@ -64,9 +68,13 @@ jobs:
 
       # free 10GB of space
       - name: Remove unnecessary files
+        if: matrix.os == 'ubuntu-latest'
         run: |
           sudo rm -rf /usr/share/dotnet
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          docker image prune --all --force
+          docker builder prune --all --force
+          docker system prune --all --force
 
       # Build the docker image for testing
       - name: Build a Docker image

diff --git a/.github/workflows/pr-tests-enclave.yml b/.github/workflows/pr-tests-enclave.yml
@@ -37,6 +37,9 @@ jobs:
         run: |
           sudo rm -rf /usr/share/dotnet
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          docker image prune --all --force
+          docker builder prune --all --force
+          docker system prune --all --force
 
       - name: Check for file changes
         uses: dorny/paths-filter@v2

diff --git a/.github/workflows/pr-tests-frontend.yml b/.github/workflows/pr-tests-frontend.yml
@@ -114,6 +114,9 @@ jobs:
         run: |
           sudo rm -rf /usr/share/dotnet
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          docker image prune --all --force
+          docker builder prune --all --force
+          docker system prune --all --force
 
       - name: Check for file changes
         uses: dorny/paths-filter@v2

diff --git a/.github/workflows/pr-tests-stack-arm64.yml b/.github/workflows/pr-tests-stack-arm64.yml
@@ -36,6 +36,9 @@ jobs:
         run: |
           sudo rm -rf /usr/share/dotnet
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          docker image prune --all --force
+          docker builder prune --all --force
+          docker system prune --all --force
 
       - name: Check for file changes
         uses: dorny/paths-filter@v2

diff --git a/.github/workflows/pr-tests-stack.yml b/.github/workflows/pr-tests-stack.yml
@@ -56,6 +56,9 @@ jobs:
         run: |
           sudo rm -rf /usr/share/dotnet
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          docker image prune --all --force
+          docker builder prune --all --force
+          docker system prune --all --force
 
       - name: Check for file changes
         uses: dorny/paths-filter@v2
@@ -247,7 +250,7 @@ jobs:
       max-parallel: 99
       matrix:
         # os: [ubuntu-latest, macos-latest, windows-latest, windows]
-        os: [om-ci-16vcpu-ubuntu2204]
+        os: [ubuntu-latest]
         python-version: ["3.11"]
         notebook-paths: ["api/0.8"]
       fail-fast: false
@@ -268,6 +271,16 @@ jobs:
 
       - uses: actions/checkout@v3
 
+      # free 10GB of space
+      - name: Remove unnecessary files
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          docker image prune --all --force
+          docker builder prune --all --force
+          docker system prune --all --force
+
       - name: Check for file changes
         uses: dorny/paths-filter@v2
         id: changes
@@ -459,7 +472,8 @@ jobs:
       max-parallel: 99
       matrix:
         # os: [ubuntu-latest, macos-latest, windows-latest, windows]
-        os: [om-ci-16vcpu-ubuntu2204]
+        # os: [om-ci-16vcpu-ubuntu2204]
+        os: [ubuntu-latest]
         python-version: ["3.11"]
         pytest-modules: ["frontend network"]
       fail-fast: false
@@ -485,6 +499,16 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      # free 10GB of space
+      - name: Remove unnecessary files
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          docker image prune --all --force
+          docker builder prune --all --force
+          docker system prune --all --force
+
       - name: Get pip cache dir
         if: steps.changes.outputs.stack == 'true'
         id: pip-cache
@@ -544,6 +568,7 @@ jobs:
         env:
           HAGRID_ART: false
           PYTEST_MODULES: "${{ matrix.pytest-modules }}"
+          GITHUB_CI: true
         shell: bash
         run: |
           # install k3d

diff --git a/.github/workflows/pr-tests-syft.yml b/.github/workflows/pr-tests-syft.yml
@@ -215,6 +215,9 @@ jobs:
         run: |
           sudo rm -rf /usr/share/dotnet
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          docker image prune --all --force
+          docker builder prune --all --force
+          docker system prune --all --force
 
       - name: Check for file changes
         uses: dorny/paths-filter@v2

diff --git a/packages/grid/backend/backend.dockerfile b/packages/grid/backend/backend.dockerfile
@@ -50,6 +50,8 @@ COPY syft/src/syft/capnp /app/syft/src/syft/capnp
 
 # install syft
 RUN --mount=type=cache,target=/root/.cache \
+    # force cpu torch
+    pip install --user torch==2.1.0 -f https://download.pytorch.org/whl/cpu/torch_stable.html \
     pip install --user -e /app/syft && \
     pip uninstall ansible ansible-core -y && \
     rm -rf ~/.local/lib/python3.11/site-packages/ansible_collections

diff --git a/tox.ini b/tox.ini
@@ -28,9 +28,11 @@ skipsdist = True
 
 [testenv]
 basepython = python3
-install_command = pip install {opts} {packages}
+install_command = pip install --find-links https://download.pytorch.org/whl/cpu/torch_stable.html {opts} {packages}
 commands =
     python --version
+setenv =
+    PIP_FIND_LINKS=https://download.pytorch.org/whl/cpu/torch_stable.html
 
 # Syft
 [testenv:syft]
@@ -622,7 +624,9 @@ allowlist_externals =
 setenv =
     ORCHESTRA_DEPLOYMENT_TYPE = {env:ORCHESTRA_DEPLOYMENT_TYPE:k8s}
     NODE_PORT = {env:NODE_PORT:9082}
+    GITHUB_CI = {env:GITHUB_CI:false}
 commands =
+    bash -c "echo Running with GITHUB_CI=$GITHUB_CI; date"
     python -c 'import syft as sy; sy.stage_protocol_changes()'
     k3d version
 
@@ -693,6 +697,12 @@ commands =
     #     deploy -b -p domain; \
     #     do ((--r))||exit;echo "retrying" && sleep 20;done)'
 
+    # free up build cache after build of images
+    bash -c 'if [[ "$GITHUB_CI" != "false" ]]; then \
+        docker image prune --all --force; \
+        docker builder prune --all --force; \
+    fi'
+
     sleep 30
 
     # wait for front end