Merge branch 'dev' into ionesio/fix_api_logs

OpenMined · Aug 28, 2024 · 16e422c · 16e422c
2 parents cb4876c + 395dcad
commit 16e422c
Show file tree

Hide file tree

Showing 19 changed files with 1,042 additions and 243 deletions.
diff --git a/.github/workflows/pr-tests-stack.yml b/.github/workflows/pr-tests-stack.yml
@@ -418,6 +418,146 @@ jobs:
           k3d cluster delete test-datasite-1 || true
           k3d registry delete k3d-registry.localhost || true
 
+  pr-tests-notebook-scenario-k8s:
+    strategy:
+      max-parallel: 99
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ["3.12"]
+      fail-fast: false
+
+    runs-on: ${{matrix.os}}
+
+    steps:
+      - name: Permission to home directory
+        run: |
+          sudo chown -R $USER:$USER $HOME
+      - uses: actions/checkout@v4
+      - name: Check for file changes
+        uses: dorny/paths-filter@v3
+        id: changes
+        with:
+          base: ${{ github.ref }}
+          token: ${{ github.token }}
+          filters: .github/file-filters.yml
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        if: steps.changes.outputs.stack == 'true'
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Add K3d Registry
+        run: |
+          sudo python ./scripts/patch_hosts.py --add-k3d-registry
+
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: true
+          large-packages: false
+
+      # free 10GB of space
+      - name: Remove unnecessary files
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+          docker image prune --all --force
+          docker builder prune --all --force
+          docker system prune --all --force
+
+      - name: Install pip dependencies
+        if: steps.changes.outputs.stack == 'true'
+        run: |
+          python -m pip install --upgrade pip
+          pip install uv==0.2.17 tox==4.16.0 tox-uv==1.9.0
+          uv --version
+
+      - name: Get uv cache dir
+        if: steps.changes.outputs.stack == 'true'
+        id: pip-cache
+        shell: bash
+        run: |
+          echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT
+
+      - name: Load github cache
+        uses: actions/cache@v4
+        if: steps.changes.outputs.stack == 'true'
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-uv-py${{ matrix.python-version }}
+          restore-keys: |
+            ${{ runner.os }}-uv-py${{ matrix.python-version }}
+
+      - name: Install kubectl
+        if: steps.changes.outputs.stack == 'true'
+        run: |
+          # cleanup apt version
+          sudo apt remove kubectl || true
+          # install kubectl 1.27
+          curl -LO https://dl.k8s.io/release/v1.27.2/bin/linux/amd64/kubectl
+          chmod +x kubectl
+          sudo install kubectl /usr/local/bin;
+
+      - name: Install helm
+        if: steps.changes.outputs.stack == 'true'
+        run: |
+          # install helm
+          curl  -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3
+          chmod 700 get_helm.sh
+          ./get_helm.sh
+
+      - name: Run Notebooks Tests
+        if: steps.changes.outputs.stack == 'true'
+        timeout-minutes: 60
+        env:
+          GITHUB_CI: true
+        shell: bash
+        run: |
+          K3D_VERSION=v5.6.3
+          DEVSPACE_VERSION=v6.3.12
+          # install k3d
+          wget https://github.com/k3d-io/k3d/releases/download/${K3D_VERSION}/k3d-linux-amd64
+          mv k3d-linux-amd64 k3d
+          chmod +x k3d
+          export PATH=`pwd`:$PATH
+          k3d version
+          curl -sSL https://github.com/loft-sh/devspace/releases/download/${DEVSPACE_VERSION}/devspace-linux-amd64 -o ./devspace
+          chmod +x devspace
+          devspace version
+          tox -e stack.test.notebook.scenario.k8s
+
+      - name: Get current timestamp
+        id: date
+        if: failure()
+        shell: bash
+        run: echo "date=$(date +%s)" >> $GITHUB_OUTPUT
+
+      - name: Collect logs from k3d
+        if: steps.changes.outputs.stack == 'true' && failure()
+        shell: bash
+        run: |
+          mkdir -p ./k8s-logs
+          kubectl describe all -A --context k3d-test-datasite-1 --namespace syft > ./k8s-logs/test-datasite-1-desc-${{ steps.date.outputs.date }}.txt
+          kubectl logs -l app.kubernetes.io/name!=random --prefix=true --context k3d-test-datasite-1 --namespace syft > ./k8s-logs/test-datasite-1-logs-${{ steps.date.outputs.date }}.txt
+          ls -la ./k8s-logs
+
+      - name: Upload logs to GitHub
+        uses: actions/upload-artifact@master
+        if: steps.changes.outputs.stack == 'true' && failure()
+        with:
+          name: k8s-logs-notebook-${{ matrix.os }}-${{ steps.date.outputs.date }}
+          path: ./k8s-logs/
+
+      - name: Cleanup k3d
+        if: steps.changes.outputs.stack == 'true' && failure()
+        shell: bash
+        run: |
+          export PATH=`pwd`:$PATH
+          k3d cluster delete test-datasite-1 || true
+          k3d registry delete k3d-registry.localhost || true
+
   pr-tests-migrations:
     strategy:
       max-parallel: 99

diff --git a/notebooks/api/0.8/05-custom-policy.ipynb b/notebooks/api/0.8/05-custom-policy.ipynb
@@ -258,18 +258,17 @@
     "    def __init__(self, *args: Any, **kwargs: Any) -> None:\n",
     "        pass\n",
     "\n",
-    "    def filter_kwargs(self, kwargs, context, code_item_id):  # stdlib\n",
+    "    def filter_kwargs(self, kwargs, context):  # stdlib\n",
     "        allowed_inputs = self.allowed_ids_only(\n",
     "            allowed_inputs=self.inputs, kwargs=kwargs, context=context\n",
     "        )\n",
     "        results = self.retrieve_from_db(\n",
-    "            code_item_id=code_item_id,\n",
     "            allowed_inputs=allowed_inputs,\n",
     "            context=context,\n",
     "        )\n",
     "        return results\n",
     "\n",
-    "    def retrieve_from_db(self, code_item_id, allowed_inputs, context):\n",
+    "    def retrieve_from_db(self, allowed_inputs, context):\n",
     "        # syft absolute\n",
     "        from syft import ServerType\n",
     "        from syft.service.action.action_object import TwinMode\n",
@@ -346,7 +345,6 @@
     "        filtered_input_kwargs = self.filter_kwargs(\n",
     "            kwargs=usr_input_kwargs,\n",
     "            context=context,\n",
-    "            code_item_id=code_item_id,\n",
     "        )\n",
     "\n",
     "        expected_input_kwargs = set()\n",