Merge pull request #9180 from madhavajay/madhava/testing

WIP Added initial scenario testing prototype
OpenMined · Aug 28, 2024 · b4a513e · b4a513e
2 parents ffb7adf + 213204b
commit b4a513e
Show file tree

Hide file tree

Showing 33 changed files with 2,393 additions and 423 deletions.
diff --git a/.github/file-filters.yml b/.github/file-filters.yml
@@ -76,3 +76,17 @@ notebooks:
   - packages/syft/**/*.ini
   - packages/syft/**/*.sh
   - packages/syft/**/*.mako
+
+notebooks_scenario:
+  - .github/workflows/pr-tests-syft.yml
+  - notebooks/scenarios/**/*.ipynb
+  - packages/syft/**/*.py
+  - packages/syft/**/*.capnp
+  - packages/syft/**/*.yml
+  - packages/syft/**/*.cfg
+  - packages/syft/**/*.dockerfile
+  - packages/syft/**/*.toml
+  - packages/syft/**/*.txt
+  - packages/syft/**/*.ini
+  - packages/syft/**/*.sh
+  - packages/syft/**/*.mako
diff --git a/.github/workflows/pr-tests-syft.yml b/.github/workflows/pr-tests-syft.yml
@@ -188,6 +188,92 @@ jobs:
           max_attempts: 3
           command: tox -e syft.test.notebook
 
+  pr-tests-syft-notebook-scenario:
+    strategy:
+      max-parallel: 99
+      matrix:
+        # Disable on windows until its flakyness is reduced.
+        # os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [ubuntu-latest, macos-latest]
+        python-version: ["3.12"]
+        deployment-type: ["python"]
+        bump-version: ["False"]
+        include:
+          - python-version: "3.11"
+            os: "ubuntu-latest"
+            deployment-type: "python"
+          - python-version: "3.10"
+            os: "ubuntu-latest"
+            deployment-type: "python"
+          - python-version: "3.12"
+            os: "ubuntu-latest"
+            deployment-type: "python"
+            bump-version: "True"
+
+    runs-on: ${{ matrix.os }}
+    steps:
+      # - name: Permission to home directory
+      #   if: matrix.os == 'ubuntu-latest'
+      #   run: |
+      #     sudo chown -R $USER:$USER $HOME
+      - name: "clean .git/config"
+        if: matrix.os == 'windows-latest'
+        continue-on-error: true
+        shell: bash
+        run: |
+          echo "deleting ${GITHUB_WORKSPACE}/.git/config"
+          rm ${GITHUB_WORKSPACE}/.git/config
+
+      - uses: actions/checkout@v4
+
+      - name: Check for file changes
+        uses: dorny/paths-filter@v3
+        id: changes
+        with:
+          base: ${{ github.ref }}
+          token: ${{ github.token }}
+          filters: .github/file-filters.yml
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        if: steps.changes.outputs.syft == 'true'  || steps.changes.outputs.notebooks_scenario == 'true'
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install pip packages
+        if: steps.changes.outputs.syft == 'true'  || steps.changes.outputs.notebooks_scenario == 'true'
+        run: |
+          python -m pip install --upgrade pip
+          pip install uv==0.2.17 tox==4.16.0 tox-uv==1.9.0
+          uv --version
+
+      - name: Get uv cache dir
+        id: pip-cache
+        if: steps.changes.outputs.syft == 'true'  || steps.changes.outputs.notebooks_scenario == 'true'
+        shell: bash
+        run: |
+          echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT
+
+      - name: Load github cache
+        uses: actions/cache@v4
+        if: steps.changes.outputs.syft == 'true'  || steps.changes.outputs.notebooks_scenario == 'true'
+        with:
+          path: ${{ steps.pip-cache.outputs.dir }}
+          key: ${{ runner.os }}-uv-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }}
+          restore-keys: |
+            ${{ runner.os }}-uv-py${{ matrix.python-version }}-
+
+      - name: Run notebook scenario tests
+        uses: nick-fields/retry@v3
+        if: steps.changes.outputs.syft == 'true'  || steps.changes.outputs.notebooks_scenario == 'true'
+        env:
+          ORCHESTRA_DEPLOYMENT_TYPE: "${{ matrix.deployment-type }}"
+          BUMP_VERSION: "${{ matrix.bump-version }}"
+        with:
+          timeout_seconds: 2400
+          max_attempts: 3
+          command: tox -e syft.test.notebook.scenario
+
   pr-tests-syft-notebook-single-container:
     strategy:
       max-parallel: 99

diff --git a/.gitignore b/.gitignore
@@ -81,4 +81,7 @@ out.*
 .git-blame-ignore-revs
 
 # migration data
-packages/grid/helm/examples/dev/migration.yaml
+packages/grid/helm/examples/dev/migration.yaml
+
+# dynaconf settings file
+**/settings.yaml
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -31,7 +31,7 @@ repos:
         exclude: ^(packages/syft/tests/mongomock)
       - id: name-tests-test
         always_run: true
-        exclude: ^(.*/tests/utils/)|^(.*fixtures.py|packages/syft/tests/mongomock)
+        exclude: ^(.*/tests/utils/)|^(.*fixtures.py|packages/syft/tests/mongomock)|^(tests/scenarios/bigquery/helpers)
       - id: requirements-txt-fixer
         always_run: true
         exclude: "packages/syft/tests/mongomock"

diff --git a/notebooks/scenarios/bigquery/01-setup-datasite.ipynb b/notebooks/scenarios/bigquery/01-setup-datasite.ipynb
@@ -6,8 +6,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# third party\n",
-    "\n",
     "# syft absolute\n",
     "import syft as sy\n",
     "from syft import test_settings"
@@ -256,6 +254,13 @@
    "source": [
     "# !docker image ls | grep bigquery"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {