Merge branch 'dev' into test_upgrades

OpenMined · Sep 18, 2024 · ca05244 · ca05244
2 parents 2a0b76d + 32896af
commit ca05244
Show file tree

Hide file tree

Showing 240 changed files with 4,896 additions and 15,932 deletions.
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.9.2-beta.2
+current_version = 0.9.2-beta.3
 tag = False
 tag_name = {new_version}
 commit = True

diff --git a/.github/workflows/container-scan.yml b/.github/workflows/container-scan.yml
@@ -224,7 +224,7 @@ jobs:
           name: syft.sbom.json
           path: syft.sbom.json
 
-  scan-mongo-latest-trivy:
+  scan-postgres-latest-trivy:
     permissions:
       contents: read # for actions/checkout to fetch code
       security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
@@ -238,24 +238,24 @@ jobs:
         continue-on-error: true
         uses: aquasecurity/trivy-action@master
         with:
-          image-ref: "mongo:7.0.0"
+          image-ref: "postgres:16.1"
           format: "cyclonedx"
-          output: "mongo-trivy-results.sbom.json"
+          output: "postgres-trivy-results.sbom.json"
           timeout: "10m0s"
 
         #Upload SBOM to GitHub Artifacts
       - name: Upload SBOM to GitHub Artifacts
         uses: actions/upload-artifact@v4
         with:
-          name: mongo-trivy-results.sbom.json
-          path: mongo-trivy-results.sbom.json
+          name: postgres-trivy-results.sbom.json
+          path: postgres-trivy-results.sbom.json
 
           #Generate sarif file
       - name: Run Trivy vulnerability scanner
         continue-on-error: true
         uses: aquasecurity/trivy-action@master
         with:
-          image-ref: "mongo:7.0.0"
+          image-ref: "postgres:16.1"
           format: "sarif"
           output: "trivy-results.sarif"
           timeout: "10m0s"
@@ -266,7 +266,7 @@ jobs:
         with:
           sarif_file: "trivy-results.sarif"
 
-  scan-mongo-latest-snyk:
+  scan-postgres-latest-snyk:
     permissions:
       contents: read # for actions/checkout to fetch code
       security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
@@ -281,7 +281,7 @@ jobs:
           # This is where you will need to introduce the Snyk API token created with your Snyk account
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
         with:
-          image: mongo:7.0.0
+          image: postgres:16.1
           args: --sarif-file-output=snyk-code.sarif
 
         # Replace any "undefined" security severity values with 0. The undefined value is used in the case

diff --git a/.isort.cfg b/.isort.cfg
@@ -20,4 +20,4 @@ import_heading_localfolder=relative
 ignore_comments=False
 force_grid_wrap=True
 honor_noqa=True
-skip_glob=packages/syft/src/syft/__init__.py,packages/grid/data/*,packages/syft/tests/mongomock/*
+skip_glob=packages/syft/src/syft/__init__.py,packages/grid/data/*
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -3,41 +3,36 @@ repos:
     rev: v4.5.0
     hooks:
       - id: check-ast
-        exclude: ^(packages/syft/tests/mongomock)
         always_run: true
       - id: trailing-whitespace
         always_run: true
-        exclude: ^(docs/|.+\.md|.bumpversion.cfg|packages/syft/tests/mongomock)
+        exclude: ^(docs/|.+\.md|.bumpversion.cfg)
       - id: check-docstring-first
         always_run: true
-        exclude: ^(packages/syft/tests/mongomock)
       - id: check-json
         always_run: true
-        exclude: ^(packages/grid/frontend/|packages/syft/tests/mongomock|.vscode)
+        exclude: ^(packages/grid/frontend/|.vscode)
       - id: check-added-large-files
         always_run: true
         exclude: ^(packages/grid/backend/wheels/.*|docs/img/header.png|docs/img/terminalizer.gif)
       - id: check-yaml
         always_run: true
-        exclude: ^(packages/grid/k8s/rendered/|packages/grid/helm/|packages/syft/tests/mongomock)
+        exclude: ^(packages/grid/k8s/rendered/|packages/grid/helm/)
       - id: check-merge-conflict
         always_run: true
         args: ["--assume-in-merge"]
       - id: check-executables-have-shebangs
         always_run: true
-        exclude: ^(packages/syft/tests/mongomock)
       - id: debug-statements
         always_run: true
-        exclude: ^(packages/syft/tests/mongomock)
       - id: name-tests-test
         always_run: true
-        exclude: ^(.*/tests/utils/)|^(.*fixtures.py|packages/syft/tests/mongomock)|^(tests/scenarios/bigquery/helpers)
+        exclude: ^(.*/tests/utils/)|^(.*fixtures.py)|^(tests/scenarios/bigquery/helpers)
       - id: requirements-txt-fixer
         always_run: true
-        exclude: "packages/syft/tests/mongomock"
       - id: mixed-line-ending
         args: ["--fix=lf"]
-        exclude: '\.bat|\.csv|\.ps1$|packages/syft/tests/mongomock'
+        exclude: '\.bat|\.csv|\.ps1$'
 
   - repo: https://github.com/MarcoGorelli/absolufy-imports # This repository has been archived by the owner on Aug 15, 2023. It is now read-only.
     rev: v0.3.1
@@ -88,7 +83,6 @@ repos:
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix, --show-fixes]
-        exclude: packages/syft/tests/mongomock
         types_or: [python, pyi, jupyter]
       - id: ruff-format
         types_or: [python, pyi, jupyter]
@@ -178,7 +172,7 @@ repos:
     rev: "v3.0.0-alpha.9-for-vscode"
     hooks:
       - id: prettier
-        exclude: ^(packages/grid/helm|packages/grid/frontend/pnpm-lock.yaml|packages/syft/tests/mongomock|.vscode)
+        exclude: ^(packages/grid/helm|packages/grid/frontend/pnpm-lock.yaml|.vscode)
 
   # - repo: meta
   #   hooks:

diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -4,6 +4,13 @@
   // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
   "version": "0.2.0",
   "configurations": [
+    {
+      "name": "Python Debugger: Current File",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${file}",
+      "console": "integratedTerminal"
+    },
     {
       "name": "Syft Debugger",
       "type": "debugpy",

diff --git a/VERSION b/VERSION
@@ -1,5 +1,5 @@
 # Mono Repo Global Version
-__version__ = "0.9.2-beta.2"
+__version__ = "0.9.2-beta.3"
 # elsewhere we can call this file: `python VERSION` and simply take the stdout
 
 # stdlib

diff --git a/docs/source/api_reference/syft.store.mongo_client.rst b/docs/source/api_reference/syft.store.mongo_client.rst
diff --git a/docs/source/api_reference/syft.store.mongo_codecs.rst b/docs/source/api_reference/syft.store.mongo_codecs.rst
diff --git a/docs/source/api_reference/syft.store.mongo_document_store.rst b/docs/source/api_reference/syft.store.mongo_document_store.rst
diff --git a/docs/source/api_reference/syft.store.rst b/docs/source/api_reference/syft.store.rst
@@ -32,8 +32,5 @@
    syft.store.kv_document_store
    syft.store.linked_obj
    syft.store.locks
-   syft.store.mongo_client
-   syft.store.mongo_codecs
-   syft.store.mongo_document_store
    syft.store.sqlite_document_store
 
diff --git a/notebooks/api/0.8/12-custom-api-endpoint.ipynb b/notebooks/api/0.8/12-custom-api-endpoint.ipynb
@@ -657,6 +657,9 @@
     "# stdlib\n",
     "import time\n",
     "\n",
+    "# syft absolute\n",
+    "from syft.service.job.job_stash import JobStatus\n",
+    "\n",
     "# Iterate over the Jobs waiting them to finish their pipelines.\n",
     "job_pool = [\n",
     "    (log_call_job, \"Logging Private Function Call\"),\n",
@@ -665,13 +668,19 @@
     "]\n",
     "for job, expected_log in job_pool:\n",
     "    updated_job = datasite_client.api.services.job.get(job.id)\n",
-    "    while updated_job.status.value != \"completed\":\n",
+    "    while updated_job.status in {JobStatus.CREATED, JobStatus.PROCESSING}:\n",
     "        updated_job = datasite_client.api.services.job.get(job.id)\n",
     "        time.sleep(1)\n",
-    "    # If they're completed. Then, check if the TwinAPI print appears in the job logs.\n",
-    "    assert expected_log in datasite_client.api.services.job.get(job.id).logs(\n",
-    "        _print=False\n",
-    "    )"
+    "\n",
+    "    assert (\n",
+    "        updated_job.status == JobStatus.COMPLETED\n",
+    "    ), f\"Job {updated_job.id} exited with status {updated_job.status} and result {updated_job.result}\"\n",
+    "    if updated_job.status == JobStatus.COMPLETED:\n",
+    "        print(f\"Job {updated_job.id} completed\")\n",
+    "        # If they're completed. Then, check if the TwinAPI print appears in the job logs.\n",
+    "        assert expected_log in datasite_client.api.services.job.get(job.id).logs(\n",
+    "            _print=False\n",
+    "        )"
    ]
   },
   {
@@ -683,6 +692,11 @@
   }
  ],
  "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
@@ -693,7 +707,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.5"
+   "version": "3.10.13"
   }
  },
  "nbformat": 4,

diff --git a/notebooks/scenarios/bigquery/000-start-and-configure-server-and-admins.ipynb b/notebooks/scenarios/bigquery/000-start-and-configure-server-and-admins.ipynb
@@ -26,7 +26,7 @@
     "\n",
     "# syft absolute\n",
     "import syft as sy\n",
-    "from syft import get_helpers  # noqa: F401\n",
+    "from syft import test_helpers  # noqa: F401\n",
     "\n",
     "# third party\n",
     "from email_helpers import get_email_server\n",

diff --git a/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb b/notebooks/scenarios/bigquery/001-scale-delete-worker-pools.ipynb
@@ -26,7 +26,7 @@
     "\n",
     "# syft absolute\n",
     "import syft as sy\n",
-    "from syft import get_helpers  # noqa: F401\n",
+    "from syft import test_helpers  # noqa: F401\n",
     "\n",
     "# third party\n",
     "from email_helpers import Timeout\n",

diff --git a/notebooks/scenarios/bigquery/010-setup-bigquery-pool.ipynb b/notebooks/scenarios/bigquery/010-setup-bigquery-pool.ipynb
@@ -24,7 +24,7 @@
     "\n",
     "# syft absolute\n",
     "import syft as sy\n",
-    "from syft import get_helpers  # noqa: F401\n",
+    "from syft import test_helpers  # noqa: F401\n",
     "from syft import test_settings\n",
     "\n",
     "# third party\n",

diff --git a/notebooks/scenarios/bigquery/011-users-emails-passwords.ipynb b/notebooks/scenarios/bigquery/011-users-emails-passwords.ipynb
@@ -28,7 +28,7 @@
     "\n",
     "# syft absolute\n",
     "import syft as sy\n",
-    "from syft import get_helpers  # noqa: F401\n",
+    "from syft import test_helpers  # noqa: F401\n",
     "\n",
     "# third party\n",
     "from email_helpers import SENDER\n",

diff --git a/notebooks/scenarios/bigquery/020-configure-api.ipynb b/notebooks/scenarios/bigquery/020-configure-api.ipynb
@@ -33,7 +33,7 @@
     "\n",
     "# syft absolute\n",
     "import syft as sy\n",
-    "from syft import get_helpers  # noqa: F401\n",
+    "from syft import test_helpers  # noqa: F401\n",
     "from syft import test_settings\n",
     "\n",
     "# third party\n",

diff --git a/notebooks/scenarios/bigquery/021-create-jobs.ipynb b/notebooks/scenarios/bigquery/021-create-jobs.ipynb
@@ -40,7 +40,7 @@
     "\n",
     "# syft absolute\n",
     "import syft as sy\n",
-    "from syft import get_helpers  # noqa: F401\n",
+    "from syft import test_helpers  # noqa: F401\n",
     "from syft.service.job.job_stash import JobStatus\n",
     "\n",
     "# third party\n",

diff --git a/notebooks/scenarios/bigquery/040-do-review-requests.ipynb b/notebooks/scenarios/bigquery/040-do-review-requests.ipynb
@@ -24,7 +24,7 @@
     "\n",
     "# syft absolute\n",
     "import syft as sy\n",
-    "from syft import get_helpers  # noqa: F401\n",
+    "from syft import test_helpers  # noqa: F401\n",
     "from syft.service.job.job_stash import Job\n",
     "\n",
     "# third party\n",