Refactor DB as document store, add Postgres, remove mongo

.github/workflows/container-scan.yml

@@ -224,7 +224,7 @@ jobs:
           name: syft.sbom.json
           path: syft.sbom.json
 
-  scan-mongo-latest-trivy:
+  scan-postgres-latest-trivy:
     permissions:
       contents: read # for actions/checkout to fetch code
       security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
@@ -238,24 +238,24 @@ jobs:
         continue-on-error: true
         uses: aquasecurity/trivy-action@master
         with:
-          image-ref: "mongo:7.0.0"
+          image-ref: "postgres:16.1"
           format: "cyclonedx"
-          output: "mongo-trivy-results.sbom.json"
+          output: "postgres-trivy-results.sbom.json"
           timeout: "10m0s"
 
         #Upload SBOM to GitHub Artifacts
       - name: Upload SBOM to GitHub Artifacts
         uses: actions/upload-artifact@v4
         with:
-          name: mongo-trivy-results.sbom.json
-          path: mongo-trivy-results.sbom.json
+          name: postgres-trivy-results.sbom.json
+          path: postgres-trivy-results.sbom.json
 
           #Generate sarif file
       - name: Run Trivy vulnerability scanner
         continue-on-error: true
         uses: aquasecurity/trivy-action@master
         with:
-          image-ref: "mongo:7.0.0"
+          image-ref: "postgres:16.1"
           format: "sarif"
           output: "trivy-results.sarif"
           timeout: "10m0s"
@@ -266,7 +266,7 @@ jobs:
         with:
           sarif_file: "trivy-results.sarif"
 
-  scan-mongo-latest-snyk:
+  scan-postgres-latest-snyk:
     permissions:
       contents: read # for actions/checkout to fetch code
       security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
@@ -281,7 +281,7 @@ jobs:
           # This is where you will need to introduce the Snyk API token created with your Snyk account
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
         with:
-          image: mongo:7.0.0
+          image: postgres:16.1
           args: --sarif-file-output=snyk-code.sarif
 
         # Replace any "undefined" security severity values with 0. The undefined value is used in the case

.isort.cfg

@@ -20,4 +20,4 @@ import_heading_localfolder=relative
 ignore_comments=False
 force_grid_wrap=True
 honor_noqa=True
-skip_glob=packages/syft/src/syft/__init__.py,packages/grid/data/*,packages/syft/tests/mongomock/*
+skip_glob=packages/syft/src/syft/__init__.py,packages/grid/data/*

.pre-commit-config.yaml

@@ -3,41 +3,36 @@ repos:
     rev: v4.5.0
     hooks:
       - id: check-ast
-        exclude: ^(packages/syft/tests/mongomock)
         always_run: true
       - id: trailing-whitespace
         always_run: true
-        exclude: ^(docs/|.+\.md|.bumpversion.cfg|packages/syft/tests/mongomock)
+        exclude: ^(docs/|.+\.md|.bumpversion.cfg)
       - id: check-docstring-first
         always_run: true
-        exclude: ^(packages/syft/tests/mongomock)
       - id: check-json
         always_run: true
-        exclude: ^(packages/grid/frontend/|packages/syft/tests/mongomock|.vscode)
+        exclude: ^(packages/grid/frontend/|.vscode)
       - id: check-added-large-files
         always_run: true
         exclude: ^(packages/grid/backend/wheels/.*|docs/img/header.png|docs/img/terminalizer.gif)
       - id: check-yaml
         always_run: true
-        exclude: ^(packages/grid/k8s/rendered/|packages/grid/helm/|packages/syft/tests/mongomock)
+        exclude: ^(packages/grid/k8s/rendered/|packages/grid/helm/)
       - id: check-merge-conflict
         always_run: true
         args: ["--assume-in-merge"]
       - id: check-executables-have-shebangs
         always_run: true
-        exclude: ^(packages/syft/tests/mongomock)
       - id: debug-statements
         always_run: true
-        exclude: ^(packages/syft/tests/mongomock)
       - id: name-tests-test
         always_run: true
-        exclude: ^(.*/tests/utils/)|^(.*fixtures.py|packages/syft/tests/mongomock)|^(tests/scenarios/bigquery/helpers)
+        exclude: ^(.*/tests/utils/)|^(.*fixtures.py)|^(tests/scenarios/bigquery/helpers)
       - id: requirements-txt-fixer
         always_run: true
-        exclude: "packages/syft/tests/mongomock"
       - id: mixed-line-ending
         args: ["--fix=lf"]
-        exclude: '\.bat|\.csv|\.ps1$|packages/syft/tests/mongomock'
+        exclude: '\.bat|\.csv|\.ps1$'
 
   - repo: https://github.com/MarcoGorelli/absolufy-imports # This repository has been archived by the owner on Aug 15, 2023. It is now read-only.
     rev: v0.3.1
@@ -88,7 +83,6 @@ repos:
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix, --show-fixes]
-        exclude: packages/syft/tests/mongomock
         types_or: [python, pyi, jupyter]
       - id: ruff-format
         types_or: [python, pyi, jupyter]
@@ -178,7 +172,7 @@ repos:
     rev: "v3.0.0-alpha.9-for-vscode"
     hooks:
       - id: prettier
-        exclude: ^(packages/grid/helm|packages/grid/frontend/pnpm-lock.yaml|packages/syft/tests/mongomock|.vscode)
+        exclude: ^(packages/grid/helm|packages/grid/frontend/pnpm-lock.yaml|.vscode)
 
   # - repo: meta
   #   hooks:

.vscode/launch.json

@@ -4,6 +4,13 @@
   // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
   "version": "0.2.0",
   "configurations": [
+    {
+      "name": "Python Debugger: Current File",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${file}",
+      "console": "integratedTerminal"
+    },
     {
       "name": "Syft Debugger",
       "type": "debugpy",

docs/source/api_reference/syft.store.rst

@@ -32,8 +32,5 @@
    syft.store.kv_document_store
    syft.store.linked_obj
    syft.store.locks
-   syft.store.mongo_client
-   syft.store.mongo_codecs
-   syft.store.mongo_document_store
    syft.store.sqlite_document_store
 

notebooks/api/0.8/12-custom-api-endpoint.ipynb

@@ -657,6 +657,9 @@
     "# stdlib\n",
     "import time\n",
     "\n",
+    "# syft absolute\n",
+    "from syft.service.job.job_stash import JobStatus\n",
+    "\n",
     "# Iterate over the Jobs waiting them to finish their pipelines.\n",
     "job_pool = [\n",
     "    (log_call_job, \"Logging Private Function Call\"),\n",
@@ -665,13 +668,19 @@
     "]\n",
     "for job, expected_log in job_pool:\n",
     "    updated_job = datasite_client.api.services.job.get(job.id)\n",
-    "    while updated_job.status.value != \"completed\":\n",
+    "    while updated_job.status in {JobStatus.CREATED, JobStatus.PROCESSING}:\n",
     "        updated_job = datasite_client.api.services.job.get(job.id)\n",
     "        time.sleep(1)\n",
-    "    # If they're completed. Then, check if the TwinAPI print appears in the job logs.\n",
-    "    assert expected_log in datasite_client.api.services.job.get(job.id).logs(\n",
-    "        _print=False\n",
-    "    )"
+    "\n",
+    "    assert (\n",
+    "        updated_job.status == JobStatus.COMPLETED\n",
+    "    ), f\"Job {updated_job.id} exited with status {updated_job.status} and result {updated_job.result}\"\n",
+    "    if updated_job.status == JobStatus.COMPLETED:\n",
+    "        print(f\"Job {updated_job.id} completed\")\n",
+    "        # If they're completed. Then, check if the TwinAPI print appears in the job logs.\n",
+    "        assert expected_log in datasite_client.api.services.job.get(job.id).logs(\n",
+    "            _print=False\n",
+    "        )"
    ]
   },
   {
@@ -683,6 +692,11 @@
   }
  ],
  "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
@@ -693,7 +707,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.5"
+   "version": "3.10.13"
   }
  },
  "nbformat": 4,

notebooks/tutorials/deployments/03-deploy-k8s-k3d.ipynb

@@ -78,7 +78,7 @@
     "If you want to deploy your Kubernetes cluster in a resource-constrained environment, use the following flags to override the default configurations. Please note that you will need at least 1 CPU and 2 GB of RAM on Docker, and some tests may not work in such low-resource environments:\n",
     "\n",
     "```sh\n",
-    "helm install my-syft openmined/syft --version $SYFT_VERSION --namespace syft --create-namespace --set ingress.className=\"traefik\" --set server.resourcesPreset=null --set seaweedfs.resourcesPreset=null --set mongo.resourcesPreset=null --set registry.resourcesPreset=null --set proxy.resourcesPreset=null --set frontend.resourcesPreset=null\n",
+    "helm install my-syft openmined/syft --version $SYFT_VERSION --namespace syft --create-namespace --set ingress.className=\"traefik\" --set server.resourcesPreset=null --set seaweedfs.resourcesPreset=null --set postgres.resourcesPreset=null --set registry.resourcesPreset=null --set proxy.resourcesPreset=null --set frontend.resourcesPreset=null\n",
     "```\n",
     "\n",
     "</div>\n",
@@ -89,7 +89,7 @@
     "If you would like to set your own default password even for the production style deployment, use the following command:\n",
     "\n",
     "```sh\n",
-    "helm install my-syft openmined/syft --version $SYFT_VERSION --namespace syft --create-namespace --set ingress.className=\"traefik\" --set global.randomizedSecrets=false --set server.secret.defaultRootPassword=\"changethis\" --set seaweedfs.secret.s3RootPassword=\"admin\" --set mongo.secret.rootPassword=\"example\"\n",
+    "helm install my-syft openmined/syft --version $SYFT_VERSION --namespace syft --create-namespace --set ingress.className=\"traefik\" --set global.randomizedSecrets=false --set server.secret.defaultRootPassword=\"changethis\" --set seaweedfs.secret.s3RootPassword=\"admin\" --set postgres.secret.rootPassword=\"example\"\n",
     "```\n",
     "</div>\n"
    ]

packages/grid/backend/backend.dockerfile

@@ -9,7 +9,7 @@ ARG TORCH_VERSION="2.2.2"
 
 # ==================== [BUILD STEP] Python Dev Base ==================== #
 
-FROM cgr.dev/chainguard/wolfi-base as syft_deps
+FROM cgr.dev/chainguard/wolfi-base AS syft_deps
 
 ARG PYTHON_VERSION
 ARG UV_VERSION
@@ -45,7 +45,7 @@ RUN --mount=type=cache,target=/root/.cache,sharing=locked \
 
 # ==================== [Final] Setup Syft Server ==================== #
 
-FROM cgr.dev/chainguard/wolfi-base as backend
+FROM cgr.dev/chainguard/wolfi-base AS backend
 
 ARG PYTHON_VERSION
 ARG UV_VERSION
@@ -84,9 +84,10 @@ ENV \
     DEFAULT_ROOT_EMAIL="[email protected]" \
     DEFAULT_ROOT_PASSWORD="changethis" \
     STACK_API_KEY="changeme" \
-    MONGO_HOST="localhost" \
-    MONGO_PORT="27017" \
-    MONGO_USERNAME="root" \
-    MONGO_PASSWORD="example"
+    POSTGRESQL_DBNAME="syftdb_postgres" \
+    POSTGRESQL_HOST="localhost" \
+    POSTGRESQL_PORT="5432" \
+    POSTGRESQL_USERNAME="syft_postgres" \
+    POSTGRESQL_PASSWORD="example"
 
 CMD ["bash", "./grid/start.sh"]

packages/grid/backend/grid/core/config.py

@@ -126,18 +126,19 @@ def get_emails_enabled(self) -> Self:
     # NETWORK_CHECK_INTERVAL: int = int(os.getenv("NETWORK_CHECK_INTERVAL", 60))
     # DATASITE_CHECK_INTERVAL: int = int(os.getenv("DATASITE_CHECK_INTERVAL", 60))
     CONTAINER_HOST: str = str(os.getenv("CONTAINER_HOST", "docker"))
-    MONGO_HOST: str = str(os.getenv("MONGO_HOST", ""))
-    MONGO_PORT: int = int(os.getenv("MONGO_PORT", 27017))
-    MONGO_USERNAME: str = str(os.getenv("MONGO_USERNAME", ""))
-    MONGO_PASSWORD: str = str(os.getenv("MONGO_PASSWORD", ""))
+    POSTGRESQL_DBNAME: str = str(os.getenv("POSTGRESQL_DBNAME", ""))
+    POSTGRESQL_HOST: str = str(os.getenv("POSTGRESQL_HOST", ""))
+    POSTGRESQL_PORT: int = int(os.getenv("POSTGRESQL_PORT", 5432))
+    POSTGRESQL_USERNAME: str = str(os.getenv("POSTGRESQL_USERNAME", ""))
+    POSTGRESQL_PASSWORD: str = str(os.getenv("POSTGRESQL_PASSWORD", ""))
     DEV_MODE: bool = True if os.getenv("DEV_MODE", "false").lower() == "true" else False
     # ZMQ stuff
     QUEUE_PORT: int = int(os.getenv("QUEUE_PORT", 5556))
     CREATE_PRODUCER: bool = (
         True if os.getenv("CREATE_PRODUCER", "false").lower() == "true" else False
     )
     N_CONSUMERS: int = int(os.getenv("N_CONSUMERS", 1))
-    SQLITE_PATH: str = os.path.expandvars("$HOME/data/db/")
+    SQLITE_PATH: str = os.path.expandvars("/tmp/data/db")
     SINGLE_CONTAINER_MODE: bool = str_to_bool(os.getenv("SINGLE_CONTAINER_MODE", False))
     CONSUMER_SERVICE_NAME: str | None = os.getenv("CONSUMER_SERVICE_NAME")
     INMEMORY_WORKERS: bool = str_to_bool(os.getenv("INMEMORY_WORKERS", True))