Merge pull request #8240 from kiendang/remove-worker-dockerfile

Merge backend and worker dockerfile

.github/workflows/cd-syft-dev.yml

@@ -116,19 +116,6 @@ jobs:
             ${{ secrets.ACR_SERVER }}/openmined/grid-frontend:${{ steps.grid.outputs.GRID_VERSION }}
           target: grid-ui-development
 
-      # TODO: Re-enable once we have Enclave up and running
-      # - name: Build and push `grid-enclave` image to registry
-      #   uses: docker/build-push-action@v5
-      #   with:
-      #     context: ./packages
-      #     file: ./packages/grid/worker/worker.dockerfile
-      #     push: true
-      #     target: worker
-      #     tags: |
-      #       ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:dev
-      #       ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:dev-${{ github.sha }}
-      #       ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:${{ steps.grid.outputs.GRID_VERSION }}
-
       - name: Build Helm Chart & Copy to infra
         if: github.ref == 'refs/heads/dev' || github.event.inputs.deploy-helm == 'true'
         shell: bash

.github/workflows/cd-syft.yml

@@ -153,25 +153,7 @@ jobs:
           digest="${{ steps.grid-frontend-build.outputs.digest }}"
           touch "/tmp/digests/grid-frontend/${digest#sha256:}"
 
-      - name: Build and push `grid-enclave` image to DockerHub
-        id: grid-enclave-build
-        uses: docker/build-push-action@v5
-        with:
-          context: ./packages
-          file: ./packages/grid/worker/worker.dockerfile
-          platforms: ${{ steps.release_metadata.outputs.release_platform }}
-          outputs: type=image,name=openmined/grid-enclave,push-by-digest=true,name-canonical=true,push=true
-          target: worker
-          cache-from: type=registry,ref=openmined/grid-enclave:cache-${{ steps.release_metadata.outputs.short_release_platform}}
-          cache-to: type=registry,ref=openmined/grid-enclave:cache-${{ steps.release_metadata.outputs.short_release_platform }},mode=max
-
-      - name: Export digest for grid-enclave
-        run: |
-          mkdir -p /tmp/digests/grid-enclave
-          digest="${{ steps.grid-enclave-build.outputs.digest }}"
-          touch "/tmp/digests/grid-enclave/${digest#sha256:}"
-
-      - name: Upload digest for grid-backend, grid-frontend and grid-enclave
+      - name: Upload digest for grid-backend and grid-frontend
         uses: actions/upload-artifact@v3
         with:
           name: digests-${{ steps.release_metadata.outputs.grid_version }}
@@ -219,14 +201,6 @@ jobs:
             -t openmined/grid-frontend:${{ needs.build-and-push-docker-images.outputs.release_tag }} \
             $(printf 'openmined/grid-frontend@sha256:%s ' *)
 
-      - name: Create manifest list and push for grid-enclave
-        working-directory: /tmp/digests/grid-enclave
-        run: |
-          docker buildx imagetools create  \
-            -t openmined/grid-enclave:${{ needs.build-and-push-docker-images.outputs.grid_version }} \
-            -t openmined/grid-enclave:${{ needs.build-and-push-docker-images.outputs.release_tag }} \
-            $(printf 'openmined/grid-enclave@sha256:%s ' *)
-
   deploy-syft:
     needs: [merge-docker-images]
     if: always() &&  needs.merge-docker-images.result == 'success'

packages/grid/backend/grid/core/config.py

@@ -14,6 +14,30 @@
 from pydantic import HttpUrl
 from pydantic import validator
 
+_truthy = {"yes", "y", "true", "t", "on", "1"}
+_falsy = {"no", "n", "false", "f", "off", "0"}
+
+
+def _distutils_strtoint(s: str) -> int:
+    """implements the deprecated distutils.util.strtoint"""
+    ls = s.lower()
+    if ls in _truthy:
+        return 1
+    if ls in _falsy:
+        return 0
+    raise ValueError(f"invalid truth value '{s}'")
+
+
+def str_to_int(bool_str: Any) -> int:
+    try:
+        return _distutils_strtoint(str(bool_str))
+    except ValueError:
+        return 0
+
+
+def str_to_bool(bool_str: Any) -> bool:
+    return bool(str_to_int(bool_str))
+
 
 class Settings(BaseSettings):
     API_V2_STR: str = "/api/v2"
@@ -109,6 +133,7 @@ def get_emails_enabled(cls, v: bool, values: Dict[str, Any]) -> bool:
     MONGO_USERNAME: str = str(os.getenv("MONGO_USERNAME", ""))
     MONGO_PASSWORD: str = str(os.getenv("MONGO_PASSWORD", ""))
     SQLITE_PATH: str = os.path.expandvars("$HOME/data/db/")
+    SINGLE_CONTAINER_MODE: bool = str_to_bool(os.getenv("SINGLE_CONTAINER_MODE", False))
 
     TEST_MODE: bool = (
         True if os.getenv("TEST_MODE", "false").lower() == "true" else False

packages/grid/backend/grid/core/node.py

@@ -1,5 +1,7 @@
 # syft absolute
+from syft.abstract_node import NodeType
 from syft.node.domain import Domain
+from syft.node.enclave import Enclave
 from syft.node.gateway import Gateway
 from syft.node.node import get_enable_warnings
 from syft.node.node import get_node_name
@@ -16,52 +18,61 @@
 # grid absolute
 from grid.core.config import settings
 
-mongo_client_config = MongoStoreClientConfig(
-    hostname=settings.MONGO_HOST,
-    port=settings.MONGO_PORT,
-    username=settings.MONGO_USERNAME,
-    password=settings.MONGO_PASSWORD,
-)
 
-mongo_store_config = MongoStoreConfig(client_config=mongo_client_config)
+def mongo_store_config() -> MongoStoreConfig:
+    mongo_client_config = MongoStoreClientConfig(
+        hostname=settings.MONGO_HOST,
+        port=settings.MONGO_PORT,
+        username=settings.MONGO_USERNAME,
+        password=settings.MONGO_PASSWORD,
+    )
+
+    return MongoStoreConfig(client_config=mongo_client_config)
+
 
+def sql_store_config() -> SQLiteStoreConfig:
+    client_config = SQLiteStoreClientConfig(path=settings.SQLITE_PATH)
+    return SQLiteStoreConfig(client_config=client_config)
+
+
+def seaweedfs_config() -> SeaweedFSConfig:
+    seaweed_client_config = SeaweedFSClientConfig(
+        host=settings.S3_ENDPOINT,
+        port=settings.S3_PORT,
+        access_key=settings.S3_ROOT_USER,
+        secret_key=settings.S3_ROOT_PWD,
+        region=settings.S3_REGION,
+        bucket_name=get_node_uid_env(),
+    )
 
-client_config = SQLiteStoreClientConfig(path=settings.SQLITE_PATH)
-sql_store_config = SQLiteStoreConfig(client_config=client_config)
+    return SeaweedFSConfig(client_config=seaweed_client_config)
 
-node_type = get_node_type()
+
+node_type = NodeType(get_node_type())
 node_name = get_node_name()
 
 node_side_type = get_node_side_type()
 enable_warnings = get_enable_warnings()
 
-seaweed_client_config = SeaweedFSClientConfig(
-    host=settings.S3_ENDPOINT,
-    port=settings.S3_PORT,
-    access_key=settings.S3_ROOT_USER,
-    secret_key=settings.S3_ROOT_PWD,
-    region=settings.S3_REGION,
-    bucket_name=get_node_uid_env(),
-)
+worker_classes = {
+    NodeType.DOMAIN: Domain,
+    NodeType.GATEWAY: Gateway,
+    NodeType.ENCLAVE: Enclave,
+}
 
-blob_storage_config = SeaweedFSConfig(client_config=seaweed_client_config)
+worker_class = worker_classes[node_type]
 
+single_container_mode = settings.SINGLE_CONTAINER_MODE
 
-if node_type == "gateway" or node_type == "network":
-    worker = Gateway(
-        name=node_name,
-        node_side_type=node_side_type,
-        action_store_config=mongo_store_config,
-        document_store_config=mongo_store_config,
-        enable_warnings=enable_warnings,
-        blob_storage_config=blob_storage_config,
-    )
-else:
-    worker = Domain(
-        name=node_name,
-        node_side_type=node_side_type,
-        action_store_config=mongo_store_config,
-        document_store_config=mongo_store_config,
-        enable_warnings=enable_warnings,
-        blob_storage_config=blob_storage_config,
-    )
+store_config = sql_store_config() if single_container_mode else mongo_store_config()
+blob_storage_config = None if single_container_mode else seaweedfs_config()
+
+worker = worker_class(
+    name=node_name,
+    node_side_type=node_side_type,
+    action_store_config=store_config,
+    document_store_config=store_config,
+    enable_warnings=enable_warnings,
+    blob_storage_config=blob_storage_config,
+    local_db=single_container_mode,
+)

packages/grid/default.env

@@ -20,7 +20,6 @@ TRAEFIK_PUBLIC_TAG=traefik-public
 
 STACK_NAME=grid-openmined-org
 DOCKER_IMAGE_BACKEND=openmined/grid-backend
-DOCKER_IMAGE_WORKER=openmined/grid-enclave
 DOCKER_IMAGE_FRONTEND=openmined/grid-frontend
 DOCKER_IMAGE_SVELTE=openmined/grid-svelte
 DOCKER_IMAGE_TRAEFIK=traefik

packages/grid/docker-compose.build.yml

@@ -5,9 +5,21 @@ services:
       context: ${RELATIVE_PATH}./frontend
       dockerfile: frontend.dockerfile
       target: "${FRONTEND_TARGET:-grid-ui-development}"
+    profiles:
+      - frontend
 
   backend:
     build:
       context: ${RELATIVE_PATH}../
       dockerfile: ./grid/backend/backend.dockerfile
       target: "backend"
+    profiles:
+      - backend
+
+  worker:
+    build:
+      context: ${RELATIVE_PATH}../
+      dockerfile: ./grid/backend/backend.dockerfile
+      target: "backend"
+    profiles:
+      - worker

packages/grid/docker-compose.dev.yml

@@ -1,6 +1,8 @@
 version: "3.8"
 services:
   proxy:
+    profiles:
+      - proxy
     ports:
       - "8080"
     extra_hosts:
@@ -10,6 +12,8 @@ services:
       - "--api.insecure=true" # admin panel no password
 
   frontend:
+    profiles:
+      - frontend
     volumes:
       - ${RELATIVE_PATH}./frontend/src:/app/src
       - ${RELATIVE_PATH}./frontend/static:/app/static
@@ -30,10 +34,26 @@ services:
   #     # - "5672" # AMQP port
 
   mongo:
+    profiles:
+      - mongo
     ports:
       - "27017"
 
   backend:
+    profiles:
+      - backend
+    volumes:
+      - ${RELATIVE_PATH}./backend/grid:/root/app/grid
+      - ${RELATIVE_PATH}../syft:/root/app/syft
+      - ${RELATIVE_PATH}./data/package-cache:/root/.cache
+    environment:
+      - DEV_MODE=True
+    stdin_open: true
+    tty: true
+
+  worker:
+    profiles:
+      - worker
     volumes:
       - ${RELATIVE_PATH}./backend/grid:/root/app/grid
       - ${RELATIVE_PATH}../syft:/root/app/syft

packages/grid/docker-compose.pull.yml

@@ -8,15 +8,29 @@ services:
 
   seaweedfs:
     image: "${DOCKER_IMAGE_SEAWEEDFS?Variable not set}:${SEAWEEDFS_VERSION}"
+    profiles:
+      - blob-storage
 
   # docker-host:
   #   image: qoomon/docker-host
 
   proxy:
     image: ${DOCKER_IMAGE_TRAEFIK?Variable not set}:${TRAEFIK_VERSION?Variable not set}
+    profiles:
+      - proxy
 
   mongo:
     image: "${MONGO_IMAGE}:${MONGO_VERSION}"
+    profiles:
+      - mongo
 
   jaeger:
     image: jaegertracing/all-in-one:1.37
+    profiles:
+      - telemetry
+
+  # Temporary fix until we refactor pull, build, launch UI step during hagrid launch
+  worker:
+    image: "${DOCKER_IMAGE_BACKEND?Variable not set}:${VERSION-latest}"
+    profiles:
+      - worker

packages/grid/docker-compose.test.yml

@@ -1,6 +1,8 @@
 version: "3.8"
 services:
   proxy:
+    profiles:
+      - proxy
     ports:
       - "8080"
     extra_hosts:
@@ -25,6 +27,14 @@ services:
       - "8333" # S3
 
   backend:
+    profiles:
+      - backend
+    environment:
+      - TEST_MODE=1
+
+  worker:
+    profiles:
+      - worker
     environment:
       - TEST_MODE=1
 

packages/grid/docker-compose.tls.yml

@@ -1,6 +1,8 @@
 version: "3.8"
 services:
   proxy:
+    profiles:
+      - proxy
     ports:
       - "${HTTPS_PORT}:${HTTPS_PORT}"
     environment: