diff --git a/.github/workflows/cd-syft-dev.yml b/.github/workflows/cd-syft-dev.yml index 1d29e2f6937..8d0169b5805 100644 --- a/.github/workflows/cd-syft-dev.yml +++ b/.github/workflows/cd-syft-dev.yml @@ -116,19 +116,6 @@ jobs: ${{ secrets.ACR_SERVER }}/openmined/grid-frontend:${{ steps.grid.outputs.GRID_VERSION }} target: grid-ui-development - # TODO: Re-enable once we have Enclave up and running - # - name: Build and push `grid-enclave` image to registry - # uses: docker/build-push-action@v5 - # with: - # context: ./packages - # file: ./packages/grid/worker/worker.dockerfile - # push: true - # target: worker - # tags: | - # ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:dev - # ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:dev-${{ github.sha }} - # ${{ secrets.ACR_SERVER }}/openmined/grid-enclave:${{ steps.grid.outputs.GRID_VERSION }} - - name: Build Helm Chart & Copy to infra if: github.ref == 'refs/heads/dev' || github.event.inputs.deploy-helm == 'true' shell: bash diff --git a/.github/workflows/cd-syft.yml b/.github/workflows/cd-syft.yml index 8e4c24e0c45..752ae0243aa 100644 --- a/.github/workflows/cd-syft.yml +++ b/.github/workflows/cd-syft.yml @@ -153,25 +153,7 @@ jobs: digest="${{ steps.grid-frontend-build.outputs.digest }}" touch "/tmp/digests/grid-frontend/${digest#sha256:}" - - name: Build and push `grid-enclave` image to DockerHub - id: grid-enclave-build - uses: docker/build-push-action@v5 - with: - context: ./packages - file: ./packages/grid/worker/worker.dockerfile - platforms: ${{ steps.release_metadata.outputs.release_platform }} - outputs: type=image,name=openmined/grid-enclave,push-by-digest=true,name-canonical=true,push=true - target: worker - cache-from: type=registry,ref=openmined/grid-enclave:cache-${{ steps.release_metadata.outputs.short_release_platform}} - cache-to: type=registry,ref=openmined/grid-enclave:cache-${{ steps.release_metadata.outputs.short_release_platform }},mode=max - - - name: Export digest for grid-enclave - run: | - mkdir -p /tmp/digests/grid-enclave - digest="${{ steps.grid-enclave-build.outputs.digest }}" - touch "/tmp/digests/grid-enclave/${digest#sha256:}" - - - name: Upload digest for grid-backend, grid-frontend and grid-enclave + - name: Upload digest for grid-backend and grid-frontend uses: actions/upload-artifact@v3 with: name: digests-${{ steps.release_metadata.outputs.grid_version }} @@ -219,14 +201,6 @@ jobs: -t openmined/grid-frontend:${{ needs.build-and-push-docker-images.outputs.release_tag }} \ $(printf 'openmined/grid-frontend@sha256:%s ' *) - - name: Create manifest list and push for grid-enclave - working-directory: /tmp/digests/grid-enclave - run: | - docker buildx imagetools create \ - -t openmined/grid-enclave:${{ needs.build-and-push-docker-images.outputs.grid_version }} \ - -t openmined/grid-enclave:${{ needs.build-and-push-docker-images.outputs.release_tag }} \ - $(printf 'openmined/grid-enclave@sha256:%s ' *) - deploy-syft: needs: [merge-docker-images] if: always() && needs.merge-docker-images.result == 'success' diff --git a/packages/grid/backend/grid/core/config.py b/packages/grid/backend/grid/core/config.py index df140cc39aa..7f974061e6a 100644 --- a/packages/grid/backend/grid/core/config.py +++ b/packages/grid/backend/grid/core/config.py @@ -14,6 +14,30 @@ from pydantic import HttpUrl from pydantic import validator +_truthy = {"yes", "y", "true", "t", "on", "1"} +_falsy = {"no", "n", "false", "f", "off", "0"} + + +def _distutils_strtoint(s: str) -> int: + """implements the deprecated distutils.util.strtoint""" + ls = s.lower() + if ls in _truthy: + return 1 + if ls in _falsy: + return 0 + raise ValueError(f"invalid truth value '{s}'") + + +def str_to_int(bool_str: Any) -> int: + try: + return _distutils_strtoint(str(bool_str)) + except ValueError: + return 0 + + +def str_to_bool(bool_str: Any) -> bool: + return bool(str_to_int(bool_str)) + class Settings(BaseSettings): API_V2_STR: str = "/api/v2" @@ -109,6 +133,7 @@ def get_emails_enabled(cls, v: bool, values: Dict[str, Any]) -> bool: MONGO_USERNAME: str = str(os.getenv("MONGO_USERNAME", "")) MONGO_PASSWORD: str = str(os.getenv("MONGO_PASSWORD", "")) SQLITE_PATH: str = os.path.expandvars("$HOME/data/db/") + SINGLE_CONTAINER_MODE: bool = str_to_bool(os.getenv("SINGLE_CONTAINER_MODE", False)) TEST_MODE: bool = ( True if os.getenv("TEST_MODE", "false").lower() == "true" else False diff --git a/packages/grid/backend/grid/core/node.py b/packages/grid/backend/grid/core/node.py index dea08672498..19d22c80567 100644 --- a/packages/grid/backend/grid/core/node.py +++ b/packages/grid/backend/grid/core/node.py @@ -1,5 +1,7 @@ # syft absolute +from syft.abstract_node import NodeType from syft.node.domain import Domain +from syft.node.enclave import Enclave from syft.node.gateway import Gateway from syft.node.node import get_enable_warnings from syft.node.node import get_node_name @@ -16,52 +18,61 @@ # grid absolute from grid.core.config import settings -mongo_client_config = MongoStoreClientConfig( - hostname=settings.MONGO_HOST, - port=settings.MONGO_PORT, - username=settings.MONGO_USERNAME, - password=settings.MONGO_PASSWORD, -) -mongo_store_config = MongoStoreConfig(client_config=mongo_client_config) +def mongo_store_config() -> MongoStoreConfig: + mongo_client_config = MongoStoreClientConfig( + hostname=settings.MONGO_HOST, + port=settings.MONGO_PORT, + username=settings.MONGO_USERNAME, + password=settings.MONGO_PASSWORD, + ) + + return MongoStoreConfig(client_config=mongo_client_config) + +def sql_store_config() -> SQLiteStoreConfig: + client_config = SQLiteStoreClientConfig(path=settings.SQLITE_PATH) + return SQLiteStoreConfig(client_config=client_config) + + +def seaweedfs_config() -> SeaweedFSConfig: + seaweed_client_config = SeaweedFSClientConfig( + host=settings.S3_ENDPOINT, + port=settings.S3_PORT, + access_key=settings.S3_ROOT_USER, + secret_key=settings.S3_ROOT_PWD, + region=settings.S3_REGION, + bucket_name=get_node_uid_env(), + ) -client_config = SQLiteStoreClientConfig(path=settings.SQLITE_PATH) -sql_store_config = SQLiteStoreConfig(client_config=client_config) + return SeaweedFSConfig(client_config=seaweed_client_config) -node_type = get_node_type() + +node_type = NodeType(get_node_type()) node_name = get_node_name() node_side_type = get_node_side_type() enable_warnings = get_enable_warnings() -seaweed_client_config = SeaweedFSClientConfig( - host=settings.S3_ENDPOINT, - port=settings.S3_PORT, - access_key=settings.S3_ROOT_USER, - secret_key=settings.S3_ROOT_PWD, - region=settings.S3_REGION, - bucket_name=get_node_uid_env(), -) +worker_classes = { + NodeType.DOMAIN: Domain, + NodeType.GATEWAY: Gateway, + NodeType.ENCLAVE: Enclave, +} -blob_storage_config = SeaweedFSConfig(client_config=seaweed_client_config) +worker_class = worker_classes[node_type] +single_container_mode = settings.SINGLE_CONTAINER_MODE -if node_type == "gateway" or node_type == "network": - worker = Gateway( - name=node_name, - node_side_type=node_side_type, - action_store_config=mongo_store_config, - document_store_config=mongo_store_config, - enable_warnings=enable_warnings, - blob_storage_config=blob_storage_config, - ) -else: - worker = Domain( - name=node_name, - node_side_type=node_side_type, - action_store_config=mongo_store_config, - document_store_config=mongo_store_config, - enable_warnings=enable_warnings, - blob_storage_config=blob_storage_config, - ) +store_config = sql_store_config() if single_container_mode else mongo_store_config() +blob_storage_config = None if single_container_mode else seaweedfs_config() + +worker = worker_class( + name=node_name, + node_side_type=node_side_type, + action_store_config=store_config, + document_store_config=store_config, + enable_warnings=enable_warnings, + blob_storage_config=blob_storage_config, + local_db=single_container_mode, +) diff --git a/packages/grid/default.env b/packages/grid/default.env index 6491bedb64f..95c6329e0f1 100644 --- a/packages/grid/default.env +++ b/packages/grid/default.env @@ -20,7 +20,6 @@ TRAEFIK_PUBLIC_TAG=traefik-public STACK_NAME=grid-openmined-org DOCKER_IMAGE_BACKEND=openmined/grid-backend -DOCKER_IMAGE_WORKER=openmined/grid-enclave DOCKER_IMAGE_FRONTEND=openmined/grid-frontend DOCKER_IMAGE_SVELTE=openmined/grid-svelte DOCKER_IMAGE_TRAEFIK=traefik diff --git a/packages/grid/docker-compose.build.yml b/packages/grid/docker-compose.build.yml index 8ff5d8599b2..a6408749c3b 100644 --- a/packages/grid/docker-compose.build.yml +++ b/packages/grid/docker-compose.build.yml @@ -5,9 +5,21 @@ services: context: ${RELATIVE_PATH}./frontend dockerfile: frontend.dockerfile target: "${FRONTEND_TARGET:-grid-ui-development}" + profiles: + - frontend backend: build: context: ${RELATIVE_PATH}../ dockerfile: ./grid/backend/backend.dockerfile target: "backend" + profiles: + - backend + + worker: + build: + context: ${RELATIVE_PATH}../ + dockerfile: ./grid/backend/backend.dockerfile + target: "backend" + profiles: + - worker diff --git a/packages/grid/docker-compose.dev.yml b/packages/grid/docker-compose.dev.yml index db3161913b7..badc3fdfafb 100644 --- a/packages/grid/docker-compose.dev.yml +++ b/packages/grid/docker-compose.dev.yml @@ -1,6 +1,8 @@ version: "3.8" services: proxy: + profiles: + - proxy ports: - "8080" extra_hosts: @@ -10,6 +12,8 @@ services: - "--api.insecure=true" # admin panel no password frontend: + profiles: + - frontend volumes: - ${RELATIVE_PATH}./frontend/src:/app/src - ${RELATIVE_PATH}./frontend/static:/app/static @@ -30,10 +34,26 @@ services: # # - "5672" # AMQP port mongo: + profiles: + - mongo ports: - "27017" backend: + profiles: + - backend + volumes: + - ${RELATIVE_PATH}./backend/grid:/root/app/grid + - ${RELATIVE_PATH}../syft:/root/app/syft + - ${RELATIVE_PATH}./data/package-cache:/root/.cache + environment: + - DEV_MODE=True + stdin_open: true + tty: true + + worker: + profiles: + - worker volumes: - ${RELATIVE_PATH}./backend/grid:/root/app/grid - ${RELATIVE_PATH}../syft:/root/app/syft diff --git a/packages/grid/docker-compose.pull.yml b/packages/grid/docker-compose.pull.yml index 576474e78a8..40812975460 100644 --- a/packages/grid/docker-compose.pull.yml +++ b/packages/grid/docker-compose.pull.yml @@ -8,15 +8,29 @@ services: seaweedfs: image: "${DOCKER_IMAGE_SEAWEEDFS?Variable not set}:${SEAWEEDFS_VERSION}" + profiles: + - blob-storage # docker-host: # image: qoomon/docker-host proxy: image: ${DOCKER_IMAGE_TRAEFIK?Variable not set}:${TRAEFIK_VERSION?Variable not set} + profiles: + - proxy mongo: image: "${MONGO_IMAGE}:${MONGO_VERSION}" + profiles: + - mongo jaeger: image: jaegertracing/all-in-one:1.37 + profiles: + - telemetry + + # Temporary fix until we refactor pull, build, launch UI step during hagrid launch + worker: + image: "${DOCKER_IMAGE_BACKEND?Variable not set}:${VERSION-latest}" + profiles: + - worker diff --git a/packages/grid/docker-compose.test.yml b/packages/grid/docker-compose.test.yml index bdf99c14294..fe1a3d71bbc 100644 --- a/packages/grid/docker-compose.test.yml +++ b/packages/grid/docker-compose.test.yml @@ -1,6 +1,8 @@ version: "3.8" services: proxy: + profiles: + - proxy ports: - "8080" extra_hosts: @@ -25,6 +27,14 @@ services: - "8333" # S3 backend: + profiles: + - backend + environment: + - TEST_MODE=1 + + worker: + profiles: + - worker environment: - TEST_MODE=1 diff --git a/packages/grid/docker-compose.tls.yml b/packages/grid/docker-compose.tls.yml index 6bfa5d7d1d7..010967b4561 100644 --- a/packages/grid/docker-compose.tls.yml +++ b/packages/grid/docker-compose.tls.yml @@ -1,6 +1,8 @@ version: "3.8" services: proxy: + profiles: + - proxy ports: - "${HTTPS_PORT}:${HTTPS_PORT}" environment: diff --git a/packages/grid/docker-compose.yml b/packages/grid/docker-compose.yml index 7a865336532..b6da43dc9b4 100644 --- a/packages/grid/docker-compose.yml +++ b/packages/grid/docker-compose.yml @@ -10,6 +10,8 @@ services: restart: always hostname: ${NODE_NAME?Variable not set} image: ${DOCKER_IMAGE_TRAEFIK?Variable not set}:${TRAEFIK_VERSION?Variable not set} + profiles: + - proxy networks: - "${TRAEFIK_PUBLIC_NETWORK?Variable not set}" - default @@ -76,9 +78,48 @@ services: # volumes: # - ./rabbitmq/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf + worker: + restart: always + image: "${DOCKER_IMAGE_BACKEND?Variable not set}:${VERSION-latest}" + profiles: + - worker + env_file: + - .env + environment: + - SERVICE_NAME=worker + - RELEASE=${RELEASE:-production} + - VERSION=${VERSION} + - VERSION_HASH=${VERSION_HASH} + - NODE_TYPE=${NODE_TYPE?Variable not set} + - NODE_NAME=${NODE_NAME?Variable not set} + - STACK_API_KEY=${STACK_API_KEY} + - PORT=${HTTP_PORT} + - IGNORE_TLS_ERRORS=${IGNORE_TLS_ERRORS?False} + - HTTP_PORT=${HTTP_PORT} + - HTTPS_PORT=${HTTPS_PORT} + - USE_BLOB_STORAGE=${USE_BLOB_STORAGE} + - CONTAINER_HOST=${CONTAINER_HOST} + - TRACE=False # TODO: Trace Mode is set to False, until jaegar is integrated + - JAEGER_HOST=${JAEGER_HOST} + - JAEGER_PORT=${JAEGER_PORT} + - ASSOCIATION_TIMEOUT=${ASSOCIATION_TIMEOUT} + - DEV_MODE=${DEV_MODE} + - NODE_SIDE_TYPE=${NODE_SIDE_TYPE} + - ENABLE_WARNINGS=${ENABLE_WARNINGS} + ports: + - "${HTTP_PORT}:${HTTP_PORT}" + volumes: + - credentials-data:/root/data/creds/ + extra_hosts: + - "host.docker.internal:host-gateway" + labels: + - "orgs.openmined.syft=this is a syft worker container" + backend: restart: always image: "${DOCKER_IMAGE_BACKEND?Variable not set}:${VERSION-latest}" + profiles: + - backend depends_on: - proxy env_file: @@ -205,6 +246,8 @@ services: mongo: image: "${MONGO_IMAGE}:${MONGO_VERSION}" + profiles: + - mongo restart: always environment: - MONGO_INITDB_ROOT_USERNAME=${MONGO_USERNAME} diff --git a/packages/grid/podman/podman-kube/podman-syft-kube-config.yaml b/packages/grid/podman/podman-kube/podman-syft-kube-config.yaml index ef4a8fa092e..bdd23d8e927 100644 --- a/packages/grid/podman/podman-kube/podman-syft-kube-config.yaml +++ b/packages/grid/podman/podman-kube/podman-syft-kube-config.yaml @@ -23,7 +23,6 @@ data: STACK_NAME: grid-openmined-org DOCKER_IMAGE_BACKEND: openmined/grid-backend - DOCKER_IMAGE_WORKER: openmined/grid-enclave DOCKER_IMAGE_FRONTEND: openmined/grid-frontend DOCKER_IMAGE_SVELTE: openmined/grid-svelte DOCKER_IMAGE_TRAEFIK: traefik diff --git a/packages/grid/worker/__init__.py b/packages/grid/worker/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/packages/grid/worker/bootstrap.py b/packages/grid/worker/bootstrap.py deleted file mode 120000 index a51aa478732..00000000000 --- a/packages/grid/worker/bootstrap.py +++ /dev/null @@ -1 +0,0 @@ -../backend/grid/bootstrap.py \ No newline at end of file diff --git a/packages/grid/worker/docker-compose.build.yml b/packages/grid/worker/docker-compose.build.yml deleted file mode 100644 index a6eb59a0835..00000000000 --- a/packages/grid/worker/docker-compose.build.yml +++ /dev/null @@ -1,9 +0,0 @@ -version: "3.8" -services: - worker: - build: - context: ${RELATIVE_PATH}../../ - dockerfile: ./grid/worker/worker.dockerfile - target: "worker" - labels: - com.docker.compose.container-number: 1 diff --git a/packages/grid/worker/docker-compose.dev.yml b/packages/grid/worker/docker-compose.dev.yml deleted file mode 100644 index 5f349600bf6..00000000000 --- a/packages/grid/worker/docker-compose.dev.yml +++ /dev/null @@ -1,12 +0,0 @@ -version: "3.8" -services: - worker: - volumes: - - ${RELATIVE_PATH}./:/app - - ${RELATIVE_PATH}../../syft/:/app/syft - - ${RELATIVE_PATH}../backend/grid/bootstrap.py:/app/bootstrap.py - # - ${RELATIVE_PATH}./data/package-cache:/root/.cache - environment: - - DEV_MODE=True - ports: - - "$HTTP_PORT" diff --git a/packages/grid/worker/docker-compose.pull.yml b/packages/grid/worker/docker-compose.pull.yml deleted file mode 100644 index 83a91755a8b..00000000000 --- a/packages/grid/worker/docker-compose.pull.yml +++ /dev/null @@ -1,4 +0,0 @@ -version: "3.8" -services: - worker: - image: "${DOCKER_IMAGE_WORKER?Variable not set}:${VERSION-latest}" diff --git a/packages/grid/worker/docker-compose.yml b/packages/grid/worker/docker-compose.yml deleted file mode 100644 index 155c33a1f6d..00000000000 --- a/packages/grid/worker/docker-compose.yml +++ /dev/null @@ -1,45 +0,0 @@ -version: "3.8" -services: - worker: - restart: always - image: "${DOCKER_IMAGE_WORKER?Variable not set}:${VERSION-latest}" - environment: - - SERVICE_NAME=worker - - RELEASE=${RELEASE:-production} - - VERSION=${VERSION} - - VERSION_HASH=${VERSION_HASH} - - NODE_TYPE=${NODE_TYPE?Variable not set} - - NODE_NAME=${NODE_NAME?Variable not set} - - STACK_API_KEY=${STACK_API_KEY} - - PORT=${HTTP_PORT} - - IGNORE_TLS_ERRORS=${IGNORE_TLS_ERRORS?False} - - HTTP_PORT=${HTTP_PORT} - - HTTPS_PORT=${HTTPS_PORT} - - USE_BLOB_STORAGE=${USE_BLOB_STORAGE} - - CONTAINER_HOST=${CONTAINER_HOST} - - TRACE=False # TODO: Trace Mode is set to False, until jaegar is integrated - - JAEGER_HOST=${JAEGER_HOST} - - JAEGER_PORT=${JAEGER_PORT} - - ASSOCIATION_TIMEOUT=${ASSOCIATION_TIMEOUT} - - DEV_MODE=${DEV_MODE} - - NODE_SIDE_TYPE=${NODE_SIDE_TYPE} - - ENABLE_WARNINGS=${ENABLE_WARNINGS} - command: "/app/start.sh" - ports: - - "${HTTP_PORT}:${HTTP_PORT}" - volumes: - - credentials-data:/storage - extra_hosts: - - "host.docker.internal:host-gateway" - labels: - - "orgs.openmined.syft=this is a syft worker container" - -volumes: - credentials-data: - labels: - - "orgs.openmined.syft=this is a syft worker container" - -networks: - worker-public: - labels: - - "orgs.openmined.syft=this is a syft worker container" diff --git a/packages/grid/worker/new_routes.py b/packages/grid/worker/new_routes.py deleted file mode 120000 index f7d6e339c47..00000000000 --- a/packages/grid/worker/new_routes.py +++ /dev/null @@ -1 +0,0 @@ -../backend/grid/api/new/new_routes.py \ No newline at end of file diff --git a/packages/grid/worker/start.sh b/packages/grid/worker/start.sh deleted file mode 100755 index b27c07e7c6b..00000000000 --- a/packages/grid/worker/start.sh +++ /dev/null @@ -1,29 +0,0 @@ -#! /usr/bin/env bash -set -e - -echo "Running start.sh with RELEASE=${RELEASE}" - -RELOAD="" -if [[ ${DEV_MODE} == "True" ]]; -then - echo "DEV_MODE Enabled" - RELOAD="--reload" - pip install -e "/app/syft[telemetry]" -fi - -export RUST_BACKTRACE=$RUST_BACKTRACE - -set +e -export NODE_PRIVATE_KEY=$(python bootstrap.py --private_key) -export NODE_UID=$(python bootstrap.py --uid) -set -e - -echo "NODE_UID=$NODE_UID" -echo "NODE_TYPE=$NODE_TYPE" - -APP_MODULE=worker:app -LOG_LEVEL=${LOG_LEVEL:-info} -HOST=${HOST:-0.0.0.0} -PORT=${PORT:-8080} - -exec uvicorn $RELOAD --host $HOST --port $PORT --log-level $LOG_LEVEL "$APP_MODULE" diff --git a/packages/grid/worker/worker.dockerfile b/packages/grid/worker/worker.dockerfile deleted file mode 100644 index f0bbb3ad860..00000000000 --- a/packages/grid/worker/worker.dockerfile +++ /dev/null @@ -1,64 +0,0 @@ -ARG PYTHON_VERSION='3.11' - -FROM python:3.11-slim as build - -# set UTC timezone -ENV TZ=Etc/UTC -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone - -RUN mkdir -p /root/.local - -RUN apt-get update && apt-get upgrade -y -RUN --mount=type=cache,sharing=locked,target=/var/cache/apt \ - DEBIAN_FRONTEND=noninteractive \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - curl python3-dev gcc make build-essential cmake git - -RUN --mount=type=cache,target=/root/.cache \ - pip install -U pip - - -WORKDIR /app - -# Backend -FROM python:$PYTHON_VERSION-slim as worker -RUN apt-get update && apt-get upgrade -y -COPY --from=build /root/.local /root/.local - -ENV PYTHONPATH=/app -ENV PATH=/root/.local/bin:$PATH - -RUN --mount=type=cache,target=/root/.cache \ - pip install -U pip - -WORKDIR /app - -# copy grid -COPY grid/worker /app/ -COPY grid/backend/grid/bootstrap.py /app/bootstrap.py -RUN chmod +x /app/start.sh - -# copy skeleton to do package install -COPY syft/setup.py /app/syft/setup.py -COPY syft/setup.cfg /app/syft/setup.cfg -COPY syft/pyproject.toml /app/syft/pyproject.toml -COPY syft/MANIFEST.in /app/syft/MANIFEST.in -COPY syft/src/syft/VERSION /app/syft/src/syft/VERSION -COPY syft/src/syft/capnp /app/syft/src/syft/capnp - -# install syft -RUN --mount=type=cache,target=/root/.cache \ - pip install --user torch==2.1.0 -f https://download.pytorch.org/whl/cpu/torch_stable.html && \ - pip install --user -e /app/syft && \ - pip uninstall ansible ansible-core -y && \ - rm -rf ~/.local/lib/python3.11/site-packages/ansible_collections - -# security patches -RUN apt purge --auto-remove linux-libc-dev -y || true -RUN apt purge --auto-remove libldap-2.5-0 -y || true - -# copy any changed source -COPY syft/src /app/syft/src - -CMD ["bash", "/app/start.sh"] diff --git a/packages/grid/worker/worker.py b/packages/grid/worker/worker.py deleted file mode 100644 index ea3b9d18315..00000000000 --- a/packages/grid/worker/worker.py +++ /dev/null @@ -1,57 +0,0 @@ -# stdlib - -# third party -from fastapi import FastAPI - -# syft absolute -from syft.abstract_node import NodeSideType -from syft.abstract_node import NodeType -from syft.client.client import API_PATH -from syft.node.domain import Domain -from syft.node.enclave import Enclave -from syft.node.gateway import Gateway -from syft.node.node import get_dev_mode -from syft.node.node import get_enable_warnings -from syft.node.node import get_node_name -from syft.node.node import get_node_side_type -from syft.node.node import get_node_type -from syft.node.routes import make_routes -from syft.protocol.data_protocol import stage_protocol_changes - -worker_classes = { - NodeType.DOMAIN: Domain, - NodeType.GATEWAY: Gateway, - NodeType.ENCLAVE: Enclave, -} - -node_name = get_node_name() -node_type = NodeType(get_node_type()) -node_side_type = NodeSideType(get_node_side_type()) -enable_warnings = get_enable_warnings() -if node_type not in worker_classes: - raise NotImplementedError(f"node_type: {node_type} is not supported") -worker_class = worker_classes[node_type] -worker = worker_class( - name=node_name, - local_db=True, - sqlite_path="/storage/", - node_type=node_type, - enable_warnings=enable_warnings, - node_side_type=node_side_type, -) -router = make_routes(worker=worker) - -app = FastAPI(title="Worker") - -if get_dev_mode(): - print("Staging protocol changes...") - status = stage_protocol_changes() - print(status) - - -@app.get("/") -async def root() -> str: - return f"OpenMined {node_type.value.capitalize()} Node Running" - - -app.include_router(router, prefix=API_PATH) diff --git a/packages/hagrid/hagrid/cli.py b/packages/hagrid/hagrid/cli.py index ae8ada75c9c..8d26c3b429f 100644 --- a/packages/hagrid/hagrid/cli.py +++ b/packages/hagrid/hagrid/cli.py @@ -142,9 +142,6 @@ def get_compose_src_path( else: path = deployment_dir(node_name) - if kwargs["deployment_type"] == "single_container": - path = path + "/worker" - os.makedirs(path, exist_ok=True) return path @@ -2142,6 +2139,8 @@ def create_launch_docker_cmd( **kwargs, ) + single_container_mode = kwargs["deployment_type"] == "single_container" + enable_oblv = bool(kwargs["oblv"]) print(" - NAME: " + str(snake_name)) print(" - TEMPLATE DIR: " + template_grid_dir) @@ -2202,6 +2201,7 @@ def create_launch_docker_cmd( "ENABLE_OBLV": str(enable_oblv).lower(), "CREDENTIALS_VOLUME": host_path, "NODE_SIDE_TYPE": kwargs["node_side_type"], + "SINGLE_CONTAINER_MODE": single_container_mode, } if "trace" in kwargs and kwargs["trace"] is True: @@ -2323,21 +2323,25 @@ def create_launch_docker_cmd( except Exception: # nosec pass - if kwargs["deployment_type"] == "single_container": - return create_launch_worker_cmd(cmd=cmd, kwargs=kwargs, build=build, tail=tail) + if single_container_mode: + cmd += " --profile worker" + else: + cmd += " --profile backend" + cmd += " --profile proxy" + cmd += " --profile mongo" - if str(node_type.input) in ["network", "gateway"]: - cmd += " --profile network" + if str(node_type.input) in ["network", "gateway"]: + cmd += " --profile network" - if use_blob_storage: - cmd += " --profile blob-storage" + if use_blob_storage: + cmd += " --profile blob-storage" - # no frontend container so expect bad gateway on the / route - if not bool(kwargs["headless"]): - cmd += " --profile frontend" + # no frontend container so expect bad gateway on the / route + if not bool(kwargs["headless"]): + cmd += " --profile frontend" - if "trace" in kwargs and kwargs["trace"]: - cmd += " --profile telemetry" + if "trace" in kwargs and kwargs["trace"]: + cmd += " --profile telemetry" final_commands = {} final_commands["Pulling"] = pull_command(cmd, kwargs) @@ -2357,25 +2361,6 @@ def create_launch_docker_cmd( return final_commands -def create_launch_worker_cmd( - cmd: str, - kwargs: TypeDict[str, Any], - build: bool, - tail: bool = True, -) -> TypeDict[str, TypeList[str]]: - final_commands = {} - final_commands["Pulling"] = pull_command(cmd, kwargs) - cmd += " --file docker-compose.yml" - - if build: - my_build_command = build_command(cmd) - final_commands["Building"] = my_build_command - - dev_mode = kwargs.get("dev", False) - final_commands["Launching"] = deploy_command(cmd, tail, dev_mode) - return final_commands - - def create_launch_vagrant_cmd(verb: GrammarVerb) -> str: host_term = verb.get_named_term_hostgrammar(name="host") node_name = verb.get_named_term_type(name="node_name") @@ -3279,16 +3264,9 @@ def create_land_docker_cmd(verb: GrammarVerb, prune_volumes: bool = False) -> st """ node_name = verb.get_named_term_type(name="node_name") snake_name = str(node_name.snake_input) - containers = shell("docker ps --format '{{.Names}}' | " + f"grep {snake_name}") - # Check if the container name belongs to worker container - grid_path = GRID_SRC_PATH() - if "proxy" in containers: - path = grid_path - env_var = ";export $(cat .env | sed 's/#.*//g' | xargs);" - else: - path = grid_path + "/worker" - env_var = ";export $(cat ../.env | sed 's/#.*//g' | xargs);" + path = GRID_SRC_PATH() + env_var = ";export $(cat .env | sed 's/#.*//g' | xargs);" cmd = "" cmd += "docker compose" diff --git a/packages/hagrid/hagrid/manifest_template.yml b/packages/hagrid/hagrid/manifest_template.yml index 054e0a9554a..fecb88df9b1 100644 --- a/packages/hagrid/hagrid/manifest_template.yml +++ b/packages/hagrid/hagrid/manifest_template.yml @@ -30,11 +30,6 @@ files: - devspace.yaml - traefik/k8s/dynamic.yml - traefik/k8s/traefik.yml - worker: - - worker/docker-compose.yml - - worker/docker-compose.dev.yml - - worker/docker-compose.pull.yml - - worker/docker-compose.build.yml podman: - podman/podman-kube/podman-syft-kube-config.yaml - podman/podman-kube/podman-syft-kube.yaml diff --git a/packages/hagrid/hagrid/orchestra.py b/packages/hagrid/hagrid/orchestra.py index 542739b304d..273ff441d3d 100644 --- a/packages/hagrid/hagrid/orchestra.py +++ b/packages/hagrid/hagrid/orchestra.py @@ -578,6 +578,8 @@ def shutdown( land_output = shell(f"hagrid land {snake_name} --force") if "Removed" in land_output: print(f" ✅ {snake_name} Container Removed") + elif "No resource found to remove for project" in land_output: + print(f" ✅ {snake_name} Container does not exist") else: print(f"❌ Unable to remove container: {snake_name} :{land_output}") diff --git a/packages/hagrid/hagrid/parse_template.py b/packages/hagrid/hagrid/parse_template.py index 3e4c5b351be..e4f8f9a124f 100644 --- a/packages/hagrid/hagrid/parse_template.py +++ b/packages/hagrid/hagrid/parse_template.py @@ -169,13 +169,8 @@ def setup_from_manifest_template( os.path.join(package_path, f) for f in template_files["common"] ] - # worker - if deployment_type == "single_container" and host_type in ["docker"]: - files_to_download += [ - os.path.join(package_path, f) for f in template_files["worker"] - ] # docker related files - elif host_type in ["docker"]: + if host_type in ["docker"]: files_to_download += [ os.path.join(package_path, f) for f in template_files["docker"] ] @@ -266,13 +261,7 @@ def render_templates( # common files files_to_render += template_files["common"] - # worker - if deployment_type == "single_container" and host_type in ["docker"]: - for template_file in template_files["worker"]: - if "default.env" not in template_file: - files_to_render.append(template_file) - - elif host_type in ["docker"]: + if host_type in ["docker"]: # docker related files for template_file in template_files["docker"]: if "default.env" not in template_file: diff --git a/packages/syftcli/manifest.yml b/packages/syftcli/manifest.yml index a7d94164f64..e2bbd293f8d 100644 --- a/packages/syftcli/manifest.yml +++ b/packages/syftcli/manifest.yml @@ -26,11 +26,6 @@ configFiles: - packages/grid/devspace.yaml - packages/grid/traefik/k8s/dynamic.yml - packages/grid/traefik/k8s/traefik.yml - worker: - - packages/grid/worker/docker-compose.yml - - packages/grid/worker/docker-compose.dev.yml - - packages/grid/worker/docker-compose.pull.yml - - packages/grid/worker/docker-compose.build.yml podman: - packages/grid/podman/podman-kube/podman-syft-kube-config.yaml - packages/grid/podman/podman-kube/podman-syft-kube.yaml diff --git a/tox.ini b/tox.ini index d967ab88d56..359d08719b4 100644 --- a/tox.ini +++ b/tox.ini @@ -291,7 +291,7 @@ commands = ; instead of docker creating it as root bash -c 'mkdir -p packages/grid/data/package-cache' - bash -c 'HAGRID_ART=$HAGRID_ART hagrid launch test-gateway-1 network to docker:9081 $HAGRID_FLAGS --no-health-checks --verbose --no-warnings --build' + bash -c 'HAGRID_ART=$HAGRID_ART hagrid launch test-gateway-1 gateway to docker:9081 $HAGRID_FLAGS --no-health-checks --verbose --no-warnings --build' bash -c 'HAGRID_ART=$HAGRID_ART hagrid launch test-domain-1 domain to docker:9082 $HAGRID_FLAGS --no-health-checks --enable-signup --verbose --no-warnings --build' ; bash -c 'HAGRID_ART=$HAGRID_ART hagrid launch test-domain-2 domain to docker:9083 --headless $HAGRID_FLAGS --enable-signup --no-health-checks --verbose --no-warnings --build'