diff --git a/packages/.dockerignore b/packages/.dockerignore index 559e945a401..a8628d4acb1 100644 --- a/packages/.dockerignore +++ b/packages/.dockerignore @@ -1,4 +1,9 @@ +**/*.pyc + grid/data grid/packer grid/.devspace -syftcli \ No newline at end of file +syftcli + +syft/tests +syft/README.md diff --git a/packages/grid/backend/backend.dockerfile b/packages/grid/backend/backend.dockerfile index 02cc5d506e3..e882794774f 100644 --- a/packages/grid/backend/backend.dockerfile +++ b/packages/grid/backend/backend.dockerfile @@ -1,67 +1,113 @@ -ARG PYTHON_VERSION='3.11' +ARG PYTHON_VERSION="3.11" +ARG TZ="Etc/UTC" -FROM python:3.11-slim as build +# change to USER="syftuser", UID=1000 and HOME="/home/$USER" for rootless +ARG USER="root" +ARG UID=0 +ARG USER_GRP=$USER:$USER +ARG HOME="/root" +ARG APPDIR="$HOME/app" -# set UTC timezone -ENV TZ=Etc/UTC -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone +# ==================== [BUILD STEP] Python Dev Base ==================== # -RUN mkdir -p /root/.local +FROM cgr.dev/chainguard/wolfi-base as python_dev -RUN apt-get update && apt-get upgrade -y -RUN --mount=type=cache,sharing=locked,target=/var/cache/apt \ - DEBIAN_FRONTEND=noninteractive \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - curl python3-dev gcc make build-essential cmake git +ARG PYTHON_VERSION +ARG TZ +ARG USER +ARG UID -RUN --mount=type=cache,target=/root/.cache \ - pip install -U pip +# Setup Python DEV +RUN apk update && \ + apk add build-base gcc tzdata python-$PYTHON_VERSION-dev py$PYTHON_VERSION-pip && \ + ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + # uncomment for creating rootless user + # && adduser -D -u $UID $USER -#install jupyterlab -RUN --mount=type=cache,target=/root/.cache \ - pip install --user jupyterlab +# ==================== [BUILD STEP] Install Syft Dependency ==================== # -WORKDIR /app +FROM python_dev as syft_deps -# Backend -FROM python:$PYTHON_VERSION-slim as backend -RUN apt-get update && apt-get upgrade -y -COPY --from=build /root/.local /root/.local +ARG APPDIR +ARG HOME +ARG UID +ARG USER +ARG USER_GRP -ENV PYTHONPATH=/app -ENV PATH=/root/.local/bin:$PATH +USER $USER +WORKDIR $APPDIR +ENV PATH=$PATH:$HOME/.local/bin -RUN --mount=type=cache,target=/root/.cache \ - pip install -U pip - -WORKDIR /app +# copy skeleton to do package install +COPY --chown=$USER_GRP syft/setup.py ./syft/setup.py +COPY --chown=$USER_GRP syft/setup.cfg ./syft/setup.cfg +COPY --chown=$USER_GRP syft/pyproject.toml ./syft/pyproject.toml +COPY --chown=$USER_GRP syft/MANIFEST.in ./syft/MANIFEST.in +COPY --chown=$USER_GRP syft/src/syft/VERSION ./syft/src/syft/VERSION +COPY --chown=$USER_GRP syft/src/syft/capnp ./syft/src/syft/capnp + +# Install all dependencies together here to avoid any version conflicts across pkgs +RUN --mount=type=cache,target=$HOME/.cache/,rw,uid=$UID \ + pip install --user torch==2.1.0 -f https://download.pytorch.org/whl/cpu/torch_stable.html && \ + pip install --user pip-autoremove jupyterlab==4.0.7 -e ./syft/ && \ + pip-autoremove ansible ansible-core -y + +# ==================== [Final] Setup Syft Server ==================== # + +FROM cgr.dev/chainguard/wolfi-base as backend + +# inherit from global +ARG APPDIR +ARG HOME +ARG PYTHON_VERSION +ARG TZ +ARG USER +ARG USER_GRP + +# Setup Python +RUN apk update && \ + apk add --no-cache tzdata bash python-$PYTHON_VERSION py$PYTHON_VERSION-pip && \ + ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone && \ + rm -rf /var/cache/apk/* && \ + # Uncomment for rootless user + # adduser -D -u 1000 $USER && \ + mkdir -p /var/log/pygrid $HOME/data/creds $HOME/data/db $HOME/.cache $HOME/.local + # chown -R $USER_GRP /var/log/pygrid $HOME/ + +USER $USER +WORKDIR $APPDIR + +# Update environment variables +ENV PATH=$PATH:$HOME/.local/bin \ + PYTHONPATH=$APPDIR \ + APPDIR=$APPDIR \ + NODE_NAME="default_node_name" \ + NODE_TYPE="domain" \ + SERVICE_NAME="backend" \ + RELEASE="production" \ + DEV_MODE="False" \ + CONTAINER_HOST="docker" \ + PORT=80\ + HTTP_PORT=80 \ + HTTPS_PORT=443 \ + DOMAIN_CONNECTION_PORT=3030 \ + IGNORE_TLS_ERRORS="False" \ + DEFAULT_ROOT_EMAIL="info@openmined.org" \ + DEFAULT_ROOT_PASSWORD="changethis" \ + STACK_API_KEY="changeme" \ + MONGO_HOST="localhost" \ + MONGO_PORT="27017" \ + MONGO_USERNAME="root" \ + MONGO_PASSWORD="example" \ + CREDENTIALS_PATH="$HOME/data/creds/credentials.json" + +# Copy pre-built jupyterlab, syft dependencies +COPY --chown=$USER_GRP --from=syft_deps $HOME/.local $HOME/.local # copy grid -COPY grid/backend /app/ +COPY --chown=$USER_GRP grid/backend/grid ./grid -# copy skeleton to do package install -COPY syft/setup.py /app/syft/setup.py -COPY syft/setup.cfg /app/syft/setup.cfg -COPY syft/pyproject.toml /app/syft/pyproject.toml -COPY syft/MANIFEST.in /app/syft/MANIFEST.in -COPY syft/src/syft/VERSION /app/syft/src/syft/VERSION -COPY syft/src/syft/capnp /app/syft/src/syft/capnp - -# install syft -RUN --mount=type=cache,target=/root/.cache \ - # force cpu torch - pip install --user torch==2.1.0 -f https://download.pytorch.org/whl/cpu/torch_stable.html \ - pip install --user -e /app/syft && \ - pip uninstall ansible ansible-core -y && \ - rm -rf ~/.local/lib/python3.11/site-packages/ansible_collections - -# security patches -RUN apt purge --auto-remove linux-libc-dev -y || true -RUN apt purge --auto-remove libldap-2.5-0 -y || true - -# copy any changed source -COPY syft/src /app/syft/src - -# change to worker-start.sh or start-reload.sh as needed -CMD ["bash", "/app/grid/start.sh"] +# copy syft +COPY --chown=$USER_GRP syft/ ./syft/ + +CMD ["bash", "./grid/start.sh"] diff --git a/packages/grid/backend/grid/bootstrap.py b/packages/grid/backend/grid/bootstrap.py index 6450591c8be..1f4b0fb8df8 100644 --- a/packages/grid/backend/grid/bootstrap.py +++ b/packages/grid/backend/grid/bootstrap.py @@ -29,7 +29,8 @@ def get_env(key: str, default: str = "") -> Optional[str]: return None -CREDENTIALS_PATH = str(get_env("CREDENTIALS_PATH", "/storage/credentials.json")) +DEFAULT_CREDENTIALS_PATH = os.path.expandvars("$HOME/data/creds/credentials.json") +CREDENTIALS_PATH = str(get_env("CREDENTIALS_PATH", DEFAULT_CREDENTIALS_PATH)) NODE_PRIVATE_KEY = "NODE_PRIVATE_KEY" NODE_UID = "NODE_UID" @@ -59,7 +60,7 @@ def save_credential(key: str, value: str) -> str: try: dirname = os.path.dirname(CREDENTIALS_PATH) if not os.path.exists(dirname): - os.mkdir(dirname) + os.makedirs(dirname, exist_ok=True) with open(CREDENTIALS_PATH, "w") as f: f.write(f"{json.dumps(credentials)}") except Exception as e: diff --git a/packages/grid/backend/grid/core/config.py b/packages/grid/backend/grid/core/config.py index 0b9948ca961..df140cc39aa 100644 --- a/packages/grid/backend/grid/core/config.py +++ b/packages/grid/backend/grid/core/config.py @@ -59,7 +59,9 @@ def get_project_name(cls, v: Optional[str], values: Dict[str, Any]) -> str: return v EMAIL_RESET_TOKEN_EXPIRE_HOURS: int = 48 - EMAIL_TEMPLATES_DIR: str = "/app/grid/email-templates/build" + EMAIL_TEMPLATES_DIR: str = os.path.expandvars( + "$HOME/app/grid/email-templates/build" + ) EMAILS_ENABLED: bool = False @validator("EMAILS_ENABLED", pre=True) @@ -106,6 +108,7 @@ def get_emails_enabled(cls, v: bool, values: Dict[str, Any]) -> bool: MONGO_PORT: int = int(os.getenv("MONGO_PORT", 0)) MONGO_USERNAME: str = str(os.getenv("MONGO_USERNAME", "")) MONGO_PASSWORD: str = str(os.getenv("MONGO_PASSWORD", "")) + SQLITE_PATH: str = os.path.expandvars("$HOME/data/db/") TEST_MODE: bool = ( True if os.getenv("TEST_MODE", "false").lower() == "true" else False diff --git a/packages/grid/backend/grid/core/node.py b/packages/grid/backend/grid/core/node.py index 06c400bbe8e..dea08672498 100644 --- a/packages/grid/backend/grid/core/node.py +++ b/packages/grid/backend/grid/core/node.py @@ -26,7 +26,7 @@ mongo_store_config = MongoStoreConfig(client_config=mongo_client_config) -client_config = SQLiteStoreClientConfig(path="/storage/") +client_config = SQLiteStoreClientConfig(path=settings.SQLITE_PATH) sql_store_config = SQLiteStoreConfig(client_config=client_config) node_type = get_node_type() diff --git a/packages/grid/backend/grid/start.sh b/packages/grid/backend/grid/start.sh index e8caa39cfef..a47b88bc717 100755 --- a/packages/grid/backend/grid/start.sh +++ b/packages/grid/backend/grid/start.sh @@ -1,35 +1,36 @@ #! /usr/bin/env bash set -e -echo "Running start.sh with RELEASE=${RELEASE}" -export GEVENT_MONKEYPATCH="False" +echo "Running start.sh with RELEASE=${RELEASE} and $(id)" +export GEVENT_MONKEYPATCH="False" APP_MODULE=grid.main:app LOG_LEVEL=${LOG_LEVEL:-info} HOST=${HOST:-0.0.0.0} PORT=${PORT:-80} RELOAD="" NODE_TYPE=${NODE_TYPE:-domain} +APPDIR=${APPDIR:-$HOME/app} + +# For debugging permissions +ls -lisa $HOME/data +ls -lisa $APPDIR/syft/ +ls -lisa $APPDIR/grid/ if [[ ${DEV_MODE} == "True" ]]; then echo "DEV_MODE Enabled" RELOAD="--reload" - pip install -e "/app/syft[telemetry]" + pip install --user -e "$APPDIR/syft[telemetry]" fi set +e -NODE_PRIVATE_KEY=$(python /app/grid/bootstrap.py --private_key) -NODE_UID=$(python /app/grid/bootstrap.py --uid) +export NODE_PRIVATE_KEY=$(python $APPDIR/grid/bootstrap.py --private_key) +export NODE_UID=$(python $APPDIR/grid/bootstrap.py --uid) +export NODE_TYPE=$NODE_TYPE set -e -echo "NODE_PRIVATE_KEY=$NODE_PRIVATE_KEY" echo "NODE_UID=$NODE_UID" echo "NODE_TYPE=$NODE_TYPE" -export NODE_UID=$NODE_UID -export NODE_PRIVATE_KEY=$NODE_PRIVATE_KEY -export NODE_TYPE=$NODE_TYPE - -# export GEVENT_MONKEYPATCH="True" exec uvicorn $RELOAD --host $HOST --port $PORT --log-level $LOG_LEVEL "$APP_MODULE" diff --git a/packages/grid/default.env b/packages/grid/default.env index 1c91483c1ea..759bab8ca01 100644 --- a/packages/grid/default.env +++ b/packages/grid/default.env @@ -9,6 +9,7 @@ HTTPS_PORT=443 HEADSCALE_PORT=8080 NETWORK_NAME=omnet RELEASE=production +CREDENTIALS_VOLUME=credentials-data # tls IGNORE_TLS_ERRORS=False @@ -109,4 +110,4 @@ OBLV_KEY_PATH="~/.oblv" DOMAIN_CONNECTION_PORT=3030 # Registation -ENABLE_SIGNUP=False \ No newline at end of file +ENABLE_SIGNUP=False diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index dde53587f46..da3082cb6b5 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -125,10 +125,17 @@ deployments: name: component-chart repo: https://charts.devspace.sh values: + # PodSecurityContext (uncomment for rootless "syftuser") + # securityContext: + # runAsNonRoot: true + # runAsUser: 1000 + # runAsGroup: 1000 + # fsGroup: 1000 + # fsGroupChangePolicy: "Always" containers: - image: "${CONTAINER_REGISTRY}/${DOCKER_IMAGE_BACKEND}:${VERSION}" volumeMounts: - - containerPath: /storage + - containerPath: /root/data/creds/ volume: name: credentials-data subPath: /credentials-data @@ -182,8 +189,6 @@ deployments: value: "${DEFAULT_ROOT_EMAIL}" - name: DEFAULT_ROOT_PASSWORD value: "${DEFAULT_ROOT_PASSWORD}" - - name: BACKEND_STORAGE_PATH - value: "/storage" volumes: - name: credentials-data size: "100Mi" @@ -313,8 +318,8 @@ dev: value: "True" logs: {} sync: - - path: ./backend/grid:/app/grid - - path: ../syft:/app/syft + - path: ./backend/grid:/root/app/grid + - path: ../syft:/root/app/syft profiles: - name: gateway diff --git a/packages/grid/docker-compose.dev.yml b/packages/grid/docker-compose.dev.yml index 3faef69b269..1aef5823b2b 100644 --- a/packages/grid/docker-compose.dev.yml +++ b/packages/grid/docker-compose.dev.yml @@ -41,8 +41,8 @@ services: backend: volumes: - - ${RELATIVE_PATH}./backend/grid:/app/grid - - ${RELATIVE_PATH}../syft:/app/syft + - ${RELATIVE_PATH}./backend/grid:/root/app/grid + - ${RELATIVE_PATH}../syft:/root/app/syft - ${RELATIVE_PATH}./data/package-cache:/root/.cache environment: - DEV_MODE=True @@ -51,16 +51,16 @@ services: # backend_stream: # volumes: - # - ${RELATIVE_PATH}./backend/grid:/app/grid - # - ${RELATIVE_PATH}../syft:/app/syft + # - ${RELATIVE_PATH}./backend/grid:/root/app/grid + # - ${RELATIVE_PATH}../syft:/root/app/syft # - ${RELATIVE_PATH}./data/package-cache:/root/.cache # environment: # - DEV_MODE=True # celeryworker: # volumes: - # - ${RELATIVE_PATH}./backend/grid:/app/grid - # - ${RELATIVE_PATH}../syft/:/app/syft + # - ${RELATIVE_PATH}./backend/grid:/root/app/grid + # - ${RELATIVE_PATH}../syft/:/root/app/syft # - ${RELATIVE_PATH}./data/package-cache:/root/.cache # environment: # - DEV_MODE=True diff --git a/packages/grid/docker-compose.yml b/packages/grid/docker-compose.yml index c02868028e9..ab4ac42c1d2 100644 --- a/packages/grid/docker-compose.yml +++ b/packages/grid/docker-compose.yml @@ -130,11 +130,9 @@ services: - ENABLE_OBLV=${ENABLE_OBLV} - DEFAULT_ROOT_EMAIL=${DEFAULT_ROOT_EMAIL} - DEFAULT_ROOT_PASSWORD=${DEFAULT_ROOT_PASSWORD} - - BACKEND_STORAGE_PATH=${BACKEND_STORAGE_PATH} - command: "/app/grid/start.sh" network_mode: service:proxy volumes: - - ${BACKEND_STORAGE_PATH}:/storage + - ${CREDENTIALS_VOLUME}:/root/data/creds/ stdin_open: true tty: true labels: @@ -168,10 +166,9 @@ services: # - DEV_MODE=${DEV_MODE} # - DOMAIN_CONNECTION_PORT=${DOMAIN_CONNECTION_PORT} # - ENABLE_OBLV=${ENABLE_OBLV} - # command: "/app/grid/start.sh" # network_mode: service:proxy # volumes: - # - credentials-data:/storage + # - credentials-data:/root/data/creds/ # celeryworker: # restart: always diff --git a/packages/grid/helm/manifests.yaml b/packages/grid/helm/manifests.yaml index cc20c5b278d..177bc6a0167 100644 --- a/packages/grid/helm/manifests.yaml +++ b/packages/grid/helm/manifests.yaml @@ -65,7 +65,7 @@ spec: - name: RELEASE value: production - name: VERSION - value: 0.8.2-beta.33 + value: 0.8.2-beta.40 - name: VERSION_HASH value: unknown - name: NODE_TYPE @@ -100,10 +100,8 @@ spec: value: info@openmined.org - name: DEFAULT_ROOT_PASSWORD value: changethis - - name: BACKEND_STORAGE_PATH - value: /storage envFrom: null - image: docker.io/openmined/grid-backend:0.8.2-beta.33 + image: docker.io/openmined/grid-backend:0.8.2-beta.40 lifecycle: null livenessProbe: null name: container-0 @@ -112,7 +110,7 @@ spec: startupProbe: null volumeDevices: null volumeMounts: - - mountPath: /storage + - mountPath: /root/data/creds/ name: credentials-data readOnly: false subPath: credentials-data @@ -220,7 +218,7 @@ spec: command: null env: - name: VERSION - value: 0.8.2-beta.33 + value: 0.8.2-beta.40 - name: VERSION_HASH value: unknown - name: NODE_TYPE @@ -228,7 +226,7 @@ spec: - name: NEXT_PUBLIC_API_URL value: ${NEXT_PUBLIC_API_URL} envFrom: null - image: docker.io/openmined/grid-frontend:0.8.2-beta.33 + image: docker.io/openmined/grid-frontend:0.8.2-beta.40 lifecycle: null livenessProbe: null name: container-0 @@ -363,7 +361,7 @@ spec: - name: MONGO_INITDB_ROOT_PASSWORD value: example envFrom: null - image: mongo:7.0.0 + image: mongo:7.0.2 lifecycle: null livenessProbe: null name: container-0 @@ -582,7 +580,7 @@ spec: - name: S3_PORT value: '8333' envFrom: null - image: chrislusf/seaweedfs:3.55 + image: chrislusf/seaweedfs:3.57 lifecycle: null livenessProbe: null name: container-0 diff --git a/packages/grid/helm/syft/templates/backend-statefulset.yaml b/packages/grid/helm/syft/templates/backend-statefulset.yaml index 4635147d65f..ca6790892e1 100644 --- a/packages/grid/helm/syft/templates/backend-statefulset.yaml +++ b/packages/grid/helm/syft/templates/backend-statefulset.yaml @@ -44,7 +44,7 @@ spec: - name: RELEASE value: production - name: VERSION - value: 0.8.2-beta.33 + value: 0.8.2-beta.40 - name: VERSION_HASH value: {{ .Values.node.settings.versionHash }} - name: NODE_TYPE @@ -79,10 +79,8 @@ spec: value: {{ .Values.secrets.syft.defaultRootEmail }} - name: DEFAULT_ROOT_PASSWORD value: {{ .Values.secrets.syft.defaultRootPassword }} - - name: BACKEND_STORAGE_PATH - value: /storage envFrom: null - image: docker.io/openmined/grid-backend:0.8.2-beta.33 + image: docker.io/openmined/grid-backend:0.8.2-beta.40 lifecycle: null livenessProbe: null name: container-0 @@ -91,7 +89,7 @@ spec: startupProbe: null volumeDevices: null volumeMounts: - - mountPath: /storage + - mountPath: /root/data/creds/ name: credentials-data readOnly: false subPath: credentials-data diff --git a/packages/grid/helm/syft/templates/frontend-deployment.yaml b/packages/grid/helm/syft/templates/frontend-deployment.yaml index e0b2ddb8346..4966d834e01 100644 --- a/packages/grid/helm/syft/templates/frontend-deployment.yaml +++ b/packages/grid/helm/syft/templates/frontend-deployment.yaml @@ -32,7 +32,7 @@ spec: command: null env: - name: VERSION - value: 0.8.2-beta.33 + value: 0.8.2-beta.40 - name: VERSION_HASH value: {{ .Values.node.settings.versionHash }} - name: NODE_TYPE @@ -40,7 +40,7 @@ spec: - name: NEXT_PUBLIC_API_URL value: ${NEXT_PUBLIC_API_URL} envFrom: null - image: docker.io/openmined/grid-frontend:0.8.2-beta.33 + image: docker.io/openmined/grid-frontend:0.8.2-beta.40 lifecycle: null livenessProbe: null name: container-0 diff --git a/packages/grid/helm/syft/templates/mongo-statefulset.yaml b/packages/grid/helm/syft/templates/mongo-statefulset.yaml index 9da6335d889..d25e01f446b 100644 --- a/packages/grid/helm/syft/templates/mongo-statefulset.yaml +++ b/packages/grid/helm/syft/templates/mongo-statefulset.yaml @@ -36,7 +36,7 @@ spec: - name: MONGO_INITDB_ROOT_PASSWORD value: {{ .Values.secrets.db.mongo.mongoInitdbRootPassword }} envFrom: null - image: mongo:7.0.0 + image: mongo:7.0.2 lifecycle: null livenessProbe: null name: container-0 diff --git a/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml b/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml index c6d82d0234c..7927ac34acc 100644 --- a/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml +++ b/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml @@ -43,7 +43,7 @@ spec: - name: S3_PORT value: '8333' envFrom: null - image: chrislusf/seaweedfs:3.55 + image: chrislusf/seaweedfs:3.57 lifecycle: null livenessProbe: null name: container-0 diff --git a/packages/grid/podman/podman-kube/podman-syft-kube-config.yaml b/packages/grid/podman/podman-kube/podman-syft-kube-config.yaml index 1684f1c3dfe..33690b5128f 100644 --- a/packages/grid/podman/podman-kube/podman-syft-kube-config.yaml +++ b/packages/grid/podman/podman-kube/podman-syft-kube-config.yaml @@ -102,7 +102,7 @@ data: # Syft SYFT_TUTORIAL_MODE: False - BACKEND_STORAGE_PATH: credentials-data + CREDENTIALS_VOLUME: credentials-data NODE_SIDE_TYPE: high # Worker diff --git a/packages/grid/worker/start.sh b/packages/grid/worker/start.sh index a6e8f004785..b27c07e7c6b 100755 --- a/packages/grid/worker/start.sh +++ b/packages/grid/worker/start.sh @@ -14,17 +14,13 @@ fi export RUST_BACKTRACE=$RUST_BACKTRACE set +e -NODE_PRIVATE_KEY=$(python bootstrap.py --private_key) -NODE_UID=$(python bootstrap.py --uid) +export NODE_PRIVATE_KEY=$(python bootstrap.py --private_key) +export NODE_UID=$(python bootstrap.py --uid) set -e -echo "NODE_PRIVATE_KEY=$NODE_PRIVATE_KEY" echo "NODE_UID=$NODE_UID" echo "NODE_TYPE=$NODE_TYPE" -export NODE_UID=$NODE_UID -export NODE_PRIVATE_KEY=$NODE_PRIVATE_KEY - APP_MODULE=worker:app LOG_LEVEL=${LOG_LEVEL:-info} HOST=${HOST:-0.0.0.0} diff --git a/packages/hagrid/hagrid/cli.py b/packages/hagrid/hagrid/cli.py index 62817c43612..f2df95575b9 100644 --- a/packages/hagrid/hagrid/cli.py +++ b/packages/hagrid/hagrid/cli.py @@ -265,7 +265,7 @@ def clean(location: str) -> None: ) @click.option("--tls", is_flag=True, help="Launch with TLS configuration") @click.option("--test", is_flag=True, help="Launch with test configuration") -@click.option("--dev", is_flag=True, help="Shortcut for development release") +@click.option("--dev", is_flag=True, help="Shortcut for development mode") @click.option( "--release", default="production", @@ -2062,9 +2062,9 @@ def build_command(cmd: str) -> TypeList[str]: return [build_cmd] -def deploy_command(cmd: str, tail: bool, release_type: str) -> TypeList[str]: +def deploy_command(cmd: str, tail: bool, dev_mode: bool) -> TypeList[str]: up_cmd = str(cmd) - up_cmd += " --file docker-compose.dev.yml" if release_type == "development" else "" + up_cmd += " --file docker-compose.dev.yml" if dev_mode else "" up_cmd += " up" if not tail: up_cmd += " -d" @@ -2178,7 +2178,7 @@ def create_launch_docker_cmd( ) # use a docker volume - backend_storage = "credentials-data" + host_path = "credentials-data" # in development use a folder mount if kwargs.get("release", "") == "development": @@ -2186,7 +2186,7 @@ def create_launch_docker_cmd( # if EDITABLE_MODE: # RELATIVE_PATH = "../" # we might need to change this for the hagrid template mode - backend_storage = f"{RELATIVE_PATH}./backend/grid/storage/{snake_name}" + host_path = f"{RELATIVE_PATH}./backend/grid/storage/{snake_name}" envs = { "RELEASE": "production", @@ -2206,7 +2206,7 @@ def create_launch_docker_cmd( generate_sec_random_password(length=48, special_chars=False) ), "ENABLE_OBLV": str(enable_oblv).lower(), - "BACKEND_STORAGE_PATH": backend_storage, + "CREDENTIALS_VOLUME": host_path, "NODE_SIDE_TYPE": kwargs["node_side_type"], } @@ -2361,9 +2361,8 @@ def create_launch_docker_cmd( my_build_command = build_command(cmd) final_commands["Building"] = my_build_command - release_type = kwargs["release"] - - final_commands["Launching"] = deploy_command(cmd, tail, release_type) + dev_mode = kwargs.get("dev", False) + final_commands["Launching"] = deploy_command(cmd, tail, dev_mode) return final_commands @@ -2373,8 +2372,6 @@ def create_launch_worker_cmd( build: bool, tail: bool = True, ) -> TypeDict[str, TypeList[str]]: - release_type = kwargs["release"] - final_commands = {} final_commands["Pulling"] = pull_command(cmd, kwargs) cmd += " --file docker-compose.yml" @@ -2383,9 +2380,8 @@ def create_launch_worker_cmd( my_build_command = build_command(cmd) final_commands["Building"] = my_build_command - release_type = kwargs["release"] - - final_commands["Launching"] = deploy_command(cmd, tail, release_type) + dev_mode = kwargs.get("dev", False) + final_commands["Launching"] = deploy_command(cmd, tail, dev_mode) return final_commands diff --git a/packages/syft/src/syft/store/sqlite_document_store.py b/packages/syft/src/syft/store/sqlite_document_store.py index 0cb1cd11b4e..8a7600ae36d 100644 --- a/packages/syft/src/syft/store/sqlite_document_store.py +++ b/packages/syft/src/syft/store/sqlite_document_store.py @@ -84,6 +84,11 @@ def _connect(self) -> None: # that different connections are used in each thread. By using a dict for the # _db and _cur we can ensure they are never shared self.file_path = self.store_config.client_config.file_path + + path = Path(self.file_path) + if not path.exists(): + path.parent.mkdir(parents=True, exist_ok=True) + self._db[thread_ident()] = sqlite3.connect( self.file_path, timeout=self.store_config.client_config.timeout, diff --git a/tox.ini b/tox.ini index 8afb2e94815..b0ff531f743 100644 --- a/tox.ini +++ b/tox.ini @@ -236,12 +236,14 @@ allowlist_externals = chcp passenv=HOME, USER setenv = - HAGRID_FLAGS = {env:HAGRID_FLAGS:--tag=local --test} + HAGRID_FLAGS = {env:HAGRID_FLAGS:--tag=local --release=development --test} EMULATION = {env:EMULATION:false} HAGRID_ART = false PYTHONIOENCODING = utf-8 PYTEST_MODULES = {env:PYTEST_MODULES:frontend network e2e security redis} commands = + bash -c "whoami; id;" + bash -c "echo Running with HAGRID_FLAGS=$HAGRID_FLAGS EMULATION=$EMULATION PYTEST_MODULES=$PYTEST_MODULES; date" ; install syft and hagrid @@ -280,9 +282,13 @@ commands = python -c 'import syft as sy; sy.stage_protocol_changes()' - bash -c 'HAGRID_ART=$HAGRID_ART hagrid launch test-gateway-1 network to docker:9081 $HAGRID_FLAGS --no-health-checks --verbose --no-warnings --dev' - bash -c 'HAGRID_ART=$HAGRID_ART hagrid launch test-domain-1 domain to docker:9082 $HAGRID_FLAGS --no-health-checks --enable-signup --verbose --no-warnings --dev' - ; bash -c 'HAGRID_ART=$HAGRID_ART hagrid launch test-domain-2 domain to docker:9083 --headless $HAGRID_FLAGS --enable-signup --no-health-checks --verbose --no-warnings --dev' + ; Make sure that pacakge-cache is owned by the current user + ; instead of docker creating it as root + bash -c 'mkdir -p packages/grid/data/package-cache' + + bash -c 'HAGRID_ART=$HAGRID_ART hagrid launch test-gateway-1 network to docker:9081 $HAGRID_FLAGS --no-health-checks --verbose --no-warnings --build' + bash -c 'HAGRID_ART=$HAGRID_ART hagrid launch test-domain-1 domain to docker:9082 $HAGRID_FLAGS --no-health-checks --enable-signup --verbose --no-warnings --build' + ; bash -c 'HAGRID_ART=$HAGRID_ART hagrid launch test-domain-2 domain to docker:9083 --headless $HAGRID_FLAGS --enable-signup --no-health-checks --verbose --no-warnings --build' ; wait for nodes to start docker ps @@ -825,7 +831,7 @@ commands = bash -c 'cd packages/grid/helm && \ helm lint syft' - bash -c "k3d cluster delete build || true" + bash -c "k3d cluster delete build; docker volume rm k3d-build-images --force; echo Done" [testenv:syft.package.helm] description = Package Helm Chart for Kubernetes