From 6c8a2a5f26eb4020066428ee9dc5ba0a8d9f318d Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Wed, 18 Oct 2023 18:30:07 +0000 Subject: [PATCH 01/19] [grid] secure base + rootless user + cached deps --- packages/grid/backend/backend.dockerfile | 109 +++++++++++++---------- packages/syft/setup.cfg | 22 ++--- 2 files changed, 76 insertions(+), 55 deletions(-) diff --git a/packages/grid/backend/backend.dockerfile b/packages/grid/backend/backend.dockerfile index 6a2208655b8..0bad6570838 100644 --- a/packages/grid/backend/backend.dockerfile +++ b/packages/grid/backend/backend.dockerfile @@ -1,65 +1,84 @@ -ARG PYTHON_VERSION='3.11' +# ==================== [BASE] Setup deps + rootless user ==================== # -FROM python:3.11-slim as build +FROM cgr.dev/chainguard/wolfi-base as base -# set UTC timezone +ARG PYTHON_VERSION="3.11" ENV TZ=Etc/UTC -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone -RUN mkdir -p /root/.local +RUN --mount=type=cache,target=/var/cache/apk \ + apk update && \ + apk add bash tzdata python-$PYTHON_VERSION py$PYTHON_VERSION-pip && \ + ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone -RUN apt-get update && apt-get upgrade -y -RUN --mount=type=cache,sharing=locked,target=/var/cache/apt \ - DEBIAN_FRONTEND=noninteractive \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - curl python3-dev gcc make build-essential cmake git +ARG HOME=/home/nonroot +ARG NONROOT=nonroot:nonroot -RUN --mount=type=cache,target=/root/.cache \ - pip install -U pip +# use wolfi-base provided rootless user +USER nonroot -#install jupyterlab -RUN --mount=type=cache,target=/root/.cache \ +ENV PATH=$PATH:$HOME/.local/bin + +# ==================== [BUILD CACHE] Jupyterlab Cache ==================== # + +FROM base as jupyter + +RUN --mount=type=cache,target=$HOME/.cache/ \ pip install --user jupyterlab -WORKDIR /app +# ==================== [BUILD CACHE] Syft deps changes ==================== # -# Backend -FROM python:$PYTHON_VERSION-slim as backend -RUN apt-get update && apt-get upgrade -y -COPY --from=build /root/.local /root/.local +FROM base as syft_deps_changes -ENV PYTHONPATH=/app -ENV PATH=/root/.local/bin:$PATH +WORKDIR $HOME/ -RUN --mount=type=cache,target=/root/.cache \ - pip install -U pip +COPY --chown=$NONROOT syft/setup.cfg $HOME/setup.cfg -WORKDIR /app +# setup.cfg might change, but ML dependencies may not +# so we take a snapshot of everything in DOCKER:CACHED +RUN awk '/DOCKER:CACHED:START/,/DOCKER:CACHED:END/ {if (!/#/) print}' $HOME/setup.cfg | sed 's/ //g' | sort > $HOME/requirements.txt; -# copy grid -COPY grid/backend /app/ +# ==================== [BUILD CACHE] Syft ML Cache ==================== # + +FROM base as syft_cached_deps + +COPY --from=syft_deps_changes $HOME/requirements.txt $HOME/requirements.txt + +# Should hopefully be run only when req.txt changes +RUN --mount=type=cache,target=$HOME/.cache/ \ + pip install --user -r $HOME/requirements.txt && \ + rm $HOME/requirements.txt + +# ==================== [MAIN] Setup Syft ==================== # + +FROM syft_cached_deps + +# Copy pre-built jupyterlab +COPY --from=jupyter --chown=$NONROOT $HOME/.local $HOME/.local + +WORKDIR $HOME/app +ENV PYTHONPATH=$HOME/app +ENV APPDIR=$HOME/app # copy skeleton to do package install -COPY syft/setup.py /app/syft/setup.py -COPY syft/setup.cfg /app/syft/setup.cfg -COPY syft/pyproject.toml /app/syft/pyproject.toml -COPY syft/MANIFEST.in /app/syft/MANIFEST.in -COPY syft/src/syft/VERSION /app/syft/src/syft/VERSION -COPY syft/src/syft/capnp /app/syft/src/syft/capnp - -# install syft -RUN --mount=type=cache,target=/root/.cache \ - pip install --user -e /app/syft && \ - pip uninstall ansible ansible-core -y && \ - rm -rf ~/.local/lib/python3.11/site-packages/ansible_collections - -# security patches -RUN apt purge --auto-remove linux-libc-dev -y || true -RUN apt purge --auto-remove libldap-2.5-0 -y || true +COPY --chown=$NONROOT syft/setup.py ./syft/setup.py +COPY --chown=$NONROOT syft/setup.cfg ./syft/setup.cfg +COPY --chown=$NONROOT syft/pyproject.toml ./syft/pyproject.toml +COPY --chown=$NONROOT syft/MANIFEST.in ./syft/MANIFEST.in +COPY --chown=$NONROOT syft/src/syft/VERSION ./syft/src/syft/VERSION +COPY --chown=$NONROOT syft/src/syft/capnp ./syft/src/syft/capnp + +# Install Syft +RUN pip install --user pip-autoremove && \ + pip install --user -e ./syft/ && \ + pip-autoremove ansible ansible-core -y && \ + pip uninstall pip-autoremove -y && \ + rm -rf $HOME/.cache/ + +# copy grid +COPY --chown=$NONROOT grid/backend . # copy any changed source -COPY syft/src /app/syft/src +COPY --chown=$NONROOT syft/src ./syft/src # change to worker-start.sh or start-reload.sh as needed -CMD ["bash", "/app/grid/start.sh"] +CMD ["bash", "./grid/start.sh"] diff --git a/packages/syft/setup.cfg b/packages/syft/setup.cfg index 94c5c5d9fa4..1497adc7cf6 100644 --- a/packages/syft/setup.cfg +++ b/packages/syft/setup.cfg @@ -31,15 +31,8 @@ syft = forbiddenfruit==0.1.4 gevent==22.10.2 gipc==1.5.0 - jaxlib==0.4.18 - jax==0.4.18 loguru==0.7.2 - networkx==2.8 - numpy>=1.23.5,<=1.24.4 - opendp==0.8.0 packaging>=23.0 - pandas==1.5.3 - pyarrow==11.0.0 pycapnp==1.3.0 pydantic[email]==1.10.13 pymongo==4.5.0 @@ -56,15 +49,24 @@ syft = uvicorn[standard]==0.23.2 fastapi==0.103.2 hagrid>=0.3 - matplotlib==3.8.0 - dm-haiku==0.0.10 itables==1.6.2 + argon2-cffi==23.1.0 + # ===== DOCKER:CACHED:START ===== # + numpy>=1.23.5,<=1.24.4 + pandas==1.5.3 + pyarrow==11.0.0 + matplotlib==3.8.0 + networkx==2.8 + jaxlib==0.4.18 + jax==0.4.18 + opendp==0.8.0 safetensors==0.4.0 transformers==4.34.0 evaluate==0.4.1 + dm-haiku==0.0.10 torch==2.1.0 recordlinkage==0.16 - argon2-cffi==23.1.0 + # ===== DOCKER:CACHED:END ===== # install_requires = %(syft)s From 3c0d1d5d0e76593417cbabbe8613315e31a9339b Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Wed, 18 Oct 2023 18:30:48 +0000 Subject: [PATCH 02/19] [grid] make syft use rootless user paths --- packages/grid/backend/grid/bootstrap.py | 2 +- packages/grid/backend/grid/core/node.py | 2 +- packages/grid/backend/grid/start.sh | 6 +++--- packages/grid/devspace.yaml | 4 ++-- packages/grid/worker/worker.py | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/grid/backend/grid/bootstrap.py b/packages/grid/backend/grid/bootstrap.py index 6450591c8be..24eef12b189 100644 --- a/packages/grid/backend/grid/bootstrap.py +++ b/packages/grid/backend/grid/bootstrap.py @@ -29,7 +29,7 @@ def get_env(key: str, default: str = "") -> Optional[str]: return None -CREDENTIALS_PATH = str(get_env("CREDENTIALS_PATH", "/storage/credentials.json")) +CREDENTIALS_PATH = str(get_env("CREDENTIALS_PATH", "./storage/credentials.json")) NODE_PRIVATE_KEY = "NODE_PRIVATE_KEY" NODE_UID = "NODE_UID" diff --git a/packages/grid/backend/grid/core/node.py b/packages/grid/backend/grid/core/node.py index d164debc14a..d8ca11abc32 100644 --- a/packages/grid/backend/grid/core/node.py +++ b/packages/grid/backend/grid/core/node.py @@ -26,7 +26,7 @@ mongo_store_config = MongoStoreConfig(client_config=mongo_client_config) -client_config = SQLiteStoreClientConfig(path="/storage/") +client_config = SQLiteStoreClientConfig(path="./storage/") sql_store_config = SQLiteStoreConfig(client_config=client_config) node_type = get_node_type() diff --git a/packages/grid/backend/grid/start.sh b/packages/grid/backend/grid/start.sh index e8caa39cfef..a25ba9629fc 100755 --- a/packages/grid/backend/grid/start.sh +++ b/packages/grid/backend/grid/start.sh @@ -15,12 +15,12 @@ if [[ ${DEV_MODE} == "True" ]]; then echo "DEV_MODE Enabled" RELOAD="--reload" - pip install -e "/app/syft[telemetry]" + pip install -e "$APPDIR/syft[telemetry]" fi set +e -NODE_PRIVATE_KEY=$(python /app/grid/bootstrap.py --private_key) -NODE_UID=$(python /app/grid/bootstrap.py --uid) +NODE_PRIVATE_KEY=$(python $APPDIR/grid/bootstrap.py --private_key) +NODE_UID=$(python $APPDIR/grid/bootstrap.py --uid) set -e echo "NODE_PRIVATE_KEY=$NODE_PRIVATE_KEY" diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index 010caa037e1..0dc1ebaf22f 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -313,8 +313,8 @@ dev: value: "True" logs: {} sync: - - path: ./backend/grid:/app/grid - - path: ../syft:/app/syft + - path: ./backend/grid:/home/nonroot/app/grid + - path: ../syft:/home/nonroot/app/syft profiles: - name: gateway diff --git a/packages/grid/worker/worker.py b/packages/grid/worker/worker.py index ea3b9d18315..e2a34ea27ad 100644 --- a/packages/grid/worker/worker.py +++ b/packages/grid/worker/worker.py @@ -34,7 +34,7 @@ worker = worker_class( name=node_name, local_db=True, - sqlite_path="/storage/", + sqlite_path="./storage/", node_type=node_type, enable_warnings=enable_warnings, node_side_type=node_side_type, From 7249a68c323dc281bad44f1097aacd1a4dbd93ca Mon Sep 17 00:00:00 2001 From: Madhava Jay Date: Thu, 19 Oct 2023 09:59:46 +1000 Subject: [PATCH 03/19] Fixed issue with missing build tools for some python packages --- packages/grid/backend/backend.dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/grid/backend/backend.dockerfile b/packages/grid/backend/backend.dockerfile index 0bad6570838..1c684561470 100644 --- a/packages/grid/backend/backend.dockerfile +++ b/packages/grid/backend/backend.dockerfile @@ -7,7 +7,7 @@ ENV TZ=Etc/UTC RUN --mount=type=cache,target=/var/cache/apk \ apk update && \ - apk add bash tzdata python-$PYTHON_VERSION py$PYTHON_VERSION-pip && \ + apk add build-base gcc python-$PYTHON_VERSION-dev bash tzdata python-$PYTHON_VERSION py$PYTHON_VERSION-pip && \ ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone ARG HOME=/home/nonroot From 9ee1995bcb928e41d38ab6e2cdcc19202848f6b9 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Fri, 20 Oct 2023 15:42:13 +0000 Subject: [PATCH 04/19] revert setup.cfg changes --- packages/syft/setup.cfg | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/packages/syft/setup.cfg b/packages/syft/setup.cfg index 1497adc7cf6..94c5c5d9fa4 100644 --- a/packages/syft/setup.cfg +++ b/packages/syft/setup.cfg @@ -31,8 +31,15 @@ syft = forbiddenfruit==0.1.4 gevent==22.10.2 gipc==1.5.0 + jaxlib==0.4.18 + jax==0.4.18 loguru==0.7.2 + networkx==2.8 + numpy>=1.23.5,<=1.24.4 + opendp==0.8.0 packaging>=23.0 + pandas==1.5.3 + pyarrow==11.0.0 pycapnp==1.3.0 pydantic[email]==1.10.13 pymongo==4.5.0 @@ -49,24 +56,15 @@ syft = uvicorn[standard]==0.23.2 fastapi==0.103.2 hagrid>=0.3 - itables==1.6.2 - argon2-cffi==23.1.0 - # ===== DOCKER:CACHED:START ===== # - numpy>=1.23.5,<=1.24.4 - pandas==1.5.3 - pyarrow==11.0.0 matplotlib==3.8.0 - networkx==2.8 - jaxlib==0.4.18 - jax==0.4.18 - opendp==0.8.0 + dm-haiku==0.0.10 + itables==1.6.2 safetensors==0.4.0 transformers==4.34.0 evaluate==0.4.1 - dm-haiku==0.0.10 torch==2.1.0 recordlinkage==0.16 - # ===== DOCKER:CACHED:END ===== # + argon2-cffi==23.1.0 install_requires = %(syft)s From 9a3df9b7b13e15d32468739e491c290bcedf4017 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Fri, 20 Oct 2023 16:32:12 +0000 Subject: [PATCH 05/19] [grid] fix duplication in build caches --- packages/.dockerignore | 7 +- packages/grid/backend/backend.dockerfile | 117 ++++++++++++----------- 2 files changed, 65 insertions(+), 59 deletions(-) diff --git a/packages/.dockerignore b/packages/.dockerignore index 559e945a401..a8628d4acb1 100644 --- a/packages/.dockerignore +++ b/packages/.dockerignore @@ -1,4 +1,9 @@ +**/*.pyc + grid/data grid/packer grid/.devspace -syftcli \ No newline at end of file +syftcli + +syft/tests +syft/README.md diff --git a/packages/grid/backend/backend.dockerfile b/packages/grid/backend/backend.dockerfile index 1c684561470..0eaab0edf2f 100644 --- a/packages/grid/backend/backend.dockerfile +++ b/packages/grid/backend/backend.dockerfile @@ -1,84 +1,85 @@ -# ==================== [BASE] Setup deps + rootless user ==================== # - -FROM cgr.dev/chainguard/wolfi-base as base - ARG PYTHON_VERSION="3.11" -ENV TZ=Etc/UTC +ARG TZ="Etc/UTC" +ARG SYFT_WORKDIR="/home/nonroot/app" -RUN --mount=type=cache,target=/var/cache/apk \ - apk update && \ - apk add build-base gcc python-$PYTHON_VERSION-dev bash tzdata python-$PYTHON_VERSION py$PYTHON_VERSION-pip && \ - ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone +# ==================== [BUILD STEP] Build Base + rootless user ==================== # -ARG HOME=/home/nonroot -ARG NONROOT=nonroot:nonroot +FROM cgr.dev/chainguard/wolfi-base as python_dev -# use wolfi-base provided rootless user -USER nonroot +ARG PYTHON_VERSION +ARG TZ -ENV PATH=$PATH:$HOME/.local/bin - -# ==================== [BUILD CACHE] Jupyterlab Cache ==================== # +# Setup Python DEV +RUN apk update && \ + apk add build-base gcc tzdata python-$PYTHON_VERSION-dev py$PYTHON_VERSION-pip && \ + ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone -FROM base as jupyter +# # ==================== [BUILD STEP] Build+Install Jupyterlab ==================== # -RUN --mount=type=cache,target=$HOME/.cache/ \ - pip install --user jupyterlab +FROM python_dev as jupyter -# ==================== [BUILD CACHE] Syft deps changes ==================== # +USER nonroot +WORKDIR /home/nonroot +ENV PATH=$PATH:/home/nonroot/.local/bin -FROM base as syft_deps_changes +RUN --mount=type=cache,target=/home/nonroot/.cache/,rw,uid=65532 \ + pip install --user jupyterlab==4.0.7 -WORKDIR $HOME/ +# ==================== [BUILD STEP] Build+Install ML Dependency ==================== # -COPY --chown=$NONROOT syft/setup.cfg $HOME/setup.cfg +FROM python_dev as syft_deps -# setup.cfg might change, but ML dependencies may not -# so we take a snapshot of everything in DOCKER:CACHED -RUN awk '/DOCKER:CACHED:START/,/DOCKER:CACHED:END/ {if (!/#/) print}' $HOME/setup.cfg | sed 's/ //g' | sort > $HOME/requirements.txt; +ARG SYFT_WORKDIR -# ==================== [BUILD CACHE] Syft ML Cache ==================== # +USER nonroot +WORKDIR $SYFT_WORKDIR +ENV PATH=$PATH:/home/nonroot/.local/bin -FROM base as syft_cached_deps +# copy skeleton to do package install +COPY --chown=nonroot:nonroot syft/setup.py ./syft/setup.py +COPY --chown=nonroot:nonroot syft/setup.cfg ./syft/setup.cfg +COPY --chown=nonroot:nonroot syft/pyproject.toml ./syft/pyproject.toml +COPY --chown=nonroot:nonroot syft/MANIFEST.in ./syft/MANIFEST.in +COPY --chown=nonroot:nonroot syft/src/syft/VERSION ./syft/src/syft/VERSION +COPY --chown=nonroot:nonroot syft/src/syft/capnp ./syft/src/syft/capnp + +# Install Syft & dependencies +RUN --mount=type=cache,target=/home/nonroot/.cache/,rw,uid=65532 \ + pip install --user pip-autoremove && \ + pip install --user -e ./syft/ && \ + pip-autoremove ansible ansible-core -y -COPY --from=syft_deps_changes $HOME/requirements.txt $HOME/requirements.txt +# ==================== [MAIN] Setup Syft ==================== # -# Should hopefully be run only when req.txt changes -RUN --mount=type=cache,target=$HOME/.cache/ \ - pip install --user -r $HOME/requirements.txt && \ - rm $HOME/requirements.txt +FROM cgr.dev/chainguard/wolfi-base as python_prod -# ==================== [MAIN] Setup Syft ==================== # +# inherit from global +ARG PYTHON_VERSION +ARG TZ +ARG SYFT_WORKDIR -FROM syft_cached_deps +# Setup Python +RUN apk update && \ + apk add --no-cache tzdata bash python-$PYTHON_VERSION py$PYTHON_VERSION-pip && \ + ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone && \ + rm -rf /var/cache/apk/* -# Copy pre-built jupyterlab -COPY --from=jupyter --chown=$NONROOT $HOME/.local $HOME/.local +USER nonroot +WORKDIR $SYFT_WORKDIR -WORKDIR $HOME/app -ENV PYTHONPATH=$HOME/app -ENV APPDIR=$HOME/app +# Update environment variables +ENV PATH=$PATH:/home/nonroot/.local/bin \ + PYTHONPATH=$SYFT_WORKDIR \ + APPDIR=$SYFT_WORKDIR -# copy skeleton to do package install -COPY --chown=$NONROOT syft/setup.py ./syft/setup.py -COPY --chown=$NONROOT syft/setup.cfg ./syft/setup.cfg -COPY --chown=$NONROOT syft/pyproject.toml ./syft/pyproject.toml -COPY --chown=$NONROOT syft/MANIFEST.in ./syft/MANIFEST.in -COPY --chown=$NONROOT syft/src/syft/VERSION ./syft/src/syft/VERSION -COPY --chown=$NONROOT syft/src/syft/capnp ./syft/src/syft/capnp - -# Install Syft -RUN pip install --user pip-autoremove && \ - pip install --user -e ./syft/ && \ - pip-autoremove ansible ansible-core -y && \ - pip uninstall pip-autoremove -y && \ - rm -rf $HOME/.cache/ +# Copy pre-built jupyterlab, syft dependencies +COPY --chown=nonroot:nonroot --from=syft_deps /home/nonroot/.local /home/nonroot/.local +COPY --chown=nonroot:nonroot --from=jupyter /home/nonroot/.local /home/nonroot/.local # copy grid -COPY --chown=$NONROOT grid/backend . +COPY --chown=nonroot:nonroot grid/backend/grid ./grid -# copy any changed source -COPY --chown=$NONROOT syft/src ./syft/src +# copy syft +COPY --chown=nonroot:nonroot syft/ ./syft/ -# change to worker-start.sh or start-reload.sh as needed CMD ["bash", "./grid/start.sh"] From 2b4030ec6bdb04a1fa27d7d004f8669ddd447596 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Fri, 20 Oct 2023 16:52:41 +0000 Subject: [PATCH 06/19] [grid] fix potential deps conflicts --- packages/grid/backend/backend.dockerfile | 46 ++++++++++-------------- 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/packages/grid/backend/backend.dockerfile b/packages/grid/backend/backend.dockerfile index 0eaab0edf2f..287798720ee 100644 --- a/packages/grid/backend/backend.dockerfile +++ b/packages/grid/backend/backend.dockerfile @@ -1,8 +1,9 @@ ARG PYTHON_VERSION="3.11" ARG TZ="Etc/UTC" ARG SYFT_WORKDIR="/home/nonroot/app" +ARG NONROOT_UG="nonroot:nonroot" -# ==================== [BUILD STEP] Build Base + rootless user ==================== # +# ==================== [BUILD STEP] Python Dev Base ==================== # FROM cgr.dev/chainguard/wolfi-base as python_dev @@ -14,42 +15,31 @@ RUN apk update && \ apk add build-base gcc tzdata python-$PYTHON_VERSION-dev py$PYTHON_VERSION-pip && \ ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone -# # ==================== [BUILD STEP] Build+Install Jupyterlab ==================== # - -FROM python_dev as jupyter - -USER nonroot -WORKDIR /home/nonroot -ENV PATH=$PATH:/home/nonroot/.local/bin - -RUN --mount=type=cache,target=/home/nonroot/.cache/,rw,uid=65532 \ - pip install --user jupyterlab==4.0.7 - -# ==================== [BUILD STEP] Build+Install ML Dependency ==================== # +# ==================== [BUILD STEP] Install Syft Dependency ==================== # FROM python_dev as syft_deps ARG SYFT_WORKDIR +ARG NONROOT_UG USER nonroot WORKDIR $SYFT_WORKDIR ENV PATH=$PATH:/home/nonroot/.local/bin # copy skeleton to do package install -COPY --chown=nonroot:nonroot syft/setup.py ./syft/setup.py -COPY --chown=nonroot:nonroot syft/setup.cfg ./syft/setup.cfg -COPY --chown=nonroot:nonroot syft/pyproject.toml ./syft/pyproject.toml -COPY --chown=nonroot:nonroot syft/MANIFEST.in ./syft/MANIFEST.in -COPY --chown=nonroot:nonroot syft/src/syft/VERSION ./syft/src/syft/VERSION -COPY --chown=nonroot:nonroot syft/src/syft/capnp ./syft/src/syft/capnp - -# Install Syft & dependencies +COPY --chown=$NONROOT_UG syft/setup.py ./syft/setup.py +COPY --chown=$NONROOT_UG syft/setup.cfg ./syft/setup.cfg +COPY --chown=$NONROOT_UG syft/pyproject.toml ./syft/pyproject.toml +COPY --chown=$NONROOT_UG syft/MANIFEST.in ./syft/MANIFEST.in +COPY --chown=$NONROOT_UG syft/src/syft/VERSION ./syft/src/syft/VERSION +COPY --chown=$NONROOT_UG syft/src/syft/capnp ./syft/src/syft/capnp + +# Install all dependencies together here to avoid any version conflicts across pkgs RUN --mount=type=cache,target=/home/nonroot/.cache/,rw,uid=65532 \ - pip install --user pip-autoremove && \ - pip install --user -e ./syft/ && \ + pip install --user pip-autoremove jupyterlab==4.0.7 -e ./syft/ && \ pip-autoremove ansible ansible-core -y -# ==================== [MAIN] Setup Syft ==================== # +# ==================== [Final] Setup Syft Server ==================== # FROM cgr.dev/chainguard/wolfi-base as python_prod @@ -57,6 +47,7 @@ FROM cgr.dev/chainguard/wolfi-base as python_prod ARG PYTHON_VERSION ARG TZ ARG SYFT_WORKDIR +ARG NONROOT_UG # Setup Python RUN apk update && \ @@ -73,13 +64,12 @@ ENV PATH=$PATH:/home/nonroot/.local/bin \ APPDIR=$SYFT_WORKDIR # Copy pre-built jupyterlab, syft dependencies -COPY --chown=nonroot:nonroot --from=syft_deps /home/nonroot/.local /home/nonroot/.local -COPY --chown=nonroot:nonroot --from=jupyter /home/nonroot/.local /home/nonroot/.local +COPY --chown=$NONROOT_UG --from=syft_deps /home/nonroot/.local /home/nonroot/.local # copy grid -COPY --chown=nonroot:nonroot grid/backend/grid ./grid +COPY --chown=$NONROOT_UG grid/backend/grid ./grid # copy syft -COPY --chown=nonroot:nonroot syft/ ./syft/ +COPY --chown=$NONROOT_UG syft/ ./syft/ CMD ["bash", "./grid/start.sh"] From 1b3841623fa50aa432e7f112258c27c68567f9aa Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Fri, 20 Oct 2023 18:44:29 +0000 Subject: [PATCH 07/19] [grid] fix incorrect backend target name --- packages/grid/backend/backend.dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/grid/backend/backend.dockerfile b/packages/grid/backend/backend.dockerfile index 287798720ee..c6b867774c4 100644 --- a/packages/grid/backend/backend.dockerfile +++ b/packages/grid/backend/backend.dockerfile @@ -41,7 +41,7 @@ RUN --mount=type=cache,target=/home/nonroot/.cache/,rw,uid=65532 \ # ==================== [Final] Setup Syft Server ==================== # -FROM cgr.dev/chainguard/wolfi-base as python_prod +FROM cgr.dev/chainguard/wolfi-base as backend # inherit from global ARG PYTHON_VERSION From c241c37a228d56dbac1aadf7051141db74296f22 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Mon, 23 Oct 2023 06:54:16 +0000 Subject: [PATCH 08/19] [grid] rename env BACKEND_STORAGE_PATH to CREDENTIALS_VOLUME --- packages/grid/default.env | 3 ++- packages/grid/devspace.yaml | 2 -- packages/grid/docker-compose.yml | 5 ++--- .../grid/podman/podman-kube/podman-syft-kube-config.yaml | 2 +- packages/hagrid/hagrid/cli.py | 6 +++--- 5 files changed, 8 insertions(+), 10 deletions(-) diff --git a/packages/grid/default.env b/packages/grid/default.env index 1c91483c1ea..759bab8ca01 100644 --- a/packages/grid/default.env +++ b/packages/grid/default.env @@ -9,6 +9,7 @@ HTTPS_PORT=443 HEADSCALE_PORT=8080 NETWORK_NAME=omnet RELEASE=production +CREDENTIALS_VOLUME=credentials-data # tls IGNORE_TLS_ERRORS=False @@ -109,4 +110,4 @@ OBLV_KEY_PATH="~/.oblv" DOMAIN_CONNECTION_PORT=3030 # Registation -ENABLE_SIGNUP=False \ No newline at end of file +ENABLE_SIGNUP=False diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index e12e50d6e7c..fec0e1e854f 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -182,8 +182,6 @@ deployments: value: "${DEFAULT_ROOT_EMAIL}" - name: DEFAULT_ROOT_PASSWORD value: "${DEFAULT_ROOT_PASSWORD}" - - name: BACKEND_STORAGE_PATH - value: "/storage" volumes: - name: credentials-data size: "100Mi" diff --git a/packages/grid/docker-compose.yml b/packages/grid/docker-compose.yml index c02868028e9..294edc2bdbb 100644 --- a/packages/grid/docker-compose.yml +++ b/packages/grid/docker-compose.yml @@ -103,6 +103,7 @@ services: backend: restart: always image: "${DOCKER_IMAGE_BACKEND?Variable not set}:${VERSION-latest}" + user: 65532:65532 depends_on: - proxy env_file: @@ -130,11 +131,9 @@ services: - ENABLE_OBLV=${ENABLE_OBLV} - DEFAULT_ROOT_EMAIL=${DEFAULT_ROOT_EMAIL} - DEFAULT_ROOT_PASSWORD=${DEFAULT_ROOT_PASSWORD} - - BACKEND_STORAGE_PATH=${BACKEND_STORAGE_PATH} - command: "/app/grid/start.sh" network_mode: service:proxy volumes: - - ${BACKEND_STORAGE_PATH}:/storage + - ${CREDENTIALS_VOLUME}:/storage stdin_open: true tty: true labels: diff --git a/packages/grid/podman/podman-kube/podman-syft-kube-config.yaml b/packages/grid/podman/podman-kube/podman-syft-kube-config.yaml index 1684f1c3dfe..33690b5128f 100644 --- a/packages/grid/podman/podman-kube/podman-syft-kube-config.yaml +++ b/packages/grid/podman/podman-kube/podman-syft-kube-config.yaml @@ -102,7 +102,7 @@ data: # Syft SYFT_TUTORIAL_MODE: False - BACKEND_STORAGE_PATH: credentials-data + CREDENTIALS_VOLUME: credentials-data NODE_SIDE_TYPE: high # Worker diff --git a/packages/hagrid/hagrid/cli.py b/packages/hagrid/hagrid/cli.py index 35852e6521f..453086e45ea 100644 --- a/packages/hagrid/hagrid/cli.py +++ b/packages/hagrid/hagrid/cli.py @@ -2174,7 +2174,7 @@ def create_launch_docker_cmd( ) # use a docker volume - backend_storage = "credentials-data" + host_path = "credentials-data" # in development use a folder mount if kwargs.get("release", "") == "development": @@ -2182,7 +2182,7 @@ def create_launch_docker_cmd( # if EDITABLE_MODE: # RELATIVE_PATH = "../" # we might need to change this for the hagrid template mode - backend_storage = f"{RELATIVE_PATH}./backend/grid/storage/{snake_name}" + host_path = f"{RELATIVE_PATH}./backend/grid/storage/{snake_name}" envs = { "RELEASE": "production", @@ -2202,7 +2202,7 @@ def create_launch_docker_cmd( generate_sec_random_password(length=48, special_chars=False) ), "ENABLE_OBLV": str(enable_oblv).lower(), - "BACKEND_STORAGE_PATH": backend_storage, + "CREDENTIALS_VOLUME": host_path, "NODE_SIDE_TYPE": kwargs["node_side_type"], } From 7d78fec94cea81aa1d988687d8b012cbd362434c Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Mon, 23 Oct 2023 06:56:25 +0000 Subject: [PATCH 09/19] [grid] use rootless /home/azureuser/data for storage --- packages/grid/backend/backend.dockerfile | 25 +++++++++++++++++-- packages/grid/backend/grid/bootstrap.py | 5 ++-- packages/grid/backend/grid/core/config.py | 5 +++- packages/grid/backend/grid/core/node.py | 2 +- packages/grid/backend/grid/start.sh | 4 +++ packages/grid/devspace.yaml | 9 ++++++- packages/grid/docker-compose.dev.yml | 14 +++++------ packages/grid/docker-compose.yml | 5 ++-- .../src/syft/store/sqlite_document_store.py | 5 ++++ 9 files changed, 57 insertions(+), 17 deletions(-) diff --git a/packages/grid/backend/backend.dockerfile b/packages/grid/backend/backend.dockerfile index c6b867774c4..cbc7ef37c41 100644 --- a/packages/grid/backend/backend.dockerfile +++ b/packages/grid/backend/backend.dockerfile @@ -53,7 +53,9 @@ ARG NONROOT_UG RUN apk update && \ apk add --no-cache tzdata bash python-$PYTHON_VERSION py$PYTHON_VERSION-pip && \ ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone && \ - rm -rf /var/cache/apk/* + rm -rf /var/cache/apk/* && \ + mkdir -p /var/log/pygrid RUN mkdir -p /home/nonroot/data/creds /home/nonroot/data/db && \ + chown -R $NONROOT_UG /var/log/pygrid /home/nonroot/data USER nonroot WORKDIR $SYFT_WORKDIR @@ -61,7 +63,26 @@ WORKDIR $SYFT_WORKDIR # Update environment variables ENV PATH=$PATH:/home/nonroot/.local/bin \ PYTHONPATH=$SYFT_WORKDIR \ - APPDIR=$SYFT_WORKDIR + APPDIR=$SYFT_WORKDIR \ + NODE_NAME="default_node_name" \ + NODE_TYPE="domain" \ + SERVICE_NAME="backend" \ + RELEASE="production" \ + DEV_MODE="False" \ + CONTAINER_HOST="docker" \ + PORT=80\ + HTTP_PORT=80 \ + HTTPS_PORT=443 \ + DOMAIN_CONNECTION_PORT=3030 \ + IGNORE_TLS_ERRORS="False" \ + DEFAULT_ROOT_EMAIL="info@openmined.org" \ + DEFAULT_ROOT_PASSWORD="changethis" \ + STACK_API_KEY="changeme" \ + MONGO_HOST="localhost" \ + MONGO_PORT="27017" \ + MONGO_USERNAME="root" \ + MONGO_PASSWORD="example" \ + CREDENTIALS_PATH="/home/nonroot/data/creds/credentials.json" # Copy pre-built jupyterlab, syft dependencies COPY --chown=$NONROOT_UG --from=syft_deps /home/nonroot/.local /home/nonroot/.local diff --git a/packages/grid/backend/grid/bootstrap.py b/packages/grid/backend/grid/bootstrap.py index 24eef12b189..1f4b0fb8df8 100644 --- a/packages/grid/backend/grid/bootstrap.py +++ b/packages/grid/backend/grid/bootstrap.py @@ -29,7 +29,8 @@ def get_env(key: str, default: str = "") -> Optional[str]: return None -CREDENTIALS_PATH = str(get_env("CREDENTIALS_PATH", "./storage/credentials.json")) +DEFAULT_CREDENTIALS_PATH = os.path.expandvars("$HOME/data/creds/credentials.json") +CREDENTIALS_PATH = str(get_env("CREDENTIALS_PATH", DEFAULT_CREDENTIALS_PATH)) NODE_PRIVATE_KEY = "NODE_PRIVATE_KEY" NODE_UID = "NODE_UID" @@ -59,7 +60,7 @@ def save_credential(key: str, value: str) -> str: try: dirname = os.path.dirname(CREDENTIALS_PATH) if not os.path.exists(dirname): - os.mkdir(dirname) + os.makedirs(dirname, exist_ok=True) with open(CREDENTIALS_PATH, "w") as f: f.write(f"{json.dumps(credentials)}") except Exception as e: diff --git a/packages/grid/backend/grid/core/config.py b/packages/grid/backend/grid/core/config.py index 0b9948ca961..df140cc39aa 100644 --- a/packages/grid/backend/grid/core/config.py +++ b/packages/grid/backend/grid/core/config.py @@ -59,7 +59,9 @@ def get_project_name(cls, v: Optional[str], values: Dict[str, Any]) -> str: return v EMAIL_RESET_TOKEN_EXPIRE_HOURS: int = 48 - EMAIL_TEMPLATES_DIR: str = "/app/grid/email-templates/build" + EMAIL_TEMPLATES_DIR: str = os.path.expandvars( + "$HOME/app/grid/email-templates/build" + ) EMAILS_ENABLED: bool = False @validator("EMAILS_ENABLED", pre=True) @@ -106,6 +108,7 @@ def get_emails_enabled(cls, v: bool, values: Dict[str, Any]) -> bool: MONGO_PORT: int = int(os.getenv("MONGO_PORT", 0)) MONGO_USERNAME: str = str(os.getenv("MONGO_USERNAME", "")) MONGO_PASSWORD: str = str(os.getenv("MONGO_PASSWORD", "")) + SQLITE_PATH: str = os.path.expandvars("$HOME/data/db/") TEST_MODE: bool = ( True if os.getenv("TEST_MODE", "false").lower() == "true" else False diff --git a/packages/grid/backend/grid/core/node.py b/packages/grid/backend/grid/core/node.py index d8ca11abc32..8051601d043 100644 --- a/packages/grid/backend/grid/core/node.py +++ b/packages/grid/backend/grid/core/node.py @@ -26,7 +26,7 @@ mongo_store_config = MongoStoreConfig(client_config=mongo_client_config) -client_config = SQLiteStoreClientConfig(path="./storage/") +client_config = SQLiteStoreClientConfig(path=settings.SQLITE_PATH) sql_store_config = SQLiteStoreConfig(client_config=client_config) node_type = get_node_type() diff --git a/packages/grid/backend/grid/start.sh b/packages/grid/backend/grid/start.sh index a25ba9629fc..8e2a36bbe95 100755 --- a/packages/grid/backend/grid/start.sh +++ b/packages/grid/backend/grid/start.sh @@ -31,5 +31,9 @@ export NODE_UID=$NODE_UID export NODE_PRIVATE_KEY=$NODE_PRIVATE_KEY export NODE_TYPE=$NODE_TYPE +# For debugging permissions +id +ls -lisa /home/nonroot/data/ + # export GEVENT_MONKEYPATCH="True" exec uvicorn $RELOAD --host $HOST --port $PORT --log-level $LOG_LEVEL "$APP_MODULE" diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index fec0e1e854f..1fe482bea37 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -125,10 +125,17 @@ deployments: name: component-chart repo: https://charts.devspace.sh values: + # PodSecurityContext + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + fsGroupChangePolicy: "Always" containers: - image: "${CONTAINER_REGISTRY}/${DOCKER_IMAGE_BACKEND}:${VERSION}" volumeMounts: - - containerPath: /storage + - containerPath: /home/nonroot/data/creds/ volume: name: credentials-data subPath: /credentials-data diff --git a/packages/grid/docker-compose.dev.yml b/packages/grid/docker-compose.dev.yml index 3faef69b269..89acc31760e 100644 --- a/packages/grid/docker-compose.dev.yml +++ b/packages/grid/docker-compose.dev.yml @@ -41,9 +41,9 @@ services: backend: volumes: - - ${RELATIVE_PATH}./backend/grid:/app/grid - - ${RELATIVE_PATH}../syft:/app/syft - - ${RELATIVE_PATH}./data/package-cache:/root/.cache + - ${RELATIVE_PATH}./backend/grid:/home/nonroot/app/grid + - ${RELATIVE_PATH}../syft:/home/nonroot/app/syft + - ${RELATIVE_PATH}./data/package-cache:/home/nonroot/.cache environment: - DEV_MODE=True stdin_open: true @@ -51,16 +51,16 @@ services: # backend_stream: # volumes: - # - ${RELATIVE_PATH}./backend/grid:/app/grid - # - ${RELATIVE_PATH}../syft:/app/syft + # - ${RELATIVE_PATH}./backend/grid:/home/nonroot/app/grid + # - ${RELATIVE_PATH}../syft:/home/nonroot/app/syft # - ${RELATIVE_PATH}./data/package-cache:/root/.cache # environment: # - DEV_MODE=True # celeryworker: # volumes: - # - ${RELATIVE_PATH}./backend/grid:/app/grid - # - ${RELATIVE_PATH}../syft/:/app/syft + # - ${RELATIVE_PATH}./backend/grid:/home/nonroot/app/grid + # - ${RELATIVE_PATH}../syft/:/home/nonroot/app/syft # - ${RELATIVE_PATH}./data/package-cache:/root/.cache # environment: # - DEV_MODE=True diff --git a/packages/grid/docker-compose.yml b/packages/grid/docker-compose.yml index 294edc2bdbb..8a61fff7b95 100644 --- a/packages/grid/docker-compose.yml +++ b/packages/grid/docker-compose.yml @@ -133,7 +133,7 @@ services: - DEFAULT_ROOT_PASSWORD=${DEFAULT_ROOT_PASSWORD} network_mode: service:proxy volumes: - - ${CREDENTIALS_VOLUME}:/storage + - ${CREDENTIALS_VOLUME}:/home/nonroot/data/creds/ stdin_open: true tty: true labels: @@ -167,10 +167,9 @@ services: # - DEV_MODE=${DEV_MODE} # - DOMAIN_CONNECTION_PORT=${DOMAIN_CONNECTION_PORT} # - ENABLE_OBLV=${ENABLE_OBLV} - # command: "/app/grid/start.sh" # network_mode: service:proxy # volumes: - # - credentials-data:/storage + # - credentials-data:/home/nonroot/data/creds/ # celeryworker: # restart: always diff --git a/packages/syft/src/syft/store/sqlite_document_store.py b/packages/syft/src/syft/store/sqlite_document_store.py index 856f604da09..0e586297e38 100644 --- a/packages/syft/src/syft/store/sqlite_document_store.py +++ b/packages/syft/src/syft/store/sqlite_document_store.py @@ -84,6 +84,11 @@ def _connect(self) -> None: # that different connections are used in each thread. By using a dict for the # _db and _cur we can ensure they are never shared self.file_path = self.store_config.client_config.file_path + + path = Path(self.file_path) + if not path.exists(): + path.parent.mkdir(parents=True, exist_ok=True) + self._db[thread_ident()] = sqlite3.connect( self.file_path, timeout=self.store_config.client_config.timeout, From c4684267afa964ce892fccf638647d1915c10470 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Mon, 23 Oct 2023 07:01:34 +0000 Subject: [PATCH 10/19] [grid] update helm charts --- packages/grid/helm/manifests.yaml | 23 +++++++++++-------- .../syft/templates/backend-statefulset.yaml | 15 +++++++----- .../syft/templates/frontend-deployment.yaml | 4 ++-- .../syft/templates/mongo-statefulset.yaml | 2 +- .../syft/templates/seaweedfs-statefulset.yaml | 2 +- tox.ini | 2 +- 6 files changed, 27 insertions(+), 21 deletions(-) diff --git a/packages/grid/helm/manifests.yaml b/packages/grid/helm/manifests.yaml index cc20c5b278d..3c2d797de91 100644 --- a/packages/grid/helm/manifests.yaml +++ b/packages/grid/helm/manifests.yaml @@ -65,7 +65,7 @@ spec: - name: RELEASE value: production - name: VERSION - value: 0.8.2-beta.33 + value: 0.8.2-beta.40 - name: VERSION_HASH value: unknown - name: NODE_TYPE @@ -100,10 +100,8 @@ spec: value: info@openmined.org - name: DEFAULT_ROOT_PASSWORD value: changethis - - name: BACKEND_STORAGE_PATH - value: /storage envFrom: null - image: docker.io/openmined/grid-backend:0.8.2-beta.33 + image: docker.io/openmined/grid-backend:0.8.2-beta.40 lifecycle: null livenessProbe: null name: container-0 @@ -112,7 +110,7 @@ spec: startupProbe: null volumeDevices: null volumeMounts: - - mountPath: /storage + - mountPath: /home/nonroot/data/creds/ name: credentials-data readOnly: false subPath: credentials-data @@ -125,7 +123,12 @@ spec: nodeSelector: null overhead: null readinessGates: null - securityContext: null + securityContext: + fsGroup: 65532 + fsGroupChangePolicy: Always + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 terminationGracePeriodSeconds: 5 tolerations: null topologySpreadConstraints: null @@ -220,7 +223,7 @@ spec: command: null env: - name: VERSION - value: 0.8.2-beta.33 + value: 0.8.2-beta.40 - name: VERSION_HASH value: unknown - name: NODE_TYPE @@ -228,7 +231,7 @@ spec: - name: NEXT_PUBLIC_API_URL value: ${NEXT_PUBLIC_API_URL} envFrom: null - image: docker.io/openmined/grid-frontend:0.8.2-beta.33 + image: docker.io/openmined/grid-frontend:0.8.2-beta.40 lifecycle: null livenessProbe: null name: container-0 @@ -363,7 +366,7 @@ spec: - name: MONGO_INITDB_ROOT_PASSWORD value: example envFrom: null - image: mongo:7.0.0 + image: mongo:7.0.2 lifecycle: null livenessProbe: null name: container-0 @@ -582,7 +585,7 @@ spec: - name: S3_PORT value: '8333' envFrom: null - image: chrislusf/seaweedfs:3.55 + image: chrislusf/seaweedfs:3.57 lifecycle: null livenessProbe: null name: container-0 diff --git a/packages/grid/helm/syft/templates/backend-statefulset.yaml b/packages/grid/helm/syft/templates/backend-statefulset.yaml index 4635147d65f..a02ef498af1 100644 --- a/packages/grid/helm/syft/templates/backend-statefulset.yaml +++ b/packages/grid/helm/syft/templates/backend-statefulset.yaml @@ -44,7 +44,7 @@ spec: - name: RELEASE value: production - name: VERSION - value: 0.8.2-beta.33 + value: 0.8.2-beta.40 - name: VERSION_HASH value: {{ .Values.node.settings.versionHash }} - name: NODE_TYPE @@ -79,10 +79,8 @@ spec: value: {{ .Values.secrets.syft.defaultRootEmail }} - name: DEFAULT_ROOT_PASSWORD value: {{ .Values.secrets.syft.defaultRootPassword }} - - name: BACKEND_STORAGE_PATH - value: /storage envFrom: null - image: docker.io/openmined/grid-backend:0.8.2-beta.33 + image: docker.io/openmined/grid-backend:0.8.2-beta.40 lifecycle: null livenessProbe: null name: container-0 @@ -91,7 +89,7 @@ spec: startupProbe: null volumeDevices: null volumeMounts: - - mountPath: /storage + - mountPath: /home/nonroot/data/creds/ name: credentials-data readOnly: false subPath: credentials-data @@ -104,7 +102,12 @@ spec: nodeSelector: null overhead: null readinessGates: null - securityContext: null + securityContext: + fsGroup: 65532 + fsGroupChangePolicy: Always + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 terminationGracePeriodSeconds: 5 tolerations: null topologySpreadConstraints: null diff --git a/packages/grid/helm/syft/templates/frontend-deployment.yaml b/packages/grid/helm/syft/templates/frontend-deployment.yaml index e0b2ddb8346..4966d834e01 100644 --- a/packages/grid/helm/syft/templates/frontend-deployment.yaml +++ b/packages/grid/helm/syft/templates/frontend-deployment.yaml @@ -32,7 +32,7 @@ spec: command: null env: - name: VERSION - value: 0.8.2-beta.33 + value: 0.8.2-beta.40 - name: VERSION_HASH value: {{ .Values.node.settings.versionHash }} - name: NODE_TYPE @@ -40,7 +40,7 @@ spec: - name: NEXT_PUBLIC_API_URL value: ${NEXT_PUBLIC_API_URL} envFrom: null - image: docker.io/openmined/grid-frontend:0.8.2-beta.33 + image: docker.io/openmined/grid-frontend:0.8.2-beta.40 lifecycle: null livenessProbe: null name: container-0 diff --git a/packages/grid/helm/syft/templates/mongo-statefulset.yaml b/packages/grid/helm/syft/templates/mongo-statefulset.yaml index 9da6335d889..d25e01f446b 100644 --- a/packages/grid/helm/syft/templates/mongo-statefulset.yaml +++ b/packages/grid/helm/syft/templates/mongo-statefulset.yaml @@ -36,7 +36,7 @@ spec: - name: MONGO_INITDB_ROOT_PASSWORD value: {{ .Values.secrets.db.mongo.mongoInitdbRootPassword }} envFrom: null - image: mongo:7.0.0 + image: mongo:7.0.2 lifecycle: null livenessProbe: null name: container-0 diff --git a/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml b/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml index c6d82d0234c..7927ac34acc 100644 --- a/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml +++ b/packages/grid/helm/syft/templates/seaweedfs-statefulset.yaml @@ -43,7 +43,7 @@ spec: - name: S3_PORT value: '8333' envFrom: null - image: chrislusf/seaweedfs:3.55 + image: chrislusf/seaweedfs:3.57 lifecycle: null livenessProbe: null name: container-0 diff --git a/tox.ini b/tox.ini index 2ee59b796f2..cf93732cdaa 100644 --- a/tox.ini +++ b/tox.ini @@ -810,7 +810,7 @@ commands = bash -c 'cd packages/grid/helm && \ helm lint syft' - bash -c "k3d cluster delete build || true" + bash -c "k3d cluster delete build; docker volume rm k3d-build-images --force; echo Done" [testenv:syft.package.helm] description = Package Helm Chart for Kubernetes From 644d364b4806d84b732a0b8da94a2bcb8380835d Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Mon, 23 Oct 2023 08:25:06 +0000 Subject: [PATCH 11/19] [grid] do not log private key --- packages/grid/worker/start.sh | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/packages/grid/worker/start.sh b/packages/grid/worker/start.sh index a6e8f004785..b27c07e7c6b 100755 --- a/packages/grid/worker/start.sh +++ b/packages/grid/worker/start.sh @@ -14,17 +14,13 @@ fi export RUST_BACKTRACE=$RUST_BACKTRACE set +e -NODE_PRIVATE_KEY=$(python bootstrap.py --private_key) -NODE_UID=$(python bootstrap.py --uid) +export NODE_PRIVATE_KEY=$(python bootstrap.py --private_key) +export NODE_UID=$(python bootstrap.py --uid) set -e -echo "NODE_PRIVATE_KEY=$NODE_PRIVATE_KEY" echo "NODE_UID=$NODE_UID" echo "NODE_TYPE=$NODE_TYPE" -export NODE_UID=$NODE_UID -export NODE_PRIVATE_KEY=$NODE_PRIVATE_KEY - APP_MODULE=worker:app LOG_LEVEL=${LOG_LEVEL:-info} HOST=${HOST:-0.0.0.0} From fd9cedf5d65906fa5b95538fb30dce931bd61b81 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Mon, 23 Oct 2023 08:25:48 +0000 Subject: [PATCH 12/19] [grid] fix mkdir --- packages/grid/backend/backend.dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/grid/backend/backend.dockerfile b/packages/grid/backend/backend.dockerfile index cbc7ef37c41..0e0dce356f7 100644 --- a/packages/grid/backend/backend.dockerfile +++ b/packages/grid/backend/backend.dockerfile @@ -54,7 +54,7 @@ RUN apk update && \ apk add --no-cache tzdata bash python-$PYTHON_VERSION py$PYTHON_VERSION-pip && \ ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone && \ rm -rf /var/cache/apk/* && \ - mkdir -p /var/log/pygrid RUN mkdir -p /home/nonroot/data/creds /home/nonroot/data/db && \ + mkdir -p /var/log/pygrid /home/nonroot/data/creds /home/nonroot/data/db && \ chown -R $NONROOT_UG /var/log/pygrid /home/nonroot/data USER nonroot From 133b71f3bfffcd8a4346a6db4940312ed6e0b686 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Mon, 23 Oct 2023 09:05:17 +0000 Subject: [PATCH 13/19] revert worker changes --- packages/grid/worker/worker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/grid/worker/worker.py b/packages/grid/worker/worker.py index e2a34ea27ad..ea3b9d18315 100644 --- a/packages/grid/worker/worker.py +++ b/packages/grid/worker/worker.py @@ -34,7 +34,7 @@ worker = worker_class( name=node_name, local_db=True, - sqlite_path="./storage/", + sqlite_path="/storage/", node_type=node_type, enable_warnings=enable_warnings, node_side_type=node_side_type, From 2698c956ac9eb97f4f987a197a1a9416a37df42b Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Mon, 23 Oct 2023 10:34:02 +0000 Subject: [PATCH 14/19] [grid] fix ci error --- packages/grid/backend/grid/start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/grid/backend/grid/start.sh b/packages/grid/backend/grid/start.sh index 8e2a36bbe95..85c1ddfa9c6 100755 --- a/packages/grid/backend/grid/start.sh +++ b/packages/grid/backend/grid/start.sh @@ -15,7 +15,7 @@ if [[ ${DEV_MODE} == "True" ]]; then echo "DEV_MODE Enabled" RELOAD="--reload" - pip install -e "$APPDIR/syft[telemetry]" + pip install --user -e "$APPDIR/syft[telemetry]" fi set +e From 2a785827ec4d98d555813206e3085b3cb52a2c2c Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Tue, 24 Oct 2023 05:06:56 +0000 Subject: [PATCH 15/19] [grid] use new syftuser(uid=1000) --- packages/grid/backend/backend.dockerfile | 56 +++++++++++-------- packages/grid/backend/grid/start.sh | 15 ++--- packages/grid/devspace.yaml | 12 ++-- packages/grid/docker-compose.dev.yml | 15 ++--- packages/grid/docker-compose.yml | 6 +- packages/grid/helm/manifests.yaml | 8 +-- .../syft/templates/backend-statefulset.yaml | 8 +-- tox.ini | 4 ++ 8 files changed, 69 insertions(+), 55 deletions(-) diff --git a/packages/grid/backend/backend.dockerfile b/packages/grid/backend/backend.dockerfile index 0e0dce356f7..6d9e104a0a4 100644 --- a/packages/grid/backend/backend.dockerfile +++ b/packages/grid/backend/backend.dockerfile @@ -1,7 +1,10 @@ ARG PYTHON_VERSION="3.11" ARG TZ="Etc/UTC" -ARG SYFT_WORKDIR="/home/nonroot/app" -ARG NONROOT_UG="nonroot:nonroot" +ARG USER="syftuser" +ARG UID=1000 +ARG USER_GRP=$USER:$USER +ARG HOME="/home/$USER" +ARG SYFT_WORKDIR="$HOME/app" # ==================== [BUILD STEP] Python Dev Base ==================== # @@ -9,33 +12,39 @@ FROM cgr.dev/chainguard/wolfi-base as python_dev ARG PYTHON_VERSION ARG TZ +ARG USER +ARG UID # Setup Python DEV RUN apk update && \ apk add build-base gcc tzdata python-$PYTHON_VERSION-dev py$PYTHON_VERSION-pip && \ - ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone && \ + adduser -D -u $UID $USER # ==================== [BUILD STEP] Install Syft Dependency ==================== # FROM python_dev as syft_deps ARG SYFT_WORKDIR -ARG NONROOT_UG +ARG USER_GRP +ARG USER +ARG HOME +ARG UID -USER nonroot +USER $USER WORKDIR $SYFT_WORKDIR -ENV PATH=$PATH:/home/nonroot/.local/bin +ENV PATH=$PATH:$HOME/.local/bin # copy skeleton to do package install -COPY --chown=$NONROOT_UG syft/setup.py ./syft/setup.py -COPY --chown=$NONROOT_UG syft/setup.cfg ./syft/setup.cfg -COPY --chown=$NONROOT_UG syft/pyproject.toml ./syft/pyproject.toml -COPY --chown=$NONROOT_UG syft/MANIFEST.in ./syft/MANIFEST.in -COPY --chown=$NONROOT_UG syft/src/syft/VERSION ./syft/src/syft/VERSION -COPY --chown=$NONROOT_UG syft/src/syft/capnp ./syft/src/syft/capnp +COPY --chown=$USER_GRP syft/setup.py ./syft/setup.py +COPY --chown=$USER_GRP syft/setup.cfg ./syft/setup.cfg +COPY --chown=$USER_GRP syft/pyproject.toml ./syft/pyproject.toml +COPY --chown=$USER_GRP syft/MANIFEST.in ./syft/MANIFEST.in +COPY --chown=$USER_GRP syft/src/syft/VERSION ./syft/src/syft/VERSION +COPY --chown=$USER_GRP syft/src/syft/capnp ./syft/src/syft/capnp # Install all dependencies together here to avoid any version conflicts across pkgs -RUN --mount=type=cache,target=/home/nonroot/.cache/,rw,uid=65532 \ +RUN --mount=type=cache,target=$HOME/.cache/,rw,uid=$UID \ pip install --user pip-autoremove jupyterlab==4.0.7 -e ./syft/ && \ pip-autoremove ansible ansible-core -y @@ -47,21 +56,24 @@ FROM cgr.dev/chainguard/wolfi-base as backend ARG PYTHON_VERSION ARG TZ ARG SYFT_WORKDIR -ARG NONROOT_UG +ARG USER_GRP +ARG USER +ARG HOME # Setup Python RUN apk update && \ apk add --no-cache tzdata bash python-$PYTHON_VERSION py$PYTHON_VERSION-pip && \ ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone && \ rm -rf /var/cache/apk/* && \ - mkdir -p /var/log/pygrid /home/nonroot/data/creds /home/nonroot/data/db && \ - chown -R $NONROOT_UG /var/log/pygrid /home/nonroot/data + adduser -D -u 1000 $USER && \ + mkdir -p /var/log/pygrid $HOME/data/creds $HOME/data/db $HOME/.cache $HOME/.local && \ + chown -R $USER_GRP /var/log/pygrid $HOME/ -USER nonroot +USER $USER WORKDIR $SYFT_WORKDIR # Update environment variables -ENV PATH=$PATH:/home/nonroot/.local/bin \ +ENV PATH=$PATH:$HOME/.local/bin \ PYTHONPATH=$SYFT_WORKDIR \ APPDIR=$SYFT_WORKDIR \ NODE_NAME="default_node_name" \ @@ -82,15 +94,15 @@ ENV PATH=$PATH:/home/nonroot/.local/bin \ MONGO_PORT="27017" \ MONGO_USERNAME="root" \ MONGO_PASSWORD="example" \ - CREDENTIALS_PATH="/home/nonroot/data/creds/credentials.json" + CREDENTIALS_PATH="$HOME/data/creds/credentials.json" # Copy pre-built jupyterlab, syft dependencies -COPY --chown=$NONROOT_UG --from=syft_deps /home/nonroot/.local /home/nonroot/.local +COPY --chown=$USER_GRP --from=syft_deps $HOME/.local $HOME/.local # copy grid -COPY --chown=$NONROOT_UG grid/backend/grid ./grid +COPY --chown=$USER_GRP grid/backend/grid ./grid # copy syft -COPY --chown=$NONROOT_UG syft/ ./syft/ +COPY --chown=$USER_GRP syft/ ./syft/ CMD ["bash", "./grid/start.sh"] diff --git a/packages/grid/backend/grid/start.sh b/packages/grid/backend/grid/start.sh index 85c1ddfa9c6..c2bde278556 100755 --- a/packages/grid/backend/grid/start.sh +++ b/packages/grid/backend/grid/start.sh @@ -4,6 +4,10 @@ set -e echo "Running start.sh with RELEASE=${RELEASE}" export GEVENT_MONKEYPATCH="False" +# For debugging permissions +id +ls -lisa ~/data/ + APP_MODULE=grid.main:app LOG_LEVEL=${LOG_LEVEL:-info} HOST=${HOST:-0.0.0.0} @@ -19,21 +23,14 @@ then fi set +e -NODE_PRIVATE_KEY=$(python $APPDIR/grid/bootstrap.py --private_key) -NODE_UID=$(python $APPDIR/grid/bootstrap.py --uid) +export NODE_PRIVATE_KEY=$(python $APPDIR/grid/bootstrap.py --private_key) +export NODE_UID=$(python $APPDIR/grid/bootstrap.py --uid) set -e -echo "NODE_PRIVATE_KEY=$NODE_PRIVATE_KEY" echo "NODE_UID=$NODE_UID" echo "NODE_TYPE=$NODE_TYPE" -export NODE_UID=$NODE_UID -export NODE_PRIVATE_KEY=$NODE_PRIVATE_KEY export NODE_TYPE=$NODE_TYPE -# For debugging permissions -id -ls -lisa /home/nonroot/data/ - # export GEVENT_MONKEYPATCH="True" exec uvicorn $RELOAD --host $HOST --port $PORT --log-level $LOG_LEVEL "$APP_MODULE" diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index 1fe482bea37..8cac6cb13cd 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -128,14 +128,14 @@ deployments: # PodSecurityContext securityContext: runAsNonRoot: true - runAsUser: 65532 - runAsGroup: 65532 - fsGroup: 65532 + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 fsGroupChangePolicy: "Always" containers: - image: "${CONTAINER_REGISTRY}/${DOCKER_IMAGE_BACKEND}:${VERSION}" volumeMounts: - - containerPath: /home/nonroot/data/creds/ + - containerPath: /home/syftuser/data/creds/ volume: name: credentials-data subPath: /credentials-data @@ -318,8 +318,8 @@ dev: value: "True" logs: {} sync: - - path: ./backend/grid:/home/nonroot/app/grid - - path: ../syft:/home/nonroot/app/syft + - path: ./backend/grid:/home/syftuser/app/grid + - path: ../syft:/home/syftuser/app/syft profiles: - name: gateway diff --git a/packages/grid/docker-compose.dev.yml b/packages/grid/docker-compose.dev.yml index 89acc31760e..7167fce7976 100644 --- a/packages/grid/docker-compose.dev.yml +++ b/packages/grid/docker-compose.dev.yml @@ -40,10 +40,11 @@ services: - "27017" backend: + user: 1000:1000 volumes: - - ${RELATIVE_PATH}./backend/grid:/home/nonroot/app/grid - - ${RELATIVE_PATH}../syft:/home/nonroot/app/syft - - ${RELATIVE_PATH}./data/package-cache:/home/nonroot/.cache + - ${RELATIVE_PATH}./backend/grid:/home/syftuser/app/grid + - ${RELATIVE_PATH}../syft:/home/syftuser/app/syft + - ${RELATIVE_PATH}./data/package-cache:/home/syftuser/.cache environment: - DEV_MODE=True stdin_open: true @@ -51,16 +52,16 @@ services: # backend_stream: # volumes: - # - ${RELATIVE_PATH}./backend/grid:/home/nonroot/app/grid - # - ${RELATIVE_PATH}../syft:/home/nonroot/app/syft + # - ${RELATIVE_PATH}./backend/grid:/home/syftuser/app/grid + # - ${RELATIVE_PATH}../syft:/home/syftuser/app/syft # - ${RELATIVE_PATH}./data/package-cache:/root/.cache # environment: # - DEV_MODE=True # celeryworker: # volumes: - # - ${RELATIVE_PATH}./backend/grid:/home/nonroot/app/grid - # - ${RELATIVE_PATH}../syft/:/home/nonroot/app/syft + # - ${RELATIVE_PATH}./backend/grid:/home/syftuser/app/grid + # - ${RELATIVE_PATH}../syft/:/home/syftuser/app/syft # - ${RELATIVE_PATH}./data/package-cache:/root/.cache # environment: # - DEV_MODE=True diff --git a/packages/grid/docker-compose.yml b/packages/grid/docker-compose.yml index 8a61fff7b95..f3c429e044f 100644 --- a/packages/grid/docker-compose.yml +++ b/packages/grid/docker-compose.yml @@ -103,7 +103,7 @@ services: backend: restart: always image: "${DOCKER_IMAGE_BACKEND?Variable not set}:${VERSION-latest}" - user: 65532:65532 + user: 1000:1000 depends_on: - proxy env_file: @@ -133,7 +133,7 @@ services: - DEFAULT_ROOT_PASSWORD=${DEFAULT_ROOT_PASSWORD} network_mode: service:proxy volumes: - - ${CREDENTIALS_VOLUME}:/home/nonroot/data/creds/ + - ${CREDENTIALS_VOLUME}:/home/syftuser/data/creds/ stdin_open: true tty: true labels: @@ -169,7 +169,7 @@ services: # - ENABLE_OBLV=${ENABLE_OBLV} # network_mode: service:proxy # volumes: - # - credentials-data:/home/nonroot/data/creds/ + # - credentials-data:/home/syftuser/data/creds/ # celeryworker: # restart: always diff --git a/packages/grid/helm/manifests.yaml b/packages/grid/helm/manifests.yaml index 3c2d797de91..a1cda8f2025 100644 --- a/packages/grid/helm/manifests.yaml +++ b/packages/grid/helm/manifests.yaml @@ -110,7 +110,7 @@ spec: startupProbe: null volumeDevices: null volumeMounts: - - mountPath: /home/nonroot/data/creds/ + - mountPath: /home/syftuser/data/creds/ name: credentials-data readOnly: false subPath: credentials-data @@ -124,11 +124,11 @@ spec: overhead: null readinessGates: null securityContext: - fsGroup: 65532 + fsGroup: 1000 fsGroupChangePolicy: Always - runAsGroup: 65532 + runAsGroup: 1000 runAsNonRoot: true - runAsUser: 65532 + runAsUser: 1000 terminationGracePeriodSeconds: 5 tolerations: null topologySpreadConstraints: null diff --git a/packages/grid/helm/syft/templates/backend-statefulset.yaml b/packages/grid/helm/syft/templates/backend-statefulset.yaml index a02ef498af1..331295fb84f 100644 --- a/packages/grid/helm/syft/templates/backend-statefulset.yaml +++ b/packages/grid/helm/syft/templates/backend-statefulset.yaml @@ -89,7 +89,7 @@ spec: startupProbe: null volumeDevices: null volumeMounts: - - mountPath: /home/nonroot/data/creds/ + - mountPath: /home/syftuser/data/creds/ name: credentials-data readOnly: false subPath: credentials-data @@ -103,11 +103,11 @@ spec: overhead: null readinessGates: null securityContext: - fsGroup: 65532 + fsGroup: 1000 fsGroupChangePolicy: Always - runAsGroup: 65532 + runAsGroup: 1000 runAsNonRoot: true - runAsUser: 65532 + runAsUser: 1000 terminationGracePeriodSeconds: 5 tolerations: null topologySpreadConstraints: null diff --git a/tox.ini b/tox.ini index cf93732cdaa..272ce295a0b 100644 --- a/tox.ini +++ b/tox.ini @@ -273,6 +273,10 @@ commands = python -c 'import syft as sy; sy.stage_protocol_changes()' + ; Make sure that pacakge-cache is owned by the current user + ; instead of docker creating it as root + bash -c 'mkdir -p packages/grid/data/package-cache' + bash -c 'HAGRID_ART=$HAGRID_ART hagrid launch test-gateway-1 network to docker:9081 $HAGRID_FLAGS --no-health-checks --verbose --no-warnings --dev' bash -c 'HAGRID_ART=$HAGRID_ART hagrid launch test-domain-1 domain to docker:9082 $HAGRID_FLAGS --no-health-checks --enable-signup --verbose --no-warnings --dev' ; bash -c 'HAGRID_ART=$HAGRID_ART hagrid launch test-domain-2 domain to docker:9083 --headless $HAGRID_FLAGS --enable-signup --no-health-checks --verbose --no-warnings --dev' From 429bfdf02e1214bf18c91e234c7567913b2b499e Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Wed, 25 Oct 2023 16:39:21 +0000 Subject: [PATCH 16/19] [tox] print user & id --- tox.ini | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 272ce295a0b..395641e78b9 100644 --- a/tox.ini +++ b/tox.ini @@ -235,6 +235,8 @@ setenv = PYTHONIOENCODING = utf-8 PYTEST_MODULES = {env:PYTEST_MODULES:frontend network e2e security redis} commands = + bash -c "whoami; id;" + bash -c "echo Running with HAGRID_FLAGS=$HAGRID_FLAGS EMULATION=$EMULATION PYTEST_MODULES=$PYTEST_MODULES; date" ; install syft and hagrid @@ -277,8 +279,8 @@ commands = ; instead of docker creating it as root bash -c 'mkdir -p packages/grid/data/package-cache' - bash -c 'HAGRID_ART=$HAGRID_ART hagrid launch test-gateway-1 network to docker:9081 $HAGRID_FLAGS --no-health-checks --verbose --no-warnings --dev' - bash -c 'HAGRID_ART=$HAGRID_ART hagrid launch test-domain-1 domain to docker:9082 $HAGRID_FLAGS --no-health-checks --enable-signup --verbose --no-warnings --dev' + bash -c 'HAGRID_ART=$HAGRID_ART hagrid launch test-gateway-1 network to docker:9081 $HAGRID_FLAGS --no-health-checks --verbose --no-warnings --build' + bash -c 'HAGRID_ART=$HAGRID_ART hagrid launch test-domain-1 domain to docker:9082 $HAGRID_FLAGS --no-health-checks --enable-signup --verbose --no-warnings --build' ; bash -c 'HAGRID_ART=$HAGRID_ART hagrid launch test-domain-2 domain to docker:9083 --headless $HAGRID_FLAGS --enable-signup --no-health-checks --verbose --no-warnings --dev' ; wait for nodes to start From 9da1be4b82d9e4368f860cfc07e1731143514c89 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Thu, 26 Oct 2023 06:25:52 +0000 Subject: [PATCH 17/19] [tox] fix integration tests --- packages/hagrid/hagrid/cli.py | 18 +++++++----------- tox.ini | 4 ++-- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/packages/hagrid/hagrid/cli.py b/packages/hagrid/hagrid/cli.py index 453086e45ea..d45197ec6de 100644 --- a/packages/hagrid/hagrid/cli.py +++ b/packages/hagrid/hagrid/cli.py @@ -261,7 +261,7 @@ def clean(location: str) -> None: ) @click.option("--tls", is_flag=True, help="Launch with TLS configuration") @click.option("--test", is_flag=True, help="Launch with test configuration") -@click.option("--dev", is_flag=True, help="Shortcut for development release") +@click.option("--dev", is_flag=True, help="Shortcut for development mode") @click.option( "--release", default="production", @@ -2058,9 +2058,9 @@ def build_command(cmd: str) -> TypeList[str]: return [build_cmd] -def deploy_command(cmd: str, tail: bool, release_type: str) -> TypeList[str]: +def deploy_command(cmd: str, tail: bool, dev_mode: bool) -> TypeList[str]: up_cmd = str(cmd) - up_cmd += " --file docker-compose.dev.yml" if release_type == "development" else "" + up_cmd += " --file docker-compose.dev.yml" if dev_mode else "" up_cmd += " up" if not tail: up_cmd += " -d" @@ -2357,9 +2357,8 @@ def create_launch_docker_cmd( my_build_command = build_command(cmd) final_commands["Building"] = my_build_command - release_type = kwargs["release"] - - final_commands["Launching"] = deploy_command(cmd, tail, release_type) + dev_mode = kwargs.get("dev", False) + final_commands["Launching"] = deploy_command(cmd, tail, dev_mode) return final_commands @@ -2369,8 +2368,6 @@ def create_launch_worker_cmd( build: bool, tail: bool = True, ) -> TypeDict[str, TypeList[str]]: - release_type = kwargs["release"] - final_commands = {} final_commands["Pulling"] = pull_command(cmd, kwargs) cmd += " --file docker-compose.yml" @@ -2379,9 +2376,8 @@ def create_launch_worker_cmd( my_build_command = build_command(cmd) final_commands["Building"] = my_build_command - release_type = kwargs["release"] - - final_commands["Launching"] = deploy_command(cmd, tail, release_type) + dev_mode = kwargs.get("dev", False) + final_commands["Launching"] = deploy_command(cmd, tail, dev_mode) return final_commands diff --git a/tox.ini b/tox.ini index 2eee61aac63..395654a6182 100644 --- a/tox.ini +++ b/tox.ini @@ -231,7 +231,7 @@ allowlist_externals = chcp passenv=HOME, USER setenv = - HAGRID_FLAGS = {env:HAGRID_FLAGS:--tag=local --test} + HAGRID_FLAGS = {env:HAGRID_FLAGS:--tag=local --release=development --test} EMULATION = {env:EMULATION:false} HAGRID_ART = false PYTHONIOENCODING = utf-8 @@ -283,7 +283,7 @@ commands = bash -c 'HAGRID_ART=$HAGRID_ART hagrid launch test-gateway-1 network to docker:9081 $HAGRID_FLAGS --no-health-checks --verbose --no-warnings --build' bash -c 'HAGRID_ART=$HAGRID_ART hagrid launch test-domain-1 domain to docker:9082 $HAGRID_FLAGS --no-health-checks --enable-signup --verbose --no-warnings --build' - ; bash -c 'HAGRID_ART=$HAGRID_ART hagrid launch test-domain-2 domain to docker:9083 --headless $HAGRID_FLAGS --enable-signup --no-health-checks --verbose --no-warnings --dev' + ; bash -c 'HAGRID_ART=$HAGRID_ART hagrid launch test-domain-2 domain to docker:9083 --headless $HAGRID_FLAGS --enable-signup --no-health-checks --verbose --no-warnings --build' ; wait for nodes to start docker ps From c3dc9870754594de4bda9be44848b48922bf3abd Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Thu, 26 Oct 2023 07:45:44 +0000 Subject: [PATCH 18/19] [grid] log permissions --- packages/grid/backend/grid/start.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/grid/backend/grid/start.sh b/packages/grid/backend/grid/start.sh index c2bde278556..f22990fec65 100755 --- a/packages/grid/backend/grid/start.sh +++ b/packages/grid/backend/grid/start.sh @@ -7,6 +7,8 @@ export GEVENT_MONKEYPATCH="False" # For debugging permissions id ls -lisa ~/data/ +ls -lisa ~/app/syft/ +ls -lisa ~/app/grid/ APP_MODULE=grid.main:app LOG_LEVEL=${LOG_LEVEL:-info} From 2167f75538910f0d4b94f61c730b9e2347143025 Mon Sep 17 00:00:00 2001 From: Yash Gorana Date: Thu, 26 Oct 2023 17:37:07 +0000 Subject: [PATCH 19/19] [grid] revert to root user --- packages/grid/backend/backend.dockerfile | 44 ++++++++++--------- packages/grid/backend/grid/start.sh | 20 ++++----- packages/grid/devspace.yaml | 20 ++++----- packages/grid/docker-compose.dev.yml | 15 +++---- packages/grid/docker-compose.yml | 5 +-- packages/grid/helm/manifests.yaml | 9 +--- .../syft/templates/backend-statefulset.yaml | 9 +--- 7 files changed, 56 insertions(+), 66 deletions(-) diff --git a/packages/grid/backend/backend.dockerfile b/packages/grid/backend/backend.dockerfile index 3a32b14501b..e882794774f 100644 --- a/packages/grid/backend/backend.dockerfile +++ b/packages/grid/backend/backend.dockerfile @@ -1,10 +1,12 @@ ARG PYTHON_VERSION="3.11" ARG TZ="Etc/UTC" -ARG USER="syftuser" -ARG UID=1000 + +# change to USER="syftuser", UID=1000 and HOME="/home/$USER" for rootless +ARG USER="root" +ARG UID=0 ARG USER_GRP=$USER:$USER -ARG HOME="/home/$USER" -ARG SYFT_WORKDIR="$HOME/app" +ARG HOME="/root" +ARG APPDIR="$HOME/app" # ==================== [BUILD STEP] Python Dev Base ==================== # @@ -18,21 +20,22 @@ ARG UID # Setup Python DEV RUN apk update && \ apk add build-base gcc tzdata python-$PYTHON_VERSION-dev py$PYTHON_VERSION-pip && \ - ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone && \ - adduser -D -u $UID $USER + ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + # uncomment for creating rootless user + # && adduser -D -u $UID $USER # ==================== [BUILD STEP] Install Syft Dependency ==================== # FROM python_dev as syft_deps -ARG SYFT_WORKDIR -ARG USER_GRP -ARG USER +ARG APPDIR ARG HOME ARG UID +ARG USER +ARG USER_GRP USER $USER -WORKDIR $SYFT_WORKDIR +WORKDIR $APPDIR ENV PATH=$PATH:$HOME/.local/bin # copy skeleton to do package install @@ -54,29 +57,30 @@ RUN --mount=type=cache,target=$HOME/.cache/,rw,uid=$UID \ FROM cgr.dev/chainguard/wolfi-base as backend # inherit from global +ARG APPDIR +ARG HOME ARG PYTHON_VERSION ARG TZ -ARG SYFT_WORKDIR -ARG USER_GRP ARG USER -ARG HOME +ARG USER_GRP # Setup Python RUN apk update && \ apk add --no-cache tzdata bash python-$PYTHON_VERSION py$PYTHON_VERSION-pip && \ ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone && \ rm -rf /var/cache/apk/* && \ - adduser -D -u 1000 $USER && \ - mkdir -p /var/log/pygrid $HOME/data/creds $HOME/data/db $HOME/.cache $HOME/.local && \ - chown -R $USER_GRP /var/log/pygrid $HOME/ + # Uncomment for rootless user + # adduser -D -u 1000 $USER && \ + mkdir -p /var/log/pygrid $HOME/data/creds $HOME/data/db $HOME/.cache $HOME/.local + # chown -R $USER_GRP /var/log/pygrid $HOME/ USER $USER -WORKDIR $SYFT_WORKDIR +WORKDIR $APPDIR # Update environment variables ENV PATH=$PATH:$HOME/.local/bin \ - PYTHONPATH=$SYFT_WORKDIR \ - APPDIR=$SYFT_WORKDIR \ + PYTHONPATH=$APPDIR \ + APPDIR=$APPDIR \ NODE_NAME="default_node_name" \ NODE_TYPE="domain" \ SERVICE_NAME="backend" \ @@ -106,4 +110,4 @@ COPY --chown=$USER_GRP grid/backend/grid ./grid # copy syft COPY --chown=$USER_GRP syft/ ./syft/ -CMD ["bash", "./grid/start.sh"] \ No newline at end of file +CMD ["bash", "./grid/start.sh"] diff --git a/packages/grid/backend/grid/start.sh b/packages/grid/backend/grid/start.sh index f22990fec65..a47b88bc717 100755 --- a/packages/grid/backend/grid/start.sh +++ b/packages/grid/backend/grid/start.sh @@ -1,21 +1,21 @@ #! /usr/bin/env bash set -e -echo "Running start.sh with RELEASE=${RELEASE}" -export GEVENT_MONKEYPATCH="False" - -# For debugging permissions -id -ls -lisa ~/data/ -ls -lisa ~/app/syft/ -ls -lisa ~/app/grid/ +echo "Running start.sh with RELEASE=${RELEASE} and $(id)" +export GEVENT_MONKEYPATCH="False" APP_MODULE=grid.main:app LOG_LEVEL=${LOG_LEVEL:-info} HOST=${HOST:-0.0.0.0} PORT=${PORT:-80} RELOAD="" NODE_TYPE=${NODE_TYPE:-domain} +APPDIR=${APPDIR:-$HOME/app} + +# For debugging permissions +ls -lisa $HOME/data +ls -lisa $APPDIR/syft/ +ls -lisa $APPDIR/grid/ if [[ ${DEV_MODE} == "True" ]]; then @@ -27,12 +27,10 @@ fi set +e export NODE_PRIVATE_KEY=$(python $APPDIR/grid/bootstrap.py --private_key) export NODE_UID=$(python $APPDIR/grid/bootstrap.py --uid) +export NODE_TYPE=$NODE_TYPE set -e echo "NODE_UID=$NODE_UID" echo "NODE_TYPE=$NODE_TYPE" -export NODE_TYPE=$NODE_TYPE - -# export GEVENT_MONKEYPATCH="True" exec uvicorn $RELOAD --host $HOST --port $PORT --log-level $LOG_LEVEL "$APP_MODULE" diff --git a/packages/grid/devspace.yaml b/packages/grid/devspace.yaml index 8cac6cb13cd..827014ad53a 100644 --- a/packages/grid/devspace.yaml +++ b/packages/grid/devspace.yaml @@ -125,17 +125,17 @@ deployments: name: component-chart repo: https://charts.devspace.sh values: - # PodSecurityContext - securityContext: - runAsNonRoot: true - runAsUser: 1000 - runAsGroup: 1000 - fsGroup: 1000 - fsGroupChangePolicy: "Always" + # PodSecurityContext (uncomment for rootless "syftuser") + # securityContext: + # runAsNonRoot: true + # runAsUser: 1000 + # runAsGroup: 1000 + # fsGroup: 1000 + # fsGroupChangePolicy: "Always" containers: - image: "${CONTAINER_REGISTRY}/${DOCKER_IMAGE_BACKEND}:${VERSION}" volumeMounts: - - containerPath: /home/syftuser/data/creds/ + - containerPath: /root/data/creds/ volume: name: credentials-data subPath: /credentials-data @@ -318,8 +318,8 @@ dev: value: "True" logs: {} sync: - - path: ./backend/grid:/home/syftuser/app/grid - - path: ../syft:/home/syftuser/app/syft + - path: ./backend/grid:/root/app/grid + - path: ../syft:/root/app/syft profiles: - name: gateway diff --git a/packages/grid/docker-compose.dev.yml b/packages/grid/docker-compose.dev.yml index 7167fce7976..1aef5823b2b 100644 --- a/packages/grid/docker-compose.dev.yml +++ b/packages/grid/docker-compose.dev.yml @@ -40,11 +40,10 @@ services: - "27017" backend: - user: 1000:1000 volumes: - - ${RELATIVE_PATH}./backend/grid:/home/syftuser/app/grid - - ${RELATIVE_PATH}../syft:/home/syftuser/app/syft - - ${RELATIVE_PATH}./data/package-cache:/home/syftuser/.cache + - ${RELATIVE_PATH}./backend/grid:/root/app/grid + - ${RELATIVE_PATH}../syft:/root/app/syft + - ${RELATIVE_PATH}./data/package-cache:/root/.cache environment: - DEV_MODE=True stdin_open: true @@ -52,16 +51,16 @@ services: # backend_stream: # volumes: - # - ${RELATIVE_PATH}./backend/grid:/home/syftuser/app/grid - # - ${RELATIVE_PATH}../syft:/home/syftuser/app/syft + # - ${RELATIVE_PATH}./backend/grid:/root/app/grid + # - ${RELATIVE_PATH}../syft:/root/app/syft # - ${RELATIVE_PATH}./data/package-cache:/root/.cache # environment: # - DEV_MODE=True # celeryworker: # volumes: - # - ${RELATIVE_PATH}./backend/grid:/home/syftuser/app/grid - # - ${RELATIVE_PATH}../syft/:/home/syftuser/app/syft + # - ${RELATIVE_PATH}./backend/grid:/root/app/grid + # - ${RELATIVE_PATH}../syft/:/root/app/syft # - ${RELATIVE_PATH}./data/package-cache:/root/.cache # environment: # - DEV_MODE=True diff --git a/packages/grid/docker-compose.yml b/packages/grid/docker-compose.yml index f3c429e044f..ab4ac42c1d2 100644 --- a/packages/grid/docker-compose.yml +++ b/packages/grid/docker-compose.yml @@ -103,7 +103,6 @@ services: backend: restart: always image: "${DOCKER_IMAGE_BACKEND?Variable not set}:${VERSION-latest}" - user: 1000:1000 depends_on: - proxy env_file: @@ -133,7 +132,7 @@ services: - DEFAULT_ROOT_PASSWORD=${DEFAULT_ROOT_PASSWORD} network_mode: service:proxy volumes: - - ${CREDENTIALS_VOLUME}:/home/syftuser/data/creds/ + - ${CREDENTIALS_VOLUME}:/root/data/creds/ stdin_open: true tty: true labels: @@ -169,7 +168,7 @@ services: # - ENABLE_OBLV=${ENABLE_OBLV} # network_mode: service:proxy # volumes: - # - credentials-data:/home/syftuser/data/creds/ + # - credentials-data:/root/data/creds/ # celeryworker: # restart: always diff --git a/packages/grid/helm/manifests.yaml b/packages/grid/helm/manifests.yaml index a1cda8f2025..177bc6a0167 100644 --- a/packages/grid/helm/manifests.yaml +++ b/packages/grid/helm/manifests.yaml @@ -110,7 +110,7 @@ spec: startupProbe: null volumeDevices: null volumeMounts: - - mountPath: /home/syftuser/data/creds/ + - mountPath: /root/data/creds/ name: credentials-data readOnly: false subPath: credentials-data @@ -123,12 +123,7 @@ spec: nodeSelector: null overhead: null readinessGates: null - securityContext: - fsGroup: 1000 - fsGroupChangePolicy: Always - runAsGroup: 1000 - runAsNonRoot: true - runAsUser: 1000 + securityContext: null terminationGracePeriodSeconds: 5 tolerations: null topologySpreadConstraints: null diff --git a/packages/grid/helm/syft/templates/backend-statefulset.yaml b/packages/grid/helm/syft/templates/backend-statefulset.yaml index 331295fb84f..ca6790892e1 100644 --- a/packages/grid/helm/syft/templates/backend-statefulset.yaml +++ b/packages/grid/helm/syft/templates/backend-statefulset.yaml @@ -89,7 +89,7 @@ spec: startupProbe: null volumeDevices: null volumeMounts: - - mountPath: /home/syftuser/data/creds/ + - mountPath: /root/data/creds/ name: credentials-data readOnly: false subPath: credentials-data @@ -102,12 +102,7 @@ spec: nodeSelector: null overhead: null readinessGates: null - securityContext: - fsGroup: 1000 - fsGroupChangePolicy: Always - runAsGroup: 1000 - runAsNonRoot: true - runAsUser: 1000 + securityContext: null terminationGracePeriodSeconds: 5 tolerations: null topologySpreadConstraints: null