From c23d7e8979fcdc3373f8e37efbdda329ea8af51d Mon Sep 17 00:00:00 2001 From: "Shah, Karan" Date: Thu, 7 Nov 2024 16:14:34 +0530 Subject: [PATCH 01/17] Download src only for packages that exist on pip Signed-off-by: Shah, Karan --- openfl-docker/licenses.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openfl-docker/licenses.sh b/openfl-docker/licenses.sh index 005a8bccfa..4eba1e8c97 100755 --- a/openfl-docker/licenses.sh +++ b/openfl-docker/licenses.sh @@ -45,6 +45,6 @@ if [ "$INSTALL_SOURCES" = "yes" ]; then # Append dependency list to all_dependencies.txt pip-licenses | awk '{for(i=1;i<=NF;i++) if(i!=2) printf $i" "; print ""}' | tee -a all_dependencies.txt - # Download source packages for Python packages with specific licenses - pip-licenses | grep -E 'GPL|MPL|EPL' | awk '{OFS="=="} {print $1,$2}' | xargs pip download --no-binary :all: + # Download source packages for Python packages (if exists) with specific licenses + pip-licenses | grep -E 'GPL|MPL|EPL' | awk '{OFS="=="} {print $1,$2}' | xargs -I {} sh -c 'pip download --no-binary :all: {} || true' fi From ec25130faf7aae4198951e692930c3d28517c1ec Mon Sep 17 00:00:00 2001 From: "Shah, Karan" Date: Fri, 8 Nov 2024 12:45:10 +0530 Subject: [PATCH 02/17] Install Gramine in base image Signed-off-by: Shah, Karan --- openfl-docker/Dockerfile.base | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/openfl-docker/Dockerfile.base b/openfl-docker/Dockerfile.base index 0b5d746aca..c85f5ca619 100644 --- a/openfl-docker/Dockerfile.base +++ b/openfl-docker/Dockerfile.base @@ -15,6 +15,7 @@ RUN --mount=type=cache,id=apt-dev,target=/var/cache/apt \ apt-get update && \ apt-get install -y \ git \ + curl \ python3-pip \ python3.10-dev \ ca-certificates \ @@ -23,6 +24,18 @@ RUN --mount=type=cache,id=apt-dev,target=/var/cache/apt \ apt-get purge -y linux-libc-dev && \ rm -rf /var/lib/apt/lists/* +# Install Gramine +RUN --mount=type=cache,id=apt-dev,target=/var/cache/apt \ + curl -fsSLo /usr/share/keyrings/gramine-keyring.gpg https://packages.gramineproject.io/gramine-keyring.gpg && \ + echo "deb [arch=amd64 signed-by=/usr/share/keyrings/gramine-keyring.gpg] https://packages.gramineproject.io/ jammy main" \ + | tee /etc/apt/sources.list.d/gramine.list && \ + curl -fsSLo /usr/share/keyrings/intel-sgx-deb.asc https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key && \ + echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel-sgx-deb.asc] https://download.01.org/intel-sgx/sgx_repo/ubuntu jammy main" \ + | tee /etc/apt/sources.list.d/intel-sgx.list && \ + apt-get update && \ + apt-get install -y gramine --no-install-recommends && \ + rm -rf /var/lib/apt/lists/* + # Create an unprivileged user. RUN groupadd -g 1001 default && \ useradd -m -u 1001 -g default user From b70e4e1dd880d4b233f2a6ef43e5ad89594c7760 Mon Sep 17 00:00:00 2001 From: "Shah, Karan" Date: Fri, 8 Nov 2024 12:45:10 +0530 Subject: [PATCH 03/17] Install Gramine in base image Signed-off-by: Shah, Karan --- openfl-docker/Dockerfile.base | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openfl-docker/Dockerfile.base b/openfl-docker/Dockerfile.base index c85f5ca619..139767b544 100644 --- a/openfl-docker/Dockerfile.base +++ b/openfl-docker/Dockerfile.base @@ -1,7 +1,7 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 # ------------------------------------ -# OpenFL Base Image +# OpenFL Base Image w/ Gramine support # $> docker build . -t openfl -f Dockerfile.base [--build-arg OPENFL_REVISION=GIT_URL@COMMIT_ID] # ------------------------------------ FROM ubuntu:22.04 AS base From 004e99f85b79a1c76ff7873f3dd665dd8e9a4faa Mon Sep 17 00:00:00 2001 From: "Shah, Karan" Date: Tue, 12 Nov 2024 22:00:07 +0530 Subject: [PATCH 04/17] Add enclave template Signed-off-by: Shah, Karan --- openfl-docker/gramine_app/Makefile | 50 +++++++++++++ .../gramine_app/fx.manifest.template | 74 +++++++++++++++++++ 2 files changed, 124 insertions(+) create mode 100644 openfl-docker/gramine_app/Makefile create mode 100755 openfl-docker/gramine_app/fx.manifest.template diff --git a/openfl-docker/gramine_app/Makefile b/openfl-docker/gramine_app/Makefile new file mode 100644 index 0000000000..5d92ca3666 --- /dev/null +++ b/openfl-docker/gramine_app/Makefile @@ -0,0 +1,50 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +ARCH_LIBDIR ?= /lib/$(shell $(CC) -dumpmachine) + +# This is a signer key on the BUILDING machine +SGX_SIGNER_KEY ?= /key.pem + +ifeq ($(DEBUG),1) +GRAMINE_LOG_LEVEL = debug +else +GRAMINE_LOG_LEVEL = error +endif + +.PHONY: all +all: fx.manifest +ifeq ($(SGX),1) +all: fx.manifest.sgx fx.sig +endif + +fx.manifest: fx.manifest.template + @echo "Making fx.manifest file" +# disable checks temp - until gramine is pegged to proper release + gramine-manifest \ + -Dlog_level=$(GRAMINE_LOG_LEVEL) \ + -Darch_libdir=$(ARCH_LIBDIR) \ + -Dno_proxy=$(no_proxy) \ + -Dhttp_proxy=$(http_proxy) \ + -Dhttps_proxy=$(https_proxy) \ + -Dentrypoint=$(shell which fx) \ + $< >$@ + +fx.manifest.sgx: fx.manifest + @echo "Making fx.manifest.sgx file" + @test -s $(SGX_SIGNER_KEY) || \ + { echo "SGX signer private key was not found, please specify SGX_SIGNER_KEY!"; exit 1; } + @gramine-sgx-sign \ + --key $(SGX_SIGNER_KEY) \ + --manifest $< \ + --output $@ | tail -n 1 | tr -d ' ' | xargs -I {} echo "fx.mr_enclave={}" + +fx.sig: fx.manifest.sgx + +.PHONY: clean +clean: + $(RM) *.manifest *.manifest.sgx *.token *.sig OUTPUT* *.PID TEST_STDOUT TEST_STDERR + $(RM) -r scripts/__pycache__ + +.PHONY: distclean +distclean: clean diff --git a/openfl-docker/gramine_app/fx.manifest.template b/openfl-docker/gramine_app/fx.manifest.template new file mode 100755 index 0000000000..23ba9ce61e --- /dev/null +++ b/openfl-docker/gramine_app/fx.manifest.template @@ -0,0 +1,74 @@ +# Copyright (C) 2024 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 +# ================================== +# OpenFL Enclave for Gramine-SGX +# ================================== + +libos.entrypoint = "{{ entrypoint }}" +loader.entrypoint.uri = "file:{{ gramine.libos }}" + +loader.log_level = "{{ log_level }}" + +loader.env.OMP_NUM_THREADS = "16" +loader.env.LD_LIBRARY_PATH = "{{ arch_libdir }}:/usr/{{ arch_libdir }}:/lib:/usr/lib" +loader.env.SSL_CERT_DIR = "/etc/ssl/certs" +# loader.env.no_proxy = "{{ no_proxy }}" +# loader.env.https_proxy = "{{ https_proxy }}" +# loader.env.http_proxy = "{{ http_proxy }}" + +loader.insecure__use_cmdline_argv = true +loader.insecure__use_host_env = true + + +# URI - path on host +# PATH - pointer inside gramine +fs.start_dir = "/workspace" +fs.mounts = [ + { uri = "file:{{ gramine.runtimedir() }}", path = "/lib" }, + { uri = "file:{{ arch_libdir }}", path = "{{ arch_libdir }}" }, + { uri = "file:/usr", path = "/usr" }, + { uri = "file:/etc/ssl/certs", path = "/etc/ssl/certs" }, + { uri = "file:/workspace", path = "/workspace" }, + { uri = "file:/keys", path = "/keys" }, + { uri = "file:/host_save_path", path = "/host_save_path" }, + { type = "tmpfs", path = "/tmp" }, +] + +sgx.debug = false +sgx.preheat_enclave = false +sgx.enclave_size = "16G" + +sys.stack.size = "4M" +sys.enable_sigterm_injection = true +sys.enable_extra_runtime_domain_names_conf = true +# sys.brk.max_size = "1M" + +sgx.trusted_files = [ + "file:{{ gramine.libos }}", + "file:{{ entrypoint }}", + "file:{{ gramine.runtimedir() }}/", + "file:{{ arch_libdir }}/", + "file:/usr/{{ arch_libdir }}/", + "file:/etc/ssl/certs/", + "file:{{ python.stdlib }}/", + "file:{{ python.distlib }}/", +{% for path in python.get_sys_path('python') %} + "file:{{ path }}{{ '/' if path.is_dir() else '' }}", +{% endfor %} + "file:/workspace/src/", +] + +sgx.allowed_files = [ + "file:/workspace/save", + "file:/workspace/plan/", + "file:/workspace/logs", + "file:/workspace/cert", + "file:/keys", + "file:/host_save_path", + "file:/workspace/data", + "file:/workspace/plan/cols.yaml", + "file:/workspace/plan/data.yaml", + "file:/workspace/plan/plan.yaml", +] +sgx.remote_attestation = "dcap" +sgx.max_threads = 512 From 1965115a397e9dd917d48b6498f78c9d9add6dd6 Mon Sep 17 00:00:00 2001 From: "Shah, Karan" Date: Tue, 12 Nov 2024 22:55:36 +0530 Subject: [PATCH 05/17] SGX option in dockerize Signed-off-by: Shah, Karan --- openfl/interface/workspace.py | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/openfl/interface/workspace.py b/openfl/interface/workspace.py index 260c71eed1..32c264a6a8 100644 --- a/openfl/interface/workspace.py +++ b/openfl/interface/workspace.py @@ -389,6 +389,22 @@ def export_() -> str: default=False, help="If set, rebuilds docker images with `--no-cache` option.", ) +@option( + "--sgx-ready", + is_flag=True, + default=False, + help="If set, builds an SGX-enabled OpenFL enclave.", +) +@option( + "--enclave-key", + "enclave_key", + type=str, + required=False, + help=( + "Path to an enclave signing key. If not provided, a new key will be generated. " + "This option is only valid when `--sgx-ready` is set." + ), +) @option( "--revision", required=False, @@ -401,7 +417,9 @@ def export_() -> str: ), ) @pass_context -def dockerize_(context, save, rebuild, revision): +def dockerize_( + context, save: bool, rebuild: bool, sgx_ready: bool, enclave_key: str, revision: str +): """Package current workspace as a Docker image.""" # Docker build options @@ -430,10 +448,24 @@ def dockerize_(context, save, rebuild, revision): _execute(base_image_build_cmd) # Build workspace image. + options = [] + options.append("--no-cache" if rebuild else "") + options = " ".join(options) + if enclave_key is None: + _execute("openssl genrsa -out key.pem -3 3072") + enclave_key = os.path.abspath("key.pem") + logging.info(f"Generated new enclave key: {enclave_key}") + else: + enclave_key = os.path.abspath(enclave_key) + if not os.path.exists(enclave_key): + raise FileNotFoundError(f"Enclave key `{enclave_key}` does not exist") + logging.info(f"Using enclave key: {enclave_key}") + logging.info("Building workspace image") ws_image_build_cmd = ( "DOCKER_BUILDKIT=1 docker build {options} " "--build-arg WORKSPACE_NAME={workspace_name} " + "--secret id=signer-key,src={enclave_key} " "-t {image_name} " "-f {dockerfile} " "{build_context}" @@ -441,6 +473,7 @@ def dockerize_(context, save, rebuild, revision): options=options, image_name=workspace_name, workspace_name=workspace_name, + enclave_key=enclave_key, dockerfile=os.path.join(SITEPACKS, "openfl-docker", "Dockerfile.workspace"), build_context=".", ) From 8bafc0554967e31631f6f9216432457f4cf30cd5 Mon Sep 17 00:00:00 2001 From: "Shah, Karan" Date: Wed, 13 Nov 2024 12:45:51 +0530 Subject: [PATCH 06/17] Remove /keys Signed-off-by: Shah, Karan --- openfl-docker/gramine_app/fx.manifest.template | 2 -- 1 file changed, 2 deletions(-) diff --git a/openfl-docker/gramine_app/fx.manifest.template b/openfl-docker/gramine_app/fx.manifest.template index 23ba9ce61e..9898021f94 100755 --- a/openfl-docker/gramine_app/fx.manifest.template +++ b/openfl-docker/gramine_app/fx.manifest.template @@ -29,7 +29,6 @@ fs.mounts = [ { uri = "file:/usr", path = "/usr" }, { uri = "file:/etc/ssl/certs", path = "/etc/ssl/certs" }, { uri = "file:/workspace", path = "/workspace" }, - { uri = "file:/keys", path = "/keys" }, { uri = "file:/host_save_path", path = "/host_save_path" }, { type = "tmpfs", path = "/tmp" }, ] @@ -63,7 +62,6 @@ sgx.allowed_files = [ "file:/workspace/plan/", "file:/workspace/logs", "file:/workspace/cert", - "file:/keys", "file:/host_save_path", "file:/workspace/data", "file:/workspace/plan/cols.yaml", From 9be298c8329c7f1c74906ea4db1de134d610c441 Mon Sep 17 00:00:00 2001 From: "Shah, Karan" Date: Wed, 13 Nov 2024 13:09:01 +0530 Subject: [PATCH 07/17] Remove /host_save_path Signed-off-by: Shah, Karan --- openfl-docker/gramine_app/fx.manifest.template | 2 -- 1 file changed, 2 deletions(-) diff --git a/openfl-docker/gramine_app/fx.manifest.template b/openfl-docker/gramine_app/fx.manifest.template index 9898021f94..da9b3c7c7b 100755 --- a/openfl-docker/gramine_app/fx.manifest.template +++ b/openfl-docker/gramine_app/fx.manifest.template @@ -29,7 +29,6 @@ fs.mounts = [ { uri = "file:/usr", path = "/usr" }, { uri = "file:/etc/ssl/certs", path = "/etc/ssl/certs" }, { uri = "file:/workspace", path = "/workspace" }, - { uri = "file:/host_save_path", path = "/host_save_path" }, { type = "tmpfs", path = "/tmp" }, ] @@ -62,7 +61,6 @@ sgx.allowed_files = [ "file:/workspace/plan/", "file:/workspace/logs", "file:/workspace/cert", - "file:/host_save_path", "file:/workspace/data", "file:/workspace/plan/cols.yaml", "file:/workspace/plan/data.yaml", From 382151f8dc4a5bd61d19a673542c7795486d2e5f Mon Sep 17 00:00:00 2001 From: "Shah, Karan" Date: Wed, 13 Nov 2024 20:33:15 +0530 Subject: [PATCH 08/17] Simplify variables and constants Signed-off-by: Shah, Karan --- openfl-docker/gramine_app/Makefile | 14 +++----- .../gramine_app/fx.manifest.template | 35 ++++++++----------- 2 files changed, 20 insertions(+), 29 deletions(-) diff --git a/openfl-docker/gramine_app/Makefile b/openfl-docker/gramine_app/Makefile index 5d92ca3666..dbe4d1ce66 100644 --- a/openfl-docker/gramine_app/Makefile +++ b/openfl-docker/gramine_app/Makefile @@ -1,11 +1,9 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 - +VENV_ROOT ?= /opt/venv +WORKSPACE_ROOT ?= /workspace ARCH_LIBDIR ?= /lib/$(shell $(CC) -dumpmachine) -# This is a signer key on the BUILDING machine -SGX_SIGNER_KEY ?= /key.pem - ifeq ($(DEBUG),1) GRAMINE_LOG_LEVEL = debug else @@ -20,14 +18,12 @@ endif fx.manifest: fx.manifest.template @echo "Making fx.manifest file" -# disable checks temp - until gramine is pegged to proper release gramine-manifest \ -Dlog_level=$(GRAMINE_LOG_LEVEL) \ -Darch_libdir=$(ARCH_LIBDIR) \ - -Dno_proxy=$(no_proxy) \ - -Dhttp_proxy=$(http_proxy) \ - -Dhttps_proxy=$(https_proxy) \ - -Dentrypoint=$(shell which fx) \ + -Dvenv_root=$(VENV_ROOT) \ + -Dentrypoint=$(VENV_ROOT)/bin/fx \ + -Dworkspace_root=$(WORKSPACE_ROOT) \ $< >$@ fx.manifest.sgx: fx.manifest diff --git a/openfl-docker/gramine_app/fx.manifest.template b/openfl-docker/gramine_app/fx.manifest.template index da9b3c7c7b..276a8d1268 100755 --- a/openfl-docker/gramine_app/fx.manifest.template +++ b/openfl-docker/gramine_app/fx.manifest.template @@ -5,30 +5,25 @@ # ================================== libos.entrypoint = "{{ entrypoint }}" -loader.entrypoint.uri = "file:{{ gramine.libos }}" - +loader.entrypoint = "file:{{ gramine.libos }}" loader.log_level = "{{ log_level }}" -loader.env.OMP_NUM_THREADS = "16" -loader.env.LD_LIBRARY_PATH = "{{ arch_libdir }}:/usr/{{ arch_libdir }}:/lib:/usr/lib" -loader.env.SSL_CERT_DIR = "/etc/ssl/certs" -# loader.env.no_proxy = "{{ no_proxy }}" -# loader.env.https_proxy = "{{ https_proxy }}" -# loader.env.http_proxy = "{{ http_proxy }}" - loader.insecure__use_cmdline_argv = true loader.insecure__use_host_env = true +loader.env.LD_LIBRARY_PATH = "{{ venv_root }}:{{ arch_libdir }}:/usr/{{ arch_libdir }}:/lib:/usr/lib" +loader.env.SSL_CERT_DIR = "/etc/ssl/certs" # URI - path on host # PATH - pointer inside gramine -fs.start_dir = "/workspace" +fs.start_dir = "{{ workspace_root }}" fs.mounts = [ { uri = "file:{{ gramine.runtimedir() }}", path = "/lib" }, { uri = "file:{{ arch_libdir }}", path = "{{ arch_libdir }}" }, { uri = "file:/usr", path = "/usr" }, { uri = "file:/etc/ssl/certs", path = "/etc/ssl/certs" }, - { uri = "file:/workspace", path = "/workspace" }, + { uri = "file:{{ workspace_root }}", path = "{{ workspace_root }}" }, + { uri = "file:{{ venv_root }}", path = "{{ venv_root }}" }, { type = "tmpfs", path = "/tmp" }, ] @@ -53,18 +48,18 @@ sgx.trusted_files = [ {% for path in python.get_sys_path('python') %} "file:{{ path }}{{ '/' if path.is_dir() else '' }}", {% endfor %} - "file:/workspace/src/", + "file:{{ venv_root }}/", + "file:{{ workspace_root }}/src/", ] sgx.allowed_files = [ - "file:/workspace/save", - "file:/workspace/plan/", - "file:/workspace/logs", - "file:/workspace/cert", - "file:/workspace/data", - "file:/workspace/plan/cols.yaml", - "file:/workspace/plan/data.yaml", - "file:/workspace/plan/plan.yaml", + "file:{{ workspace_root }}/save", + "file:{{ workspace_root }}/logs", + "file:{{ workspace_root }}/cert", + "file:{{ workspace_root }}/data", + "file:{{ workspace_root }}/plan/cols.yaml", + "file:{{ workspace_root }}/plan/data.yaml", + "file:{{ workspace_root }}/plan/plan.yaml", ] sgx.remote_attestation = "dcap" sgx.max_threads = 512 From e0e95f97ee6e456945b5119cca3e60daf6ffadf7 Mon Sep 17 00:00:00 2001 From: "Shah, Karan" Date: Thu, 14 Nov 2024 13:06:20 +0530 Subject: [PATCH 09/17] Cleanup manifest and makefile; less hardcode Signed-off-by: Shah, Karan --- openfl-docker/gramine_app/Makefile | 12 +++++-- .../gramine_app/fx.manifest.template | 36 +++++++++++-------- 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/openfl-docker/gramine_app/Makefile b/openfl-docker/gramine_app/Makefile index dbe4d1ce66..4dbc8ef142 100644 --- a/openfl-docker/gramine_app/Makefile +++ b/openfl-docker/gramine_app/Makefile @@ -1,8 +1,16 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -VENV_ROOT ?= /opt/venv -WORKSPACE_ROOT ?= /workspace +# ------------------------------------ +# Makefile for Gramine application within a container +# Usage: +# 1. Activate the python venv. +# 2. Provide paths VENV_ROOT and WORKSPACE_ROOT. +# 3. make SGX=0/1 [SGX_SIGNER_KEY=] +# ------------------------------------ +VENV_ROOT ?= $(shell dirname $(shell dirname $(shell which python))) +WORKSPACE_ROOT ?= $(shell pwd) ARCH_LIBDIR ?= /lib/$(shell $(CC) -dumpmachine) +SGX_SIGNER_KEY ?= /key.pem ifeq ($(DEBUG),1) GRAMINE_LOG_LEVEL = debug diff --git a/openfl-docker/gramine_app/fx.manifest.template b/openfl-docker/gramine_app/fx.manifest.template index 276a8d1268..93ad6c0d01 100755 --- a/openfl-docker/gramine_app/fx.manifest.template +++ b/openfl-docker/gramine_app/fx.manifest.template @@ -1,8 +1,9 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -# ================================== -# OpenFL Enclave for Gramine-SGX -# ================================== +# ------------------------------------- +# Enclave Manifest for OpenFL TaskRunner API. +# This defines the configuration for the Gramine loader to run a Python application. +# ------------------------------------- libos.entrypoint = "{{ entrypoint }}" loader.entrypoint = "file:{{ gramine.libos }}" @@ -14,28 +15,35 @@ loader.insecure__use_host_env = true loader.env.LD_LIBRARY_PATH = "{{ venv_root }}:{{ arch_libdir }}:/usr/{{ arch_libdir }}:/lib:/usr/lib" loader.env.SSL_CERT_DIR = "/etc/ssl/certs" -# URI - path on host -# PATH - pointer inside gramine +# Filesystem configuration within Gramine LibOS fs.start_dir = "{{ workspace_root }}" fs.mounts = [ + # System mounts (URI: path on host, PATH: pointer inside gramine) { uri = "file:{{ gramine.runtimedir() }}", path = "/lib" }, { uri = "file:{{ arch_libdir }}", path = "{{ arch_libdir }}" }, - { uri = "file:/usr", path = "/usr" }, { uri = "file:/etc/ssl/certs", path = "/etc/ssl/certs" }, + { uri = "file:/usr", path = "/usr" }, + { type = "tmpfs", path = "/tmp" }, + # User-defined mounts specific to the application. { uri = "file:{{ workspace_root }}", path = "{{ workspace_root }}" }, { uri = "file:{{ venv_root }}", path = "{{ venv_root }}" }, - { type = "tmpfs", path = "/tmp" }, ] -sgx.debug = false -sgx.preheat_enclave = false -sgx.enclave_size = "16G" - +# System configuration sys.stack.size = "4M" +sys.brk.max_size = "1M" sys.enable_sigterm_injection = true sys.enable_extra_runtime_domain_names_conf = true -# sys.brk.max_size = "1M" +# SGX configuration +sgx.debug = false +sgx.enclave_size = "16G" +sgx.preheat_enclave = false +sgx.remote_attestation = "dcap" +sgx.max_threads = 512 + +# List of trusted files, that are hashed and signed by the enclave. +# If these files change after signing of an enclave, application cannot run. sgx.trusted_files = [ "file:{{ gramine.libos }}", "file:{{ entrypoint }}", @@ -52,6 +60,8 @@ sgx.trusted_files = [ "file:{{ workspace_root }}/src/", ] +# List of allowed files that SGX enclave does NOT verify with signatures. +# One should be conservative as to which files are allowed, these can be modified by enclave. sgx.allowed_files = [ "file:{{ workspace_root }}/save", "file:{{ workspace_root }}/logs", @@ -61,5 +71,3 @@ sgx.allowed_files = [ "file:{{ workspace_root }}/plan/data.yaml", "file:{{ workspace_root }}/plan/plan.yaml", ] -sgx.remote_attestation = "dcap" -sgx.max_threads = 512 From fe914554ed1f05d02cadb252fd2ffd9ebc918322 Mon Sep 17 00:00:00 2001 From: "Shah, Karan" Date: Thu, 14 Nov 2024 13:09:41 +0530 Subject: [PATCH 10/17] Make venv default Signed-off-by: Shah, Karan --- openfl-docker/Dockerfile.base | 19 +++++++++++-------- openfl-docker/Dockerfile.workspace | 20 +++++++++++++++----- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/openfl-docker/Dockerfile.base b/openfl-docker/Dockerfile.base index 139767b544..f58d83747f 100644 --- a/openfl-docker/Dockerfile.base +++ b/openfl-docker/Dockerfile.base @@ -18,12 +18,18 @@ RUN --mount=type=cache,id=apt-dev,target=/var/cache/apt \ curl \ python3-pip \ python3.10-dev \ + python3.10-venv \ ca-certificates \ build-essential \ --no-install-recommends && \ apt-get purge -y linux-libc-dev && \ rm -rf /var/lib/apt/lists/* +# Create a python virtual environment. +RUN python3.10 -m venv /opt/venv && \ + /opt/venv/bin/pip install --no-cache-dir --upgrade pip setuptools wheel +ENV PATH=/opt/venv/bin:$PATH + # Install Gramine RUN --mount=type=cache,id=apt-dev,target=/var/cache/apt \ curl -fsSLo /usr/share/keyrings/gramine-keyring.gpg https://packages.gramineproject.io/gramine-keyring.gpg && \ @@ -36,17 +42,14 @@ RUN --mount=type=cache,id=apt-dev,target=/var/cache/apt \ apt-get install -y gramine --no-install-recommends && \ rm -rf /var/lib/apt/lists/* +# Install OpenFL. +ARG OPENFL_REVISION=https://github.com/securefederatedai/openfl.git@v1.6 +RUN pip install --no-cache-dir git+${OPENFL_REVISION} && \ + INSTALL_SOURCES=yes /opt/venv/lib/python3.10/site-packages/openfl-docker/licenses.sh + # Create an unprivileged user. RUN groupadd -g 1001 default && \ useradd -m -u 1001 -g default user USER user -WORKDIR /home/user -ENV PATH=/home/user/.local/bin:$PATH - -# Install OpenFL. -ARG OPENFL_REVISION=https://github.com/securefederatedai/openfl.git@v1.6 -RUN pip install --no-cache-dir -U pip setuptools wheel && \ - pip install --no-cache-dir git+${OPENFL_REVISION} && \ - INSTALL_SOURCES=yes /home/user/.local/lib/python3.10/site-packages/openfl-docker/licenses.sh CMD ["/bin/bash"] diff --git a/openfl-docker/Dockerfile.workspace b/openfl-docker/Dockerfile.workspace index 08446663c6..3b644b741a 100644 --- a/openfl-docker/Dockerfile.workspace +++ b/openfl-docker/Dockerfile.workspace @@ -6,13 +6,23 @@ ARG BASE_IMAGE=openfl:latest FROM ${BASE_IMAGE} +USER root SHELL ["/bin/bash", "-o", "pipefail", "-c"] -USER user +# Import workspace +WORKDIR / ARG WORKSPACE_NAME -COPY ${WORKSPACE_NAME}.zip . -RUN fx workspace import --archive ${WORKSPACE_NAME}.zip && \ - pip install --no-cache-dir -r ${WORKSPACE_NAME}/requirements.txt +COPY ${WORKSPACE_NAME}.zip /workspace.zip +RUN fx workspace import --archive /workspace.zip && \ + pip install --no-cache-dir -r /workspace/requirements.txt + +# Build enclaves +WORKDIR /workspace +RUN --mount=type=secret,id=signer-key,dst=/key.pem \ + cp -r /opt/venv/lib/python3.10/site-packages/openfl-docker/gramine_app/* /workspace/ && \ + mkdir /mrenclave && \ + make SGX=1 SGX_SIGNER_KEY=/key.pem >> /mrenclave/fx && \ + echo "$(cat /mrenclave/fx)" -WORKDIR /home/user/${WORKSPACE_NAME} +USER user CMD ["/bin/bash"] \ No newline at end of file From 80e34ec3ad690507d9f21d9e58943f89bd267107 Mon Sep 17 00:00:00 2001 From: "Shah, Karan" Date: Thu, 14 Nov 2024 13:53:25 +0530 Subject: [PATCH 11/17] Remove --sgx-ready flag Signed-off-by: Shah, Karan --- openfl-docker/gramine_app/fx.manifest.template | 2 +- openfl/interface/workspace.py | 15 ++------------- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/openfl-docker/gramine_app/fx.manifest.template b/openfl-docker/gramine_app/fx.manifest.template index 93ad6c0d01..928dff0f56 100755 --- a/openfl-docker/gramine_app/fx.manifest.template +++ b/openfl-docker/gramine_app/fx.manifest.template @@ -6,7 +6,7 @@ # ------------------------------------- libos.entrypoint = "{{ entrypoint }}" -loader.entrypoint = "file:{{ gramine.libos }}" +loader.entrypoint.uri = "file:{{ gramine.libos }}" loader.log_level = "{{ log_level }}" loader.insecure__use_cmdline_argv = true diff --git a/openfl/interface/workspace.py b/openfl/interface/workspace.py index 32c264a6a8..2da2b1bc04 100644 --- a/openfl/interface/workspace.py +++ b/openfl/interface/workspace.py @@ -389,21 +389,12 @@ def export_() -> str: default=False, help="If set, rebuilds docker images with `--no-cache` option.", ) -@option( - "--sgx-ready", - is_flag=True, - default=False, - help="If set, builds an SGX-enabled OpenFL enclave.", -) @option( "--enclave-key", "enclave_key", type=str, required=False, - help=( - "Path to an enclave signing key. If not provided, a new key will be generated. " - "This option is only valid when `--sgx-ready` is set." - ), + help="Path to an enclave signing key. If not provided, a new key will be generated. ", ) @option( "--revision", @@ -417,9 +408,7 @@ def export_() -> str: ), ) @pass_context -def dockerize_( - context, save: bool, rebuild: bool, sgx_ready: bool, enclave_key: str, revision: str -): +def dockerize_(context, save: bool, rebuild: bool, enclave_key: str, revision: str): """Package current workspace as a Docker image.""" # Docker build options From 5952ef0d9fc30689f3d9df459f661d6e0c91a1f9 Mon Sep 17 00:00:00 2001 From: "Shah, Karan" Date: Thu, 14 Nov 2024 14:21:36 +0530 Subject: [PATCH 12/17] Store enclave measurement in file Signed-off-by: Shah, Karan --- openfl-docker/Dockerfile.workspace | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/openfl-docker/Dockerfile.workspace b/openfl-docker/Dockerfile.workspace index 3b644b741a..0165f557c1 100644 --- a/openfl-docker/Dockerfile.workspace +++ b/openfl-docker/Dockerfile.workspace @@ -1,7 +1,12 @@ # Copyright (C) 2024 Intel Corporation # SPDX-License-Identifier: Apache-2.0 # ------------------------------------ -# Workspace Image +# Gramine-ready Workspace Image +# Usage: +# $> docker build . -t openfl-workspace -f Dockerfile.workspace \ +# [--build-arg BASE_IMAGE=openfl:latest] \ +# [--build-arg WORKSPACE_NAME=WORKSPACE_NAME] \ +# [--secret id=signer-key,src=signer-key.pem] # ------------------------------------ ARG BASE_IMAGE=openfl:latest FROM ${BASE_IMAGE} @@ -20,9 +25,9 @@ RUN fx workspace import --archive /workspace.zip && \ WORKDIR /workspace RUN --mount=type=secret,id=signer-key,dst=/key.pem \ cp -r /opt/venv/lib/python3.10/site-packages/openfl-docker/gramine_app/* /workspace/ && \ - mkdir /mrenclave && \ - make SGX=1 SGX_SIGNER_KEY=/key.pem >> /mrenclave/fx && \ - echo "$(cat /mrenclave/fx)" + make SGX=1 SGX_SIGNER_KEY=/key.pem >> fx.mr_enclave && \ + echo "$(cat fx.mr_enclave)" && \ + chown -R user /workspace USER user CMD ["/bin/bash"] \ No newline at end of file From f76875e1e4e5ea7ed00906da632a823dce6756ac Mon Sep 17 00:00:00 2001 From: "Shah, Karan" Date: Thu, 14 Nov 2024 14:27:09 +0530 Subject: [PATCH 13/17] Rename dockerization.yml to tr_docker_native.yml Signed-off-by: Shah, Karan --- .github/workflows/{dockerization.yml => tr_docker_native.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename .github/workflows/{dockerization.yml => tr_docker_native.yml} (98%) diff --git a/.github/workflows/dockerization.yml b/.github/workflows/tr_docker_native.yml similarity index 98% rename from .github/workflows/dockerization.yml rename to .github/workflows/tr_docker_native.yml index 81d29f5f2a..b333b54059 100644 --- a/.github/workflows/dockerization.yml +++ b/.github/workflows/tr_docker_native.yml @@ -1,5 +1,5 @@ # Tests an FL experiment in a Dockerized environment. -name: Dockerization +name: Task Runner API (Docker/native) on: pull_request: From 31c209a397945b6b01b87a647d98893a8d0205ac Mon Sep 17 00:00:00 2001 From: "Shah, Karan" Date: Thu, 14 Nov 2024 14:27:45 +0530 Subject: [PATCH 14/17] Add TaskRunner Gramine-Direct CI E2E test Signed-off-by: Shah, Karan --- .../workflows/tr_docker_gramine_direct.yml | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 .github/workflows/tr_docker_gramine_direct.yml diff --git a/.github/workflows/tr_docker_gramine_direct.yml b/.github/workflows/tr_docker_gramine_direct.yml new file mode 100644 index 0000000000..c415f9055f --- /dev/null +++ b/.github/workflows/tr_docker_gramine_direct.yml @@ -0,0 +1,97 @@ +# Tests an FL experiment in a Dockerized environment. +name: Task Runner API (Docker/gramine-direct) + +on: + pull_request: + branches: [ develop ] + types: [opened, synchronize, reopened, ready_for_review] + +permissions: + contents: read + +jobs: + build: + if: github.event.pull_request.draft == false + runs-on: ubuntu-latest + timeout-minutes: 10 + + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.8 + uses: actions/setup-python@v3 + with: + python-version: "3.8" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install . + + - name: Create workspace image + run: | + fx workspace create --prefix example_workspace --template keras_cnn_mnist + cd example_workspace + fx plan initialize -a localhost + fx workspace dockerize --save --revision https://github.com/${GITHUB_REPOSITORY}.git@${{ github.event.pull_request.head.sha }} + + - name: Create certificate authority for workspace + run: | + cd example_workspace + fx workspace certify + + - name: Create signed cert for collaborator + run: | + cd example_workspace + fx collaborator create -d 1 -n charlie --silent + fx collaborator generate-cert-request -n charlie --silent + fx collaborator certify --request-pkg col_charlie_to_agg_cert_request.zip --silent + + # Pack the collaborator's private key, signed cert, and data.yaml into a tarball + tarfiles="plan/data.yaml agg_to_col_charlie_signed_cert.zip" + for entry in cert/client/*; do + if [[ "$entry" == *.key ]]; then + tarfiles="$tarfiles $entry" + fi + done + + tar -cf cert_col_charlie.tar $tarfiles + + # Clean up + rm -f $tarfiles + rm -f col_charlie_to_agg_cert_request.zip + + - name: Create signed cert for aggregator + run: | + cd example_workspace + fx aggregator generate-cert-request --fqdn localhost + fx aggregator certify --fqdn localhost --silent + + # Pack all files that aggregator needs to start training + tar -cf cert_agg.tar plan cert save + + # Remove the directories after archiving + rm -rf plan cert save + + - name: Load workspace image + run: | + cd example_workspace + docker load -i example_workspace.tar + + - name: Run aggregator and collaborator + run: | + cd example_workspace + + set -x + docker run --rm \ + --network host \ + --security-opt seccomp=unconfined \ + --mount type=bind,source=./cert_agg.tar,target=/certs.tar \ + --env KERAS_HOME=/tmp \ + example_workspace bash -c "tar -xf /certs.tar && gramine-direct fx aggregator start" & + + # TODO: Run with two collaborators instead. + docker run --rm \ + --network host \ + --security-opt seccomp=unconfined \ + --mount type=bind,source=./cert_col_charlie.tar,target=/certs.tar \ + --env KERAS_HOME=/tmp \ + example_workspace bash -c "tar -xf /certs.tar && fx collaborator certify --import agg_to_col_charlie_signed_cert.zip && gramine-direct fx collaborator start -n charlie" \ No newline at end of file From ff1619cfd3b7de32d3adcfc8ccd08ea711880b48 Mon Sep 17 00:00:00 2001 From: "Shah, Karan" Date: Thu, 14 Nov 2024 14:36:05 +0530 Subject: [PATCH 15/17] Update command help Signed-off-by: Shah, Karan --- openfl/interface/workspace.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/openfl/interface/workspace.py b/openfl/interface/workspace.py index 2da2b1bc04..b138ad67db 100644 --- a/openfl/interface/workspace.py +++ b/openfl/interface/workspace.py @@ -394,7 +394,11 @@ def export_() -> str: "enclave_key", type=str, required=False, - help="Path to an enclave signing key. If not provided, a new key will be generated. ", + help=( + "Path to an enclave signing key. If not provided, a key will be auto-generated in the workspace. " + "Note that this command builds a TEE-ready image, key is NOT packaged along with the image. " + "You have the flexibility to not run inside a TEE later." + ), ) @option( "--revision", @@ -409,7 +413,7 @@ def export_() -> str: ) @pass_context def dockerize_(context, save: bool, rebuild: bool, enclave_key: str, revision: str): - """Package current workspace as a Docker image.""" + """Package current workspace as a TEE-ready Docker image.""" # Docker build options options = [] From 12756b12c198560c4af8f2998d98dea24c4d572a Mon Sep 17 00:00:00 2001 From: "Shah, Karan" Date: Thu, 14 Nov 2024 14:40:54 +0530 Subject: [PATCH 16/17] Rename tests Signed-off-by: Shah, Karan --- .github/workflows/tr_docker_gramine_direct.yml | 2 +- .github/workflows/tr_docker_native.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tr_docker_gramine_direct.yml b/.github/workflows/tr_docker_gramine_direct.yml index c415f9055f..d8f7480ea1 100644 --- a/.github/workflows/tr_docker_gramine_direct.yml +++ b/.github/workflows/tr_docker_gramine_direct.yml @@ -1,5 +1,5 @@ # Tests an FL experiment in a Dockerized environment. -name: Task Runner API (Docker/gramine-direct) +name: TaskRunner (docker/gramine-direct) on: pull_request: diff --git a/.github/workflows/tr_docker_native.yml b/.github/workflows/tr_docker_native.yml index b333b54059..899fcd8296 100644 --- a/.github/workflows/tr_docker_native.yml +++ b/.github/workflows/tr_docker_native.yml @@ -1,5 +1,5 @@ # Tests an FL experiment in a Dockerized environment. -name: Task Runner API (Docker/native) +name: TaskRunner (docker/native) on: pull_request: From e4139ae0c2246b54ca297e7a690b20154fa8617b Mon Sep 17 00:00:00 2001 From: "Shah, Karan" Date: Thu, 14 Nov 2024 15:44:58 +0530 Subject: [PATCH 17/17] Add README.md Signed-off-by: Shah, Karan --- openfl-docker/README.md | 89 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 openfl-docker/README.md diff --git a/openfl-docker/README.md b/openfl-docker/README.md new file mode 100644 index 0000000000..da8540770d --- /dev/null +++ b/openfl-docker/README.md @@ -0,0 +1,89 @@ +# Using OpenFL within a Container + +OpenFL can be used within a container for simulating Federated Learning experiments, or to deploy real-world experiments within Trusted Execution Environments (TEEs). + +## Base Image + +To develop or simulate experiments within a container, build the base image (or pull one from docker hub). + +```shell +# Pull latest stable base image +$> docker pull intel/openfl + +# Or, build a base image from the latest source code +$> docker build . -t openfl -f Dockerfile.base \ + --build-arg OPENFL_REVISION=https://github.com/securefederatedai/openfl.git@develop +``` + +Run the container: +```shell +user@vm:~/openfl$ docker run -it --rm openfl:latest bash +user@7b40624c207a:/$ fx +OpenFL - Open Federated Learning + +BASH COMPLETE ACTIVATION + +Run in terminal: + _FX_COMPLETE=bash_source fx > ~/.fx-autocomplete.sh + source ~/.fx-autocomplete.sh +If ~/.fx-autocomplete.sh has already exist: + source ~/.fx-autocomplete.sh + +CORRECT USAGE + +fx [options] [command] [subcommand] [args] +``` + +## Deployment +This section assumes familiarity with the [TaskRunner API](https://openfl.readthedocs.io/en/latest/about/features_index/taskrunner.html#running-the-task-runner). + +### Building a workspace image +OpenFL supports [Gramine-based](https://gramine.readthedocs.io/en/stable/) TEEs that run within SGX. + +To build a TEE-ready workspace image, run the following command from an existing workspace directory. Ensure PKI setup and plan confirmations are done before this step. + +```shell +# Optional, generate an enclave signing key (auto-generated otherwise) +user@vm:~/example_workspace$ openssl genrsa -out key.pem -3 3072 +user@vm:~/example_workspace$ fx workspace dockerize --enclave-key ./key.pem --save +``` +This command builds the base image and a TEE-ready workspace image. Refer to `fx workspace dockerize --help` for more details. + +A signed docker image named `example_workspace.tar` will be saved in the workspace. This image (along with respective PKI certificates) can be shared across participating entities. + +### Running without a TEE +Using native `fx` command within the image will run the experiment without TEEs. + +```shell +# Aggregator +docker run --rm \ + --network host \ + --mount type=bind,source=./certs.tar,target=/certs.tar \ + example_workspace bash -c "fx aggregator start ..." + +# Collaborator(s) +docker run --rm \ + --network host \ + --mount type=bind,source=./certs.tar,target=/certs.tar \ + example_workspace bash -c "fx collaborator start ..." +``` + +### Running within a TEE +To run `fx` within a TEE, mount SGX device and AESMD volumes. In addition, prefix the `fx` command with `gramine-sgx` directive. +```shell +# Aggregator +docker run --rm \ + --network host \ + --device=/dev/sgx_enclave \ + -v /var/run/aesmd/aesm.socket:/var/run/aesmd/aesm.socket \ + --mount type=bind,source=./certs.tar,target=/certs.tar \ + example_workspace bash -c "gramine-sgx fx aggregator start ..." + +# Collaborator(s) +docker run --rm \ + --network host \ + --device=/dev/sgx_enclave \ + -v /var/run/aesmd/aesm.socket:/var/run/aesmd/aesm.socket \ + --mount type=bind,source=./certs.tar,target=/certs.tar \ + example_workspace bash -c "gramine-sgx fx collaborator start ..." +``` \ No newline at end of file