diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index ff25b97d..713f8e81 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -81,31 +81,25 @@ jobs: sed -e 's#\(.*container_instance = "\)\(.*\)$#\1'$PAYLOAD_ARTIFACTS'\2#g' config/config.toml | sudo tee /etc/enclave-cc/config.toml working-directory: ${{ github.workspace }}/src/github.com/confidential-containers/enclave-cc/src/shim - - name: Build agent-enclave bundle + - name: Build unified bundle run: | mkdir $PAYLOAD_ARTIFACTS - docker build . -f tools/packaging/build/agent-enclave-bundle/Dockerfile --build-arg SGX_MODE=${SGX_MODE} --build-arg KBC=${KBC} -t agent-instance:build - docker export $(docker create agent-instance:build) | tee > ${PAYLOAD_ARTIFACTS}/agent-instance.tar + docker build . -f tools/packaging/build/unified-bundle/Dockerfile --build-arg SGX_MODE=${SGX_MODE} --build-arg KBC=${KBC} -t unified-instance:build + docker export $(docker create unified-instance:build) | tee > ${PAYLOAD_ARTIFACTS}/unified-instance.tar working-directory: ${{ github.workspace }}/src/github.com/confidential-containers/enclave-cc - name: Install config.json for agent-enclave bundle run: | jq -a -f sgx-mode-config.filter config.json.template | tee ${PAYLOAD_ARTIFACTS}/config.json - working-directory: ${{ github.workspace }}/src/github.com/confidential-containers/enclave-cc/tools/packaging/build/agent-enclave-bundle - - - name: Build boot-instance bundle - run: | - docker build . -f tools/packaging/build/boot-instance-bundle/Dockerfile --build-arg SGX_MODE=${SGX_MODE} -t boot-instance:build - docker export $(docker create boot-instance:build) | tee > ${PAYLOAD_ARTIFACTS}/boot-instance.tar - working-directory: ${{ github.workspace }}/src/github.com/confidential-containers/enclave-cc + working-directory: ${{ github.workspace }}/src/github.com/confidential-containers/enclave-cc/tools/packaging/build/unified-bundle - name: Install enclave-cc bundles run: | mkdir -p opt/confidential-containers/share/enclave-cc-agent-instance/rootfs - sudo tar -xf agent-instance.tar -C opt/confidential-containers/share/enclave-cc-agent-instance/rootfs + sudo tar -xf unified-instance.tar -C opt/confidential-containers/share/enclave-cc-agent-instance/rootfs cp config.json opt/confidential-containers/share/enclave-cc-agent-instance/ mkdir -p opt/confidential-containers/share/enclave-cc-boot-instance/rootfs - sudo tar -xf boot-instance.tar -C opt/confidential-containers/share/enclave-cc-boot-instance/rootfs + sudo tar -xf unified-instance.tar -C opt/confidential-containers/share/enclave-cc-boot-instance/rootfs working-directory: ${{env.PAYLOAD_ARTIFACTS}} - name: Install decrypt_config.conf and ocicrypt.conf for agent-enclave bundle diff --git a/tools/packaging/build/build_payload.sh b/tools/packaging/build/build_payload.sh index d6739704..4b61fe13 100755 --- a/tools/packaging/build/build_payload.sh +++ b/tools/packaging/build/build_payload.sh @@ -22,16 +22,12 @@ export PAYLOAD_ARTIFACTS="${SCRIPT_ROOT}/payload_artifacts" mkdir -p ${PAYLOAD_ARTIFACTS} # build pre-installed OCI bundle for agent enclave container -pushd ${SCRIPT_ROOT}/agent-enclave-bundle -docker build ${ENCLAVE_CC_ROOT} -f ${SCRIPT_ROOT}/agent-enclave-bundle/Dockerfile --build-arg SGX_MODE=${SGX_MODE} --build-arg KBC=${KBC} -t agent-instance +pushd ${SCRIPT_ROOT}/unified-bundle +docker build ${ENCLAVE_CC_ROOT} -f ${SCRIPT_ROOT}/unified-bundle/Dockerfile --build-arg SGX_MODE=${SGX_MODE} --build-arg KBC=${KBC} -t unified-instance jq -a -f sgx-mode-config.filter config.json.template | tee ${PAYLOAD_ARTIFACTS}/config.json -docker export $(docker create agent-instance) | tee > ${PAYLOAD_ARTIFACTS}/agent-instance.tar +docker export $(docker create unified-instance) | tee > ${PAYLOAD_ARTIFACTS}/unified-instance.tar popd -# build pre-installed OCI bundle for boot instance -docker build ${ENCLAVE_CC_ROOT} -f ${SCRIPT_ROOT}/boot-instance-bundle/Dockerfile --build-arg SGX_MODE=${SGX_MODE} -t boot-instance -docker export $(docker create boot-instance) | tee > ${PAYLOAD_ARTIFACTS}/boot-instance.tar - # build shim-rune binary: "containerd-shim-rune-v2" pushd ${ENCLAVE_CC_ROOT}/src/shim docker run --pull always -t -v ${PWD}:/build --workdir /build golang:${GO_VERSION}-bullseye make binaries @@ -55,5 +51,5 @@ fi popd #cleanup -docker rmi ${IMAGE} boot-instance agent-instance -f +docker rmi ${IMAGE} unified-instance -f rm -rf payload_artifacts diff --git a/tools/packaging/build/unified-bundle/Dockerfile b/tools/packaging/build/unified-bundle/Dockerfile new file mode 100644 index 00000000..4fdc7182 --- /dev/null +++ b/tools/packaging/build/unified-bundle/Dockerfile @@ -0,0 +1,135 @@ +FROM ubuntu:20.04 as builder + +ARG KBC=cc-kbc + +RUN apt-get update && \ + env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + make \ + ca-certificates \ + curl \ + wget \ + gnupg + +ARG OCCLUM_VERSION=0.29.7-1 +ARG SGXSDK_VERSION=2_22_100 +ARG RUST_VERSION=1.72.0 + +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain=${RUST_VERSION} +RUN echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel-sgx.gpg] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main" | tee -a /etc/apt/sources.list.d/intel-sgx.list \ + && wget -qO - https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | gpg --dearmor --output /usr/share/keyrings/intel-sgx.gpg \ + && wget -qO - https://download.01.org/intel-sgx/sgx_repo/ubuntu/apt_preference_files/99sgx_${SGXSDK_VERSION}_focal_custom_version.cfg | tee -a /etc/apt/preferences.d/99sgx_sdk.cfg \ + && apt-get update \ + && env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + libsgx-uae-service \ + libsgx-dcap-ql \ + libsgx-dcap-ql-dev \ + libsgx-dcap-quote-verify \ + libsgx-dcap-quote-verify-dev \ + libsgx-quote-ex \ + libsgx-dcap-default-qpl + +RUN echo "deb [arch=amd64 signed-by=/usr/share/keyrings/occlum.gpg] https://occlum.io/occlum-package-repos/debian focal main" | tee -a /etc/apt/sources.list.d/occlum.list \ + && wget -qO - https://occlum.io/occlum-package-repos/debian/public.key | gpg --dearmor --output /usr/share/keyrings/occlum.gpg \ + && apt-get update +RUN env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + binutils \ + build-essential \ + cmake \ + git \ + libssl-dev \ + jq \ + tzdata \ + binutils \ + libclang-dev \ + libfuse2 \ + libfuse3-3 \ + ca-certificates \ + protobuf-compiler \ + rsync \ + occlum-toolchains-glibc \ + occlum-pal=${OCCLUM_VERSION} \ + occlum-sgx-tools=${OCCLUM_VERSION} \ + occlum-runtime=${OCCLUM_VERSION} \ + occlum=${OCCLUM_VERSION} + +COPY src/ /enclave-cc/src/ +RUN cd /enclave-cc/src/enclave-agent && \ + . $HOME/.cargo/env && \ + make ${KBC} +RUN cd /enclave-cc/src/runtime-boot/init && \ + . $HOME/.cargo/env && \ + cargo build --release + +WORKDIR /run/unified +COPY tools/packaging/build/unified-bundle/enclave-agent-${KBC}.yaml . +COPY tools/packaging/build/unified-bundle/boot_template.yaml . +COPY tools/packaging/build/unified-bundle/init.yaml . +COPY tools/packaging/build/unified-bundle/agent-instance.filter . +COPY tools/packaging/build/unified-bundle/boot-instance.filter . + +ARG SGX_MODE=HW +RUN export PATH="$PATH:/opt/occlum/build/bin" && \ + occlum new occlum_instance && cd occlum_instance && \ + copy_bom -f ../enclave-agent-${KBC}.yaml --root image \ + --include-dir /opt/occlum/etc/template && \ + mv Occlum.json Occlum.json.orig && \ + jq -a -f ../agent-instance.filter Occlum.json.orig > Occlum.json && \ + SGX_MODE=${SGX_MODE} occlum build && \ + if [ $SGX_MODE="SIM" ]; then \ + export DEBUG="--debug"; \ + fi; occlum package ${DEBUG} + +RUN export PATH="$PATH:/opt/occlum/build/bin" && \ + occlum new boot_instance && cd boot_instance && \ + copy_bom -f ../boot_template.yaml --root image \ + --include-dir /opt/occlum/etc/template && \ + copy_bom -f ../init.yaml --root initfs \ + --include-dir /opt/occlum/etc/template && \ + mv Occlum.json Occlum.json.orig && \ + jq -a -f ../boot-instance.filter Occlum.json.orig > Occlum.json && \ + SGX_MODE=${SGX_MODE} occlum build && \ + if [ $SGX_MODE="SIM" ]; then \ + export DEBUG="--debug"; \ + fi; occlum package ${DEBUG} + +FROM ubuntu:20.04 + +RUN apt-get update && \ + env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates \ + wget \ + gnupg + +ARG OCCLUM_VERSION=0.29.7-1 +ARG SGXSDK_VERSION=2_22_100 + +RUN echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel-sgx.gpg] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main" | tee -a /etc/apt/sources.list.d/intel-sgx.list \ + && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/occlum.gpg] https://occlum.io/occlum-package-repos/debian focal main" | tee -a /etc/apt/sources.list.d/occlum.list \ + && wget -qO - https://occlum.io/occlum-package-repos/debian/public.key | gpg --dearmor --output /usr/share/keyrings/occlum.gpg \ + && wget -qO - https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | gpg --dearmor --output /usr/share/keyrings/intel-sgx.gpg \ + && wget -qO - https://download.01.org/intel-sgx/sgx_repo/ubuntu/apt_preference_files/99sgx_${SGXSDK_VERSION}_focal_custom_version.cfg | tee -a /etc/apt/preferences.d/99sgx_sdk.cfg \ + && apt-get update \ + && env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + libsgx-uae-service \ + libsgx-dcap-ql \ + libsgx-dcap-quote-verify \ + libsgx-quote-ex \ + libsgx-dcap-default-qpl \ + tzdata \ + libfuse2 \ + libfuse3-3 \ + occlum-runtime=${OCCLUM_VERSION} \ + occlum-sgx-tools=${OCCLUM_VERSION} + +WORKDIR /run/rune +COPY --from=builder /run/unified/occlum_instance/occlum_instance.tar.gz . +COPY --from=builder /run/unified/boot_instance/boot_instance.tar.gz . + +RUN tar xf occlum_instance.tar.gz && \ + tar xf boot_instance.tar.gz && \ + echo "/run/rune/occlum_instance/build/lib/" | tee /etc/ld.so.conf.d/occlum-pal.conf && \ + echo "/opt/occlum/sgxsdk-tools/sdk_libs/" | tee -a /etc/ld.so.conf.d/occlum-pal.conf && \ + ldconfig && \ + /opt/occlum/sgxsdk-tools/bin/x64/sgx_sign dump -dumpfile /run/agent.txt -cssfile /run/sigstruct1.bin -enclave occlum_instance/build/lib/libocclum-libos.signed.so && \ + /opt/occlum/sgxsdk-tools/bin/x64/sgx_sign dump -dumpfile /run/boot.txt -cssfile /run/sigstruct2.bin -enclave boot_instance/build/lib/libocclum-libos.signed.so && \ + rm *.tar.gz diff --git a/tools/packaging/build/unified-bundle/agent-instance.filter b/tools/packaging/build/unified-bundle/agent-instance.filter new file mode 100644 index 00000000..25951a0b --- /dev/null +++ b/tools/packaging/build/unified-bundle/agent-instance.filter @@ -0,0 +1,12 @@ +.resource_limits.user_space_size = "2000MB" | +.resource_limits.kernel_space_heap_size = "1024MB" | +.resource_limits.kernel_space_stack_size= "128MB" | +.resource_limits.max_num_of_threads = 8 | +.entry_points = ["/bin/enclave-agent"] | +.mount |= . + +[{ + "target": "/etc/", + "type": "hostfs", + "source": "/configs" +}] | +if $ENV.SGX_MODE == "SIM" then .metadata.debuggable = true else .metadata.debuggable = false end diff --git a/tools/packaging/build/unified-bundle/boot-instance.filter b/tools/packaging/build/unified-bundle/boot-instance.filter new file mode 100644 index 00000000..23b8739e --- /dev/null +++ b/tools/packaging/build/unified-bundle/boot-instance.filter @@ -0,0 +1,4 @@ +.resource_limits.user_space_size = "600MB" | +.resource_limits.kernel_space_stack_size= "2MB" | +.env.default |= . + ["LD_LIBRARY_PATH=/opt/occlum/glibc/lib:/lib/x86_64-linux-gnu/:/usr/local/lib/x86_64-linux-gnu:/usr/lib/x86_64-linux-gnu:/usr/local/lib"] | +if $ENV.SGX_MODE == "SIM" then .metadata.debuggable = true else .metadata.debuggable = false end diff --git a/tools/packaging/build/unified-bundle/boot_template.yaml b/tools/packaging/build/unified-bundle/boot_template.yaml new file mode 100644 index 00000000..87a511a5 --- /dev/null +++ b/tools/packaging/build/unified-bundle/boot_template.yaml @@ -0,0 +1,7 @@ +includes: + - base.yaml +targets: + - target: /bin + copy: + - files: + - /usr/bin/hostname diff --git a/tools/packaging/build/unified-bundle/config.json.template b/tools/packaging/build/unified-bundle/config.json.template new file mode 100644 index 00000000..a5b5d785 --- /dev/null +++ b/tools/packaging/build/unified-bundle/config.json.template @@ -0,0 +1,203 @@ +{ + "ociVersion": "1.0.2-dev", + "process": { + "terminal": false, + "user": { + "uid": 0, + "gid": 0 + }, + "args": [ + "/run/rune/occlum_instance/build/bin/occlum-run", + "/bin/enclave-agent", + "-c", + "/etc/decrypt_config.conf", + "-o", + "/etc/ocicrypt.conf", + "-l", + "tcp://127.0.0.1:7788" + ], + "env": [ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + "TERM=xterm" + ], + "cwd": "/run/rune/occlum_instance", + "capabilities": { + "bounding": [ + "CAP_AUDIT_WRITE", + "CAP_KILL", + "CAP_NET_BIND_SERVICE" + ], + "effective": [ + "CAP_AUDIT_WRITE", + "CAP_KILL", + "CAP_NET_BIND_SERVICE" + ], + "inheritable": [ + "CAP_AUDIT_WRITE", + "CAP_KILL", + "CAP_NET_BIND_SERVICE" + ], + "permitted": [ + "CAP_AUDIT_WRITE", + "CAP_KILL", + "CAP_NET_BIND_SERVICE" + ], + "ambient": [ + "CAP_AUDIT_WRITE", + "CAP_KILL", + "CAP_NET_BIND_SERVICE" + ] + }, + "rlimits": [ + { + "type": "RLIMIT_NOFILE", + "hard": 65535, + "soft": 65535 + } + ], + "noNewPrivileges": true + }, + "root": { + "path": "rootfs" + }, + "hostname": "rune", + "mounts": [ + { + "destination": "/proc", + "type": "proc", + "source": "proc" + }, + { + "destination": "/etc/resolv.conf", + "type": "bind", + "source": "/etc/resolv.conf", + "options": [ + "rbind", + "rprivate", + "ro" + ] + }, + { + "destination": "/etc/hostname", + "type": "bind", + "source": "/etc/hostname", + "options": [ + "rbind", + "rprivate", + "ro" + ] + }, + { + "destination": "/dev", + "type": "tmpfs", + "source": "tmpfs", + "options": [ + "nosuid", + "strictatime", + "mode=755", + "size=65536k" + ] + }, + { + "destination": "/dev/pts", + "type": "devpts", + "source": "devpts", + "options": [ + "nosuid", + "noexec", + "newinstance", + "ptmxmode=0666", + "mode=0620", + "gid=5" + ] + }, + { + "destination": "/dev/shm", + "type": "tmpfs", + "source": "shm", + "options": [ + "nosuid", + "noexec", + "nodev", + "mode=1777", + "size=65536k" + ] + }, + { + "destination": "/dev/mqueue", + "type": "mqueue", + "source": "mqueue", + "options": [ + "nosuid", + "noexec", + "nodev" + ] + }, + { + "destination": "/sys", + "type": "sysfs", + "source": "sysfs", + "options": [ + "nosuid", + "noexec", + "nodev", + "ro" + ] + }, + { + "destination": "/sys/fs/cgroup", + "type": "cgroup", + "source": "cgroup", + "options": [ + "nosuid", + "noexec", + "nodev", + "relatime", + "ro" + ] + } + ], + "linux": { + "resources": { + "devices": [ + { + "allow": false, + "access": "rwm" + } + ] + }, + "namespaces": [ + { + "type": "pid" + }, + { + "type": "ipc" + }, + { + "type": "uts" + }, + { + "type": "mount" + } + ], + "maskedPaths": [ + "/proc/acpi", + "/proc/asound", + "/proc/kcore", + "/proc/keys", + "/proc/latency_stats", + "/proc/timer_list", + "/proc/timer_stats", + "/proc/sched_debug", + "/sys/firmware", + "/proc/scsi" + ], + "readonlyPaths": [ + "/proc/bus", + "/proc/fs", + "/proc/irq", + "/proc/sys", + "/proc/sysrq-trigger" + ] + } +} diff --git a/tools/packaging/build/unified-bundle/enclave-agent-cc-kbc.yaml b/tools/packaging/build/unified-bundle/enclave-agent-cc-kbc.yaml new file mode 100644 index 00000000..e2d93ab6 --- /dev/null +++ b/tools/packaging/build/unified-bundle/enclave-agent-cc-kbc.yaml @@ -0,0 +1,20 @@ +includes: + - base.yaml +targets: + - target: /bin + copy: + - files: + - /enclave-cc/src/enclave-agent/target/release/enclave-agent + # copy libnss_files + - target: /opt/occlum/glibc/lib + copy: + - files: + - /opt/occlum/glibc/lib/libnss_files.so.2 + - /opt/occlum/glibc/lib/libnss_dns.so.2 + - /lib/x86_64-linux-gnu/libdl.so.2 + - /usr/lib/x86_64-linux-gnu/libssl.so.1.1 + - /usr/lib/x86_64-linux-gnu/libcrypto.so.1.1 + - target: /usr/lib/ssl + copy: + - dirs: + - /etc/ssl/certs diff --git a/tools/packaging/build/unified-bundle/enclave-agent-sample-kbc.yaml b/tools/packaging/build/unified-bundle/enclave-agent-sample-kbc.yaml new file mode 100644 index 00000000..e2d93ab6 --- /dev/null +++ b/tools/packaging/build/unified-bundle/enclave-agent-sample-kbc.yaml @@ -0,0 +1,20 @@ +includes: + - base.yaml +targets: + - target: /bin + copy: + - files: + - /enclave-cc/src/enclave-agent/target/release/enclave-agent + # copy libnss_files + - target: /opt/occlum/glibc/lib + copy: + - files: + - /opt/occlum/glibc/lib/libnss_files.so.2 + - /opt/occlum/glibc/lib/libnss_dns.so.2 + - /lib/x86_64-linux-gnu/libdl.so.2 + - /usr/lib/x86_64-linux-gnu/libssl.so.1.1 + - /usr/lib/x86_64-linux-gnu/libcrypto.so.1.1 + - target: /usr/lib/ssl + copy: + - dirs: + - /etc/ssl/certs diff --git a/tools/packaging/build/unified-bundle/init.yaml b/tools/packaging/build/unified-bundle/init.yaml new file mode 100644 index 00000000..cc3a15ca --- /dev/null +++ b/tools/packaging/build/unified-bundle/init.yaml @@ -0,0 +1,7 @@ +includes: + - base.yaml +targets: + - target: /bin/ + copy: + - files: + - /enclave-cc/src/runtime-boot/init/target/release/init diff --git a/tools/packaging/build/unified-bundle/sgx-mode-config.filter b/tools/packaging/build/unified-bundle/sgx-mode-config.filter new file mode 100644 index 00000000..2e1bf899 --- /dev/null +++ b/tools/packaging/build/unified-bundle/sgx-mode-config.filter @@ -0,0 +1,31 @@ +if $ENV.SGX_MODE == "SIM" then +.process.env |= . + ["OCCLUM_RELEASE_ENCLAVE=0"] +else +.process.env |= . + ["OCCLUM_RELEASE_ENCLAVE=1", "SGX_AESM_ADDR=1"] | +.linux.devices |= . + +[{ + "path": "/dev/sgx_enclave", + "type": "c", + "major": 10, + "minor": 125, + "fileMode": 438 +}] | +.linux.resources.devices |= . + +[{ + "allow": true, + "type": "c", + "major": 10, + "minor": 125, + "access": "rwm" +}] | +.mounts |= . + +[{ + "destination": "/var/run/aesmd", + "type": "bind", + "source": "/var/run/aesmd", + "options": [ + "rbind", + "rw" + ] +}] +end diff --git a/tools/packaging/deploy/enclave-cc-deploy.sh b/tools/packaging/deploy/enclave-cc-deploy.sh index e588e153..b9637d31 100755 --- a/tools/packaging/deploy/enclave-cc-deploy.sh +++ b/tools/packaging/deploy/enclave-cc-deploy.sh @@ -63,11 +63,11 @@ function get_container_runtime() { function install_artifacts() { echo "copying enclave-cc artifacts onto host" mkdir -p /opt/confidential-containers/share/enclave-cc-agent-instance/rootfs - tar -xf agent-instance.tar -C /opt/confidential-containers/share/enclave-cc-agent-instance/rootfs + tar -xf unified-instance.tar -C /opt/confidential-containers/share/enclave-cc-agent-instance/rootfs cp config.json /opt/confidential-containers/share/enclave-cc-agent-instance mkdir -p /opt/confidential-containers/share/enclave-cc-boot-instance/rootfs - tar -xf boot-instance.tar -C /opt/confidential-containers/share/enclave-cc-boot-instance/rootfs + tar -xf unified-instance.tar -C /opt/confidential-containers/share/enclave-cc-boot-instance/rootfs cp shim-rune-config.toml /etc/enclave-cc/config.toml