diff --git a/.gitignore b/.gitignore index cf06367..3cfd882 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,6 @@ acme.json # Proxy Build proxy/build + +# Filesystem +*.ext4 diff --git a/buildroot/README.md b/buildroot/README.md new file mode 100644 index 0000000..13ac642 --- /dev/null +++ b/buildroot/README.md @@ -0,0 +1,80 @@ +# Buildroot + +To build the HAL for Linux, you need to install [Buildroot](https://buildroot.org/). Checkout [README.md](./linux/README.md) for more information. + +## To run using qemu + +After following the steps in [README.md](./linux/README.md), you will have bzImage and rootfs.cpio.gz files. + +Next we need to create a filesystem image. We will use `mkfs.ext4` to create the filesystem image. + +```bash +dd if=/dev/zero of=rootfs.img bs=1M count=30720 +mkfs.ext4 ./rootfs.img +``` + +Now we can run the QEMU VM with the filesystem image. + +```bash +sudo bash buildroot/qemu.sh start_cvm +``` + +If you want to start a normal VM, you can run: + +```bash +sudo bash buildroot/qemu.sh start +``` + +Login to the VM using the following credentials: + +- Username: `root` + +Attest the VM by running the following command: + +```bash +bash /cube/attest.sh +``` + +You will see a report similar to the following: + +```bash +The AMD ARK was self-signed! +The AMD ASK was signed by the AMD ARK! +The VCEK was signed by the AMD ASK! +Reported TCB Boot Loader from certificate matches the attestation report. +Reported TCB TEE from certificate matches the attestation report. +Reported TCB SNP from certificate matches the attestation report. +Reported TCB Microcode from certificate matches the attestation report. +Chip ID from certificate matches the attestation report. +VEK signed the Attestation Report! +Measurement from SNP Attestation Report: daa2e216eafd8c6404b72157a130500ab0c0944064c8e1009ebf5e910371caf57a6711654108a01a69baaa1a05759cf0 +``` + +Clone the repository. Since this is a private repository, you need to create a classic personal access token with `repo` and `read:packages` permissions. + +```bash +git clone https://github.com/ultravioletrs/cube.git +``` + +Your username is your github username and your password is the access token you generated in step above. + +Login to the docker registry + +```bash +docker login ghcr.io +``` + +Your username is your github username and your password is the access token you generated in step above. + +Pull the docker images + +```bash +cd cube/docker-compose/ +docker compose pull +``` + +Start the docker composition + +```bash +docker compose up -d +``` diff --git a/buildroot/linux/Config.in b/buildroot/linux/Config.in index 8b13789..835cef4 100644 --- a/buildroot/linux/Config.in +++ b/buildroot/linux/Config.in @@ -1 +1,2 @@ - +source "$BR2_EXTERNAL_CUBE_PATH/package/setup/Config.in" +source "$BR2_EXTERNAL_CUBE_PATH/package/snpguest/Config.in" diff --git a/buildroot/linux/README.md b/buildroot/linux/README.md index cba37e9..6040229 100644 --- a/buildroot/linux/README.md +++ b/buildroot/linux/README.md @@ -8,8 +8,8 @@ HAL uses [Buildroot](https://buildroot.org/)'s [_External Tree_ mechanism](https ```bash git clone https://gitlab.com/buildroot.org/buildroot.git +git clone https://github.com/ultravioletrs/cube.git cd buildroot -git checkout 2023.08 make BR2_EXTERNAL=../cube/buildroot/linux cube_defconfig # Execute 'make menuconfig' only if you want to make additional configuration changes to Buildroot. make menuconfig diff --git a/buildroot/linux/board/cube/README.md b/buildroot/linux/board/cube/README.md deleted file mode 100644 index 1af41ad..0000000 --- a/buildroot/linux/board/cube/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# Cube - -Run the emulation with: - -```bash -qemu-system-x86_64 -M pc -kernel output/images/bzImage -drive file=output/images/rootfs.ext2,if=virtio,format=raw -append "rootwait root=/dev/vda console=tty1 console=ttyS0" -serial stdio -net nic,model=virtio -net user # cube_defconfig -``` - -Optionally add `-smp N` to emulate a SMP system with N CPUs. - -The login prompt will appear in the graphical window. diff --git a/buildroot/linux/board/cube/linux.config b/buildroot/linux/board/cube/linux.config index 5c8c3ac..ad5e14d 100644 --- a/buildroot/linux/board/cube/linux.config +++ b/buildroot/linux/board/cube/linux.config @@ -34,6 +34,7 @@ CONFIG_VIRTIO_INPUT=y CONFIG_VIRTIO_MMIO=y CONFIG_VIRTIO_MMIO_CMDLINE_DEVICES=y CONFIG_EXT4_FS=y +CONFIG_EXT4_FS_POSIX_ACL=y CONFIG_AUTOFS4_FS=y CONFIG_TMPFS=y CONFIG_TMPFS_POSIX_ACL=y @@ -65,3 +66,15 @@ CONFIG_PREEMPT_DYNAMIC=n CONFIG_DEBUG_PREEMPT=n CONFIG_CGROUP_MISC=y CONFIG_X86_CPUID=y +CONFIG_X86_MSR=y + +CONFIG_KERNEL_EXT4_FS_SECURITY=y +CONFIG_EXT4_FS_SECURITY=y + +CONFIG_QUOTA=y +CONFIG_QUOTA_NETLINK_INTERFACE=y +CONFIG_PRINT_QUOTA_WARNING=y +CONFIG_QUOTA_DEBUG=y +CONFIG_QUOTA_TREE=y +CONFIG_QFMT_V2=y +CONFIG_EXT4_FS_QUOTA=y diff --git a/buildroot/linux/board/cube/readme.txt b/buildroot/linux/board/cube/readme.txt new file mode 100644 index 0000000..05fd393 --- /dev/null +++ b/buildroot/linux/board/cube/readme.txt @@ -0,0 +1,7 @@ +Run the emulation with: + + qemu-system-x86_64 -M pc -kernel output/images/bzImage -drive file=output/images/rootfs.ext2,if=virtio,format=raw -append "rootwait root=/dev/vda console=tty1 console=ttyS0" -serial stdio -net nic,model=virtio -net user # cube_defconfig + +Optionally add -smp N to emulate a SMP system with N CPUs. + +The login prompt will appear in the graphical window. diff --git a/buildroot/linux/configs/cube_defconfig b/buildroot/linux/configs/cube_defconfig index a07bd51..5ff0891 100644 --- a/buildroot/linux/configs/cube_defconfig +++ b/buildroot/linux/configs/cube_defconfig @@ -2,7 +2,7 @@ BR2_x86_64=y # System -BR2_TARGET_GENERIC_HOSTNAME="Cube AI" +BR2_TARGET_GENERIC_HOSTNAME="cube" BR2_TARGET_GENERIC_ISSUE="Welcome to Cube AI" BR2_PACKAGE_DHCP=y BR2_PACKAGE_DHCP_CLIENT=y @@ -39,6 +39,7 @@ BR2_LINUX_KERNEL_PATCH="" BR2_LINUX_KERNEL_USE_CUSTOM_CONFIG=y BR2_LINUX_KERNEL_CUSTOM_CONFIG_FILE="$(BR2_EXTERNAL_CUBE_PATH)/board/cube/linux.config" BR2_LINUX_KERNEL_NEEDS_HOST_LIBELF=y +BR2_LINUX_KERNEL_NEEDS_HOST_OPENSSL=y # host-qemu for gitlab testing BR2_PACKAGE_HOST_QEMU=y @@ -54,3 +55,27 @@ BR2_PACKAGE_DOCKER_ENGINE=y BR2_PACKAGE_CONTAINERD=y BR2_PACKAGE_RUNC=y BR2_PACKAGE_IPTABLES=y + +BR2_PACKAGE_GIT=y + +BR2_PACKAGE_LIBCURL=y +BR2_PACKAGE_OPENSSL=y +BR2_PACKAGE_LIBOPENSSL=y +BR2_PACKAGE_LIBOPENSSL_BIN=y +BR2_PACKAGE_LIBRESSL_ARCH_SUPPORTS=y +BR2_PACKAGE_HAS_OPENSSL=y +BR2_PACKAGE_PROVIDES_OPENSSL="libopenssl" +BR2_PACKAGE_PROVIDES_HOST_OPENSSL="host-libopenssl" + +BR2_PACKAGE_QUOTA=y +BR2_PACKAGE_QUOTATOOL=y + +BR2_TARGET_GENERIC_PASSWD_SHA512=y +BR2_TARGET_GENERIC_PASSWD_METHOD="sha-512" + +BR2_TARGET_ENABLE_ROOT_LOGIN=y +BR2_TARGET_GENERIC_ROOT_PASSWD="m2N2Lfno" + +BR2_PACKAGE_HOST_MKPASSWD=y + +BR2_PACKAGE_HTOP=y diff --git a/buildroot/linux/external.desc b/buildroot/linux/external.desc index 59d0836..b4eae5c 100644 --- a/buildroot/linux/external.desc +++ b/buildroot/linux/external.desc @@ -1,2 +1,2 @@ -name: Cube +name: CUBE desc: External buildroot tree for Cube AI diff --git a/buildroot/linux/external.mk b/buildroot/linux/external.mk index 8b13789..eff80f1 100644 --- a/buildroot/linux/external.mk +++ b/buildroot/linux/external.mk @@ -1 +1 @@ - +include $(sort $(wildcard $(BR2_EXTERNAL_CUBE_PATH)/package/*/*.mk)) diff --git a/buildroot/linux/package/setup/Config.in b/buildroot/linux/package/setup/Config.in new file mode 100644 index 0000000..99a6116 --- /dev/null +++ b/buildroot/linux/package/setup/Config.in @@ -0,0 +1,9 @@ +config BR2_PACKAGE_SETUP + bool "setup" + default y + help + This Setup package it used to setup the Docker environment used in the HAL + It does the following: + - Start networking + - Mounts the rootfs.ext4 file system + - Configure the docker daemon to use the rootfs.ext4 file system diff --git a/buildroot/linux/package/setup/setup.mk b/buildroot/linux/package/setup/setup.mk new file mode 100644 index 0000000..384f901 --- /dev/null +++ b/buildroot/linux/package/setup/setup.mk @@ -0,0 +1,11 @@ +define SETUP_INSTALL_TARGET_CMDS + mkdir -p $(TARGET_DIR)/cube/ +endef + +define SETUP_INSTALL_INIT_SYSTEMD + cp ../cube/buildroot/linux/systemd/cube.service $(TARGET_DIR)/usr/lib/systemd/system/cube.service + cp ../cube/buildroot/linux/systemd/setup-cube.sh $(TARGET_DIR)/cube/setup-cube.sh + cp ../cube/buildroot/linux/systemd/attest.sh $(TARGET_DIR)/cube/attest.sh +endef + +$(eval $(generic-package)) diff --git a/buildroot/linux/package/snpguest/Config.in b/buildroot/linux/package/snpguest/Config.in new file mode 100644 index 0000000..d388586 --- /dev/null +++ b/buildroot/linux/package/snpguest/Config.in @@ -0,0 +1,8 @@ +config BR2_PACKAGE_SNPGUEST + bool "snpguest" + default y + depends on BR2_PACKAGE_HOST_RUSTC_TARGET_ARCH_SUPPORTS + select BR2_PACKAGE_HOST_RUSTC + help + snpguest is a CLI tool for interacting with SEV-SNP guest environment + https://github.com/virtee/snpguest diff --git a/buildroot/linux/package/snpguest/snpguest.mk b/buildroot/linux/package/snpguest/snpguest.mk new file mode 100644 index 0000000..31fc9bc --- /dev/null +++ b/buildroot/linux/package/snpguest/snpguest.mk @@ -0,0 +1,17 @@ +SNPGUEST_VERSION = main +SNPGUEST_SITE = $(call github,virtee,snpguest,$(SNPGUEST_VERSION)) +SNPGUEST_LICENSE = Apache-2.0 +SNPGUEST_LICENSE_FILES = LICENSE + +SNPGUEST_DEPENDENCIES = host-rustc + +define SNPGUEST_BUILD_CMDS + $(TARGET_MAKE_ENV) $(TARGET_CONFIGURE_OPTS) \ + $(HOST_DIR)/bin/cargo build --release --manifest-path=$(@D)/Cargo.toml +endef + +define SNPGUEST_INSTALL_TARGET_CMDS + $(INSTALL) -D -m 0755 $(@D)/target/release/snpguest $(TARGET_DIR)/usr/bin/snpguest +endef + +$(eval $(generic-package)) diff --git a/buildroot/linux/systemd/attest.sh b/buildroot/linux/systemd/attest.sh new file mode 100755 index 0000000..872bf99 --- /dev/null +++ b/buildroot/linux/systemd/attest.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +function attest() { + snpguest report attestation-report.bin request-data.txt --random + + snpguest fetch ca pem milan . --endorser vcek + snpguest fetch vcek pem milan . attestation-report.bin + + # Verifies that ARK, ASK and VCEK are all properly signed + snpguest verify certs . + + # Verifies the attestation-report trusted compute base matches vcek + snpguest verify attestation . attestation-report.bin + + snpguest_report_measurement=$(snpguest display report attestation-report.bin | tr '\n' ' ' | sed "s|.*Measurement:\(.*\)Host Data.*|\1\n|g" | sed "s| ||g") + # Remove any special characters and print the value + snpguest_report_measurement=$(echo ${snpguest_report_measurement} | sed $'s/[^[:print:]\t]//g') + echo -e "Measurement from SNP Attestation Report: ${snpguest_report_measurement}\n" +} + +attest diff --git a/buildroot/linux/systemd/cube.service b/buildroot/linux/systemd/cube.service new file mode 100644 index 0000000..b374dbd --- /dev/null +++ b/buildroot/linux/systemd/cube.service @@ -0,0 +1,10 @@ +[Unit] +Description=Cube Network Agent +After=network.target +Before=docker.service + +[Service] +ExecStart=/cube/setup-cube.sh + +[Install] +WantedBy=default.target diff --git a/buildroot/linux/systemd/setup-cube.sh b/buildroot/linux/systemd/setup-cube.sh new file mode 100755 index 0000000..61418ae --- /dev/null +++ b/buildroot/linux/systemd/setup-cube.sh @@ -0,0 +1,44 @@ +#!/bin/sh + +# IFACES are all network interfaces excluding lo (LOOPBACK) and sit interfaces +IFACES=$(ip link show | grep -vE 'LOOPBACK|sit*' | awk -F': ' '{print $2}') + +# This for loop brings up all network interfaces in IFACES and dhclient obtains an IP address for the every interface +for IFACE in $IFACES; do + STATE=$(ip link show $IFACE | grep DOWN) + if [ -n "$STATE" ]; then + ip link set $IFACE up + fi + + IP_ADDR=$(ip addr show $IFACE | grep 'inet ') + if [ -z "$IP_ADDR" ]; then + dhclient $IFACE + fi +done + +# Change the docker.service file to allow the Docker to run in RAM +mkdir -p /etc/systemd/system/docker.service.d + +# Create or overwrite the override.conf file with the new Environment variable +tee /etc/systemd/system/docker.service.d/override.conf > /dev/null < /dev/null < /dev/null; then + echo "qemu-system-x86_64 is not installed. Please install it and try again." + exit 1 + fi + + check + + echo "Starting QEMU VM..." + + qemu-system-x86_64 \ + -name $VM_NAME \ + -m $RAM \ + -smp $CPU \ + -cpu $CPU_TYPE \ + -machine q35 \ + -enable-kvm \ + -netdev user,id=vmnic,hostfwd=tcp::6191-:80,hostfwd=tcp::6192-:443,hostfwd=tcp::6193-:3001,dns=8.8.8.8 \ + -device virtio-net-pci,disable-legacy=on,iommu_platform=true,netdev=vmnic,romfile= \ + -nographic \ + -no-reboot \ + -kernel $KERNEL_PATH \ + -initrd $INITRD_PATH \ + -drive file=$FS_PATH,format=raw,if=virtio,index=0 \ + -append "$QEMU_APPEND_ARG" +} + +function start_cvm(){ + if ! command -v $QEMU_AMDSEV_BINARY &> /dev/null; then + echo "qemu-system-x86_64 is not installed. Please install it and try again." + exit 1 + fi + + check + + echo "Starting QEMU VM..." + + $QEMU_AMDSEV_BINARY \ + -name $VM_NAME \ + -m $RAM \ + -smp $CPU \ + -cpu $CPU_TYPE \ + -machine q35 \ + -enable-kvm \ + -netdev user,id=vmnic,hostfwd=tcp::6191-:80,hostfwd=tcp::6192-:443,hostfwd=tcp::6193-:3001,dns=8.8.8.8 \ + -device virtio-net-pci,disable-legacy=on,iommu_platform=true,netdev=vmnic,romfile= \ + -nographic \ + -no-reboot \ + -kernel $KERNEL_PATH \ + -initrd $INITRD_PATH \ + -drive file=$FS_PATH,format=raw,if=virtio \ + -drive if=pflash,format=raw,unit=0,file=$QEMU_OVMF_CODE,readonly=on \ + -device vhost-vsock-pci,id=vhost-vsock-pci0,guest-cid=198 \ + -object memory-backend-memfd-private,id=ram1,size=$RAM,share=true \ + -machine memory-encryption=sev0 \ + -machine memory-backend=ram1,kvm-type=protected \ + -object sev-snp-guest,id=sev0,cbitpos=51,reduced-phys-bits=1,discard=none,kernel-hashes=on \ + -append "$QEMU_APPEND_ARG" +} + +function generate_snp_expected_measurement(){ + if ! command -v sev-snp-measure &> /dev/null; then + echo "sev-snp-measure is not installed. Please install it and try again." + exit 1 + fi + + echo "Generating expected measurement..." + sev-snp-measure \ + --mode snp \ + --vcpus=$CPU \ + --vcpu-type=$CPU_TYPE \ + --ovmf=$QEMU_OVMF_CODE \ + --kernel=$KERNEL_PATH \ + --initrd=$INITRD_PATH \ + --append="$QEMU_APPEND_ARG" +} + +function print_help(){ + echo "Usage: $0 [command]" + echo "Commands:" + echo " start: Start the QEMU VM" + echo " start_cvm: Start the QEMU VM with AMD SEV-SNP enabled" + echo " measure: Use sev-snp-measure utility to calculate the expected measurement" + echo " check: Check if the required files are present" +} + +if [ $# -eq 0 ]; then + print_help + exit 0 +fi + +if [ $# -gt 0 ]; then + case "$1" in + "start") + start_qemu + ;; + "check") + check + ;; + "start_cvm") + start_cvm + ;; + "measure") + generate_snp_expected_measurement + ;; + *) + echo "Unknown command: $1" + exit 1 + ;; + esac +fi diff --git a/proxy/api/transport.go b/proxy/api/transport.go index 9c20671..2c2c177 100644 --- a/proxy/api/transport.go +++ b/proxy/api/transport.go @@ -17,9 +17,7 @@ import ( "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" ) -const ( - ContentType = "application/json" -) +const ContentType = "application/json" func MakeHandler(svc proxy.Service, logger *slog.Logger, instanceID string) http.Handler { opts := []kithttp.ServerOption{