Skip to content

Commit

Permalink
resourcegen: add mysql deployment
Browse files Browse the repository at this point in the history
  • Loading branch information
@davidweisse
davidweisse committed Dec 16, 2024
1 parent 7386451 commit 7511ed0
Show file tree
Hide file tree
Showing 4 changed files with 205 additions and 46 deletions.
41 changes: 41 additions & 0 deletions internal/kuberesource/parts.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -473,3 +473,44 @@ func ServiceMeshProxy() *applycorev1.ContainerApplyConfiguration {
WithPort(intstr.FromInt(15006))),
)
}

// CryptsetupInitCommand returns the init command for the cryptsetup
// container to setup an encrypted LUKS mount.
func CryptsetupInitCommand() string {
return `#!/bin/bash
set -e
# device is the path to the block device to be encrypted.
device="/dev/csi0"
# workload_secret_path is the path to the Contrast workload secret.
workload_secret_path="/contrast/secrets/workload-secret-seed"
# tmp_key_path is the path to a temporary key file.
tmp_key_path="/dev/shm/key"
# disk_encryption_key_path is the path to the disk encryption key.
disk_encryption_key_path="/dev/shm/disk-key"
# If the device is not already a LUKS device, format it.
if ! cryptsetup isLuks "${device}"; then
# Generate a random key for the first initialization.
dd if=/dev/urandom bs=1 count=32 2>/dev/null | base64 -w0 > "${tmp_key_path}"
cryptsetup luksFormat --pbkdf-memory=10240 $device "${tmp_key_path}" </dev/null
uuid=$(blkid "${device}" -s UUID -o value)
openssl kdf -keylen 32 -kdfopt digest:SHA2-256 -kdfopt hexkey:$(cat "${workload_secret_path}") \
-kdfopt info:${uuid} -binary HKDF | base64 -w0 > "${disk_encryption_key_path}" 2>/dev/null
cryptsetup luksChangeKey --pbkdf-memory=10240 "${device}" --key-file "${tmp_key_path}" "${disk_encryption_key_path}"
cryptsetup open "${device}" state -d "${disk_encryption_key_path}"
mkfs.ext4 /dev/mapper/state
cryptsetup close state
fi
# No matter if this is the first initialization derive the key (again) and open the device.
uuid=$(blkid "${device}" -s UUID -o value)
openssl kdf -keylen 32 -kdfopt digest:SHA2-256 -kdfopt hexkey:$(cat "${workload_secret_path}") \
-kdfopt info:${uuid} -binary HKDF | base64 -w0 > "${disk_encryption_key_path}" 2>/dev/null
cryptsetup luksUUID "${device}"
cryptsetup open "${device}" state -d "${disk_encryption_key_path}"
mount /dev/mapper/state /state
touch /done
sleep inf
`
}
2 changes: 2 additions & 0 deletions internal/kuberesource/resourcegen/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,8 @@ func main() {
subResources = kuberesource.PatchRuntimeHandlers(subResources, "contrast-cc")
case "volume-stateful-set":
subResources = kuberesource.PatchRuntimeHandlers(kuberesource.VolumeStatefulSet(), "contrast-cc")
case "mysql":
subResources = kuberesource.PatchRuntimeHandlers(kuberesource.MySQL(), "contrast-cc")
default:
log.Fatalf("Error: unknown set: %s\n", set)
}
Expand Down
203 changes: 158 additions & 45 deletions internal/kuberesource/sets.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -502,47 +502,6 @@ func Emojivoto(smMode serviceMeshMode) []any {
// VolumeStatefulSet returns a stateful set for testing volume mounts and the
// mounting of encrypted luks volumes using the workload-secret.
func VolumeStatefulSet() []any {
initCommand := `#!/bin/bash
# cryptsetup complains if there is no /run directory.
mkdir /run
set -e
# device is the path to the block device to be encrypted.
device="/dev/csi0"
# workload_secret_path is the path to the Contrast workload secret.
workload_secret_path="/contrast/secrets/workload-secret-seed"
# tmp_key_path is the path to a temporary key file.
tmp_key_path="/dev/shm/key"
# disk_encryption_key_path is the path to the disk encryption key.
disk_encryption_key_path="/dev/shm/disk-key"
# If the device is not already a LUKS device, format it.
if ! cryptsetup isLuks "${device}"; then
# Generate a random key for the first initialization.
dd if=/dev/urandom bs=1 count=32 2>/dev/null | base64 | tr -d '\n' > "${tmp_key_path}"
cryptsetup luksFormat --pbkdf-memory=10240 $device "${tmp_key_path}" </dev/null
# Now we can get the LUKS UUID and deterministically derive the disk encryption key from the workload secret.
uuid=$(blkid "${device}" -s UUID -o value)
openssl kdf -keylen 32 -kdfopt digest:SHA2-256 -kdfopt key:$(cat "${workload_secret_path}") \
-kdfopt info:${uuid} -binary HKDF | base64 | tr -d '\n' > "${disk_encryption_key_path}"
cryptsetup luksChangeKey --pbkdf-memory=10240 "${device}" --key-file "${tmp_key_path}" "${disk_encryption_key_path}"
cryptsetup open "${device}" state -d "${disk_encryption_key_path}"
mkfs.ext4 /dev/mapper/state
cryptsetup close state
fi
# No matter if this is the first initialization derive the key (again) and open the device.
uuid=$(blkid "${device}" -s UUID -o value)
openssl kdf -keylen 32 -kdfopt digest:SHA2-256 -kdfopt key:$(cat "${workload_secret_path}") \
-kdfopt info:${uuid} -binary HKDF | base64 | tr -d '\n' > "${disk_encryption_key_path}"
cryptsetup luksUUID "${device}"
cryptsetup open "${device}" state -d "${disk_encryption_key_path}"
mkdir -p /srv/state
mount /dev/mapper/state /srv/state
touch /done
sleep inf
`

vss := StatefulSet("volume-tester", "").
WithSpec(StatefulSetSpec().
WithPersistentVolumeClaimRetentionPolicy(applyappsv1.StatefulSetPersistentVolumeClaimRetentionPolicy().
Expand All @@ -561,16 +520,19 @@ sleep inf
Container().
WithName("volume-tester-init").
WithImage("ghcr.io/edgelesssys/contrast/cryptsetup:latest").
WithCommand("/bin/sh", "-c", initCommand).
WithCommand("/bin/sh", "-c", CryptsetupInitCommand()).
WithVolumeDevices(
applycorev1.VolumeDevice().
WithName("state").
WithDevicePath("/dev/csi0"),
).
WithVolumeMounts(
VolumeMount().
WithName("run").
WithMountPath("/run"),
VolumeMount().
WithName("share").
WithMountPath("/srv").
WithMountPath("/state").
WithMountPropagation(corev1.MountPropagationBidirectional),
VolumeMount().
WithName("contrast-secrets").
Expand All @@ -588,7 +550,7 @@ sleep inf
WithFailureThreshold(20).
WithPeriodSeconds(5).
WithExec(applycorev1.ExecAction().
WithCommand("/bin/sh", "-c", "cat /done"),
WithCommand("/bin/test", "-f", "/done"),
),
).
WithRestartPolicy(
Expand All @@ -603,11 +565,14 @@ sleep inf
WithVolumeMounts(
VolumeMount().
WithName("share").
WithMountPath("/srv").
WithMountPath("/state").
WithMountPropagation(corev1.MountPropagationHostToContainer),
),
).
WithVolumes(
applycorev1.Volume().
WithName("run").
WithEmptyDir(applycorev1.EmptyDirVolumeSource()),
applycorev1.Volume().
WithName("share").
WithEmptyDir(applycorev1.EmptyDirVolumeSource()),
Expand All @@ -627,3 +592,151 @@ sleep inf

return []any{vss}
}

// MySQL returns the resources for deploying a MySQL database
// with an encrypted luks volume using the workload-secret.
func MySQL() []any {
backend := StatefulSet("mysql-backend", "").
WithAnnotations(map[string]string{smIngressConfigAnnotationKey: ""}).
WithSpec(StatefulSetSpec().
WithPersistentVolumeClaimRetentionPolicy(applyappsv1.StatefulSetPersistentVolumeClaimRetentionPolicy().
WithWhenDeleted(appsv1.DeletePersistentVolumeClaimRetentionPolicyType).
WithWhenScaled(appsv1.DeletePersistentVolumeClaimRetentionPolicyType)).
WithReplicas(1).
WithSelector(LabelSelector().
WithMatchLabels(map[string]string{"app.kubernetes.io/name": "mysql-backend"}),
).
WithServiceName("mysql-backend").
WithTemplate(PodTemplateSpec().
WithLabels(map[string]string{"app.kubernetes.io/name": "mysql-backend"}).
WithSpec(
PodSpec().
WithInitContainers(
Container().
WithName("luks-setup").
WithImage("ghcr.io/edgelesssys/contrast/cryptsetup:latest").
WithCommand("/bin/sh", "-c", CryptsetupInitCommand()).
WithVolumeDevices(
applycorev1.VolumeDevice().
WithName("state").
WithDevicePath("/dev/csi0"),
).
WithVolumeMounts(
VolumeMount().
WithName("run").
WithMountPath("/run"),
VolumeMount().
WithName("share").
WithMountPath("/state").
WithMountPropagation(corev1.MountPropagationBidirectional),
VolumeMount().
WithName("contrast-secrets").
WithMountPath("/contrast"),
).
WithSecurityContext(
applycorev1.SecurityContext().
WithPrivileged(true),
).
WithResources(ResourceRequirements().
WithMemoryLimitAndRequest(100),
).
WithStartupProbe(
Probe().
WithFailureThreshold(20).
WithPeriodSeconds(5).
WithExec(applycorev1.ExecAction().
WithCommand("/bin/test", "-f", "/done"),
),
).
WithRestartPolicy(
corev1.ContainerRestartPolicyAlways,
),
).
WithContainers(
Container().
WithName("mysql-backend").
WithImage("docker.io/library/mysql:latest").
WithEnv(NewEnvVar("MYSQL_ALLOW_EMPTY_PASSWORD", "1")).
WithPorts(
ContainerPort().
WithName("mysql").
WithContainerPort(3306),
).
WithVolumeMounts(
VolumeMount().
WithName("share").
WithMountPath("/var/lib/mysql").
WithMountPropagation(corev1.MountPropagationHostToContainer),
).
WithResources(ResourceRequirements().
WithMemoryLimitAndRequest(1000),
),
).
WithVolumes(
applycorev1.Volume().
WithName("run").
WithEmptyDir(applycorev1.EmptyDirVolumeSource()),
applycorev1.Volume().
WithName("share").
WithEmptyDir(applycorev1.EmptyDirVolumeSource()),
),
),
).
WithVolumeClaimTemplates(applycorev1.PersistentVolumeClaim("state", "").
WithSpec(applycorev1.PersistentVolumeClaimSpec().
WithVolumeMode(corev1.PersistentVolumeBlock).
WithAccessModes(corev1.ReadWriteOnce).
WithResources(applycorev1.VolumeResourceRequirements().
WithRequests(map[corev1.ResourceName]resource.Quantity{corev1.ResourceStorage: resource.MustParse("1Gi")}),
),
),
),
)

backendService := ServiceForStatefulSet(backend)

clientCmd := `#!/bin/bash
while ! mysqladmin ping -h 127.137.0.1 -u root --silent; do
echo "Waiting for MySQL server ...";
sleep 5;
done
mysql -h 127.137.0.1 -u root -e "CREATE DATABASE my_db;"
mysql -h 127.137.0.1 -u root -D my_db -e "CREATE TABLE my_table (id INT NOT NULL AUTO_INCREMENT, uuid CHAR(36), PRIMARY KEY (id));"
while true; do
mysql -h 127.137.0.1 -u root -D my_db -e "INSERT INTO my_table (uuid) VALUES (UUID());"
mysql -h 127.137.0.1 -u root -D my_db -e "SELECT * FROM my_table;"
sleep 5;
done
`

client := Deployment("mysql-client", "").
WithAnnotations(map[string]string{smEgressConfigAnnotationKey: "mysql-backend#127.137.0.1:3306#mysql-backend:3306"}).
WithSpec(DeploymentSpec().
WithReplicas(1).
WithSelector(LabelSelector().
WithMatchLabels(map[string]string{"app.kubernetes.io/name": "mysql-client"}),
).
WithTemplate(PodTemplateSpec().
WithLabels(map[string]string{"app.kubernetes.io/name": "mysql-client"}).
WithSpec(
PodSpec().
WithContainers(
Container().
WithName("mysql-client").
WithImage("docker.io/library/mysql:latest").
WithEnv(NewEnvVar("MYSQL_ALLOW_EMPTY_PASSWORD", "1")).
WithCommand("/bin/sh", "-c", clientCmd).
WithResources(ResourceRequirements().
WithMemoryLimitAndRequest(1000),
),
),
),
),
)

return []any{
backend,
backendService,
client,
}
}
5 changes: 4 additions & 1 deletion justfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ apply target=default_deploy_target:
kubectl apply -f ./{{ workspace_dir }}/runtime
exit 0
;;
"openssl" | "emojivoto" | "volume-stateful-set")
"openssl" | "emojivoto" | "volume-stateful-set" | "mysql")
:
;;
*)
Expand Down Expand Up @@ -303,6 +303,9 @@ wait-for-workload target=default_deploy_target:
"volume-stateful-set")
nix run .#scripts.kubectl-wait-ready -- $ns volume-tester
;;
"mysql")
nix run .#scripts.kubectl-wait-ready -- $ns mysql
;;
*)
echo "Please register workloads of new targets in wait-for-workload"
exit 1
Expand Down

0 comments on commit 7511ed0

Please sign in to comment.