diff --git a/cli/generate.go b/cli/generate.go index c0707ea543..f90abb0f3a 100644 --- a/cli/generate.go +++ b/cli/generate.go @@ -102,12 +102,32 @@ func findGenerateTargets(args []string) ([]string, error) { return nil, fmt.Errorf("failed to walk %s: %w", path, err) } } + + paths = filterNonCoCoRuntime("kata-cc-isolation", paths) + if len(paths) == 0 { return nil, fmt.Errorf("no .yml/.yaml files found") } return paths, nil } +func filterNonCoCoRuntime(runtimeClassName string, paths []string) []string { + var filtered []string + for _, path := range paths { + data, err := os.ReadFile(path) + if err != nil { + log.Printf("failed to read %s: %v", path, err) + continue + } + if !bytes.Contains(data, []byte(runtimeClassName)) { + log.Printf("%s is not a CoCo runtime, ignoring", path) + continue + } + filtered = append(filtered, path) + } + return filtered +} + func generatePolicies(ctx context.Context, regoPath, policyPath string, yamlPaths []string) error { for _, yamlPath := range yamlPaths { policyHash, err := generatePolicyForFile(ctx, regoPath, policyPath, yamlPath) diff --git a/coordinator/intercom.go b/coordinator/intercom.go index f4909ece1d..d8d821ce0b 100644 --- a/coordinator/intercom.go +++ b/coordinator/intercom.go @@ -65,6 +65,6 @@ func (i *intercomServer) NewMeshCert(ctx context.Context, req *intercom.NewMeshC return &intercom.NewMeshCertResponse{ // TODO(3u13r): Replace the CA Cert the intermediate CA cert CaCert: i.caChainGetter.GetCACert(), - CertChain: append(intermCert, cert...), + CertChain: append(cert, intermCert...), }, nil } diff --git a/coordinator/main.go b/coordinator/main.go index 6129ab5523..43f3c996af 100644 --- a/coordinator/main.go +++ b/coordinator/main.go @@ -3,6 +3,7 @@ package main import ( "log" "net" + "os" "github.com/edgelesssys/nunki/internal/ca" "github.com/edgelesssys/nunki/internal/coordapi" @@ -12,7 +13,12 @@ import ( func main() { log.Println("Coordinator started") - caInstance, err := ca.New() + namespace, ok := os.LookupEnv("NAMESPACE") + if !ok { + log.Fatalf("NAMESPACE environment variable not set") + } + + caInstance, err := ca.New(namespace) if err != nil { log.Fatalf("failed to create CA: %v", err) } diff --git a/deployments/emojivoto/coordinator.yml b/deployments/emojivoto/coordinator.yml new file mode 100644 index 0000000000..589d730989 --- /dev/null +++ b/deployments/emojivoto/coordinator.yml @@ -0,0 +1,44 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: coordinator-kbs + namespace: edg-emojivoto +spec: + selector: + matchLabels: + run: coordinator-kbs + replicas: 1 + template: + metadata: + labels: + run: coordinator-kbs + spec: + runtimeClassName: kata-cc-isolation + containers: + - name: coordinator-kbs + image: "ghcr.io/edgelesssys/nunki/coordinator-kbs:latest" + imagePullPolicy: Always + ports: + - containerPort: 7777 + - containerPort: 1313 + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace +--- +apiVersion: v1 +kind: Service +metadata: + name: coordinator-kbs + namespace: edg-emojivoto +spec: + ports: + - name: intercom + port: 7777 + protocol: TCP + - name: coordapi + port: 1313 + protocol: TCP + selector: + run: coordinator-kbs diff --git a/deployments/emojivoto/emoji.yml b/deployments/emojivoto/emoji.yml new file mode 100644 index 0000000000..ca806aed5d --- /dev/null +++ b/deployments/emojivoto/emoji.yml @@ -0,0 +1,84 @@ +kind: ServiceAccount +apiVersion: v1 +metadata: + name: emoji + namespace: edg-emojivoto +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: emoji + namespace: edg-emojivoto + labels: + app.kubernetes.io/name: emoji + app.kubernetes.io/part-of: emojivoto + app.kubernetes.io/version: v11 +spec: + replicas: 1 + selector: + matchLabels: + app: emoji-svc + version: v11 + template: + metadata: + labels: + app: emoji-svc + version: v11 + spec: + runtimeClassName: kata-cc-isolation + initContainers: + - name: initializer + image: "ghcr.io/edgelesssys/nunki/initializer:latest" + imagePullPolicy: Always + env: + - name: COORDINATOR_HOST + value: coordinator-kbs.edg-emojivoto + volumeMounts: + - name: tls-certs + mountPath: /tls-config + serviceAccountName: emoji + containers: + - env: + - name: GRPC_PORT + value: "8080" + - name: PROM_PORT + value: "8801" + - name: EDG_CERT_PATH + value: /tls-config/certChain.pem + - name: EDG_CA_PATH + value: /tls-config/CACert.pem + - name: EDG_KEY_PATH + value: /tls-config/key.pem + image: ghcr.io/3u13r/emojivoto-emoji-svc:coco-1 + imagePullPolicy: Always + name: emoji-svc + ports: + - containerPort: 8080 + name: grpc + - containerPort: 8801 + name: prom + resources: + requests: + cpu: 100m + volumeMounts: + - name: tls-certs + mountPath: /tls-config + volumes: + - name: tls-certs + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: emoji-svc + namespace: edg-emojivoto +spec: + selector: + app: emoji-svc + ports: + - name: grpc + port: 8080 + targetPort: 8080 + - name: prom + port: 8801 + targetPort: 8801 diff --git a/deployments/emojivoto/initializer.yml b/deployments/emojivoto/initializer.yml new file mode 100644 index 0000000000..5b1fc32f7d --- /dev/null +++ b/deployments/emojivoto/initializer.yml @@ -0,0 +1,40 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: workload + namespace: edg-emojivoto +spec: + selector: + matchLabels: + run: workload + replicas: 1 + template: + metadata: + labels: + run: workload + spec: + runtimeClassName: kata-cc-isolation + initContainers: + - name: initializer + image: "ghcr.io/edgelesssys/nunki/initializer:latest" + imagePullPolicy: Always + env: + - name: COORDINATOR_HOST + value: coordinator-kbs.edg-emojivoto + volumeMounts: + - name: tls-certs + mountPath: /tls-config + containers: + - name: workload + image: "fedora:38" + imagePullPolicy: Always + command: + - /bin/bash + - "-c" + - echo Workload started ; sleep inf + volumeMounts: + - name: tls-certs + mountPath: /tls-config + volumes: + - name: tls-certs + emptyDir: {} diff --git a/deployments/emojivoto/ns.yml b/deployments/emojivoto/ns.yml new file mode 100644 index 0000000000..321162eb8e --- /dev/null +++ b/deployments/emojivoto/ns.yml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: edg-emojivoto diff --git a/deployments/emojivoto/portforwarder.yml b/deployments/emojivoto/portforwarder.yml new file mode 100644 index 0000000000..b23352a779 --- /dev/null +++ b/deployments/emojivoto/portforwarder.yml @@ -0,0 +1,45 @@ +apiVersion: v1 +kind: Pod +metadata: + name: port-forwarder-coordinator + namespace: default +spec: + containers: + - name: port-forwarder + image: nixery.dev/shell/socat + env: + - name: LISTEN_PORT + value: "1313" + - name: FORWARD_HOST + value: coordinator-kbs.edg-emojivoto + - name: FORWARD_PORT + value: "1313" + command: + - /bin/bash + - "-c" + - echo Starting port-forward with socat; exec socat -d -d TCP-LISTEN:${LISTEN_PORT},fork TCP:${FORWARD_HOST}:${FORWARD_PORT} + ports: + - containerPort: 1313 +--- +apiVersion: v1 +kind: Pod +metadata: + name: port-forwarder-2 + namespace: default +spec: + containers: + - name: port-forwarder + image: nixery.dev/shell/socat + env: + - name: LISTEN_PORT + value: "8080" + - name: FORWARD_HOST + value: web-svc.edg-emojivoto + - name: FORWARD_PORT + value: "443" + command: + - /bin/bash + - "-c" + - echo Starting port-forward with socat; exec socat -d -d TCP-LISTEN:${LISTEN_PORT},fork TCP:${FORWARD_HOST}:${FORWARD_PORT} + ports: + - containerPort: 8080 diff --git a/deployments/emojivoto/vote-bot.yml b/deployments/emojivoto/vote-bot.yml new file mode 100644 index 0000000000..3bcb35762a --- /dev/null +++ b/deployments/emojivoto/vote-bot.yml @@ -0,0 +1,33 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: vote-bot + namespace: edg-emojivoto + labels: + app.kubernetes.io/name: vote-bot + app.kubernetes.io/part-of: emojivoto + app.kubernetes.io/version: v11 +spec: + replicas: 1 + selector: + matchLabels: + app: vote-bot + version: v11 + template: + metadata: + labels: + app: vote-bot + version: v11 + spec: + containers: + - command: + - emojivoto-vote-bot + env: + - name: WEB_HOST + value: web-svc.edg-emojivoto:443 + image: ghcr.io/3u13r/emojivoto-web:coco-1 + imagePullPolicy: Always + name: vote-bot + resources: + requests: + cpu: 10m diff --git a/deployments/emojivoto/voting.yml b/deployments/emojivoto/voting.yml new file mode 100644 index 0000000000..d15c071ad5 --- /dev/null +++ b/deployments/emojivoto/voting.yml @@ -0,0 +1,84 @@ +kind: ServiceAccount +apiVersion: v1 +metadata: + name: voting + namespace: edg-emojivoto +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: voting + namespace: edg-emojivoto + labels: + app.kubernetes.io/name: voting + app.kubernetes.io/part-of: emojivoto + app.kubernetes.io/version: v11 +spec: + replicas: 1 + selector: + matchLabels: + app: voting-svc + version: v11 + template: + metadata: + labels: + app: voting-svc + version: v11 + spec: + runtimeClassName: kata-cc-isolation + initContainers: + - name: initializer + image: "ghcr.io/edgelesssys/nunki/initializer:latest" + imagePullPolicy: Always + env: + - name: COORDINATOR_HOST + value: coordinator-kbs.edg-emojivoto + volumeMounts: + - name: tls-certs + mountPath: /tls-config + serviceAccountName: voting + containers: + - env: + - name: GRPC_PORT + value: "8080" + - name: PROM_PORT + value: "8801" + - name: EDG_CERT_PATH + value: /tls-config/certChain.pem + - name: EDG_CA_PATH + value: /tls-config/CACert.pem + - name: EDG_KEY_PATH + value: /tls-config/key.pem + image: ghcr.io/3u13r/emojivoto-voting-svc:coco-1 + imagePullPolicy: Always + name: voting-svc + ports: + - containerPort: 8080 + name: grpc + - containerPort: 8801 + name: prom + resources: + requests: + cpu: 100m + volumeMounts: + - name: tls-certs + mountPath: /tls-config + volumes: + - name: tls-certs + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: voting-svc + namespace: edg-emojivoto +spec: + selector: + app: voting-svc + ports: + - name: grpc + port: 8080 + targetPort: 8080 + - name: prom + port: 8801 + targetPort: 8801 diff --git a/deployments/emojivoto/web.yml b/deployments/emojivoto/web.yml new file mode 100644 index 0000000000..e1a28a4695 --- /dev/null +++ b/deployments/emojivoto/web.yml @@ -0,0 +1,86 @@ +kind: ServiceAccount +apiVersion: v1 +metadata: + name: web + namespace: edg-emojivoto +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: web + namespace: edg-emojivoto + labels: + app.kubernetes.io/name: web + app.kubernetes.io/part-of: emojivoto + app.kubernetes.io/version: v11 +spec: + replicas: 1 + selector: + matchLabels: + app: web-svc + version: v11 + template: + metadata: + labels: + app: web-svc + version: v11 + spec: + runtimeClassName: kata-cc-isolation + initContainers: + - name: initializer + image: "ghcr.io/edgelesssys/nunki/initializer:latest" + imagePullPolicy: Always + env: + - name: COORDINATOR_HOST + value: coordinator-kbs.edg-emojivoto + volumeMounts: + - name: tls-certs + mountPath: /tls-config + serviceAccountName: web + containers: + - env: + - name: WEB_PORT + value: "8080" + - name: EMOJISVC_HOST + value: emoji-svc.edg-emojivoto:8080 + - name: VOTINGSVC_HOST + value: voting-svc.edg-emojivoto:8080 + - name: INDEX_BUNDLE + value: dist/index_bundle.js + - name: EDG_CERT_PATH + value: /tls-config/certChain.pem + - name: EDG_CA_PATH + value: /tls-config/CACert.pem + - name: EDG_KEY_PATH + value: /tls-config/key.pem + - name: EDG_DISABLE_CLIENT_AUTH + value: "true" + image: ghcr.io/3u13r/emojivoto-web:coco-1 + imagePullPolicy: Always + name: web-svc + ports: + - containerPort: 8080 + name: https + resources: + requests: + cpu: 100m + volumeMounts: + - name: tls-certs + mountPath: /tls-config + volumes: + - name: tls-certs + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: web-svc + namespace: edg-emojivoto +spec: + type: ClusterIP + selector: + app: web-svc + ports: + - name: https + port: 443 + targetPort: 8080 diff --git a/deployments/simple/coordinator.yml b/deployments/simple/coordinator.yml index f1470fabf9..f3355c9f63 100644 --- a/deployments/simple/coordinator.yml +++ b/deployments/simple/coordinator.yml @@ -1,4 +1,3 @@ ---- apiVersion: apps/v1 kind: Deployment metadata: @@ -22,6 +21,11 @@ spec: ports: - containerPort: 7777 - containerPort: 1313 + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace --- apiVersion: v1 kind: Service diff --git a/deployments/simple/initializer.yml b/deployments/simple/initializer.yml index 73ddef72c4..0871a21219 100644 --- a/deployments/simple/initializer.yml +++ b/deployments/simple/initializer.yml @@ -1,4 +1,3 @@ ---- apiVersion: apps/v1 kind: Deployment metadata: diff --git a/deployments/simple/ns.yml b/deployments/simple/ns.yml index 827d64b987..55893d5330 100644 --- a/deployments/simple/ns.yml +++ b/deployments/simple/ns.yml @@ -1,4 +1,3 @@ ---- apiVersion: v1 kind: Namespace metadata: diff --git a/deployments/simple/portforwarder.yml b/deployments/simple/portforwarder.yml index 2f35ceed13..9bb68469e3 100644 --- a/deployments/simple/portforwarder.yml +++ b/deployments/simple/portforwarder.yml @@ -1,9 +1,8 @@ ---- apiVersion: v1 kind: Pod metadata: - name: port-forwarder - namespace: edg-coco + name: port-forwarder-coordinator + namespace: default spec: containers: - name: port-forwarder diff --git a/internal/ca/ca.go b/internal/ca/ca.go index bf69feb555..3bf8d3ebca 100644 --- a/internal/ca/ca.go +++ b/internal/ca/ca.go @@ -8,7 +8,6 @@ import ( "crypto/x509/pkix" "encoding/pem" "fmt" - "math/big" "time" "github.com/edgelesssys/nunki/internal/crypto" @@ -21,11 +20,18 @@ type CA struct { intermPrivKey *rsa.PrivateKey intermCert *x509.Certificate intermPEM []byte + + namespace string } -func New() (*CA, error) { +func New(namespace string) (*CA, error) { + rootSerialNumber, err := crypto.GenerateCertificateSerialNumber() + if err != nil { + return nil, err + } + root := &x509.Certificate{ - SerialNumber: big.NewInt(2019), + SerialNumber: rootSerialNumber, Subject: pkix.Name{CommonName: "system:coordinator-kbs:root"}, NotBefore: time.Now(), NotAfter: time.Now().AddDate(10, 0, 0), @@ -33,7 +39,7 @@ func New() (*CA, error) { KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, BasicConstraintsValid: true, } - rootPrivKey, err := rsa.GenerateKey(rand.Reader, 4098) + rootPrivKey, err := rsa.GenerateKey(rand.Reader, 4096) if err != nil { return nil, fmt.Errorf("failed to generate RSA private key: %w", err) } @@ -47,8 +53,12 @@ func New() (*CA, error) { Bytes: rootBytes, }) + intermSerialNumber, err := crypto.GenerateCertificateSerialNumber() + if err != nil { + return nil, err + } interm := &x509.Certificate{ - SerialNumber: big.NewInt(2020), + SerialNumber: intermSerialNumber, Subject: pkix.Name{CommonName: "system:coordinator-kbs:intermediate"}, NotBefore: time.Now(), NotAfter: time.Now().AddDate(10, 0, 0), @@ -56,7 +66,7 @@ func New() (*CA, error) { KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, BasicConstraintsValid: true, } - intermPrivKey, err := rsa.GenerateKey(rand.Reader, 4098) + intermPrivKey, err := rsa.GenerateKey(rand.Reader, 4096) if err != nil { return nil, fmt.Errorf("failed to generate RSA private key: %w", err) } @@ -77,6 +87,7 @@ func New() (*CA, error) { intermPrivKey: intermPrivKey, intermCert: interm, intermPEM: intermPEM.Bytes(), + namespace: namespace, }, nil } @@ -97,6 +108,7 @@ func (c *CA) NewAttestedMeshCert(commonName string, extensions []pkix.Extension, KeyUsage: x509.KeyUsageDigitalSignature, BasicConstraintsValid: true, ExtraExtensions: extensions, + DNSNames: []string{fmt.Sprintf("*.%s", c.namespace)}, } certBytes, err := x509.CreateCertificate(rand.Reader, certTemplate, c.intermCert, subjectPublicKey, c.intermPrivKey) diff --git a/justfile b/justfile index 3d99dce752..20d9c3afdf 100644 --- a/justfile +++ b/justfile @@ -1,5 +1,5 @@ # Undeploy, rebuild, deploy. -default: undeploy coordinator initializer deploy +default target=default_deploy_target: undeploy coordinator initializer (deploy target) # Build the coordinator, containerize and push it. coordinator: @@ -13,10 +13,11 @@ default_deploy_target := "simple" worspace_dir := "workspace" # Generate policies, apply Kubernetes manifests. -deploy target=default_deploy_target: generate apply +deploy target=default_deploy_target: (generate target) apply # Generate policies, update manifest. generate target=default_deploy_target: + #!/usr/bin/env bash mkdir -p ./{{worspace_dir}} rm -rf ./{{worspace_dir}}/deployment cp -R ./deployments/{{target}} ./{{worspace_dir}}/deployment @@ -25,25 +26,25 @@ generate target=default_deploy_target: | with(select(.spec.template.spec.containers[].image | contains(\"coordinator-kbs\")); \ .spec.template.spec.containers[0].image = \"${container_registry}/coordinator-kbs:latest\")" \ ./{{worspace_dir}}/deployment/coordinator.yml - nix run .#yq-go -- -i ". \ - | with(select(.spec.template.spec.initContainers[].image | contains(\"initializer\")); \ - .spec.template.spec.initContainers[0].image = \"${container_registry}/initializer:latest\")" \ - ./{{worspace_dir}}/deployment/initializer.yml + for f in ./{{worspace_dir}}/deployment/*.yml; do + nix run .#yq-go -- -i ". \ + | with(select(.spec.template.spec.initContainers[].image | contains(\"initializer\")); \ + .spec.template.spec.initContainers[0].image = \"${container_registry}/initializer:latest\")" \ + "${f}" + done nix run .#cli -- generate \ -m ./{{worspace_dir}}/manifest.json \ -p tools \ -s genpolicy-msft.json \ - ./{{worspace_dir}}/deployment/{coordinator,initializer}.yml + ./{{worspace_dir}}/deployment/*.yml # Apply Kubernetes manifests from /deployment -apply target=default_deploy_target: +apply: kubectl apply -f ./{{worspace_dir}}/deployment/ns.yml - kubectl apply -f ./{{worspace_dir}}/deployment/coordinator.yml - kubectl apply -f ./{{worspace_dir}}/deployment/initializer.yml - kubectl apply -f ./{{worspace_dir}}/deployment/portforwarder.yml + kubectl apply -f ./{{worspace_dir}}/deployment # Delete Kubernetes manifests. -undeploy target=default_deploy_target: +undeploy: -kubectl delete -f ./{{worspace_dir}}/deployment # Create a CoCo-enabled AKS cluster. @@ -53,13 +54,13 @@ create: # Set the manifest at the coordinator. set: #!/usr/bin/env bash - kubectl -n edg-coco port-forward pod/port-forwarder 1313 & + kubectl port-forward pod/port-forwarder-coordinator 1313 & PID=$! sleep 1 nix run .#cli -- set \ -m ./{{worspace_dir}}/manifest.json \ -c localhost:1313 \ - ./{{worspace_dir}}/deployment/{coordinator,initializer}.yml + ./{{worspace_dir}}/deployment/*.yml kill $PID diff --git a/treefmt.nix b/treefmt.nix index 6d206c9d06..b05b6369f3 100644 --- a/treefmt.nix +++ b/treefmt.nix @@ -7,5 +7,6 @@ _: shellcheck.enable = true; shfmt.enable = true; statix.enable = true; + yamlfmt.enable = true; }; }