WIP

.github/workflows/e2e_openssl_baremetal.yml

@@ -0,0 +1,56 @@
+name: e2e test openssl baremetal
+
+on:
+  workflow_dispatch:
+    inputs:
+      skip-undeploy:
+        description: "Skip undeploy"
+        required: false
+        type: boolean
+        default: false
+  pull_request:
+    paths-ignore:
+      - dev-docs/**
+      - docs/**
+      - rfc/**
+
+env:
+  container_registry: ghcr.io/edgelesssys
+  DO_NOT_TRACK: 1
+
+jobs:
+  test:
+    runs-on:
+      labels: snp
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - name: Log in to ghcr.io Container registry
+        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - uses: nicknovitski/nix-develop@a2060d116a50b36dfab02280af558e73ab52427d # v1.1.0
+      - name: Create justfile.env
+        run: |
+          cat <<EOF > justfile.env
+          container_registry=${{ env.container_registry }}
+          default_platform="K3s-QEMU-SNP"
+          EOF
+      - name: Build and prepare deployments
+        run: |
+          just coordinator initializer openssl port-forwarder node-installer
+      - name: E2E Test
+        run: |
+          nix shell .#contrast.e2e --command openssl.test -test.v \
+            --image-replacements workspace/just.containerlookup \
+            --namespace-file workspace/e2e.namespace \
+            --platform K3s-QEMU-SNP \
+            --skip-undeploy="${{ inputs.skip-undeploy && 'true' || 'false' }}"
+      - name: Cleanup
+        if: cancelled() && !inputs.skip-undeploy
+        run: |
+          kubectl delete ns "$(cat workspace/e2e.namespace)" --timeout 5m

cli/cmd/generate.go

@@ -131,6 +131,8 @@ func runGenerate(cmd *cobra.Command, args []string) error {
 	switch flags.referenceValuesPlatform {
 	case platforms.AKSCloudHypervisorSNP:
 		defaultManifest = manifest.DefaultAKS()
+	case platforms.K3sQEMUSNP:
+		defaultManifest = manifest.DefaultBaremetalSNP()
 	}
 
 	defaultManifestData, err := json.MarshalIndent(&defaultManifest, "", "  ")

e2e/genpolicy/genpolicy_test.go

@@ -19,21 +19,25 @@ import (
 	"github.com/edgelesssys/contrast/e2e/internal/contrasttest"
 	"github.com/edgelesssys/contrast/e2e/internal/kubeclient"
 	"github.com/edgelesssys/contrast/internal/kuberesource"
+	"github.com/edgelesssys/contrast/node-installer/platforms"
 	"github.com/stretchr/testify/require"
 )
 
 var (
-	imageReplacementsFile, namespaceFile string
-	skipUndeploy                         bool
+	imageReplacementsFile, namespaceFile, platform string
+	skipUndeploy                                   bool
 )
 
 // TestGenpolicy runs regression tests for generated policies.
 func TestGenpolicy(t *testing.T) {
 	testCases := kuberesource.GenpolicyRegressionTests()
 
+	p, err := platforms.FromString(platform)
+	require.NoError(t, err)
+
 	for name, deploy := range testCases {
 		t.Run(name, func(t *testing.T) {
-			ct := contrasttest.New(t, imageReplacementsFile, namespaceFile, skipUndeploy)
+			ct := contrasttest.New(t, imageReplacementsFile, namespaceFile, p, skipUndeploy)
 
 			ct.Init(t, []any{deploy})
 
@@ -67,6 +71,7 @@ func TestGenpolicy(t *testing.T) {
 func TestMain(m *testing.M) {
 	flag.StringVar(&imageReplacementsFile, "image-replacements", "", "path to image replacements file")
 	flag.StringVar(&namespaceFile, "namespace-file", "", "file to store the namespace in")
+	flag.StringVar(&platform, "platform", "", "Deployment platform")
 	flag.BoolVar(&skipUndeploy, "skip-undeploy", false, "skip undeploy step in the test")
 	flag.Parse()
 

e2e/getdents/getdents_test.go

@@ -19,6 +19,7 @@ import (
 	"github.com/edgelesssys/contrast/e2e/internal/contrasttest"
 	"github.com/edgelesssys/contrast/e2e/internal/kubeclient"
 	"github.com/edgelesssys/contrast/internal/kuberesource"
+	"github.com/edgelesssys/contrast/node-installer/platforms"
 	"github.com/stretchr/testify/require"
 )
 
@@ -27,12 +28,14 @@ const (
 )
 
 var (
-	imageReplacementsFile, namespaceFile string
-	skipUndeploy                         bool
+	imageReplacementsFile, namespaceFile, platform string
+	skipUndeploy                                   bool
 )
 
 func TestGetDEnts(t *testing.T) {
-	ct := contrasttest.New(t, imageReplacementsFile, namespaceFile, skipUndeploy)
+	p, err := platforms.FromString(platform)
+	require.NoError(t, err)
+	ct := contrasttest.New(t, imageReplacementsFile, namespaceFile, p, skipUndeploy)
 
 	resources, err := kuberesource.GetDEnts()
 	require.NoError(t, err)
@@ -83,6 +86,7 @@ func TestGetDEnts(t *testing.T) {
 func TestMain(m *testing.M) {
 	flag.StringVar(&imageReplacementsFile, "image-replacements", "", "path to image replacements file")
 	flag.StringVar(&namespaceFile, "namespace-file", "", "file to store the namespace in")
+	flag.StringVar(&platform, "platform", "", "Deployment platform")
 	flag.BoolVar(&skipUndeploy, "skip-undeploy", false, "skip undeploy step in the test")
 	flag.Parse()
 

e2e/internal/contrasttest/assets/allow-all.rego

@@ -0,0 +1,38 @@
+package agent_policy
+
+default AddARPNeighborsRequest := true
+default AddSwapRequest := true
+default CloseStdinRequest := true
+default CopyFileRequest := true
+default CreateContainerRequest := true
+default CreateSandboxRequest := true
+default DestroySandboxRequest := true
+default ExecProcessRequest := true
+default GetMetricsRequest := true
+default GetOOMEventRequest := true
+default GuestDetailsRequest := true
+default ListInterfacesRequest := true
+default ListRoutesRequest := true
+default MemHotplugByProbeRequest := true
+default OnlineCPUMemRequest := true
+default PauseContainerRequest := true
+default PullImageRequest := true
+default ReadStreamRequest := true
+default RemoveContainerRequest := true
+default RemoveStaleVirtiofsShareMountsRequest := true
+default ReseedRandomDevRequest := true
+default ResumeContainerRequest := true
+default SetGuestDateTimeRequest := true
+default SetPolicyRequest := true
+default SignalProcessRequest := true
+default StartContainerRequest := true
+default StartTracingRequest := true
+default StatsContainerRequest := true
+default StopTracingRequest := true
+default TtyWinResizeRequest := true
+default UpdateContainerRequest := true
+default UpdateEphemeralMountsRequest := true
+default UpdateInterfaceRequest := true
+default UpdateRoutesRequest := true
+default WaitProcessRequest := true
+default WriteStreamRequest := true

e2e/internal/contrasttest/contrasttest.go

@@ -8,9 +8,11 @@ import (
 	"context"
 	"crypto/rand"
 	"crypto/x509"
+	_ "embed"
 	"encoding/hex"
 	"fmt"
 	"io"
+	"io/fs"
 	"os"
 	"path"
 	"regexp"
@@ -28,13 +30,17 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+//go:embed assets/allow-all.rego
+var allowAllRegoRules []byte
+
 // ContrastTest is the Contrast test helper struct.
 type ContrastTest struct {
 	// inputs, usually filled by New()
 	Namespace             string
 	WorkDir               string
 	ImageReplacements     map[string]string
 	ImageReplacementsFile string
+	Platform              platforms.Platform
 	NamespaceFile         string
 	SkipUndeploy          bool
 	Kubeclient            *kubeclient.Kubeclient
@@ -46,14 +52,14 @@ type ContrastTest struct {
 }
 
 // New creates a new contrasttest.T object bound to the given test.
-func New(t *testing.T, imageReplacements, namespaceFile string, skipUndeploy bool) *ContrastTest {
+func New(t *testing.T, imageReplacements, namespaceFile string, platform platforms.Platform, skipUndeploy bool) *ContrastTest {
 	return &ContrastTest{
 		Namespace:             makeNamespace(t),
 		WorkDir:               t.TempDir(),
-		ImageReplacementsFile: imageReplacements,
-		NamespaceFile:         namespaceFile,
-		SkipUndeploy:          skipUndeploy,
-		Kubeclient:            kubeclient.NewForTest(t),
+		ImageReplacementsFile: imageReplacements, Platform: platform,
+		NamespaceFile: namespaceFile,
+		SkipUndeploy:  skipUndeploy,
+		Kubeclient:    kubeclient.NewForTest(t),
 	}
 }
 
@@ -140,10 +146,18 @@ func (ct *ContrastTest) Init(t *testing.T, resources []any) {
 func (ct *ContrastTest) Generate(t *testing.T) {
 	require := require.New(t)
 
+	// The policy specified in the rego rules doesn't work for these platforms
+	// yet. Use the allow-all policy instead.
+	switch ct.Platform {
+	case platforms.K3sQEMUSNP:
+		err := os.WriteFile(path.Join(ct.WorkDir, "rules.rego"), allowAllRegoRules, fs.ModePerm)
+		require.NoError(err)
+	}
+
 	args := append(
 		ct.commonArgs(),
 		"--image-replacements", ct.ImageReplacementsFile,
-		"--reference-values", "aks-clh-snp",
+		"--reference-values", ct.Platform.String(),
 		path.Join(ct.WorkDir, "resources.yaml"),
 	)
 
@@ -247,7 +261,7 @@ func (ct *ContrastTest) commonArgs() []string {
 func (ct *ContrastTest) installRuntime(t *testing.T) {
 	require := require.New(t)
 
-	resources, err := kuberesource.Runtime(platforms.AKSCloudHypervisorSNP)
+	resources, err := kuberesource.Runtime(ct.Platform)
 	require.NoError(err)
 	resources = kuberesource.PatchImages(resources, ct.ImageReplacements)
 	resources = kuberesource.PatchNamespaces(resources, ct.Namespace)

e2e/openssl/openssl_test.go

@@ -21,6 +21,7 @@ import (
 	"github.com/edgelesssys/contrast/e2e/internal/kubeclient"
 	"github.com/edgelesssys/contrast/internal/kuberesource"
 	"github.com/edgelesssys/contrast/internal/manifest"
+	"github.com/edgelesssys/contrast/node-installer/platforms"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -35,13 +36,15 @@ const (
 )
 
 var (
-	imageReplacementsFile, namespaceFile string
-	skipUndeploy                         bool
+	imageReplacementsFile, namespaceFile, platform string
+	skipUndeploy                                   bool
 )
 
 // TestOpenSSL runs e2e tests on the example OpenSSL deployment.
 func TestOpenSSL(t *testing.T) {
-	ct := contrasttest.New(t, imageReplacementsFile, namespaceFile, skipUndeploy)
+	p, err := platforms.FromString(platform)
+	require.NoError(t, err)
+	ct := contrasttest.New(t, imageReplacementsFile, namespaceFile, p, skipUndeploy)
 
 	resources := kuberesource.OpenSSL()
 
@@ -216,6 +219,7 @@ func TestOpenSSL(t *testing.T) {
 func TestMain(m *testing.M) {
 	flag.StringVar(&imageReplacementsFile, "image-replacements", "", "path to image replacements file")
 	flag.StringVar(&namespaceFile, "namespace-file", "", "file to store the namespace in")
+	flag.StringVar(&platform, "platform", "", "Deployment platform")
 	flag.BoolVar(&skipUndeploy, "skip-undeploy", false, "skip undeploy step in the test")
 	flag.Parse()
 

e2e/policy/policy_test.go

@@ -21,6 +21,7 @@ import (
 	"github.com/edgelesssys/contrast/internal/kubeapi"
 	"github.com/edgelesssys/contrast/internal/kuberesource"
 	"github.com/edgelesssys/contrast/internal/manifest"
+	"github.com/edgelesssys/contrast/node-installer/platforms"
 	"github.com/prometheus/common/expfmt"
 	"github.com/stretchr/testify/require"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -33,12 +34,14 @@ const (
 )
 
 var (
-	imageReplacementsFile, namespaceFile string
-	skipUndeploy                         bool
+	imageReplacementsFile, namespaceFile, platform string
+	skipUndeploy                                   bool
 )
 
 func TestPolicy(t *testing.T) {
-	ct := contrasttest.New(t, imageReplacementsFile, namespaceFile, skipUndeploy)
+	p, err := platforms.FromString(platform)
+	require.NoError(t, err)
+	ct := contrasttest.New(t, imageReplacementsFile, namespaceFile, p, skipUndeploy)
 
 	resources := kuberesource.OpenSSL()
 
@@ -141,6 +144,7 @@ func TestPolicy(t *testing.T) {
 func TestMain(m *testing.M) {
 	flag.StringVar(&imageReplacementsFile, "image-replacements", "", "path to image replacements file")
 	flag.StringVar(&namespaceFile, "namespace-file", "", "file to store the namespace in")
+	flag.StringVar(&platform, "platform", "", "Deployment platform")
 	flag.BoolVar(&skipUndeploy, "skip-undeploy", false, "skip undeploy step in the test")
 	flag.Parse()
 

e2e/servicemesh/servicemesh_test.go

@@ -20,18 +20,21 @@ import (
 	"github.com/edgelesssys/contrast/e2e/internal/contrasttest"
 	"github.com/edgelesssys/contrast/e2e/internal/kubeclient"
 	"github.com/edgelesssys/contrast/internal/kuberesource"
+	"github.com/edgelesssys/contrast/node-installer/platforms"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 var (
-	imageReplacementsFile, namespaceFile string
-	skipUndeploy                         bool
+	imageReplacementsFile, namespaceFile, platform string
+	skipUndeploy                                   bool
 )
 
 // TestIngressEgress tests that the ingress and egress proxies work as configured.
 func TestIngressEgress(t *testing.T) {
-	ct := contrasttest.New(t, imageReplacementsFile, namespaceFile, skipUndeploy)
+	p, err := platforms.FromString(platform)
+	require.NoError(t, err)
+	ct := contrasttest.New(t, imageReplacementsFile, namespaceFile, p, skipUndeploy)
 
 	resources := kuberesource.Emojivoto(kuberesource.ServiceMeshIngressEgress)
 
@@ -145,6 +148,7 @@ func TestIngressEgress(t *testing.T) {
 func TestMain(m *testing.M) {
 	flag.StringVar(&imageReplacementsFile, "image-replacements", "", "path to image replacements file")
 	flag.StringVar(&namespaceFile, "namespace-file", "", "file to store the namespace in")
+	flag.StringVar(&platform, "platform", "", "Deployment platform")
 	flag.BoolVar(&skipUndeploy, "skip-undeploy", false, "skip undeploy step in the test")
 	flag.Parse()
 

internal/manifest/constants.go

@@ -29,6 +29,19 @@ func DefaultAKS() Manifest {
 	return mnfst
 }
 
+func DefaultBaremetalSNP() Manifest {
+	mnfst := Default()
+	mnfst.ReferenceValues.SNP = SNPReferenceValues{
+		MinimumTCB: SNPTCB{
+			BootloaderVersion: toPtr(SVN(0)),
+			TEEVersion:        toPtr(SVN(0)),
+			SNPVersion:        toPtr(SVN(0)),
+			MicrocodeVersion:  toPtr(SVN(0)),
+		},
+	}
+	return mnfst
+}
+
 func toPtr[T any](t T) *T {
 	return &t
 }

justfile

@@ -29,7 +29,7 @@ tardev-snapshotter: (push "tardev-snapshotter")
 
 default_cli := "contrast.cli"
 default_deploy_target := "openssl"
-default_platform := "AKS-CLH-SNP"
+default_platform := "$default_platform"
 workspace_dir := "workspace"
 
 # Build the node-installer, containerize and push it.
@@ -39,7 +39,7 @@ node-installer platform=default_platform: tardev-snapshotter
         "AKS-CLH-SNP")
             just push "node-installer-microsoft"
         ;;
-        "K3s-QEMU-TDX"|"RKE2-QEMU-TDX")
+        "K3s-QEMU-TDX"|"K3s-QEMU-SNP"|"RKE2-QEMU-TDX")
             just push "node-installer-kata"
         ;;
         *)