diff --git a/.github/workflows/e2e_openssl_baremetal.yml b/.github/workflows/e2e_openssl_baremetal.yml new file mode 100644 index 0000000000..cc29089aca --- /dev/null +++ b/.github/workflows/e2e_openssl_baremetal.yml @@ -0,0 +1,56 @@ +name: e2e test openssl baremetal + +on: + workflow_dispatch: + inputs: + skip-undeploy: + description: "Skip undeploy" + required: false + type: boolean + default: false + pull_request: + paths-ignore: + - dev-docs/** + - docs/** + - rfc/** + +env: + container_registry: ghcr.io/edgelesssys + DO_NOT_TRACK: 1 + +jobs: + test: + runs-on: + labels: snp + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - name: Log in to ghcr.io Container registry + uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - uses: nicknovitski/nix-develop@a2060d116a50b36dfab02280af558e73ab52427d # v1.1.0 + - name: Create justfile.env + run: | + cat < justfile.env + container_registry=${{ env.container_registry }} + default_platform="K3s-QEMU-SNP" + EOF + - name: Build and prepare deployments + run: | + just coordinator initializer openssl port-forwarder node-installer + - name: E2E Test + run: | + nix shell .#contrast.e2e --command openssl.test -test.v \ + --image-replacements workspace/just.containerlookup \ + --namespace-file workspace/e2e.namespace \ + --platform K3s-QEMU-SNP \ + --skip-undeploy="${{ inputs.skip-undeploy && 'true' || 'false' }}" + - name: Cleanup + if: cancelled() && !inputs.skip-undeploy + run: | + kubectl delete ns "$(cat workspace/e2e.namespace)" --timeout 5m diff --git a/cli/cmd/generate.go b/cli/cmd/generate.go index e9184b7f5b..4da9ee166a 100644 --- a/cli/cmd/generate.go +++ b/cli/cmd/generate.go @@ -131,6 +131,8 @@ func runGenerate(cmd *cobra.Command, args []string) error { switch flags.referenceValuesPlatform { case platforms.AKSCloudHypervisorSNP: defaultManifest = manifest.DefaultAKS() + case platforms.K3sQEMUSNP: + defaultManifest = manifest.DefaultBaremetalSNP() } defaultManifestData, err := json.MarshalIndent(&defaultManifest, "", " ") diff --git a/e2e/genpolicy/genpolicy_test.go b/e2e/genpolicy/genpolicy_test.go index 0d54cb93e8..29b3b75abf 100644 --- a/e2e/genpolicy/genpolicy_test.go +++ b/e2e/genpolicy/genpolicy_test.go @@ -19,21 +19,25 @@ import ( "github.com/edgelesssys/contrast/e2e/internal/contrasttest" "github.com/edgelesssys/contrast/e2e/internal/kubeclient" "github.com/edgelesssys/contrast/internal/kuberesource" + "github.com/edgelesssys/contrast/node-installer/platforms" "github.com/stretchr/testify/require" ) var ( - imageReplacementsFile, namespaceFile string - skipUndeploy bool + imageReplacementsFile, namespaceFile, platform string + skipUndeploy bool ) // TestGenpolicy runs regression tests for generated policies. func TestGenpolicy(t *testing.T) { testCases := kuberesource.GenpolicyRegressionTests() + p, err := platforms.FromString(platform) + require.NoError(t, err) + for name, deploy := range testCases { t.Run(name, func(t *testing.T) { - ct := contrasttest.New(t, imageReplacementsFile, namespaceFile, skipUndeploy) + ct := contrasttest.New(t, imageReplacementsFile, namespaceFile, p, skipUndeploy) ct.Init(t, []any{deploy}) @@ -67,6 +71,7 @@ func TestGenpolicy(t *testing.T) { func TestMain(m *testing.M) { flag.StringVar(&imageReplacementsFile, "image-replacements", "", "path to image replacements file") flag.StringVar(&namespaceFile, "namespace-file", "", "file to store the namespace in") + flag.StringVar(&platform, "platform", "", "Deployment platform") flag.BoolVar(&skipUndeploy, "skip-undeploy", false, "skip undeploy step in the test") flag.Parse() diff --git a/e2e/getdents/getdents_test.go b/e2e/getdents/getdents_test.go index 1a8bff1969..690b25e466 100644 --- a/e2e/getdents/getdents_test.go +++ b/e2e/getdents/getdents_test.go @@ -19,6 +19,7 @@ import ( "github.com/edgelesssys/contrast/e2e/internal/contrasttest" "github.com/edgelesssys/contrast/e2e/internal/kubeclient" "github.com/edgelesssys/contrast/internal/kuberesource" + "github.com/edgelesssys/contrast/node-installer/platforms" "github.com/stretchr/testify/require" ) @@ -27,12 +28,14 @@ const ( ) var ( - imageReplacementsFile, namespaceFile string - skipUndeploy bool + imageReplacementsFile, namespaceFile, platform string + skipUndeploy bool ) func TestGetDEnts(t *testing.T) { - ct := contrasttest.New(t, imageReplacementsFile, namespaceFile, skipUndeploy) + p, err := platforms.FromString(platform) + require.NoError(t, err) + ct := contrasttest.New(t, imageReplacementsFile, namespaceFile, p, skipUndeploy) resources, err := kuberesource.GetDEnts() require.NoError(t, err) @@ -83,6 +86,7 @@ func TestGetDEnts(t *testing.T) { func TestMain(m *testing.M) { flag.StringVar(&imageReplacementsFile, "image-replacements", "", "path to image replacements file") flag.StringVar(&namespaceFile, "namespace-file", "", "file to store the namespace in") + flag.StringVar(&platform, "platform", "", "Deployment platform") flag.BoolVar(&skipUndeploy, "skip-undeploy", false, "skip undeploy step in the test") flag.Parse() diff --git a/e2e/internal/contrasttest/assets/allow-all.rego b/e2e/internal/contrasttest/assets/allow-all.rego new file mode 100644 index 0000000000..7ac8134f03 --- /dev/null +++ b/e2e/internal/contrasttest/assets/allow-all.rego @@ -0,0 +1,38 @@ +package agent_policy + +default AddARPNeighborsRequest := true +default AddSwapRequest := true +default CloseStdinRequest := true +default CopyFileRequest := true +default CreateContainerRequest := true +default CreateSandboxRequest := true +default DestroySandboxRequest := true +default ExecProcessRequest := true +default GetMetricsRequest := true +default GetOOMEventRequest := true +default GuestDetailsRequest := true +default ListInterfacesRequest := true +default ListRoutesRequest := true +default MemHotplugByProbeRequest := true +default OnlineCPUMemRequest := true +default PauseContainerRequest := true +default PullImageRequest := true +default ReadStreamRequest := true +default RemoveContainerRequest := true +default RemoveStaleVirtiofsShareMountsRequest := true +default ReseedRandomDevRequest := true +default ResumeContainerRequest := true +default SetGuestDateTimeRequest := true +default SetPolicyRequest := true +default SignalProcessRequest := true +default StartContainerRequest := true +default StartTracingRequest := true +default StatsContainerRequest := true +default StopTracingRequest := true +default TtyWinResizeRequest := true +default UpdateContainerRequest := true +default UpdateEphemeralMountsRequest := true +default UpdateInterfaceRequest := true +default UpdateRoutesRequest := true +default WaitProcessRequest := true +default WriteStreamRequest := true diff --git a/e2e/internal/contrasttest/contrasttest.go b/e2e/internal/contrasttest/contrasttest.go index 7742804ea9..f3c0160c02 100644 --- a/e2e/internal/contrasttest/contrasttest.go +++ b/e2e/internal/contrasttest/contrasttest.go @@ -8,9 +8,11 @@ import ( "context" "crypto/rand" "crypto/x509" + _ "embed" "encoding/hex" "fmt" "io" + "io/fs" "os" "path" "regexp" @@ -28,6 +30,9 @@ import ( "github.com/stretchr/testify/require" ) +//go:embed assets/allow-all.rego +var allowAllRegoRules []byte + // ContrastTest is the Contrast test helper struct. type ContrastTest struct { // inputs, usually filled by New() @@ -35,6 +40,7 @@ type ContrastTest struct { WorkDir string ImageReplacements map[string]string ImageReplacementsFile string + Platform platforms.Platform NamespaceFile string SkipUndeploy bool Kubeclient *kubeclient.Kubeclient @@ -46,14 +52,14 @@ type ContrastTest struct { } // New creates a new contrasttest.T object bound to the given test. -func New(t *testing.T, imageReplacements, namespaceFile string, skipUndeploy bool) *ContrastTest { +func New(t *testing.T, imageReplacements, namespaceFile string, platform platforms.Platform, skipUndeploy bool) *ContrastTest { return &ContrastTest{ Namespace: makeNamespace(t), WorkDir: t.TempDir(), - ImageReplacementsFile: imageReplacements, - NamespaceFile: namespaceFile, - SkipUndeploy: skipUndeploy, - Kubeclient: kubeclient.NewForTest(t), + ImageReplacementsFile: imageReplacements, Platform: platform, + NamespaceFile: namespaceFile, + SkipUndeploy: skipUndeploy, + Kubeclient: kubeclient.NewForTest(t), } } @@ -140,10 +146,18 @@ func (ct *ContrastTest) Init(t *testing.T, resources []any) { func (ct *ContrastTest) Generate(t *testing.T) { require := require.New(t) + // The policy specified in the rego rules doesn't work for these platforms + // yet. Use the allow-all policy instead. + switch ct.Platform { + case platforms.K3sQEMUSNP: + err := os.WriteFile(path.Join(ct.WorkDir, "rules.rego"), allowAllRegoRules, fs.ModePerm) + require.NoError(err) + } + args := append( ct.commonArgs(), "--image-replacements", ct.ImageReplacementsFile, - "--reference-values", "aks-clh-snp", + "--reference-values", ct.Platform.String(), path.Join(ct.WorkDir, "resources.yaml"), ) @@ -247,7 +261,7 @@ func (ct *ContrastTest) commonArgs() []string { func (ct *ContrastTest) installRuntime(t *testing.T) { require := require.New(t) - resources, err := kuberesource.Runtime(platforms.AKSCloudHypervisorSNP) + resources, err := kuberesource.Runtime(ct.Platform) require.NoError(err) resources = kuberesource.PatchImages(resources, ct.ImageReplacements) resources = kuberesource.PatchNamespaces(resources, ct.Namespace) diff --git a/e2e/openssl/openssl_test.go b/e2e/openssl/openssl_test.go index 0a05c88167..106354a7d8 100644 --- a/e2e/openssl/openssl_test.go +++ b/e2e/openssl/openssl_test.go @@ -21,6 +21,7 @@ import ( "github.com/edgelesssys/contrast/e2e/internal/kubeclient" "github.com/edgelesssys/contrast/internal/kuberesource" "github.com/edgelesssys/contrast/internal/manifest" + "github.com/edgelesssys/contrast/node-installer/platforms" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -35,13 +36,15 @@ const ( ) var ( - imageReplacementsFile, namespaceFile string - skipUndeploy bool + imageReplacementsFile, namespaceFile, platform string + skipUndeploy bool ) // TestOpenSSL runs e2e tests on the example OpenSSL deployment. func TestOpenSSL(t *testing.T) { - ct := contrasttest.New(t, imageReplacementsFile, namespaceFile, skipUndeploy) + p, err := platforms.FromString(platform) + require.NoError(t, err) + ct := contrasttest.New(t, imageReplacementsFile, namespaceFile, p, skipUndeploy) resources := kuberesource.OpenSSL() @@ -216,6 +219,7 @@ func TestOpenSSL(t *testing.T) { func TestMain(m *testing.M) { flag.StringVar(&imageReplacementsFile, "image-replacements", "", "path to image replacements file") flag.StringVar(&namespaceFile, "namespace-file", "", "file to store the namespace in") + flag.StringVar(&platform, "platform", "", "Deployment platform") flag.BoolVar(&skipUndeploy, "skip-undeploy", false, "skip undeploy step in the test") flag.Parse() diff --git a/e2e/policy/policy_test.go b/e2e/policy/policy_test.go index 0fd02696ad..60d3c8e050 100644 --- a/e2e/policy/policy_test.go +++ b/e2e/policy/policy_test.go @@ -21,6 +21,7 @@ import ( "github.com/edgelesssys/contrast/internal/kubeapi" "github.com/edgelesssys/contrast/internal/kuberesource" "github.com/edgelesssys/contrast/internal/manifest" + "github.com/edgelesssys/contrast/node-installer/platforms" "github.com/prometheus/common/expfmt" "github.com/stretchr/testify/require" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -33,12 +34,14 @@ const ( ) var ( - imageReplacementsFile, namespaceFile string - skipUndeploy bool + imageReplacementsFile, namespaceFile, platform string + skipUndeploy bool ) func TestPolicy(t *testing.T) { - ct := contrasttest.New(t, imageReplacementsFile, namespaceFile, skipUndeploy) + p, err := platforms.FromString(platform) + require.NoError(t, err) + ct := contrasttest.New(t, imageReplacementsFile, namespaceFile, p, skipUndeploy) resources := kuberesource.OpenSSL() @@ -141,6 +144,7 @@ func TestPolicy(t *testing.T) { func TestMain(m *testing.M) { flag.StringVar(&imageReplacementsFile, "image-replacements", "", "path to image replacements file") flag.StringVar(&namespaceFile, "namespace-file", "", "file to store the namespace in") + flag.StringVar(&platform, "platform", "", "Deployment platform") flag.BoolVar(&skipUndeploy, "skip-undeploy", false, "skip undeploy step in the test") flag.Parse() diff --git a/e2e/servicemesh/servicemesh_test.go b/e2e/servicemesh/servicemesh_test.go index 555b9cfdce..ae355d56a6 100644 --- a/e2e/servicemesh/servicemesh_test.go +++ b/e2e/servicemesh/servicemesh_test.go @@ -20,18 +20,21 @@ import ( "github.com/edgelesssys/contrast/e2e/internal/contrasttest" "github.com/edgelesssys/contrast/e2e/internal/kubeclient" "github.com/edgelesssys/contrast/internal/kuberesource" + "github.com/edgelesssys/contrast/node-installer/platforms" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) var ( - imageReplacementsFile, namespaceFile string - skipUndeploy bool + imageReplacementsFile, namespaceFile, platform string + skipUndeploy bool ) // TestIngressEgress tests that the ingress and egress proxies work as configured. func TestIngressEgress(t *testing.T) { - ct := contrasttest.New(t, imageReplacementsFile, namespaceFile, skipUndeploy) + p, err := platforms.FromString(platform) + require.NoError(t, err) + ct := contrasttest.New(t, imageReplacementsFile, namespaceFile, p, skipUndeploy) resources := kuberesource.Emojivoto(kuberesource.ServiceMeshIngressEgress) @@ -145,6 +148,7 @@ func TestIngressEgress(t *testing.T) { func TestMain(m *testing.M) { flag.StringVar(&imageReplacementsFile, "image-replacements", "", "path to image replacements file") flag.StringVar(&namespaceFile, "namespace-file", "", "file to store the namespace in") + flag.StringVar(&platform, "platform", "", "Deployment platform") flag.BoolVar(&skipUndeploy, "skip-undeploy", false, "skip undeploy step in the test") flag.Parse() diff --git a/internal/manifest/constants.go b/internal/manifest/constants.go index 1e0900b18e..ec88faeac7 100644 --- a/internal/manifest/constants.go +++ b/internal/manifest/constants.go @@ -29,6 +29,19 @@ func DefaultAKS() Manifest { return mnfst } +func DefaultBaremetalSNP() Manifest { + mnfst := Default() + mnfst.ReferenceValues.SNP = SNPReferenceValues{ + MinimumTCB: SNPTCB{ + BootloaderVersion: toPtr(SVN(0)), + TEEVersion: toPtr(SVN(0)), + SNPVersion: toPtr(SVN(0)), + MicrocodeVersion: toPtr(SVN(0)), + }, + } + return mnfst +} + func toPtr[T any](t T) *T { return &t } diff --git a/justfile b/justfile index 5d4145f5dd..0a1e8f4489 100644 --- a/justfile +++ b/justfile @@ -29,7 +29,7 @@ tardev-snapshotter: (push "tardev-snapshotter") default_cli := "contrast.cli" default_deploy_target := "openssl" -default_platform := "AKS-CLH-SNP" +default_platform := "$default_platform" workspace_dir := "workspace" # Build the node-installer, containerize and push it. @@ -39,7 +39,7 @@ node-installer platform=default_platform: tardev-snapshotter "AKS-CLH-SNP") just push "node-installer-microsoft" ;; - "K3s-QEMU-TDX"|"RKE2-QEMU-TDX") + "K3s-QEMU-TDX"|"K3s-QEMU-SNP"|"RKE2-QEMU-TDX") just push "node-installer-kata" ;; *) diff --git a/packages/by-name/contrast/package.nix b/packages/by-name/contrast/package.nix index 71dd1acd02..f18c3a1905 100644 --- a/packages/by-name/contrast/package.nix +++ b/packages/by-name/contrast/package.nix @@ -6,6 +6,7 @@ buildGoModule, buildGoTest, microsoft, + kata, genpolicy ? microsoft.genpolicy, contrast, installShellFiles, @@ -41,10 +42,10 @@ let ]; }; - launchDigest = builtins.readFile "${microsoft.runtime-class-files}/launch-digest.hex"; + launchDigest = builtins.readFile "${kata.runtime-class-files}/launch-digest.hex"; runtimeHandler = lib.removeSuffix "\n" ( - builtins.readFile "${microsoft.runtime-class-files}/runtime-handler" + builtins.readFile "${kata.runtime-class-files}/runtime-handler" ); packageOutputs = [ @@ -75,6 +76,7 @@ buildGoModule rec { (path.append root "cli/cmd/assets/image-replacements.txt") (path.append root "internal/attestation/snp/Milan.pem") (path.append root "internal/attestation/snp/Genoa.pem") + (path.append root "e2e/internal/contrasttest/assets/allow-all.rego") (path.append root "node-installer") (fileset.difference (fileset.fileFilter (file: hasSuffix ".go" file.name) root) ( path.append root "service-mesh" diff --git a/packages/by-name/kata/runtime-class-files/package.nix b/packages/by-name/kata/runtime-class-files/package.nix index cd60e7f4c5..5e039bb18c 100644 --- a/packages/by-name/kata/runtime-class-files/package.nix +++ b/packages/by-name/kata/runtime-class-files/package.nix @@ -13,8 +13,8 @@ let image = kata.kata-image; kernel = "${kata.kata-kernel-uvm}/bzImage"; - qemu-bin = "${qemu-static}/bin/qemu-system-x86_64"; - qemu-share = "${qemu-static}/share/qemu"; + qemu-bin = "${qemu-static.override { snpSupport = true; }}/bin/qemu-system-x86_64"; + qemu-share = "${qemu-static.override { snpSupport = true; }}/share/qemu"; ovmf = "${OVMF-SNP}/FV/OVMF.fd"; @@ -32,7 +32,7 @@ stdenvNoCC.mkDerivation { # TODO(msanft): perform the actual launch digest calculation. buildPhase = '' mkdir -p $out - sha256sum ${image} ${kernel} ${qemu-bin} ${qemu-share}/kvmvapic.bin ${qemu-share}/linuxboot_dma.bin ${qemu-share}/efi-virtio.rom ${containerd-shim-contrast-cc-v2} ${ovmf} | sha256sum | cut -d " " -f 1 > $out/launch-digest.hex + sha384sum ${image} ${kernel} ${qemu-bin} ${qemu-share}/kvmvapic.bin ${qemu-share}/linuxboot_dma.bin ${qemu-share}/efi-virtio.rom ${containerd-shim-contrast-cc-v2} ${ovmf} | sha384sum | cut -d " " -f 1 > $out/launch-digest.hex printf "contrast-cc-%s" "$(cat $out/launch-digest.hex | head -c 32)" > $out/runtime-handler '';