diff --git a/e2e/internal/kubeclient/deploy.go b/e2e/internal/kubeclient/deploy.go index 69aa14e060..8a5ca7371e 100644 --- a/e2e/internal/kubeclient/deploy.go +++ b/e2e/internal/kubeclient/deploy.go @@ -99,6 +99,38 @@ func (c *Kubeclient) WaitForDeployment(ctx context.Context, namespace, name stri } } +func (c *Kubeclient) WaitForLoadBalancer(ctx context.Context, namespace, name string) (string, error) { + watcher, err := c.client.CoreV1().Services(namespace).Watch(ctx, metav1.ListOptions{FieldSelector: "metadata.name=" + name}) + if err != nil { + return "", err + } + for { + select { + case evt := <-watcher.ResultChan(): + switch evt.Type { + case watch.Added: + fallthrough + case watch.Modified: + svc, ok := evt.Object.(*corev1.Service) + if !ok { + return "", fmt.Errorf("watcher received unexpected type %T", evt.Object) + } + for _, ingress := range svc.Status.LoadBalancer.Ingress { + if ingress.IP != "" { + return ingress.IP, nil + } + } + case watch.Deleted: + return "", fmt.Errorf("service %s/%s was deleted while waiting for it", namespace, name) + default: + c.log.Warn("ignoring unexpected watch event", "type", evt.Type, "object", evt.Object) + } + case <-ctx.Done(): + return "", ctx.Err() + } + } +} + func (c *Kubeclient) toJSON(a any) string { s, err := json.Marshal(a) if err != nil { diff --git a/e2e/release/release_test.go b/e2e/release/release_test.go new file mode 100644 index 0000000000..4316e5acfe --- /dev/null +++ b/e2e/release/release_test.go @@ -0,0 +1,219 @@ +//go:build e2e +// +build e2e + +package release + +import ( + "context" + "crypto/rand" + "encoding/hex" + "flag" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path" + "testing" + "time" + + "github.com/edgelesssys/contrast/e2e/internal/kubeclient" + "github.com/edgelesssys/contrast/e2e/internal/kuberesource" + "github.com/edgelesssys/contrast/internal/kubeapi" + "github.com/google/go-github/v61/github" + "github.com/stretchr/testify/require" +) + +// namespace the tests are executed in. +const ( + opensslFrontend = "openssl-frontend" + opensslBackend = "openssl-backend" +) + +var ( + owner = flag.String("owner", "edgelesssys", "TODO") + repo = flag.String("repo", "contrast", "TODO") + tag = flag.String("tag", "", "TODO") + namespace = flag.String("namespace", "", "TODO") + keep = flag.Bool("keep", false, "") +) + +func copyRelease(t *testing.T, releaseDir string) string { + require := require.New(t) + + dir := t.TempDir() + infos, err := os.ReadDir(releaseDir) + require.NoError(err) + for _, info := range infos { + src, err := os.Open(path.Join(releaseDir, info.Name())) + require.NoError(err) + dst, err := os.OpenFile(path.Join(dir, info.Name()), os.O_CREATE|os.O_RDWR, 0o777) + require.NoError(err) + _, err = io.Copy(dst, src) + require.NoError(err) + src.Close() + dst.Close() + } + + return dir +} + +func fetchRelease(ctx context.Context, t *testing.T) string { + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + require := require.New(t) + gh := github.NewClient(nil).WithAuthToken(os.Getenv("GH_TOKEN")) + + dir := t.TempDir() + + // Find our target release. There is GetReleaseByTag, but we may be looking for a draft release. + rels, _, err := gh.Repositories.ListReleases(ctx, *owner, *repo, nil) + require.NoError(err) + var release *github.RepositoryRelease + for _, rel := range rels { + if *rel.TagName == *tag { + release = rel + break + } + } + require.NotNil(release) + + for _, asset := range release.Assets { + f, err := os.OpenFile(path.Join(dir, *asset.Name), os.O_CREATE|os.O_RDWR, 0o777) + require.NoError(err) + body, _, err := gh.Repositories.DownloadReleaseAsset(ctx, *owner, *repo, *asset.ID, http.DefaultClient) + require.NoError(err) + _, err = io.Copy(f, body) + require.NoError(err) + f.Close() + } + + return dir +} + +func TestRelease(t *testing.T) { + ctx := context.Background() + k := kubeclient.NewForTest(t) + + if *namespace == "" { + *namespace = randomNamespace(t) + } + + require.True(t, t.Run("create-namespace", func(t *testing.T) { + require := require.New(t) + ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) + defer cancel() + + res, err := kuberesource.ResourcesToUnstructured([]any{kuberesource.Namespace(*namespace)}) + require.NoError(err) + require.NoError(k.Apply(ctx, res...)) + }), "the namespace is required for subsequent tests to run") + + t.Cleanup(func() { + if *keep { + return + } + ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) + defer cancel() + + res, err := kuberesource.ResourcesToUnstructured([]any{kuberesource.Namespace(*namespace)}) + if err != nil { + return + } + k.Delete(ctx, res...) + }) + + // dir := fetchRelease(ctx, t) + dir := copyRelease(t, "/tmp/releasetest") + + contrast := &runner{path.Join(dir, "contrast")} + + for _, sub := range []string{"help"} { + contrast.Run(t, ctx, 2*time.Second, sub) + } + + var coordinatorIP string + require.True(t, t.Run("apply-coordinator", func(t *testing.T) { + require := require.New(t) + ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + + yaml, err := os.ReadFile(path.Join(dir, "coordinator.yml")) + require.NoError(err) + resources, err := kubeapi.UnmarshalUnstructuredK8SResource(yaml) + require.NoError(err) + + for _, r := range resources { + require.NoError(k.PatchNamespace(*namespace, r)) + } + + require.NoError(k.Apply(ctx, resources...)) + require.NoError(k.WaitForDeployment(ctx, *namespace, "coordinator")) + coordinatorIP, err = k.WaitForLoadBalancer(ctx, *namespace, "coordinator") + require.NoError(err) + }), "the coordinator is required for subsequent tests to run") + + require.True(t, t.Run("unpack-deployment", func(t *testing.T) { + require := require.New(t) + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + cmd := exec.CommandContext(ctx, "unzip", "emojivoto-demo.zip") + cmd.Dir = dir + out, err := cmd.CombinedOutput() + require.NoError(err, "output:\n%s", string(out)) + + infos, err := os.ReadDir(path.Join(dir, "deployment")) + require.NoError(err) + for _, info := range infos { + name := path.Join(path.Join(dir, "deployment"), info.Name()) + yaml, err := os.ReadFile(name) + resources, err := kubeapi.UnmarshalUnstructuredK8SResource(yaml) + require.NoError(err) + + for _, r := range resources { + require.NoError(k.PatchNamespace(*namespace, r)) + } + newYAML, err := kuberesource.EncodeUnstructured(resources) + require.NoError(err) + require.NoError(os.WriteFile(name, newYAML, 0o644)) + + } + }), "unpacking needs to succeed for subsequent tests to run") + + contrast.Run(t, ctx, 2*time.Minute, "generate", "deployment/") + contrast.Run(t, ctx, 1*time.Minute, "set", "-c", coordinatorIP+":1313", "deployment/") + contrast.Run(t, ctx, 1*time.Minute, "verify", "-c", coordinatorIP+":1313") + + // TODO: apply and test emojivoto +} + +type runner struct { + path string +} + +func (r *runner) Run(t *testing.T, ctx context.Context, timeout time.Duration, args ...string) { + require.True(t, t.Run(args[0], func(t *testing.T) { + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + cmd := exec.CommandContext(ctx, r.path, args...) + cmd.Dir = path.Dir(r.path) + out, err := cmd.CombinedOutput() + require.NoError(t, err, "output:\n%s", string(out)) + }), args[0]+" needs to succeed for subsequent tests to run") +} + +func randomNamespace(t *testing.T) string { + buf := make([]byte, 4) + n, err := rand.Read(buf) + require.NoError(t, err) + require.Equal(t, 4, n) + return "releasetest-" + hex.EncodeToString(buf) +} + +func TestMain(m *testing.M) { + flag.Parse() + + fmt.Printf("Release: %s/%s@%s\n", *owner, *repo, *tag) + os.Exit(m.Run()) +} diff --git a/go.mod b/go.mod index 0b90bea650..931cf7ed98 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/edgelesssys/contrast go 1.21 require ( + github.com/google/go-github/v61 v61.0.0 github.com/google/go-sev-guest v0.11.1 github.com/spf13/afero v1.11.0 github.com/spf13/cobra v1.8.0 @@ -32,6 +33,7 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-configfs-tsm v0.2.2 // indirect + github.com/google/go-querystring v1.1.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/logger v1.1.1 // indirect github.com/google/uuid v1.6.0 // indirect diff --git a/go.sum b/go.sum index 73c598726a..9a9bfe95b1 100644 --- a/go.sum +++ b/go.sum @@ -25,12 +25,17 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-configfs-tsm v0.2.2 h1:YnJ9rXIOj5BYD7/0DNnzs8AOp7UcvjfTvt215EWcs98= github.com/google/go-configfs-tsm v0.2.2/go.mod h1:EL1GTDFMb5PZQWDviGfZV9n87WeGTR/JUg13RfwkgRo= +github.com/google/go-github/v61 v61.0.0 h1:VwQCBwhyE9JclCI+22/7mLB1PuU9eowCXKY5pNlu1go= +github.com/google/go-github/v61 v61.0.0/go.mod h1:0WR+KmsWX75G2EbpyGsGmradjo3IiciuI4BmdVCobQY= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/go-sev-guest v0.11.1 h1:gnww4U8fHV5DCPz4gykr1s8SEX1fFNcxCBy+vvXN24k= github.com/google/go-sev-guest v0.11.1/go.mod h1:qBOfb+JmgsUI3aUyzQoGC13Kpp9zwLeWvuyXmA9q77w= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=