Skip to content

Commit

Permalink
wip: e2e release test
Browse files Browse the repository at this point in the history
  • Loading branch information
burgerdev committed Apr 19, 2024
1 parent e99a151 commit 06af4a2
Show file tree
Hide file tree
Showing 4 changed files with 258 additions and 0 deletions.
32 changes: 32 additions & 0 deletions e2e/internal/kubeclient/deploy.go
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,38 @@ func (c *Kubeclient) WaitForDeployment(ctx context.Context, namespace, name stri
}
}

func (c *Kubeclient) WaitForLoadBalancer(ctx context.Context, namespace, name string) (string, error) {
watcher, err := c.client.CoreV1().Services(namespace).Watch(ctx, metav1.ListOptions{FieldSelector: "metadata.name=" + name})
if err != nil {
return "", err
}
for {
select {
case evt := <-watcher.ResultChan():
switch evt.Type {
case watch.Added:
fallthrough
case watch.Modified:
svc, ok := evt.Object.(*corev1.Service)
if !ok {
return "", fmt.Errorf("watcher received unexpected type %T", evt.Object)
}
for _, ingress := range svc.Status.LoadBalancer.Ingress {
if ingress.IP != "" {
return ingress.IP, nil
}
}
case watch.Deleted:
return "", fmt.Errorf("service %s/%s was deleted while waiting for it", namespace, name)
default:
c.log.Warn("ignoring unexpected watch event", "type", evt.Type, "object", evt.Object)
}
case <-ctx.Done():
return "", ctx.Err()
}
}
}

func (c *Kubeclient) toJSON(a any) string {
s, err := json.Marshal(a)
if err != nil {
Expand Down
219 changes: 219 additions & 0 deletions e2e/release/release_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,219 @@
//go:build e2e
// +build e2e

package release

import (
"context"
"crypto/rand"
"encoding/hex"
"flag"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"path"
"testing"
"time"

"github.com/edgelesssys/contrast/e2e/internal/kubeclient"
"github.com/edgelesssys/contrast/e2e/internal/kuberesource"
"github.com/edgelesssys/contrast/internal/kubeapi"
"github.com/google/go-github/v61/github"
"github.com/stretchr/testify/require"
)

// namespace the tests are executed in.
const (
opensslFrontend = "openssl-frontend"
opensslBackend = "openssl-backend"
)

var (
owner = flag.String("owner", "edgelesssys", "TODO")
repo = flag.String("repo", "contrast", "TODO")
tag = flag.String("tag", "", "TODO")
namespace = flag.String("namespace", "", "TODO")
keep = flag.Bool("keep", false, "")
)

func copyRelease(t *testing.T, releaseDir string) string {
require := require.New(t)

dir := t.TempDir()
infos, err := os.ReadDir(releaseDir)
require.NoError(err)
for _, info := range infos {
src, err := os.Open(path.Join(releaseDir, info.Name()))
require.NoError(err)
dst, err := os.OpenFile(path.Join(dir, info.Name()), os.O_CREATE|os.O_RDWR, 0o777)
require.NoError(err)
_, err = io.Copy(dst, src)
require.NoError(err)
src.Close()
dst.Close()
}

return dir
}

func fetchRelease(ctx context.Context, t *testing.T) string {
ctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
defer cancel()
require := require.New(t)
gh := github.NewClient(nil).WithAuthToken(os.Getenv("GH_TOKEN"))

dir := t.TempDir()

// Find our target release. There is GetReleaseByTag, but we may be looking for a draft release.
rels, _, err := gh.Repositories.ListReleases(ctx, *owner, *repo, nil)
require.NoError(err)
var release *github.RepositoryRelease
for _, rel := range rels {
if *rel.TagName == *tag {
release = rel
break
}
}
require.NotNil(release)

for _, asset := range release.Assets {
f, err := os.OpenFile(path.Join(dir, *asset.Name), os.O_CREATE|os.O_RDWR, 0o777)
require.NoError(err)
body, _, err := gh.Repositories.DownloadReleaseAsset(ctx, *owner, *repo, *asset.ID, http.DefaultClient)
require.NoError(err)
_, err = io.Copy(f, body)
require.NoError(err)
f.Close()
}

return dir
}

func TestRelease(t *testing.T) {
ctx := context.Background()
k := kubeclient.NewForTest(t)

if *namespace == "" {
*namespace = randomNamespace(t)
}

require.True(t, t.Run("create-namespace", func(t *testing.T) {
require := require.New(t)
ctx, cancel := context.WithTimeout(ctx, 1*time.Minute)
defer cancel()

res, err := kuberesource.ResourcesToUnstructured([]any{kuberesource.Namespace(*namespace)})
require.NoError(err)
require.NoError(k.Apply(ctx, res...))
}), "the namespace is required for subsequent tests to run")

t.Cleanup(func() {
if *keep {
return
}
ctx, cancel := context.WithTimeout(ctx, 1*time.Minute)
defer cancel()

res, err := kuberesource.ResourcesToUnstructured([]any{kuberesource.Namespace(*namespace)})
if err != nil {
return
}
k.Delete(ctx, res...)
})

// dir := fetchRelease(ctx, t)
dir := copyRelease(t, "/tmp/releasetest")

contrast := &runner{path.Join(dir, "contrast")}

for _, sub := range []string{"help"} {
contrast.Run(t, ctx, 2*time.Second, sub)
}

var coordinatorIP string
require.True(t, t.Run("apply-coordinator", func(t *testing.T) {
require := require.New(t)
ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
defer cancel()

yaml, err := os.ReadFile(path.Join(dir, "coordinator.yml"))
require.NoError(err)
resources, err := kubeapi.UnmarshalUnstructuredK8SResource(yaml)
require.NoError(err)

for _, r := range resources {
require.NoError(k.PatchNamespace(*namespace, r))
}

require.NoError(k.Apply(ctx, resources...))
require.NoError(k.WaitForDeployment(ctx, *namespace, "coordinator"))
coordinatorIP, err = k.WaitForLoadBalancer(ctx, *namespace, "coordinator")
require.NoError(err)
}), "the coordinator is required for subsequent tests to run")

require.True(t, t.Run("unpack-deployment", func(t *testing.T) {
require := require.New(t)
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()

cmd := exec.CommandContext(ctx, "unzip", "emojivoto-demo.zip")
cmd.Dir = dir
out, err := cmd.CombinedOutput()
require.NoError(err, "output:\n%s", string(out))

infos, err := os.ReadDir(path.Join(dir, "deployment"))
require.NoError(err)
for _, info := range infos {
name := path.Join(path.Join(dir, "deployment"), info.Name())
yaml, err := os.ReadFile(name)
resources, err := kubeapi.UnmarshalUnstructuredK8SResource(yaml)
require.NoError(err)

for _, r := range resources {
require.NoError(k.PatchNamespace(*namespace, r))
}
newYAML, err := kuberesource.EncodeUnstructured(resources)
require.NoError(err)
require.NoError(os.WriteFile(name, newYAML, 0o644))

}
}), "unpacking needs to succeed for subsequent tests to run")

contrast.Run(t, ctx, 2*time.Minute, "generate", "deployment/")
contrast.Run(t, ctx, 1*time.Minute, "set", "-c", coordinatorIP+":1313", "deployment/")
contrast.Run(t, ctx, 1*time.Minute, "verify", "-c", coordinatorIP+":1313")

// TODO: apply and test emojivoto
}

type runner struct {
path string
}

func (r *runner) Run(t *testing.T, ctx context.Context, timeout time.Duration, args ...string) {
require.True(t, t.Run(args[0], func(t *testing.T) {
ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
cmd := exec.CommandContext(ctx, r.path, args...)
cmd.Dir = path.Dir(r.path)
out, err := cmd.CombinedOutput()
require.NoError(t, err, "output:\n%s", string(out))
}), args[0]+" needs to succeed for subsequent tests to run")
}

func randomNamespace(t *testing.T) string {
buf := make([]byte, 4)
n, err := rand.Read(buf)
require.NoError(t, err)
require.Equal(t, 4, n)
return "releasetest-" + hex.EncodeToString(buf)
}

func TestMain(m *testing.M) {
flag.Parse()

fmt.Printf("Release: %s/%s@%s\n", *owner, *repo, *tag)
os.Exit(m.Run())
}
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ module github.com/edgelesssys/contrast
go 1.21

require (
github.com/google/go-github/v61 v61.0.0
github.com/google/go-sev-guest v0.11.1
github.com/spf13/afero v1.11.0
github.com/spf13/cobra v1.8.0
Expand Down Expand Up @@ -32,6 +33,7 @@ require (
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/gnostic-models v0.6.8 // indirect
github.com/google/go-configfs-tsm v0.2.2 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/logger v1.1.1 // indirect
github.com/google/uuid v1.6.0 // indirect
Expand Down
5 changes: 5 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,17 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=
github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-configfs-tsm v0.2.2 h1:YnJ9rXIOj5BYD7/0DNnzs8AOp7UcvjfTvt215EWcs98=
github.com/google/go-configfs-tsm v0.2.2/go.mod h1:EL1GTDFMb5PZQWDviGfZV9n87WeGTR/JUg13RfwkgRo=
github.com/google/go-github/v61 v61.0.0 h1:VwQCBwhyE9JclCI+22/7mLB1PuU9eowCXKY5pNlu1go=
github.com/google/go-github/v61 v61.0.0/go.mod h1:0WR+KmsWX75G2EbpyGsGmradjo3IiciuI4BmdVCobQY=
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
github.com/google/go-sev-guest v0.11.1 h1:gnww4U8fHV5DCPz4gykr1s8SEX1fFNcxCBy+vvXN24k=
github.com/google/go-sev-guest v0.11.1/go.mod h1:qBOfb+JmgsUI3aUyzQoGC13Kpp9zwLeWvuyXmA9q77w=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
Expand Down

0 comments on commit 06af4a2

Please sign in to comment.