Skip to content

Commit

Permalink
e2e: add test for release candidates
Browse files Browse the repository at this point in the history
  • Loading branch information
burgerdev committed Apr 24, 2024
1 parent 3652fb9 commit bb2c1d0
Show file tree
Hide file tree
Showing 5 changed files with 331 additions and 2 deletions.
59 changes: 59 additions & 0 deletions e2e/internal/kubeclient/deploy.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ import (
"context"
"encoding/json"
"fmt"
"net"
"sort"
"strconv"
"time"

appsv1 "k8s.io/api/apps/v1"
Expand Down Expand Up @@ -99,6 +101,63 @@ func (c *Kubeclient) WaitForDeployment(ctx context.Context, namespace, name stri
}
}

// WaitForLoadBalancer waits until the given service is configured with an external IP and returns it.
func (c *Kubeclient) WaitForLoadBalancer(ctx context.Context, namespace, name string) (string, error) {
watcher, err := c.client.CoreV1().Services(namespace).Watch(ctx, metav1.ListOptions{FieldSelector: "metadata.name=" + name})
if err != nil {
return "", err
}
var ip string
var port int
loop:
for {
select {
case evt := <-watcher.ResultChan():
switch evt.Type {
case watch.Added:
fallthrough
case watch.Modified:
svc, ok := evt.Object.(*corev1.Service)
if !ok {
return "", fmt.Errorf("watcher received unexpected type %T", evt.Object)
}
for _, ingress := range svc.Status.LoadBalancer.Ingress {
if ingress.IP != "" {
ip = ingress.IP
// TODO(burgerdev): deal with more than one port, and protocols other than TCP
port = int(svc.Spec.Ports[0].Port)
break loop
}
}
case watch.Deleted:
return "", fmt.Errorf("service %s/%s was deleted while waiting for it", namespace, name)
default:
c.log.Warn("ignoring unexpected watch event", "type", evt.Type, "object", evt.Object)
}
case <-ctx.Done():
return "", fmt.Errorf("LoadBalancer %s/%s did not get a public IP before %w", namespace, name, ctx.Err())
}
}

ticker := time.NewTicker(time.Second)
defer ticker.Stop()

dialer := &net.Dialer{}
for {
select {
case <-ticker.C:
conn, err := dialer.DialContext(ctx, "tcp", net.JoinHostPort(ip, strconv.Itoa(port)))
if err == nil {
conn.Close()
return ip, nil
}
c.log.Info("probe failed", "namespace", namespace, "name", name, "error", err)
case <-ctx.Done():
return "", fmt.Errorf("LoadBalancer %s/%s never responded to probing before %w", namespace, name, ctx.Err())
}
}
}

func (c *Kubeclient) toJSON(a any) string {
s, err := json.Marshal(a)
if err != nil {
Expand Down
263 changes: 263 additions & 0 deletions e2e/release/release_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,263 @@
//go:build e2e
// +build e2e

package release

import (
"context"
"crypto/rand"
"crypto/tls"
"crypto/x509"
"encoding/hex"
"flag"
"io"
"net"
"net/http"
"os"
"os/exec"
"path"
"path/filepath"
"testing"
"time"

"github.com/edgelesssys/contrast/e2e/internal/kubeclient"
"github.com/edgelesssys/contrast/e2e/internal/kuberesource"
"github.com/edgelesssys/contrast/internal/kubeapi"
"github.com/google/go-github/v61/github"
"github.com/stretchr/testify/require"
)

const (
tokenEnvVar = "GH_TOKEN"
)

var (
owner = flag.String("owner", "edgelesssys", "Github repository owner")
repo = flag.String("repo", "contrast", "Github repository")
tag = flag.String("tag", "", "tag name of the release to download")
namespace = flag.String("namespace", "", "k8s namespace to install resources to (will be deleted unless --keep is set)")
keep = flag.Bool("keep", false, "don't delete test resources and deployment")
)

// TestRelease downloads a release from Github, sets up the coordinator, installs the demo
// deployment and runs some simple smoke tests.
func TestRelease(t *testing.T) {
ctx := context.Background()
k := kubeclient.NewForTest(t)

if *namespace == "" {
*namespace = randomNamespace(t)
t.Logf("Created test namespace %s", *namespace)
}

require.True(t, t.Run("create-namespace", func(t *testing.T) {
require := require.New(t)
ctx, cancel := context.WithTimeout(ctx, 1*time.Minute)
defer cancel()

res, err := kuberesource.ResourcesToUnstructured([]any{kuberesource.Namespace(*namespace)})
require.NoError(err)
require.NoError(k.Apply(ctx, res...))
}), "the namespace is required for subsequent tests to run")

t.Cleanup(func() {
if *keep {
return
}
ctx, cancel := context.WithTimeout(ctx, 1*time.Minute)
defer cancel()

res, err := kuberesource.ResourcesToUnstructured([]any{kuberesource.Namespace(*namespace)})
if err != nil {
return
}
k.Delete(ctx, res...)
})

dir := fetchRelease(ctx, t)

contrast := &contrast{dir}

for _, sub := range []string{"help"} {
contrast.Run(t, ctx, 2*time.Second, sub)
}

var coordinatorIP string
require.True(t, t.Run("apply-coordinator", func(t *testing.T) {
require := require.New(t)
ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
defer cancel()

yaml, err := os.ReadFile(path.Join(dir, "coordinator.yml"))
require.NoError(err)
resources, err := kubeapi.UnmarshalUnstructuredK8SResource(yaml)
require.NoError(err)

for _, r := range resources {
r.SetNamespace(*namespace)
}

require.NoError(k.Apply(ctx, resources...))
require.NoError(k.WaitForDeployment(ctx, *namespace, "coordinator"))
coordinatorIP, err = k.WaitForLoadBalancer(ctx, *namespace, "coordinator")
require.NoError(err)
}), "the coordinator is required for subsequent tests to run")

require.True(t, t.Run("unpack-deployment", func(t *testing.T) {
require := require.New(t)
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()

cmd := exec.CommandContext(ctx, "unzip", "emojivoto-demo.zip")
cmd.Dir = dir
out, err := cmd.CombinedOutput()
require.NoError(err, "output:\n%s", string(out))

infos, err := os.ReadDir(path.Join(dir, "deployments"))
require.NoError(err)
for _, info := range infos {
name := path.Join(path.Join(dir, "deployments"), info.Name())
yaml, err := os.ReadFile(name)
require.NoError(err)
resources, err := kubeapi.UnmarshalUnstructuredK8SResource(yaml)
require.NoError(err)

for _, r := range resources {
require.NoError(k.PatchNamespace(*namespace, r))
}
newYAML, err := kuberesource.EncodeUnstructured(resources)
require.NoError(err)
require.NoError(os.WriteFile(name, newYAML, 0o644))

}
}), "unpacking needs to succeed for subsequent tests to run")

contrast.Run(t, ctx, 2*time.Minute, "generate", "deployments/")
contrast.Run(t, ctx, 1*time.Minute, "set", "-c", coordinatorIP+":1313", "deployments/")
contrast.Run(t, ctx, 1*time.Minute, "verify", "-c", coordinatorIP+":1313")

require.True(t, t.Run("apply-demo", func(t *testing.T) {
require := require.New(t)
ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
defer cancel()

files, err := filepath.Glob(path.Join(dir, "deployments", "*.yml"))
require.NoError(err)
for _, file := range files {
yaml, err := os.ReadFile(file)
require.NoError(err)
resources, err := kubeapi.UnmarshalUnstructuredK8SResource(yaml)
require.NoError(err)
require.NoError(k.Apply(ctx, resources...))
}

require.NoError(k.WaitForDeployment(ctx, *namespace, "vote-bot"))
require.NoError(k.WaitForDeployment(ctx, *namespace, "voting"))
require.NoError(k.WaitForDeployment(ctx, *namespace, "emoji"))
require.NoError(k.WaitForDeployment(ctx, *namespace, "web"))
}), "applying the demo is required for subsequent tests to run")

t.Run("test-demo", func(t *testing.T) {
require := require.New(t)
ctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
defer cancel()

emojiwebIP, err := k.WaitForLoadBalancer(ctx, *namespace, "web-svc")
require.NoError(err)

cfg := &tls.Config{RootCAs: x509.NewCertPool()}
pem, err := os.ReadFile(path.Join(dir, "verify", "mesh-root.pem"))
require.NoError(err)
require.True(cfg.RootCAs.AppendCertsFromPEM(pem))

c := http.Client{
Transport: &http.Transport{
DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) {
return (&net.Dialer{}).DialContext(ctx, "tcp", net.JoinHostPort(emojiwebIP, "443"))
},
TLSClientConfig: cfg,
},
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://web", nil)
require.NoError(err)
resp, err := c.Do(req)
require.NoError(err)
require.Equal(http.StatusOK, resp.StatusCode)
})
}

type contrast struct {
dir string
}

func (c *contrast) Run(t *testing.T, ctx context.Context, timeout time.Duration, args ...string) {
require.True(t, t.Run(args[0], func(t *testing.T) {
ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()

args = append([]string{"--log-level", "debug"}, args...)
cmd := exec.CommandContext(ctx, "./contrast", args...)
cmd.Dir = c.dir
out, err := cmd.CombinedOutput()
require.NoError(t, err, "output:\n%s", string(out))
}), args[0]+" needs to succeed for subsequent tests to run")
}

func randomNamespace(t *testing.T) string {
buf := make([]byte, 4)
n, err := rand.Read(buf)
require.NoError(t, err)
require.Equal(t, 4, n)
return "releasetest-" + hex.EncodeToString(buf)
}

// fetchRelease downloads the release corresponding to the global tag variable and returns the directory.
func fetchRelease(ctx context.Context, t *testing.T) string {
require := require.New(t)
ctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
defer cancel()

token := os.Getenv(tokenEnvVar)
require.NotEmpty(token, "environment variable %q must contain a Github access token", tokenEnvVar)
gh := github.NewClient(nil).WithAuthToken(token)

var dir string
if *keep {
var err error
dir, err = os.MkdirTemp("", "releasetest-")
require.NoError(err)
t.Logf("Created test directory %s", dir)
} else {
dir = t.TempDir()
}

// Find our target release. There is GetReleaseByTag, but we may be looking for a draft release.
rels, resp, err := gh.Repositories.ListReleases(ctx, *owner, *repo, nil)
require.NoError(err)
var release *github.RepositoryRelease
for _, rel := range rels {
t.Logf("Checking release %q", *rel.TagName)
if *rel.TagName == *tag {
release = rel
break
}
}
require.NotNil(release, "release %q not found among %d releases\nGithub response:\n%#v", *tag, len(rels), resp)

for _, asset := range release.Assets {
f, err := os.OpenFile(path.Join(dir, *asset.Name), os.O_CREATE|os.O_RDWR, 0o777)
require.NoError(err)
body, _, err := gh.Repositories.DownloadReleaseAsset(ctx, *owner, *repo, *asset.ID, http.DefaultClient)
require.NoError(err, "could not fetch release asset %q (id: %d)", asset.Name, asset.ID)
_, err = io.Copy(f, body)
require.NoError(err)
f.Close()
}

return dir
}

func TestMain(m *testing.M) {
flag.Parse()
os.Exit(m.Run())
}
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ module github.com/edgelesssys/contrast
go 1.21

require (
github.com/google/go-github/v61 v61.0.0
github.com/google/go-sev-guest v0.11.1
github.com/spf13/afero v1.11.0
github.com/spf13/cobra v1.8.0
Expand Down Expand Up @@ -32,6 +33,7 @@ require (
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/gnostic-models v0.6.8 // indirect
github.com/google/go-configfs-tsm v0.2.2 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/logger v1.1.1 // indirect
github.com/google/uuid v1.6.0 // indirect
Expand Down
5 changes: 5 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,17 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I=
github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-configfs-tsm v0.2.2 h1:YnJ9rXIOj5BYD7/0DNnzs8AOp7UcvjfTvt215EWcs98=
github.com/google/go-configfs-tsm v0.2.2/go.mod h1:EL1GTDFMb5PZQWDviGfZV9n87WeGTR/JUg13RfwkgRo=
github.com/google/go-github/v61 v61.0.0 h1:VwQCBwhyE9JclCI+22/7mLB1PuU9eowCXKY5pNlu1go=
github.com/google/go-github/v61 v61.0.0/go.mod h1:0WR+KmsWX75G2EbpyGsGmradjo3IiciuI4BmdVCobQY=
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
github.com/google/go-sev-guest v0.11.1 h1:gnww4U8fHV5DCPz4gykr1s8SEX1fFNcxCBy+vvXN24k=
github.com/google/go-sev-guest v0.11.1/go.mod h1:qBOfb+JmgsUI3aUyzQoGC13Kpp9zwLeWvuyXmA9q77w=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
Expand Down
4 changes: 2 additions & 2 deletions packages/by-name/contrast/package.nix
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ let

ldflags = [ "-s" ];

subPackages = [ "e2e/openssl" "e2e/servicemesh" ];
subPackages = [ "e2e/openssl" "e2e/servicemesh" "e2e/release" ];
};

runtimeHandler = lib.removeSuffix "\n" (builtins.readFile "${runtime-class-files}/runtime-handler");
Expand Down Expand Up @@ -51,7 +51,7 @@ buildGoModule rec {
};

proxyVendor = true;
vendorHash = "sha256-m67gNUGvb4z7OyHvJdOX7SZKgBWn11OEA28oJiQjpXI=";
vendorHash = "sha256-tM+z5RoZ2ClB88OvenYMu3DVUXWnjFEF7xK9p6/06jc=";

subPackages = packageOutputs ++ [ "e2e/internal/kuberesource/resourcegen" ];

Expand Down

0 comments on commit bb2c1d0

Please sign in to comment.