Skip to content

Commit

Permalink
Merge pull request gruntwork-io#1378 from jguionnet/feat/add-remote-r…
Browse files Browse the repository at this point in the history
…epo-support-in-helm

Feat/add snapshot based testing for helm
  • Loading branch information
denis256 authored Jan 8, 2024
2 parents 807138f + b4b4d46 commit 28c0f43
Show file tree
Hide file tree
Showing 8 changed files with 534 additions and 15 deletions.
20 changes: 18 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ require (
github.com/oracle/oci-go-sdk v7.1.0+incompatible
github.com/pquerna/otp v1.2.0
github.com/sirupsen/logrus v1.8.1
github.com/stretchr/testify v1.8.2
github.com/stretchr/testify v1.8.4
github.com/tmccombs/hcl2json v0.3.3
github.com/urfave/cli v1.22.2
github.com/zclconf/go-cty v1.9.1
Expand All @@ -48,6 +48,8 @@ require (

require (
cloud.google.com/go/cloudbuild v1.9.0
github.com/gonvenience/ytbx v1.4.4
github.com/homeport/dyff v1.6.0
github.com/slack-go/slack v0.10.3
gotest.tools/v3 v3.0.3
)
Expand All @@ -63,6 +65,7 @@ require (
github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
github.com/Azure/go-autorest/logger v0.2.1 // indirect
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
github.com/BurntSushi/toml v1.3.2 // indirect
github.com/agext/levenshtein v1.2.3 // indirect
github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
Expand All @@ -83,6 +86,11 @@ require (
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/gonvenience/bunt v1.3.5 // indirect
github.com/gonvenience/neat v1.3.12 // indirect
github.com/gonvenience/term v1.0.2 // indirect
github.com/gonvenience/text v1.0.7 // indirect
github.com/gonvenience/wrap v1.1.2 // indirect
github.com/google/gnostic-models v0.6.8 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/gofuzz v1.2.0 // indirect
Expand All @@ -97,9 +105,14 @@ require (
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.15.11 // indirect
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-ciede2000 v0.0.0-20170301095244-782e8c62fec3 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/mitchellh/go-ps v1.0.0 // indirect
github.com/mitchellh/go-testing-interface v1.14.1 // indirect
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
github.com/mitchellh/hashstructure v1.1.0 // indirect
github.com/moby/spdystream v0.2.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
Expand All @@ -110,10 +123,13 @@ require (
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/satori/go.uuid v1.2.0 // indirect
github.com/sergi/go-diff v1.3.1 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/texttheater/golang-levenshtein v1.0.1 // indirect
github.com/ulikunitz/xz v0.5.10 // indirect
github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 // indirect
go.opencensus.io v0.24.0 // indirect
golang.org/x/sync v0.1.0 // indirect
golang.org/x/sync v0.4.0 // indirect
golang.org/x/sys v0.13.0 // indirect
golang.org/x/term v0.13.0 // indirect
golang.org/x/text v0.13.0 // indirect
Expand Down
66 changes: 54 additions & 12 deletions go.sum

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions modules/helm/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,5 @@ type Options struct {
Logger *logger.Logger // Set a non-default logger that should be used. See the logger package for more info. Use logger.Discard to not print the output while executing the command.
ExtraArgs map[string][]string // Extra arguments to pass to the helm install/upgrade/rollback/delete and helm repo add commands. The key signals the command (e.g., install) while the values are the extra arguments to pass through.
BuildDependencies bool // If true, helm dependencies will be built before rendering template, installing or upgrade the chart.
SnapshotPath string // The path to the snapshot directory when using snapshot based testing. Empty string means use default ($PWD/__snapshot__).
}
125 changes: 125 additions & 0 deletions modules/helm/template.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,11 @@ import (

"github.com/gruntwork-io/terratest/modules/files"
"github.com/gruntwork-io/terratest/modules/testing"

"os"

"github.com/gonvenience/ytbx"
"github.com/homeport/dyff/pkg/dyff"
)

// RenderTemplate runs `helm template` to render the template given the provided options and returns stdout/stderr from
Expand Down Expand Up @@ -134,3 +139,123 @@ func UnmarshalK8SYamlE(t testing.TestingT, yamlData string, destinationObj inter
}
return nil
}

// UpdateSnapshot creates or updates the k8s manifest snapshot of a chart (e.g bitnami/nginx).
// It is one of the two functions needed to implement snapshot based testing for helm.
// see https://github.com/gruntwork-io/terratest/issues/1377
// A snapshot is used to compare the current manifests of a chart with the previous manifests.
// A global diff is run against the two snapshosts and the number of differences is returned.
func UpdateSnapshot(t testing.TestingT, options *Options, yamlData string, releaseName string) {
require.NoError(t, UpdateSnapshotE(t, options, yamlData, releaseName))
}

// UpdateSnapshotE creates or updates the k8s manifest snapshot of a chart (e.g bitnami/nginx).
// It is one of the two functions needed to implement snapshot based testing for helm.
// see https://github.com/gruntwork-io/terratest/issues/1377
// A snapshot is used to compare the current manifests of a chart with the previous manifests.
// A global diff is run against the two snapshosts and the number of differences is returned.
// It will failed the test if there is an error while writing the manifests' snapshot in the file system
func UpdateSnapshotE(t testing.TestingT, options *Options, yamlData string, releaseName string) error {

var snapshotDir = "__snapshot__"
if options.SnapshotPath != "" {
snapshotDir = options.SnapshotPath
}
// Create a directory if not exists
if !files.FileExists(snapshotDir) {
if err := os.Mkdir(snapshotDir, 0755); err != nil {
return errors.WithStackTrace(err)
}
}

filename := filepath.Join(snapshotDir, releaseName+".yaml")
// Open a file in write mode
file, err := os.Create(filename)
if err != nil {
return errors.WithStackTrace(err)
}
defer file.Close()

// Write the k8s manifest into the file
if _, err = file.WriteString(yamlData); err != nil {
return errors.WithStackTrace(err)
}

if options.Logger != nil {
options.Logger.Logf(t, "helm chart manifest written into file: %s", filename)
}
return nil
}

// DiffAgainstSnapshot compare the current manifests of a chart (e.g bitnami/nginx)
// with the previous manifests stored in the snapshot.
// see https://github.com/gruntwork-io/terratest/issues/1377
// It returns the number of difference between the two manifests or -1 in case of error
// It will fail the test if there is an error while reading or writing the two manifests in the file system
func DiffAgainstSnapshot(t testing.TestingT, options *Options, yamlData string, releaseName string) int {
numberOfDiffs, err := DiffAgainstSnapshotE(t, options, yamlData, releaseName)
require.NoError(t, err)
return numberOfDiffs
}

// DiffAgainstSnapshotE compare the current manifests of a chart (e.g bitnami/nginx)
// with the previous manifests stored in the snapshot.
// see https://github.com/gruntwork-io/terratest/issues/1377
// It returns the number of difference between the manifests or -1 in case of error
func DiffAgainstSnapshotE(t testing.TestingT, options *Options, yamlData string, releaseName string) (int, error) {

var snapshotDir = "__snapshot__"
if options.SnapshotPath != "" {
snapshotDir = options.SnapshotPath
}

// load the yaml snapshot file
snapshot := filepath.Join(snapshotDir, releaseName+".yaml")
from, err := ytbx.LoadFile(snapshot)
if err != nil {
return -1, errors.WithStackTrace(err)
}

// write the current manifest into a file as `dyff` does not support string input
currentManifests := releaseName + ".yaml"
file, err := os.Create(currentManifests)
if err != nil {
return -1, errors.WithStackTrace(err)
}

if _, err = file.WriteString(yamlData); err != nil {
return -1, errors.WithStackTrace(err)
}
defer file.Close()
defer os.Remove(currentManifests)

to, err := ytbx.LoadFile(currentManifests)
if err != nil {
return -1, errors.WithStackTrace(err)
}

// compare the two manifests using `dyff`
compOpt := dyff.KubernetesEntityDetection(false)

// create a report
report, err := dyff.CompareInputFiles(from, to, compOpt)
if err != nil {
return -1, errors.WithStackTrace(err)
}

// write any difference to stdout
reportWriter := &dyff.HumanReport{
Report: report,
DoNotInspectCerts: false,
NoTableStyle: false,
OmitHeader: false,
UseGoPatchPaths: false,
}

err = reportWriter.WriteReport(os.Stdout)
if err != nil {
return -1, errors.WithStackTrace(err)
}
// return the number of diffs to use in assertion while testing: 0 = no differences
return len(reportWriter.Diffs), nil
}
82 changes: 81 additions & 1 deletion modules/helm/template_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
appsv1 "k8s.io/api/apps/v1"

"github.com/gruntwork-io/terratest/modules/k8s"
"github.com/gruntwork-io/terratest/modules/logger"
"github.com/gruntwork-io/terratest/modules/random"
)

Expand All @@ -25,7 +26,7 @@ func TestRemoteChartRender(t *testing.T) {
const (
remoteChartSource = "https://charts.bitnami.com/bitnami"
remoteChartName = "nginx"
remoteChartVersion = "13.2.23"
remoteChartVersion = "13.2.24"
)

t.Parallel()
Expand All @@ -45,6 +46,7 @@ func TestRemoteChartRender(t *testing.T) {
"image.tag": remoteChartVersion,
},
KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName),
Logger: logger.Discard,
}

// Run RenderTemplate to render the template and capture the output. Note that we use the version without `E`, since
Expand All @@ -65,3 +67,81 @@ func TestRemoteChartRender(t *testing.T) {
require.Equal(t, len(deploymentContainers), 1)
require.Equal(t, deploymentContainers[0].Image, expectedContainerImage)
}

// Test that we can dump all the manifest locally a remote chart (e.g bitnami/nginx)
// so that I can use them later to compare between two versions of the same chart for example
func TestRemoteChartRenderDump(t *testing.T) {
const (
remoteChartSource = "https://charts.bitnami.com/bitnami"
remoteChartName = "nginx"
remoteChartVersion = "13.2.20"
// need to set a fix name for the namespace so it is not flag as a difference
namespaceName = "dump-ns"
)

releaseName := remoteChartName

options := &Options{
SetValues: map[string]string{
"image.repository": remoteChartName,
"image.registry": "",
"image.tag": remoteChartVersion,
},
KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName),
Logger: logger.Discard,
}

// Run RenderTemplate to render the template and capture the output. Note that we use the version without `E`, since
// we want to assert that the template renders without any errors.
output := RenderRemoteTemplate(t, options, remoteChartSource, releaseName, []string{})

// Now we use kubernetes/client-go library to render the template output into the Deployment struct. This will
// ensure the Deployment resource is rendered correctly.
var deployment appsv1.Deployment
UnmarshalK8SYaml(t, output, &deployment)

// Verify the namespace matches the expected supplied namespace.
require.Equal(t, namespaceName, deployment.Namespace)

// write chart manifest to a local filesystem directory
options = &Options{
Logger: logger.Default,
SnapshotPath: "__chart_manifests_snapshot__",
}
UpdateSnapshot(t, options, output, releaseName)
}

// Test that we can diff all the manifest to a local snapshot using a remote chart (e.g bitnami/nginx)
func TestRemoteChartRenderDiff(t *testing.T) {
const (
remoteChartSource = "https://charts.bitnami.com/bitnami"
remoteChartName = "nginx"
remoteChartVersion = "13.2.24"
// need to set a fix name for the namespace so it is not flag as a difference
namespaceName = "dump-ns"
)

releaseName := remoteChartName
options := &Options{
SetValues: map[string]string{
"image.repository": remoteChartName,
"image.registry": "",
"image.tag": remoteChartVersion,
},
KubectlOptions: k8s.NewKubectlOptions("", "", namespaceName),
Logger: logger.Discard,
SnapshotPath: "__chart_manifests_snapshot__",
}

// Run RenderTemplate to render the template and capture the output. Note that we use the version without `E`, since
// we want to assert that the template renders without any errors.
output := RenderRemoteTemplate(t, options, remoteChartSource, releaseName, []string{})

// Now we use kubernetes/client-go library to render the template output into the Deployment struct. This will
// ensure the Deployment resource is rendered correctly.
var deployment appsv1.Deployment
UnmarshalK8SYaml(t, output, &deployment)

// run the diff and assert there is only one difference: the image name
require.Equal(t, 1, DiffAgainstSnapshot(t, options, output, releaseName))
}
30 changes: 30 additions & 0 deletions test/fixtures/helm/keda-values.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
metricsServer:
replicaCount: 3
operator:
name: keda-operator
replicaCount: 3
podAnnotations:
keda:
sidecar.istio.io/inject: "false"
metricsAdapter:
sidecar.istio.io/inject: "false"
podDisruptionBudget:
metricServer:
minAvailable: 1
operator:
minAvailable: 1
resources:
metricServer:
limits:
cpu: 100m
memory: 1234Mi
requests:
cpu: 50m
memory: 128Mi
operator:
limits:
cpu: 100m
memory: 1111Mi
requests:
cpu: 50m
memory: 888Mi
Loading

0 comments on commit 28c0f43

Please sign in to comment.