From f10346230ee7296f32ceb333fc0c928fa73d0a59 Mon Sep 17 00:00:00 2001 From: Ahmed Elsabbahy Date: Mon, 6 Mar 2017 09:35:58 -0500 Subject: [PATCH] Template Support (#200) Add text/template support to goss to allow using conditions, var files, and environment variables (see README and manual changes). * text/template support * Update tests to use text/template * Update documentation --- README.md | 22 ++++- add.go | 4 +- cmd/goss/goss.go | 13 ++- development/build_images.sh | 1 + development/push_images.sh | 1 - docs/manual.md | 111 ++++++++++++++++++++- integration-tests/Dockerfile_arch.md5 | 2 +- integration-tests/goss/alpine3/goss.yaml | 8 -- integration-tests/goss/centos7/goss.yaml | 8 -- integration-tests/goss/goss-service.yaml | 7 ++ integration-tests/goss/goss-shared.yaml | 10 +- integration-tests/goss/precise/goss.yaml | 8 -- integration-tests/goss/vars.yaml | 15 +++ integration-tests/goss/wheezy/goss.yaml | 8 -- integration-tests/test.sh | 8 +- store.go | 119 ++++++++++++++++++----- validate.go | 9 +- 17 files changed, 283 insertions(+), 71 deletions(-) create mode 100644 integration-tests/goss/vars.yaml diff --git a/README.md b/README.md index 1f7a64863..6d7273c1a 100644 --- a/README.md +++ b/README.md @@ -104,6 +104,11 @@ Total Duration: 0.021s # <- yeah, it's that fast.. Count: 15, Failed: 0 ``` +* Edit it to use [templates](https://github.com/aelsabbahy/goss/blob/master/docs/manual.md#templates), and run with a vars file +``` +goss --vars vars.yaml validate +``` + * keep running it until the system enters a valid state or we timeout ``` goss validate --retry-timeout 30s --sleep 1s @@ -118,10 +123,11 @@ goss serve --format json & curl localhost:8080/healthz ``` -### Patterns, matchers and metadata -Goss files can be manually edited to match: +### Manually editing Goss files +Goss files can be manually edited to use: * [Patterns](https://github.com/aelsabbahy/goss/blob/master/docs/manual.md#patterns) -* [Advanced Matchers](https://github.com/aelsabbahy/goss/blob/master/docs/manual.md#advanced-matchers). +* [Advanced Matchers](https://github.com/aelsabbahy/goss/blob/master/docs/manual.md#advanced-matchers) +* [Templates](https://github.com/aelsabbahy/goss/blob/master/docs/manual.md#templates) * `title` and `meta` (arbitrary data) attributes are persisted when adding other resources with `goss add` Some examples: @@ -153,6 +159,16 @@ package: - have-len: 3 - not: contain-element: 4.4.0 + + # Loaded from --vars YAML/JSON file + {{.Vars.package}}: + installed: true + +{{if eq .Env.OS "centos"}} + # This test is only when $OS environment variable is set to "centos" + libselinux: + installed: true +{{end}} ``` ## Supported resources diff --git a/add.go b/add.go index 5ad9b5605..5a114554b 100644 --- a/add.go +++ b/add.go @@ -14,7 +14,7 @@ import ( // Simple wrapper to add multiple resources func AddResources(fileName, resourceName string, keys []string, c *cli.Context) error { - setStoreFormatFromFileName(fileName) + OutStoreFormat = getStoreFormatFromFileName(fileName) config := util.Config{ IgnoreList: c.GlobalStringSlice("exclude-attr"), Timeout: int(c.Duration("timeout") / time.Millisecond), @@ -159,7 +159,7 @@ func AddResource(fileName string, gossConfig GossConfig, resourceName, key strin // Simple wrapper to add multiple resources func AutoAddResources(fileName string, keys []string, c *cli.Context) error { - setStoreFormatFromFileName(fileName) + OutStoreFormat = getStoreFormatFromFileName(fileName) config := util.Config{ IgnoreList: c.GlobalStringSlice("exclude-attr"), Timeout: int(c.Duration("timeout") / time.Millisecond), diff --git a/cmd/goss/goss.go b/cmd/goss/goss.go index e0aa09faa..a11f6792e 100644 --- a/cmd/goss/goss.go +++ b/cmd/goss/goss.go @@ -27,6 +27,11 @@ func main() { Usage: "Goss file to read from / write to", EnvVar: "GOSS_FILE", }, + cli.StringFlag{ + Name: "vars", + Usage: "json/yaml file containing variables for template", + EnvVar: "GOSS_VARS", + }, cli.StringFlag{ Name: "package", Usage: "Package type to use [rpm, deb, apk, pacman]", @@ -118,8 +123,14 @@ func main() { Name: "render", Aliases: []string{"r"}, Usage: "render gossfile after imports", + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "debug, d", + Usage: fmt.Sprintf("Print debugging info when rendering"), + }, + }, Action: func(c *cli.Context) error { - fmt.Print(goss.RenderJSON(c.GlobalString("gossfile"))) + fmt.Print(goss.RenderJSON(c)) return nil }, }, diff --git a/development/build_images.sh b/development/build_images.sh index 819608411..fd5bb0299 100755 --- a/development/build_images.sh +++ b/development/build_images.sh @@ -7,6 +7,7 @@ INTEGRATION_TEST_DIR="$SCRIPT_DIR/../integration-tests/" for docker_file in $INTEGRATION_TEST_DIR/Dockerfile_*; do + [[ $docker_file == *.md5 ]] && continue os=$(cut -d '_' -f2 <<<"$docker_file") docker build -t "aelsabbahy/goss_${os}:latest" - < "$docker_file" done diff --git a/development/push_images.sh b/development/push_images.sh index b33335d37..fcbc45574 100755 --- a/development/push_images.sh +++ b/development/push_images.sh @@ -16,4 +16,3 @@ popd for image in $images; do docker push "${image}:latest" done - diff --git a/docs/manual.md b/docs/manual.md index 992786d7d..d18da7d5e 100644 --- a/docs/manual.md +++ b/docs/manual.md @@ -14,6 +14,7 @@ * [validate, v \- Validate the system](#validate-v---validate-the-system) * [Important note about goss file format](#important-note-about-goss-file-format) * [Available tests](#available-tests) + * [addr](#addr) * [command](#command) * [dns](#dns) * [file](#file) @@ -30,7 +31,7 @@ * [user](#user) * [Patterns](#patterns) * [Advanced Matchers](#advanced-matchers) - +* [Templates](#templates) ## Usage @@ -54,6 +55,7 @@ COMMANDS: GLOBAL OPTIONS: --gossfile, -g "./goss.yaml" Goss file to read from / write to [$GOSS_FILE] + --vars value json/yaml file containing variables for template [$GOSS_VARS] --package Package type to use [rpm, deb, apk, pacman] --help, -h show help --generate-bash-completion @@ -71,6 +73,13 @@ Valid formats: * **YAML** (default) * **JSON** +### --vars +The file to read variables from when rendering gossfile [templates](#templates). + +Valid formats: +* **YAML** (default) +* **JSON** + ### --package The package type to check for. @@ -188,6 +197,10 @@ process: ### render, r - Render gossfile after importing all referenced gossfiles This command allows you to keep your tests separated and render a single, valid, gossfile, by including them with the `gossfile` directive. +#### Flags +##### --debug +This prints the rendered golang template prior to printing the parsed JSON/YAML gossfile. + #### Example: ```bash @@ -676,7 +689,7 @@ For the attributes that use patterns (ex. `file`, `command` `output`), each patt **NOTE:** Pattern attributes do not support [Advanced Matchers](#advanced-matchers) -**NOTE:** Regex support is based on golangs regex engine documented [here](https://golang.org/pkg/regexp/syntax/) +**NOTE:** Regex support is based on golang's regex engine documented [here](https://golang.org/pkg/regexp/syntax/) **NOTE:** You will **need** the double backslash (`\\`) escape for Regex special entities, for example `\\s` for blank spaces. @@ -738,3 +751,97 @@ package: For more information see: * [gomega_test.go](https://github.com/aelsabbahy/goss/blob/master/resource/gomega_test.go) - For a complete set of supported json -> Gomega mapping * [gomega](https://onsi.github.io/gomega/) - Gomega matchers reference + +## Templates + +Goss test files can leverage golang's [text/template](https://golang.org/pkg/text/template/) to allow for dynamic or conditional tests. + +Available variables: +* `{{.Env}}` - Containing environment variables +* `{{.Vars}}` - Containing the values defined in [--vars](#global-options) file + +Available functions beyond text/template [built-in functions](https://golang.org/pkg/text/template/#hdr-Functions): +* `mkSlice` - Retuns a slice of all the arguments. See examples below for usage. + +**NOTE:** gossfiles containing text/template `{{}}` controls will no longer work with `goss add/autoadd`. One way to get around this is to split your template and static goss files and use [gossfile](#gossfile) to import. + +### Examples + +Using [puppetlabs/facter](https://github.com/puppetlabs/facter) or [chef/ohai](https://github.com/chef/ohai) as external tools to provide vars. +```bash +$ goss --vars <(ohai) validate +$ goss --vars <(facter -j) validate +``` + +Using `mkSlice` to define a loop locally. +```yaml +file: +{{- range mkSlice "/etc/passwd" "/etc/group"}} + {{.}}: + exists: true + mode: "0644" + owner: root + group: root + filetype: file +{{end}} +``` + +Using Env variables and a vars file: + +**vars.yaml:** +```yaml +centos: + packages: + kernel: + - "4.9.11-centos" + - "4.9.11-centos2" +debian: + packages: + kernel: + - "4.9.11-debian" + - "4.9.11-debian2" +users: + - user1 + - user2 +``` + +**goss.yaml:** +```yaml +package: +# Looping over a variables defined in a vars.yaml using $OS environment variable as a lookup key +{{range $name, $vers := index .Vars .Env.OS "packages"}} + {{$name}}: + installed: true + versions: + {{range $vers}} + - {{.}} + {{end}} +{{end}} + +# This test is only when OS=centos variable is defined +{{if eq .Env.OS "centos"}} + libselinux: + installed: true +{{end}} + +# Loop over users +user: +{{range .Vars.users}} + {{.}}: + exists: true + groups: + - {{.}} + home: /home/{{.}} + shell: /bin/bash +{{end}} +``` + +Rendered results: +```bash +# To validate: +$ OS=centos goss --vars vars.yaml validate +# To render: +$ OS=centos goss --vars vars.yaml render +# To render with debugging enabled: +$ OS=centos goss --vars vars.yaml render --debug +``` diff --git a/integration-tests/Dockerfile_arch.md5 b/integration-tests/Dockerfile_arch.md5 index 51f58353e..5e6285f75 100644 --- a/integration-tests/Dockerfile_arch.md5 +++ b/integration-tests/Dockerfile_arch.md5 @@ -1 +1 @@ -b430b4a6a2ea84679f9ae705a65b2c25 Dockerfile_arch +f73a8fa9e9c0940ce2bfc09c13c3aaf5 Dockerfile_arch diff --git a/integration-tests/goss/alpine3/goss.yaml b/integration-tests/goss/alpine3/goss.yaml index fa01b3ce1..377f610a8 100644 --- a/integration-tests/goss/alpine3/goss.yaml +++ b/integration-tests/goss/alpine3/goss.yaml @@ -1,13 +1,5 @@ --- -package: - apache2: - installed: true - versions: - - 2.4.23-r1 service: - apache2: - enabled: true - running: true autofs: enabled: false running: false diff --git a/integration-tests/goss/centos7/goss.yaml b/integration-tests/goss/centos7/goss.yaml index a7731c494..300a6fa4a 100644 --- a/integration-tests/goss/centos7/goss.yaml +++ b/integration-tests/goss/centos7/goss.yaml @@ -1,13 +1,5 @@ --- -package: - httpd: - installed: true - versions: - - 2.4.6 service: - httpd: - enabled: true - running: true autofs: enabled: false running: false diff --git a/integration-tests/goss/goss-service.yaml b/integration-tests/goss/goss-service.yaml index bfb0c4f0f..13217b78e 100644 --- a/integration-tests/goss/goss-service.yaml +++ b/integration-tests/goss/goss-service.yaml @@ -3,3 +3,10 @@ service: foobar: enabled: false running: false +{{if eq .Env.OS "centos7"}} + httpd: +{{else}} + apache2: +{{end}} + enabled: true + running: true diff --git a/integration-tests/goss/goss-shared.yaml b/integration-tests/goss/goss-shared.yaml index b8663232e..d1ef5585b 100644 --- a/integration-tests/goss/goss-shared.yaml +++ b/integration-tests/goss/goss-shared.yaml @@ -11,7 +11,8 @@ command: stderr: - not found file: - "/etc/passwd": +{{range mkSlice "/etc/passwd" "/etc/group"}} + {{.}}: exists: true mode: '0644' owner: root @@ -19,6 +20,7 @@ file: filetype: file contains: - root +{{end}} "/goss/hellogoss.txt": exists: true md5: 7c9bb14b3bf178e82c00c2a4398c93cd @@ -34,6 +36,12 @@ file: package: foobar: installed: false +{{- range $name, $ver := index .Vars .Env.OS "packages"}} + {{$name}}: + installed: true + versions: + - {{$ver}} +{{end}} addr: tcp://google.com:22: reachable: false diff --git a/integration-tests/goss/precise/goss.yaml b/integration-tests/goss/precise/goss.yaml index 87536c9e1..a4566b776 100644 --- a/integration-tests/goss/precise/goss.yaml +++ b/integration-tests/goss/precise/goss.yaml @@ -1,13 +1,5 @@ --- -package: - apache2: - installed: true - versions: - - 2.2.22-1ubuntu1.11 service: - apache2: - enabled: true - running: true autofs: enabled: true running: true diff --git a/integration-tests/goss/vars.yaml b/integration-tests/goss/vars.yaml new file mode 100644 index 000000000..f9b9f613d --- /dev/null +++ b/integration-tests/goss/vars.yaml @@ -0,0 +1,15 @@ +--- +alpine3: + packages: + apache2: "2.4.23-r1" +arch: + packages: +centos7: + packages: + httpd: "2.4.6" +precise: + packages: + apache2: "2.2.22-1ubuntu1.11" +wheezy: + packages: + apache2: "2.2.22-13+deb7u7" diff --git a/integration-tests/goss/wheezy/goss.yaml b/integration-tests/goss/wheezy/goss.yaml index 4b8435f88..8ceb0e3d2 100644 --- a/integration-tests/goss/wheezy/goss.yaml +++ b/integration-tests/goss/wheezy/goss.yaml @@ -1,13 +1,5 @@ --- -package: - apache2: - installed: true - versions: - - 2.2.22-13+deb7u7 service: - apache2: - enabled: true - running: true autofs: enabled: false running: false diff --git a/integration-tests/test.sh b/integration-tests/test.sh index 73f9831ed..501420a9b 100755 --- a/integration-tests/test.sh +++ b/integration-tests/test.sh @@ -32,7 +32,7 @@ docker_exec() { if docker ps -a | grep "$container_name";then docker rm -vf "$container_name" fi -opts=(--cap-add SYS_ADMIN -v "$PWD/goss:/goss" -d --name "$container_name" $(seccomp_opts)) +opts=(--env OS=$os --cap-add SYS_ADMIN -v "$PWD/goss:/goss" -d --name "$container_name" $(seccomp_opts)) id=$(docker run "${opts[@]}" "aelsabbahy/goss_$os" /sbin/init) ip=$(docker inspect --format '{{ .NetworkSettings.IPAddress }}' "$id") trap "rv=\$?; docker rm -vf $id; exit \$rv" INT TERM EXIT @@ -40,13 +40,13 @@ trap "rv=\$?; docker rm -vf $id; exit \$rv" INT TERM EXIT [[ $os != "arch" ]] && docker_exec "/goss/$os/goss-linux-$arch" -g "/goss/goss-wait.yaml" validate -r 10s -s 100ms && sleep 1 #out=$(docker exec "$container_name" bash -c "time /goss/$os/goss-linux-$arch -g /goss/$os/goss.yaml validate") -out=$(docker_exec "/goss/$os/goss-linux-$arch" -g "/goss/$os/goss.yaml" validate) +out=$(docker_exec "/goss/$os/goss-linux-$arch" --vars "/goss/vars.yaml" -g "/goss/$os/goss.yaml" validate) echo "$out" if [[ $os == "arch" ]]; then - egrep -q 'Count: 56, Failed: 0' <<<"$out" + egrep -q 'Count: 62, Failed: 0' <<<"$out" else - egrep -q 'Count: 70, Failed: 0' <<<"$out" + egrep -q 'Count: 76, Failed: 0' <<<"$out" fi if [[ ! $os == "arch" ]]; then diff --git a/store.go b/store.go index 4da915657..bb007e9fd 100644 --- a/store.go +++ b/store.go @@ -1,6 +1,7 @@ package goss import ( + "bytes" "encoding/json" "fmt" "io/ioutil" @@ -10,66 +11,135 @@ import ( "reflect" "sort" "strings" + "text/template" "gopkg.in/yaml.v2" "github.com/aelsabbahy/goss/resource" + "github.com/urfave/cli" ) const ( - JSON = iota + UNSET = iota + JSON YAML - UNSET ) -var StoreFormat = UNSET +var OutStoreFormat = UNSET +var TemplateFilter func(data []byte) []byte +var debug = false -func setStoreFormatFromFileName(f string) { +func getStoreFormatFromFileName(f string) int { ext := filepath.Ext(f) switch ext { case ".json": - StoreFormat = JSON + return JSON case ".yaml", ".yml": - StoreFormat = YAML + return YAML default: log.Fatalf("Unknown file extension: %v", ext) } + return 0 } -func setStoreFormatFromData(data []byte) { +func getStoreFormatFromData(data []byte) int { var v interface{} if err := unmarshalJSON(data, &v); err == nil { - StoreFormat = JSON - return + return JSON } if err := unmarshalYAML(data, &v); err == nil { - StoreFormat = YAML - return + return YAML } log.Fatalf("Unable to determine format from content") + return 0 } // Reads json file returning GossConfig func ReadJSON(filePath string) GossConfig { - // FIXME: Any problems with this? - setStoreFormatFromFileName(filePath) file, err := ioutil.ReadFile(filePath) if err != nil { fmt.Printf("File error: %v\n", err) os.Exit(1) } - return ReadJSONData(file) + return ReadJSONData(file, false) +} + +type TmplVars struct { + Vars map[string]interface{} +} + +func (t *TmplVars) Env() map[string]string { + env := make(map[string]string) + for _, i := range os.Environ() { + sep := strings.Index(i, "=") + env[i[0:sep]] = i[sep+1:] + } + return env +} + +func varsFromFile(varsFile string) (map[string]interface{}, error) { + var vars map[string]interface{} + if varsFile == "" { + return vars, nil + } + data, err := ioutil.ReadFile(varsFile) + if err != nil { + return vars, err + } + format := getStoreFormatFromData(data) + if err := unmarshal(data, &vars, format); err != nil { + return vars, err + } + return vars, nil +} + +func mkSlice(args ...interface{}) []interface{} { + return args +} + +func NewTemplateFilter(varsFile string) func([]byte) []byte { + vars, err := varsFromFile(varsFile) + if err != nil { + fmt.Printf("Error: loading vars file '%s'\n%v\n", varsFile, err) + os.Exit(1) + } + tVars := &TmplVars{Vars: vars} + + f := func(data []byte) []byte { + funcMap := map[string]interface{}{"mkSlice": mkSlice} + t := template.New("test").Funcs(template.FuncMap(funcMap)) + tmpl, err := t.Parse(string(data)) + if err != nil { + log.Fatal(err) + } + tmpl.Option("missingkey=error") + var doc bytes.Buffer + err = tmpl.Execute(&doc, tVars) + if err != nil { + log.Fatal(err) + } + return doc.Bytes() + } + return f } // Reads json byte array returning GossConfig -func ReadJSONData(data []byte) GossConfig { - if StoreFormat == UNSET { - setStoreFormatFromData(data) +func ReadJSONData(data []byte, detectFormat bool) GossConfig { + if TemplateFilter != nil { + data = TemplateFilter(data) + if debug { + fmt.Println("DEBUG: file after text/template render") + fmt.Println(string(data)) + } + } + format := OutStoreFormat + if detectFormat == true { + format = getStoreFormatFromData(data) } gossConfig := NewGossConfig() // Horrible, but will do for now - if err := unmarshal(data, gossConfig); err != nil { + if err := unmarshal(data, gossConfig, format); err != nil { // FIXME: really dude.. this is so ugly fmt.Printf("Error: %v\n", err) os.Exit(1) @@ -78,8 +148,13 @@ func ReadJSONData(data []byte) GossConfig { } // Reads json file recursively returning string -func RenderJSON(filePath string) string { +func RenderJSON(c *cli.Context) string { + filePath := c.GlobalString("gossfile") + varsFile := c.GlobalString("vars") + debug = c.Bool("debug") + TemplateFilter = NewTemplateFilter(varsFile) path := filepath.Dir(filePath) + OutStoreFormat = getStoreFormatFromFileName(filePath) gossConfig := mergeJSONData(ReadJSON(filePath), 0, path) b, err := marshal(gossConfig) @@ -158,7 +233,7 @@ func resourcePrint(fileName string, res resource.ResourceRead) { } func marshal(gossConfig interface{}) ([]byte, error) { - switch StoreFormat { + switch OutStoreFormat { case JSON: return marshalJSON(gossConfig) case YAML: @@ -168,8 +243,8 @@ func marshal(gossConfig interface{}) ([]byte, error) { } } -func unmarshal(data []byte, v interface{}) error { - switch StoreFormat { +func unmarshal(data []byte, v interface{}, storeFormat int) error { + switch storeFormat { case JSON: return unmarshalJSON(data, v) case YAML: diff --git a/validate.go b/validate.go index 225427712..b08e35430 100644 --- a/validate.go +++ b/validate.go @@ -21,6 +21,7 @@ func getGossConfig(c *cli.Context) GossConfig { var fh *os.File var path, source string var gossConfig GossConfig + TemplateFilter = NewTemplateFilter(c.GlobalString("vars")) specFile := c.GlobalString("gossfile") if specFile == "-" { source = "STDIN" @@ -30,13 +31,17 @@ func getGossConfig(c *cli.Context) GossConfig { fmt.Printf("Error: %v\n", err) os.Exit(1) } - gossConfig = mergeJSONData(ReadJSONData(data), 0, path) + OutStoreFormat = getStoreFormatFromData(data) + gossConfig = ReadJSONData(data, true) } else { source = specFile path = filepath.Dir(specFile) - gossConfig = mergeJSONData(ReadJSON(specFile), 0, path) + OutStoreFormat = getStoreFormatFromFileName(specFile) + gossConfig = ReadJSON(specFile) } + gossConfig = mergeJSONData(gossConfig, 0, path) + if len(gossConfig.Resources()) == 0 { fmt.Printf("Error: found 0 tests, source: %v\n", source) os.Exit(1)