Merge pull request #77 from lukasjarosch/40-partial-templates

Partial Template Support

examples/partial_templates/compiled/example/_partials/no_data.md

@@ -0,0 +1 @@
+

examples/partial_templates/compiled/example/_partials/with_data.md

@@ -0,0 +1 @@
+

examples/partial_templates/compiled/example/main.md

@@ -0,0 +1,31 @@
+# main template
+
+---
+
+
+
+this partial does not use any data
+
+
+
+---
+
+{network:
+    foo: bar
+skipper:
+    use:
+        - network
+ example}
+
+---
+
+
+
+# example
+
+```
+foo: bar
+
+```
+
+

examples/partial_templates/go.mod

@@ -0,0 +1,38 @@
+module github.com/lukasjarosch/skipper/examples/partial_tempaltes
+
+go 1.19
+
+require (
+	github.com/lukasjarosch/skipper v0.0.0-20221027074243-61e46553f948
+	github.com/spf13/afero v1.10.0
+)
+
+replace github.com/lukasjarosch/skipper => ../../
+
+require (
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/keyvault/azkeys v0.10.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 // indirect
+	github.com/Masterminds/goutils v1.1.1 // indirect
+	github.com/Masterminds/semver/v3 v3.2.1 // indirect
+	github.com/Masterminds/sprig/v3 v3.2.3 // indirect
+	github.com/golang-jwt/jwt/v5 v5.0.0 // indirect
+	github.com/google/uuid v1.3.1 // indirect
+	github.com/huandu/xstrings v1.4.0 // indirect
+	github.com/imdario/mergo v0.3.16 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
+	github.com/shopspring/decimal v1.3.1 // indirect
+	github.com/spf13/cast v1.5.1 // indirect
+	golang.org/x/crypto v0.14.0 // indirect
+	golang.org/x/net v0.17.0 // indirect
+	golang.org/x/sys v0.13.0 // indirect
+	golang.org/x/text v0.13.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

examples/partial_templates/inventory/classes/network.yaml

@@ -0,0 +1,3 @@
+---
+network:
+  foo: bar

examples/partial_templates/inventory/targets/example.yaml

@@ -0,0 +1,5 @@
+---
+target:
+  skipper:
+    use:
+      - network

examples/partial_templates/main.go

@@ -0,0 +1,56 @@
+package main
+
+import (
+	"path"
+
+	"github.com/lukasjarosch/skipper"
+	"github.com/spf13/afero"
+)
+
+var (
+	fileSystem    = afero.NewOsFs()
+	inventoryPath = "inventory"
+	classPath     = path.Join(inventoryPath, "classes")
+	targetPath    = path.Join(inventoryPath, "targets")
+	secretPath    = path.Join(inventoryPath, "secrets")
+	templatePath  = path.Join(inventoryPath, "..", "templates")
+	outputPath    = "compiled"
+
+	target = "example"
+)
+
+func main() {
+	inventory, err := skipper.NewInventory(fileSystem, classPath, targetPath, secretPath)
+	if err != nil {
+		panic(err)
+	}
+
+	// Process the inventory, given the target name
+	data, err := inventory.Data(target, nil, false, false)
+	if err != nil {
+		panic(err)
+	}
+
+	templateOutputPath := path.Join(outputPath, target)
+	templater, err := skipper.NewTemplater(fileSystem, templatePath, templateOutputPath, nil, []string{})
+	if err != nil {
+		panic(err)
+	}
+
+	skipperConfig, err := inventory.GetSkipperConfig(target)
+	if err != nil {
+		panic(err)
+	}
+
+	// execute templates  ----------------------------------------------------------------------------------
+	err = templater.ExecuteAll(skipper.DefaultTemplateContext(data, target), false, nil)
+	if err != nil {
+		panic(err)
+	}
+
+	// copy files as specified in the target config (base path is template root)
+	err = skipper.CopyFilesByConfig(fileSystem, skipperConfig.Copies, templatePath, templateOutputPath)
+	if err != nil {
+		panic(err)
+	}
+}

examples/partial_templates/templates/_partials/no_data.md

@@ -0,0 +1,5 @@
+{{ define "no_data" }}
+
+this partial does not use any data
+
+{{ end }}

examples/partial_templates/templates/_partials/with_data.md

@@ -0,0 +1,9 @@
+{{ define "with_data" }}
+
+# {{ .TargetName }}
+
+```
+{{ .Network }}
+```
+
+{{ end }}

examples/partial_templates/templates/main.md

@@ -0,0 +1,13 @@
+# main template
+
+---
+
+{{ template "no_data" }}
+
+---
+
+{{ . }}
+
+---
+
+{{ template "with_data" context "TargetName" .TargetName "Network" .Inventory.network }}

template.go

@@ -5,6 +5,7 @@ import (
 	"bytes"
 	"fmt"
 	"io/fs"
+	"mime"
 	"path/filepath"
 	"regexp"
 	"strconv"
@@ -17,7 +18,6 @@ import (
 )
 
 var customFuncs map[string]any = map[string]any{
-
 	"tfStringArray": func(input []interface{}) string {
 		var s []string
 		for _, v := range input {
@@ -41,10 +41,27 @@ var customFuncs map[string]any = map[string]any{
 		}
 		return time.Now().AddDate(y, m, d).Format(time.RFC3339)
 	},
+
+	"context": func(values ...interface{}) (map[string]interface{}, error) {
+		if len(values)%2 != 0 {
+			return nil, fmt.Errorf("uneven amount of values")
+		}
+		context := make(map[string]interface{}, len(values)/2)
+		for i := 0; i < len(values); i += 2 {
+			key, ok := values[i].(string)
+			if !ok {
+				return nil, fmt.Errorf("map key must be a string")
+			}
+			context[key] = values[i+1]
+		}
+
+		return context, nil
+	},
 }
 
 type Templater struct {
 	Files            []*File
+	partialTemplates []*File
 	IgnoreRegex      []*regexp.Regexp
 	templateRootPath string
 	outputRootPath   string
@@ -91,20 +108,67 @@ func NewTemplater(fileSystem afero.Fs, templateRootPath, outputRootPath string,
 			return nil
 		}
 
+		// Not all files are actually templates, there may very well exist other files (such as images)
+		// which we do not want to register as templates as they would not parse.
+		ignoredMimeTypes := []*regexp.Regexp{
+			regexp.MustCompile(`image/.*`),
+			regexp.MustCompile(`video/.*`),
+		}
+		mimeType := mime.TypeByExtension(filepath.Ext(info.Name()))
+
+		for _, ignoredMimeRegex := range ignoredMimeTypes {
+			if !ignoredMimeRegex.MatchString(mimeType) {
+				continue
+			}
+			return nil // skip this file
+		}
+
+		// Load and register template file
 		file, err := NewFile(filePath)
 		if err != nil {
 			return err
 		}
+		err = file.Load(t.templateFs)
+		if err != nil {
+			return err
+		}
 		t.Files = append(t.Files, file)
 		return nil
 	})
 	if err != nil {
 		return nil, fmt.Errorf("error walking over template path: %w", err)
 	}
 
+	t.DiscoverPartials()
+
 	return t, nil
 }
 
+// DiscoverPartials will iterate over all registered files and check each of them
+// whether an additional template is defined (e.g. using the 'define' directive).
+// If so, the file is added to the list of partial templates which is made available
+// during template execution.
+// This ensures that every template can access partial templates.
+// Subsequent calls to this method will reset the 'partialTemplates' field.
+func (t *Templater) DiscoverPartials() {
+	t.partialTemplates = []*File{}
+
+	for _, tplFile := range t.Files {
+		if t.isPathIgnored(tplFile.Path) {
+			continue
+		}
+
+		// There exists at least one template named 'test'.
+		// If there is more than one, the file contains a partial template definition.
+		test := template.New(tplFile.Path).Funcs(t.templateFuncs)
+		test.Parse(string(tplFile.Bytes))
+		if len(test.Templates()) == 1 {
+			continue
+		}
+		t.partialTemplates = append(t.partialTemplates, tplFile)
+	}
+}
+
 // DefaultTemplateContext returns the default template context with an 'Inventory' field where the Data is located.
 // Additionally it adds the 'TargetName' field for convenience.
 func DefaultTemplateContext(data Data, targetName string) any {
@@ -134,7 +198,6 @@ func (t *Templater) Execute(template *File, data any, allowNoValue bool, renameC
 
 // execute is the main rendering function for templates
 func (t *Templater) execute(tplFile *File, data any, targetPath string, allowNoValue bool) error {
-
 	// if the template matches any IgnoreRegex, just copy the file to the targetPath
 	// without rendering it as template
 	for _, v := range t.IgnoreRegex {
@@ -147,17 +210,28 @@ func (t *Templater) execute(tplFile *File, data any, targetPath string, allowNoV
 		}
 	}
 
-	err := tplFile.Load(t.templateFs)
-	if err != nil {
-		return err
+	// create new target template with the attached functions
+	tpl := template.New(tplFile.Path).Funcs(t.templateFuncs)
+
+	// Add every discovered partial template in case its needed.
+	for _, partialTemplate := range t.partialTemplates {
+		if t.isPathIgnored(partialTemplate.Path) {
+			continue
+		}
+
+		var err error
+		tpl, err = tpl.Parse(string(partialTemplate.Bytes))
+		if err != nil {
+			return fmt.Errorf("failed to parse template %s: %w", partialTemplate.Path, err)
+		}
 	}
 
-	tpl := template.New(tplFile.Path).Funcs(t.templateFuncs)
+	// now, finally parse the template we're actually aiming to render
+	var err error
 	tpl, err = tpl.Parse(string(tplFile.Bytes))
 	if err != nil {
 		return fmt.Errorf("failed to parse template %s: %w", tplFile.Path, err)
 	}
-
 	out := new(bytes.Buffer)
 	err = tpl.Execute(out, data)
 	if err != nil {
@@ -194,7 +268,6 @@ func (t *Templater) ExecuteComponents(data any, components []ComponentConfig, al
 
 	for _, component := range components {
 		for _, input := range component.InputPaths {
-
 			file := t.getTemplateByPath(input)
 
 			if file == nil {
@@ -240,3 +313,14 @@ func (t *Templater) getTemplateByPath(path string) *File {
 	}
 	return nil
 }
+
+// isPathIgnored provides a quick way to check whether a given path should be
+// ignored based on the 'IgnoreRegex' field.
+func (t *Templater) isPathIgnored(filePath string) bool {
+	for _, v := range t.IgnoreRegex {
+		if v.MatchString(filePath) {
+			return true
+		}
+	}
+	return false
+}