stash: start reference implementation

class.go

@@ -81,7 +81,7 @@ func NewClass(filePath string, codec Codec, identifier ClassIdentifier) (*Class,
 
 	className := PathFileBaseName(filePath)
 	if identifier.Last() != className {
-		return nil, fmt.Errorf("class name must be last segment of classIdentifier: %w", ErrInvalidClassIdentifier)
+		return nil, fmt.Errorf("class name '%s' must be last segment of classIdentifier '%s': %w", className, identifier, ErrInvalidClassIdentifier)
 	}
 
 	return &Class{

data/data.go

@@ -23,13 +23,17 @@ func Walk(data interface{}, walkFn WalkFunc) error {
 
 // walk implements a basic DFS and traverses over every
 // node in the data, calling the WalkFunc on each of them.
-func walk(parent interface{}, path Path, walkFn WalkFunc) error {
+func walk(parent interface{}, currentPath Path, walkFn WalkFunc) error {
 	if parent == nil {
 		return nil
 	}
 
-	parentValue := reflect.ValueOf(parent)
+	// Make a copy of path, otherwise the reference is reused
+	// which causes weird append behavior.
+	// See: https://stackoverflow.com/a/20277579
+	path := NewPath(currentPath.String())
 
+	parentValue := reflect.ValueOf(parent)
 	switch parentValue.Kind() {
 	case reflect.Map:
 		// ignore the root (empty) path and only recurse into 'actual paths'

reference.go

@@ -0,0 +1,115 @@
+package skipper
+
+import (
+	"errors"
+	"fmt"
+	"regexp"
+	"strings"
+
+	"github.com/lukasjarosch/skipper/data"
+)
+
+// TODO: handle reference-to-reference
+// TODO: PathReferences / KeyReferences
+// TODO: handle cyclic references
+
+var (
+	// ReferenceRegex defines the strings which are valid references
+	// See: https://regex101.com/r/lIuuep/1
+	ReferenceRegex = regexp.MustCompile(`\${(?P<reference>[\w-]+(?:\:[\w-]+)*)}`)
+
+	ErrUndefinedReference = fmt.Errorf("undefined reference")
+)
+
+// Reference is a reference to a value with a different path.
+type Reference struct {
+	// Path is the path where the reference is defined
+	Path data.Path
+	// TargetPath is the path the reference points to
+	TargetPath data.Path
+}
+
+func (ref Reference) Name() string {
+	return fmt.Sprintf("${%s}", strings.ReplaceAll(ref.TargetPath.String(), ".", ":"))
+}
+
+type ResolvedReference struct {
+	Reference
+	// TargetValue is the value to which the TargetPath points to
+	// This can be [data.NilValue]. In that case there is no target
+	// value but a target reference.
+	TargetValue data.Value
+	// TargetReference is non-nil if the Reference points to another [Reference]
+	TargetReference *Reference
+}
+
+// ReferenceParser is responsible for discovering and resolving references.
+type ReferenceParser struct {
+	source ReferenceSourceWalker
+}
+
+type ReferenceSourceWalker interface {
+	WalkValues(func(path data.Path, value data.Value) error) error
+}
+
+type ReferenceSourceGetter interface {
+	GetPath(path data.Path) (data.Value, error)
+}
+
+var ErrReferenceSourceIsNil = fmt.Errorf("source is nil")
+
+func ParseReferences(source ReferenceSourceWalker) ([]Reference, error) {
+	if source == nil {
+		return nil, ErrReferenceSourceIsNil
+	}
+
+	var references []Reference
+	source.WalkValues(func(path data.Path, value data.Value) error {
+		referenceMatches := ReferenceRegex.FindAllStringSubmatch(value.String(), -1)
+
+		if referenceMatches != nil {
+			for _, match := range referenceMatches {
+				references = append(references, Reference{
+					Path:       path,
+					TargetPath: ReferencePathToPath(match[1]),
+				})
+			}
+		}
+
+		return nil
+	})
+
+	return references, nil
+}
+
+func ResolveReferences(references []Reference, resolveSource ReferenceSourceGetter) ([]ResolvedReference, error) {
+	if resolveSource == nil {
+		return nil, ErrReferenceSourceIsNil
+	}
+
+	var errs error
+	var resolvedReferences []ResolvedReference
+	for _, reference := range references {
+		val, err := resolveSource.GetPath(reference.TargetPath)
+		if err != nil {
+			errs = errors.Join(errs, fmt.Errorf("%w %s at %s: %w", ErrUndefinedReference, reference.Name(), reference.Path, err))
+			continue
+		}
+
+		resolvedReferences = append(resolvedReferences, ResolvedReference{
+			Reference:   reference,
+			TargetValue: val,
+		})
+	}
+	if errs != nil {
+		return nil, errs
+	}
+
+	return resolvedReferences, nil
+}
+
+// ReferencePathToPath converts the path used within references (colon-separated) to a proper [data.Path]
+func ReferencePathToPath(referencePath string) data.Path {
+	referencePath = strings.ReplaceAll(referencePath, ":", ".")
+	return data.NewPath(referencePath)
+}

reference_test.go

@@ -0,0 +1,195 @@
+package skipper_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	. "github.com/lukasjarosch/skipper"
+	"github.com/lukasjarosch/skipper/codec"
+	"github.com/lukasjarosch/skipper/data"
+)
+
+// func makeInventory(t *testing.T) *Inventory {
+// 	dataFiles, err := DiscoverFiles("testdata/references/data", codec.YamlPathSelector)
+// 	assert.NoError(t, err)
+// 	dataRegistry, err := NewRegistryFromFiles(dataFiles, func(filePaths []string) ([]*Class, error) {
+// 		return ClassLoader("testdata/references/data", filePaths, codec.NewYamlCodec())
+// 	})
+// 	assert.NoError(t, err)
+//
+// 	targetFiles, err := DiscoverFiles("testdata/references/targets", codec.YamlPathSelector)
+// 	assert.NoError(t, err)
+// 	targetsRegistry, err := NewRegistryFromFiles(targetFiles, func(filePaths []string) ([]*Class, error) {
+// 		return ClassLoader("testdata/references/targets", filePaths, codec.NewYamlCodec())
+// 	})
+// 	assert.NoError(t, err)
+//
+// 	inventory, err := NewInventory()
+// 	assert.NoError(t, err)
+//
+// 	err = inventory.RegisterScope(DataScope, dataRegistry)
+// 	assert.NoError(t, err)
+// 	err = inventory.RegisterScope(TargetsScope, targetsRegistry)
+// 	assert.NoError(t, err)
+//
+// 	err = inventory.SetDefaultScope(DataScope)
+// 	assert.NoError(t, err)
+//
+// 	return inventory
+// }
+
+var (
+	localReferences = []Reference{
+		{
+			Path:       data.NewPath("simple.departments.engineering.manager"),
+			TargetPath: data.NewPath("employees.0.name"),
+		},
+		{
+			Path:       data.NewPath("simple.departments.analytics.manager"),
+			TargetPath: data.NewPath("simple.employees.1.name"),
+		},
+		{
+			Path:       data.NewPath("simple.departments.marketing.manager"),
+			TargetPath: data.NewPath("simple.employees.2.name"),
+		},
+		{
+			Path:       data.NewPath("simple.projects.Project_X.department"),
+			TargetPath: data.NewPath("simple.departments.engineering.name"),
+		},
+	}
+	localResolvedReferences = []ResolvedReference{
+		{
+			Reference: Reference{
+				Path:       data.NewPath("simple.departments.engineering.manager"),
+				TargetPath: data.NewPath("employees.0.name"),
+			},
+			TargetValue:     data.NewValue("John Doe"),
+			TargetReference: nil,
+		},
+		{
+			Reference: Reference{
+				Path:       data.NewPath("simple.departments.analytics.manager"),
+				TargetPath: data.NewPath("simple.employees.1.name"),
+			},
+			TargetValue:     data.NewValue("Jane Smith"),
+			TargetReference: nil,
+		},
+		{
+			Reference: Reference{
+				Path:       data.NewPath("simple.departments.marketing.manager"),
+				TargetPath: data.NewPath("simple.employees.2.name"),
+			},
+			TargetValue:     data.NewValue("Michael Johnson"),
+			TargetReference: nil,
+		},
+		{
+			Reference: Reference{
+				Path:       data.NewPath("simple.projects.Project_X.department"),
+				TargetPath: data.NewPath("simple.departments.engineering.name"),
+			},
+			TargetValue:     data.NewValue("Engineering"),
+			TargetReference: nil,
+		},
+	}
+)
+
+func TestParseReferences(t *testing.T) {
+	_, err := ParseReferences(nil)
+	assert.ErrorIs(t, err, ErrReferenceSourceIsNil)
+
+	class, err := NewClass("testdata/references/local/simple.yaml", codec.NewYamlCodec(), data.NewPathFromOsPath("simple"))
+	assert.NoError(t, err)
+
+	references, err := ParseReferences(class)
+	assert.NoError(t, err)
+	for _, reference := range references {
+		assert.Contains(t, localReferences, reference)
+	}
+}
+
+func TestResolveReferencesSimple(t *testing.T) {
+	class, err := NewClass("testdata/references/local/simple.yaml", codec.NewYamlCodec(), data.NewPathFromOsPath("simple"))
+	assert.NoError(t, err)
+
+	// Test: resolve all valid references which have a direct TargetValue
+	resolved, err := ResolveReferences(localReferences, class)
+	assert.NoError(t, err)
+	assert.Len(t, resolved, len(localResolvedReferences), "Every Reference should emit a ResolveReference")
+	for _, resolved := range resolved {
+		assert.Contains(t, localResolvedReferences, resolved, "ResolvedReference should be returned")
+		assert.Nil(t, resolved.TargetReference)
+	}
+
+	// Test: references with invalid TargetPath
+	invalidReferences := []Reference{
+		{
+			Path:       data.NewPath("simple.departments.marketing.name"),
+			TargetPath: data.NewPath("invalid.path"),
+		},
+		{
+			Path:       data.NewPath("simple.departments.marketing.name"),
+			TargetPath: data.NewPath("another.invalid.path"),
+		},
+	}
+	resolved, err = ResolveReferences(invalidReferences, class)
+	assert.ErrorIs(t, err, ErrUndefinedReference)
+	assert.Nil(t, resolved)
+
+	// TODO: reference to reference
+	// TODO: cycle
+}
+
+func TestResolveReferencesMap(t *testing.T) {
+	class, err := NewClass("testdata/references/local/nested.yaml", codec.NewYamlCodec(), data.NewPathFromOsPath("nested"))
+	assert.NoError(t, err)
+
+	references, err := ParseReferences(class)
+	assert.NoError(t, err)
+	assert.NotNil(t, references)
+
+	resolved, err := ResolveReferences(references, class)
+	assert.NoError(t, err)
+	assert.Len(t, resolved, len(references))
+
+	expected := []ResolvedReference{
+		{
+			Reference: Reference{
+				Path:       data.NewPath("nested.target"),
+				TargetPath: data.NewPath("source"),
+			},
+			TargetReference: nil,
+			TargetValue: data.NewValue(map[string]interface{}{
+				"foo": "bar",
+				"bar": "baz",
+			}),
+		},
+		{
+			Reference: Reference{
+				Path:       data.NewPath("nested.target_array"),
+				TargetPath: data.NewPath("source_array"),
+			},
+			TargetReference: nil,
+			TargetValue:     data.NewValue([]interface{}{"foo", "bar", "baz"}),
+		},
+		{
+			Reference: Reference{
+				Path:       data.NewPath("nested.target_nested_map"),
+				TargetPath: data.NewPath("nested_map"),
+			},
+			TargetReference: nil,
+			TargetValue: data.NewValue(map[string]interface{}{
+				"foo": map[string]interface{}{
+					"bar": map[string]interface{}{
+						"baz": "qux",
+					},
+				},
+			}),
+		},
+	}
+	assert.Len(t, resolved, len(expected))
+	for _, res := range resolved {
+		assert.Contains(t, expected, res)
+		assert.Nil(t, res.TargetReference)
+	}
+}

registry_test.go

@@ -170,3 +170,6 @@ func TestRegistryPostSetHook(t *testing.T) {
 	assert.NotNil(t, val.Raw)
 	assert.Equal(t, val.Raw, "very_juicy")
 }
+
+func TestRegistryWalk(t *testing.T) {
+}

testdata/references/local/nested.yaml

@@ -0,0 +1,15 @@
+nested:
+  source:
+    foo: bar
+    bar: baz
+  source_array:
+    - foo
+    - bar
+    - baz
+  nested_map:
+    foo:
+      bar:
+        baz: qux
+  target: ${source}
+  target_array: ${source_array}
+  target_nested_map: ${nested_map}