diff --git a/class.go b/class.go index a4f083a..d168c72 100644 --- a/class.go +++ b/class.go @@ -81,7 +81,7 @@ func NewClass(filePath string, codec Codec, identifier ClassIdentifier) (*Class, className := PathFileBaseName(filePath) if identifier.Last() != className { - return nil, fmt.Errorf("class name must be last segment of classIdentifier: %w", ErrInvalidClassIdentifier) + return nil, fmt.Errorf("class name '%s' must be last segment of classIdentifier '%s': %w", className, identifier, ErrInvalidClassIdentifier) } return &Class{ diff --git a/data/data.go b/data/data.go index dca3e61..2c807be 100644 --- a/data/data.go +++ b/data/data.go @@ -23,13 +23,17 @@ func Walk(data interface{}, walkFn WalkFunc) error { // walk implements a basic DFS and traverses over every // node in the data, calling the WalkFunc on each of them. -func walk(parent interface{}, path Path, walkFn WalkFunc) error { +func walk(parent interface{}, currentPath Path, walkFn WalkFunc) error { if parent == nil { return nil } - parentValue := reflect.ValueOf(parent) + // Make a copy of path, otherwise the reference is reused + // which causes weird append behavior. + // See: https://stackoverflow.com/a/20277579 + path := NewPath(currentPath.String()) + parentValue := reflect.ValueOf(parent) switch parentValue.Kind() { case reflect.Map: // ignore the root (empty) path and only recurse into 'actual paths' diff --git a/reference.go b/reference.go new file mode 100644 index 0000000..c75b823 --- /dev/null +++ b/reference.go @@ -0,0 +1,115 @@ +package skipper + +import ( + "errors" + "fmt" + "regexp" + "strings" + + "github.com/lukasjarosch/skipper/data" +) + +// TODO: handle reference-to-reference +// TODO: PathReferences / KeyReferences +// TODO: handle cyclic references + +var ( + // ReferenceRegex defines the strings which are valid references + // See: https://regex101.com/r/lIuuep/1 + ReferenceRegex = regexp.MustCompile(`\${(?P[\w-]+(?:\:[\w-]+)*)}`) + + ErrUndefinedReference = fmt.Errorf("undefined reference") +) + +// Reference is a reference to a value with a different path. +type Reference struct { + // Path is the path where the reference is defined + Path data.Path + // TargetPath is the path the reference points to + TargetPath data.Path +} + +func (ref Reference) Name() string { + return fmt.Sprintf("${%s}", strings.ReplaceAll(ref.TargetPath.String(), ".", ":")) +} + +type ResolvedReference struct { + Reference + // TargetValue is the value to which the TargetPath points to + // This can be [data.NilValue]. In that case there is no target + // value but a target reference. + TargetValue data.Value + // TargetReference is non-nil if the Reference points to another [Reference] + TargetReference *Reference +} + +// ReferenceParser is responsible for discovering and resolving references. +type ReferenceParser struct { + source ReferenceSourceWalker +} + +type ReferenceSourceWalker interface { + WalkValues(func(path data.Path, value data.Value) error) error +} + +type ReferenceSourceGetter interface { + GetPath(path data.Path) (data.Value, error) +} + +var ErrReferenceSourceIsNil = fmt.Errorf("source is nil") + +func ParseReferences(source ReferenceSourceWalker) ([]Reference, error) { + if source == nil { + return nil, ErrReferenceSourceIsNil + } + + var references []Reference + source.WalkValues(func(path data.Path, value data.Value) error { + referenceMatches := ReferenceRegex.FindAllStringSubmatch(value.String(), -1) + + if referenceMatches != nil { + for _, match := range referenceMatches { + references = append(references, Reference{ + Path: path, + TargetPath: ReferencePathToPath(match[1]), + }) + } + } + + return nil + }) + + return references, nil +} + +func ResolveReferences(references []Reference, resolveSource ReferenceSourceGetter) ([]ResolvedReference, error) { + if resolveSource == nil { + return nil, ErrReferenceSourceIsNil + } + + var errs error + var resolvedReferences []ResolvedReference + for _, reference := range references { + val, err := resolveSource.GetPath(reference.TargetPath) + if err != nil { + errs = errors.Join(errs, fmt.Errorf("%w %s at %s: %w", ErrUndefinedReference, reference.Name(), reference.Path, err)) + continue + } + + resolvedReferences = append(resolvedReferences, ResolvedReference{ + Reference: reference, + TargetValue: val, + }) + } + if errs != nil { + return nil, errs + } + + return resolvedReferences, nil +} + +// ReferencePathToPath converts the path used within references (colon-separated) to a proper [data.Path] +func ReferencePathToPath(referencePath string) data.Path { + referencePath = strings.ReplaceAll(referencePath, ":", ".") + return data.NewPath(referencePath) +} diff --git a/reference_test.go b/reference_test.go new file mode 100644 index 0000000..5270252 --- /dev/null +++ b/reference_test.go @@ -0,0 +1,195 @@ +package skipper_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + . "github.com/lukasjarosch/skipper" + "github.com/lukasjarosch/skipper/codec" + "github.com/lukasjarosch/skipper/data" +) + +// func makeInventory(t *testing.T) *Inventory { +// dataFiles, err := DiscoverFiles("testdata/references/data", codec.YamlPathSelector) +// assert.NoError(t, err) +// dataRegistry, err := NewRegistryFromFiles(dataFiles, func(filePaths []string) ([]*Class, error) { +// return ClassLoader("testdata/references/data", filePaths, codec.NewYamlCodec()) +// }) +// assert.NoError(t, err) +// +// targetFiles, err := DiscoverFiles("testdata/references/targets", codec.YamlPathSelector) +// assert.NoError(t, err) +// targetsRegistry, err := NewRegistryFromFiles(targetFiles, func(filePaths []string) ([]*Class, error) { +// return ClassLoader("testdata/references/targets", filePaths, codec.NewYamlCodec()) +// }) +// assert.NoError(t, err) +// +// inventory, err := NewInventory() +// assert.NoError(t, err) +// +// err = inventory.RegisterScope(DataScope, dataRegistry) +// assert.NoError(t, err) +// err = inventory.RegisterScope(TargetsScope, targetsRegistry) +// assert.NoError(t, err) +// +// err = inventory.SetDefaultScope(DataScope) +// assert.NoError(t, err) +// +// return inventory +// } + +var ( + localReferences = []Reference{ + { + Path: data.NewPath("simple.departments.engineering.manager"), + TargetPath: data.NewPath("employees.0.name"), + }, + { + Path: data.NewPath("simple.departments.analytics.manager"), + TargetPath: data.NewPath("simple.employees.1.name"), + }, + { + Path: data.NewPath("simple.departments.marketing.manager"), + TargetPath: data.NewPath("simple.employees.2.name"), + }, + { + Path: data.NewPath("simple.projects.Project_X.department"), + TargetPath: data.NewPath("simple.departments.engineering.name"), + }, + } + localResolvedReferences = []ResolvedReference{ + { + Reference: Reference{ + Path: data.NewPath("simple.departments.engineering.manager"), + TargetPath: data.NewPath("employees.0.name"), + }, + TargetValue: data.NewValue("John Doe"), + TargetReference: nil, + }, + { + Reference: Reference{ + Path: data.NewPath("simple.departments.analytics.manager"), + TargetPath: data.NewPath("simple.employees.1.name"), + }, + TargetValue: data.NewValue("Jane Smith"), + TargetReference: nil, + }, + { + Reference: Reference{ + Path: data.NewPath("simple.departments.marketing.manager"), + TargetPath: data.NewPath("simple.employees.2.name"), + }, + TargetValue: data.NewValue("Michael Johnson"), + TargetReference: nil, + }, + { + Reference: Reference{ + Path: data.NewPath("simple.projects.Project_X.department"), + TargetPath: data.NewPath("simple.departments.engineering.name"), + }, + TargetValue: data.NewValue("Engineering"), + TargetReference: nil, + }, + } +) + +func TestParseReferences(t *testing.T) { + _, err := ParseReferences(nil) + assert.ErrorIs(t, err, ErrReferenceSourceIsNil) + + class, err := NewClass("testdata/references/local/simple.yaml", codec.NewYamlCodec(), data.NewPathFromOsPath("simple")) + assert.NoError(t, err) + + references, err := ParseReferences(class) + assert.NoError(t, err) + for _, reference := range references { + assert.Contains(t, localReferences, reference) + } +} + +func TestResolveReferencesSimple(t *testing.T) { + class, err := NewClass("testdata/references/local/simple.yaml", codec.NewYamlCodec(), data.NewPathFromOsPath("simple")) + assert.NoError(t, err) + + // Test: resolve all valid references which have a direct TargetValue + resolved, err := ResolveReferences(localReferences, class) + assert.NoError(t, err) + assert.Len(t, resolved, len(localResolvedReferences), "Every Reference should emit a ResolveReference") + for _, resolved := range resolved { + assert.Contains(t, localResolvedReferences, resolved, "ResolvedReference should be returned") + assert.Nil(t, resolved.TargetReference) + } + + // Test: references with invalid TargetPath + invalidReferences := []Reference{ + { + Path: data.NewPath("simple.departments.marketing.name"), + TargetPath: data.NewPath("invalid.path"), + }, + { + Path: data.NewPath("simple.departments.marketing.name"), + TargetPath: data.NewPath("another.invalid.path"), + }, + } + resolved, err = ResolveReferences(invalidReferences, class) + assert.ErrorIs(t, err, ErrUndefinedReference) + assert.Nil(t, resolved) + + // TODO: reference to reference + // TODO: cycle +} + +func TestResolveReferencesMap(t *testing.T) { + class, err := NewClass("testdata/references/local/nested.yaml", codec.NewYamlCodec(), data.NewPathFromOsPath("nested")) + assert.NoError(t, err) + + references, err := ParseReferences(class) + assert.NoError(t, err) + assert.NotNil(t, references) + + resolved, err := ResolveReferences(references, class) + assert.NoError(t, err) + assert.Len(t, resolved, len(references)) + + expected := []ResolvedReference{ + { + Reference: Reference{ + Path: data.NewPath("nested.target"), + TargetPath: data.NewPath("source"), + }, + TargetReference: nil, + TargetValue: data.NewValue(map[string]interface{}{ + "foo": "bar", + "bar": "baz", + }), + }, + { + Reference: Reference{ + Path: data.NewPath("nested.target_array"), + TargetPath: data.NewPath("source_array"), + }, + TargetReference: nil, + TargetValue: data.NewValue([]interface{}{"foo", "bar", "baz"}), + }, + { + Reference: Reference{ + Path: data.NewPath("nested.target_nested_map"), + TargetPath: data.NewPath("nested_map"), + }, + TargetReference: nil, + TargetValue: data.NewValue(map[string]interface{}{ + "foo": map[string]interface{}{ + "bar": map[string]interface{}{ + "baz": "qux", + }, + }, + }), + }, + } + assert.Len(t, resolved, len(expected)) + for _, res := range resolved { + assert.Contains(t, expected, res) + assert.Nil(t, res.TargetReference) + } +} diff --git a/registry_test.go b/registry_test.go index ae0572b..e19a054 100644 --- a/registry_test.go +++ b/registry_test.go @@ -170,3 +170,6 @@ func TestRegistryPostSetHook(t *testing.T) { assert.NotNil(t, val.Raw) assert.Equal(t, val.Raw, "very_juicy") } + +func TestRegistryWalk(t *testing.T) { +} diff --git a/testdata/references/local/nested.yaml b/testdata/references/local/nested.yaml new file mode 100644 index 0000000..c189e7b --- /dev/null +++ b/testdata/references/local/nested.yaml @@ -0,0 +1,15 @@ +nested: + source: + foo: bar + bar: baz + source_array: + - foo + - bar + - baz + nested_map: + foo: + bar: + baz: qux + target: ${source} + target_array: ${source_array} + target_nested_map: ${nested_map} diff --git a/testdata/references/local/simple.yaml b/testdata/references/local/simple.yaml new file mode 100644 index 0000000..81066f6 --- /dev/null +++ b/testdata/references/local/simple.yaml @@ -0,0 +1,51 @@ +simple: + employees: + - name: John Doe + position: Software Engineer + department: Engineering + salary: 80000 + hire_date: 2022-01-15 + - name: Jane Smith + position: Data Analyst + department: Analytics + salary: 70000 + hire_date: 2021-09-20 + - name: Michael Johnson + position: Marketing Manager + department: Marketing + salary: 90000 + hire_date: 2020-11-10 + + departments: + engineering: + name: Engineering + location: San Francisco + manager: ${employees:0:name} + analytics: + name: Analytics + location: New York + manager: ${simple:employees:1:name} + marketing: + name: Marketing + location: Los Angeles + manager: ${simple:employees:2:name} + + projects: + Project_X: + department: ${simple:departments:engineering:name} + start_date: 2022-02-01 + end_date: 2022-08-01 + budget: 100000 + status: In Progress + Data_Analysis_Project: + department: Analytics + start_date: 2022-03-15 + end_date: 2022-10-15 + budget: 80000 + status: Completed + Marketing_Campaign: + department: Marketing + start_date: 2022-05-01 + end_date: 2022-12-01 + budget: 120000 + status: Planning