diff --git a/internal/util/invalid_serialization_detector.go b/internal/util/invalid_serialization_detector.go index 773c53c9..09e670ec 100644 --- a/internal/util/invalid_serialization_detector.go +++ b/internal/util/invalid_serialization_detector.go @@ -19,6 +19,7 @@ func NewInvalidSerializationDetectorSchema() *InvalidSerializationDetectorSchema // the data has been serialized or unserialized at least once (since, if it is // never operated on, then it's trivial to claim that it was never doubly done). type InvalidSerializationDetectorSchema struct { + schema.ScalarType SerializeCnt int UnserializeCnt int } @@ -89,15 +90,6 @@ func (d *InvalidSerializationDetectorSchema) SerializeType(data string) (any, er return d.Serialize(data) } -// ApplyScope is for applying a scope to the references. Does not apply to this object. -func (d *InvalidSerializationDetectorSchema) ApplyScope(_ schema.Scope, _ string) {} - -// ValidateReferences validates that all necessary references from scopes have been applied. -// Does not apply to this object. -func (d *InvalidSerializationDetectorSchema) ValidateReferences() error { - return nil -} - // TypeID returns the category of type this type is. Returns string because // the valid states of this type include all strings. func (d *InvalidSerializationDetectorSchema) TypeID() schema.TypeID { diff --git a/workflow/any.go b/workflow/any.go index ae66b1b0..50eaab3e 100644 --- a/workflow/any.go +++ b/workflow/any.go @@ -17,11 +17,7 @@ func newAnySchemaWithExpressions() *anySchemaWithExpressions { // anySchemaWithExpressions is a wildcard allowing maps, lists, integers, strings, bools, and floats. It also allows // expression objects. type anySchemaWithExpressions struct { - anySchema schema.AnySchema -} - -func (a *anySchemaWithExpressions) ReflectedType() reflect.Type { - return a.anySchema.ReflectedType() + schema.AnySchema } func (a *anySchemaWithExpressions) Unserialize(data any) (any, error) { @@ -39,59 +35,19 @@ func (a *anySchemaWithExpressions) ValidateCompatibility(dataOrType any) error { // Assume okay return nil } - return a.anySchema.ValidateCompatibility(dataOrType) + return a.AnySchema.ValidateCompatibility(dataOrType) } func (a *anySchemaWithExpressions) Serialize(data any) (any, error) { return a.checkAndConvert(data) } -func (a *anySchemaWithExpressions) ApplyScope(scope schema.Scope, namespace string) { - a.anySchema.ApplyScope(scope, namespace) -} - -func (a *anySchemaWithExpressions) ValidateReferences() error { - return a.anySchema.ValidateReferences() -} - -func (a *anySchemaWithExpressions) TypeID() schema.TypeID { - return a.anySchema.TypeID() -} - func (a *anySchemaWithExpressions) checkAndConvert(data any) (any, error) { if _, ok := data.(expressions.Expression); ok { return data, nil } t := reflect.ValueOf(data) switch t.Kind() { - case reflect.Int: - fallthrough - case reflect.Uint: - fallthrough - case reflect.Int8: - fallthrough - case reflect.Uint8: - fallthrough - case reflect.Int16: - fallthrough - case reflect.Uint16: - fallthrough - case reflect.Int32: - fallthrough - case reflect.Uint32: - fallthrough - case reflect.Uint64: - fallthrough - case reflect.Int64: - fallthrough - case reflect.Float32: - fallthrough - case reflect.Float64: - fallthrough - case reflect.String: - fallthrough - case reflect.Bool: - return a.anySchema.Unserialize(data) case reflect.Slice: result := make([]any, t.Len()) for i := 0; i < t.Len(); i++ { @@ -118,8 +74,6 @@ func (a *anySchemaWithExpressions) checkAndConvert(data any) (any, error) { } return result, nil default: - return nil, &schema.ConstraintError{ - Message: fmt.Sprintf("unsupported data type for 'any' type: %T", data), - } + return a.AnySchema.Unserialize(data) } } diff --git a/workflow/executor.go b/workflow/executor.go index 8f47fcc9..9c1c8816 100644 --- a/workflow/executor.go +++ b/workflow/executor.go @@ -227,7 +227,7 @@ func (e *executor) processInput(workflow *Workflow) (schema.Scope, error) { if !ok { return nil, fmt.Errorf("bug: unserialized input is not a scope") } - typedInput.ApplyScope(typedInput, schema.DEFAULT_NAMESPACE) + typedInput.ApplySelf() return typedInput, nil } @@ -317,7 +317,7 @@ func applyLifecycleScopes( stepLifecycles map[string]step.Lifecycle[step.LifecycleStageWithSchema], typedInput schema.Scope, ) error { - allNamespaces := make(map[string]schema.Scope) + allNamespaces := make(map[string]map[string]*schema.ObjectSchema) for workflowStepID, stepLifecycle := range stepLifecycles { for _, stage := range stepLifecycle.Stages { prefix := "$.steps." + workflowStepID + "." + stage.ID + "." @@ -335,9 +335,9 @@ func applyLifecycleScopes( // applyAllNamespaces applies all namespaces to the given scope. // This function also validates references, and lists the namespaces // and their objects in the event of a reference failure. -func applyAllNamespaces(allNamespaces map[string]schema.Scope, scopeToApplyTo schema.Scope) error { - for namespace, scope := range allNamespaces { - scopeToApplyTo.ApplyScope(scope, namespace) +func applyAllNamespaces(allNamespaces map[string]map[string]*schema.ObjectSchema, scopeToApplyTo schema.Scope) error { + for namespace, objects := range allNamespaces { + scopeToApplyTo.ApplyNamespace(objects, namespace) } err := scopeToApplyTo.ValidateReferences() if err == nil { @@ -345,9 +345,9 @@ func applyAllNamespaces(allNamespaces map[string]schema.Scope, scopeToApplyTo sc } // Now on the error path. Provide useful debug info. availableObjects := "" - for namespace, scope := range allNamespaces { + for namespace, objects := range allNamespaces { availableObjects += "\n\t" + namespace + ":" - for objectID := range scope.Objects() { + for objectID := range objects { availableObjects += " " + objectID } } @@ -359,13 +359,13 @@ func applyAllNamespaces(allNamespaces map[string]schema.Scope, scopeToApplyTo sc ) } -func addOutputNamespacedScopes(allNamespaces map[string]schema.Scope, stage step.LifecycleStageWithSchema, prefix string) { +func addOutputNamespacedScopes(allNamespaces map[string]map[string]*schema.ObjectSchema, stage step.LifecycleStageWithSchema, prefix string) { for outputID, outputSchema := range stage.Outputs { addScopesWithReferences(allNamespaces, outputSchema.Schema(), prefix+outputID) } } -func addInputNamespacedScopes(allNamespaces map[string]schema.Scope, stage step.LifecycleStageWithSchema, prefix string) { +func addInputNamespacedScopes(allNamespaces map[string]map[string]*schema.ObjectSchema, stage step.LifecycleStageWithSchema, prefix string) { for inputID, inputSchemaProperty := range stage.InputSchema { var inputSchemaType = inputSchemaProperty.Type() // Extract item values from lists (like for ForEach) @@ -379,19 +379,19 @@ func addInputNamespacedScopes(allNamespaces map[string]schema.Scope, stage step. } // Adds the scope to the namespace map, as well as all resolved references from external namespaces. -func addScopesWithReferences(allNamespaces map[string]schema.Scope, scope schema.Scope, prefix string) { - // First, just adds the scope - allNamespaces[prefix] = scope +func addScopesWithReferences(allNamespaces map[string]map[string]*schema.ObjectSchema, scope schema.Scope, prefix string) { + // First, just adds the scope's objects. + allNamespaces[prefix] = scope.Objects() // Next, checks all properties for resolved references that reference objects outside of this scope. rootObject := scope.RootObject() for propertyID, property := range rootObject.Properties() { if property.Type().TypeID() == schema.TypeIDRef { refProperty := property.Type().(schema.Ref) - if refProperty.Namespace() != schema.DEFAULT_NAMESPACE { + if refProperty.Namespace() != schema.SelfNamespace { // Found a reference to an object that is not included in the scope. Add it to the map. var referencedObject any = refProperty.GetObject() refObjectSchema := referencedObject.(*schema.ObjectSchema) - allNamespaces[prefix+"."+propertyID] = schema.NewScopeSchema(refObjectSchema) + allNamespaces[prefix+"."+propertyID] = map[string]*schema.ObjectSchema{refObjectSchema.ID(): refObjectSchema} } } } diff --git a/workflow/executor_unit_test.go b/workflow/executor_unit_test.go index cc533c9e..09ee97fb 100644 --- a/workflow/executor_unit_test.go +++ b/workflow/executor_unit_test.go @@ -59,17 +59,17 @@ var testLifecycleStage = step.LifecycleStageWithSchema{ } func TestAddOutputNamespacedScopes(t *testing.T) { - allNamespaces := make(map[string]schema.Scope) + allNamespaces := make(map[string]map[string]*schema.ObjectSchema) namespacePrefix := "TEST_PREFIX_" addOutputNamespacedScopes(allNamespaces, testLifecycleStage, namespacePrefix) expectedNamespace := namespacePrefix + "outputC" assert.Equals(t, len(allNamespaces), 1) assert.MapContainsKey(t, expectedNamespace, allNamespaces) - assert.Equals(t, "testObjectC", allNamespaces[expectedNamespace].Root()) + assert.MapContainsKey(t, "testObjectC", allNamespaces[expectedNamespace]) } func TestAddInputNamespacedScopes(t *testing.T) { - allNamespaces := make(map[string]schema.Scope) + allNamespaces := make(map[string]map[string]*schema.ObjectSchema) namespacePrefix := "TEST_PREFIX_" expectedInputs := map[string]string{ "scopeA": "testObjectA", @@ -81,12 +81,12 @@ func TestAddInputNamespacedScopes(t *testing.T) { for expectedInput, expectedObject := range expectedInputs { expectedNamespace := namespacePrefix + expectedInput assert.MapContainsKey(t, expectedNamespace, allNamespaces) - assert.Equals(t, expectedObject, allNamespaces[expectedNamespace].Root()) + assert.MapContainsKey(t, expectedObject, allNamespaces[expectedNamespace]) } } func TestAddScopesWithMissingCache(t *testing.T) { - allNamespaces := make(map[string]schema.Scope) + allNamespaces := make(map[string]map[string]*schema.ObjectSchema) externalRef3 := schema.NewNamespacedRefSchema("scopeTestObjectC", "not-applied-namespace", nil) notAppliedExternalRefProperty := schema.NewPropertySchema( externalRef3, @@ -117,7 +117,7 @@ func TestAddScopesWithMissingCache(t *testing.T) { func TestAddScopesWithReferences(t *testing.T) { // Test that the scope itself and the resolved references are added. - allNamespaces := make(map[string]schema.Scope) + allNamespaces := make(map[string]map[string]*schema.ObjectSchema) internalRef := schema.NewRefSchema("scopeTestObjectA", nil) internalRefProperty := schema.NewPropertySchema( internalRef, @@ -178,9 +178,9 @@ func TestAddScopesWithReferences(t *testing.T) { "scopeTestObjectB", map[string]*schema.PropertySchema{}, ), ) - testScope.ApplyScope(scopeToApply, "test-namespace-1") - testScope.ApplyScope(scopeToApply, "$.test-namespace-2") - testScope.ApplyScope(testScope, schema.DEFAULT_NAMESPACE) + testScope.ApplyNamespace(scopeToApply.Objects(), "test-namespace-1") + testScope.ApplyNamespace(scopeToApply.Objects(), "$.test-namespace-2") + testScope.ApplySelf() addScopesWithReferences(allNamespaces, testScope, "$") assert.Equals(t, len(allNamespaces), 3) assert.MapContainsKey(t, "$", allNamespaces) @@ -229,17 +229,18 @@ func TestApplyAllNamespaces_Pass(t *testing.T) { assert.Panics(t, func() { ref2Schema.GetObject() }) - allNamespaces := make(map[string]schema.Scope) - allNamespaces["test-namespace-1"] = schema.NewScopeSchema( - schema.NewObjectSchema( + allNamespaces := make(map[string]map[string]*schema.ObjectSchema) + // Test one manual map creation, and one using scope's `Objects()` method. + allNamespaces["test-namespace-1"] = map[string]*schema.ObjectSchema{ + "scopeTestObjectB": schema.NewObjectSchema( "scopeTestObjectB", map[string]*schema.PropertySchema{}, ), - ) + } allNamespaces["test-namespace-2"] = schema.NewScopeSchema( schema.NewObjectSchema( "scopeTestObjectC", map[string]*schema.PropertySchema{}, ), - ) + ).Objects() // Call the function under test and validate that all caches are set. assert.NoError(t, applyAllNamespaces(allNamespaces, testScope)) assert.NoError(t, testScope.ValidateReferences()) @@ -289,12 +290,12 @@ func TestApplyAllNamespaces_MissingNamespace(t *testing.T) { ref2Schema.GetObject() }) // Only add test-namespace-1 - allNamespaces := make(map[string]schema.Scope) + allNamespaces := make(map[string]map[string]*schema.ObjectSchema) allNamespaces["test-namespace-1"] = schema.NewScopeSchema( schema.NewObjectSchema( "scopeTestObjectB", map[string]*schema.PropertySchema{}, ), - ) + ).Objects() err := applyAllNamespaces(allNamespaces, testScope) assert.Error(t, err) assert.Contains(t, err.Error(), "error validating references") @@ -324,12 +325,12 @@ func TestApplyAllNamespaces_InvalidObject(t *testing.T) { }, ), ) - allNamespaces := make(map[string]schema.Scope) + allNamespaces := make(map[string]map[string]*schema.ObjectSchema) allNamespaces["test-namespace-1"] = schema.NewScopeSchema( schema.NewObjectSchema( "scopeTestObjectB", map[string]*schema.PropertySchema{}, ), - ) + ).Objects() assert.PanicsContains( t, func() { diff --git a/workflow/workflow_test.go b/workflow/workflow_test.go index 657b91e2..dc8bb471 100644 --- a/workflow/workflow_test.go +++ b/workflow/workflow_test.go @@ -1328,6 +1328,45 @@ func TestDelayedDisabledStepWorkflow(t *testing.T) { assert.Equals(t, toggledOutputMap["message"], "Step toggled_wait/wait disabled") } +var testExpressionWithExtraWhitespace = ` +version: v0.2.0 +input: + root: RootObject + objects: + RootObject: + id: RootObject + properties: {} +steps: + wait_1: + plugin: + src: "n/a" + deployment_type: "builtin" + step: wait + input: + wait_time_ms: 0 +outputs: + a: + leading-whitespace: !expr " $.steps.wait_1.outputs.success.message" + trailing-whitespace: !expr "$.steps.wait_1.outputs.success.message " + # Use | instead of |- to keep the newline at the end. + trailing-newline: !expr | + $.steps.wait_1.outputs.success.message +` + +func TestExpressionWithWhitespace(t *testing.T) { + preparedWorkflow := assert.NoErrorR[workflow.ExecutableWorkflow](t)( + getTestImplPreparedWorkflow(t, testExpressionWithExtraWhitespace), + ) + outputID, outputData, err := preparedWorkflow.Execute(context.Background(), map[string]any{}) + assert.NoError(t, err) + assert.Equals(t, outputID, "a") + assert.Equals(t, outputData.(map[any]any), map[any]any{ + "leading-whitespace": "Plugin slept for 0 ms.", + "trailing-whitespace": "Plugin slept for 0 ms.", + "trailing-newline": "Plugin slept for 0 ms.", + }) +} + func createTestExecutableWorkflow(t *testing.T, workflowStr string, workflowCtx map[string][]byte) (workflow.ExecutableWorkflow, error) { logConfig := log.Config{ Level: log.LevelDebug,