refactor: centralise config/secrets/dbs/mocks into ModuleContext (#1395)

- Reduces reliance on `configuration` package in go-runtime
- Removed CallOverrider protocol:
- `call.go` now asks ModuleContext for any custom behaviour as required
by the test set up. This includes verb mocks, whether to allow calling
the verb directly, or custom errors in testing environments

backend/controller/controller.go

@@ -640,11 +640,11 @@ func (s *Service) GetModuleContext(ctx context.Context, req *connect.Request[ftl
 	if !ok {
 		return nil, connect.NewError(connect.CodeNotFound, fmt.Errorf("module %q not found", req.Msg.Module))
 	}
-	moduleContext, err := modulecontext.FromEnvironment(ctx, module.Name)
+	moduleContext, err := modulecontext.FromEnvironment(ctx, module.Name, false)
 	if err != nil {
 		return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("could not get module context: %w", err))
 	}
-	response, err := moduleContext.ToProto(ctx)
+	response, err := moduleContext.ToProto(module.Name)
 	if err != nil {
 		return nil, connect.NewError(connect.CodeInternal, fmt.Errorf("could not marshal module context: %w", err))
 	}

common/configuration/manager.go

@@ -44,9 +44,7 @@ func configFromEnvironment() []string {
 func NewDefaultSecretsManagerFromEnvironment(ctx context.Context) (*Manager[Secrets], error) {
 	var cr Resolver[Secrets] = ProjectConfigResolver[Secrets]{Config: configFromEnvironment()}
 	return DefaultSecretsMixin{
-		InlineProvider: InlineProvider[Secrets]{
-			Inline: true,
-		},
+		InlineProvider: InlineProvider[Secrets]{},
 	}.NewSecretsManager(ctx, cr)
 }
 
@@ -55,9 +53,7 @@ func NewDefaultSecretsManagerFromEnvironment(ctx context.Context) (*Manager[Secr
 func NewDefaultConfigurationManagerFromEnvironment(ctx context.Context) (*Manager[Configuration], error) {
 	cr := ProjectConfigResolver[Configuration]{Config: configFromEnvironment()}
 	return DefaultConfigMixin{
-		InlineProvider: InlineProvider[Configuration]{
-			Inline: true,
-		},
+		InlineProvider: InlineProvider[Configuration]{},
 	}.NewConfigurationManager(ctx, cr)
 }
 

common/configuration/manager_test.go

@@ -2,6 +2,7 @@ package configuration
 
 import (
 	"context"
+	"fmt"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -16,13 +17,7 @@ import (
 
 func TestManager(t *testing.T) {
 	keyring.MockInit() // There's no way to undo this :\
-
-	config := filepath.Join(t.TempDir(), "ftl-project.toml")
-	existing, err := os.ReadFile("testdata/ftl-project.toml")
-	assert.NoError(t, err)
-	err = os.WriteFile(config, existing, 0600)
-	assert.NoError(t, err)
-
+	config := tempConfigPath(t, "testdata/ftl-project.toml", "manager")
 	ctx := log.ContextWithNewDefaultLogger(context.Background())
 
 	t.Run("Secrets", func(t *testing.T) {
@@ -60,6 +55,64 @@ func TestManager(t *testing.T) {
 	})
 }
 
+// TestMapPriority checks that module specific configs beat global configs when flattening for a module
+func TestMapPriority(t *testing.T) {
+	ctx := log.ContextWithNewDefaultLogger(context.Background())
+	config := tempConfigPath(t, "", "map")
+	cm, err := New(ctx,
+		ProjectConfigResolver[Configuration]{Config: []string{config}},
+		[]Provider[Configuration]{
+			InlineProvider[Configuration]{
+				Inline: true,
+			},
+		})
+	assert.NoError(t, err)
+	moduleName := "test"
+
+	// Set 50 configs and 50 global configs
+	// It's hard to tell if module config beats global configs because we are dealing with unordered maps, or because the logic is correct
+	// Repeating it 50 times hopefully gives us a good chance of catching inconsistencies
+	for i := range 50 {
+		key := fmt.Sprintf("key%d", i)
+
+		strValue := "HelloWorld"
+		globalStrValue := "GlobalHelloWorld"
+		if i%2 == 0 {
+			// sometimes try setting the module config first
+			assert.NoError(t, cm.Set(ctx, Ref{Module: optional.Some(moduleName), Name: key}, strValue))
+			assert.NoError(t, cm.Set(ctx, Ref{Module: optional.None[string](), Name: key}, globalStrValue))
+		} else {
+			// other times try setting the global config first
+			assert.NoError(t, cm.Set(ctx, Ref{Module: optional.None[string](), Name: key}, globalStrValue))
+			assert.NoError(t, cm.Set(ctx, Ref{Module: optional.Some(moduleName), Name: key}, strValue))
+		}
+	}
+	result, err := cm.MapForModule(ctx, moduleName)
+	assert.NoError(t, err)
+
+	for i := range 50 {
+		key := fmt.Sprintf("key%d", i)
+		assert.Equal(t, `"HelloWorld"`, string(result[key]), "module configs should beat global configs")
+	}
+}
+
+func tempConfigPath(t *testing.T, existingPath string, prefix string) string {
+	t.Helper()
+
+	config := filepath.Join(t.TempDir(), fmt.Sprintf("%s-ftl-project.toml", prefix))
+	var existing []byte
+	var err error
+	if existingPath == "" {
+		existing = []byte{}
+	} else {
+		existing, err = os.ReadFile(existingPath)
+		assert.NoError(t, err)
+	}
+	err = os.WriteFile(config, existing, 0600)
+	assert.NoError(t, err)
+	return config
+}
+
 // nolint
 func testManager[R Role](
 	t *testing.T,

go-runtime/ftl/call.go

@@ -10,6 +10,7 @@ import (
 	ftlv1 "github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1"
 	"github.com/TBD54566975/ftl/backend/protos/xyz/block/ftl/v1/ftlv1connect"
 	"github.com/TBD54566975/ftl/go-runtime/encoding"
+	"github.com/TBD54566975/ftl/go-runtime/modulecontext"
 	"github.com/TBD54566975/ftl/internal/rpc"
 )
 
@@ -19,37 +20,44 @@ func call[Req, Resp any](ctx context.Context, callee Ref, req Req) (resp Resp, e
 		return resp, fmt.Errorf("%s: failed to marshal request: %w", callee, err)
 	}
 
-	if overrider, ok := CallOverriderFromContext(ctx); ok {
-		override, uncheckedResp, err := overrider.OverrideCall(ctx, callee, req)
+	behavior, err := modulecontext.FromContext(ctx).BehaviorForVerb(modulecontext.Ref(callee))
+	if err != nil {
+		return resp, fmt.Errorf("%s: %w", callee, err)
+	}
+	switch behavior := behavior.(type) {
+	case modulecontext.MockBehavior:
+		uncheckedResp, err := behavior.Mock(ctx, req)
 		if err != nil {
 			return resp, fmt.Errorf("%s: %w", callee, err)
 		}
-		if override {
-			if resp, ok = uncheckedResp.(Resp); ok {
-				return resp, nil
-			}
-			return resp, fmt.Errorf("%s: overridden verb had invalid response type %T, expected %v", callee, uncheckedResp, reflect.TypeFor[Resp]())
+		if r, ok := uncheckedResp.(Resp); ok {
+			return r, nil
 		}
-	}
-
-	client := rpc.ClientFromContext[ftlv1connect.VerbServiceClient](ctx)
-	cresp, err := client.Call(ctx, connect.NewRequest(&ftlv1.CallRequest{Verb: callee.ToProto(), Body: reqData}))
-	if err != nil {
-		return resp, fmt.Errorf("%s: failed to call Verb: %w", callee, err)
-	}
-	switch cresp := cresp.Msg.Response.(type) {
-	case *ftlv1.CallResponse_Error_:
-		return resp, fmt.Errorf("%s: %s", callee, cresp.Error.Message)
-
-	case *ftlv1.CallResponse_Body:
-		err = encoding.Unmarshal(cresp.Body, &resp)
+		return resp, fmt.Errorf("%s: overridden verb had invalid response type %T, expected %v", callee, uncheckedResp, reflect.TypeFor[Resp]())
+	case modulecontext.DirectBehavior:
+		panic("not implemented")
+	case modulecontext.StandardBehavior:
+		client := rpc.ClientFromContext[ftlv1connect.VerbServiceClient](ctx)
+		cresp, err := client.Call(ctx, connect.NewRequest(&ftlv1.CallRequest{Verb: callee.ToProto(), Body: reqData}))
 		if err != nil {
-			return resp, fmt.Errorf("%s: failed to decode response: %w", callee, err)
+			return resp, fmt.Errorf("%s: failed to call Verb: %w", callee, err)
 		}
-		return resp, nil
+		switch cresp := cresp.Msg.Response.(type) {
+		case *ftlv1.CallResponse_Error_:
+			return resp, fmt.Errorf("%s: %s", callee, cresp.Error.Message)
 
+		case *ftlv1.CallResponse_Body:
+			err = encoding.Unmarshal(cresp.Body, &resp)
+			if err != nil {
+				return resp, fmt.Errorf("%s: failed to decode response: %w", callee, err)
+			}
+			return resp, nil
+
+		default:
+			panic(fmt.Sprintf("%s: invalid response type %T", callee, cresp))
+		}
 	default:
-		panic(fmt.Sprintf("%s: invalid response type %T", callee, cresp))
+		panic(fmt.Sprintf("unknown behavior: %s", behavior))
 	}
 }
 

go-runtime/ftl/config.go

@@ -6,7 +6,7 @@ import (
 	"runtime"
 	"strings"
 
-	"github.com/TBD54566975/ftl/common/configuration"
+	"github.com/TBD54566975/ftl/go-runtime/modulecontext"
 )
 
 // ConfigType is a type that can be used as a configuration value.
@@ -32,9 +32,7 @@ func (c ConfigValue[T]) GoString() string {
 
 // Get returns the value of the configuration key from FTL.
 func (c ConfigValue[T]) Get(ctx context.Context) (out T) {
-	cm := configuration.ConfigFromContext(ctx)
-	ref := configuration.NewRef(c.Module, c.Name)
-	err := cm.Get(ctx, ref, &out)
+	err := modulecontext.FromContext(ctx).GetConfig(c.Name, &out)
 	if err != nil {
 		panic(fmt.Errorf("failed to get %s: %w", c, err))
 	}

go-runtime/ftl/config_test.go

@@ -6,22 +6,21 @@ import (
 
 	"github.com/alecthomas/assert/v2"
 
-	"github.com/TBD54566975/ftl/common/configuration"
-	"github.com/TBD54566975/ftl/common/projectconfig"
+	"github.com/TBD54566975/ftl/go-runtime/modulecontext"
 	"github.com/TBD54566975/ftl/internal/log"
 )
 
 func TestConfig(t *testing.T) {
 	ctx := log.ContextWithNewDefaultLogger(context.Background())
-	cr := configuration.ProjectConfigResolver[configuration.Configuration]{Config: []string{"testdata/ftl-project.toml"}}
-	assert.Equal(t, []string{"testdata/ftl-project.toml"}, projectconfig.ConfigPaths(cr.Config))
-	cm, err := configuration.NewConfigurationManager(ctx, cr)
-	assert.NoError(t, err)
-	ctx = configuration.ContextWithConfig(ctx, cm)
+
+	moduleCtx := modulecontext.New()
+	ctx = moduleCtx.ApplyToContext(ctx)
+
 	type C struct {
 		One string
 		Two string
 	}
 	config := Config[C]("test")
+	assert.NoError(t, moduleCtx.SetConfig("test", C{"one", "two"}))
 	assert.Equal(t, C{"one", "two"}, config.Get(ctx))
 }

go-runtime/ftl/database.go

@@ -27,8 +27,8 @@ func (d Database) String() string { return fmt.Sprintf("database %q", d.Name) }
 
 // Get returns the sql db connection for the database.
 func (d Database) Get(ctx context.Context) *sql.DB {
-	provider := modulecontext.DBProviderFromContext(ctx)
-	db, err := provider.Get(d.Name)
+	provider := modulecontext.FromContext(ctx)
+	db, err := provider.GetDatabase(d.Name, d.DBType)
 	if err != nil {
 		panic(err.Error())
 	}

go-runtime/ftl/ftltest/ftltest.go

@@ -6,9 +6,6 @@ import (
 	"fmt"
 	"reflect"
 
-	"github.com/alecthomas/types/optional"
-
-	cf "github.com/TBD54566975/ftl/common/configuration"
 	"github.com/TBD54566975/ftl/go-runtime/ftl"
 	"github.com/TBD54566975/ftl/go-runtime/modulecontext"
 	"github.com/TBD54566975/ftl/internal/log"
@@ -17,15 +14,12 @@ import (
 // Context suitable for use in testing FTL verbs with provided options
 func Context(options ...func(context.Context) error) context.Context {
 	ctx := log.ContextWithNewDefaultLogger(context.Background())
-	context, err := modulecontext.FromEnvironment(ctx, ftl.Module())
+	context, err := modulecontext.FromEnvironment(ctx, ftl.Module(), true)
 	if err != nil {
 		panic(err)
 	}
 	ctx = context.ApplyToContext(ctx)
 
-	mockProvider := newMockVerbProvider()
-	ctx = ftl.ApplyCallOverriderToContext(ctx, mockProvider)
-
 	for _, option := range options {
 		err = option(ctx)
 		if err != nil {
@@ -49,8 +43,7 @@ func WithConfig[T ftl.ConfigType](config ftl.ConfigValue[T], value T) func(conte
 		if config.Module != ftl.Module() {
 			return fmt.Errorf("config %v does not match current module %s", config.Module, ftl.Module())
 		}
-		cm := cf.ConfigFromContext(ctx)
-		return cm.Set(ctx, cf.Ref{Module: optional.Some(config.Module), Name: config.Name}, value)
+		return modulecontext.FromContext(ctx).SetConfig(config.Name, value)
 	}
 }
 
@@ -68,8 +61,7 @@ func WithSecret[T ftl.SecretType](secret ftl.SecretValue[T], value T) func(conte
 		if secret.Module != ftl.Module() {
 			return fmt.Errorf("secret %v does not match current module %s", secret.Module, ftl.Module())
 		}
-		sm := cf.SecretsFromContext(ctx)
-		return sm.Set(ctx, cf.Ref{Module: optional.Some(secret.Module), Name: secret.Name}, value)
+		return modulecontext.FromContext(ctx).SetSecret(secret.Name, value)
 	}
 }
 
@@ -87,21 +79,13 @@ func WithSecret[T ftl.SecretType](secret ftl.SecretValue[T], value T) func(conte
 func WhenVerb[Req any, Resp any](verb ftl.Verb[Req, Resp], fake func(ctx context.Context, req Req) (resp Resp, err error)) func(context.Context) error {
 	return func(ctx context.Context) error {
 		ref := ftl.FuncRef(verb)
-		overrider, ok := ftl.CallOverriderFromContext(ctx)
-		if !ok {
-			return fmt.Errorf("could not override %v with a fake, context not set up with call overrider", ref)
-		}
-		mockProvider, ok := overrider.(*mockVerbProvider)
-		if !ok {
-			return fmt.Errorf("could not override %v with a fake, call overrider is not a MockProvider", ref)
-		}
-		mockProvider.mocks[ref] = func(ctx context.Context, req any) (resp any, err error) {
+		modulecontext.FromContext(ctx).SetMockVerb(modulecontext.Ref(ref), func(ctx context.Context, req any) (resp any, err error) {
 			request, ok := req.(Req)
 			if !ok {
 				return nil, fmt.Errorf("invalid request type %T for %v, expected %v", req, ref, reflect.TypeFor[Req]())
 			}
 			return fake(ctx, request)
-		}
+		})
 		return nil
 	}
 }