Merge branch 'main' into moe/python-protos

.github/actions/build-cache/action.yml

@@ -56,3 +56,12 @@ runs:
         key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }}
         restore-keys: |
           ${{ runner.os }}-pnpm-
+    - name: Restore Playwright Browser Cache
+      id: cache-playwright
+      uses: actions/cache/restore@v4
+      with:
+        path: ~/.cache/ms-playwright
+        key: ${{ runner.os }}-playwright-${{ hashFiles('**/pnpm-lock.yaml') }}
+        restore-keys: |
+          ${{ runner.os }}-playwright-
+

.github/workflows/writecache.yml

@@ -28,6 +28,8 @@ jobs:
         run: go mod download -x
       - name: Download pnpm dependencies
         run: pnpm install
+      - name: Download playwright browsers
+        run: npx playwright install
       - id: find-go-build-cache
         shell: bash
         run: echo "cache=$(go env GOCACHE)" >> "$GITHUB_OUTPUT"
@@ -66,6 +68,12 @@ jobs:
             ~/.pnpm-store
             node_modules
           key: ${{ runner.os }}-pnpm-${{ hashFiles('**/pnpm-lock.yaml') }}
+      - name: Save Playwright Browser Cache
+        id: cache-playwright
+        uses: actions/cache/save@v4
+        with:
+          path: ~/.cache/ms-playwright
+          key: ${{ runner.os }}-playwright-${{ hashFiles('**/pnpm-lock.yaml') }}
       - name: Save Docker Images
         uses: ScribeMD/[email protected]
         with:

backend/controller/admin/admin.go

@@ -120,8 +120,7 @@ func (s *AdminService) ConfigSet(ctx context.Context, req *connect.Request[ftlv1
 		return nil, err
 	}
 
-	pkey := configProviderKey(req.Msg.Provider)
-	err = s.cm.SetJSON(ctx, pkey, refFromConfigRef(req.Msg.GetRef()), req.Msg.Value)
+	err = s.cm.SetJSON(ctx, refFromConfigRef(req.Msg.GetRef()), req.Msg.Value)
 	if err != nil {
 		return nil, fmt.Errorf("failed to set config: %w", err)
 	}
@@ -130,8 +129,7 @@ func (s *AdminService) ConfigSet(ctx context.Context, req *connect.Request[ftlv1
 
 // ConfigUnset unsets the config value at the given ref.
 func (s *AdminService) ConfigUnset(ctx context.Context, req *connect.Request[ftlv1.UnsetConfigRequest]) (*connect.Response[ftlv1.UnsetConfigResponse], error) {
-	pkey := configProviderKey(req.Msg.Provider)
-	err := s.cm.Unset(ctx, pkey, refFromConfigRef(req.Msg.GetRef()))
+	err := s.cm.Unset(ctx, refFromConfigRef(req.Msg.GetRef()))
 	if err != nil {
 		return nil, fmt.Errorf("failed to unset config: %w", err)
 	}
@@ -146,10 +144,6 @@ func (s *AdminService) SecretsList(ctx context.Context, req *connect.Request[ftl
 	}
 	secrets := []*ftlv1.ListSecretsResponse_Secret{}
 	for _, secret := range listing {
-		if req.Msg.Provider != nil && manager.ProviderKeyForAccessor(secret.Accessor) != secretProviderKey(req.Msg.Provider) {
-			// Skip secrets that don't match the provider in the request
-			continue
-		}
 		module, ok := secret.Module.Get()
 		if req.Msg.Module != nil && *req.Msg.Module != "" && module != *req.Msg.Module {
 			continue
@@ -192,34 +186,14 @@ func (s *AdminService) SecretGet(ctx context.Context, req *connect.Request[ftlv1
 	return connect.NewResponse(&ftlv1.GetSecretResponse{Value: vb}), nil
 }
 
-func secretProviderKey(p *ftlv1.SecretProvider) configuration.ProviderKey {
-	if p == nil {
-		return ""
-	}
-	switch *p {
-	case ftlv1.SecretProvider_SECRET_INLINE:
-		return providers.InlineProviderKey
-	case ftlv1.SecretProvider_SECRET_ENVAR:
-		return providers.EnvarProviderKey
-	case ftlv1.SecretProvider_SECRET_KEYCHAIN:
-		return providers.KeychainProviderKey
-	case ftlv1.SecretProvider_SECRET_OP:
-		return providers.OnePasswordProviderKey
-	case ftlv1.SecretProvider_SECRET_ASM:
-		return providers.ASMProviderKey
-	}
-	return ""
-}
-
 // SecretSet sets the secret at the given ref to the provided value.
 func (s *AdminService) SecretSet(ctx context.Context, req *connect.Request[ftlv1.SetSecretRequest]) (*connect.Response[ftlv1.SetSecretResponse], error) {
 	err := s.validateAgainstSchema(ctx, true, refFromConfigRef(req.Msg.GetRef()), req.Msg.Value)
 	if err != nil {
 		return nil, err
 	}
 
-	pkey := secretProviderKey(req.Msg.Provider)
-	err = s.sm.SetJSON(ctx, pkey, refFromConfigRef(req.Msg.GetRef()), req.Msg.Value)
+	err = s.sm.SetJSON(ctx, refFromConfigRef(req.Msg.GetRef()), req.Msg.Value)
 	if err != nil {
 		return nil, fmt.Errorf("failed to set secret: %w", err)
 	}
@@ -228,8 +202,7 @@ func (s *AdminService) SecretSet(ctx context.Context, req *connect.Request[ftlv1
 
 // SecretUnset unsets the secret value at the given ref.
 func (s *AdminService) SecretUnset(ctx context.Context, req *connect.Request[ftlv1.UnsetSecretRequest]) (*connect.Response[ftlv1.UnsetSecretResponse], error) {
-	pkey := secretProviderKey(req.Msg.Provider)
-	err := s.sm.Unset(ctx, pkey, refFromConfigRef(req.Msg.GetRef()))
+	err := s.sm.Unset(ctx, refFromConfigRef(req.Msg.GetRef()))
 	if err != nil {
 		return nil, fmt.Errorf("failed to unset secret: %w", err)
 	}

backend/controller/admin/admin_test.go

@@ -22,18 +22,14 @@ import (
 )
 
 func TestAdminService(t *testing.T) {
+	t.Skip("This will be replaced soon")
 	config := tempConfigPath(t, "testdata/ftl-project.toml", "admin")
 	ctx := log.ContextWithNewDefaultLogger(context.Background())
 
-	cm, err := manager.NewConfigurationManager(ctx, routers.ProjectConfig[configuration.Configuration]{Config: config})
+	cm, err := manager.New(ctx, routers.ProjectConfig[configuration.Configuration]{Config: config}, providers.Inline[configuration.Configuration]{})
 	assert.NoError(t, err)
 
-	sm, err := manager.New(ctx,
-		routers.ProjectConfig[configuration.Secrets]{Config: config},
-		[]configuration.Provider[configuration.Secrets]{
-			providers.Envar[configuration.Secrets]{},
-			providers.Inline[configuration.Secrets]{},
-		})
+	sm, err := manager.New(ctx, routers.ProjectConfig[configuration.Secrets]{Config: config}, providers.Inline[configuration.Secrets]{})
 	assert.NoError(t, err)
 	admin := NewAdminService(cm, sm, &diskSchemaRetriever{})
 	assert.NotZero(t, admin)
@@ -207,18 +203,12 @@ func TestAdminValidation(t *testing.T) {
 	config := tempConfigPath(t, "testdata/ftl-project.toml", "admin")
 	ctx := log.ContextWithNewDefaultLogger(context.Background())
 
-	cm, err := manager.NewConfigurationManager(ctx, routers.ProjectConfig[configuration.Configuration]{Config: config})
+	cm, err := manager.New(ctx, routers.ProjectConfig[configuration.Configuration]{Config: config}, providers.Inline[configuration.Configuration]{})
 	assert.NoError(t, err)
 
-	sm, err := manager.New(ctx,
-		routers.ProjectConfig[configuration.Secrets]{Config: config},
-		[]configuration.Provider[configuration.Secrets]{
-			providers.Envar[configuration.Secrets]{},
-			providers.Inline[configuration.Secrets]{},
-		})
+	sm, err := manager.New(ctx, routers.ProjectConfig[configuration.Secrets]{Config: config}, providers.Inline[configuration.Secrets]{})
 	assert.NoError(t, err)
 	admin := NewAdminService(cm, sm, &mockSchemaRetriever{})
-	assert.NotZero(t, admin)
 
 	testSetConfig(t, ctx, admin, "batmobile", "color", "Black", "")
 	testSetConfig(t, ctx, admin, "batmobile", "color", "Red", "JSON validation failed: Red is not a valid variant of enum batmobile.Color")

backend/controller/admin/client.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"net"
 	"net/url"
+	"time"
 
 	"connectrpc.com/connect"
 
@@ -54,7 +55,9 @@ func ShouldUseLocalClient(ctx context.Context, adminClient ftlv1connect.AdminSer
 	if err != nil {
 		return false, err
 	}
-	_, err = adminClient.Ping(ctx, connect.NewRequest(&ftlv1.PingRequest{}))
+	timeoutCtx, cancel := context.WithTimeout(ctx, time.Millisecond*500)
+	defer cancel()
+	_, err = adminClient.Ping(timeoutCtx, connect.NewRequest(&ftlv1.PingRequest{}))
 	if isConnectUnavailableError(err) && isLocal {
 		return true, nil
 	}
@@ -63,7 +66,9 @@ func ShouldUseLocalClient(ctx context.Context, adminClient ftlv1connect.AdminSer
 
 func isConnectUnavailableError(err error) bool {
 	var connectErr *connect.Error
-	if errors.As(err, &connectErr) {
+	if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+		return true
+	} else if errors.As(err, &connectErr) {
 		return connectErr.Code() == connect.CodeUnavailable
 	}
 	return false

backend/controller/admin/local_client_test.go

@@ -57,15 +57,10 @@ func TestAdminNoValidationWithNoSchema(t *testing.T) {
 	config := tempConfigPath(t, "testdata/ftl-project.toml", "admin")
 	ctx := log.ContextWithNewDefaultLogger(context.Background())
 
-	cm, err := manager.NewConfigurationManager(ctx, routers.ProjectConfig[cf.Configuration]{Config: config})
+	cm, err := manager.New(ctx, routers.ProjectConfig[cf.Configuration]{Config: config}, providers.NewInline[cf.Configuration]())
 	assert.NoError(t, err)
 
-	sm, err := manager.New(ctx,
-		routers.ProjectConfig[cf.Secrets]{Config: config},
-		[]cf.Provider[cf.Secrets]{
-			providers.Envar[cf.Secrets]{},
-			providers.Inline[cf.Secrets]{},
-		})
+	sm, err := manager.New(ctx, routers.ProjectConfig[cf.Secrets]{Config: config}, providers.NewInline[cf.Secrets]())
 	assert.NoError(t, err)
 
 	dsr := &diskSchemaRetriever{deployRoot: optional.Some(string(t.TempDir()))}

backend/controller/admin/testdata/ftl-project.toml

@@ -1,4 +1,7 @@
 name = "testdata"
+secrets-provider = "inline"
+config-provider = "inline"
+
 [global]
 [global.secrets]
 bar = "envar://baza"
@@ -10,6 +13,6 @@ foo = "inline://ImZvb2JhciI"
 mutable = "inline://ImhlbGxvd29ybGQi"
 
 [modules]
-  [modules.echo]
-    [modules.echo.configuration]
-      default = "inline://ImFub255bW91cyI"
+[modules.echo]
+[modules.echo.configuration]
+default = "inline://ImFub255bW91cyI"

backend/controller/controller.go

@@ -13,7 +13,6 @@ import (
 	"io"
 	"net/http"
 	"net/url"
-	goslices "slices"
 	"sort"
 	"strings"
 	"sync"
@@ -132,7 +131,15 @@ func (c *Config) OpenDBAndInstrument() (*sql.DB, error) {
 }
 
 // Start the Controller. Blocks until the context is cancelled.
-func Start(ctx context.Context, config Config, runnerScaling scaling.RunnerScaling, conn *sql.DB, devel bool) error {
+func Start(
+	ctx context.Context,
+	config Config,
+	runnerScaling scaling.RunnerScaling,
+	cm *cf.Manager[configuration.Configuration],
+	sm *cf.Manager[configuration.Secrets],
+	conn *sql.DB,
+	devel bool,
+) error {
 	config.SetDefaults()
 
 	logger := log.FromContext(ctx)
@@ -153,16 +160,13 @@ func Start(ctx context.Context, config Config, runnerScaling scaling.RunnerScali
 		logger.Infof("Web console available at: %s", config.Bind)
 	}
 
-	svc, err := New(ctx, conn, config, devel, runnerScaling)
+	svc, err := New(ctx, conn, cm, sm, config, devel, runnerScaling)
 	if err != nil {
 		return err
 	}
 	logger.Debugf("Listening on %s", config.Bind)
 	logger.Debugf("Advertising as %s", config.Advertise)
 
-	cm := cf.ConfigFromContext(ctx)
-	sm := cf.SecretsFromContext(ctx)
-
 	admin := admin.NewAdminService(cm, sm, svc.dal)
 	console := console.NewService(svc.dal, svc.timeline)
 
@@ -222,6 +226,9 @@ type Service struct {
 	key                model.ControllerKey
 	deploymentLogsSink *deploymentLogsSink
 
+	cm *cf.Manager[configuration.Configuration]
+	sm *cf.Manager[configuration.Secrets]
+
 	tasks                   *scheduledtask.Scheduler
 	cronJobs                *cronjobs.Service
 	pubSub                  *pubsub.Service
@@ -243,7 +250,15 @@ type Service struct {
 	runnerScaling           scaling.RunnerScaling
 }
 
-func New(ctx context.Context, conn *sql.DB, config Config, devel bool, runnerScaling scaling.RunnerScaling) (*Service, error) {
+func New(
+	ctx context.Context,
+	conn *sql.DB,
+	cm *cf.Manager[configuration.Configuration],
+	sm *cf.Manager[configuration.Secrets],
+	config Config,
+	devel bool,
+	runnerScaling scaling.RunnerScaling,
+) (*Service, error) {
 	key := config.Key
 	if config.Key.IsZero() {
 		key = model.NewControllerKey(config.Bind.Hostname(), config.Bind.Port())
@@ -265,6 +280,8 @@ func New(ctx context.Context, conn *sql.DB, config Config, devel bool, runnerSca
 	scheduler := scheduledtask.New(ctx, key, ldb)
 
 	svc := &Service{
+		cm:                      cm,
+		sm:                      sm,
 		tasks:                   scheduler,
 		dbleaser:                ldb,
 		conn:                    conn,
@@ -729,20 +746,17 @@ func (s *Service) Ping(ctx context.Context, req *connect.Request[ftlv1.PingReque
 func (s *Service) GetModuleContext(ctx context.Context, req *connect.Request[ftlv1.ModuleContextRequest], resp *connect.ServerStream[ftlv1.ModuleContextResponse]) error {
 	name := req.Msg.Module
 
-	cm := cf.ConfigFromContext(ctx)
-	sm := cf.SecretsFromContext(ctx)
-
 	// Initialize checksum to -1; a zero checksum does occur when the context contains no settings
 	lastChecksum := int64(-1)
 
 	for {
 		h := sha.New()
 
-		configs, err := cm.MapForModule(ctx, name)
+		configs, err := s.cm.MapForModule(ctx, name)
 		if err != nil {
 			return connect.NewError(connect.CodeInternal, fmt.Errorf("could not get configs: %w", err))
 		}
-		secrets, err := sm.MapForModule(ctx, name)
+		secrets, err := s.sm.MapForModule(ctx, name)
 		if err != nil {
 			return connect.NewError(connect.CodeInternal, fmt.Errorf("could not get secrets: %w", err))
 		}
@@ -1043,22 +1057,11 @@ func (s *Service) CreateDeployment(ctx context.Context, req *connect.Request[ftl
 		return nil, fmt.Errorf("invalid module schema: %w", err)
 	}
 
-	sm := cf.SecretsFromContext(ctx)
 	for _, d := range module.Decls {
 		if db, ok := d.(*schema.Database); ok && db.Runtime != nil {
 			key := dsnSecretKey(module.Name, db.Name)
 
-			// TODO: Use a cluster specific default provider
-			allKeys := sm.ProviderKeys()
-			var providerKey configuration.ProviderKey
-			for _, key := range []configuration.ProviderKey{"inline", "asm"} {
-				if goslices.Contains(allKeys, key) {
-					providerKey = key
-					break
-				}
-			}
-
-			if err := sm.Set(ctx, providerKey, configuration.NewRef(module.Name, key), db.Runtime.DSN); err != nil {
+			if err := s.sm.Set(ctx, configuration.NewRef(module.Name, key), db.Runtime.DSN); err != nil {
 				return nil, fmt.Errorf("could not set database secret %s: %w", key, err)
 			}
 			logger.Infof("Database declaration: %s -> %s", db.Name, db.Runtime.DSN)

backend/controller/leases/testdata/go/leases/go.mod

@@ -24,10 +24,10 @@ require (
 	github.com/alecthomas/participle/v2 v2.1.1 // indirect
 	github.com/alecthomas/repr v0.4.0 // indirect
 	github.com/alecthomas/types v0.16.0 // indirect
-	github.com/aws/aws-sdk-go-v2 v1.32.2 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 // indirect
-	github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.34.2 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.32.3 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22 // indirect
+	github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.34.3 // indirect
 	github.com/aws/smithy-go v1.22.0 // indirect
 	github.com/benbjohnson/clock v1.3.5 // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect

backend/controller/sql/testdata/go/database/go.mod

@@ -21,10 +21,10 @@ require (
 	github.com/alecthomas/participle/v2 v2.1.1 // indirect
 	github.com/alecthomas/repr v0.4.0 // indirect
 	github.com/alecthomas/types v0.16.0 // indirect
-	github.com/aws/aws-sdk-go-v2 v1.32.2 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.21 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.21 // indirect
-	github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.34.2 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.32.3 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.22 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.22 // indirect
+	github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.34.3 // indirect
 	github.com/aws/smithy-go v1.22.0 // indirect
 	github.com/benbjohnson/clock v1.3.5 // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect