diff --git a/cmd/ftl/cmd_config.go b/cmd/ftl/cmd_config.go new file mode 100644 index 0000000000..6708c6ac58 --- /dev/null +++ b/cmd/ftl/cmd_config.go @@ -0,0 +1,168 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + + "github.com/alecthomas/kong" + + "github.com/TBD54566975/ftl/common/configuration" +) + +type mutableConfigProviderMixin struct { + configuration.InlineProvider + configuration.EnvarProvider[configuration.EnvarTypeConfig] +} + +func (s *mutableConfigProviderMixin) newConfigManager(ctx context.Context, resolver configuration.Resolver) (*configuration.Manager, error) { + return configuration.New(ctx, resolver, []configuration.Provider{s.InlineProvider, s.EnvarProvider}) +} + +type configCmd struct { + configuration.ProjectConfigResolver[configuration.FromConfig] + + List configListCmd `cmd:"" help:"List configuration."` + Get configGetCmd `cmd:"" help:"Get a configuration value."` + Set configSetCmd `cmd:"" help:"Set a configuration value."` + Unset configUnsetCmd `cmd:"" help:"Unset a configuration value."` +} + +func (s *configCmd) newConfigManager(ctx context.Context) (*configuration.Manager, error) { + mp := mutableConfigProviderMixin{} + _ = kong.ApplyDefaults(&mp) + return mp.newConfigManager(ctx, s.ProjectConfigResolver) +} + +func (s *configCmd) Help() string { + return ` +Configuration values are used to store non-sensitive information such as URLs, +etc. +` +} + +type configListCmd struct { + Values bool `help:"List configuration values."` + Module string `optional:"" arg:"" placeholder:"MODULE" help:"List configuration only in this module."` +} + +func (s *configListCmd) Run(ctx context.Context, scmd *configCmd) error { + sm, err := scmd.newConfigManager(ctx) + if err != nil { + return err + } + listing, err := sm.List(ctx) + if err != nil { + return err + } + for _, config := range listing { + module, ok := config.Module.Get() + if s.Module != "" && module != s.Module { + continue + } + if ok { + fmt.Printf("%s.%s", module, config.Name) + } else { + fmt.Print(config.Name) + } + if s.Values { + var value any + err := sm.Get(ctx, config.Ref, &value) + if err != nil { + fmt.Printf(" (error: %s)\n", err) + } else { + data, _ := json.Marshal(value) + fmt.Printf(" = %s\n", data) + } + } else { + fmt.Println() + } + } + return nil + +} + +type configGetCmd struct { + Ref configuration.Ref `arg:"" help:"Configuration reference in the form [.]."` +} + +func (s *configGetCmd) Help() string { + return ` +Returns a JSON-encoded configuration value. +` +} + +func (s *configGetCmd) Run(ctx context.Context, scmd *configCmd) error { + sm, err := scmd.newConfigManager(ctx) + if err != nil { + return err + } + var value any + err = sm.Get(ctx, s.Ref, &value) + if err != nil { + return err + } + + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + err = enc.Encode(value) + if err != nil { + return fmt.Errorf("%s: %w", s.Ref, err) + } + return nil +} + +type configSetCmd struct { + mutableConfigProviderMixin + + JSON bool `help:"Assume input value is JSON."` + Ref configuration.Ref `arg:"" help:"Configuration reference in the form [.]."` + Value *string `arg:"" placeholder:"VALUE" help:"Configuration value (read from stdin if omitted)." optional:""` +} + +func (s *configSetCmd) Run(ctx context.Context, scmd *configCmd) error { + sm, err := s.newConfigManager(ctx, scmd.ProjectConfigResolver) + if err != nil { + return err + } + + if err := sm.Mutable(); err != nil { + return err + } + + var config []byte + if s.Value != nil { + config = []byte(*s.Value) + } else { + config, err = io.ReadAll(os.Stdin) + if err != nil { + return fmt.Errorf("failed to read config from stdin: %w", err) + } + } + + var configValue any + if s.JSON { + if err := json.Unmarshal(config, &configValue); err != nil { + return fmt.Errorf("config is not valid JSON: %w", err) + } + } else { + configValue = string(config) + } + return sm.Set(ctx, s.Ref, configValue) +} + +type configUnsetCmd struct { + mutableConfigProviderMixin + + Ref configuration.Ref `arg:"" help:"Configuration reference in the form [.]."` +} + +func (s *configUnsetCmd) Run(ctx context.Context, scmd *configCmd) error { + sm, err := s.newConfigManager(ctx, scmd.ProjectConfigResolver) + if err != nil { + return err + } + return sm.Unset(ctx, s.Ref) +} diff --git a/cmd/ftl/cmd_secret.go b/cmd/ftl/cmd_secret.go new file mode 100644 index 0000000000..f33502e83e --- /dev/null +++ b/cmd/ftl/cmd_secret.go @@ -0,0 +1,182 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + + "github.com/alecthomas/kong" + "github.com/mattn/go-isatty" + "golang.org/x/term" + + "github.com/TBD54566975/ftl/common/configuration" +) + +type mutableSecretProviderMixin struct { + configuration.InlineProvider + configuration.KeychainProvider + configuration.EnvarProvider[configuration.EnvarTypeSecrets] + configuration.OnePasswordProvider +} + +func (s *mutableSecretProviderMixin) newSecretsManager(ctx context.Context, resolver configuration.Resolver) (*configuration.Manager, error) { + return configuration.New(ctx, resolver, []configuration.Provider{ + s.InlineProvider, s.KeychainProvider, s.EnvarProvider, s.OnePasswordProvider, + }) +} + +type secretCmd struct { + configuration.ProjectConfigResolver[configuration.FromSecrets] + + List secretListCmd `cmd:"" help:"List secrets."` + Get secretGetCmd `cmd:"" help:"Get a secret."` + Set secretSetCmd `cmd:"" help:"Set a secret."` + Unset secretUnsetCmd `cmd:"" help:"Unset a secret."` +} + +func (s *secretCmd) newSecretsManager(ctx context.Context) (*configuration.Manager, error) { + mp := mutableSecretProviderMixin{} + _ = kong.ApplyDefaults(&mp) + return mp.newSecretsManager(ctx, s.ProjectConfigResolver) +} + +func (s *secretCmd) Help() string { + return ` +Secrets are used to store sensitive information such as passwords, tokens, and +keys. When setting a secret, the value is read from a password prompt if stdin +is a terminal, otherwise it is read from stdin directly. Secrets can be stored +in the project's configuration file, in the system keychain, in environment +variables, and so on. +` +} + +type secretListCmd struct { + Values bool `help:"List secret values."` + Module string `optional:"" arg:"" placeholder:"MODULE" help:"List secrets only in this module."` +} + +func (s *secretListCmd) Run(ctx context.Context, scmd *secretCmd) error { + sm, err := scmd.newSecretsManager(ctx) + if err != nil { + return err + } + listing, err := sm.List(ctx) + if err != nil { + return err + } + for _, secret := range listing { + module, ok := secret.Module.Get() + if s.Module != "" && module != s.Module { + continue + } + if ok { + fmt.Printf("%s.%s", module, secret.Name) + } else { + fmt.Print(secret.Name) + } + if s.Values { + var value any + err := sm.Get(ctx, secret.Ref, &value) + if err != nil { + fmt.Printf(" (error: %s)\n", err) + } else { + data, _ := json.Marshal(value) + fmt.Printf(" = %s\n", data) + } + } else { + fmt.Println() + } + } + return nil + +} + +type secretGetCmd struct { + Ref configuration.Ref `arg:"" help:"Secret reference in the form [.]."` +} + +func (s *secretGetCmd) Help() string { + return ` +Returns a JSON-encoded secret value. +` +} + +func (s *secretGetCmd) Run(ctx context.Context, scmd *secretCmd) error { + sm, err := scmd.newSecretsManager(ctx) + if err != nil { + return err + } + var value any + err = sm.Get(ctx, s.Ref, &value) + if err != nil { + return err + } + + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + err = enc.Encode(value) + if err != nil { + return fmt.Errorf("%s: %w", s.Ref, err) + } + return nil +} + +type secretSetCmd struct { + mutableSecretProviderMixin + + JSON bool `help:"Assume input value is JSON."` + Ref configuration.Ref `arg:"" help:"Secret reference in the form [.]."` +} + +func (s *secretSetCmd) Run(ctx context.Context, scmd *secretCmd) error { + sm, err := s.newSecretsManager(ctx, scmd.ProjectConfigResolver) + if err != nil { + return err + } + + if err := sm.Mutable(); err != nil { + return err + } + + // Prompt for a secret if stdin is a terminal, otherwise read from stdin. + var secret []byte + if isatty.IsTerminal(0) { + fmt.Print("Secret: ") + secret, err = term.ReadPassword(0) + fmt.Println() + if err != nil { + return err + } + } else { + secret, err = io.ReadAll(os.Stdin) + if err != nil { + return fmt.Errorf("failed to read secret from stdin: %w", err) + } + } + + var secretValue any + if s.JSON { + if err := json.Unmarshal(secret, &secretValue); err != nil { + return fmt.Errorf("secret is not valid JSON: %w", err) + } + } else { + secretValue = string(secret) + } + return sm.Set(ctx, s.Ref, secretValue) +} + +type secretUnsetCmd struct { + mutableSecretProviderMixin + + Ref configuration.Ref `arg:"" help:"Secret reference in the form [.]."` +} + +func (s *secretUnsetCmd) Run(ctx context.Context, scmd *secretCmd) error { + sm, err := s.newSecretsManager(ctx, scmd.ProjectConfigResolver) + if err != nil { + return err + } + return sm.Unset(ctx, s.Ref) +} diff --git a/cmd/ftl/main.go b/cmd/ftl/main.go index 852d5134b7..39ccf0e73a 100644 --- a/cmd/ftl/main.go +++ b/cmd/ftl/main.go @@ -20,7 +20,6 @@ import ( type CLI struct { Version kong.VersionFlag `help:"Show version."` - Config kong.ConfigFlag `help:"Load configuration from TOML file." placeholder:"FILE"` LogConfig log.Config `embed:"" prefix:"log-" group:"Logging:"` Endpoint *url.URL `default:"http://127.0.0.1:8892" help:"FTL endpoint to bind/connect to." env:"FTL_ENDPOINT"` @@ -38,6 +37,8 @@ type CLI struct { Build buildCmd `cmd:"" help:"Build an FTL module."` Deploy deployCmd `cmd:"" help:"Create a new deployment."` Download downloadCmd `cmd:"" help:"Download a deployment."` + Secret secretCmd `cmd:"" help:"Manage secrets."` + Config configCmd `cmd:"" help:"Manage configuration."` } var cli CLI @@ -46,6 +47,8 @@ func main() { kctx := kong.Parse(&cli, kong.Description(`FTL - Towards a 𝝺-calculus for large-scale systems`), kong.Configuration(kongtoml.Loader, ".ftl.toml", "~/.ftl.toml"), + kong.ShortUsageOnError(), + kong.HelpOptions{Compact: true, WrapUpperBound: 80}, kong.AutoGroup(func(parent kong.Visitable, flag *kong.Flag) *kong.Group { node, ok := parent.(*kong.Command) if !ok { diff --git a/common/configuration/1password.go b/common/configuration/1password.go new file mode 100644 index 0000000000..444f294b82 --- /dev/null +++ b/common/configuration/1password.go @@ -0,0 +1,54 @@ +package configuration + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/url" + + "github.com/TBD54566975/ftl/internal/exec" + "github.com/TBD54566975/ftl/internal/log" +) + +// OnePasswordProvider is a configuration provider that reads passwords from +// 1Password vaults via the "op" command line tool. +type OnePasswordProvider struct { + OnePassword bool `name:"op" help:"Write 1Password secret references - does not write to 1Password." group:"Provider:" xor:"configwriter"` +} + +var _ MutableProvider = OnePasswordProvider{} + +func (o OnePasswordProvider) Key() string { return "op" } +func (o OnePasswordProvider) Delete(ctx context.Context, ref Ref) error { return nil } + +func (o OnePasswordProvider) Load(ctx context.Context, ref Ref, key *url.URL) ([]byte, error) { + _, err := exec.LookPath("op") + if err != nil { + return nil, fmt.Errorf("1Password CLI tool \"op\" not found: %w", err) + } + output, err := exec.Capture(ctx, ".", "op", "read", "-n", key.String()) + if err != nil { + lines := bytes.Split(output, []byte("\n")) + logger := log.FromContext(ctx) + for _, line := range lines { + logger.Warnf("%s", line) + } + return nil, fmt.Errorf("error running 1password CLI tool \"op\": %w", err) + } + return json.Marshal(string(output)) +} + +func (o OnePasswordProvider) Store(ctx context.Context, ref Ref, value []byte) (*url.URL, error) { + var opref string + if err := json.Unmarshal(value, &opref); err != nil { + return nil, fmt.Errorf("1Password value must be a JSON string containing a 1Password secret refererence: %w", err) + } + u, err := url.Parse(opref) + if err != nil { + return nil, fmt.Errorf("invalid 1Password item ID: %w", err) + } + return u, nil +} + +func (o OnePasswordProvider) Writer() bool { return o.OnePassword } diff --git a/common/configuration/api.go b/common/configuration/api.go new file mode 100644 index 0000000000..c6110e1de4 --- /dev/null +++ b/common/configuration/api.go @@ -0,0 +1,100 @@ +// Package configuration is a generic configuration and secret management API. +package configuration + +import ( + "context" + "errors" + "net/url" + "strings" + + "github.com/alecthomas/types/optional" +) + +// ErrNotFound is returned when a configuration entry is not found or cannot be resolved. +var ErrNotFound = errors.New("not found") + +type Entry struct { + Ref + Accessor *url.URL +} + +// A Ref is a reference to a configuration value. +type Ref struct { + Module optional.Option[string] + Name string +} + +// NewRef creates a new Ref. +// +// If [module] is empty, the Ref is considered to be a global configuration value. +func NewRef(module, name string) Ref { + return Ref{Module: optional.Zero(module), Name: name} +} + +func ParseRef(s string) (Ref, error) { + ref := Ref{} + err := ref.UnmarshalText([]byte(s)) + return ref, err +} + +func (k Ref) String() string { + if m, ok := k.Module.Get(); ok { + return m + "." + k.Name + } + return k.Name +} + +func (k *Ref) UnmarshalText(text []byte) error { + s := string(text) + if i := strings.Index(s, "."); i != -1 { + k.Module = optional.Some(s[:i]) + k.Name = s[i+1:] + } else { + k.Name = s + } + return nil +} + +// A Resolver resolves configuration names to keys that are then used to load +// values from a Provider. +// +// This indirection allows for the storage of configuration values to be +// abstracted from the configuration itself. For example, the ftl-project.toml +// file contains per-module and global configuration maps, but the secrets +// themselves may be stored in a separate secret store such as a system keychain. +type Resolver interface { + Get(ctx context.Context, ref Ref) (key *url.URL, err error) + Set(ctx context.Context, ref Ref, key *url.URL) error + Unset(ctx context.Context, ref Ref) error + List(ctx context.Context) ([]Entry, error) +} + +// Provider is a generic interface for storing and retrieving configuration and secrets. +type Provider interface { + Key() string + Load(ctx context.Context, ref Ref, key *url.URL) ([]byte, error) +} + +// A MutableProvider is a Provider that can update configuration. +type MutableProvider interface { + Provider + // Writer returns true if this provider should be used to store configuration. + // + // Only one provider should return true. + // + // To be usable from the CLI, each provider must be a Kong-compatible struct + // containing a flag that this method should return. For example: + // + // type InlineProvider struct { + // Inline bool `help:"Write values inline." group:"Provider:" xor:"configwriter"` + // } + // + // func (i InlineProvider) Writer() bool { return i.Inline } + // + // The "xor" tag is used to ensure that only one writer is selected. + Writer() bool + // Store a configuration value and return its key. + Store(ctx context.Context, ref Ref, value []byte) (*url.URL, error) + // Delete a configuration value. + Delete(ctx context.Context, ref Ref) error +} diff --git a/common/configuration/envar.go b/common/configuration/envar.go new file mode 100644 index 0000000000..fd25be630e --- /dev/null +++ b/common/configuration/envar.go @@ -0,0 +1,100 @@ +package configuration + +import ( + "context" + "encoding/base64" + "fmt" + "net/url" + "os" +) + +type EnvarType interface{ prefix() string } + +type EnvarTypeConfig struct{} + +func (EnvarTypeConfig) prefix() string { return "FTL_CONFIG_" } + +type EnvarTypeSecrets struct{} + +func (EnvarTypeSecrets) prefix() string { return "FTL_SECRET_" } + +// EnvarProvider is a configuration provider that reads from environment variables. +type EnvarProvider[T EnvarType] struct { + Envar bool `help:"Print configuration as environment variables." xor:"configwriter" group:"Provider:"` +} + +var _ MutableProvider = EnvarProvider[EnvarTypeConfig]{} + +func (EnvarProvider[T]) Key() string { return "envar" } + +func (e EnvarProvider[T]) Load(ctx context.Context, ref Ref, key *url.URL) ([]byte, error) { + // FTL__[]_ where and are base64 encoded. + envar := e.key(ref) + + value, ok := os.LookupEnv(envar) + if ok { + return base64.RawStdEncoding.DecodeString(value) + } + return nil, fmt.Errorf("environment variable %q is not set: %w", envar, ErrNotFound) +} + +func (e EnvarProvider[T]) Delete(ctx context.Context, ref Ref) error { + return nil +} + +func (e EnvarProvider[T]) Store(ctx context.Context, ref Ref, value []byte) (*url.URL, error) { + envar := e.key(ref) + fmt.Printf("%s=%s\n", envar, base64.RawStdEncoding.EncodeToString(value)) + return &url.URL{Scheme: "envar", Host: ref.Name}, nil +} + +func (e EnvarProvider[T]) Writer() bool { return e.Envar } + +func (e EnvarProvider[T]) key(ref Ref) string { + key := e.prefix() + if m, ok := ref.Module.Get(); ok { + key += base64.RawStdEncoding.EncodeToString([]byte(m)) + "_" + } + key += base64.RawStdEncoding.EncodeToString([]byte(ref.Name)) + return key +} + +func (EnvarProvider[T]) prefix() string { + var t T + return t.prefix() +} + +// I don't think there's a need to parse environment variables, but let's keep +// this around for a bit just in case, as it was a PITA to write. +// +// func (e EnvarProvider[T]) entryForEnvar(env string) (Entry, error) { +// parts := strings.SplitN(env, "=", 2) +// if !strings.HasPrefix(parts[0], e.prefix()) { +// return Entry{}, fmt.Errorf("invalid environment variable %q", parts[0]) +// } +// accessor, err := url.Parse(parts[1]) +// if err != nil { +// return Entry{}, fmt.Errorf("invalid URL %q: %w", parts[1], err) +// } +// // FTL__[]_ +// nameParts := strings.SplitN(parts[0], "_", 4) +// if len(nameParts) < 4 { +// return Entry{}, fmt.Errorf("invalid environment variable %q", parts[0]) +// } +// var module optional.Option[string] +// if nameParts[2] != "" { +// decoded, err := base64.RawStdEncoding.DecodeString(nameParts[2]) +// if err != nil { +// return Entry{}, fmt.Errorf("invalid encoded module %q: %w", nameParts[2], err) +// } +// module = optional.Some(string(decoded)) +// } +// decoded, err := base64.RawStdEncoding.DecodeString(nameParts[3]) +// if err != nil { +// return Entry{}, fmt.Errorf("invalid encoded name %q: %w", nameParts[3], err) +// } +// return Entry{ +// Ref: Ref{module, string(decoded)}, +// Accessor: accessor, +// }, nil +// } diff --git a/common/configuration/inline.go b/common/configuration/inline.go new file mode 100644 index 0000000000..887d6f0a1a --- /dev/null +++ b/common/configuration/inline.go @@ -0,0 +1,36 @@ +package configuration + +import ( + "context" + "encoding/base64" + "fmt" + "net/url" +) + +// InlineProvider is a configuration provider that stores configuration in its key. +type InlineProvider struct { + Inline bool `help:"Write values inline in the configuration file." group:"Provider:" xor:"configwriter"` +} + +var _ MutableProvider = InlineProvider{} + +func (InlineProvider) Key() string { return "inline" } + +func (i InlineProvider) Writer() bool { return i.Inline } + +func (InlineProvider) Load(ctx context.Context, ref Ref, key *url.URL) ([]byte, error) { + data, err := base64.RawStdEncoding.DecodeString(key.Host) + if err != nil { + return nil, fmt.Errorf("invalid base64 data in inline configuration: %w", err) + } + return data, nil +} + +func (InlineProvider) Store(ctx context.Context, ref Ref, value []byte) (*url.URL, error) { + b64 := base64.RawStdEncoding.EncodeToString(value) + return &url.URL{Scheme: "inline", Host: b64}, nil +} + +func (InlineProvider) Delete(ctx context.Context, ref Ref) error { + return nil +} diff --git a/common/configuration/keychain.go b/common/configuration/keychain.go new file mode 100644 index 0000000000..4a6eb66221 --- /dev/null +++ b/common/configuration/keychain.go @@ -0,0 +1,55 @@ +package configuration + +import ( + "context" + "errors" + "fmt" + "net/url" + "strings" + + keyring "github.com/zalando/go-keyring" +) + +type KeychainProvider struct { + Keychain bool `help:"Write to the system keychain." group:"Provider:" xor:"configwriter"` +} + +var _ MutableProvider = KeychainProvider{} + +func (k KeychainProvider) Key() string { return "keychain" } + +func (k KeychainProvider) Writer() bool { return k.Keychain } + +func (k KeychainProvider) Load(ctx context.Context, ref Ref, key *url.URL) ([]byte, error) { + value, err := keyring.Get(k.serviceName(ref), key.Host) + if err != nil { + if errors.Is(err, keyring.ErrNotFound) { + return nil, fmt.Errorf("no keychain entry for %q: %w", key.Host, ErrNotFound) + } + return nil, err + } + return []byte(value), nil +} + +func (k KeychainProvider) Store(ctx context.Context, ref Ref, value []byte) (*url.URL, error) { + err := keyring.Set(k.serviceName(ref), ref.Name, string(value)) + if err != nil { + return nil, err + } + return &url.URL{Scheme: "keychain", Host: ref.Name}, nil +} + +func (k KeychainProvider) Delete(ctx context.Context, ref Ref) error { + err := keyring.Delete(k.serviceName(ref), ref.Name) + if err != nil { + if errors.Is(err, keyring.ErrNotFound) { + return fmt.Errorf("no keychain entry for %q: %w", ref, ErrNotFound) + } + return err + } + return nil +} + +func (k KeychainProvider) serviceName(ref Ref) string { + return "ftl-secret-" + strings.ReplaceAll(ref.String(), ".", "-") +} diff --git a/common/configuration/manager.go b/common/configuration/manager.go new file mode 100644 index 0000000000..ec89d48566 --- /dev/null +++ b/common/configuration/manager.go @@ -0,0 +1,118 @@ +package configuration + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" +) + +// Manager is a high-level configuration manager that abstracts the details of +// the Resolver and Provider interfaces. +type Manager struct { + providers map[string]Provider + writer MutableProvider + resolver Resolver +} + +// New configuration manager. +func New(ctx context.Context, resolver Resolver, providers []Provider) (*Manager, error) { + m := &Manager{ + providers: map[string]Provider{}, + } + for _, p := range providers { + m.providers[p.Key()] = p + if mutable, ok := p.(MutableProvider); ok && mutable.Writer() { + if m.writer != nil { + return nil, fmt.Errorf("multiple writers %s and %s", m.writer.Key(), p.Key()) + } + m.writer = mutable + } + } + m.resolver = resolver + return m, nil +} + +// Mutable returns an error if the configuration manager doesn't have a +// writeable provider configured. +func (m *Manager) Mutable() error { + if m.writer != nil { + return nil + } + writers := []string{} + for _, p := range m.providers { + if mutable, ok := p.(MutableProvider); ok { + writers = append(writers, "--"+mutable.Key()) + } + } + return fmt.Errorf("no writeable configuration provider available, specify one of %s", strings.Join(writers, ", ")) +} + +// Get a configuration value from the active providers. +// +// "value" must be a pointer to a Go type that can be unmarshalled from JSON. +func (m *Manager) Get(ctx context.Context, ref Ref, value any) error { + key, err := m.resolver.Get(ctx, ref) + if err != nil { + return err + } + provider, ok := m.providers[key.Scheme] + if !ok { + return fmt.Errorf("no provider for scheme %q", key.Scheme) + } + data, err := provider.Load(ctx, ref, key) + if err != nil { + return fmt.Errorf("%s: %w", ref, err) + } + return json.Unmarshal(data, value) +} + +// Set a configuration value in the active writing provider. +// +// "value" must be a Go type that can be marshalled to JSON. +func (m *Manager) Set(ctx context.Context, ref Ref, value any) error { + if err := m.Mutable(); err != nil { + return err + } + data, err := json.Marshal(value) + if err != nil { + return err + } + key, err := m.writer.Store(ctx, ref, data) + if err != nil { + return err + } + return m.resolver.Set(ctx, ref, key) +} + +// Unset a configuration value in all providers. +func (m *Manager) Unset(ctx context.Context, ref Ref) error { + for _, provider := range m.providers { + if mutable, ok := provider.(MutableProvider); ok { + if err := mutable.Delete(ctx, ref); err != nil && !errors.Is(err, ErrNotFound) { + return err + } + } + } + return m.resolver.Unset(ctx, ref) +} + +func (m *Manager) List(ctx context.Context) ([]Entry, error) { + entries := []Entry{} + for _, provider := range m.providers { + if resolver, ok := provider.(Resolver); ok { + subentries, err := resolver.List(ctx) + if err != nil { + return nil, fmt.Errorf("%s: %w", provider.Key(), err) + } + entries = append(entries, subentries...) + } + } + subentries, err := m.resolver.List(ctx) + if err != nil { + return nil, err + } + entries = append(entries, subentries...) + return entries, nil +} diff --git a/common/configuration/manager_test.go b/common/configuration/manager_test.go new file mode 100644 index 0000000000..aafb356525 --- /dev/null +++ b/common/configuration/manager_test.go @@ -0,0 +1,88 @@ +package configuration + +import ( + "context" + "net/url" + "os" + "path/filepath" + "testing" + + "github.com/alecthomas/assert/v2" + "github.com/zalando/go-keyring" + + "github.com/TBD54566975/ftl/internal/log" +) + +func TestManager(t *testing.T) { + keyring.MockInit() // There's no way to undo this :\ + + config := filepath.Join(t.TempDir(), "ftl-project.toml") + existing, err := os.ReadFile("testdata/ftl-project.toml") + assert.NoError(t, err) + err = os.WriteFile(config, existing, 0600) + assert.NoError(t, err) + + ctx := log.ContextWithNewDefaultLogger(context.Background()) + cf, err := New(ctx, + ProjectConfigResolver[FromConfig]{Config: config}, + []Provider{ + EnvarProvider[EnvarTypeConfig]{}, + InlineProvider{Inline: true}, // Writer + KeychainProvider{}, + }) + assert.NoError(t, err) + + actual, err := cf.List(ctx) + assert.NoError(t, err) + + expected := []Entry{ + {Ref: Ref{Name: "baz"}, Accessor: URL("envar://baz")}, + {Ref: Ref{Name: "foo"}, Accessor: URL("inline://ImJhciI")}, + {Ref: Ref{Name: "keychain"}, Accessor: URL("keychain://keychain")}, + } + + assert.Equal(t, expected, actual) + + // Try to get value from missing envar + var bazValue map[string]string + + err = cf.Get(ctx, Ref{Name: "baz"}, &bazValue) + assert.IsError(t, err, ErrNotFound) + + // Set the envar and try again. + t.Setenv("FTL_CONFIG_YmF6", "eyJiYXoiOiJ3YXoifQ") // baz={"baz": "waz"} + + err = cf.Get(ctx, Ref{Name: "baz"}, &bazValue) + assert.NoError(t, err) + assert.Equal(t, map[string]string{"baz": "waz"}, bazValue) + + var fooValue string + err = cf.Get(ctx, Ref{Name: "foo"}, &fooValue) + assert.NoError(t, err) + assert.Equal(t, "bar", fooValue) + + err = cf.Get(ctx, Ref{Name: "nonexistent"}, &fooValue) + assert.IsError(t, err, ErrNotFound) + + // Change value. + err = cf.Set(ctx, Ref{Name: "foo"}, "hello") + assert.NoError(t, err) + + err = cf.Get(ctx, Ref{Name: "foo"}, &fooValue) + assert.NoError(t, err) + assert.Equal(t, "hello", fooValue) + + // Delete value + err = cf.Unset(ctx, Ref{Name: "foo"}) + assert.NoError(t, err) + err = cf.Get(ctx, Ref{Name: "foo"}, &fooValue) + assert.IsError(t, err, ErrNotFound) +} + +func URL(s string) *url.URL { + u, err := url.Parse(s) + if err != nil { + panic(err) + } + return u +} diff --git a/common/configuration/projectconfig.go b/common/configuration/projectconfig.go new file mode 100644 index 0000000000..ec16f05d9a --- /dev/null +++ b/common/configuration/projectconfig.go @@ -0,0 +1,161 @@ +package configuration + +import ( + "context" + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + "sort" + + "github.com/alecthomas/types/optional" + "golang.org/x/exp/maps" + + pc "github.com/TBD54566975/ftl/common/projectconfig" + "github.com/TBD54566975/ftl/internal" +) + +type FromConfigOrSecrets interface { + get(config pc.ConfigAndSecrets) map[string]*pc.URL + set(config *pc.ConfigAndSecrets, mapping map[string]*pc.URL) +} + +type FromConfig struct{} + +func (f FromConfig) get(config pc.ConfigAndSecrets) map[string]*pc.URL { return config.Config } + +func (f FromConfig) set(config *pc.ConfigAndSecrets, mapping map[string]*pc.URL) { + config.Config = mapping +} + +type FromSecrets struct{} + +func (f FromSecrets) get(config pc.ConfigAndSecrets) map[string]*pc.URL { return config.Secrets } + +func (f FromSecrets) set(config *pc.ConfigAndSecrets, mapping map[string]*pc.URL) { + config.Secrets = mapping +} + +// ProjectConfigResolver is parametric Resolver that loads values from either a +// project's configuration or secrets maps based on the type parameter. +type ProjectConfigResolver[From FromConfigOrSecrets] struct { + Config string `help:"Load project configuration from TOML file." placeholder:"FILE" type:"existingfile"` +} + +var _ Resolver = (*ProjectConfigResolver[FromConfig])(nil) + +func (p ProjectConfigResolver[T]) Get(ctx context.Context, ref Ref) (*url.URL, error) { + mapping, err := p.getMapping(ref.Module) + if err != nil { + return nil, err + } + key, ok := mapping[ref.Name] + if !ok { + return nil, fmt.Errorf("no such key %q: %w", ref.Name, ErrNotFound) + } + return (*url.URL)(key), nil +} + +func (p ProjectConfigResolver[T]) List(ctx context.Context) ([]Entry, error) { + config, err := p.loadConfig() + if err != nil { + return nil, err + } + entries := []Entry{} + moduleNames := maps.Keys(config.Modules) + moduleNames = append(moduleNames, "") + for _, moduleName := range moduleNames { + module := optional.Zero(moduleName) + mapping, err := p.getMapping(module) + if err != nil { + return nil, err + } + for name, key := range mapping { + entries = append(entries, Entry{ + Ref: Ref{module, name}, + Accessor: (*url.URL)(key), + }) + } + } + sort.Slice(entries, func(i, j int) bool { + im, _ := entries[i].Module.Get() + jm, _ := entries[j].Module.Get() + return im < jm || (im == jm && entries[i].Name < entries[j].Name) + }) + return entries, nil +} + +func (p ProjectConfigResolver[T]) Set(ctx context.Context, ref Ref, key *url.URL) error { + mapping, err := p.getMapping(ref.Module) + if err != nil { + return err + } + mapping[ref.Name] = (*pc.URL)(key) + return p.setMapping(ref.Module, mapping) +} + +func (p ProjectConfigResolver[From]) Unset(ctx context.Context, ref Ref) error { + mapping, err := p.getMapping(ref.Module) + if err != nil { + return err + } + delete(mapping, ref.Name) + return p.setMapping(ref.Module, mapping) +} + +func (p ProjectConfigResolver[T]) configPath() string { + if p.Config != "" { + return p.Config + } + return filepath.Join(internal.GitRoot("."), "ftl-project.toml") +} + +func (p ProjectConfigResolver[T]) loadConfig() (pc.Config, error) { + configPath := p.configPath() + config, err := pc.Load(configPath) + if errors.Is(err, os.ErrNotExist) { + return pc.Config{}, nil + } else if err != nil { + return pc.Config{}, err + } + return config, nil +} + +func (p ProjectConfigResolver[T]) getMapping(module optional.Option[string]) (map[string]*pc.URL, error) { + config, err := p.loadConfig() + if err != nil { + return nil, err + } + var t T + if m, ok := module.Get(); ok { + if config.Modules == nil { + return map[string]*pc.URL{}, nil + } + return t.get(config.Modules[m]), nil + } + mapping := t.get(config.Global) + if mapping == nil { + mapping = map[string]*pc.URL{} + } + return mapping, nil +} + +func (p ProjectConfigResolver[T]) setMapping(module optional.Option[string], mapping map[string]*pc.URL) error { + config, err := p.loadConfig() + if err != nil { + return err + } + var t T + if m, ok := module.Get(); ok { + if config.Modules == nil { + config.Modules = map[string]pc.ConfigAndSecrets{} + } + moduleConfig := config.Modules[m] + t.set(&moduleConfig, mapping) + config.Modules[m] = moduleConfig + } else { + t.set(&config.Global, mapping) + } + return pc.Save(p.configPath(), config) +} diff --git a/common/configuration/testdata/ftl-project.toml b/common/configuration/testdata/ftl-project.toml new file mode 100644 index 0000000000..2d7a31cc7f --- /dev/null +++ b/common/configuration/testdata/ftl-project.toml @@ -0,0 +1,5 @@ +[global] +[global.configuration] +baz = "envar://baz" +foo = "inline://ImJhciI" +keychain = "keychain://keychain" diff --git a/common/projectconfig/projectconfig.go b/common/projectconfig/projectconfig.go new file mode 100644 index 0000000000..33d9729087 --- /dev/null +++ b/common/projectconfig/projectconfig.go @@ -0,0 +1,53 @@ +package projectconfig + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/BurntSushi/toml" +) + +type ConfigAndSecrets struct { + Config map[string]*URL `toml:"configuration"` + Secrets map[string]*URL `toml:"secrets"` +} + +type Config struct { + Global ConfigAndSecrets `toml:"global"` + Modules map[string]ConfigAndSecrets `toml:"modules"` +} + +// Load project config from a file. +func Load(path string) (Config, error) { + config := Config{} + md, err := toml.DecodeFile(path, &config) + if err != nil { + return Config{}, err + } + if len(md.Undecoded()) > 0 { + keys := make([]string, len(md.Undecoded())) + for i, key := range md.Undecoded() { + keys[i] = key.String() + } + return Config{}, fmt.Errorf("unknown configuration keys: %s", strings.Join(keys, ", ")) + } + return config, nil +} + +// Save project config atomically to a file. +func Save(path string, config Config) error { + w, err := os.CreateTemp(filepath.Dir(path), filepath.Base(path)) + if err != nil { + return err + } + defer os.Remove(w.Name()) //nolint:errcheck + defer w.Close() //nolint:errcheck + + enc := toml.NewEncoder(w) + if err := enc.Encode(config); err != nil { + return err + } + return os.Rename(w.Name(), path) +} diff --git a/common/projectconfig/projectconfig_test.go b/common/projectconfig/projectconfig_test.go new file mode 100644 index 0000000000..9e3864b729 --- /dev/null +++ b/common/projectconfig/projectconfig_test.go @@ -0,0 +1,27 @@ +package projectconfig + +import ( + "testing" + + "github.com/alecthomas/assert/v2" +) + +func TestProjectConfig(t *testing.T) { + actual, err := Load("testdata/ftl-project.toml") + assert.NoError(t, err) + expected := Config{ + Modules: map[string]ConfigAndSecrets{ + "module": { + Config: map[string]*URL{ + "githubAccessToken": MustParseURL("keychain://githubAccessToken"), + }, + Secrets: map[string]*URL{ + "companyApiKey": MustParseURL("op://devel/yj3jfj2vzsbiwqabprflnl27lm/companyApiKey"), + "encryptionKey": MustParseURL("inline://notASensitiveSecret"), + }, + }, + }, + } + + assert.Equal(t, expected, actual) +} diff --git a/common/projectconfig/testdata/ftl-project.toml b/common/projectconfig/testdata/ftl-project.toml new file mode 100644 index 0000000000..68b6645ef8 --- /dev/null +++ b/common/projectconfig/testdata/ftl-project.toml @@ -0,0 +1,6 @@ +[modules.module.configuration] +githubAccessToken = "keychain://githubAccessToken" + +[modules.module.secrets] +encryptionKey = "inline://notASensitiveSecret" +companyApiKey = "op://devel/yj3jfj2vzsbiwqabprflnl27lm/companyApiKey" diff --git a/common/projectconfig/url.go b/common/projectconfig/url.go new file mode 100644 index 0000000000..1503f97cd5 --- /dev/null +++ b/common/projectconfig/url.go @@ -0,0 +1,43 @@ +package projectconfig + +import ( + "fmt" + "net/url" +) + +// A URL that supports marshalling and unmarshalling which is, very +// frustratingly, not possible with the stdlib url.URL. +type URL url.URL + +func ParseURL(rawurl string) (*URL, error) { + parsed, err := url.Parse(rawurl) + if err != nil { + return nil, err + } + return (*URL)(parsed), nil +} + +func MustParseURL(rawurl string) *URL { + parsed, err := ParseURL(rawurl) + if err != nil { + panic(err) + } + return parsed +} + +func (u *URL) UnmarshalText(text []byte) error { + parsed, err := url.Parse(string(text)) + if err != nil { + return err + } + *u = URL(*parsed) + return nil +} + +func (u URL) MarshalText() ([]byte, error) { + return []byte((*url.URL)(&u).String()), nil +} + +func (u URL) GoString() string { + return fmt.Sprintf("projectconfig.MustParseURL(%q)", (*url.URL)(&u).String()) +} diff --git a/go.mod b/go.mod index 4438b16344..42ae841e88 100644 --- a/go.mod +++ b/go.mod @@ -51,6 +51,7 @@ require ( golang.org/x/mod v0.15.0 golang.org/x/net v0.21.0 golang.org/x/sync v0.6.0 + golang.org/x/term v0.17.0 golang.org/x/tools v0.18.0 google.golang.org/protobuf v1.32.0 modernc.org/sqlite v1.29.2 diff --git a/go.sum b/go.sum index a70839f9c1..d1b796a0b8 100644 --- a/go.sum +++ b/go.sum @@ -244,6 +244,8 @@ golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=