From 5a60f47b755762f9b5fcf1fccf501928b0831f28 Mon Sep 17 00:00:00 2001 From: Wes Date: Tue, 17 Oct 2023 16:15:09 -0700 Subject: [PATCH] feat: add 'ftl serve' command for local dev (#497) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #470 ```bash ftl serve --help Usage: ftl serve Start the FTL server. Flags: -h, --help Show context-sensitive help. --version Show version. --config=FILE Load configuration from TOML file. --endpoint=http://127.0.0.1:8892 FTL endpoint to bind/connect to ($FTL_ENDPOINT). --authenticators=HOST=EXE,… Authenticators to use for FTL endpoints ($FTL_AUTHENTICATORS). Logging: --log-level=info Log level ($LOG_LEVEL). --log-json Log in JSON format ($LOG_JSON). Command flags: --bind=http://localhost:8892 Starting endpoint to bind to and advertise to. Each controller and runner will increment the port by 1 -c, --num-controllers=1 Number of controllers to start. -r, --num-runners=10 Number of runners to start. ``` --------- Co-authored-by: github-actions[bot] --- backend/runner/runner.go | 2 +- cmd/ftl/cmd_serve.go | 102 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 101 insertions(+), 3 deletions(-) diff --git a/backend/runner/runner.go b/backend/runner/runner.go index ad788b3d8c..f0e8047978 100644 --- a/backend/runner/runner.go +++ b/backend/runner/runner.go @@ -41,7 +41,7 @@ type Config struct { ControllerEndpoint *url.URL `name:"ftl-endpoint" help:"Controller endpoint." env:"FTL_ENDPOINT" default:"http://localhost:8892"` TemplateDir string `help:"Template directory to copy into each deployment, if any." type:"existingdir"` DeploymentDir string `help:"Directory to store deployments in." default:"${deploymentdir}"` - Language []string `short:"l" help:"Languages the runner supports." env:"FTL_LANGUAGE" required:""` + Language []string `short:"l" help:"Languages the runner supports." env:"FTL_LANGUAGE" default:"go,kotlin"` HeartbeatPeriod time.Duration `help:"Minimum period between heartbeats." default:"3s"` HeartbeatJitter time.Duration `help:"Jitter to add to heartbeat period." default:"2s"` } diff --git a/cmd/ftl/cmd_serve.go b/cmd/ftl/cmd_serve.go index a6a1480cd2..c9b0853d53 100644 --- a/cmd/ftl/cmd_serve.go +++ b/cmd/ftl/cmd_serve.go @@ -2,14 +2,112 @@ package main import ( "context" + "encoding/binary" + "fmt" + "net/url" + "os" + "path/filepath" + "strconv" + "github.com/alecthomas/errors" + "github.com/alecthomas/kong" + "golang.org/x/sync/errgroup" + + "github.com/TBD54566975/ftl/backend/common/log" "github.com/TBD54566975/ftl/backend/controller" + "github.com/TBD54566975/ftl/backend/runner" ) type serveCmd struct { - controller.Config + Bind *url.URL `help:"Starting endpoint to bind to and advertise to. Each controller and runner will increment the port by 1" default:"http://localhost:8892"` + Controllers int `short:"c" help:"Number of controllers to start." default:"1"` + Runners int `short:"r" help:"Number of runners to start." default:"10"` } func (s *serveCmd) Run(ctx context.Context) error { - return controller.Start(ctx, s.Config) + logger := log.FromContext(ctx) + logger.Infof("Starting %d controller(s) and %d runner(s)", s.Controllers, s.Runners) + + wg, ctx := errgroup.WithContext(ctx) + + controllerAddresses := make([]*url.URL, 0, s.Controllers) + nextBind := s.Bind + + for i := 0; i < s.Controllers; i++ { + controllerAddresses = append(controllerAddresses, nextBind) + config := controller.Config{ + Bind: nextBind, + } + if err := kong.ApplyDefaults(&config); err != nil { + return errors.WithStack(err) + } + + scope := fmt.Sprintf("controller-%d", i) + controllerCtx := log.ContextWithLogger(ctx, logger.Scope(scope)) + + wg.Go(func() error { return controller.Start(controllerCtx, config) }) + + var err error + nextBind, err = incrementPort(nextBind) + if err != nil { + return errors.WithStack(err) + } + } + + cacheDir, err := os.UserCacheDir() + if err != nil { + return errors.WithStack(err) + } + + for i := 0; i < s.Runners; i++ { + controllerEndpoint := controllerAddresses[i%len(controllerAddresses)] + fmt.Printf("controllerEndpoint: %s runner: %s\n", controllerEndpoint, nextBind) + config := runner.Config{ + Bind: nextBind, + ControllerEndpoint: controllerEndpoint, + } + + name := fmt.Sprintf("runner%d", i) + if err := kong.ApplyDefaults(&config, kong.Vars{ + "deploymentdir": filepath.Join(cacheDir, "ftl-runner", name, "deployments"), + "language": "go,kotlin", + }); err != nil { + return errors.WithStack(err) + } + + // Create a readable ULID for the runner. + var ulid [16]byte + binary.BigEndian.PutUint32(ulid[10:], uint32(i)) + ulidStr := fmt.Sprintf("%025X", ulid) + err := config.Key.Scan(ulidStr) + if err != nil { + return errors.WithStack(err) + } + + runnerCtx := log.ContextWithLogger(ctx, logger.Scope(name)) + + wg.Go(func() error { return runner.Start(runnerCtx, config) }) + + nextBind, err = incrementPort(nextBind) + if err != nil { + return errors.WithStack(err) + } + } + + if err := wg.Wait(); err != nil { + return errors.WithStack(err) + } + return nil +} + +func incrementPort(baseURL *url.URL) (*url.URL, error) { + newURL := *baseURL + + newPort, err := strconv.Atoi(newURL.Port()) + if err != nil { + return nil, errors.WithStack(err) + } + + newURL.Host = fmt.Sprintf("%s:%d", baseURL.Hostname(), newPort+1) + return &newURL, nil }