Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: add flags to ftl-go for multi-platform support #358

Merged
merged 1 commit into from
Sep 6, 2023
Merged

fix: add flags to ftl-go for multi-platform support #358

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion backend/controller/controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ func New(ctx context.Context, db *dal.DAL, config Config) (*Service, error) {
go runWithRetries(ctx, time.Second*10, time.Second*20, svc.reapStaleControllers)
go runWithRetries(ctx, config.RunnerTimeout, time.Second*10, svc.reapStaleRunners)
go runWithRetries(ctx, config.DeploymentReservationTimeout, time.Second*20, svc.releaseExpiredReservations)
go runWithRetries(ctx, config.RunnerTimeout, time.Second*10, svc.reconcileDeployments)
go runWithRetries(ctx, time.Second*1, time.Second*5, svc.reconcileDeployments)
return svc, nil
}

Expand Down
86 changes: 52 additions & 34 deletions cmd/ftl-go/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,15 +36,15 @@ import (

type watchCmd struct{}

func (w *watchCmd) Run(ctx context.Context, c *cli, client ftlv1connect.ControllerServiceClient, importRoot ImportRoot) error {
err := buildRemoteModules(ctx, client, c.Root, importRoot)
func (w *watchCmd) Run(ctx context.Context, c *cli, client ftlv1connect.ControllerServiceClient, bctx BuildContext) error {
err := buildRemoteModules(ctx, client, bctx)
if err != nil {
return errors.WithStack(err)
}

wg, ctx := errgroup.WithContext(ctx)
wg.Go(func() error { return pullModules(ctx, client, c.Root, importRoot) })
wg.Go(func() error { return pushModules(ctx, client, c.Root, c.WatchFrequency, importRoot) })
wg.Go(func() error { return pullModules(ctx, client, bctx) })
wg.Go(func() error { return pushModules(ctx, client, c.WatchFrequency, bctx) })

if err := wg.Wait(); err != nil {
return errors.WithStack(err)
Expand All @@ -56,31 +56,48 @@ type deployCmd struct {
Name string `arg:"" required:"" help:"Name of module to deploy."`
}

func (d *deployCmd) Run(ctx context.Context, c *cli, client ftlv1connect.ControllerServiceClient, importRoot ImportRoot) error {
return errors.WithStack(pushModule(ctx, client, filepath.Join(c.Root, d.Name), importRoot))
func (d *deployCmd) Run(ctx context.Context, c *cli, client ftlv1connect.ControllerServiceClient, bctx BuildContext) error {
return errors.WithStack(pushModule(ctx, client, filepath.Join(c.Root, d.Name), bctx))
}

type cli struct {
LogConfig log.Config `embed:""`
FTL string `env:"FTL_ENDPOINT" help:"FTL endpoint to connect to." default:"http://localhost:8892"`
WatchFrequency time.Duration `short:"w" default:"500ms" help:"Frequency to watch for changes to local FTL modules."`
Root string `short:"r" type:"existingdir" help:"Root directory to sync FTL modules into." default:"."`
OS string `short:"o" help:"OS to build for." env:"GOOS"`
Arch string `short:"a" help:"Architecture to build for." env:"GOARCH"`

Watch watchCmd `cmd:"" default:"" help:"Watch for and rebuild local and remote FTL modules."`
Deploy deployCmd `cmd:"" help:"Deploy a local FTL module."`
}

type BuildContext struct {
OS string
Arch string
Root string
ImportRoot
}

func main() {
c := &cli{}
kctx := kong.Parse(c)

client := rpc.Dial(ftlv1connect.NewControllerServiceClient, c.FTL, log.Warn)
logger := log.Configure(os.Stderr, c.LogConfig)
ctx := log.ContextWithLogger(context.Background(), logger)

importRoot, err := findImportRoot(c.Root)
kctx.FatalIfErrorf(err)

kctx.Bind(importRoot)
bctx := BuildContext{
OS: c.OS,
Arch: c.Arch,
Root: c.Root,
ImportRoot: importRoot,
}

kctx.Bind(bctx)
kctx.BindTo(ctx, (*context.Context)(nil))
kctx.BindTo(client, (*ftlv1connect.ControllerServiceClient)(nil))
err = kctx.Run()
Expand Down Expand Up @@ -124,23 +141,23 @@ func findImportRoot(root string) (importRoot ImportRoot, err error) {
}, nil
}

func pushModules(ctx context.Context, client ftlv1connect.ControllerServiceClient, root string, watchFrequency time.Duration, importRoot ImportRoot) error {
func pushModules(ctx context.Context, client ftlv1connect.ControllerServiceClient, watchFrequency time.Duration, bctx BuildContext) error {
logger := log.FromContext(ctx)
entries, err := os.ReadDir(root)
entries, err := os.ReadDir(bctx.Root)
if err != nil {
return errors.Wrap(err, "failed to read root directory")
}
for _, entry := range entries {
if !entry.IsDir() {
continue
}
dir := filepath.Join(root, entry.Name())
dir := filepath.Join(bctx.Root, entry.Name())
if _, err := os.Stat(filepath.Join(dir, "generated_ftl_module.go")); err == nil {
continue
}

logger.Infof("Pushing local FTL module %q", entry.Name())
err := pushModule(ctx, client, dir, importRoot)
err := pushModule(ctx, client, dir, bctx)
if err != nil {
if connect.CodeOf(err) == connect.CodeAlreadyExists {
logger.Infof("Module %q already exists, skipping", entry.Name())
Expand All @@ -150,7 +167,7 @@ func pushModules(ctx context.Context, client ftlv1connect.ControllerServiceClien
}
}

logger.Infof("Watching %s for changes", root)
logger.Infof("Watching %s for changes", bctx.Root)
wg, ctx := errgroup.WithContext(ctx)
watch := watcher.New()
defer watch.Close()
Expand All @@ -164,15 +181,15 @@ func pushModules(ctx context.Context, client ftlv1connect.ControllerServiceClien
if event.IsDir() ||
strings.Contains(event.Path, "/.") ||
strings.Contains(event.Path, "/generated_ftl_module.go") ||
!strings.HasPrefix(event.Path, root) ||
!strings.HasPrefix(event.Path, bctx.Root) ||
strings.Contains(event.Path, "/build/") {
continue
}
dir := strings.TrimPrefix(event.Path, root+"/")
dir = filepath.Join(root, strings.Split(dir, "/")[0])
dir := strings.TrimPrefix(event.Path, bctx.Root+"/")
dir = filepath.Join(bctx.Root, strings.Split(dir, "/")[0])
logger.Infof("Detected change to %s, pushing module", dir)

err := pushModule(ctx, client, dir, importRoot)
err := pushModule(ctx, client, dir, bctx)
if err != nil {
logger.Errorf(err, "failed to rebuild module")
}
Expand All @@ -182,15 +199,15 @@ func pushModules(ctx context.Context, client ftlv1connect.ControllerServiceClien
}
}
})
err = watch.AddRecursive(root)
err = watch.AddRecursive(bctx.Root)
if err != nil {
return errors.Wrap(err, "failed to watch root directory")
}
wg.Go(func() error { return errors.WithStack(watch.Start(watchFrequency)) })
return errors.WithStack(wg.Wait())
}

func pushModule(ctx context.Context, client ftlv1connect.ControllerServiceClient, dir string, importRoot ImportRoot) error {
func pushModule(ctx context.Context, client ftlv1connect.ControllerServiceClient, dir string, bctx BuildContext) error {
logger := log.FromContext(ctx)

sch, err := compile.ExtractModuleSchema(dir)
Expand All @@ -203,13 +220,14 @@ func pushModule(ctx context.Context, client ftlv1connect.ControllerServiceClient
return nil
}

tmpDir, err := generateBuildDir(dir, sch, importRoot)
tmpDir, err := generateBuildDir(dir, sch, bctx)
if err != nil {
return errors.Wrap(err, "failed to generate build directory")
}

logger.Infof("Building module %s in %s", sch.Name, tmpDir)
cmd := exec.Command(ctx, log.Info, tmpDir, "go", "build", "-o", "main", "-trimpath", "-ldflags=-s -w -buildid=", ".")
cmd.Env = append(cmd.Environ(), "GOOS="+bctx.OS, "GOARCH="+bctx.Arch, "CGO_ENABLED=0")
if err := cmd.Run(); err != nil {
return errors.Wrap(err, "failed to build module")
}
Expand Down Expand Up @@ -317,7 +335,7 @@ func uploadArtefacts(ctx context.Context, client ftlv1connect.ControllerServiceC
return nil
}

func generateBuildDir(dir string, sch *schema.Module, importRoot ImportRoot) (string, error) {
func generateBuildDir(dir string, sch *schema.Module, bctx BuildContext) (string, error) {
cacheDir, err := os.UserCacheDir()
if err != nil {
return "", errors.Wrap(err, "failed to get user cache directory")
Expand All @@ -328,20 +346,20 @@ func generateBuildDir(dir string, sch *schema.Module, importRoot ImportRoot) (st
return "", errors.Wrap(err, "failed to create build directory")
}
mainFile := filepath.Join(tmpDir, "main.go")
if err := generate.File(mainFile, importRoot.FTLBasePkg, generate.Main, sch); err != nil {
if err := generate.File(mainFile, bctx.FTLBasePkg, generate.Main, sch); err != nil {
return "", errors.Wrap(err, "failed to generate main.go")
}
goWorkFile := filepath.Join(tmpDir, "go.work")
if err := generate.File(goWorkFile, importRoot.FTLBasePkg, generate.GenerateGoWork, []string{
importRoot.GoModuleDir,
if err := generate.File(goWorkFile, bctx.FTLBasePkg, generate.GenerateGoWork, []string{
bctx.GoModuleDir,
}); err != nil {
return "", errors.Wrap(err, "failed to generate go.work")
}
goModFile := filepath.Join(tmpDir, "go.mod")
replace := map[string]string{
importRoot.Module.Module.Mod.Path: importRoot.GoModuleDir,
bctx.Module.Module.Mod.Path: bctx.GoModuleDir,
}
if err := generate.File(goModFile, importRoot.FTLBasePkg, generate.GenerateGoMod, generate.GoModConfig{
if err := generate.File(goModFile, bctx.FTLBasePkg, generate.GenerateGoMod, generate.GoModConfig{
Replace: replace,
}); err != nil {
return "", errors.Wrap(err, "failed to generate go.mod")
Expand All @@ -358,53 +376,53 @@ func hasVerbs(sch *schema.Module) bool {
return false
}

func pullModules(ctx context.Context, client ftlv1connect.ControllerServiceClient, root string, importRoot ImportRoot) error {
func pullModules(ctx context.Context, client ftlv1connect.ControllerServiceClient, bctx BuildContext) error {
resp, err := client.PullSchema(ctx, connect.NewRequest(&ftlv1.PullSchemaRequest{}))
if err != nil {
return errors.Wrap(err, "failed to pull schema")
}
for resp.Receive() {
msg := resp.Msg()
err = generateModuleFromSchema(ctx, msg.Schema, root, importRoot)
err = generateModuleFromSchema(ctx, msg.Schema, bctx)
if err != nil {
return errors.Wrap(err, "failed to sync module")
}
}
return errors.Wrap(resp.Err(), "failed to pull schema")
}

func buildRemoteModules(ctx context.Context, client ftlv1connect.ControllerServiceClient, root string, importRoot ImportRoot) error {
func buildRemoteModules(ctx context.Context, client ftlv1connect.ControllerServiceClient, bctx BuildContext) error {
fullSchema, err := client.GetSchema(ctx, connect.NewRequest(&ftlv1.GetSchemaRequest{}))
if err != nil {
return errors.Wrap(err, "failed to retrieve schema")
}
for _, module := range fullSchema.Msg.Schema.Modules {
err := generateModuleFromSchema(ctx, module, root, importRoot)
err := generateModuleFromSchema(ctx, module, bctx)
if err != nil {
return errors.Wrap(err, "failed to generate module")
}
}
return err
}

func generateModuleFromSchema(ctx context.Context, msg *pschema.Module, root string, importRoot ImportRoot) error {
func generateModuleFromSchema(ctx context.Context, msg *pschema.Module, bctx BuildContext) error {
sch, err := schema.ModuleFromProto(msg)
if err != nil {
return errors.Wrap(err, "failed to parse schema")
}
dir := filepath.Join(root, sch.Name)
dir := filepath.Join(bctx.Root, sch.Name)
if _, err := os.Stat(dir); err == nil {
if _, err = os.Stat(filepath.Join(dir, "generated_ftl_module.go")); errors.Is(err, os.ErrNotExist) {
return nil
}
}
if err := generateModule(ctx, dir, sch, importRoot); err != nil {
if err := generateModule(ctx, dir, sch, bctx); err != nil {
return errors.Wrap(err, "failed to generate module")
}
return nil
}

func generateModule(ctx context.Context, dir string, sch *schema.Module, importRoot ImportRoot) error {
func generateModule(ctx context.Context, dir string, sch *schema.Module, bctx BuildContext) error {
logger := log.FromContext(ctx)
logger.Infof("Generating stubs for FTL module %s", sch.Name)
err := os.MkdirAll(dir, 0750)
Expand All @@ -417,7 +435,7 @@ func generateModule(ctx context.Context, dir string, sch *schema.Module, importR
}
defer w.Close() //nolint:gosec
defer os.Remove(w.Name())
err = generate.ExternalModule(w, sch, importRoot.FTLBasePkg)
err = generate.ExternalModule(w, sch, bctx.FTLBasePkg)
if err != nil {
return errors.Wrap(err, "failed to generate stubs")
}
Expand Down
7 changes: 4 additions & 3 deletions scripts/integration-tests
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,11 +60,13 @@ deploy_echo_kotlin() (
deploy_time_go() (
info "Deploying time"
cd examples
ftl-go deploy time
# Pull a supported platforms from the cluster.
platform="$(ftl status | jq -r '.runners[].labels | "\(.os)-\(.arch)"' | sort | uniq | head -1)"
ftl-go --os "${platform%-*}" --arch "${platform#*-}" deploy time
)

wait_for_deploys() {
wait_for "deployments to come up" '[ "$(ftl ps --json | jq -r .module | sort | paste -sd " " -)" == "echo time" ]'
wait_for "deployments to come up" 'ftl status | jq -r ".routes[].module" | sort | paste -sd " " - | grep -q "echo time"'
}

build_release
Expand All @@ -79,7 +81,6 @@ deploy_echo_kotlin
wait_for_deploys

info "Calling echo"
wait_for "echo to respond" 'ftl call echo.echo'
message="$(ftl call echo.echo '{"name": "Alice"}' | jq -r .message)"
[[ "$message" =~ "Hello, Alice! The time is " ]] || error "Unexpected response from echo: $message"
info "Success!"