fix: embed runner template for ftl serve (#544)

In development mode this will build the template directory first, in
release mode it will embed a zip file containing the template directory.

Fixes #541

Bitfile

@@ -43,6 +43,8 @@ CLIENT_IN = frontend/src/**/*
 NODE_MODULES_OUT = frontend/node_modules
 NODE_MODULES_IN = frontend/package{,-lock}.json
 
+RUNNER_TEMPLATE_ZIP = backend/controller/scaling/localscaling/template.zip
+
 #virtual release:
 #  inputs: %{RELEASE}/ftl %{RELEASE}/ftl-controller %{RELEASE}/ftl-runner
 
@@ -60,7 +62,7 @@ NODE_MODULES_IN = frontend/package{,-lock}.json
 %{RELEASE}/ftl-runner: %{RELEASE} %{GO_SOURCES} cmd/ftl-runner/**/*.go
   build: go build -o %{OUT} -tags release -ldflags "-X main.version=%{VERSION} -X main.timestamp=$(date +%s)" ./cmd/ftl-runner
 
-%{RELEASE}/ftl: %{RELEASE} %{GO_SOURCES} cmd/ftl/**/*.go {go,kotlin}-runtime/scaffolding.zip %{CLIENT_OUT}
+%{RELEASE}/ftl: %{RELEASE} %{GO_SOURCES} cmd/ftl/**/*.go %{CLIENT_OUT} **/*.zip
   build: go build -o %{OUT} -tags release -ldflags "-X main.version=%{VERSION} -X main.timestamp=$(date +%s)" ./cmd/ftl
 
 %{RELEASE}/ftl-initdb: %{RELEASE} %{GO_SOURCES} cmd/ftl-initdb/**/*.go
@@ -107,6 +109,9 @@ kotlin-runtime/scaffolding.zip: kotlin-runtime/scaffolding/**/*
 %{KT_RUNTIME_RUNNER_TEMPLATE_OUT}: %{KT_RUNTIME_OUT} %(dirname %{KT_RUNTIME_RUNNER_TEMPLATE_OUT})%
   build: install -m 0600 %{KT_RUNTIME_OUT} %{OUT}
 
+%{RUNNER_TEMPLATE_ZIP}: %{KT_RUNTIME_RUNNER_TEMPLATE_OUT}
+  build: cd build/template && zip -q --symlinks -r ../../%{OUT} .
+
 %{COMMON_LOG_OUT}: %{COMMON_LOG_IN}
   build: go generate %{IN}
 

backend/controller/scaling/localscaling/devel.go

@@ -0,0 +1,26 @@
+//go:build !release
+
+package localscaling
+
+import (
+	"context"
+	"path/filepath"
+	"sync"
+
+	"github.com/TBD54566975/ftl/backend/common/exec"
+	"github.com/TBD54566975/ftl/backend/common/log"
+	"github.com/TBD54566975/ftl/internal"
+)
+
+var templateDirOnce sync.Once
+
+func templateDir(ctx context.Context) string {
+	templateDirOnce.Do(func() {
+		cmd := exec.Command(ctx, log.Info, internal.FTLSourceRoot(), "bit", "build/template/ftl/jars/ftl-runtime.jar")
+		err := cmd.Run()
+		if err != nil {
+			panic(err)
+		}
+	})
+	return filepath.Join(internal.FTLSourceRoot(), "build/template")
+}

backend/controller/scaling/local_scaling.go → backend/controller/scaling/localscaling/local_scaling.go

@@ -1,4 +1,4 @@
-package scaling
+package localscaling
 
 import (
 	"context"
@@ -15,10 +15,11 @@ import (
 	"github.com/TBD54566975/ftl/backend/common/bind"
 	"github.com/TBD54566975/ftl/backend/common/log"
 	"github.com/TBD54566975/ftl/backend/common/model"
+	"github.com/TBD54566975/ftl/backend/controller/scaling"
 	"github.com/TBD54566975/ftl/backend/runner"
 )
 
-var _ RunnerScaling = (*LocalScaling)(nil)
+var _ scaling.RunnerScaling = (*LocalScaling)(nil)
 
 type LocalScaling struct {
 	lock     sync.Mutex
@@ -78,6 +79,7 @@ func (l *LocalScaling) SetReplicas(ctx context.Context, replicas int, idleRunner
 		config := runner.Config{
 			Bind:               l.portAllocator.Next(),
 			ControllerEndpoint: controllerEndpoint,
+			TemplateDir:        templateDir(ctx),
 		}
 
 		name := fmt.Sprintf("runner%d", i)

backend/controller/scaling/localscaling/release.go

@@ -0,0 +1,40 @@
+//go:build release
+
+package localscaling
+
+import (
+	"archive/zip"
+	"bytes"
+	"context"
+	_ "embed"
+	"os"
+	"path/filepath"
+	"sync"
+
+	"github.com/TBD54566975/ftl/internal"
+)
+
+//go:embed template.zip
+var archive []byte
+
+var templateDirOnce sync.Once
+
+func templateDir(ctx context.Context) string {
+	cacheDir, err := os.UserCacheDir()
+	if err != nil {
+		panic(err)
+	}
+	cacheDir = filepath.Join(cacheDir, "ftl-runner-template")
+	templateDirOnce.Do(func() {
+		_ = os.RemoveAll(cacheDir)
+		zr, err := zip.NewReader(bytes.NewReader(archive), int64(len(archive)))
+		if err != nil {
+			panic(err)
+		}
+		err = internal.UnzipDir(zr, cacheDir)
+		if err != nil {
+			panic(err)
+		}
+	})
+	return cacheDir
+}

cmd/ftl/cmd_serve.go

@@ -16,7 +16,7 @@ import (
 	"github.com/TBD54566975/ftl/backend/common/log"
 	"github.com/TBD54566975/ftl/backend/controller"
 	"github.com/TBD54566975/ftl/backend/controller/databasetesting"
-	"github.com/TBD54566975/ftl/backend/controller/scaling"
+	"github.com/TBD54566975/ftl/backend/controller/scaling/localscaling"
 )
 
 type serveCmd struct {
@@ -50,7 +50,7 @@ func (s *serveCmd) Run(ctx context.Context) error {
 		controllerAddresses = append(controllerAddresses, portAllocator.Next())
 	}
 
-	runnerScaling, err := scaling.NewLocalScaling(portAllocator, controllerAddresses)
+	runnerScaling, err := localscaling.NewLocalScaling(portAllocator, controllerAddresses)
 	if err != nil {
 		return errors.WithStack(err)
 	}

go-runtime/devel.go

@@ -3,45 +3,8 @@
 package goruntime
 
 import (
-	"archive/zip"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strings"
-
 	"github.com/TBD54566975/ftl/internal"
 )
 
 // Files is the FTL Go runtime scaffolding files.
-var Files = func() *zip.Reader {
-	cmd := exec.Command("git", "rev-parse", "--show-toplevel")
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		panic(err)
-	}
-	dir := filepath.Join(strings.TrimSpace(string(out)), "go-runtime", "scaffolding")
-	w, err := os.CreateTemp("", "")
-	if err != nil {
-		panic(err)
-	}
-	defer os.Remove(w.Name()) // This is okay because the zip.Reader will keep it open.
-	if err != nil {
-		panic(err)
-	}
-
-	err = internal.ZipDir(dir, w.Name())
-	if err != nil {
-		panic(err)
-	}
-
-	info, err := w.Stat()
-	if err != nil {
-		panic(err)
-	}
-	_, _ = w.Seek(0, 0)
-	zr, err := zip.NewReader(w, info.Size())
-	if err != nil {
-		panic(err)
-	}
-	return zr
-}()
+var Files = internal.ZipRelativeToCaller("scaffolding")

internal/source_root.go

@@ -0,0 +1,12 @@
+//go:build !release
+
+package internal
+
+import (
+	"os"
+	"path/filepath"
+)
+
+func FTLSourceRoot() string {
+	return filepath.Clean(filepath.Join(filepath.Dir(os.Args[0]), "..", ".."))
+}