From 58ddc8a3e444d268c9ece247d599b19690a42237 Mon Sep 17 00:00:00 2001 From: Alec Thomas Date: Fri, 3 Nov 2023 13:15:56 +1100 Subject: [PATCH] fix: embed runner template for `ftl serve` (#544) In development mode this will build the template directory first, in release mode it will embed a zip file containing the template directory. Fixes #541 --- Bitfile | 7 +++- .../controller/scaling/localscaling/devel.go | 26 ++++++++++++ .../{ => localscaling}/local_scaling.go | 6 ++- .../scaling/localscaling/release.go | 40 +++++++++++++++++++ cmd/ftl/cmd_serve.go | 4 +- go-runtime/devel.go | 39 +----------------- internal/source_root.go | 12 ++++++ 7 files changed, 91 insertions(+), 43 deletions(-) create mode 100644 backend/controller/scaling/localscaling/devel.go rename backend/controller/scaling/{ => localscaling}/local_scaling.go (94%) create mode 100644 backend/controller/scaling/localscaling/release.go create mode 100644 internal/source_root.go diff --git a/Bitfile b/Bitfile index 39d003ad22..9e66c98789 100644 --- a/Bitfile +++ b/Bitfile @@ -43,6 +43,8 @@ CLIENT_IN = frontend/src/**/* NODE_MODULES_OUT = frontend/node_modules NODE_MODULES_IN = frontend/package{,-lock}.json +RUNNER_TEMPLATE_ZIP = backend/controller/scaling/localscaling/template.zip + #virtual release: # inputs: %{RELEASE}/ftl %{RELEASE}/ftl-controller %{RELEASE}/ftl-runner @@ -60,7 +62,7 @@ NODE_MODULES_IN = frontend/package{,-lock}.json %{RELEASE}/ftl-runner: %{RELEASE} %{GO_SOURCES} cmd/ftl-runner/**/*.go build: go build -o %{OUT} -tags release -ldflags "-X main.version=%{VERSION} -X main.timestamp=$(date +%s)" ./cmd/ftl-runner -%{RELEASE}/ftl: %{RELEASE} %{GO_SOURCES} cmd/ftl/**/*.go {go,kotlin}-runtime/scaffolding.zip %{CLIENT_OUT} +%{RELEASE}/ftl: %{RELEASE} %{GO_SOURCES} cmd/ftl/**/*.go %{CLIENT_OUT} **/*.zip build: go build -o %{OUT} -tags release -ldflags "-X main.version=%{VERSION} -X main.timestamp=$(date +%s)" ./cmd/ftl %{RELEASE}/ftl-initdb: %{RELEASE} %{GO_SOURCES} cmd/ftl-initdb/**/*.go @@ -107,6 +109,9 @@ kotlin-runtime/scaffolding.zip: kotlin-runtime/scaffolding/**/* %{KT_RUNTIME_RUNNER_TEMPLATE_OUT}: %{KT_RUNTIME_OUT} %(dirname %{KT_RUNTIME_RUNNER_TEMPLATE_OUT})% build: install -m 0600 %{KT_RUNTIME_OUT} %{OUT} +%{RUNNER_TEMPLATE_ZIP}: %{KT_RUNTIME_RUNNER_TEMPLATE_OUT} + build: cd build/template && zip -q --symlinks -r ../../%{OUT} . + %{COMMON_LOG_OUT}: %{COMMON_LOG_IN} build: go generate %{IN} diff --git a/backend/controller/scaling/localscaling/devel.go b/backend/controller/scaling/localscaling/devel.go new file mode 100644 index 0000000000..57fd007ade --- /dev/null +++ b/backend/controller/scaling/localscaling/devel.go @@ -0,0 +1,26 @@ +//go:build !release + +package localscaling + +import ( + "context" + "path/filepath" + "sync" + + "github.com/TBD54566975/ftl/backend/common/exec" + "github.com/TBD54566975/ftl/backend/common/log" + "github.com/TBD54566975/ftl/internal" +) + +var templateDirOnce sync.Once + +func templateDir(ctx context.Context) string { + templateDirOnce.Do(func() { + cmd := exec.Command(ctx, log.Info, internal.FTLSourceRoot(), "bit", "build/template/ftl/jars/ftl-runtime.jar") + err := cmd.Run() + if err != nil { + panic(err) + } + }) + return filepath.Join(internal.FTLSourceRoot(), "build/template") +} diff --git a/backend/controller/scaling/local_scaling.go b/backend/controller/scaling/localscaling/local_scaling.go similarity index 94% rename from backend/controller/scaling/local_scaling.go rename to backend/controller/scaling/localscaling/local_scaling.go index 2a1319f627..0932048b64 100644 --- a/backend/controller/scaling/local_scaling.go +++ b/backend/controller/scaling/localscaling/local_scaling.go @@ -1,4 +1,4 @@ -package scaling +package localscaling import ( "context" @@ -15,10 +15,11 @@ import ( "github.com/TBD54566975/ftl/backend/common/bind" "github.com/TBD54566975/ftl/backend/common/log" "github.com/TBD54566975/ftl/backend/common/model" + "github.com/TBD54566975/ftl/backend/controller/scaling" "github.com/TBD54566975/ftl/backend/runner" ) -var _ RunnerScaling = (*LocalScaling)(nil) +var _ scaling.RunnerScaling = (*LocalScaling)(nil) type LocalScaling struct { lock sync.Mutex @@ -78,6 +79,7 @@ func (l *LocalScaling) SetReplicas(ctx context.Context, replicas int, idleRunner config := runner.Config{ Bind: l.portAllocator.Next(), ControllerEndpoint: controllerEndpoint, + TemplateDir: templateDir(ctx), } name := fmt.Sprintf("runner%d", i) diff --git a/backend/controller/scaling/localscaling/release.go b/backend/controller/scaling/localscaling/release.go new file mode 100644 index 0000000000..eeaae87836 --- /dev/null +++ b/backend/controller/scaling/localscaling/release.go @@ -0,0 +1,40 @@ +//go:build release + +package localscaling + +import ( + "archive/zip" + "bytes" + "context" + _ "embed" + "os" + "path/filepath" + "sync" + + "github.com/TBD54566975/ftl/internal" +) + +//go:embed template.zip +var archive []byte + +var templateDirOnce sync.Once + +func templateDir(ctx context.Context) string { + cacheDir, err := os.UserCacheDir() + if err != nil { + panic(err) + } + cacheDir = filepath.Join(cacheDir, "ftl-runner-template") + templateDirOnce.Do(func() { + _ = os.RemoveAll(cacheDir) + zr, err := zip.NewReader(bytes.NewReader(archive), int64(len(archive))) + if err != nil { + panic(err) + } + err = internal.UnzipDir(zr, cacheDir) + if err != nil { + panic(err) + } + }) + return cacheDir +} diff --git a/cmd/ftl/cmd_serve.go b/cmd/ftl/cmd_serve.go index 88f6f36a12..d43ce62327 100644 --- a/cmd/ftl/cmd_serve.go +++ b/cmd/ftl/cmd_serve.go @@ -16,7 +16,7 @@ import ( "github.com/TBD54566975/ftl/backend/common/log" "github.com/TBD54566975/ftl/backend/controller" "github.com/TBD54566975/ftl/backend/controller/databasetesting" - "github.com/TBD54566975/ftl/backend/controller/scaling" + "github.com/TBD54566975/ftl/backend/controller/scaling/localscaling" ) type serveCmd struct { @@ -50,7 +50,7 @@ func (s *serveCmd) Run(ctx context.Context) error { controllerAddresses = append(controllerAddresses, portAllocator.Next()) } - runnerScaling, err := scaling.NewLocalScaling(portAllocator, controllerAddresses) + runnerScaling, err := localscaling.NewLocalScaling(portAllocator, controllerAddresses) if err != nil { return errors.WithStack(err) } diff --git a/go-runtime/devel.go b/go-runtime/devel.go index 1df533b6e9..3e35421389 100644 --- a/go-runtime/devel.go +++ b/go-runtime/devel.go @@ -3,45 +3,8 @@ package goruntime import ( - "archive/zip" - "os" - "os/exec" - "path/filepath" - "strings" - "github.com/TBD54566975/ftl/internal" ) // Files is the FTL Go runtime scaffolding files. -var Files = func() *zip.Reader { - cmd := exec.Command("git", "rev-parse", "--show-toplevel") - out, err := cmd.CombinedOutput() - if err != nil { - panic(err) - } - dir := filepath.Join(strings.TrimSpace(string(out)), "go-runtime", "scaffolding") - w, err := os.CreateTemp("", "") - if err != nil { - panic(err) - } - defer os.Remove(w.Name()) // This is okay because the zip.Reader will keep it open. - if err != nil { - panic(err) - } - - err = internal.ZipDir(dir, w.Name()) - if err != nil { - panic(err) - } - - info, err := w.Stat() - if err != nil { - panic(err) - } - _, _ = w.Seek(0, 0) - zr, err := zip.NewReader(w, info.Size()) - if err != nil { - panic(err) - } - return zr -}() +var Files = internal.ZipRelativeToCaller("scaffolding") diff --git a/internal/source_root.go b/internal/source_root.go new file mode 100644 index 0000000000..f70ca2d562 --- /dev/null +++ b/internal/source_root.go @@ -0,0 +1,12 @@ +//go:build !release + +package internal + +import ( + "os" + "path/filepath" +) + +func FTLSourceRoot() string { + return filepath.Clean(filepath.Join(filepath.Dir(os.Args[0]), "..", "..")) +}