From d556e28c3e19a0e1c13d5c9fc619ee2072b3ee89 Mon Sep 17 00:00:00 2001 From: Alec Thomas Date: Mon, 20 Jan 2025 17:01:03 +1100 Subject: [PATCH] chore: remove `ftl schema import` (#4118) This was experimental LLM code. We will revisit. --- Justfile | 4 +- .../leases/testdata/go/leases/go.mod | 1 - .../sql/testdata/go/database/go.mod | 1 - .../controller/sql/testdata/go/mysql/go.mod | 1 - bin/.go-size-analyzer-1.7.6.pkg | 1 + bin/gsa | 1 + examples/go/mysql/go.mod | 1 - examples/go/postgres/go.mod | 1 - frontend/cli/cmd_schema.go | 1 - frontend/cli/cmd_schema_import.go | 200 ------------------ .../ftl/ftltest/testdata/go/subscriber/go.mod | 1 - .../ftl/ftltest/testdata/go/verbtypes/go.mod | 1 - .../ftl/ftltest/testdata/go/wrapped/go.mod | 1 - go-runtime/ftl/testdata/go/mapper/go.mod | 1 - .../ftl/testdata/go/typeregistry/go.mod | 1 - go.mod | 3 +- go.sum | 5 - 17 files changed, 5 insertions(+), 220 deletions(-) create mode 120000 bin/.go-size-analyzer-1.7.6.pkg create mode 120000 bin/gsa delete mode 100644 frontend/cli/cmd_schema_import.go diff --git a/Justfile b/Justfile index d0bb456517..b5f549f9cd 100644 --- a/Justfile +++ b/Justfile @@ -177,9 +177,9 @@ _build-go-binary-fast dir binary="": binary="${2:-$(basename "$1")}" if [ "${FTL_DEBUG:-}" = "true" ]; then - go build -o "{{RELEASE}}/${binary}" -tags release -gcflags=all="-N -l" -ldflags "-X github.com/block/ftl.Version={{VERSION}} -X github.com/block/ftl.timestamp={{TIMESTAMP}}" "$1" + go build -o "{{RELEASE}}/${binary}" -tags release -gcflags=all="-N -l" -ldflags "-s -w -X github.com/block/ftl.Version={{VERSION}} -X github.com/block/ftl.timestamp={{TIMESTAMP}}" "$1" else - mk "{{RELEASE}}/${binary}" : !(build|integration|infrastructure|node_modules|Procfile*|Dockerfile*) -- go build -o "{{RELEASE}}/${binary}" -tags release -ldflags "-X github.com/block/ftl.Version={{VERSION}} -X github.com/block/ftl.timestamp={{TIMESTAMP}}" "$1" + mk "{{RELEASE}}/${binary}" : !(build|integration|infrastructure|node_modules|Procfile*|Dockerfile*) -- go build -o "{{RELEASE}}/${binary}" -tags release -ldflags "-s -w -X github.com/block/ftl.Version={{VERSION}} -X github.com/block/ftl.timestamp={{TIMESTAMP}}" "$1" fi # Build the ZIP files that are embedded in the FTL release binaries diff --git a/backend/controller/leases/testdata/go/leases/go.mod b/backend/controller/leases/testdata/go/leases/go.mod index c9330e95f6..1595042e31 100644 --- a/backend/controller/leases/testdata/go/leases/go.mod +++ b/backend/controller/leases/testdata/go/leases/go.mod @@ -47,7 +47,6 @@ require ( github.com/block/scaffolder v1.3.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/chzyer/readline v1.5.1 // indirect - github.com/containerd/log v0.1.0 // indirect github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect github.com/danieljoos/wincred v1.2.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect diff --git a/backend/controller/sql/testdata/go/database/go.mod b/backend/controller/sql/testdata/go/database/go.mod index 2b43fb0f9f..f71a1c8d70 100644 --- a/backend/controller/sql/testdata/go/database/go.mod +++ b/backend/controller/sql/testdata/go/database/go.mod @@ -44,7 +44,6 @@ require ( github.com/block/scaffolder v1.3.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/chzyer/readline v1.5.1 // indirect - github.com/containerd/log v0.1.0 // indirect github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect github.com/danieljoos/wincred v1.2.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect diff --git a/backend/controller/sql/testdata/go/mysql/go.mod b/backend/controller/sql/testdata/go/mysql/go.mod index be1e4252e5..da16e7174f 100644 --- a/backend/controller/sql/testdata/go/mysql/go.mod +++ b/backend/controller/sql/testdata/go/mysql/go.mod @@ -44,7 +44,6 @@ require ( github.com/block/scaffolder v1.3.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/chzyer/readline v1.5.1 // indirect - github.com/containerd/log v0.1.0 // indirect github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect github.com/danieljoos/wincred v1.2.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect diff --git a/bin/.go-size-analyzer-1.7.6.pkg b/bin/.go-size-analyzer-1.7.6.pkg new file mode 120000 index 0000000000..383f4511d4 --- /dev/null +++ b/bin/.go-size-analyzer-1.7.6.pkg @@ -0,0 +1 @@ +hermit \ No newline at end of file diff --git a/bin/gsa b/bin/gsa new file mode 120000 index 0000000000..d7e28b7bc3 --- /dev/null +++ b/bin/gsa @@ -0,0 +1 @@ +.go-size-analyzer-1.7.6.pkg \ No newline at end of file diff --git a/examples/go/mysql/go.mod b/examples/go/mysql/go.mod index 095f17b1c6..e07201c6dc 100644 --- a/examples/go/mysql/go.mod +++ b/examples/go/mysql/go.mod @@ -44,7 +44,6 @@ require ( github.com/block/scaffolder v1.3.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/chzyer/readline v1.5.1 // indirect - github.com/containerd/log v0.1.0 // indirect github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect github.com/danieljoos/wincred v1.2.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect diff --git a/examples/go/postgres/go.mod b/examples/go/postgres/go.mod index bb7cbd3f4a..b49b58ac1c 100644 --- a/examples/go/postgres/go.mod +++ b/examples/go/postgres/go.mod @@ -44,7 +44,6 @@ require ( github.com/block/scaffolder v1.3.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/chzyer/readline v1.5.1 // indirect - github.com/containerd/log v0.1.0 // indirect github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect github.com/danieljoos/wincred v1.2.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect diff --git a/frontend/cli/cmd_schema.go b/frontend/cli/cmd_schema.go index ed0b96c06f..a46cae9be9 100644 --- a/frontend/cli/cmd_schema.go +++ b/frontend/cli/cmd_schema.go @@ -4,5 +4,4 @@ type schemaCmd struct { Get getSchemaCmd `default:"" cmd:"" help:"Retrieve the cluster FTL schema."` Diff schemaDiffCmd `cmd:"" help:"Print any schema differences between this cluster and another cluster. Returns an exit code of 1 if there are differences."` Generate schemaGenerateCmd `cmd:"" help:"Stream the schema from the cluster and generate files from the template."` - Import schemaImportCmd `cmd:"" help:"Import messages to the FTL schema."` } diff --git a/frontend/cli/cmd_schema_import.go b/frontend/cli/cmd_schema_import.go deleted file mode 100644 index 7b096d99bb..0000000000 --- a/frontend/cli/cmd_schema_import.go +++ /dev/null @@ -1,200 +0,0 @@ -package main - -import ( - "context" - "fmt" - "net" - "os" - "path/filepath" - - "github.com/alecthomas/types/optional" - "github.com/tmc/langchaingo/llms" - "github.com/tmc/langchaingo/llms/ollama" - - "github.com/block/ftl/internal/container" - "github.com/block/ftl/internal/log" -) - -type schemaImportCmd struct { - OllamaPort int `help:"Port to use for ollama." default:"11434"` - Go importGoCmd `cmd:"" help:"Import types for an FTL Go module."` - Kotlin importKotlinCmd `cmd:"" help:"Import types for an FTL Kotlin module."` -} - -type importGoCmd struct { - Dir string `arg:"" required:"" help:"Directory to import from."` -} - -type importKotlinCmd struct { - Dir string `arg:"" required:"" help:"Directory to import from."` -} - -const ollamaContainerName = "ftl-ollama-1" -const ollamaVolume = "ollama:/root/.ollama" -const ollamaModel = "llama2" - -func (i *importKotlinCmd) getPrompt() (string, error) { - input, err := createInputString(i.Dir) - if err != nil { - return "", err - } - - return fmt.Sprintf("I will provide you with an input string containing the contents of a file which must be translated into Kotlin. "+ - "The resulting Kotlin file should translate every object from the input into its own Kotlin data class, written as `data class...`. "+ - "The data class parameters should each be declared as `val`."+ - "Everything in the provided input that is not a type or object declaration must be ignored."+ - "No fields should be modified or added—the result must be an exact translation of the provided input, retaining case."+ - "The result should reside in a package named `ftl`."+ - "Your output should only include the resultant Kotlin data classes with no additional functions."+ - "Please provide output that is only the resultant Kotlin file, with no additional text or explanation. The initial string to translate is this: %s", - input, - ), nil -} - -func (i *importGoCmd) getPrompt() (string, error) { - input, err := createInputString(i.Dir) - if err != nil { - return "", err - } - - return fmt.Sprintf("I will provide you with an input string containing the contents of a file which must be translated into Go. "+ - "The resulting Go file should represent every object as its own Go struct, written as `type...`. "+ - "Everything in the provided input that is not a type or object declaration must be ignored."+ - "No fields should be modified or added—the result must be an exact translation of the provided input, retaining case."+ - "The result should reside in a package named `ftl`."+ - "Your output should only include the resultant Go types with no additional functions."+ - "Please provide output that is only the resultant Go file, with no additional text or explanation. The initial string to translate is this: %s", - input, - ), nil -} - -func (i *importGoCmd) Run(ctx context.Context, parent *schemaImportCmd) error { - err := parent.setup(ctx) - if err != nil { - return err - } - - prompt, err := i.getPrompt() - if err != nil { - return err - } - - err = query(ctx, prompt) - if err != nil { - return err - } - - return nil -} - -func (i *importKotlinCmd) Run(ctx context.Context, parent *schemaImportCmd) error { - err := parent.setup(ctx) - if err != nil { - return err - } - - prompt, err := i.getPrompt() - if err != nil { - return err - } - - err = query(ctx, prompt) - if err != nil { - return err - } - - return nil -} - -func query(ctx context.Context, prompt string) error { - logger := log.FromContext(ctx) - - logger.Debugf("The import schema command relies on the %s AI chat model to translate schemas from other "+ - "languages into FTL compliant objects in the specified language. Output may vary and results should be inspected "+ - "for correctness. It is suggested that if the results are not satisfactory, you try again.\n\n", ollamaModel) - - llm, err := ollama.New(ollama.WithModel("llama2")) - if err != nil { - logger.Errorf(err, "failed to call ollama") - } - - completion, err := llm.Call(ctx, fmt.Sprintf("Human: %s \nAssistant:", prompt), - llms.WithTemperature(0.8), - llms.WithStreamingFunc(func(ctx context.Context, chunk []byte) error { - fmt.Print(string(chunk)) - return nil - }), - ) - if err != nil { - logger.Errorf(err, "failed to call ollama") - } - - _ = completion - return nil -} - -func (s *schemaImportCmd) setup(ctx context.Context) error { - logger := log.FromContext(ctx) - - exists, err := container.DoesExist(ctx, ollamaContainerName, optional.None[string]()) - if err != nil { - return err - } - - if !exists { - logger.Debugf("Creating docker container '%s' for ollama", ollamaContainerName) - - // check if port s.OllamaPort is already in use - l, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", s.OllamaPort)) - if err != nil { - return fmt.Errorf("port %d is already in use: %w", s.OllamaPort, err) - } - _ = l.Close() - - err = container.Pull(ctx, "ollama/ollama") - if err != nil { - return err - } - - err = container.Run(ctx, "ollama/ollama", ollamaContainerName, map[int]int{s.OllamaPort: 11434}, optional.Some(ollamaVolume)) - if err != nil { - return err - } - } else { - // Start the existing container - err = container.Start(ctx, ollamaContainerName) - if err != nil { - return err - } - } - - // Initialize Ollama - err = container.Exec(ctx, ollamaContainerName, "ollama", "run", ollamaModel) - if err != nil { - return err - } - - return nil -} - -func createInputString(dir string) (string, error) { - var result string - err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if !info.IsDir() { - fileContent, err := os.ReadFile(path) - if err != nil { - return err - } - result += string(fileContent) + "\n" - } - return nil - }) - - if err != nil { - return "", err - } - return result, nil -} diff --git a/go-runtime/ftl/ftltest/testdata/go/subscriber/go.mod b/go-runtime/ftl/ftltest/testdata/go/subscriber/go.mod index e4adde4f0e..e0086a9c85 100644 --- a/go-runtime/ftl/ftltest/testdata/go/subscriber/go.mod +++ b/go-runtime/ftl/ftltest/testdata/go/subscriber/go.mod @@ -44,7 +44,6 @@ require ( github.com/block/scaffolder v1.3.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/chzyer/readline v1.5.1 // indirect - github.com/containerd/log v0.1.0 // indirect github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect github.com/danieljoos/wincred v1.2.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect diff --git a/go-runtime/ftl/ftltest/testdata/go/verbtypes/go.mod b/go-runtime/ftl/ftltest/testdata/go/verbtypes/go.mod index ecca696aa3..ea96421317 100644 --- a/go-runtime/ftl/ftltest/testdata/go/verbtypes/go.mod +++ b/go-runtime/ftl/ftltest/testdata/go/verbtypes/go.mod @@ -46,7 +46,6 @@ require ( github.com/block/scaffolder v1.3.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/chzyer/readline v1.5.1 // indirect - github.com/containerd/log v0.1.0 // indirect github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect github.com/danieljoos/wincred v1.2.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect diff --git a/go-runtime/ftl/ftltest/testdata/go/wrapped/go.mod b/go-runtime/ftl/ftltest/testdata/go/wrapped/go.mod index 5f5d0977e6..9d5615f620 100644 --- a/go-runtime/ftl/ftltest/testdata/go/wrapped/go.mod +++ b/go-runtime/ftl/ftltest/testdata/go/wrapped/go.mod @@ -44,7 +44,6 @@ require ( github.com/block/scaffolder v1.3.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/chzyer/readline v1.5.1 // indirect - github.com/containerd/log v0.1.0 // indirect github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect github.com/danieljoos/wincred v1.2.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect diff --git a/go-runtime/ftl/testdata/go/mapper/go.mod b/go-runtime/ftl/testdata/go/mapper/go.mod index 7d64f2724a..a5b1fe43ed 100644 --- a/go-runtime/ftl/testdata/go/mapper/go.mod +++ b/go-runtime/ftl/testdata/go/mapper/go.mod @@ -44,7 +44,6 @@ require ( github.com/block/scaffolder v1.3.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/chzyer/readline v1.5.1 // indirect - github.com/containerd/log v0.1.0 // indirect github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect github.com/danieljoos/wincred v1.2.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect diff --git a/go-runtime/ftl/testdata/go/typeregistry/go.mod b/go-runtime/ftl/testdata/go/typeregistry/go.mod index cbc094753a..8bcf01ab16 100644 --- a/go-runtime/ftl/testdata/go/typeregistry/go.mod +++ b/go-runtime/ftl/testdata/go/typeregistry/go.mod @@ -44,7 +44,6 @@ require ( github.com/block/scaffolder v1.3.0 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/chzyer/readline v1.5.1 // indirect - github.com/containerd/log v0.1.0 // indirect github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect github.com/danieljoos/wincred v1.2.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect diff --git a/go.mod b/go.mod index 5697036f38..0ab9142d0d 100644 --- a/go.mod +++ b/go.mod @@ -60,7 +60,6 @@ require ( github.com/tliron/commonlog v0.2.19 github.com/tliron/glsp v0.2.2 github.com/tliron/kutil v0.3.26 - github.com/tmc/langchaingo v0.1.12 github.com/zalando/go-keyring v0.2.6 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 go.opentelemetry.io/otel v1.33.0 @@ -114,6 +113,7 @@ require ( github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f // indirect github.com/cockroachdb/pebble v0.0.0-20221207173255-0f086d933dac // indirect github.com/cockroachdb/redact v1.1.3 // indirect + github.com/containerd/log v0.1.0 // indirect github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect github.com/cubicdaiya/gonp v1.0.4 // indirect github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect @@ -189,7 +189,6 @@ require ( github.com/pingcap/log v1.1.0 // indirect github.com/pingcap/tidb/pkg/parser v0.0.0-20231103154709-4f00ece106b1 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pkoukk/tiktoken-go v0.1.6 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/rivo/uniseg v0.2.0 // indirect github.com/riywo/loginshell v0.0.0-20200815045211-7d26008be1ab // indirect diff --git a/go.sum b/go.sum index 3554f7395c..1c8e3a4773 100644 --- a/go.sum +++ b/go.sum @@ -31,7 +31,6 @@ github.com/IBM/sarama v1.44.0 h1:puNKqcScjSAgVLramjsuovZrS0nJZFVsrvuUymkWqhE= github.com/IBM/sarama v1.44.0/go.mod h1:MxQ9SvGfvKIorbk077Ff6DUnBlGpidiQOtU2vuBaxVw= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= -github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= @@ -577,8 +576,6 @@ github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsK github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkoukk/tiktoken-go v0.1.6 h1:JF0TlJzhTbrI30wCvFuiw6FzP2+/bR+FIxUdgEAcUsw= -github.com/pkoukk/tiktoken-go v0.1.6/go.mod h1:9NiV+i9mJKGj1rYOT+njbv+ZwA/zJxYdewGl6qVatpg= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -682,8 +679,6 @@ github.com/tliron/glsp v0.2.2 h1:IKPfwpE8Lu8yB6Dayta+IyRMAbTVunudeauEgjXBt+c= github.com/tliron/glsp v0.2.2/go.mod h1:GMVWDNeODxHzmDPvYbYTCs7yHVaEATfYtXiYJ9w1nBg= github.com/tliron/kutil v0.3.26 h1:G+dicQLvzm3zdOMrrQFLBfHJXtk57fEu2kf1IFNyJxw= github.com/tliron/kutil v0.3.26/go.mod h1:1/HRVAb+fnRIRnzmhu0FPP+ZJKobrpwHStDVMuaXDzY= -github.com/tmc/langchaingo v0.1.12 h1:yXwSu54f3b1IKw0jJ5/DWu+qFVH1NBblwC0xddBzGJE= -github.com/tmc/langchaingo v0.1.12/go.mod h1:cd62xD6h+ouk8k/QQFhOsjRYBSA1JJ5UVKXSIgm7Ni4= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=