bug: String() is unsafe, so rename to Print() and make a safer version

pkg/auth/auth.go

@@ -23,7 +23,7 @@ func Authorize(ctx engine.Context, input string) (runner.AuthorizerResponse, err
 
 	var result bool
 	err := survey.AskOne(&survey.Confirm{
-		Help:    fmt.Sprintf("The full source of the tools is as follows:\n\n%s", ctx.Tool.String()),
+		Help:    fmt.Sprintf("The full source of the tools is as follows:\n\n%s", ctx.Tool.Print()),
 		Default: true,
 		Message: ConfirmMessage(ctx, input),
 	}, &result)

pkg/cli/fmt.go

@@ -43,9 +43,9 @@ func (e *Fmt) Run(_ *cobra.Command, args []string) error {
 	}
 
 	if e.Write && loc != "" {
-		return os.WriteFile(loc, []byte(doc.String()), 0644)
+		return os.WriteFile(loc, []byte(doc.Print()), 0644)
 	}
 
-	fmt.Print(doc.String())
+	fmt.Print(doc.Print())
 	return nil
 }

pkg/cli/gptscript.go

@@ -14,7 +14,6 @@ import (
 	"github.com/fatih/color"
 	"github.com/gptscript-ai/cmd"
 	gptscript2 "github.com/gptscript-ai/go-gptscript"
-	"github.com/gptscript-ai/gptscript/pkg/assemble"
 	"github.com/gptscript-ai/gptscript/pkg/auth"
 	"github.com/gptscript-ai/gptscript/pkg/builtin"
 	"github.com/gptscript-ai/gptscript/pkg/cache"
@@ -58,7 +57,6 @@ type GPTScript struct {
 	// Input should not be using GPTSCRIPT_INPUT env var because that is the same value that is set in tool executions
 	Input                    string   `usage:"Read input from a file (\"-\" for stdin)" short:"f" env:"GPTSCRIPT_INPUT_FILE"`
 	SubTool                  string   `usage:"Use tool of this name, not the first tool in file" local:"true"`
-	Assemble                 bool     `usage:"Assemble tool to a single artifact, saved to --output" hidden:"true" local:"true"`
 	ListModels               bool     `usage:"List the models available and exit" local:"true"`
 	ListTools                bool     `usage:"List built-in tools and exit" local:"true"`
 	ListenAddress            string   `usage:"Server listen address" default:"127.0.0.1:0" hidden:"true"`
@@ -439,20 +437,6 @@ func (r *GPTScript) Run(cmd *cobra.Command, args []string) (retErr error) {
 		return cmd.Help()
 	}
 
-	if r.Assemble {
-		var out io.Writer = os.Stdout
-		if r.Output != "" && r.Output != "-" {
-			f, err := os.Create(r.Output)
-			if err != nil {
-				return fmt.Errorf("opening %s: %w", r.Output, err)
-			}
-			defer f.Close()
-			out = f
-		}
-
-		return assemble.Assemble(prg, out)
-	}
-
 	toolInput, err := input.FromCLI(r.Input, args)
 	if err != nil {
 		return err

pkg/loader/loader.go

@@ -17,7 +17,6 @@ import (
 
 	"github.com/getkin/kin-openapi/openapi3"
 	"github.com/gptscript-ai/gptscript/internal"
-	"github.com/gptscript-ai/gptscript/pkg/assemble"
 	"github.com/gptscript-ai/gptscript/pkg/builtin"
 	"github.com/gptscript-ai/gptscript/pkg/cache"
 	"github.com/gptscript-ai/gptscript/pkg/hash"
@@ -132,36 +131,6 @@ func loadLocal(base *source, name string) (*source, bool, error) {
 	}, true, nil
 }
 
-func loadProgram(data []byte, into *types.Program, targetToolName, defaultModel string) (types.Tool, error) {
-	var ext types.Program
-
-	if err := json.Unmarshal(data[len(assemble.Header):], &ext); err != nil {
-		return types.Tool{}, err
-	}
-
-	into.ToolSet = make(map[string]types.Tool, len(ext.ToolSet))
-	for k, v := range ext.ToolSet {
-		if builtinTool, ok := builtin.DefaultModel(k, defaultModel); ok {
-			v = builtinTool
-		}
-		into.ToolSet[k] = v
-	}
-
-	tool := into.ToolSet[ext.EntryToolID]
-	if targetToolName == "" {
-		return tool, nil
-	}
-
-	tool, ok := into.ToolSet[tool.LocalTools[strings.ToLower(targetToolName)]]
-	if !ok {
-		return tool, &types.ErrToolNotFound{
-			ToolName: targetToolName,
-		}
-	}
-
-	return tool, nil
-}
-
 func loadOpenAPI(prg *types.Program, data []byte) *openapi3.T {
 	var (
 		openAPICacheKey     = hash.Digest(data)
@@ -189,14 +158,6 @@ func loadOpenAPI(prg *types.Program, data []byte) *openapi3.T {
 func readTool(ctx context.Context, cache *cache.Client, prg *types.Program, base *source, targetToolName, defaultModel string) ([]types.Tool, error) {
 	data := base.Content
 
-	if bytes.HasPrefix(data, assemble.Header) {
-		tool, err := loadProgram(data, prg, targetToolName, defaultModel)
-		if err != nil {
-			return nil, err
-		}
-		return []types.Tool{tool}, nil
-	}
-
 	var (
 		tools     []types.Tool
 		isOpenAPI bool
@@ -231,11 +192,19 @@ func readTool(ctx context.Context, cache *cache.Client, prg *types.Program, base
 	// If we didn't get any tools from trying to parse it as OpenAPI, try to parse it as a GPTScript
 	if len(tools) == 0 {
 		var err error
-		tools, err = parser.ParseTools(bytes.NewReader(data), parser.Options{
-			AssignGlobals: true,
-		})
-		if err != nil {
-			return nil, err
+		_, marshaled, ok := strings.Cut(string(data), "#!GPTSCRIPT")
+		if ok {
+			err = json.Unmarshal([]byte(marshaled), &tools)
+			if err != nil {
+				return nil, fmt.Errorf("error parsing marshalled script: %w", err)
+			}
+		} else {
+			tools, err = parser.ParseTools(bytes.NewReader(data), parser.Options{
+				AssignGlobals: true,
+			})
+			if err != nil {
+				return nil, err
+			}
 		}
 	}
 

pkg/openai/client.go

@@ -282,10 +282,7 @@ func toMessages(request types.CompletionRequest, compat bool) (result []openai.C
 				chatMessage.ToolCalls = append(chatMessage.ToolCalls, toToolCall(*content.ToolCall))
 			}
 			if content.Text != "" {
-				chatMessage.MultiContent = append(chatMessage.MultiContent, openai.ChatMessagePart{
-					Type: openai.ChatMessagePartTypeText,
-					Text: content.Text,
-				})
+				chatMessage.MultiContent = append(chatMessage.MultiContent, textToMultiContent(content.Text)...)
 			}
 		}
 
@@ -307,6 +304,35 @@ func toMessages(request types.CompletionRequest, compat bool) (result []openai.C
 	return
 }
 
+const imagePrefix = "data:image/png;base64,"
+
+func textToMultiContent(text string) []openai.ChatMessagePart {
+	var chatParts []openai.ChatMessagePart
+	parts := strings.Split(text, "\n")
+	for i := len(parts) - 1; i >= 0; i-- {
+		if strings.HasPrefix(parts[i], imagePrefix) {
+			chatParts = append(chatParts, openai.ChatMessagePart{
+				Type: openai.ChatMessagePartTypeImageURL,
+				ImageURL: &openai.ChatMessageImageURL{
+					URL: parts[i],
+				},
+			})
+			parts = parts[:i]
+		} else {
+			break
+		}
+	}
+	if len(parts) > 0 {
+		chatParts = append(chatParts, openai.ChatMessagePart{
+			Type: openai.ChatMessagePartTypeText,
+			Text: strings.Join(parts, "\n"),
+		})
+	}
+
+	slices.Reverse(chatParts)
+	return chatParts
+}
+
 func (c *Client) Call(ctx context.Context, messageRequest types.CompletionRequest, env []string, status chan<- types.CompletionStatus) (*types.CompletionMessage, error) {
 	if err := c.ValidAuth(); err != nil {
 		if err := c.RetrieveAPIKey(ctx, env); err != nil {

pkg/openai/client_test.go

@@ -9,6 +9,44 @@ import (
 	"github.com/hexops/valast"
 )
 
+func TestTextToMultiContent(t *testing.T) {
+	autogold.Expect([]openai.ChatMessagePart{{
+		Type: "text",
+		Text: "hi\ndata:image/png;base64,xxxxx\n",
+	}}).Equal(t, textToMultiContent("hi\ndata:image/png;base64,xxxxx\n"))
+
+	autogold.Expect([]openai.ChatMessagePart{
+		{
+			Type: "text",
+			Text: "hi",
+		},
+		{
+			Type:     "image_url",
+			ImageURL: &openai.ChatMessageImageURL{URL: "data:image/png;base64,xxxxx"},
+		},
+	}).Equal(t, textToMultiContent("hi\ndata:image/png;base64,xxxxx"))
+
+	autogold.Expect([]openai.ChatMessagePart{{
+		Type:     "image_url",
+		ImageURL: &openai.ChatMessageImageURL{URL: "data:image/png;base64,xxxxx"},
+	}}).Equal(t, textToMultiContent("data:image/png;base64,xxxxx"))
+
+	autogold.Expect([]openai.ChatMessagePart{
+		{
+			Type: "text",
+			Text: "\none\ntwo",
+		},
+		{
+			Type:     "image_url",
+			ImageURL: &openai.ChatMessageImageURL{URL: "data:image/png;base64,xxxxx"},
+		},
+		{
+			Type:     "image_url",
+			ImageURL: &openai.ChatMessageImageURL{URL: "data:image/png;base64,yyyyy"},
+		},
+	}).Equal(t, textToMultiContent("\none\ntwo\ndata:image/png;base64,xxxxx\ndata:image/png;base64,yyyyy"))
+}
+
 func Test_appendMessage(t *testing.T) {
 	autogold.Expect(types.CompletionMessage{Content: []types.ContentPart{
 		{ToolCall: &types.CompletionToolCall{

pkg/parser/parser.go

@@ -263,7 +263,7 @@ func writeSep(buf *strings.Builder, lastText bool) {
 	}
 }
 
-func (d Document) String() string {
+func (d Document) Print() string {
 	buf := strings.Builder{}
 	lastText := false
 	for _, node := range d.Nodes {
@@ -274,7 +274,7 @@ func (d Document) String() string {
 		}
 		if node.ToolNode != nil {
 			writeSep(&buf, lastText)
-			buf.WriteString(node.ToolNode.Tool.String())
+			buf.WriteString(node.ToolNode.Tool.Print())
 			lastText = false
 		}
 	}

pkg/parser/parser_test.go

@@ -304,7 +304,7 @@ body
 !metadata:first:package.json
 foo=base
 f
-`).Equal(t, tools[0].String())
+`).Equal(t, tools[0].Print())
 }
 
 func TestFormatWithBadInstruction(t *testing.T) {
@@ -316,9 +316,9 @@ func TestFormatWithBadInstruction(t *testing.T) {
 			Instructions: "foo: bar",
 		},
 	}
-	autogold.Expect("Name: foo\n===\nfoo: bar\n").Equal(t, input.String())
+	autogold.Expect("Name: foo\n===\nfoo: bar\n").Equal(t, input.Print())
 
-	tools, err := ParseTools(strings.NewReader(input.String()))
+	tools, err := ParseTools(strings.NewReader(input.Print()))
 	require.NoError(t, err)
 	if reflect.DeepEqual(input, tools[0]) {
 		t.Errorf("expected %v, got %v", input, tools[0])

pkg/runner/runner.go

@@ -655,6 +655,17 @@ func (r *Runner) newDispatcher(ctx context.Context) dispatcher {
 	return newParallelDispatcher(ctx)
 }
 
+func idForToolCall(id string, state *engine.Return) string {
+	if state == nil || state.State == nil {
+		return id
+	}
+	tc, ok := state.State.Pending[id]
+	if !ok || tc.Index == nil {
+		return id
+	}
+	return fmt.Sprintf("%03d", *tc.Index)
+}
+
 func (r *Runner) subCalls(callCtx engine.Context, monitor Monitor, env []string, state *State, toolCategory engine.ToolCategory) (*State, []SubCallResult, error) {
 	var (
 		resultLock  sync.Mutex
@@ -698,7 +709,9 @@ func (r *Runner) subCalls(callCtx engine.Context, monitor Monitor, env []string,
 
 	// Sort the id so if sequential the results are predictable
 	ids := maps.Keys(state.Continuation.Calls)
-	sort.Strings(ids)
+	sort.Slice(ids, func(i, j int) bool {
+		return idForToolCall(ids[i], state.Continuation) < idForToolCall(ids[j], state.Continuation)
+	})
 
 	for _, id := range ids {
 		call := state.Continuation.Calls[id]

pkg/sdkserver/routes.go

@@ -114,7 +114,7 @@ func (s *server) listTools(w http.ResponseWriter, r *http.Request) {
 		// Don't print instructions
 		tool.Instructions = ""
 
-		lines = append(lines, tool.String())
+		lines = append(lines, tool.Print())
 	}
 
 	writeResponse(logger, w, map[string]any{"stdout": strings.Join(lines, "\n---\n")})
@@ -339,5 +339,5 @@ func (s *server) fmtDocument(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	writeResponse(logger, w, map[string]string{"stdout": doc.String()})
+	writeResponse(logger, w, map[string]string{"stdout": doc.Print()})
 }

pkg/sdkserver/types.go

@@ -1,8 +1,8 @@
 package sdkserver
 
 import (
+	"encoding/json"
 	"maps"
-	"strings"
 	"time"
 
 	"github.com/gptscript-ai/gptscript/pkg/cache"
@@ -30,15 +30,12 @@ const (
 type toolDefs []types.ToolDef
 
 func (t toolDefs) String() string {
-	s := new(strings.Builder)
-	for i, tool := range t {
-		s.WriteString(tool.String())
-		if i != len(t)-1 {
-			s.WriteString("\n\n---\n\n")
-		}
+	data, err := json.Marshal(t)
+	if err != nil {
+		panic(err)
 	}
 
-	return s.String()
+	return "#!GPTSCRIPT" + string(data)
 }
 
 type (

pkg/types/tool.go

@@ -389,6 +389,14 @@ func (t Tool) GetToolRefsFromNames(names []string) (result []ToolReference, _ er
 }
 
 func (t ToolDef) String() string {
+	data, err := json.Marshal([]any{t})
+	if err != nil {
+		panic(err)
+	}
+	return "#!GPTSCRIPT" + string(data)
+}
+
+func (t ToolDef) Print() string {
 	buf := &strings.Builder{}
 	if t.Parameters.GlobalModelName != "" {
 		_, _ = fmt.Fprintf(buf, "Global Model Name: %s\n", t.Parameters.GlobalModelName)