From 42e6ee684401a36bacf3367c9ce9cb0367a21fef Mon Sep 17 00:00:00 2001 From: Brace Sproul Date: Wed, 8 Nov 2023 15:04:38 -0800 Subject: [PATCH] Brace/openai assistant (#3171) * Feat: openai assistants * stash * Added base api struct * chore: lint files * initial auto function calling implementation * option for awaiting function calls in invoke * refactors, nits * spelling nit * chore: lint files * nit * match py * schema file * fix agent executor typing * improve tests * update comment * LFG * drop console log * cleanup code * refactor * improve, better docs * cr * improve test and entrypoint * eslint disable any * Fix typing * Fix types * Fix build * Update langchain/src/experimental/openai_assistant/index.ts * Use types instead of classes * Add docs --------- Co-authored-by: jacoblee93 --- .vscode/settings.json | 7 +- .../agents/agent_types/openai_assistant.mdx | 196 ++++++++++++ .../test-exports-bun/src/entrypoints.js | 1 + .../test-exports-cf/src/entrypoints.js | 1 + .../test-exports-cjs/src/entrypoints.js | 1 + .../test-exports-esbuild/src/entrypoints.js | 1 + .../test-exports-esm/src/entrypoints.js | 1 + .../test-exports-vercel/src/entrypoints.js | 1 + .../test-exports-vite/src/entrypoints.js | 1 + langchain/.gitignore | 3 + langchain/package.json | 8 + langchain/scripts/create-entrypoints.js | 1 + langchain/src/agents/agent.ts | 79 ++--- langchain/src/agents/executor.ts | 47 +-- .../src/agents/tests/runnable.int.test.ts | 66 ++++ langchain/src/agents/types.ts | 18 +- .../experimental/openai_assistant/index.ts | 287 ++++++++++++++++++ .../experimental/openai_assistant/schema.ts | 22 ++ .../tests/openai_assistant.int.test.ts | 136 +++++++++ langchain/src/load/import_map.ts | 1 + langchain/src/tools/convert_to_openai.ts | 13 + langchain/tsconfig.json | 1 + 22 files changed, 812 insertions(+), 80 deletions(-) create mode 100644 docs/docs/modules/agents/agent_types/openai_assistant.mdx create mode 100644 langchain/src/agents/tests/runnable.int.test.ts create mode 100644 langchain/src/experimental/openai_assistant/index.ts create mode 100644 langchain/src/experimental/openai_assistant/schema.ts create mode 100644 langchain/src/experimental/openai_assistant/tests/openai_assistant.int.test.ts diff --git a/.vscode/settings.json b/.vscode/settings.json index 992d8b3a5280..1049252fccad 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -9,5 +9,10 @@ "https://json.schemastore.org/github-workflow.json": "./.github/workflows/deploy.yml" }, "typescript.tsdk": "node_modules/typescript/lib", - "cSpell.words": ["Upstash"] + "cSpell.words": [ + "Upstash" + ], + "cSpell.enableFiletypes": [ + "mdx" + ] } diff --git a/docs/docs/modules/agents/agent_types/openai_assistant.mdx b/docs/docs/modules/agents/agent_types/openai_assistant.mdx new file mode 100644 index 000000000000..627e78f27122 --- /dev/null +++ b/docs/docs/modules/agents/agent_types/openai_assistant.mdx @@ -0,0 +1,196 @@ +# OpenAI Assistant + +:::info +The [OpenAI Assistant API](https://platform.openai.com/docs/assistants/overview) is still in beta. +::: + +OpenAI released a new API for a conversational agent like system called Assistant. + +You can interact with OpenAI Assistants using OpenAI tools or custom tools. When using exclusively OpenAI tools, you can just invoke the assistant directly and get final answers. When using custom tools, you can run the assistant and tool execution loop using the built-in `AgentExecutor` or write your own executor. +OpenAI assistants currently have access to two tools hosted by OpenAI: [code interpreter](https://platform.openai.com/docs/assistants/tools/code-interpreter), and [knowledge retrieval](https://platform.openai.com/docs/assistants/tools/knowledge-retrieval). + +We've implemented the assistant API in LangChain with some helpful abstractions. In this guide we'll go over those, and show how to use them to create powerful assistants. + +## Creating an assistant + +Creating an assistant is easy. Use the `createAssistant` method and pass in a model ID, and optionally more parameters to further customize your assistant. + +```typescript +import { OpenAIAssistantRunnable } from "experimental/openai_assistant"; + +const assistant = await OpenAIAssistantRunnable.createAssistant({ + model: "gpt-4-1106-preview", +}); +const assistantResponse = await assistant.invoke({ + content: "Hello world!", +}); +console.log(assistantResponse); +/** + [ + { + id: 'msg_OBH60nkVI40V9zY2PlxMzbEI', + thread_id: 'thread_wKpj4cu1XaYEVeJlx4yFbWx5', + role: 'assistant', + content: [ + { + type: 'text', + value: 'Hello there! What can I do for you?' + } + ], + assistant_id: 'asst_RtW03Vs6laTwqSSMCQpVND7i', + run_id: 'run_4Ve5Y9fyKMcSxHbaNHOFvdC6', + } + ] + */ +``` + +If you have an existing assistant, you can pass it directly into the constructor: + +```typescript +const assistant = new OpenAIAssistantRunnable({ + assistantId: "asst_RtW03Vs6laTwqSSMCQpVND7i", + // asAgent: true +}); +``` + +In this next example we'll show how you can turn your assistant into an agent. + +## Assistant as an agent + +```typescript +import { AgentExecutor } from "langchain/agents"; +import { StructuredTool } from "langchain/tools"; +import { OpenAIAssistantRunnable } from "experimental/openai_assistant"; +``` + +The first step is to define a list of tools you want to pass to your assistant. +Here we'll only define one for simplicity's sake, however the assistant API allows for passing in a list of tools, and from there the model can use multiple tools at once. +Read more about the run steps lifecycle [here](https://platform.openai.com/docs/assistants/how-it-works/runs-and-run-steps) + +:::note +Only models released >= 1106 are able to use multiple tools at once. See the full list of OpenAI models [here](https://platform.openai.com/docs/models). +::: + +```typescript +function getCurrentWeather(location: string, _unit = "fahrenheit") { + if (location.toLowerCase().includes("tokyo")) { + return JSON.stringify({ location, temperature: "10", unit: "celsius" }); + } else if (location.toLowerCase().includes("san francisco")) { + return JSON.stringify({ location, temperature: "72", unit: "fahrenheit" }); + } else { + return JSON.stringify({ location, temperature: "22", unit: "celsius" }); + } +} +class WeatherTool extends StructuredTool { + schema = z.object({ + location: z.string().describe("The city and state, e.g. San Francisco, CA"), + unit: z.enum(["celsius", "fahrenheit"]).optional(), + }); + + name = "get_current_weather"; + + description = "Get the current weather in a given location"; + + constructor() { + super(...arguments); + } + + async _call(input: { location: string; unit: string }) { + const { location, unit } = input; + const result = getCurrentWeather(location, unit); + return result; + } +} +const tools = [new WeatherTool()]; +``` + +In the above code we've defined three things: + +- A function for the agent to call if the model requests it. +- A tool class which we'll pass to the `AgentExecutor` +- The tool list we can use to pass to our `OpenAIAssistantRunnable` and `AgentExecutor` + +Next, we construct the `OpenAIAssistantRunnable` and pass it to the `AgentExecutor`. + +```typescript +const agent = await OpenAIAssistantRunnable.createAssistant({ + model: "gpt-3.5-turbo-1106", + instructions: + "You are a weather bot. Use the provided functions to answer questions.", + name: "Weather Assistant", + tools, + asAgent: true, +}); +const agentExecutor = AgentExecutor.fromAgentAndTools({ + agent, + tools, +}); +``` + +Note how we're setting `asAgent` to `true`, this input parameter tells the `OpenAIAssistantRunnable` to return different, agent-acceptable outputs for actions or finished conversations. + +Above we're also doing something a little different from the first example by passing in input parameters for `instructions` and `name`. +These are optional parameters, with the instructions being passed as extra context to the model, and the name being used to identify the assistant in the OpenAI dashboard. + +Finally to invoke our executor we call the `.invoke` method in the exact same way as we did in the first example. + +```typescript +const assistantResponse = await agentExecutor.invoke({ + content: "What's the weather in Tokyo and San Francisco?", +}); +console.log(assistantResponse); +/** +{ + output: 'The current weather in San Francisco is 72°F, and in Tokyo, it is 10°C.' +} +*/ +``` + +Here we asked a question which contains two sub questions inside: `What's the weather in Tokyo?` and `What's the weather in San Francisco?`. +In order for the `OpenAIAssistantRunnable` to answer that it returned two sets of function call arguments for each question, demonstrating it's ability to call multiple functions at once. + +## Assistant tools + +OpenAI currently offers two tools for the assistant API: a [code interpreter](https://platform.openai.com/docs/assistants/tools/code-interpreter) and a [knowledge retrieval](https://platform.openai.com/docs/assistants/tools/knowledge-retrieval) tool. +You can offer these tools to the assistant simply by passing them in as part of the `tools` parameter when creating the assistant. + +```typescript +const assistant = await OpenAIAssistantRunnable.createAssistant({ + model: "gpt-3.5-turbo-1106", + instructions: + "You are a helpful assistant that provides answers to math problems.", + name: "Math Assistant", + tools: [{ type: "code_interpreter" }], +}); +``` + +Since we're passing `code_interpreter` as a tool, the assistant will now be able to execute Python code, allowing for more complex tasks normal LLMs are not capable of doing well, like math. + +```typescript +const assistantResponse = await assistant.invoke({ + content: "What's 10 - 4 raised to the 2.7", +}); +console.log(assistantResponse); +/** +[ + { + id: 'msg_OBH60nkVI40V9zY2PlxMzbEI', + thread_id: 'thread_wKpj4cu1XaYEVeJlx4yFbWx5', + role: 'assistant', + content: [ + { + type: 'text', + text: { + value: 'The result of 10 - 4 raised to the 2.7 is approximately -32.22.', + annotations: [] + } + } + ], + assistant_id: 'asst_RtW03Vs6laTwqSSMCQpVND7i', + run_id: 'run_4Ve5Y9fyKMcSxHbaNHOFvdC6', + } +] +*/ +``` + +Here the assistant was able to utilize the `code_interpreter` tool to calculate the answer to our question. diff --git a/environment_tests/test-exports-bun/src/entrypoints.js b/environment_tests/test-exports-bun/src/entrypoints.js index 910e629351d5..5404725a9335 100644 --- a/environment_tests/test-exports-bun/src/entrypoints.js +++ b/environment_tests/test-exports-bun/src/entrypoints.js @@ -90,6 +90,7 @@ export * from "langchain/util/document"; export * from "langchain/util/math"; export * from "langchain/util/time"; export * from "langchain/experimental/autogpt"; +export * from "langchain/experimental/openai_assistant"; export * from "langchain/experimental/babyagi"; export * from "langchain/experimental/generative_agents"; export * from "langchain/experimental/plan_and_execute"; diff --git a/environment_tests/test-exports-cf/src/entrypoints.js b/environment_tests/test-exports-cf/src/entrypoints.js index 910e629351d5..5404725a9335 100644 --- a/environment_tests/test-exports-cf/src/entrypoints.js +++ b/environment_tests/test-exports-cf/src/entrypoints.js @@ -90,6 +90,7 @@ export * from "langchain/util/document"; export * from "langchain/util/math"; export * from "langchain/util/time"; export * from "langchain/experimental/autogpt"; +export * from "langchain/experimental/openai_assistant"; export * from "langchain/experimental/babyagi"; export * from "langchain/experimental/generative_agents"; export * from "langchain/experimental/plan_and_execute"; diff --git a/environment_tests/test-exports-cjs/src/entrypoints.js b/environment_tests/test-exports-cjs/src/entrypoints.js index 5d901d5f1d8a..447aca01fe7c 100644 --- a/environment_tests/test-exports-cjs/src/entrypoints.js +++ b/environment_tests/test-exports-cjs/src/entrypoints.js @@ -90,6 +90,7 @@ const util_document = require("langchain/util/document"); const util_math = require("langchain/util/math"); const util_time = require("langchain/util/time"); const experimental_autogpt = require("langchain/experimental/autogpt"); +const experimental_openai_assistant = require("langchain/experimental/openai_assistant"); const experimental_babyagi = require("langchain/experimental/babyagi"); const experimental_generative_agents = require("langchain/experimental/generative_agents"); const experimental_plan_and_execute = require("langchain/experimental/plan_and_execute"); diff --git a/environment_tests/test-exports-esbuild/src/entrypoints.js b/environment_tests/test-exports-esbuild/src/entrypoints.js index da609092add0..b6503d4a9740 100644 --- a/environment_tests/test-exports-esbuild/src/entrypoints.js +++ b/environment_tests/test-exports-esbuild/src/entrypoints.js @@ -90,6 +90,7 @@ import * as util_document from "langchain/util/document"; import * as util_math from "langchain/util/math"; import * as util_time from "langchain/util/time"; import * as experimental_autogpt from "langchain/experimental/autogpt"; +import * as experimental_openai_assistant from "langchain/experimental/openai_assistant"; import * as experimental_babyagi from "langchain/experimental/babyagi"; import * as experimental_generative_agents from "langchain/experimental/generative_agents"; import * as experimental_plan_and_execute from "langchain/experimental/plan_and_execute"; diff --git a/environment_tests/test-exports-esm/src/entrypoints.js b/environment_tests/test-exports-esm/src/entrypoints.js index da609092add0..b6503d4a9740 100644 --- a/environment_tests/test-exports-esm/src/entrypoints.js +++ b/environment_tests/test-exports-esm/src/entrypoints.js @@ -90,6 +90,7 @@ import * as util_document from "langchain/util/document"; import * as util_math from "langchain/util/math"; import * as util_time from "langchain/util/time"; import * as experimental_autogpt from "langchain/experimental/autogpt"; +import * as experimental_openai_assistant from "langchain/experimental/openai_assistant"; import * as experimental_babyagi from "langchain/experimental/babyagi"; import * as experimental_generative_agents from "langchain/experimental/generative_agents"; import * as experimental_plan_and_execute from "langchain/experimental/plan_and_execute"; diff --git a/environment_tests/test-exports-vercel/src/entrypoints.js b/environment_tests/test-exports-vercel/src/entrypoints.js index 910e629351d5..5404725a9335 100644 --- a/environment_tests/test-exports-vercel/src/entrypoints.js +++ b/environment_tests/test-exports-vercel/src/entrypoints.js @@ -90,6 +90,7 @@ export * from "langchain/util/document"; export * from "langchain/util/math"; export * from "langchain/util/time"; export * from "langchain/experimental/autogpt"; +export * from "langchain/experimental/openai_assistant"; export * from "langchain/experimental/babyagi"; export * from "langchain/experimental/generative_agents"; export * from "langchain/experimental/plan_and_execute"; diff --git a/environment_tests/test-exports-vite/src/entrypoints.js b/environment_tests/test-exports-vite/src/entrypoints.js index 910e629351d5..5404725a9335 100644 --- a/environment_tests/test-exports-vite/src/entrypoints.js +++ b/environment_tests/test-exports-vite/src/entrypoints.js @@ -90,6 +90,7 @@ export * from "langchain/util/document"; export * from "langchain/util/math"; export * from "langchain/util/time"; export * from "langchain/experimental/autogpt"; +export * from "langchain/experimental/openai_assistant"; export * from "langchain/experimental/babyagi"; export * from "langchain/experimental/generative_agents"; export * from "langchain/experimental/plan_and_execute"; diff --git a/langchain/.gitignore b/langchain/.gitignore index 3a1dbb7ed6d2..d304224d985f 100644 --- a/langchain/.gitignore +++ b/langchain/.gitignore @@ -742,6 +742,9 @@ util/time.d.ts experimental/autogpt.cjs experimental/autogpt.js experimental/autogpt.d.ts +experimental/openai_assistant.cjs +experimental/openai_assistant.js +experimental/openai_assistant.d.ts experimental/babyagi.cjs experimental/babyagi.js experimental/babyagi.d.ts diff --git a/langchain/package.json b/langchain/package.json index 278e467b68a3..1afddcdf4522 100644 --- a/langchain/package.json +++ b/langchain/package.json @@ -754,6 +754,9 @@ "experimental/autogpt.cjs", "experimental/autogpt.js", "experimental/autogpt.d.ts", + "experimental/openai_assistant.cjs", + "experimental/openai_assistant.js", + "experimental/openai_assistant.d.ts", "experimental/babyagi.cjs", "experimental/babyagi.js", "experimental/babyagi.d.ts", @@ -2628,6 +2631,11 @@ "import": "./experimental/autogpt.js", "require": "./experimental/autogpt.cjs" }, + "./experimental/openai_assistant": { + "types": "./experimental/openai_assistant.d.ts", + "import": "./experimental/openai_assistant.js", + "require": "./experimental/openai_assistant.cjs" + }, "./experimental/babyagi": { "types": "./experimental/babyagi.d.ts", "import": "./experimental/babyagi.js", diff --git a/langchain/scripts/create-entrypoints.js b/langchain/scripts/create-entrypoints.js index 17eca09e038a..810ce2c63a4b 100644 --- a/langchain/scripts/create-entrypoints.js +++ b/langchain/scripts/create-entrypoints.js @@ -292,6 +292,7 @@ const entrypoints = { "util/time": "util/time", // experimental "experimental/autogpt": "experimental/autogpt/index", + "experimental/openai_assistant": "experimental/openai_assistant/index", "experimental/babyagi": "experimental/babyagi/index", "experimental/generative_agents": "experimental/generative_agents/index", "experimental/plan_and_execute": "experimental/plan_and_execute/index", diff --git a/langchain/src/agents/agent.ts b/langchain/src/agents/agent.ts index f979ff7bf887..547b52dad270 100644 --- a/langchain/src/agents/agent.ts +++ b/langchain/src/agents/agent.ts @@ -124,23 +124,50 @@ export abstract class BaseSingleActionAgent extends BaseAgent { ): Promise; } +/** + * Abstract base class for multi-action agents in LangChain. Extends the + * BaseAgent class and provides additional functionality specific to + * multi-action agents. + */ +export abstract class BaseMultiActionAgent extends BaseAgent { + _agentActionType(): string { + return "multi" as const; + } + + /** + * Decide what to do, given some input. + * + * @param steps - Steps the LLM has taken so far, along with observations from each. + * @param inputs - User inputs. + * @param callbackManager - Callback manager. + * + * @returns Actions specifying what tools to use. + */ + abstract plan( + steps: AgentStep[], + inputs: ChainValues, + callbackManager?: CallbackManager + ): Promise; +} + +function isAgentAction(input: unknown): input is AgentAction { + return !Array.isArray(input) && (input as AgentAction)?.tool !== undefined; +} + /** * Class representing a single action agent which accepts runnables. * Extends the BaseSingleActionAgent class and provides methods for * planning agent actions with runnables. */ -export class RunnableAgent< - RunInput extends ChainValues & { - agent_scratchpad?: string | BaseMessage[]; - stop?: string[]; - }, - RunOutput extends AgentAction | AgentFinish -> extends BaseSingleActionAgent { +export class RunnableAgent extends BaseMultiActionAgent { protected lc_runnable = true; lc_namespace = ["langchain", "agents", "runnable"]; - runnable: Runnable; + runnable: Runnable< + ChainValues & { steps: AgentStep[] }, + AgentAction[] | AgentAction | AgentFinish + >; stop?: string[]; @@ -148,7 +175,7 @@ export class RunnableAgent< return []; } - constructor(fields: RunnableAgentInput) { + constructor(fields: RunnableAgentInput) { super(); this.runnable = fields.runnable; this.stop = fields.stop; @@ -156,9 +183,9 @@ export class RunnableAgent< async plan( steps: AgentStep[], - inputs: RunInput, + inputs: ChainValues, callbackManager?: CallbackManager - ): Promise { + ): Promise { const invokeInput = { ...inputs, steps }; const output = await this.runnable.invoke(invokeInput, { @@ -166,34 +193,12 @@ export class RunnableAgent< runName: "RunnableAgent", }); - return output; - } -} + if (isAgentAction(output)) { + return [output]; + } -/** - * Abstract base class for multi-action agents in LangChain. Extends the - * BaseAgent class and provides additional functionality specific to - * multi-action agents. - */ -export abstract class BaseMultiActionAgent extends BaseAgent { - _agentActionType(): string { - return "multi" as const; + return output; } - - /** - * Decide what to do, given some input. - * - * @param steps - Steps the LLM has taken so far, along with observations from each. - * @param inputs - User inputs. - * @param callbackManager - Callback manager. - * - * @returns Actions specifying what tools to use. - */ - abstract plan( - steps: AgentStep[], - inputs: ChainValues, - callbackManager?: CallbackManager - ): Promise; } /** diff --git a/langchain/src/agents/executor.ts b/langchain/src/agents/executor.ts index 62076359e0d2..324bd989babd 100644 --- a/langchain/src/agents/executor.ts +++ b/langchain/src/agents/executor.ts @@ -10,7 +10,6 @@ import { AgentAction, AgentFinish, AgentStep, - BaseMessage, ChainValues, } from "../schema/index.js"; import { CallbackManagerForChainRun } from "../callbacks/manager.js"; @@ -31,19 +30,14 @@ type ExtractToolType = T extends { ToolType: infer Tool } * AgentExecutor. It extends ChainInputs and includes additional * properties specific to agent execution. */ -export interface AgentExecutorInput< - RunInput extends ChainValues & { - agent_scratchpad?: string | BaseMessage[]; - stop?: string[]; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - } = any, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - RunOutput extends AgentAction | AgentFinish = any -> extends ChainInputs { +export interface AgentExecutorInput extends ChainInputs { agent: | BaseSingleActionAgent | BaseMultiActionAgent - | Runnable; + | Runnable< + ChainValues & { steps?: AgentStep[] }, + AgentAction[] | AgentAction | AgentFinish + >; tools: ExtractToolType[]; returnIntermediateSteps?: boolean; maxIterations?: number; @@ -54,6 +48,9 @@ export interface AgentExecutorInput< | ((e: OutputParserException | ToolInputParsingException) => string); } +// TODO: Type properly with { intermediateSteps?: AgentStep[] }; +export type AgentExecutorOutput = ChainValues; + /** * Tool that just returns the query. * Used for exception tracking. @@ -72,15 +69,7 @@ export class ExceptionTool extends Tool { * A chain managing an agent using tools. * @augments BaseChain */ -export class AgentExecutor< - RunInput extends ChainValues & { - agent_scratchpad?: string | BaseMessage[]; - stop?: string[]; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - } = any, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - RunOutput extends AgentAction | AgentFinish = any -> extends BaseChain { +export class AgentExecutor extends BaseChain { static lc_name() { return "AgentExecutor"; } @@ -123,7 +112,7 @@ export class AgentExecutor< return this.agent.returnValues; } - constructor(input: AgentExecutorInput) { + constructor(input: AgentExecutorInput) { let agent: BaseSingleActionAgent | BaseMultiActionAgent; if (Runnable.isRunnable(input.agent)) { agent = new RunnableAgent({ runnable: input.agent }); @@ -153,15 +142,7 @@ export class AgentExecutor< } /** Create from agent and a list of tools. */ - static fromAgentAndTools< - RunInput extends ChainValues & { - agent_scratchpad?: string | BaseMessage[]; - stop?: string[]; - }, - RunOutput extends AgentAction | AgentFinish - >( - fields: AgentExecutorInput - ): AgentExecutor { + static fromAgentAndTools(fields: AgentExecutorInput): AgentExecutor { return new AgentExecutor(fields); } @@ -179,14 +160,16 @@ export class AgentExecutor< async _call( inputs: ChainValues, runManager?: CallbackManagerForChainRun - ): Promise { + ): Promise { const toolsByName = Object.fromEntries( this.tools.map((t) => [t.name.toLowerCase(), t]) ); const steps: AgentStep[] = []; let iterations = 0; - const getOutput = async (finishStep: AgentFinish) => { + const getOutput = async ( + finishStep: AgentFinish + ): Promise => { const { returnValues } = finishStep; const additional = await this.agent.prepareForOutput(returnValues, steps); diff --git a/langchain/src/agents/tests/runnable.int.test.ts b/langchain/src/agents/tests/runnable.int.test.ts new file mode 100644 index 000000000000..995db66f7f1f --- /dev/null +++ b/langchain/src/agents/tests/runnable.int.test.ts @@ -0,0 +1,66 @@ +/* eslint-disable no-process-env */ +import { test } from "@jest/globals"; +import { AgentExecutor } from "../executor.js"; +import { ChatOpenAI } from "../../chat_models/openai.js"; +import { ChatPromptTemplate, MessagesPlaceholder } from "../../prompts/chat.js"; +import { + AIMessage, + AgentStep, + BaseMessage, + FunctionMessage, +} from "../../schema/index.js"; +import { RunnableSequence } from "../../schema/runnable/base.js"; +import { SerpAPI } from "../../tools/serpapi.js"; +import { formatToOpenAIFunction } from "../../tools/convert_to_openai.js"; +import { Calculator } from "../../tools/calculator.js"; +import { OpenAIFunctionsAgentOutputParser } from "../openai/output_parser.js"; + +test("Runnable variant", async () => { + const tools = [new Calculator(), new SerpAPI()]; + const model = new ChatOpenAI({ modelName: "gpt-4", temperature: 0 }); + + const prompt = ChatPromptTemplate.fromMessages([ + ["ai", "You are a helpful assistant"], + ["human", "{input}"], + new MessagesPlaceholder("agent_scratchpad"), + ]); + + const modelWithTools = model.bind({ + functions: [...tools.map((tool) => formatToOpenAIFunction(tool))], + }); + + const formatAgentSteps = (steps: AgentStep[]): BaseMessage[] => + steps.flatMap(({ action, observation }) => { + if ("messageLog" in action && action.messageLog !== undefined) { + const log = action.messageLog as BaseMessage[]; + return log.concat(new FunctionMessage(observation, action.tool)); + } else { + return [new AIMessage(action.log)]; + } + }); + + const runnableAgent = RunnableSequence.from([ + { + input: (i: { input: string; steps: AgentStep[] }) => i.input, + agent_scratchpad: (i: { input: string; steps: AgentStep[] }) => + formatAgentSteps(i.steps), + }, + prompt, + modelWithTools, + new OpenAIFunctionsAgentOutputParser(), + ]); + + const executor = AgentExecutor.fromAgentAndTools({ + agent: runnableAgent, + tools, + }); + + console.log("Loaded agent executor"); + + const query = "What is the weather in New York?"; + console.log(`Calling agent executor with query: ${query}`); + const result = await executor.call({ + input: query, + }); + console.log(result); +}); diff --git a/langchain/src/agents/types.ts b/langchain/src/agents/types.ts index d784724a4be4..a0fe0343be4d 100644 --- a/langchain/src/agents/types.ts +++ b/langchain/src/agents/types.ts @@ -24,16 +24,14 @@ export interface AgentInput { * Interface defining the input for creating an agent that uses runnables. * It includes the Runnable instance, and an optional list of stop strings. */ -export interface RunnableAgentInput< - RunInput extends ChainValues & { - agent_scratchpad?: string | BaseMessage[]; - stop?: string[]; - // eslint-disable-next-line @typescript-eslint/no-explicit-any - } = any, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - RunOutput extends AgentAction | AgentFinish = any -> { - runnable: Runnable; +export interface RunnableAgentInput { + runnable: Runnable< + ChainValues & { + agent_scratchpad?: string | BaseMessage[]; + stop?: string[]; + }, + AgentAction[] | AgentAction | AgentFinish + >; stop?: string[]; } diff --git a/langchain/src/experimental/openai_assistant/index.ts b/langchain/src/experimental/openai_assistant/index.ts new file mode 100644 index 000000000000..6b1de22abbea --- /dev/null +++ b/langchain/src/experimental/openai_assistant/index.ts @@ -0,0 +1,287 @@ +import { type ClientOptions, OpenAI as OpenAIClient } from "openai"; +import { Runnable } from "../../schema/runnable/base.js"; +import { sleep } from "../../util/time.js"; +import type { RunnableConfig } from "../../schema/runnable/config.js"; +import type { + OpenAIAssistantFinish, + OpenAIAssistantAction, + OpenAIToolType, +} from "./schema.js"; +import { StructuredTool } from "../../tools/base.js"; +import { formatToOpenAIAssistantTool } from "../../tools/convert_to_openai.js"; + +type ThreadMessage = OpenAIClient.Beta.Threads.ThreadMessage; +type RequiredActionFunctionToolCall = + OpenAIClient.Beta.Threads.RequiredActionFunctionToolCall; + +type ExtractRunOutput = + AsAgent extends true + ? OpenAIAssistantFinish | OpenAIAssistantAction[] + : ThreadMessage[] | RequiredActionFunctionToolCall[]; + +export type OpenAIAssistantRunnableInput< + AsAgent extends boolean | undefined = undefined +> = { + client?: OpenAIClient; + clientOptions?: ClientOptions; + assistantId: string; + pollIntervalMs?: number; + asAgent?: AsAgent; +}; + +export class OpenAIAssistantRunnable< + AsAgent extends boolean | undefined, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + RunInput extends Record = Record +> extends Runnable> { + lc_namespace = ["langchain", "experimental", "openai_assistant"]; + + private client: OpenAIClient; + + assistantId: string; + + pollIntervalMs = 1000; + + asAgent?: AsAgent; + + constructor(fields: OpenAIAssistantRunnableInput) { + super(fields); + this.client = fields.client ?? new OpenAIClient(fields?.clientOptions); + this.assistantId = fields.assistantId; + this.asAgent = fields.asAgent ?? this.asAgent; + } + + static async createAssistant({ + model, + name, + instructions, + tools, + client, + clientOptions, + asAgent, + pollIntervalMs, + }: Omit, "assistantId"> & { + model: string; + name?: string; + instructions?: string; + tools?: OpenAIToolType | Array; + }) { + const formattedTools = + tools?.map((tool) => { + // eslint-disable-next-line no-instanceof/no-instanceof + if (tool instanceof StructuredTool) { + return formatToOpenAIAssistantTool(tool); + } + return tool; + }) ?? []; + const oaiClient = client ?? new OpenAIClient(clientOptions); + const assistant = await oaiClient.beta.assistants.create({ + name, + instructions, + tools: formattedTools, + model, + }); + + return new this({ + client: oaiClient, + assistantId: assistant.id, + asAgent, + pollIntervalMs, + }); + } + + async invoke( + input: RunInput, + _options?: RunnableConfig + ): Promise> { + let run: OpenAIClient.Beta.Threads.Run; + if (this.asAgent && input.steps && input.steps.length > 0) { + const parsedStepsInput = await this._parseStepsInput(input); + run = await this.client.beta.threads.runs.submitToolOutputs( + parsedStepsInput.threadId, + parsedStepsInput.runId, + { + tool_outputs: parsedStepsInput.toolOutputs, + } + ); + } else if (!("threadId" in input)) { + const thread = { + messages: [ + { + role: "user", + content: input.content, + file_ids: input.fileIds, + metadata: input.messagesMetadata, + }, + ], + metadata: input.threadMetadata, + }; + run = await this._createThreadAndRun({ + ...input, + thread, + }); + } else if (!("runId" in input)) { + await this.client.beta.threads.messages.create(input.threadId, { + content: input.content, + role: "user", + file_ids: input.file_ids, + metadata: input.messagesMetadata, + }); + run = await this._createRun(input); + } else { + // Submitting tool outputs to an existing run, outside the AgentExecutor + // framework. + run = await this.client.beta.threads.runs.submitToolOutputs( + input.runId, + input.threadId, + { + tool_outputs: input.toolOutputs, + } + ); + } + + return this._getResponse(run.id, run.thread_id); + } + + private async _parseStepsInput(input: RunInput): Promise { + const { + action: { runId, threadId }, + } = input.steps[input.steps.length - 1]; + const run = await this._waitForRun(runId, threadId); + const toolCalls = run.required_action?.submit_tool_outputs.tool_calls; + if (!toolCalls) { + return input; + } + const toolOutputs = toolCalls.flatMap((toolCall) => { + const matchedAction = ( + input.steps as { + action: OpenAIAssistantAction; + observation: string; + }[] + ).find((step) => step.action.toolCallId === toolCall.id); + + return matchedAction + ? [ + { + output: matchedAction.observation, + tool_call_id: matchedAction.action.toolCallId, + }, + ] + : []; + }); + return { toolOutputs, runId, threadId } as unknown as RunInput; + } + + private async _createRun({ + instructions, + model, + tools, + metadata, + threadId, + }: RunInput) { + const run = this.client.beta.threads.runs.create(threadId, { + assistant_id: this.assistantId, + instructions, + model, + tools, + metadata, + }); + return run; + } + + private async _createThreadAndRun(input: RunInput) { + const params: Record = [ + "instructions", + "model", + "tools", + "run_metadata", + ] + .filter((key) => key in input) + .reduce((obj, key) => { + const newObj = obj; + newObj[key] = input[key]; + return newObj; + }, {} as Record); + const run = this.client.beta.threads.createAndRun({ + ...params, + thread: input.thread, + assistant_id: this.assistantId, + }); + return run; + } + + private async _waitForRun(runId: string, threadId: string) { + let inProgress = true; + let run = {} as OpenAIClient.Beta.Threads.Run; + while (inProgress) { + run = await this.client.beta.threads.runs.retrieve(threadId, runId); + inProgress = ["in_progress", "queued"].includes(run.status); + if (inProgress) { + await sleep(this.pollIntervalMs); + } + } + return run; + } + + private async _getResponse( + runId: string, + threadId: string + ): Promise>; + + private async _getResponse( + runId: string, + threadId: string + ): Promise< + | OpenAIAssistantFinish + | OpenAIAssistantAction[] + | ThreadMessage[] + | RequiredActionFunctionToolCall[] + > { + const run = await this._waitForRun(runId, threadId); + if (run.status === "completed") { + const messages = await this.client.beta.threads.messages.list(threadId, { + order: "asc", + }); + const newMessages = messages.data.filter((msg) => msg.run_id === runId); + if (!this.asAgent) { + return newMessages; + } + const answer = newMessages.flatMap((msg) => msg.content); + if (answer.every((item) => item.type === "text")) { + const answerString = answer + .map((item) => item.type === "text" && item.text.value) + .join("\n"); + return { + returnValues: { + output: answerString, + }, + log: "", + runId, + threadId, + }; + } + } else if (run.status === "requires_action") { + if (!this.asAgent) { + return run.required_action?.submit_tool_outputs.tool_calls ?? []; + } + const actions: OpenAIAssistantAction[] = []; + run.required_action?.submit_tool_outputs.tool_calls.forEach((item) => { + const functionCall = item.function; + const args = JSON.parse(functionCall.arguments); + actions.push({ + tool: functionCall.name, + toolInput: args, + toolCallId: item.id, + log: "", + runId, + threadId, + }); + }); + return actions; + } + const runInfo = JSON.stringify(run, null, 2); + throw new Error( + `Unexpected run status ${run.status}.\nFull run info:\n\n${runInfo}` + ); + } +} diff --git a/langchain/src/experimental/openai_assistant/schema.ts b/langchain/src/experimental/openai_assistant/schema.ts new file mode 100644 index 000000000000..10d4ce2658f4 --- /dev/null +++ b/langchain/src/experimental/openai_assistant/schema.ts @@ -0,0 +1,22 @@ +import type { OpenAI as OpenAIClient } from "openai"; +import type { AgentFinish, AgentAction } from "../../schema/index.js"; + +export type OpenAIAssistantFinish = AgentFinish & { + runId: string; + + threadId: string; +}; + +export type OpenAIAssistantAction = AgentAction & { + toolCallId: string; + + runId: string; + + threadId: string; +}; + +export type OpenAIToolType = Array< + | OpenAIClient.Beta.AssistantCreateParams.AssistantToolsCode + | OpenAIClient.Beta.AssistantCreateParams.AssistantToolsRetrieval + | OpenAIClient.Beta.AssistantCreateParams.AssistantToolsFunction +>; diff --git a/langchain/src/experimental/openai_assistant/tests/openai_assistant.int.test.ts b/langchain/src/experimental/openai_assistant/tests/openai_assistant.int.test.ts new file mode 100644 index 000000000000..605cdee1ad6d --- /dev/null +++ b/langchain/src/experimental/openai_assistant/tests/openai_assistant.int.test.ts @@ -0,0 +1,136 @@ +import { z } from "zod"; +import { OpenAI as OpenAIClient } from "openai"; +import { AgentExecutor } from "../../../agents/executor.js"; +import { StructuredTool } from "../../../tools/base.js"; +import { OpenAIAssistantRunnable } from "../index.js"; + +function getCurrentWeather(location: string, _unit = "fahrenheit") { + if (location.toLowerCase().includes("tokyo")) { + return JSON.stringify({ location, temperature: "10", unit: "celsius" }); + } else if (location.toLowerCase().includes("san francisco")) { + return JSON.stringify({ location, temperature: "72", unit: "fahrenheit" }); + } else { + return JSON.stringify({ location, temperature: "22", unit: "celsius" }); + } +} + +function convertWeatherToHumanReadable(location: string, temperature: string) { + if (temperature.length > 1) { + return JSON.stringify({ location, temperature, readable: "warm" }); + } + return JSON.stringify({ location, temperature, readable: "cold" }); +} + +class WeatherTool extends StructuredTool { + schema = z.object({ + location: z.string().describe("The city and state, e.g. San Francisco, CA"), + unit: z.enum(["celsius", "fahrenheit"]).optional(), + }); + + name = "get_current_weather"; + + description = "Get the current weather in a given location"; + + constructor() { + super(...arguments); + } + + async _call(input: { location: string; unit: string }) { + const { location, unit } = input; + const result = getCurrentWeather(location, unit); + return result; + } +} + +class HumanReadableChecker extends StructuredTool { + schema = z.object({ + location: z.string().describe("The city and state, e.g. San Francisco, CA"), + temperature: z.string().describe("The temperature in degrees"), + }); + + name = "get_human_readable_weather"; + + description = + "Check whether or not the weather in a given location is warm or cold"; + + constructor() { + super(...arguments); + } + + async _call(input: { location: string; temperature: string }) { + const { location, temperature } = input; + const result = convertWeatherToHumanReadable(location, temperature); + return result; + } +} + +test("OpenAIAssistantRunnable can be passed as an agent", async () => { + const tools = [new WeatherTool(), new HumanReadableChecker()]; + const agent = await OpenAIAssistantRunnable.createAssistant({ + model: "gpt-3.5-turbo-1106", + instructions: + "You are a weather bot. Use the provided functions to answer questions.", + name: "Weather Assistant", + tools, + asAgent: true, + }); + const agentExecutor = AgentExecutor.fromAgentAndTools({ + agent, + tools, + }); + const assistantResponse = await agentExecutor.invoke({ + content: + "What's the weather in San Francisco and Tokyo? And will it be warm or cold in those places?", + }); + console.log(assistantResponse); + /** + { + output: "The weather in San Francisco, CA is currently 72°F and it's warm. In Tokyo, Japan, the temperature is 10°C and it's also warm." + } + */ +}); + +test("OpenAIAssistantRunnable is invokeable", async () => { + const assistant = await OpenAIAssistantRunnable.createAssistant({ + model: "gpt-4", + instructions: + "You are a helpful assistant that provides answers to math problems.", + name: "Math Assistant", + tools: [{ type: "code_interpreter" }], + }); + const assistantResponse = await assistant.invoke({ + content: "What's 10 - 4 raised to the 2.7", + }); + console.log(assistantResponse); + /** + [ + { + id: 'msg_egqSo3AZTWJ0DAelzR6DdKbs', + object: 'thread.message', + created_at: 1699409656, + thread_id: 'thread_lAktOZkUetJ7Gl3hzMFdi42E', + role: 'assistant', + content: [ [Object] ], + file_ids: [], + assistant_id: 'asst_fPjLqVmN21EFGLNQb8iZckEy', + run_id: 'run_orPmWI9ri1HnqBXmX7LCWWax', + metadata: {} + } + ] + */ + const content = ( + assistantResponse as OpenAIClient.Beta.Threads.ThreadMessage[] + ).flatMap((res) => res.content); + console.log(content); + /** + [ + { + type: 'text', + text: { + value: '10 - 4 raised to the 2.7 is approximately -32.22.', + annotations: [] + } + } + ] + */ +}); diff --git a/langchain/src/load/import_map.ts b/langchain/src/load/import_map.ts index c31a66622d73..01f53c8d235b 100644 --- a/langchain/src/load/import_map.ts +++ b/langchain/src/load/import_map.ts @@ -91,6 +91,7 @@ export * as util__document from "../util/document.js"; export * as util__math from "../util/math.js"; export * as util__time from "../util/time.js"; export * as experimental__autogpt from "../experimental/autogpt/index.js"; +export * as experimental__openai_assistant from "../experimental/openai_assistant/index.js"; export * as experimental__babyagi from "../experimental/babyagi/index.js"; export * as experimental__generative_agents from "../experimental/generative_agents/index.js"; export * as experimental__plan_and_execute from "../experimental/plan_and_execute/index.js"; diff --git a/langchain/src/tools/convert_to_openai.ts b/langchain/src/tools/convert_to_openai.ts index 935bf6046306..d2bf7c12388d 100644 --- a/langchain/src/tools/convert_to_openai.ts +++ b/langchain/src/tools/convert_to_openai.ts @@ -31,3 +31,16 @@ export function formatToOpenAITool( }, }; } + +export function formatToOpenAIAssistantTool( + tool: StructuredTool +): OpenAIClient.Beta.AssistantCreateParams.AssistantToolsFunction { + return { + type: "function", + function: { + name: tool.name, + description: tool.description, + parameters: zodToJsonSchema(tool.schema), + }, + }; +} diff --git a/langchain/tsconfig.json b/langchain/tsconfig.json index 4aaa26619fec..b5dc042a417e 100644 --- a/langchain/tsconfig.json +++ b/langchain/tsconfig.json @@ -280,6 +280,7 @@ "src/util/math.ts", "src/util/time.ts", "src/experimental/autogpt/index.ts", + "src/experimental/openai_assistant/index.ts", "src/experimental/babyagi/index.ts", "src/experimental/generative_agents/index.ts", "src/experimental/plan_and_execute/index.ts",