-
Notifications
You must be signed in to change notification settings - Fork 549
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #233 from langchain-ai/jacob/langgraph-cli
Update instructions to use LangGraph.js CLI
- Loading branch information
Showing
18 changed files
with
949 additions
and
158 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,90 @@ | ||
import { expect } from "vitest"; | ||
import * as ls from "langsmith/vitest"; | ||
import { z } from "zod"; | ||
import { ChatOpenAI } from "@langchain/openai"; | ||
|
||
import { graph } from "../src/agent/open-canvas/index"; | ||
import { QUERY_ROUTING_DATA } from "./data/query_routing"; | ||
import { CODEGEN_DATA } from "./data/codegen"; | ||
|
||
ls.describe("query routing", () => { | ||
ls.test( | ||
"routes followups with questions to update artifact", | ||
{ | ||
inputs: QUERY_ROUTING_DATA.inputs, | ||
referenceOutputs: QUERY_ROUTING_DATA.referenceOutputs, | ||
}, | ||
async ({ inputs, referenceOutputs }) => { | ||
const generatePathNode = graph.nodes.generatePath; | ||
const res = await generatePathNode.invoke(inputs, { | ||
configurable: { | ||
customModelName: "gpt-4o-mini", | ||
}, | ||
}); | ||
ls.logOutputs(res); | ||
expect(res).toEqual(referenceOutputs); | ||
} | ||
); | ||
}); | ||
|
||
const qualityEvaluator = async (params: { | ||
inputs: string; | ||
outputs: string; | ||
}) => { | ||
const judge = new ChatOpenAI({ model: "gpt-4o" }).withStructuredOutput( | ||
z.object({ | ||
justification: z | ||
.string() | ||
.describe("reasoning for why you are assigning a given quality score"), | ||
quality_score: z | ||
.number() | ||
.describe( | ||
"quality score for how well the generated code answers the query." | ||
), | ||
}), | ||
{ | ||
name: "judge", | ||
} | ||
); | ||
const EVAL_PROMPT = [ | ||
`Given the following user query and generated code, judge whether the`, | ||
`code satisfies the user's query. Return a quality score between 1 and 10,`, | ||
`where a 1 would be completely irrelevant to the user's input, and 10 would be a perfectly accurate code sample.`, | ||
`A 5 would be a code sample that is partially on target, but is missing some aspect of a user's request.`, | ||
`Justify your answer.\n`, | ||
`<query>\n${params.inputs}\n</query>\n`, | ||
`<generated_code>\n${params.outputs}\n</generated_code>`, | ||
].join(" "); | ||
const res = await judge.invoke(EVAL_PROMPT); | ||
return { | ||
key: "quality", | ||
score: res.quality_score, | ||
comment: res.justification, | ||
}; | ||
}; | ||
|
||
ls.describe("codegen", () => { | ||
ls.test( | ||
"generate code with an LLM agent when asked", | ||
{ | ||
inputs: CODEGEN_DATA.inputs, | ||
referenceOutputs: {}, | ||
}, | ||
async ({ inputs }) => { | ||
const generateArtifactNode = graph.nodes.generateArtifact; | ||
const res = await generateArtifactNode.invoke(inputs, { | ||
configurable: { | ||
customModelName: "gpt-4o-mini", | ||
}, | ||
}); | ||
ls.logOutputs(res); | ||
const generatedCode = (res.artifact?.contents[0] as any).code; | ||
expect(generatedCode).toBeDefined(); | ||
const wrappedEvaluator = ls.wrapEvaluator(qualityEvaluator); | ||
await wrappedEvaluator({ | ||
inputs: inputs.messages[0].content, | ||
outputs: generatedCode, | ||
}); | ||
} | ||
); | ||
}); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,10 @@ | ||
import { HumanMessage } from "@langchain/core/messages"; | ||
|
||
export const CODEGEN_DATA: Record<string, any> = { | ||
inputs: { | ||
messages: [ | ||
new HumanMessage("Write me code for an LLM agent that does scraping"), | ||
], | ||
next: "generateArtifact", | ||
}, | ||
}; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,30 @@ | ||
import { AIMessage, HumanMessage } from "@langchain/core/messages"; | ||
|
||
export const QUERY_ROUTING_DATA: Record<string, any> = { | ||
inputs: { | ||
messages: [ | ||
new HumanMessage( | ||
"generate code for an LLM agent that can scrape the web" | ||
), | ||
new AIMessage( | ||
"I've crafted a web scraper for you that fetches and parses content from a specified URL. Let me know if you need any modifications or additional features!" | ||
), | ||
new HumanMessage("Where's the LLM?"), | ||
], | ||
artifact: { | ||
currentIndex: 1, | ||
contents: [ | ||
{ | ||
index: 1, | ||
type: "code" as const, | ||
title: "Web Scraper LLM Agent", | ||
code: "import requests\nfrom bs4 import BeautifulSoup\n\nclass WebScraper:\n def __init__(self, url):\n self.url = url\n self.content = None\n\n def fetch_content(self):\n try:\n response = requests.get(self.url)\n response.raise_for_status() # Check for HTTP errors\n self.content = response.text\n except requests.RequestException as e:\n print(f\"Error fetching {self.url}: {e}\")\n\n def parse_content(self):\n if self.content:\n soup = BeautifulSoup(self.content, 'html.parser')\n return soup\n else:\n print(\"No content to parse. Please fetch content first.\")\n return None\n\n def scrape(self):\n self.fetch_content()\n return self.parse_content()\n\n# Example usage:\nif __name__ == '__main__':\n url = 'https://example.com'\n scraper = WebScraper(url)\n parsed_content = scraper.scrape()\n print(parsed_content)", | ||
language: "python" as const, | ||
}, | ||
], | ||
}, | ||
}, | ||
referenceOutputs: { | ||
next: "rewriteArtifact", | ||
}, | ||
}; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
import { defineConfig } from "vitest/config"; | ||
import path from "path"; | ||
|
||
export default defineConfig({ | ||
test: { | ||
include: ["**/*.eval.?(c|m)[jt]s"], | ||
reporters: ["langsmith/vitest/reporter"], | ||
setupFiles: ["dotenv/config"], | ||
}, | ||
resolve: { | ||
alias: { | ||
"@": path.resolve(__dirname, "./src"), | ||
}, | ||
}, | ||
}); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.