From 0c156d40a428d6a5824504dbd3361735cb12e5a2 Mon Sep 17 00:00:00 2001 From: bracesproul Date: Mon, 29 Apr 2024 13:22:53 -0700 Subject: [PATCH] update --- .../faq/evaluator-implementations.mdx | 45 ++-- docs/evaluation/faq/manage-datasets.mdx | 134 ++++++---- docs/index.mdx | 92 +++---- docs/monitoring/faq/threads.mdx | 2 +- docs/test.py | 28 ++ .../faq/customizing_trace_attributes.mdx | 242 +++++++++++------- docs/tracing/faq/logging_and_viewing.mdx | 220 ++++++++++------ docs/tracing/faq/logging_feedback.mdx | 31 ++- docs/tracing/faq/querying_feedback.mdx | 80 ++++-- docs/tracing/faq/querying_traces.mdx | 163 +++++++----- 10 files changed, 642 insertions(+), 395 deletions(-) create mode 100644 docs/test.py diff --git a/docs/evaluation/faq/evaluator-implementations.mdx b/docs/evaluation/faq/evaluator-implementations.mdx index 80d3476d..be002899 100644 --- a/docs/evaluation/faq/evaluator-implementations.mdx +++ b/docs/evaluation/faq/evaluator-implementations.mdx @@ -57,10 +57,12 @@ Three QA evaluators you can load are: `"qa"`, `"context_qa"`, `"cot_qa"`. Based , @@ -79,7 +81,8 @@ out the reference docs for more information on the expected prompt format. tabs={[ PythonBlock(`from langchain.chat_models import ChatAnthropic from langchain_core.prompts.prompt import PromptTemplate -from langsmith.evaluation import LangChainStringEvaluator\n +from langsmith.evaluation import LangChainStringEvaluator + _PROMPT_TEMPLATE = """You are an expert professor specialized in grading students' answers to questions. You are grading the following question: {input} @@ -89,11 +92,13 @@ You are grading the following predicted answer: {prediction} Respond with CORRECT or INCORRECT: Grade: -"""\n +""" + PROMPT = PromptTemplate( input_variables=["input", "reference", "prediction"], template=_PROMPT_TEMPLATE ) -eval_llm = ChatAnthropic(temperature=0.0)\n +eval_llm = ChatAnthropic(temperature=0.0) + qa_evaluator = LangChainStringEvaluator("qa", config={"llm": eval_llm, "prompt": PROMPT}) context_qa_evaluator = LangChainStringEvaluator("context_qa", config={"llm": eval_llm}) cot_qa_evaluator = LangChainStringEvaluator("cot_qa", config={"llm": eval_llm}) @@ -117,7 +122,8 @@ If you don't have ground truth reference labels, you can evaluate your run again , @@ -180,7 +187,8 @@ If you have ground truth reference labels, you can evaluate your run against cus , @@ -236,7 +245,8 @@ Evaluating extraction and function calling applications often comes down to vali , data="", @@ -277,7 +288,8 @@ To measure the similarity between a predicted string and a reference, you can us , data="", @@ -315,14 +328,16 @@ The pattern is provided as a string in the example outputs of the dataset. The e , data="", diff --git a/docs/evaluation/faq/manage-datasets.mdx b/docs/evaluation/faq/manage-datasets.mdx index 76fc9d8e..45835efd 100644 --- a/docs/evaluation/faq/manage-datasets.mdx +++ b/docs/evaluation/faq/manage-datasets.mdx @@ -65,19 +65,26 @@ Note that you can add arbitrary metadata to each example, such as a note or a so @@ -127,24 +144,28 @@ To create datasets from existing runs, you can use the same approach. Below is a @@ -254,8 +286,10 @@ You can programmatically fetch the datasets from LangSmith using the `list_datas @@ -296,7 +330,8 @@ You can filter datasets by type. Below is an example querying for chat datasets. diff --git a/docs/index.mdx b/docs/index.mdx index e98ebd28..6452f9f8 100644 --- a/docs/index.mdx +++ b/docs/index.mdx @@ -102,34 +102,36 @@ client = Client() dataset_name = "Sample Dataset" dataset = client.create_dataset( -dataset_name, description="A sample dataset in LangSmith." + dataset_name, description="A sample dataset in LangSmith." ) client.create_examples( -inputs=[ -{"postfix": "to LangSmith"}, -{"postfix": "to Evaluations in LangSmith"}, -], -outputs=[ -{"output": "Welcome to LangSmith"}, -{"output": "Welcome to Evaluations in LangSmith"}, -], -dataset_id=dataset.id, + inputs=[ + {"postfix": "to LangSmith"}, + {"postfix": "to Evaluations in LangSmith"}, + ], + outputs=[ + {"output": "Welcome to LangSmith"}, + {"output": "Welcome to Evaluations in LangSmith"}, + ], + dataset_id=dataset.id, ) # Define your evaluator + def exact_match(run, example): -return {"score": run.outputs["output"] == example.outputs["output"]} + return {"score": run.outputs["output"] == example.outputs["output"]} + experiment_results = evaluate( -lambda input: "Welcome " + input["postfix"], # Your AI system goes here -data=dataset_name, # The data to predict and grade over -evaluators=[exact_match], # The evaluators to score the results -experiment_prefix="sample-experiment", # The name of the experiment -metadata={"version": "1.0.0", "revision_id": "beta"}, + lambda input: "Welcome " + input["postfix"], # Your AI system goes here + data=dataset_name, # The data to predict and grade over + evaluators=[exact_match], # The evaluators to score the results + experiment_prefix="sample-experiment", # The name of the experiment + metadata={"version": "1.0.0", "revision_id": "beta"}, ) `), -TypeScriptBlock(`import { Client, Run, Example } from "langsmith"; + TypeScriptBlock(`import { Client, Run, Example } from "langsmith"; import { runOnDataset } from "langchain/smith"; import { EvaluationResult } from "langsmith/evaluation"; @@ -138,46 +140,46 @@ const client = new Client(); // Define dataset: these are your test cases const datasetName = "Sample Dataset"; const dataset = await client.createDataset(datasetName, { -description: "A sample dataset in LangSmith.", + description: "A sample dataset in LangSmith.", }); await client.createExamples({ -inputs: [ -{ postfix: "to LangSmith" }, -{ postfix: "to Evaluations in LangSmith" }, -], -outputs: [ -{ output: "Welcome to LangSmith" }, -{ output: "Welcome to Evaluations in LangSmith" }, -], -datasetId: dataset.id, + inputs: [ + { postfix: "to LangSmith" }, + { postfix: "to Evaluations in LangSmith" }, + ], + outputs: [ + { output: "Welcome to LangSmith" }, + { output: "Welcome to Evaluations in LangSmith" }, + ], + datasetId: dataset.id, }); // Define your evaluator const exactMatch = async ({ -run, -example, + run, + example, }: { -run: Run; -example?: Example; + run: Run; + example?: Example; }): Promise => { -return { -key: "exact_match", -score: run.outputs?.output === example?.outputs?.output ? 1 : 0, -}; + return { + key: "exact_match", + score: run.outputs?.output === example?.outputs?.output ? 1 : 0, + }; }; await runOnDataset( -(input: { postfix: string }) => ({ output: \`Welcome $\{input.postfix\}\` }), // Your AI system goes here -datasetName, // The data to predict and grade over -{ -evaluationConfig: { customEvaluators: [exactMatch] }, -projectMetadata: { -version: "1.0.0", -revision_id: "beta", -}, -} + (input: { postfix: string }) => ({ output: \`Welcome $\{input.postfix\}\` }), // Your AI system goes here + datasetName, // The data to predict and grade over + { + evaluationConfig: { customEvaluators: [exactMatch] }, + projectMetadata: { + version: "1.0.0", + revision_id: "beta", + }, + } ); -`) +`), ]} groupId="client-language" /> diff --git a/docs/monitoring/faq/threads.mdx b/docs/monitoring/faq/threads.mdx index b7f26fee..cda981cd 100644 --- a/docs/monitoring/faq/threads.mdx +++ b/docs/monitoring/faq/threads.mdx @@ -47,7 +47,7 @@ response = assistant(messages, langsmith_extra={"metadata": {"session_id": sessi messages = messages + [response, {"role": "user", "content": "what's my name?"}] response = assistant(messages, langsmith_extra={"metadata": {"session_id": session_id}}) `), - TypeScriptBlock(`from langchain_anthropic import ChatAnthropic + PythonBlock(`from langchain_anthropic import ChatAnthropic from langchain_core.prompts import ChatPromptTemplate from langchain_core.messages import HumanMessage, AIMessage from langchain_core.output_parsers import StrOutputParser diff --git a/docs/test.py b/docs/test.py new file mode 100644 index 00000000..19630a58 --- /dev/null +++ b/docs/test.py @@ -0,0 +1,28 @@ +from langsmith import Client + +example_inputs = [ + ("What is the largest mammal?", "The blue whale"), + ("What do mammals and birds have in common?", "They are both warm-blooded"), + ("What are reptiles known for?", "Having scales"), + ( + "What's the main characteristic of amphibians?", + "They live both in water and on land", + ), +] + +client = Client() +dataset_name = "Elementary Animal Questions" + +# Storing inputs in a dataset lets us +# run chains and LLMs over a shared set of examples. +dataset = client.create_dataset( + dataset_name=dataset_name, + description="Questions and answers about animal phylogenetics.", +) +for input_prompt, output_answer in example_inputs: + client.create_example( + inputs={"question": input_prompt}, + outputs={"answer": output_answer}, + metadata={"source": "Wikipedia"}, + dataset_id=dataset.id, + ) \ No newline at end of file diff --git a/docs/tracing/faq/customizing_trace_attributes.mdx b/docs/tracing/faq/customizing_trace_attributes.mdx index abc203bb..68a73f3e 100644 --- a/docs/tracing/faq/customizing_trace_attributes.mdx +++ b/docs/tracing/faq/customizing_trace_attributes.mdx @@ -32,41 +32,53 @@ When global environment variables are too broad, you can also set the project na tabs={[ PythonBlock(`import openai from langsmith import traceable -from langsmith.run_trees import RunTree\n -client = openai.Client()\n +from langsmith.run_trees import RunTree + +client = openai.Client() + messages = [ {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Hello!"} -]\n + {"role": "user", "content": "Hello!"}, +] + + # Use the @traceable decorator with the 'project_name' parameter to log traces to LangSmith # Ensure that the LANGCHAIN_TRACING_V2 environment variables are set for @traceable to work @traceable( run_type="llm", name="OpenAI Call Decorator", ) -def call_openai( - messages: list[dict], model: str = "gpt-3.5-turbo" -) -> str: - return client.chat.completions.create( - model=model, - messages=messages, - ).choices[0].message.content\n +def call_openai(messages: list[dict], model: str = "gpt-3.5-turbo") -> str: + return ( + client.chat.completions.create( + model=model, + messages=messages, + ) + .choices[0] + .message.content + ) + + # You can specify the Project via the project_name parameter call_openai( messages, # highlight-next-line langsmith_extra={"project_name": "My Project"}, -)\n +) + # The wrapped OpenAI client accepts all the same langsmith_extra parameters # as @traceable decorated functions, and logs traces to langsmith automatically from langsmith import wrappers + wrapped_client = wrappers.wrap_openai(client) wrapped_client.chat.completions.create( model="gpt-3.5-turbo", messages=messages, # highlight-next-line langsmith_extra={"project_name": "My Project"}, -)\n\n +) + + # Alternatively, create a RunTree object # You can set the project name using the project_name parameter rt = RunTree( @@ -74,7 +86,7 @@ rt = RunTree( name="OpenAI Call RunTree", inputs={"messages": messages}, # highlight-next-line - project_name="My Project" + project_name="My Project", ) chat_completion = client.chat.completions.create( model="gpt-3.5-turbo", @@ -85,28 +97,32 @@ rt.end(outputs=chat_completion) rt.post() `), TypeScriptBlock(`import OpenAI from "openai"; -import { RunTree } from "langsmith";\n -const client = new OpenAI()\n +import { RunTree } from "langsmith"; + +const client = new OpenAI(); + const messages = [ - {role: "system", content: "You are a helpful assistant."}, - {role: "user", content: "Hello!"} -]\n + { role: "system", content: "You are a helpful assistant." }, + { role: "user", content: "Hello!" }, +]; + // Create a RunTree object // You can set the project name using the project_name parameter const rt = new RunTree({ - run_type: "llm", - name: "OpenAI Call RunTree", - inputs: { messages }, - // highlight-next-line - project_name: "My Project" -}) + run_type: "llm", + name: "OpenAI Call RunTree", + inputs: { messages }, + // highlight-next-line + project_name: "My Project", +}); const chatCompletion = await client.chat.completions.create({ - model: "gpt-3.5-turbo", - messages: messages, + model: "gpt-3.5-turbo", + messages: messages, }); // End and submit the run -rt.end(chatCompletion) -await rt.postRun()`), +rt.end(chatCompletion); +await rt.postRun(); +`), LangChainPyBlock(`# You can set the project name for a specific tracer instance: from langchain.callbacks.tracers import LangChainTracer\n tracer = LangChainTracer(project_name="My Project") @@ -140,12 +156,16 @@ For more information on metadata and tags, see the [Concepts](/tracing/concepts) tabs={[ PythonBlock(`import openai from langsmith import traceable -from langsmith.run_trees import RunTree\n -client = openai.Client()\n +from langsmith.run_trees import RunTree + +client = openai.Client() + messages = [ {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Hello!"} -]\n + {"role": "user", "content": "Hello!"}, +] + + # Use the @traceable decorator with tags and metadata # Ensure that the LANGCHAIN_TRACING_V2 environment variables are set for @traceable to work @traceable( @@ -154,22 +174,30 @@ messages = [ # highlight-next-line tags=["my-tag"], # highlight-next-line - metadata={"my-key": "my-value"} + metadata={"my-key": "my-value"}, ) -def call_openai( - messages: list[dict], model: str = "gpt-3.5-turbo" -) -> str: - return client.chat.completions.create( - model=model, - messages=messages, - ).choices[0].message.content\n +def call_openai(messages: list[dict], model: str = "gpt-3.5-turbo") -> str: + return ( + client.chat.completions.create( + model=model, + messages=messages, + ) + .choices[0] + .message.content + ) + + call_openai( messages, - # You can provide tags and metadata at invocation time + # You can provide tags and metadata at invocation time # via the langsmith_extra parameter # highlight-next-line - langsmith_extra={"tags": ["my-other-tag"], "metadata": {"my-other-key": "my-value"}} -)\n + langsmith_extra={ + "tags": ["my-other-tag"], + "metadata": {"my-other-key": "my-value"}, + }, +) + # Alternatively, you can create a RunTree object with tags and metadata rt = RunTree( run_type="llm", @@ -178,7 +206,7 @@ rt = RunTree( # highlight-next-line tags=["my-tag"], # highlight-next-line - extra={"metadata": {"my-key": "my-value"}} + extra={"metadata": {"my-key": "my-value"}}, ) chat_completion = client.chat.completions.create( model="gpt-3.5-turbo", @@ -186,31 +214,36 @@ chat_completion = client.chat.completions.create( ) # End and submit the run rt.end(outputs=chat_completion) -rt.post()`), +rt.post() +`), TypeScriptBlock(`import OpenAI from "openai"; -import { RunTree } from "langsmith";\n -const client = new OpenAI();\n +import { RunTree } from "langsmith"; + +const client = new OpenAI(); + const messages = [ - {role: "system", content: "You are a helpful assistant."}, - {role: "user", content: "Hello!"} -]\n + { role: "system", content: "You are a helpful assistant." }, + { role: "user", content: "Hello!" }, +]; + // Create a RunTree object const rt = new RunTree({ - run_type: "llm", - name: "OpenAI Call RunTree", - inputs: { messages }, - // highlight-next-line - tags: ["my-tag"], - // highlight-next-line - extra: {metadata: {"my-key": "my-value"}} -}) + run_type: "llm", + name: "OpenAI Call RunTree", + inputs: { messages }, + // highlight-next-line + tags: ["my-tag"], + // highlight-next-line + extra: { metadata: { "my-key": "my-value" } }, +}); const chatCompletion = await client.chat.completions.create({ - model: "gpt-3.5-turbo", - messages: messages, + model: "gpt-3.5-turbo", + messages: messages, }); // End and submit the run -rt.end(chatCompletion) -await rt.postRun()`), +rt.end(chatCompletion); +await rt.postRun(); +`), LangChainPyBlock(`from langchain_openai import ChatOpenAI from langchain_core.prompts import ChatPromptTemplate from langchain_core.output_parsers import StrOutputParser\n @@ -249,12 +282,16 @@ When you create a run, you can specify a name for the run. This name is used to tabs={[ PythonBlock(`import openai from langsmith import traceable -from langsmith.run_trees import RunTree\n -client = openai.Client()\n +from langsmith.run_trees import RunTree + +client = openai.Client() + messages = [ {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Hello!"} -]\n + {"role": "user", "content": "Hello!"}, +] + + # Use the @traceable decorator with the name parameter # Ensure that the LANGCHAIN_TRACING_V2 environment variables are set for @traceable to work @traceable( @@ -262,14 +299,19 @@ messages = [ # highlight-next-line name="OpenAI Call Decorator", ) -def call_openai( - messages: list[dict], model: str = "gpt-3.5-turbo" -) -> str: - return client.chat.completions.create( - model=model, - messages=messages, - ).choices[0].message.content\n -call_openai(messages)\n +def call_openai(messages: list[dict], model: str = "gpt-3.5-turbo") -> str: + return ( + client.chat.completions.create( + model=model, + messages=messages, + ) + .choices[0] + .message.content + ) + + +call_openai(messages) + # Alternatively, use the name parameter of the RunTree object rt = RunTree( run_type="llm", @@ -283,28 +325,33 @@ chat_completion = client.chat.completions.create( ) # End and submit the run rt.end(outputs=chat_completion) -rt.post()`), +rt.post() +`), TypeScriptBlock(`import OpenAI from "openai"; -import { RunTree } from "langsmith";\n -const client = new OpenAI();\n +import { RunTree } from "langsmith"; + +const client = new OpenAI(); + const messages = [ - {role: "system", content: "You are a helpful assistant."}, - {role: "user", content: "Hello!"} -]\n + { role: "system", content: "You are a helpful assistant." }, + { role: "user", content: "Hello!" }, +]; + // Create a RunTree object const rt = new RunTree({ - run_type: "llm", - // highlight-next-line - name: "OpenAI Call RunTree", - inputs: { messages }, -}) + run_type: "llm", + // highlight-next-line + name: "OpenAI Call RunTree", + inputs: { messages }, +}); const chatCompletion = await client.chat.completions.create({ - model: "gpt-3.5-turbo", - messages: messages, + model: "gpt-3.5-turbo", + messages: messages, }); // End and submit the run -rt.end(chatCompletion) -await rt.postRun()`), +rt.end(chatCompletion); +await rt.postRun(); +`), LangChainPyBlock(`# When tracing within LangChain, run names default to the class name of the traced object (e.g., 'ChatOpenAI'). # (Note: this is not currently supported directly on LLM objects.) ... @@ -350,11 +397,13 @@ You can also customize and override this behavior for a given Client instance. T tabs={[ PythonBlock(`import openai from langsmith import Client -from langsmith.wrappers import wrap_openai\n +from langsmith.wrappers import wrap_openai + openai_client = wrap_openai(openai.Client()) langsmith_client = Client( hide_inputs=lambda inputs: {}, hide_outputs=lambda outputs: {} -)\n +) + # The linked run will have its metadata present, but the inputs will be hidden openai_client.chat.completions.create( model="gpt-3.5-turbo", @@ -363,7 +412,8 @@ openai_client.chat.completions.create( {"role": "user", "content": "Hello!"}, ], langsmith_extra={"client": langsmith_client}, -)\n +) + # The linked run will not have hidden inputs and outputs openai_client.chat.completions.create( model="gpt-3.5-turbo", @@ -375,11 +425,13 @@ openai_client.chat.completions.create( `), TypeScriptBlock(`import { Client } from "langsmith"; import { wrapOpenAI } from "langsmith/wrappers"; -import OpenAI from "openai";\n +import OpenAI from "openai"; + const langsmithClient = new Client({ hideInputs: (inputs) => ({}), hideOutputs: (outputs) => ({}), -});\n +}); + (async () => { // The linked run will have its metadata present, but the inputs will be hidden const filteredOAIClient = wrapOpenAI(new OpenAI(), { diff --git a/docs/tracing/faq/logging_and_viewing.mdx b/docs/tracing/faq/logging_and_viewing.mdx index 7c09fb3b..b2f17451 100644 --- a/docs/tracing/faq/logging_and_viewing.mdx +++ b/docs/tracing/faq/logging_and_viewing.mdx @@ -126,47 +126,63 @@ To log traces to a different project, see [this section](/tracing/faq/customizin tabs={[ PythonBlock(`import openai from langsmith import traceable -from langsmith.wrappers import wrap_openai\n -client = wrap_openai(openai.Client())\n +from langsmith.wrappers import wrap_openai + +client = wrap_openai(openai.Client()) + @traceable(run_type="tool", name="Retrieve Context") def my_tool(question: str) -> str: - return "During this morning's meeting, we solved all world conflict."\n + return "During this morning's meeting, we solved all world conflict." + @traceable(name="Chat Pipeline") def chat_pipeline(question: str): context = my_tool(question) messages = [ { "role": "system", "content": "You are a helpful assistant. Please respond to the user's request only based on the given context." }, - { "role": "user", "content": f"Question: {question}\nContext: {context}"} + { "role": "user", "content": f"Question: {question} +Context: {context}"} ] chat_completion = client.chat.completions.create( model="gpt-3.5-turbo", messages=messages ) - return chat_completion.choices[0].message.content\n + return chat_completion.choices[0].message.content + chat_pipeline("Can you summarize this morning's meetings?")`), TypeScriptBlock(`import OpenAI from "openai"; import { traceable } from "langsmith/traceable"; -import { wrapOpenAI } from "langsmith/wrappers";\n -const client = wrapOpenAI(new OpenAI());\n -const myTool = traceable(async (question: string) => { +import { wrapOpenAI } from "langsmith/wrappers"; + +const client = wrapOpenAI(new OpenAI()); + +const myTool = traceable( + async (question: string) => { return "During this morning's meeting, we solved all world conflict."; -}, { name: "Retrieve Context", run_type: "tool" });\n -const chatPipeline = traceable(async (question: string) => { + }, + { name: "Retrieve Context", run_type: "tool" } +); + +const chatPipeline = traceable( + async (question: string) => { const context = await myTool(question); const messages = [ - { - role: "system", - content: - "You are a helpful assistant. Please respond to the user's request only based on the given context.", - }, - { role: "user", content: \`Question: \${question} Context: \${context}\` }, + { + role: "system", + content: + "You are a helpful assistant. Please respond to the user's request only based on the given context.", + }, + { role: "user", content: \`Question: \${question} Context: \${context}\` }, ]; const chatCompletion = await client.chat.completions.create({ - model: "gpt-3.5-turbo", - messages: messages, + model: "gpt-3.5-turbo", + messages: messages, }); return chatCompletion.choices[0].message.content; -}, { name: "Chat Pipeline" });\n -await chatPipeline("Can you summarize this morning's meetings?");`), + }, + { name: "Chat Pipeline" } +); + +await chatPipeline("Can you summarize this morning's meetings?"); +`), ]} groupId="client-language" /> @@ -181,19 +197,25 @@ necessary for this method. tabs={[ PythonBlock(`import openai from langsmith.run_trees import RunTree + # This can be a user input to your app question = "Can you summarize this morning's meetings?" # Create a top-level run pipeline = RunTree( - name="Chat Pipeline", - run_type="chain", - inputs={"question": question} + name="Chat Pipeline", run_type="chain", inputs={"question": question} ) # This can be retrieved in a retrieval step context = "During this morning's meeting, we solved all world conflict." messages = [ - { "role": "system", "content": "You are a helpful assistant. Please respond to the user's request only based on the given context." }, - { "role": "user", "content": f"Question: {question}\\nContext: {context}"} + { + "role": "system", + "content": "You are a helpful assistant. Please respond to the user's request only based on the given context.", + }, + { + "role": "user", + "content": f"Question: {question}\ +Context: {context}", + }, ] # Create a child run child_llm_run = pipeline.create_child( @@ -210,39 +232,51 @@ chat_completion = client.chat.completions.create( child_llm_run.end(outputs=chat_completion) child_llm_run.post() pipeline.end(outputs={"answer": chat_completion.choices[0].message.content}) -pipeline.post()`), +pipeline.post() +`), TypeScriptBlock(`import OpenAI from "openai"; import { RunTree } from "langsmith"; // This can be a user input to your app const question = "Can you summarize this morning's meetings?"; const pipeline = new RunTree({ - name: "Chat Pipeline", - run_type: "chain", - inputs: { question } + name: "Chat Pipeline", + run_type: "chain", + inputs: { question }, }); // This can be retrieved in a retrieval step const context = "During this morning's meeting, we solved all world conflict."; const messages = [ - { role: "system", content: "You are a helpful assistant. Please respond to the user's request only based on the given context." }, - { role: "user", content: \`Question: \${question}\nContext: \${context}\` } + { + role: "system", + content: + "You are a helpful assistant. Please respond to the user's request only based on the given context.", + }, + { + role: "user", + content: \`Question: \${question} +Context: \${context}\`, + }, ]; // Create a child run const childRun = await pipeline.createChild({ - name: "OpenAI Call", - run_type: "llm", - inputs: { messages }, + name: "OpenAI Call", + run_type: "llm", + inputs: { messages }, }); // Generate a completion const client = new OpenAI(); const chatCompletion = await client.chat.completions.create({ - model: "gpt-3.5-turbo", - messages: messages, + model: "gpt-3.5-turbo", + messages: messages, }); // End the runs and log them childRun.end(chatCompletion); await childRun.postRun(); -pipeline.end({ outputs: { answer: chatCompletion.choices[0].message.content } }); -await pipeline.postRun();`), +pipeline.end({ + outputs: { answer: chatCompletion.choices[0].message.content }, +}); +await pipeline.postRun(); +`), ]} groupId="client-language" /> @@ -349,22 +383,25 @@ The example below shows how to get the run ID of a logged run using the LangSmit str: - return client.chat.completions.create( - model=model, - messages=messages, - ).choices[0].message.content +def call_openai(messages: list[dict], model: str = "gpt-3.5-turbo") -> str: + return ( + client.chat.completions.create( + model=model, + messages=messages, + ) + .choices[0] + .message.content + ) + + result = call_openai( messages, langsmith_extra={ "run_id": run_id, }, ) -print("Traceable Run ID: ", run_id)`), +print("Traceable Run ID: ", run_id) +`), TypeScriptBlock(`import OpenAI from "openai"; import { RunTree } from "langsmith"; -import {v4 as uuidv4} from "uuid";\n -const client = new OpenAI();\n +import { v4 as uuidv4 } from "uuid"; + +const client = new OpenAI(); + const messages = [ - {role: "system", content: "You are a helpful assistant."}, - {role: "user", content: "Is sunshine food for you?"} -];\n + { role: "system", content: "You are a helpful assistant." }, + { role: "user", content: "Is sunshine food for you?" }, +]; + const runId = uuidv4(); const rt = new RunTree({ - run_type: "llm", - name: "OpenAI Call RunTree", - inputs: { messages }, - id: runId -}) + run_type: "llm", + name: "OpenAI Call RunTree", + inputs: { messages }, + id: runId, +}); const chatCompletion = await client.chat.completions.create({ - model: "gpt-3.5-turbo", - messages: messages, + model: "gpt-3.5-turbo", + messages: messages, }); -rt.end(chatCompletion) -await rt.postRun() -console.log("Run ID: ", runId);`), +rt.end(chatCompletion); +await rt.postRun(); +console.log("Run ID: ", runId); +`), APIBlock(`import openai import requests\n from datetime import datetime @@ -467,13 +519,16 @@ use the LangSmith client. Below is an example. To get the run ID of a run, you c ") -print(run.url)`), - TypeScriptBlock(`import { Client } from "langsmith";\n +print(run.url) +`), + TypeScriptBlock(`import { Client } from "langsmith"; + const client = new Client(); -const runUrl = await client.getRunUrl({runId: ""}); +const runUrl = await client.getRunUrl({ runId: "" }); console.log(runUrl); `), ]} @@ -488,12 +543,15 @@ Below is an example using the SDK: "}); +await client.deleteProject({ projectName: "" }); `), ]} groupId="client-language" diff --git a/docs/tracing/faq/logging_feedback.mdx b/docs/tracing/faq/logging_feedback.mdx index 31b30db5..6e59cf98 100644 --- a/docs/tracing/faq/logging_feedback.mdx +++ b/docs/tracing/faq/logging_feedback.mdx @@ -24,28 +24,31 @@ To get the run_id of a logged run, see [this guide](/tracing/faq/logging_and_vie @@ -37,13 +41,15 @@ Below are some examples of ways to list feedback using the available arguments: "])`), + PythonBlock(`run_feedback = client.list_feedback(run_ids=[""]) +`), TypeScriptBlock(`const runFeedback: Feedback[] = []; for await (const feedback of client.listFeedback({ runIds: [""], })) { runFeedback.push(feedback); -}`), +} +`), ]} groupId="client-language" /> @@ -60,7 +66,8 @@ for await (const feedback of client.listFeedback({ feedbackKeys: ["correctness"], })) { correctnessFeedback.push(feedback); -}`), +} +`), ]} groupId="client-language" /> @@ -77,7 +84,8 @@ for await (const feedback of client.listFeedback({ feedbackSourceTypes: ["model"], })) { modelFeedback.push(feedback); -}`), +} +`), ]} groupId="client-language" /> @@ -93,13 +101,19 @@ After querying for a set of runs, you can compare the model-generated and human ", filter='gt(start_time, "2023-07-15T00:00:00Z")')\n +runs = client.list_runs( + project_name="", filter='gt(start_time, "2023-07-15T00:00:00Z")' +) + # Extract run IDs -run_ids = [run.id for run in runs]\n +run_ids = [run.id for run in runs] + # Fetch model-generated feedback for the runs -model_feedback = client.list_feedback(run_ids=run_ids, feedback_source_type=["model"])\n +model_feedback = client.list_feedback(run_ids=run_ids, feedback_source_type=["model"]) + # Fetch human feedback for the runs -human_feedback = client.list_feedback(run_ids=run_ids, feedback_source_type=["api"])`), +human_feedback = client.list_feedback(run_ids=run_ids, feedback_source_type=["api"]) +`), TypeScriptBlock(`// Query for runs const runs: Run[] = []; for await (const run of client.listRuns({ @@ -107,9 +121,11 @@ for await (const run of client.listRuns({ filter: 'gt(start_time, "2023-07-15T00:00:00Z")', })) { runs.push(run); -}\n +} + // Extract run IDs -const runIds = runs.map(run => run.id);\n +const runIds = runs.map((run) => run.id); + // Fetch model-generated feedback for the runs const modelFeedback: Feedback[] = []; for await (const feedback of client.listFeedback({ @@ -117,7 +133,8 @@ for await (const feedback of client.listFeedback({ feedbackSourceTypes: ["model"], })) { modelFeedback.push(feedback); -}\n +} + // Fetch human feedback for the runs const humanFeedback: Feedback[] = []; for await (const feedback of client.listFeedback({ @@ -125,7 +142,8 @@ for await (const feedback of client.listFeedback({ feedbackSourceTypes: ["api"], })) { humanFeedback.push(feedback); -}`), +} +`), ]} groupId="client-language" /> @@ -137,13 +155,21 @@ If you're interested in analyzing feedback for a specific key, such as 'correctn ", filter='gt(start_time, "2023-07-15T00:00:00Z")')\n -# Extract run IDs -run_ids = [run.id for run in runs]\n +runs = client.list_runs( + project_name="", filter='gt(start_time, "2023-07-15T00:00:00Z")' +) + +# Extract run IDs +run_ids = [run.id for run in runs] + # Fetch correctness feedback for the runs -correctness_feedback = client.list_feedback(run_ids=run_ids, feedback_key=["correctness"])\n +correctness_feedback = client.list_feedback( + run_ids=run_ids, feedback_key=["correctness"] +) + # Analyze the correctness scores -scores = [feedback.score for feedback in correctness_feedback]`), +scores = [feedback.score for feedback in correctness_feedback] +`), TypeScriptBlock(`// Query for runs const runs: Run[] = []; for await (const run of client.listRuns({ @@ -151,9 +177,11 @@ for await (const run of client.listRuns({ filter: 'gt(start_time, "2023-07-15T00:00:00Z")', })) { runs.push(run); -}\n +} + // Extract run IDs -const runIds = runs.map(run => run.id);\n +const runIds = runs.map((run) => run.id); + // Fetch correctness feedback for the runs const correctnessFeedback: Feedback[] = []; for await (const feedback of client.listFeedback({ @@ -161,9 +189,11 @@ for await (const feedback of client.listFeedback({ feedbackKeys: ["correctness"], })) { correctnessFeedback.push(feedback); -}\n +} + // Analyze the correctness scores -const scores = correctnessFeedback.map(feedback => feedback.score);`), +const scores = correctnessFeedback.map((feedback) => feedback.score); +`), ]} groupId="client-language" /> diff --git a/docs/tracing/faq/querying_traces.mdx b/docs/tracing/faq/querying_traces.mdx index 099f01dd..166663e4 100644 --- a/docs/tracing/faq/querying_traces.mdx +++ b/docs/tracing/faq/querying_traces.mdx @@ -38,10 +38,14 @@ For simple queries, such as filtering by project, run time, name, or run ID's, y @@ -61,7 +65,8 @@ for await (const run of client.listRuns({ projectName: "", })) { projectRuns.push(run); -};`), +} +`), ]} groupId="client-language" /> @@ -74,7 +79,8 @@ for await (const run of client.listRuns({ project_name="", start_time=datetime.now() - timedelta(days=1), run_type="llm", -)`), +) +`), TypeScriptBlock(`const todaysLlmRuns: Run[] = []; for await (const run of client.listRuns({ projectName: "", @@ -82,7 +88,8 @@ for await (const run of client.listRuns({ runType: "llm", })) { todaysLlmRuns.push(run); -};`), +} +`), ]} groupId="client-language" /> @@ -93,17 +100,16 @@ Root runs (or run traces), are runs that have no parents. These are assigned an ", is_root=True) +`), TypeScriptBlock(`const rootRuns: Run[] = []; for await (const run of client.listRuns({ projectName: "", isRoot: 1, })) { rootRuns.push(run); -};`), +} +`), ]} groupId="client-language" /> @@ -121,7 +127,8 @@ for await (const run of client.listRuns({ error: false, })) { correctRuns.push(run); -};`), +} +`), ]} groupId="client-language" /> @@ -132,8 +139,12 @@ If you have a list of run IDs, you can list them directly: @@ -186,15 +198,17 @@ The following examples assume you have configured your environment appropriately ", filter: 'eq(name, "extractor")', - traceFilter: 'and(eq(feedback_key, "user_score"), eq(feedback_score, 1))' -})`), + traceFilter: 'and(eq(feedback_key, "user_score"), eq(feedback_score, 1))', +}); +`), ]} groupId="client-language" /> @@ -204,13 +218,15 @@ The following examples assume you have configured your environment appropriately ", - filter: 'and(eq(feedback_key, "star_rating"), gt(feedback_score, 4))' -})`), + projectName: "", + filter: 'and(eq(feedback_key, "star_rating"), gt(feedback_score, 4))', +}); +`), ]} groupId="client-language" /> @@ -306,13 +322,15 @@ had `total_tokens` greater than 5000 ", - filter: 'and(eq(run_type, "chain"), gt(latency, 10), gt(total_tokens, 5000))' -})`), + filter: 'and(eq(run_type, "chain"), gt(latency, 10), gt(total_tokens, 5000))', +}); +`), ]} groupId="client-language" /> @@ -391,30 +409,27 @@ about your runs. @@ -428,13 +443,15 @@ environment metadata, you can use the same pattern as above: @@ -448,13 +465,15 @@ runs based on a conversation ID in this way, you can search for that ID in the m @@ -468,13 +487,15 @@ have a specific `conversation_id` in their metadata: @@ -491,14 +512,16 @@ This type of query is useful if you want to extract a specific run conditional o project_name="", filter='eq(name, "RetrieveDocs")', trace_filter='and(eq(feedback_key, "user_score"), eq(feedback_score, 1))', - tree_filter='eq(name, "ExpandQuery")' -)`), + tree_filter='eq(name, "ExpandQuery")', +) +`), TypeScriptBlock(`client.listRuns({ projectName: "", filter: 'eq(name, "RetrieveDocs")', traceFilter: 'and(eq(feedback_key, "user_score"), eq(feedback_score, 1))', - treeFilter: 'eq(name, "ExpandQuery")' -})`), + treeFilter: 'eq(name, "ExpandQuery")', +}); +`), ]} groupId="client-language" />