LLMonitor Callback Handler: support for future OpenAI kwargs (langcha…

…in-ai#3209) * LLMonitor: future-proof support of new OpenAI kwargs * LLMonitor: fixes for ChatBedrock and ChatVertexAI * remove console.log * Fix type error
malandis · Nov 13, 2023 · ff04a95 · ff04a95
1 parent 9c1d800
commit ff04a95
Show file tree

Hide file tree

Showing 4 changed files with 66 additions and 34 deletions.
diff --git a/docs/docs/ecosystem/integrations/llmonitor.md b/docs/docs/ecosystem/integrations/llmonitor.md
@@ -4,15 +4,15 @@ This page covers how to use [LLMonitor](https://llmonitor.com?utm_source=langcha
 
 ## What is LLMonitor?
 
-LLMonitor is an [open-source](https://github.com/llmonitor/llmonitor) observability and analytics platform that provides tracing, analytics, feedback tracking and more for conversational AI.
+LLMonitor is an [open-source](https://github.com/llmonitor/llmonitor) observability and analytics platform that provides tracing, analytics, feedback tracking and way more for AI apps.
 
 <video controls width='100%' >
   <source src='https://llmonitor.com/videos/demo-annotated.mp4'/>
 </video>
 
 ## Installation
 
-Start by installing the llmonitor package in your project:
+Start by installing the LLMonitor package in your project:
 
 ```bash
 npm install llmonitor
@@ -128,7 +128,7 @@ import { LLMonitorHandler } from "langchain/callbacks/handlers/llmonitor";
 import monitor from "llmonitor";
 
 const chat = new ChatOpenAI({
-  modelName: "gpt-3.5-turbo",
+  modelName: "gpt-4",
   callbacks: [new LLMonitorHandler()],
 });
 

diff --git a/langchain/package.json b/langchain/package.json
@@ -923,7 +923,7 @@
     "jest": "^29.5.0",
     "jest-environment-node": "^29.6.4",
     "jsdom": "^22.1.0",
-    "llmonitor": "^0.5.8",
+    "llmonitor": "^0.5.9",
     "lodash": "^4.17.21",
     "mammoth": "^1.5.1",
     "ml-matrix": "^6.10.4",
@@ -1029,7 +1029,7 @@
     "ignore": "^5.2.0",
     "ioredis": "^5.3.2",
     "jsdom": "*",
-    "llmonitor": "^0.5.8",
+    "llmonitor": "^0.5.9",
     "lodash": "^4.17.21",
     "mammoth": "*",
     "mongodb": "^5.2.0",

diff --git a/langchain/src/callbacks/handlers/llmonitor.ts b/langchain/src/callbacks/handlers/llmonitor.ts
@@ -15,7 +15,7 @@ import { Serialized } from "../../load/serializable.js";
 
 import { BaseCallbackHandler, BaseCallbackHandlerInput } from "../base.js";
 
-type Role = "user" | "ai" | "system" | "function";
+type Role = "user" | "ai" | "system" | "function" | "tool";
 
 // Langchain Helpers
 // Input can be either a single message, an array of message, or an array of array of messages (batch requests)
@@ -27,6 +27,7 @@ const parseRole = (id: string[]): Role => {
   if (roleHint.includes("System")) return "system";
   if (roleHint.includes("AI")) return "ai";
   if (roleHint.includes("Function")) return "function";
+  if (roleHint.includes("Tool")) return "tool";
 
   return "ai";
 };
@@ -35,6 +36,16 @@ type Message = BaseMessage | Generation | string;
 
 type OutputMessage = ChatMessage | string;
 
+const PARAMS_TO_CAPTURE = [
+  "stop",
+  "stop_sequences",
+  "function_call",
+  "functions",
+  "tools",
+  "tool_choice",
+  "response_format",
+];
+
 export const convertToLLMonitorMessages = (
   input: Message | Message[] | Message[][]
 ): OutputMessage | OutputMessage[] | OutputMessage[][] => {
@@ -53,12 +64,10 @@ export const convertToLLMonitorMessages = (
       const obj = message.kwargs;
       const text = message.text ?? obj.content;
 
-      const functionCall = obj.additional_kwargs?.function_call;
-
       return {
         role,
         text,
-        functionCall,
+        ...(obj.additional_kwargs ?? {}),
       };
     } catch (e) {
       // if parsing fails, return the original message
@@ -101,6 +110,36 @@ const parseOutput = (rawOutput: Record<string, unknown>) => {
   return rawOutput;
 };
 
+const parseExtraAndName = (
+  llm: Serialized,
+  extraParams?: KVMap,
+  metadata?: KVMap
+) => {
+  const params = {
+    ...(extraParams?.invocation_params ?? {}),
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    // @ts-ignore this is a valid property
+    ...(llm?.kwargs ?? {}),
+    ...(metadata || {}),
+  };
+
+  const { model, model_name, modelName, model_id, userId, userProps, ...rest } =
+    params;
+
+  const name = model || modelName || model_name || model_id || llm.id.at(-1);
+
+  // Filter rest to only include params we want to capture
+  const extra = Object.fromEntries(
+    Object.entries(rest).filter(
+      ([key]) =>
+        PARAMS_TO_CAPTURE.includes(key) ||
+        ["string", "number", "boolean"].includes(typeof rest[key])
+    )
+  ) as cJSON;
+
+  return { name, extra, userId, userProps };
+};
+
 export interface Run extends BaseRun {
   id: string;
   child_runs: this[];
@@ -148,21 +187,18 @@ export class LLMonitorHandler
     tags?: string[],
     metadata?: KVMap
   ): Promise<void> {
-    const params = {
-      ...(extraParams?.invocation_params || {}),
-      ...(metadata || {}),
-    };
-
-    const { model, model_name, modelName, userId, userProps, ...rest } = params;
-
-    const name = model || modelName || model_name || llm.id.at(-1);
+    const { name, extra, userId, userProps } = parseExtraAndName(
+      llm,
+      extraParams,
+      metadata
+    );
 
     await this.monitor.trackEvent("llm", "start", {
       runId,
       parentRunId,
       name,
       input: convertToLLMonitorMessages(prompts),
-      extra: rest,
+      extra,
       userId,
       userProps,
       tags,
@@ -179,22 +215,18 @@ export class LLMonitorHandler
     tags?: string[],
     metadata?: KVMap
   ): Promise<void> {
-    const params = {
-      ...(extraParams?.invocation_params || {}),
-      ...(metadata || {}),
-    };
-
-    // Expand them so they're excluded from the "extra" field
-    const { model, model_name, modelName, userId, userProps, ...rest } = params;
-
-    const name = model || modelName || model_name || llm.id.at(-1);
+    const { name, extra, userId, userProps } = parseExtraAndName(
+      llm,
+      extraParams,
+      metadata
+    );
 
     await this.monitor.trackEvent("llm", "start", {
       runId,
       parentRunId,
       name,
       input: convertToLLMonitorMessages(messages),
-      extra: rest,
+      extra,
       userId,
       userProps,
       tags,

diff --git a/yarn.lock b/yarn.lock
@@ -21681,7 +21681,7 @@ __metadata:
     jsonpointer: ^5.0.1
     langchainhub: ~0.0.6
     langsmith: ~0.0.48
-    llmonitor: ^0.5.8
+    llmonitor: ^0.5.9
     lodash: ^4.17.21
     mammoth: ^1.5.1
     ml-distance: ^4.0.0
@@ -21795,7 +21795,7 @@ __metadata:
     ignore: ^5.2.0
     ioredis: ^5.3.2
     jsdom: "*"
-    llmonitor: ^0.5.8
+    llmonitor: ^0.5.9
     lodash: ^4.17.21
     mammoth: "*"
     mongodb: ^5.2.0
@@ -22235,9 +22235,9 @@ __metadata:
   languageName: node
   linkType: hard
 
-"llmonitor@npm:^0.5.8":
-  version: 0.5.9
-  resolution: "llmonitor@npm:0.5.9"
+"llmonitor@npm:^0.5.9":
+  version: 0.5.10
+  resolution: "llmonitor@npm:0.5.10"
   dependencies:
     unctx: ^2.3.1
   peerDependencies:
@@ -22248,7 +22248,7 @@ __metadata:
       optional: true
     react:
       optional: true
-  checksum: ce57cd3488fefb25a3c771f61ea13d394282c57a75c7e6b5964a87bc0e26a26ecabdb93495d9b60f1657824b9153906cebbb71879452006e978a2f1e4332c6a4
+  checksum: 52c3121f86b3aceccd518323640f6009b73ae08333046d72486920bcd0d3d464bfa65e458c370a75a4676c4692f256fdd9a33b7883b51c0dabad53f9a507cab9
   languageName: node
   linkType: hard