diff --git a/js/package.json b/js/package.json index a941749c0..d149b61d2 100644 --- a/js/package.json +++ b/js/package.json @@ -1,6 +1,6 @@ { "name": "langsmith", - "version": "0.1.58", + "version": "0.1.59", "description": "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform.", "packageManager": "yarn@1.22.19", "files": [ diff --git a/js/src/client.ts b/js/src/client.ts index 7aebb76e3..b7ef0beef 100644 --- a/js/src/client.ts +++ b/js/src/client.ts @@ -32,6 +32,8 @@ import { TracerSession, TracerSessionResult, ValueType, + AnnotationQueue, + RunWithAnnotationQueueInfo, } from "./schemas.js"; import { convertLangChainMessageToExample, @@ -3103,6 +3105,208 @@ export class Client { return results; } + /** + * API for managing annotation queues + */ + + /** + * List the annotation queues on the LangSmith API. + * @param options - The options for listing annotation queues + * @param options.queueIds - The IDs of the queues to filter by + * @param options.name - The name of the queue to filter by + * @param options.nameContains - The substring that the queue name should contain + * @param options.limit - The maximum number of queues to return + * @returns An iterator of AnnotationQueue objects + */ + public async *listAnnotationQueues( + options: { + queueIds?: string[]; + name?: string; + nameContains?: string; + limit?: number; + } = {} + ): AsyncIterableIterator { + const { queueIds, name, nameContains, limit } = options; + const params = new URLSearchParams(); + if (queueIds) { + queueIds.forEach((id, i) => { + assertUuid(id, `queueIds[${i}]`); + params.append("ids", id); + }); + } + if (name) params.append("name", name); + if (nameContains) params.append("name_contains", nameContains); + params.append( + "limit", + (limit !== undefined ? Math.min(limit, 100) : 100).toString() + ); + + let count = 0; + for await (const queues of this._getPaginated( + "/annotation-queues", + params + )) { + yield* queues; + count++; + if (limit !== undefined && count >= limit) break; + } + } + + /** + * Create an annotation queue on the LangSmith API. + * @param options - The options for creating an annotation queue + * @param options.name - The name of the annotation queue + * @param options.description - The description of the annotation queue + * @param options.queueId - The ID of the annotation queue + * @returns The created AnnotationQueue object + */ + public async createAnnotationQueue(options: { + name: string; + description?: string; + queueId?: string; + }): Promise { + const { name, description, queueId } = options; + const body = { + name, + description, + id: queueId || uuid.v4(), + }; + + const response = await this.caller.call( + _getFetchImplementation(), + `${this.apiUrl}/annotation-queues`, + { + method: "POST", + headers: { ...this.headers, "Content-Type": "application/json" }, + body: JSON.stringify( + Object.fromEntries( + Object.entries(body).filter(([_, v]) => v !== undefined) + ) + ), + signal: AbortSignal.timeout(this.timeout_ms), + ...this.fetchOptions, + } + ); + await raiseForStatus(response, "create annotation queue"); + const data = await response.json(); + return data as AnnotationQueue; + } + + /** + * Read an annotation queue with the specified queue ID. + * @param queueId - The ID of the annotation queue to read + * @returns The AnnotationQueue object + */ + public async readAnnotationQueue(queueId: string): Promise { + // TODO: Replace when actual endpoint is added + const queueIteratorResult = await this.listAnnotationQueues({ + queueIds: [queueId], + }).next(); + if (queueIteratorResult.done) { + throw new Error(`Annotation queue with ID ${queueId} not found`); + } + return queueIteratorResult.value; + } + + /** + * Update an annotation queue with the specified queue ID. + * @param queueId - The ID of the annotation queue to update + * @param options - The options for updating the annotation queue + * @param options.name - The new name for the annotation queue + * @param options.description - The new description for the annotation queue + */ + public async updateAnnotationQueue( + queueId: string, + options: { + name: string; + description?: string; + } + ): Promise { + const { name, description } = options; + const response = await this.caller.call( + _getFetchImplementation(), + `${this.apiUrl}/annotation-queues/${assertUuid(queueId, "queueId")}`, + { + method: "PATCH", + headers: { ...this.headers, "Content-Type": "application/json" }, + body: JSON.stringify({ name, description }), + signal: AbortSignal.timeout(this.timeout_ms), + ...this.fetchOptions, + } + ); + await raiseForStatus(response, "update annotation queue"); + } + + /** + * Delete an annotation queue with the specified queue ID. + * @param queueId - The ID of the annotation queue to delete + */ + public async deleteAnnotationQueue(queueId: string): Promise { + const response = await this.caller.call( + _getFetchImplementation(), + `${this.apiUrl}/annotation-queues/${assertUuid(queueId, "queueId")}`, + { + method: "DELETE", + headers: { ...this.headers, Accept: "application/json" }, + signal: AbortSignal.timeout(this.timeout_ms), + ...this.fetchOptions, + } + ); + await raiseForStatus(response, "delete annotation queue"); + } + + /** + * Add runs to an annotation queue with the specified queue ID. + * @param queueId - The ID of the annotation queue + * @param runIds - The IDs of the runs to be added to the annotation queue + */ + public async addRunsToAnnotationQueue( + queueId: string, + runIds: string[] + ): Promise { + const response = await this.caller.call( + _getFetchImplementation(), + `${this.apiUrl}/annotation-queues/${assertUuid(queueId, "queueId")}/runs`, + { + method: "POST", + headers: { ...this.headers, "Content-Type": "application/json" }, + body: JSON.stringify( + runIds.map((id, i) => assertUuid(id, `runIds[${i}]`).toString()) + ), + signal: AbortSignal.timeout(this.timeout_ms), + ...this.fetchOptions, + } + ); + await raiseForStatus(response, "add runs to annotation queue"); + } + + /** + * Get a run from an annotation queue at the specified index. + * @param queueId - The ID of the annotation queue + * @param index - The index of the run to retrieve + * @returns A Promise that resolves to a RunWithAnnotationQueueInfo object + * @throws {Error} If the run is not found at the given index or for other API-related errors + */ + public async getRunFromAnnotationQueue( + queueId: string, + index: number + ): Promise { + const baseUrl = `/annotation-queues/${assertUuid(queueId, "queueId")}/run`; + const response = await this.caller.call( + _getFetchImplementation(), + `${this.apiUrl}${baseUrl}/${index}`, + { + method: "GET", + headers: this.headers, + signal: AbortSignal.timeout(this.timeout_ms), + ...this.fetchOptions, + } + ); + + await raiseForStatus(response, "get run from annotation queue"); + return await response.json(); + } + protected async _currentTenantIsOwner(owner: string): Promise { const settings = await this._getSettings(); return owner == "-" || settings.tenant_handle === owner; @@ -3228,7 +3432,7 @@ export class Client { ListCommitsResponse >( `/commits/${promptOwnerAndName}/`, - {} as URLSearchParams, + new URLSearchParams(), (res) => res.commits )) { yield* commits; diff --git a/js/src/index.ts b/js/src/index.ts index 54ead368d..a43ebda35 100644 --- a/js/src/index.ts +++ b/js/src/index.ts @@ -14,4 +14,4 @@ export { RunTree, type RunTreeConfig } from "./run_trees.js"; export { overrideFetchImplementation } from "./singletons/fetch.js"; // Update using yarn bump-version -export const __version__ = "0.1.58"; +export const __version__ = "0.1.59"; diff --git a/js/src/schemas.ts b/js/src/schemas.ts index 4d73f29aa..7275f6d39 100644 --- a/js/src/schemas.ts +++ b/js/src/schemas.ts @@ -474,3 +474,31 @@ export interface LangSmithSettings { created_at: string; tenant_handle?: string; } + +export interface AnnotationQueue { + /** The unique identifier of the annotation queue. */ + id: string; + + /** The name of the annotation queue. */ + name: string; + + /** An optional description of the annotation queue. */ + description?: string; + + /** The timestamp when the annotation queue was created. */ + created_at: string; + + /** The timestamp when the annotation queue was last updated. */ + updated_at: string; + + /** The ID of the tenant associated with the annotation queue. */ + tenant_id: string; +} + +export interface RunWithAnnotationQueueInfo extends BaseRun { + /** The last time this run was reviewed. */ + last_reviewed_time?: string; + + /** The time this run was added to the queue. */ + added_at?: string; +} diff --git a/js/src/tests/client.int.test.ts b/js/src/tests/client.int.test.ts index 1846538f5..ddf160fa3 100644 --- a/js/src/tests/client.int.test.ts +++ b/js/src/tests/client.int.test.ts @@ -1147,3 +1147,96 @@ test("clonePublicDataset method can clone a dataset", async () => { } } }); + +test("annotationqueue crud", async () => { + const client = new Client(); + const queueName = `test-queue-${uuidv4().substring(0, 8)}`; + const projectName = `test-project-${uuidv4().substring(0, 8)}`; + const queueId = uuidv4(); + + try { + // 1. Create an annotation queue + const queue = await client.createAnnotationQueue({ + name: queueName, + description: "Initial description", + queueId, + }); + expect(queue).toBeDefined(); + expect(queue.name).toBe(queueName); + + // 1a. Get the annotation queue + const fetchedQueue = await client.readAnnotationQueue(queue.id); + expect(fetchedQueue).toBeDefined(); + expect(fetchedQueue.name).toBe(queueName); + + // 1b. List annotation queues and check nameContains + const listedQueues = await toArray( + client.listAnnotationQueues({ nameContains: queueName }) + ); + expect(listedQueues.length).toBeGreaterThan(0); + expect(listedQueues.some((q) => q.id === queue.id)).toBe(true); + + // 2. Create a run in a random project + await client.createProject({ projectName }); + const runId = uuidv4(); + await client.createRun({ + id: runId, + name: "Test Run", + run_type: "chain", + inputs: { foo: "bar" }, + outputs: { baz: "qux" }, + project_name: projectName, + }); + + // Wait for run to be found in the db + const maxWaitTime = 30000; // 30 seconds + const startTime = Date.now(); + let foundRun = null; + + while (Date.now() - startTime < maxWaitTime) { + try { + foundRun = await client.readRun(runId); + if (foundRun) break; + } catch (error) { + // If run is not found, getRun might throw an error + // We'll ignore it and keep trying + } + await new Promise((resolve) => setTimeout(resolve, 1000)); // Wait for 1 second before trying again + } + + if (!foundRun) { + throw new Error( + `Run with ID ${runId} not found after ${maxWaitTime / 1000} seconds` + ); + } + + // 3. Add the run to the annotation queue + await client.addRunsToAnnotationQueue(fetchedQueue.id, [runId]); + + // 4. Update the annotation queue description and check that it is updated + const newDescription = "Updated description"; + await client.updateAnnotationQueue(queue.id, { + name: queueName, + description: newDescription, + }); + const updatedQueue = await client.readAnnotationQueue(queue.id); + expect(updatedQueue.description).toBe(newDescription); + + // Get the run from the annotation queue + const run = await client.getRunFromAnnotationQueue(queueId, 0); + expect(run).toBeDefined(); + expect(run.id).toBe(runId); + expect(run.name).toBe("Test Run"); + expect(run.run_type).toBe("chain"); + expect(run.inputs).toEqual({ foo: "bar" }); + expect(run.outputs).toEqual({ baz: "qux" }); + } finally { + // 6. Delete the annotation queue + await client.deleteAnnotationQueue(queueId); + + // Clean up the project + if (await client.hasProject({ projectName })) { + await client.deleteProject({ projectName }); + } + } +}); diff --git a/js/src/utils/_uuid.ts b/js/src/utils/_uuid.ts index 714235131..51d71f020 100644 --- a/js/src/utils/_uuid.ts +++ b/js/src/utils/_uuid.ts @@ -1,7 +1,12 @@ import * as uuid from "uuid"; -export function assertUuid(str: string): void { +export function assertUuid(str: string, which?: string): string { if (!uuid.validate(str)) { - throw new Error(`Invalid UUID: ${str}`); + const msg = + which !== undefined + ? `Invalid UUID for ${which}: ${str}` + : `Invalid UUID: ${str}`; + throw new Error(msg); } + return str; } diff --git a/python/langsmith/client.py b/python/langsmith/client.py index cb0e863f0..123f869f6 100644 --- a/python/langsmith/client.py +++ b/python/langsmith/client.py @@ -4681,28 +4681,30 @@ def add_runs_to_annotation_queue( ) ls_utils.raise_for_status_with_text(response) - def list_runs_from_annotation_queue( - self, queue_id: ID_TYPE, *, limit: Optional[int] = None - ) -> Iterator[ls_schemas.RunWithAnnotationQueueInfo]: - """List runs from an annotation queue with the specified queue ID. + def get_run_from_annotation_queue( + self, queue_id: ID_TYPE, *, index: int + ) -> ls_schemas.RunWithAnnotationQueueInfo: + """Get a run from an annotation queue at the specified index. Args: queue_id (ID_TYPE): The ID of the annotation queue. + index (int): The index of the run to retrieve. - Yields: - ls_schemas.RunWithAnnotationQueueInfo: An iterator of runs from the - annotation queue. + Returns: + ls_schemas.RunWithAnnotationQueueInfo: The run at the specified index. + + Raises: + ls_utils.LangSmithNotFoundError: If the run is not found at the given index. + ls_utils.LangSmithError: For other API-related errors. """ - path = f"/annotation-queues/{_as_uuid(queue_id, 'queue_id')}/runs" - limit_ = min(limit, 100) if limit is not None else 100 - for i, run in enumerate( - self._get_paginated_list( - path, params={"headers": self._headers, "limit": limit_} - ) - ): - yield ls_schemas.RunWithAnnotationQueueInfo(**run) - if limit is not None and i + 1 >= limit: - break + base_url = f"/annotation-queues/{_as_uuid(queue_id, 'queue_id')}/run" + response = self.request_with_retries( + "GET", + f"{base_url}/{index}", + headers=self._headers, + ) + ls_utils.raise_for_status_with_text(response) + return ls_schemas.RunWithAnnotationQueueInfo(**response.json()) def create_comparative_experiment( self, diff --git a/python/pyproject.toml b/python/pyproject.toml index ff7ff3f4f..46582df93 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "langsmith" -version = "0.1.121" +version = "0.1.122" description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform." authors = ["LangChain "] license = "MIT"