From f967361ad53cce6f639fba52a4491919feb3f6be Mon Sep 17 00:00:00 2001 From: nick-w-nick <43578531+nick-w-nick@users.noreply.github.com> Date: Wed, 18 Dec 2024 20:35:54 -0500 Subject: [PATCH 01/30] added isBoolean function to allow the value to be casted --- langchain-core/src/structured_query/utils.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/langchain-core/src/structured_query/utils.ts b/langchain-core/src/structured_query/utils.ts index 05e699e83394..b70df772be4c 100644 --- a/langchain-core/src/structured_query/utils.ts +++ b/langchain-core/src/structured_query/utils.ts @@ -72,13 +72,20 @@ export function isString(value: unknown): boolean { ); } +/** + * Checks if the provided value is a boolean. + */ +export function isBoolean(value: unknown): boolean { + return typeof value === "boolean"; +} + /** * Casts a value that might be string or number to actual string or number. * Since LLM might return back an integer/float as a string, we need to cast * it back to a number, as many vector databases can't handle number as string * values as a comparator. */ -export function castValue(input: unknown): string | number { +export function castValue(input: unknown): string | number | boolean { let value; if (isString(input)) { value = input as string; @@ -86,6 +93,8 @@ export function castValue(input: unknown): string | number { value = parseInt(input as string, 10); } else if (isFloat(input)) { value = parseFloat(input as string); + } else if (isBoolean(input)) { + value = Boolean(input); } else { throw new Error("Unsupported value type"); } From bf549bc63a9b7e9e01e2e671f3558248cb8893c4 Mon Sep 17 00:00:00 2001 From: nick-w-nick <43578531+nick-w-nick@users.noreply.github.com> Date: Wed, 18 Dec 2024 20:37:37 -0500 Subject: [PATCH 02/30] added function to get allowed comparators based on input type --- .../src/structured_query/functional.ts | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/langchain-core/src/structured_query/functional.ts b/langchain-core/src/structured_query/functional.ts index 7a0c2e894ad0..f91788d040c0 100644 --- a/langchain-core/src/structured_query/functional.ts +++ b/langchain-core/src/structured_query/functional.ts @@ -65,7 +65,31 @@ export class FunctionalTranslator extends BaseTranslator { formatFunction(): string { throw new Error("Not implemented"); } - + + /** + * Returns the allowed comparators based + * on the data type of the input. + * @param input The input value to get the allowed comparators for. + * @returns An array of allowed comparators for the input data type. + */ + getAllowedComparatorsForInput(input: string | number | boolean): Comparator[] { + const inputType = typeof input; + switch (inputType) { + case "string": { + return [Comparators.eq, Comparators.ne, Comparators.gt, Comparators.gte, Comparators.lt, Comparators.lte]; + } + case "number": { + return [Comparators.eq, Comparators.ne, Comparators.gt, Comparators.gte, Comparators.lt, Comparators.lte]; + } + case "boolean": { + return [Comparators.eq, Comparators.ne]; + } + default: { + throw new Error(`Unsupported data type: ${inputType}`); + } + } + } + /** * Returns a function that performs a comparison based on the provided * comparator. @@ -159,6 +183,9 @@ export class FunctionalTranslator extends BaseTranslator { const { comparator, attribute, value } = comparison; const undefinedTrue = [Comparators.ne]; if (this.allowedComparators.includes(comparator)) { + if (this.getAllowedComparatorsForInput(value).includes(comparator)) { + throw new Error(`'${comparator}' comparator not allowed to be used with ${typeof value}`); + } const comparatorFunction = this.getComparatorFunction(comparator); return (document: Document) => { const documentValue = document.metadata[attribute]; From 1b97531d4d964efa7f5cae768be8112c450a0fa6 Mon Sep 17 00:00:00 2001 From: nick-w-nick <43578531+nick-w-nick@users.noreply.github.com> Date: Wed, 18 Dec 2024 20:38:00 -0500 Subject: [PATCH 03/30] updated ValueType types for applicable operators --- langchain-core/src/structured_query/functional.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/langchain-core/src/structured_query/functional.ts b/langchain-core/src/structured_query/functional.ts index f91788d040c0..78964d0d33fc 100644 --- a/langchain-core/src/structured_query/functional.ts +++ b/langchain-core/src/structured_query/functional.ts @@ -17,8 +17,8 @@ import { castValue, isFilterEmpty } from "./utils.js"; * the result of a comparison operation. */ type ValueType = { - eq: string | number; - ne: string | number; + eq: string | number | boolean; + ne: string | number | boolean; lt: string | number; lte: string | number; gt: string | number; From 2f43b70496e1fe5340539a013cb9c787333eccaf Mon Sep 17 00:00:00 2001 From: nick-w-nick <43578531+nick-w-nick@users.noreply.github.com> Date: Wed, 18 Dec 2024 20:38:20 -0500 Subject: [PATCH 04/30] added boolean to visitor comparison result type --- langchain-core/src/structured_query/ir.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/langchain-core/src/structured_query/ir.ts b/langchain-core/src/structured_query/ir.ts index d2bfa6215d81..d4c3f04315ce 100644 --- a/langchain-core/src/structured_query/ir.ts +++ b/langchain-core/src/structured_query/ir.ts @@ -82,7 +82,7 @@ export type VisitorOperationResult = { */ export type VisitorComparisonResult = { [attr: string]: { - [comparator: string]: string | number; + [comparator: string]: string | number | boolean; }; }; From f83470d4741b1959a83d4dbcbc8513e621a51c9c Mon Sep 17 00:00:00 2001 From: nick-w-nick <43578531+nick-w-nick@users.noreply.github.com> Date: Wed, 18 Dec 2024 20:38:41 -0500 Subject: [PATCH 05/30] added test for isBoolean --- langchain-core/src/structured_query/tests/utils.test.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/langchain-core/src/structured_query/tests/utils.test.ts b/langchain-core/src/structured_query/tests/utils.test.ts index 94ac85e84dfa..71c6458e34f7 100644 --- a/langchain-core/src/structured_query/tests/utils.test.ts +++ b/langchain-core/src/structured_query/tests/utils.test.ts @@ -1,6 +1,6 @@ /* eslint-disable no-process-env */ import { test, expect } from "@jest/globals"; -import { castValue, isFloat, isInt, isString } from "../utils.js"; +import { castValue, isFloat, isInt, isString, isBoolean } from "../utils.js"; test("Casting values correctly", () => { const stringString = [ @@ -28,6 +28,8 @@ test("Casting values correctly", () => { const floatFloat = ["1.1", 2.2, 3.3]; + const booleanBoolean = [true, false]; + stringString.map(castValue).forEach((value) => { expect(typeof value).toBe("string"); expect(isString(value)).toBe(true); @@ -54,4 +56,9 @@ test("Casting values correctly", () => { expect(typeof value).toBe("number"); expect(isFloat(value)).toBe(true); }); + + booleanBoolean.map(castValue).forEach((value) => { + expect(typeof value).toBe("boolean"); + expect(isBoolean(value)).toBe(true); + }); }); From cb8c08b32a946d32e6949bcbe7bf6da19e14d5f8 Mon Sep 17 00:00:00 2001 From: nick-w-nick <43578531+nick-w-nick@users.noreply.github.com> Date: Wed, 18 Dec 2024 20:57:06 -0500 Subject: [PATCH 06/30] added boolean to Comparison class --- langchain-core/src/structured_query/ir.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/langchain-core/src/structured_query/ir.ts b/langchain-core/src/structured_query/ir.ts index d4c3f04315ce..29bf3fafe624 100644 --- a/langchain-core/src/structured_query/ir.ts +++ b/langchain-core/src/structured_query/ir.ts @@ -155,7 +155,7 @@ export class Comparison extends FilterDirective { constructor( public comparator: Comparator, public attribute: string, - public value: string | number + public value: string | number | boolean ) { super(); } From 74b643a448aab91c8af2bcd57d9314320f173417 Mon Sep 17 00:00:00 2001 From: nick-w-nick <43578531+nick-w-nick@users.noreply.github.com> Date: Wed, 18 Dec 2024 21:00:56 -0500 Subject: [PATCH 07/30] updated query parser test to include boolean comparisons --- .../src/chains/query_constructor/tests/query_parser.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/langchain/src/chains/query_constructor/tests/query_parser.test.ts b/langchain/src/chains/query_constructor/tests/query_parser.test.ts index 62dc607e79f1..10428a2e6272 100644 --- a/langchain/src/chains/query_constructor/tests/query_parser.test.ts +++ b/langchain/src/chains/query_constructor/tests/query_parser.test.ts @@ -17,6 +17,7 @@ const correctQuery = new StructuredQuery( ]), new Comparison(Comparators.lt, "length", 180), new Comparison(Comparators.eq, "genre", "pop"), + new Comparison(Comparators.eq, "hasLyrics", true), ]) ); @@ -35,7 +36,7 @@ test("StructuredQueryOutputParser test", async () => { const exampleOutput = `json\`\`\` { "query": "teenager love", - "filter": "and(or(eq(\\"artist\\", \\"Taylor Swift\\"), eq(\\"artist\\", \\"Katy Perry\\")), lt(\\"length\\", 180), eq(\\"genre\\", \\"pop\\"))" + "filter": "and(or(eq(\\"artist\\", \\"Taylor Swift\\"), eq(\\"artist\\", \\"Katy Perry\\")), lt(\\"length\\", 180), eq(\\"genre\\", \\"pop\\"), eq(\\"hasLyrics\\", true))" } \`\`\``; From 9d8c25ea6abfb472bbdffad54c4e01322dae7534 Mon Sep 17 00:00:00 2001 From: nick-w-nick <43578531+nick-w-nick@users.noreply.github.com> Date: Wed, 18 Dec 2024 21:01:26 -0500 Subject: [PATCH 08/30] updated query chain test to include a boolean attribute and comparison --- .../tests/query_chain.int.test.ts | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/langchain/src/chains/query_constructor/tests/query_chain.int.test.ts b/langchain/src/chains/query_constructor/tests/query_chain.int.test.ts index 332ab6afdb08..88be5ae7b379 100644 --- a/langchain/src/chains/query_constructor/tests/query_chain.int.test.ts +++ b/langchain/src/chains/query_constructor/tests/query_chain.int.test.ts @@ -33,6 +33,10 @@ test("Query Chain Test", async () => { new Comparison(Comparators.lt, "length", 90), ]) ); + const sq6 = new StructuredQuery( + "", + new Comparison(Comparators.eq, "isReleased", true) + ); const filter1 = { length: { $lt: 90 } }; const filter3 = { rating: { $gt: 8.5 } }; @@ -43,6 +47,7 @@ test("Query Chain Test", async () => { { length: { $lt: 90 } }, ], }; + const filter6 = { isReleased: { $eq: true } }; const attributeInfo: AttributeInfo[] = [ { @@ -70,6 +75,11 @@ test("Query Chain Test", async () => { description: "The length of the movie in minutes", type: "number", }, + { + name: "isReleased", + description: "Whether the movie has been released", + type: "boolean", + } ]; const documentContents = "Brief summary of a movie"; @@ -100,22 +110,28 @@ test("Query Chain Test", async () => { query: "Which movies are either comedy or drama and are less than 90 minutes?", }); + const c6 = queryChain.invoke({ + query: "Which movies have already been released?", + }); - const [r1, r3, r4, r5] = await Promise.all([c1, c3, c4, c5]); + const [r1, r3, r4, r5, r6] = await Promise.all([c1, c3, c4, c5, c6]); expect(r1).toMatchObject(sq1); expect(r3).toMatchObject(sq3); expect(r4).toMatchObject(sq4); expect(r5).toMatchObject(sq5); + expect(r6).toMatchObject(sq6); const testTranslator = new BasicTranslator(); const { filter: parsedFilter1 } = testTranslator.visitStructuredQuery(r1); const { filter: parsedFilter3 } = testTranslator.visitStructuredQuery(r3); const { filter: parsedFilter4 } = testTranslator.visitStructuredQuery(r4); const { filter: parsedFilter5 } = testTranslator.visitStructuredQuery(r5); + const { filter: parsedFilter6 } = testTranslator.visitStructuredQuery(r6); expect(parsedFilter1).toMatchObject(filter1); expect(parsedFilter3).toMatchObject(filter3); expect(parsedFilter4).toMatchObject(filter4); expect(parsedFilter5).toMatchObject(filter5); + expect(parsedFilter6).toMatchObject(filter6); }); From 15c46bd93d1f95f7f6a0434aa125b4bdc3c2b496 Mon Sep 17 00:00:00 2001 From: nick-w-nick <43578531+nick-w-nick@users.noreply.github.com> Date: Wed, 18 Dec 2024 21:38:43 -0500 Subject: [PATCH 09/30] added boolean type to query transform traversal --- langchain/src/chains/query_constructor/parser.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/langchain/src/chains/query_constructor/parser.ts b/langchain/src/chains/query_constructor/parser.ts index 49998a274718..b9e862d172ea 100644 --- a/langchain/src/chains/query_constructor/parser.ts +++ b/langchain/src/chains/query_constructor/parser.ts @@ -94,7 +94,7 @@ export class QueryTransformer { return new Comparison( funcName as Comparator, traverse(node.args[0]) as string, - traverse(node.args[1]) as string | number + traverse(node.args[1]) as string | number | boolean ); } throw new Error("Comparator must have exactly 2 arguments"); From 00e772171fe685de4e7500c8019d5ac10f96e995 Mon Sep 17 00:00:00 2001 From: nick-w-nick <43578531+nick-w-nick@users.noreply.github.com> Date: Wed, 18 Dec 2024 21:48:01 -0500 Subject: [PATCH 10/30] updated function to accept an input type rather than the value itself --- langchain-core/src/structured_query/functional.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/langchain-core/src/structured_query/functional.ts b/langchain-core/src/structured_query/functional.ts index 78964d0d33fc..4f5ab2fa945c 100644 --- a/langchain-core/src/structured_query/functional.ts +++ b/langchain-core/src/structured_query/functional.ts @@ -67,13 +67,11 @@ export class FunctionalTranslator extends BaseTranslator { } /** - * Returns the allowed comparators based - * on the data type of the input. + * Returns the allowed comparators for a given data type. * @param input The input value to get the allowed comparators for. * @returns An array of allowed comparators for the input data type. */ - getAllowedComparatorsForInput(input: string | number | boolean): Comparator[] { - const inputType = typeof input; + getAllowedComparatorsForType(inputType: string): Comparator[] { switch (inputType) { case "string": { return [Comparators.eq, Comparators.ne, Comparators.gt, Comparators.gte, Comparators.lt, Comparators.lte]; @@ -183,7 +181,7 @@ export class FunctionalTranslator extends BaseTranslator { const { comparator, attribute, value } = comparison; const undefinedTrue = [Comparators.ne]; if (this.allowedComparators.includes(comparator)) { - if (this.getAllowedComparatorsForInput(value).includes(comparator)) { + if (this.getAllowedComparatorsForType(typeof value).includes(comparator)) { throw new Error(`'${comparator}' comparator not allowed to be used with ${typeof value}`); } const comparatorFunction = this.getComparatorFunction(comparator); From db0b4ed4daecd5dcfed26c0fa9642165a1ce1c43 Mon Sep 17 00:00:00 2001 From: nick-w-nick <43578531+nick-w-nick@users.noreply.github.com> Date: Wed, 18 Dec 2024 22:05:24 -0500 Subject: [PATCH 11/30] fixed conditional to throw error if comparator is not valid with the provided value type --- langchain-core/src/structured_query/functional.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/langchain-core/src/structured_query/functional.ts b/langchain-core/src/structured_query/functional.ts index 4f5ab2fa945c..30b0fef887e2 100644 --- a/langchain-core/src/structured_query/functional.ts +++ b/langchain-core/src/structured_query/functional.ts @@ -181,7 +181,7 @@ export class FunctionalTranslator extends BaseTranslator { const { comparator, attribute, value } = comparison; const undefinedTrue = [Comparators.ne]; if (this.allowedComparators.includes(comparator)) { - if (this.getAllowedComparatorsForType(typeof value).includes(comparator)) { + if (!this.getAllowedComparatorsForType(typeof value).includes(comparator)) { throw new Error(`'${comparator}' comparator not allowed to be used with ${typeof value}`); } const comparatorFunction = this.getComparatorFunction(comparator); From f55a5ce9f7df974477270501332134d227930c16 Mon Sep 17 00:00:00 2001 From: nick-w-nick <43578531+nick-w-nick@users.noreply.github.com> Date: Thu, 19 Dec 2024 00:23:09 -0500 Subject: [PATCH 12/30] added basic tests for FunctionalTranslator --- .../structured_query/tests/functional.test.ts | 303 ++++++++++++++++++ 1 file changed, 303 insertions(+) create mode 100644 langchain-core/src/structured_query/tests/functional.test.ts diff --git a/langchain-core/src/structured_query/tests/functional.test.ts b/langchain-core/src/structured_query/tests/functional.test.ts new file mode 100644 index 000000000000..fffe2a901c7a --- /dev/null +++ b/langchain-core/src/structured_query/tests/functional.test.ts @@ -0,0 +1,303 @@ +import { Document } from "../../documents/document.js"; +import { FunctionalTranslator } from "../functional.js"; +import { Comparators, Visitor } from "../ir.js"; + +describe("FunctionalTranslator", () => { + const translator = new FunctionalTranslator(); + + describe("getAllowedComparatorsForType", () => { + test("string", () => { + expect(translator.getAllowedComparatorsForType("string")).toEqual([ + Comparators.eq, + Comparators.ne, + Comparators.gt, + Comparators.gte, + Comparators.lt, + Comparators.lte, + ]); + }); + test("number", () => { + expect(translator.getAllowedComparatorsForType("number")).toEqual([ + Comparators.eq, + Comparators.ne, + Comparators.gt, + Comparators.gte, + Comparators.lt, + Comparators.lte, + ]); + }); + test("boolean", () => { + expect(translator.getAllowedComparatorsForType("boolean")).toEqual([ + Comparators.eq, + Comparators.ne, + ]); + }); + test("unsupported", () => { + expect(() => + translator.getAllowedComparatorsForType("unsupported") + ).toThrow("Unsupported data type: unsupported"); + }); + }); + + describe("visitComparison", () => { + const attributeByType = { + string: "stringValue", + number: "numberValue", + boolean: "booleanValue", + }; + + describe("returns true or false for valid comparisons", () => { + const inputValues = { + string: "value", + number: 1, + boolean: true, + }; + + const validDocumentsByComparator: { [key in string]: Document>[] } = { + [Comparators.eq]: [ + new Document({ + pageContent: "", + metadata: { + stringValue: "value", + numberValue: 1, + booleanValue: true, + }, + }), + ], + [Comparators.ne]: [ + new Document({ + pageContent: "", + metadata: { + stringValue: "not-value", + numberValue: 0, + booleanValue: false, + }, + }), + ], + [Comparators.gt]: [ + new Document({ + pageContent: "", + metadata: { + stringValue: "valueee", + numberValue: 2, + booleanValue: true, + }, + }), + ], + [Comparators.gte]: [ + new Document({ + pageContent: "", + metadata: { + stringValue: "valueee", + numberValue: 2, + booleanValue: true, + }, + }), + new Document({ + pageContent: "", + metadata: { + stringValue: "value", + numberValue: 1, + booleanValue: true, + }, + }), + ], + [Comparators.lt]: [ + new Document({ + pageContent: "", + metadata: { + stringValue: "val", + numberValue: 0, + booleanValue: true, + }, + }), + ], + [Comparators.lte]: [ + new Document({ + pageContent: "", + metadata: { + stringValue: "val", + numberValue: 0, + booleanValue: true, + }, + }), + new Document({ + pageContent: "", + metadata: { + stringValue: "value", + numberValue: 1, + booleanValue: true, + }, + }), + ], + }; + + const invalidDocumentsByComparator: { [key in string]: Document>[] } = { + [Comparators.eq]: [ + new Document({ + pageContent: "", + metadata: { + stringValue: "not-value", + numberValue: 0, + booleanValue: false, + }, + }), + ], + [Comparators.ne]: [ + new Document({ + pageContent: "", + metadata: { + stringValue: "value", + numberValue: 1, + booleanValue: true, + }, + }), + ], + [Comparators.gt]: [ + new Document({ + pageContent: "", + metadata: { + stringValue: "value", + numberValue: 1, + booleanValue: true, + }, + }), + ], + [Comparators.gte]: [ + new Document({ + pageContent: "", + metadata: { + stringValue: "val", + numberValue: 0, + booleanValue: false, + }, + }), + ], + [Comparators.lt]: [ + new Document({ + pageContent: "", + metadata: { + stringValue: "valueee", + numberValue: 2, + booleanValue: true, + }, + }), + ], + [Comparators.lte]: [ + new Document({ + pageContent: "", + metadata: { + stringValue: "valueee", + numberValue: 2, + booleanValue: true, + }, + }), + ], + }; + + const stringComparators = translator.getAllowedComparatorsForType("string"); + for (const comparator of stringComparators) { + const value = inputValues.string; + const attribute = attributeByType.string; + const validDocuments = validDocumentsByComparator[comparator]; + for (const validDocument of validDocuments) { + test(`${value} -> ${comparator} -> ${validDocument.metadata[attribute]}`, () => { + const comparison = translator.visitComparison({ + attribute, + comparator, + value, + exprName: "Comparison", + accept: (visitor: Visitor) => visitor, + }); + const result = comparison(validDocument); + expect(result).toBeTruthy(); + }); + } + + const invalidDocuments = invalidDocumentsByComparator[comparator]; + for (const invalidDocument of invalidDocuments) { + test(`${value} -> ${comparator} -> ${invalidDocument.metadata[attribute]}`, () => { + const comparison = translator.visitComparison({ + attribute, + comparator, + value, + exprName: "Comparison", + accept: (visitor: Visitor) => visitor, + }); + const result = comparison(invalidDocument); + expect(result).toBeFalsy(); + }); + } + } + + const numberComparators = translator.getAllowedComparatorsForType("number"); + for (const comparator of numberComparators) { + const value = inputValues.number; + const attribute = attributeByType.number; + const validDocuments = validDocumentsByComparator[comparator]; + for (const validDocument of validDocuments) { + test(`${value} -> ${comparator} -> ${validDocument.metadata[attribute]}`, () => { + const comparison = translator.visitComparison({ + attribute, + comparator, + value, + exprName: "Comparison", + accept: (visitor: Visitor) => visitor, + }); + const result = comparison(validDocument); + expect(result).toBeTruthy(); + }); + } + + const invalidDocuments = invalidDocumentsByComparator[comparator]; + for (const invalidDocument of invalidDocuments) { + test(`${value} -> ${comparator} -> ${invalidDocument.metadata[attribute]}`, () => { + const comparison = translator.visitComparison({ + attribute, + comparator, + value, + exprName: "Comparison", + accept: (visitor: Visitor) => visitor, + }); + const result = comparison(invalidDocument); + expect(result).toBeFalsy(); + }); + } + } + + const booleanComparators = translator.getAllowedComparatorsForType("boolean"); + for (const comparator of booleanComparators) { + const value = inputValues.boolean; + const attribute = attributeByType.boolean; + const validDocuments = validDocumentsByComparator[comparator]; + for (const validDocument of validDocuments) { + test(`${value} -> ${comparator} -> ${validDocument.metadata[attribute]}`, () => { + const comparison = translator.visitComparison({ + attribute, + comparator, + value, + exprName: "Comparison", + accept: (visitor: Visitor) => visitor, + }); + const result = comparison(validDocument); + expect(result).toBeTruthy(); + }); + } + const invalidDocuments = invalidDocumentsByComparator[comparator]; + for (const invalidDocument of invalidDocuments) { + test(`${value} -> ${comparator} -> ${invalidDocument.metadata[attribute]}`, () => { + const comparison = translator.visitComparison({ + attribute, + comparator, + value, + exprName: "Comparison", + accept: (visitor: Visitor) => visitor, + }); + const result = comparison(invalidDocument); + expect(result).toBeFalsy(); + }); + } + } + }); + }); +}); From 379e466bf9d412a29d06a7db3f4858688d448da9 Mon Sep 17 00:00:00 2001 From: nick-w-nick <43578531+nick-w-nick@users.noreply.github.com> Date: Thu, 19 Dec 2024 00:30:53 -0500 Subject: [PATCH 13/30] fixed typo --- langchain-core/src/structured_query/base.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/langchain-core/src/structured_query/base.ts b/langchain-core/src/structured_query/base.ts index d82f615d63c4..89efdb486af2 100644 --- a/langchain-core/src/structured_query/base.ts +++ b/langchain-core/src/structured_query/base.ts @@ -100,7 +100,7 @@ export class BasicTranslator< this.allowedComparators.indexOf(func as Comparator) === -1 ) { throw new Error( - `Comparator ${func} not allowed. Allowed operators: ${this.allowedComparators.join( + `Comparator ${func} not allowed. Allowed comparators: ${this.allowedComparators.join( ", " )}` ); From 10523aed4127294c4a8b76fa3e8955825034eda5 Mon Sep 17 00:00:00 2001 From: nick-w-nick <43578531+nick-w-nick@users.noreply.github.com> Date: Thu, 19 Dec 2024 00:32:58 -0500 Subject: [PATCH 14/30] removed extra symbol --- langchain-core/src/structured_query/base.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/langchain-core/src/structured_query/base.ts b/langchain-core/src/structured_query/base.ts index 89efdb486af2..0b2436b6e5b8 100644 --- a/langchain-core/src/structured_query/base.ts +++ b/langchain-core/src/structured_query/base.ts @@ -119,7 +119,7 @@ export class BasicTranslator< } else { throw new Error("Unknown comparator or operator"); } - return `$${func}`; + return `${func}`; } /** From d538d8b4526311f89537a463a7267e5e43f44ffe Mon Sep 17 00:00:00 2001 From: nick-w-nick <43578531+nick-w-nick@users.noreply.github.com> Date: Thu, 19 Dec 2024 01:12:57 -0500 Subject: [PATCH 15/30] lint fixes --- langchain-core/src/structured_query/tests/functional.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/langchain-core/src/structured_query/tests/functional.test.ts b/langchain-core/src/structured_query/tests/functional.test.ts index fffe2a901c7a..a66507220b57 100644 --- a/langchain-core/src/structured_query/tests/functional.test.ts +++ b/langchain-core/src/structured_query/tests/functional.test.ts @@ -1,3 +1,4 @@ +import { test, expect } from "@jest/globals"; import { Document } from "../../documents/document.js"; import { FunctionalTranslator } from "../functional.js"; import { Comparators, Visitor } from "../ir.js"; @@ -53,7 +54,7 @@ describe("FunctionalTranslator", () => { boolean: true, }; - const validDocumentsByComparator: { [key in string]: Document>[] } = { + const validDocumentsByComparator: { [key in string]: Document>[] } = { [Comparators.eq]: [ new Document({ pageContent: "", @@ -132,7 +133,7 @@ describe("FunctionalTranslator", () => { ], }; - const invalidDocumentsByComparator: { [key in string]: Document>[] } = { + const invalidDocumentsByComparator: { [key in string]: Document>[] } = { [Comparators.eq]: [ new Document({ pageContent: "", From 2bd18a7dcc758ac4a0df73d72c7292594138acac Mon Sep 17 00:00:00 2001 From: nick-w-nick <43578531+nick-w-nick@users.noreply.github.com> Date: Thu, 19 Dec 2024 02:14:28 -0500 Subject: [PATCH 16/30] readded symbol --- langchain-core/src/structured_query/base.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/langchain-core/src/structured_query/base.ts b/langchain-core/src/structured_query/base.ts index 0b2436b6e5b8..89efdb486af2 100644 --- a/langchain-core/src/structured_query/base.ts +++ b/langchain-core/src/structured_query/base.ts @@ -119,7 +119,7 @@ export class BasicTranslator< } else { throw new Error("Unknown comparator or operator"); } - return `${func}`; + return `$${func}`; } /** From 8fb1fcdecf6a300af13185c6b878ea004cb06666 Mon Sep 17 00:00:00 2001 From: nick-w-nick <43578531+nick-w-nick@users.noreply.github.com> Date: Thu, 19 Dec 2024 02:26:34 -0500 Subject: [PATCH 17/30] added clarifying comments to test fakes --- langchain-core/src/structured_query/tests/functional.test.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/langchain-core/src/structured_query/tests/functional.test.ts b/langchain-core/src/structured_query/tests/functional.test.ts index a66507220b57..fecca3c0523b 100644 --- a/langchain-core/src/structured_query/tests/functional.test.ts +++ b/langchain-core/src/structured_query/tests/functional.test.ts @@ -86,6 +86,7 @@ describe("FunctionalTranslator", () => { }), ], [Comparators.gte]: [ + // test for greater than new Document({ pageContent: "", metadata: { @@ -94,6 +95,7 @@ describe("FunctionalTranslator", () => { booleanValue: true, }, }), + // test for equal to new Document({ pageContent: "", metadata: { @@ -114,6 +116,7 @@ describe("FunctionalTranslator", () => { }), ], [Comparators.lte]: [ + // test for less than new Document({ pageContent: "", metadata: { @@ -122,6 +125,7 @@ describe("FunctionalTranslator", () => { booleanValue: true, }, }), + // test for equal to new Document({ pageContent: "", metadata: { From fc79a32f40e4fc6cf2c113520e654c408fa8fbf1 Mon Sep 17 00:00:00 2001 From: nick-w-nick <43578531+nick-w-nick@users.noreply.github.com> Date: Thu, 19 Dec 2024 02:34:58 -0500 Subject: [PATCH 18/30] updated test to get input values by attribute instead of type --- .../structured_query/tests/functional.test.ts | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/langchain-core/src/structured_query/tests/functional.test.ts b/langchain-core/src/structured_query/tests/functional.test.ts index fecca3c0523b..dc63e39c7563 100644 --- a/langchain-core/src/structured_query/tests/functional.test.ts +++ b/langchain-core/src/structured_query/tests/functional.test.ts @@ -41,17 +41,17 @@ describe("FunctionalTranslator", () => { }); describe("visitComparison", () => { - const attributeByType = { + const attributesByType = { string: "stringValue", number: "numberValue", boolean: "booleanValue", }; describe("returns true or false for valid comparisons", () => { - const inputValues = { - string: "value", - number: 1, - boolean: true, + const inputValuesByAttribute: { [key in string]: string | number | boolean } = { + stringValue: "value", + numberValue: 1, + booleanValue: true, }; const validDocumentsByComparator: { [key in string]: Document>[] } = { @@ -202,8 +202,8 @@ describe("FunctionalTranslator", () => { const stringComparators = translator.getAllowedComparatorsForType("string"); for (const comparator of stringComparators) { - const value = inputValues.string; - const attribute = attributeByType.string; + const attribute = attributesByType.string; + const value = inputValuesByAttribute[attribute]; const validDocuments = validDocumentsByComparator[comparator]; for (const validDocument of validDocuments) { test(`${value} -> ${comparator} -> ${validDocument.metadata[attribute]}`, () => { @@ -237,8 +237,8 @@ describe("FunctionalTranslator", () => { const numberComparators = translator.getAllowedComparatorsForType("number"); for (const comparator of numberComparators) { - const value = inputValues.number; - const attribute = attributeByType.number; + const attribute = attributesByType.number; + const value = inputValuesByAttribute[attribute]; const validDocuments = validDocumentsByComparator[comparator]; for (const validDocument of validDocuments) { test(`${value} -> ${comparator} -> ${validDocument.metadata[attribute]}`, () => { @@ -272,8 +272,8 @@ describe("FunctionalTranslator", () => { const booleanComparators = translator.getAllowedComparatorsForType("boolean"); for (const comparator of booleanComparators) { - const value = inputValues.boolean; - const attribute = attributeByType.boolean; + const attribute = attributesByType.boolean; + const value = inputValuesByAttribute[attribute]; const validDocuments = validDocumentsByComparator[comparator]; for (const validDocument of validDocuments) { test(`${value} -> ${comparator} -> ${validDocument.metadata[attribute]}`, () => { From 56ee535fb74decd39b32a2cda20c2ce0081af43f Mon Sep 17 00:00:00 2001 From: nick-w-nick <43578531+nick-w-nick@users.noreply.github.com> Date: Thu, 19 Dec 2024 02:36:15 -0500 Subject: [PATCH 19/30] moved variable into test scenario --- .../src/structured_query/tests/functional.test.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/langchain-core/src/structured_query/tests/functional.test.ts b/langchain-core/src/structured_query/tests/functional.test.ts index dc63e39c7563..b3d185957c0a 100644 --- a/langchain-core/src/structured_query/tests/functional.test.ts +++ b/langchain-core/src/structured_query/tests/functional.test.ts @@ -41,13 +41,13 @@ describe("FunctionalTranslator", () => { }); describe("visitComparison", () => { - const attributesByType = { - string: "stringValue", - number: "numberValue", - boolean: "booleanValue", - }; - describe("returns true or false for valid comparisons", () => { + const attributesByType = { + string: "stringValue", + number: "numberValue", + boolean: "booleanValue", + }; + const inputValuesByAttribute: { [key in string]: string | number | boolean } = { stringValue: "value", numberValue: 1, From af87e116b55608a39ad5ed725b92b52b763870d0 Mon Sep 17 00:00:00 2001 From: nick-w-nick <43578531+nick-w-nick@users.noreply.github.com> Date: Thu, 19 Dec 2024 02:45:32 -0500 Subject: [PATCH 20/30] added clarifying comments --- langchain-core/src/structured_query/tests/functional.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/langchain-core/src/structured_query/tests/functional.test.ts b/langchain-core/src/structured_query/tests/functional.test.ts index b3d185957c0a..32c598db28e4 100644 --- a/langchain-core/src/structured_query/tests/functional.test.ts +++ b/langchain-core/src/structured_query/tests/functional.test.ts @@ -54,6 +54,7 @@ describe("FunctionalTranslator", () => { booleanValue: true, }; + // documents that will match against the comparison const validDocumentsByComparator: { [key in string]: Document>[] } = { [Comparators.eq]: [ new Document({ @@ -137,6 +138,7 @@ describe("FunctionalTranslator", () => { ], }; + // documents that will not match against the comparison const invalidDocumentsByComparator: { [key in string]: Document>[] } = { [Comparators.eq]: [ new Document({ From d3376c4c15694bf7a5b936d33ac201dabe590220 Mon Sep 17 00:00:00 2001 From: nick-w-nick <43578531+nick-w-nick@users.noreply.github.com> Date: Thu, 19 Dec 2024 03:01:58 -0500 Subject: [PATCH 21/30] refactored repeated tests into function that generates the tests --- .../structured_query/tests/functional.test.ts | 134 +++++------------- 1 file changed, 35 insertions(+), 99 deletions(-) diff --git a/langchain-core/src/structured_query/tests/functional.test.ts b/langchain-core/src/structured_query/tests/functional.test.ts index 32c598db28e4..c620c1d81762 100644 --- a/langchain-core/src/structured_query/tests/functional.test.ts +++ b/langchain-core/src/structured_query/tests/functional.test.ts @@ -202,109 +202,45 @@ describe("FunctionalTranslator", () => { ], }; - const stringComparators = translator.getAllowedComparatorsForType("string"); - for (const comparator of stringComparators) { - const attribute = attributesByType.string; - const value = inputValuesByAttribute[attribute]; - const validDocuments = validDocumentsByComparator[comparator]; - for (const validDocument of validDocuments) { - test(`${value} -> ${comparator} -> ${validDocument.metadata[attribute]}`, () => { - const comparison = translator.visitComparison({ - attribute, - comparator, - value, - exprName: "Comparison", - accept: (visitor: Visitor) => visitor, + function generateComparatorTestsForType(type: "string" | "number" | "boolean") { + const comparators = translator.getAllowedComparatorsForType(type); + for (const comparator of comparators) { + const attribute = attributesByType[type]; + const value = inputValuesByAttribute[attribute]; + const validDocuments = validDocumentsByComparator[comparator]; + for (const validDocument of validDocuments) { + test(`${value} -> ${comparator} -> ${validDocument.metadata[attribute]}`, () => { + const comparison = translator.visitComparison({ + attribute, + comparator, + value, + exprName: "Comparison", + accept: (visitor: Visitor) => visitor, + }); + const result = comparison(validDocument); + expect(result).toBeTruthy(); }); - const result = comparison(validDocument); - expect(result).toBeTruthy(); - }); - } - - const invalidDocuments = invalidDocumentsByComparator[comparator]; - for (const invalidDocument of invalidDocuments) { - test(`${value} -> ${comparator} -> ${invalidDocument.metadata[attribute]}`, () => { - const comparison = translator.visitComparison({ - attribute, - comparator, - value, - exprName: "Comparison", - accept: (visitor: Visitor) => visitor, - }); - const result = comparison(invalidDocument); - expect(result).toBeFalsy(); - }); - } - } - - const numberComparators = translator.getAllowedComparatorsForType("number"); - for (const comparator of numberComparators) { - const attribute = attributesByType.number; - const value = inputValuesByAttribute[attribute]; - const validDocuments = validDocumentsByComparator[comparator]; - for (const validDocument of validDocuments) { - test(`${value} -> ${comparator} -> ${validDocument.metadata[attribute]}`, () => { - const comparison = translator.visitComparison({ - attribute, - comparator, - value, - exprName: "Comparison", - accept: (visitor: Visitor) => visitor, - }); - const result = comparison(validDocument); - expect(result).toBeTruthy(); - }); - } - - const invalidDocuments = invalidDocumentsByComparator[comparator]; - for (const invalidDocument of invalidDocuments) { - test(`${value} -> ${comparator} -> ${invalidDocument.metadata[attribute]}`, () => { - const comparison = translator.visitComparison({ - attribute, - comparator, - value, - exprName: "Comparison", - accept: (visitor: Visitor) => visitor, - }); - const result = comparison(invalidDocument); - expect(result).toBeFalsy(); - }); - } - } - - const booleanComparators = translator.getAllowedComparatorsForType("boolean"); - for (const comparator of booleanComparators) { - const attribute = attributesByType.boolean; - const value = inputValuesByAttribute[attribute]; - const validDocuments = validDocumentsByComparator[comparator]; - for (const validDocument of validDocuments) { - test(`${value} -> ${comparator} -> ${validDocument.metadata[attribute]}`, () => { - const comparison = translator.visitComparison({ - attribute, - comparator, - value, - exprName: "Comparison", - accept: (visitor: Visitor) => visitor, - }); - const result = comparison(validDocument); - expect(result).toBeTruthy(); - }); - } - const invalidDocuments = invalidDocumentsByComparator[comparator]; - for (const invalidDocument of invalidDocuments) { - test(`${value} -> ${comparator} -> ${invalidDocument.metadata[attribute]}`, () => { - const comparison = translator.visitComparison({ - attribute, - comparator, - value, - exprName: "Comparison", - accept: (visitor: Visitor) => visitor, + } + const invalidDocuments = invalidDocumentsByComparator[comparator]; + for (const invalidDocument of invalidDocuments) { + test(`${value} -> ${comparator} -> ${invalidDocument.metadata[attribute]}`, () => { + const comparison = translator.visitComparison({ + attribute, + comparator, + value, + exprName: "Comparison", + accept: (visitor: Visitor) => visitor, + }); + const result = comparison(invalidDocument); + expect(result).toBeFalsy(); }); - const result = comparison(invalidDocument); - expect(result).toBeFalsy(); - }); + } } } + + generateComparatorTestsForType("string"); + generateComparatorTestsForType("number"); + generateComparatorTestsForType("boolean"); }); }); }); From f432c9ba8e8c2f9292f7b7001984d0ce3f973d84 Mon Sep 17 00:00:00 2001 From: nick-w-nick <43578531+nick-w-nick@users.noreply.github.com> Date: Thu, 19 Dec 2024 03:11:42 -0500 Subject: [PATCH 22/30] updated test value for consistency --- langchain-core/src/structured_query/tests/functional.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/langchain-core/src/structured_query/tests/functional.test.ts b/langchain-core/src/structured_query/tests/functional.test.ts index c620c1d81762..cf457168c2ec 100644 --- a/langchain-core/src/structured_query/tests/functional.test.ts +++ b/langchain-core/src/structured_query/tests/functional.test.ts @@ -176,7 +176,7 @@ describe("FunctionalTranslator", () => { metadata: { stringValue: "val", numberValue: 0, - booleanValue: false, + booleanValue: true, }, }), ], From 824d090a7f777807e3ec7821f7321a74a9cc48f1 Mon Sep 17 00:00:00 2001 From: jacoblee93 Date: Thu, 19 Dec 2024 13:43:41 -0800 Subject: [PATCH 23/30] Format --- .../src/structured_query/functional.ts | 30 +++++++++++++++---- .../structured_query/tests/functional.test.ts | 26 ++++++++++------ .../src/structured_query/tests/utils.test.ts | 2 +- langchain-core/src/structured_query/utils.ts | 2 +- .../tests/query_chain.int.test.ts | 8 ++--- 5 files changed, 47 insertions(+), 21 deletions(-) diff --git a/langchain-core/src/structured_query/functional.ts b/langchain-core/src/structured_query/functional.ts index 30b0fef887e2..2475ba1ff471 100644 --- a/langchain-core/src/structured_query/functional.ts +++ b/langchain-core/src/structured_query/functional.ts @@ -65,7 +65,7 @@ export class FunctionalTranslator extends BaseTranslator { formatFunction(): string { throw new Error("Not implemented"); } - + /** * Returns the allowed comparators for a given data type. * @param input The input value to get the allowed comparators for. @@ -74,10 +74,24 @@ export class FunctionalTranslator extends BaseTranslator { getAllowedComparatorsForType(inputType: string): Comparator[] { switch (inputType) { case "string": { - return [Comparators.eq, Comparators.ne, Comparators.gt, Comparators.gte, Comparators.lt, Comparators.lte]; + return [ + Comparators.eq, + Comparators.ne, + Comparators.gt, + Comparators.gte, + Comparators.lt, + Comparators.lte, + ]; } case "number": { - return [Comparators.eq, Comparators.ne, Comparators.gt, Comparators.gte, Comparators.lt, Comparators.lte]; + return [ + Comparators.eq, + Comparators.ne, + Comparators.gt, + Comparators.gte, + Comparators.lt, + Comparators.lte, + ]; } case "boolean": { return [Comparators.eq, Comparators.ne]; @@ -87,7 +101,7 @@ export class FunctionalTranslator extends BaseTranslator { } } } - + /** * Returns a function that performs a comparison based on the provided * comparator. @@ -181,8 +195,12 @@ export class FunctionalTranslator extends BaseTranslator { const { comparator, attribute, value } = comparison; const undefinedTrue = [Comparators.ne]; if (this.allowedComparators.includes(comparator)) { - if (!this.getAllowedComparatorsForType(typeof value).includes(comparator)) { - throw new Error(`'${comparator}' comparator not allowed to be used with ${typeof value}`); + if ( + !this.getAllowedComparatorsForType(typeof value).includes(comparator) + ) { + throw new Error( + `'${comparator}' comparator not allowed to be used with ${typeof value}` + ); } const comparatorFunction = this.getComparatorFunction(comparator); return (document: Document) => { diff --git a/langchain-core/src/structured_query/tests/functional.test.ts b/langchain-core/src/structured_query/tests/functional.test.ts index cf457168c2ec..2db797093c80 100644 --- a/langchain-core/src/structured_query/tests/functional.test.ts +++ b/langchain-core/src/structured_query/tests/functional.test.ts @@ -47,15 +47,19 @@ describe("FunctionalTranslator", () => { number: "numberValue", boolean: "booleanValue", }; - - const inputValuesByAttribute: { [key in string]: string | number | boolean } = { + + const inputValuesByAttribute: { + [key in string]: string | number | boolean; + } = { stringValue: "value", numberValue: 1, booleanValue: true, }; - + // documents that will match against the comparison - const validDocumentsByComparator: { [key in string]: Document>[] } = { + const validDocumentsByComparator: { + [key in string]: Document>[]; + } = { [Comparators.eq]: [ new Document({ pageContent: "", @@ -137,9 +141,11 @@ describe("FunctionalTranslator", () => { }), ], }; - + // documents that will not match against the comparison - const invalidDocumentsByComparator: { [key in string]: Document>[] } = { + const invalidDocumentsByComparator: { + [key in string]: Document>[]; + } = { [Comparators.eq]: [ new Document({ pageContent: "", @@ -201,8 +207,10 @@ describe("FunctionalTranslator", () => { }), ], }; - - function generateComparatorTestsForType(type: "string" | "number" | "boolean") { + + function generateComparatorTestsForType( + type: "string" | "number" | "boolean" + ) { const comparators = translator.getAllowedComparatorsForType(type); for (const comparator of comparators) { const attribute = attributesByType[type]; @@ -237,7 +245,7 @@ describe("FunctionalTranslator", () => { } } } - + generateComparatorTestsForType("string"); generateComparatorTestsForType("number"); generateComparatorTestsForType("boolean"); diff --git a/langchain-core/src/structured_query/tests/utils.test.ts b/langchain-core/src/structured_query/tests/utils.test.ts index 71c6458e34f7..53b413702c2f 100644 --- a/langchain-core/src/structured_query/tests/utils.test.ts +++ b/langchain-core/src/structured_query/tests/utils.test.ts @@ -56,7 +56,7 @@ test("Casting values correctly", () => { expect(typeof value).toBe("number"); expect(isFloat(value)).toBe(true); }); - + booleanBoolean.map(castValue).forEach((value) => { expect(typeof value).toBe("boolean"); expect(isBoolean(value)).toBe(true); diff --git a/langchain-core/src/structured_query/utils.ts b/langchain-core/src/structured_query/utils.ts index b70df772be4c..92c9639b365b 100644 --- a/langchain-core/src/structured_query/utils.ts +++ b/langchain-core/src/structured_query/utils.ts @@ -76,7 +76,7 @@ export function isString(value: unknown): boolean { * Checks if the provided value is a boolean. */ export function isBoolean(value: unknown): boolean { - return typeof value === "boolean"; + return typeof value === "boolean"; } /** diff --git a/langchain/src/chains/query_constructor/tests/query_chain.int.test.ts b/langchain/src/chains/query_constructor/tests/query_chain.int.test.ts index 88be5ae7b379..59279fe70ef1 100644 --- a/langchain/src/chains/query_constructor/tests/query_chain.int.test.ts +++ b/langchain/src/chains/query_constructor/tests/query_chain.int.test.ts @@ -76,10 +76,10 @@ test("Query Chain Test", async () => { type: "number", }, { - name: "isReleased", - description: "Whether the movie has been released", - type: "boolean", - } + name: "isReleased", + description: "Whether the movie has been released", + type: "boolean", + }, ]; const documentContents = "Brief summary of a movie"; From eeaafd4d3b131a4acacb53070941173683b500b3 Mon Sep 17 00:00:00 2001 From: nick-w-nick <43578531+nick-w-nick@users.noreply.github.com> Date: Fri, 20 Dec 2024 12:19:27 -0500 Subject: [PATCH 24/30] added boolean to vectara types --- libs/langchain-community/src/structured_query/vectara.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/langchain-community/src/structured_query/vectara.ts b/libs/langchain-community/src/structured_query/vectara.ts index ae9ca8dc7506..ab67b23e4174 100644 --- a/libs/langchain-community/src/structured_query/vectara.ts +++ b/libs/langchain-community/src/structured_query/vectara.ts @@ -27,7 +27,7 @@ export type VectaraVisitorStructuredQueryResult = { filter?: { filter?: VectaraOperationResult | VectaraComparisonResult }; }; -type Value = number | string; +type Value = number | string | boolean; function processValue(value: Value): string { /** Convert a value to a string and add single quotes if it is a string. */ if (typeof value === "string") { From 21f150474d19aa3707f4a10cd1213fc2ba484d22 Mon Sep 17 00:00:00 2001 From: nick-w-nick <43578531+nick-w-nick@users.noreply.github.com> Date: Fri, 20 Dec 2024 12:19:51 -0500 Subject: [PATCH 25/30] added supabase boolean column functionality --- .../langchain-community/src/structured_query/supabase.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/libs/langchain-community/src/structured_query/supabase.ts b/libs/langchain-community/src/structured_query/supabase.ts index 602f46bfebfa..f799e8640215 100644 --- a/libs/langchain-community/src/structured_query/supabase.ts +++ b/libs/langchain-community/src/structured_query/supabase.ts @@ -1,5 +1,6 @@ import { isFilterEmpty, + isBoolean, isFloat, isInt, isObject, @@ -28,8 +29,8 @@ import { * structured query. It can be a string or a number. */ type ValueType = { - eq: string | number; - ne: string | number; + eq: string | number | boolean; + ne: string | number | boolean; lt: string | number; lte: string | number; gt: string | number; @@ -126,7 +127,7 @@ export class SupabaseTranslator< * @param includeType Whether to include the data type in the column name. * @returns The built column name. */ - buildColumnName(attr: string, value: string | number, includeType = true) { + buildColumnName(attr: string, value: string | number | boolean, includeType = true) { let column = ""; if (isString(value)) { column = `metadata->>${attr}`; @@ -134,6 +135,8 @@ export class SupabaseTranslator< column = `metadata->${attr}${includeType ? "::int" : ""}`; } else if (isFloat(value)) { column = `metadata->${attr}${includeType ? "::float" : ""}`; + } else if(isBoolean(value)) { + column = `metadata->${attr}${includeType ? "::bool" : ""}`; } else { throw new Error("Data type not supported"); } From 21d5f7bdb3fbda78234228bff3789e1bdbbc7acd Mon Sep 17 00:00:00 2001 From: nick-w-nick <43578531+nick-w-nick@users.noreply.github.com> Date: Sat, 21 Dec 2024 00:49:23 -0500 Subject: [PATCH 26/30] Revert "added boolean to vectara types" This reverts commit eeaafd4d3b131a4acacb53070941173683b500b3. --- libs/langchain-community/src/structured_query/vectara.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/langchain-community/src/structured_query/vectara.ts b/libs/langchain-community/src/structured_query/vectara.ts index ab67b23e4174..ae9ca8dc7506 100644 --- a/libs/langchain-community/src/structured_query/vectara.ts +++ b/libs/langchain-community/src/structured_query/vectara.ts @@ -27,7 +27,7 @@ export type VectaraVisitorStructuredQueryResult = { filter?: { filter?: VectaraOperationResult | VectaraComparisonResult }; }; -type Value = number | string | boolean; +type Value = number | string; function processValue(value: Value): string { /** Convert a value to a string and add single quotes if it is a string. */ if (typeof value === "string") { From 6ff92298b89e69b305e198d250cbd203b8800c82 Mon Sep 17 00:00:00 2001 From: nick-w-nick <43578531+nick-w-nick@users.noreply.github.com> Date: Sat, 21 Dec 2024 00:50:09 -0500 Subject: [PATCH 27/30] Revert "added supabase boolean column functionality" This reverts commit 21f150474d19aa3707f4a10cd1213fc2ba484d22. --- .../langchain-community/src/structured_query/supabase.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/libs/langchain-community/src/structured_query/supabase.ts b/libs/langchain-community/src/structured_query/supabase.ts index f799e8640215..602f46bfebfa 100644 --- a/libs/langchain-community/src/structured_query/supabase.ts +++ b/libs/langchain-community/src/structured_query/supabase.ts @@ -1,6 +1,5 @@ import { isFilterEmpty, - isBoolean, isFloat, isInt, isObject, @@ -29,8 +28,8 @@ import { * structured query. It can be a string or a number. */ type ValueType = { - eq: string | number | boolean; - ne: string | number | boolean; + eq: string | number; + ne: string | number; lt: string | number; lte: string | number; gt: string | number; @@ -127,7 +126,7 @@ export class SupabaseTranslator< * @param includeType Whether to include the data type in the column name. * @returns The built column name. */ - buildColumnName(attr: string, value: string | number | boolean, includeType = true) { + buildColumnName(attr: string, value: string | number, includeType = true) { let column = ""; if (isString(value)) { column = `metadata->>${attr}`; @@ -135,8 +134,6 @@ export class SupabaseTranslator< column = `metadata->${attr}${includeType ? "::int" : ""}`; } else if (isFloat(value)) { column = `metadata->${attr}${includeType ? "::float" : ""}`; - } else if(isBoolean(value)) { - column = `metadata->${attr}${includeType ? "::bool" : ""}`; } else { throw new Error("Data type not supported"); } From 83a9db64540a3758a26fe5d1d39df357f4d08838 Mon Sep 17 00:00:00 2001 From: nick-w-nick <43578531+nick-w-nick@users.noreply.github.com> Date: Sat, 21 Dec 2024 00:55:50 -0500 Subject: [PATCH 28/30] made comparison generic with a backwards compatible default --- langchain-core/src/structured_query/functional.ts | 2 +- langchain-core/src/structured_query/ir.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/langchain-core/src/structured_query/functional.ts b/langchain-core/src/structured_query/functional.ts index 2475ba1ff471..05f3be96858e 100644 --- a/langchain-core/src/structured_query/functional.ts +++ b/langchain-core/src/structured_query/functional.ts @@ -191,7 +191,7 @@ export class FunctionalTranslator extends BaseTranslator { * @param comparison The comparison part of a structured query. * @returns A function that takes a `Document` as an argument and returns a boolean based on the comparison. */ - visitComparison(comparison: Comparison): this["VisitComparisonOutput"] { + visitComparison(comparison: Comparison): this["VisitComparisonOutput"] { const { comparator, attribute, value } = comparison; const undefinedTrue = [Comparators.ne]; if (this.allowedComparators.includes(comparator)) { diff --git a/langchain-core/src/structured_query/ir.ts b/langchain-core/src/structured_query/ir.ts index 29bf3fafe624..bbfcbd098654 100644 --- a/langchain-core/src/structured_query/ir.ts +++ b/langchain-core/src/structured_query/ir.ts @@ -149,13 +149,13 @@ export abstract class FilterDirective extends Expression {} * Class representing a comparison filter directive. It extends the * FilterDirective class. */ -export class Comparison extends FilterDirective { +export class Comparison extends FilterDirective { exprName = "Comparison" as const; constructor( public comparator: Comparator, public attribute: string, - public value: string | number | boolean + public value: ValueTypes ) { super(); } From 62eac9299a6baaeba85c3f6b107535f46537ebff Mon Sep 17 00:00:00 2001 From: nick-w-nick <43578531+nick-w-nick@users.noreply.github.com> Date: Thu, 26 Dec 2024 11:43:15 -0500 Subject: [PATCH 29/30] updated typecasting --- langchain/src/chains/query_constructor/parser.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/langchain/src/chains/query_constructor/parser.ts b/langchain/src/chains/query_constructor/parser.ts index b9e862d172ea..48c08350b3e2 100644 --- a/langchain/src/chains/query_constructor/parser.ts +++ b/langchain/src/chains/query_constructor/parser.ts @@ -91,10 +91,11 @@ export class QueryTransformer { } if (funcName in Comparators) { if (node.args && node.args.length === 2) { + const [ attribute, value ] = node.args; return new Comparison( funcName as Comparator, - traverse(node.args[0]) as string, - traverse(node.args[1]) as string | number | boolean + traverse(attribute) as string, + traverse(value) as string | number ); } throw new Error("Comparator must have exactly 2 arguments"); From b1ead948491d9029c972de0b7435b19c8983b7a1 Mon Sep 17 00:00:00 2001 From: nick-w-nick <43578531+nick-w-nick@users.noreply.github.com> Date: Thu, 26 Dec 2024 11:47:16 -0500 Subject: [PATCH 30/30] formatting fixes --- langchain-core/src/structured_query/functional.ts | 4 +++- langchain/src/chains/query_constructor/parser.ts | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/langchain-core/src/structured_query/functional.ts b/langchain-core/src/structured_query/functional.ts index 05f3be96858e..3b57b63bc6dc 100644 --- a/langchain-core/src/structured_query/functional.ts +++ b/langchain-core/src/structured_query/functional.ts @@ -191,7 +191,9 @@ export class FunctionalTranslator extends BaseTranslator { * @param comparison The comparison part of a structured query. * @returns A function that takes a `Document` as an argument and returns a boolean based on the comparison. */ - visitComparison(comparison: Comparison): this["VisitComparisonOutput"] { + visitComparison( + comparison: Comparison + ): this["VisitComparisonOutput"] { const { comparator, attribute, value } = comparison; const undefinedTrue = [Comparators.ne]; if (this.allowedComparators.includes(comparator)) { diff --git a/langchain/src/chains/query_constructor/parser.ts b/langchain/src/chains/query_constructor/parser.ts index 48c08350b3e2..c340395a860a 100644 --- a/langchain/src/chains/query_constructor/parser.ts +++ b/langchain/src/chains/query_constructor/parser.ts @@ -91,7 +91,7 @@ export class QueryTransformer { } if (funcName in Comparators) { if (node.args && node.args.length === 2) { - const [ attribute, value ] = node.args; + const [attribute, value] = node.args; return new Comparison( funcName as Comparator, traverse(attribute) as string,