feat: optional token separators for search index schema (#382)

src/__tests__/fixtures/json-schema/search/orders.json

@@ -61,5 +61,8 @@
 				}
 			}
 		}
+	},
+	"options": {
+		"token_separators": ["/"]
 	}
 }

src/__tests__/fixtures/schema/search/orders.ts

@@ -43,7 +43,7 @@ export class OrderStatus {
 	createdAt: Date;
 }
 
-@TigrisSearchIndex(ORDERS_INDEX_NAME)
+@TigrisSearchIndex(ORDERS_INDEX_NAME, { tokenSeparators: ["/"] })
 export class Order {
 	@SearchField(TigrisDataTypes.UUID, { sort: true })
 	orderId: string;

src/__tests__/search/schema.spec.ts

@@ -6,14 +6,15 @@ import { DecoratedSchemaProcessor, IndexSchema } from "../../schema/decorated-sc
 import { MATRICES_INDEX_NAME, Matrix, MatrixSchema } from "../fixtures/schema/search/matrices";
 import { TigrisCollection } from "../../decorators/tigris-collection";
 import { Field } from "../../decorators/tigris-field";
-import { TigrisDataTypes } from "../../types";
+import { SearchIndexOptions, TigrisDataTypes } from "../../types";
 import { IncorrectVectorDefError } from "../../error";
 
 type SchemaTestCase<T extends TigrisIndexType> = {
 	schemaClass: T;
 	expectedSchema: TigrisIndexSchema<any>;
 	name: string;
 	expectedJSON: string;
+	expectedOptions?: SearchIndexOptions;
 };
 
 const schemas: Array<SchemaTestCase<any>> = [
@@ -22,12 +23,14 @@ const schemas: Array<SchemaTestCase<any>> = [
 		expectedSchema: OrderSchema,
 		name: ORDERS_INDEX_NAME,
 		expectedJSON: "orders.json",
+		expectedOptions: { tokenSeparators: ["/"] },
 	},
 	{
 		schemaClass: Matrix,
 		expectedSchema: MatrixSchema,
 		name: MATRICES_INDEX_NAME,
 		expectedJSON: "matrices.json",
+		expectedOptions: undefined,
 	},
 ];
 
@@ -37,10 +40,11 @@ describe.each(schemas)("Schema conversion for: '$name'", (tc) => {
 	test("Convert decorated class to TigrisSchema", () => {
 		const generated: IndexSchema<unknown> = processor.processIndex(tc.schemaClass);
 		expect(generated.schema).toStrictEqual(tc.expectedSchema);
+		expect(generated.options).toStrictEqual(tc.expectedOptions);
 	});
 
 	test("Convert TigrisIndexSchema to JSON spec", () => {
-		expect(Utility._indexSchematoJSON(tc.name, tc.expectedSchema)).toBe(
+		expect(Utility._indexSchematoJSON(tc.name, tc.expectedSchema, tc.expectedOptions)).toBe(
 			readJSONFileAsObj("src/__tests__/fixtures/json-schema/search/" + tc.expectedJSON)
 		);
 	});

src/decorators/metadata/search-index-metadata.ts

@@ -1,5 +1,8 @@
+import { SearchIndexOptions } from "../../types";
+
 /**@internal*/
 export interface SearchIndexMetadata {
 	readonly indexName: string;
 	readonly target: Function;
+	readonly options: SearchIndexOptions;
 }

src/decorators/tigris-search-index.ts

@@ -1,15 +1,18 @@
 import { getDecoratorMetaStorage } from "../globals";
+import { SearchIndexOptions } from "../types";
 
 /**
  * TigrisSearchIndex decorator is used to mark a class as a schema/data model for Search Index.
  *
  * @param name - Name of Index
+ * @param options - Search index options
  */
-export function TigrisSearchIndex(name: string): ClassDecorator {
+export function TigrisSearchIndex(name: string, options?: SearchIndexOptions): ClassDecorator {
 	return function (target) {
 		getDecoratorMetaStorage().indexes.push({
 			indexName: name,
 			target: target,
+			options: options,
 		});
 	};
 }

src/schema/decorated-schema-processor.ts

@@ -2,6 +2,7 @@ import { DecoratorMetaStorage } from "../decorators/metadata/decorator-meta-stor
 import { getDecoratorMetaStorage } from "../globals";
 import {
 	CollectionFieldOptions,
+	SearchIndexOptions,
 	TigrisCollectionType,
 	TigrisDataTypes,
 	TigrisSchema,
@@ -20,6 +21,7 @@ export type CollectionSchema<T extends TigrisCollectionType> = {
 export type IndexSchema<T extends TigrisIndexType> = {
 	name: string;
 	schema: TigrisIndexSchema<T>;
+	options?: SearchIndexOptions;
 };
 
 /** @internal */
@@ -47,6 +49,7 @@ export class DecoratedSchemaProcessor {
 		return {
 			name: index.indexName,
 			schema: schema as TigrisIndexSchema<typeof cls>,
+			options: index.options,
 		};
 	}
 

src/search/search.ts

@@ -9,6 +9,7 @@ import {
 	ListIndexesRequest as ProtoListIndexesRequest,
 } from "../proto/server/v1/search_pb";
 import { DecoratedSchemaProcessor } from "../schema/decorated-schema-processor";
+import { SearchIndexOptions } from "../types";
 
 export class Search {
 	private readonly client: SearchClient;
@@ -37,6 +38,7 @@ export class Search {
 		let indexName: string;
 		let mayBeClass: new () => TigrisIndexType;
 		let schema: TigrisIndexSchema<T>;
+		let schemaOptions: SearchIndexOptions;
 
 		if (typeof nameOrClass === "string") {
 			indexName = nameOrClass as string;
@@ -65,9 +67,10 @@ export class Search {
 			schema = generatedIndex.schema as TigrisIndexSchema<T>;
 			// if indexName is not provided, use the one from model class
 			indexName = indexName ?? generatedIndex.name;
+			schemaOptions = generatedIndex.options;
 		}
 
-		const rawJSONSchema: string = Utility._indexSchematoJSON(indexName, schema);
+		const rawJSONSchema: string = Utility._indexSchematoJSON(indexName, schema, schemaOptions);
 		const createOrUpdateIndexRequest = new ProtoCreateIndexRequest()
 			.setProject(this.projectName)
 			.setName(indexName)

src/types.ts

@@ -705,6 +705,10 @@ export type PrimaryKeyOptions = {
 	autoGenerate?: boolean;
 };
 
+export type SearchIndexOptions = {
+	tokenSeparators?: Array<string>;
+};
+
 /**
  * Generates all possible paths for type parameter T. By recursively iterating over its keys. While
  * iterating the keys it makes the keys available in string form and in non string form both. For

src/utility.ts

@@ -8,6 +8,7 @@ import {
 	FindQueryOptions,
 	GroupByField,
 	ReadFields,
+	SearchIndexOptions,
 	SortOrder,
 	TigrisDataTypes,
 	TigrisSchema,
@@ -211,9 +212,18 @@ export const Utility = {
 		return toReturn;
 	},
 
-	_indexSchematoJSON<T>(indexName: string, schema: TigrisIndexSchema<T>): string {
+	_indexSchematoJSON<T>(
+		indexName: string,
+		schema: TigrisIndexSchema<T>,
+		options?: SearchIndexOptions
+	): string {
 		const root = { title: indexName, type: "object" };
 		root["properties"] = this._getSchemaProperties(schema, {}, {});
+		if (options && Array.isArray(options.tokenSeparators) && options.tokenSeparators.length > 0) {
+			root["options"] = {
+				token_separators: options.tokenSeparators,
+			};
+		}
 		return Utility.objToJsonString(root);
 	},