feat(): add executed query length to search response (#1733)

packages/common/src/search/_internal/portalSearchGroups.ts

@@ -11,7 +11,7 @@ import {
   IHubSearchResult,
   IQuery,
 } from "../types";
-import { getNextFunction } from "../utils";
+import { getNextFunction, getKilobyteSizeOfQuery } from "../utils";
 import { expandPredicate } from "./expandPredicate";
 
 /**
@@ -106,6 +106,7 @@ async function searchPortal(
       resp.total,
       searchPortal
     ),
+    executedQuerySize: getKilobyteSizeOfQuery(searchOptions.q),
   };
 }
 

packages/common/src/search/_internal/portalSearchItems.ts

@@ -18,7 +18,12 @@ import {
   IPredicate,
   IQuery,
 } from "../types";
-import { addDefaultItemSearchPredicates, getNextFunction } from "../utils";
+import {
+  addDefaultItemSearchPredicates,
+  getNextFunction,
+  getKilobyteSizeOfQuery,
+  expandPortalQuery,
+} from "../utils";
 import { convertPortalAggregations } from "./portalSearchUtils";
 import { expandPredicate } from "./expandPredicate";
 import HubError from "../../HubError";
@@ -59,6 +64,9 @@ export function portalSearchItemsAsItems(
 }
 
 /**
+ * DEPRECATED - use expandPortalQuery instead
+ *
+ *
  * @internal
  * Expand an IQuery by applying well-known filters and predicates,
  * and then expanding all the predicates into IMatchOption objects.
@@ -103,7 +111,7 @@ function processSearchParams(options: IHubSearchOptions, query: IQuery) {
     );
   }
 
-  const updatedQuery = expandQuery(query);
+  const updatedQuery = expandPortalQuery(query);
 
   // Serialize the all the groups for portal
   const so = serializeQueryForPortal(updatedQuery);
@@ -166,6 +174,7 @@ async function searchPortalAsItem(
       resp.total,
       searchPortalAsItem
     ),
+    executedQuerySize: getKilobyteSizeOfQuery(searchOptions.q),
   };
 }
 
@@ -209,6 +218,7 @@ async function searchPortalAsHubSearchResult(
       resp.total,
       searchPortalAsHubSearchResult
     ),
+    executedQuerySize: getKilobyteSizeOfQuery(searchOptions.q),
   };
 }
 

packages/common/src/search/_internal/portalSearchUsers.ts

@@ -19,6 +19,7 @@ import {
 import { getNextFunction } from "../utils";
 import { expandPredicate } from "./expandPredicate";
 import { cloneObject } from "../../util";
+import { getKilobyteSizeOfQuery } from "../utils";
 
 function buildSearchOptions(
   query: IQuery,
@@ -203,6 +204,7 @@ async function searchPortal(
       resp.total,
       searchPortal
     ),
+    executedQuerySize: getKilobyteSizeOfQuery(searchOptions.q),
   };
 }
 
@@ -232,6 +234,7 @@ async function searchCommunity(
       resp.total,
       searchCommunity
     ),
+    executedQuerySize: getKilobyteSizeOfQuery(searchOptions.q),
   };
 }
 

packages/common/src/search/explainQueryResult.ts

@@ -3,6 +3,7 @@ import { IFilter, IPredicate, IQuery } from "./types";
 import { expandQuery } from "./_internal/portalSearchItems";
 import { cloneObject } from "../util";
 import { explainFilter } from "./_internal/explainFilter";
+import { expandPortalQuery } from "./utils";
 
 /**
  * Explanation of why a result matched a query
@@ -123,7 +124,7 @@ export async function explainQueryResult(
   }
 
   // Expand the query so we have a standardized structure to work with
-  const expandedQuery = expandQuery(query);
+  const expandedQuery = expandPortalQuery(query);
 
   // iterate the filters on the query and get explanations for each
   const filterExplanations: IFilterExplanation[] = [];

packages/common/src/search/types/IHubSearchResponse.ts

@@ -1,5 +1,6 @@
 import { IMessage } from "../../types/IMessage";
 import { IHubAggregation } from "./IHubAggregation";
+import { Kilobyte } from "./types";
 
 /**
  * Defines a generic search response interface with parameterized result type
@@ -36,4 +37,10 @@ export interface IHubSearchResponse<T> {
    * Array of messages / warnings
    */
   messages?: IMessage[];
+
+  /**
+   * The length of the query string that was just executed in the search,
+   * measured in kilobytes
+   */
+  executedQuerySize?: Kilobyte;
 }

packages/common/src/search/types/types.ts

@@ -176,3 +176,10 @@ export interface ICatalogSearchResponse {
 
 export interface ISearchResponseHash
   extends Record<string, IHubSearchResponse<IHubSearchResult>> {}
+
+/**
+ * Type wrapper for a kilobyte
+ * This is complete syntactic sugar, it makes sizes easier to understand
+ * with units as a type
+ */
+export type Kilobyte = number;

packages/common/src/search/utils.ts

@@ -8,6 +8,7 @@ import {
   IGroup,
   ISearchGroupUsersOptions,
   ISearchOptions,
+  SearchQueryBuilder,
 } from "@esri/arcgis-rest-portal";
 import { isPageType } from "../content/_internal/internalContentUtils";
 import { IHubSite } from "../core";
@@ -22,14 +23,20 @@ import {
   IWellKnownApis,
   IApiDefinition,
   NamedApis,
+  Kilobyte,
 } from "./types/types";
 import { WellKnownCollection } from "./wellKnownCatalog";
 import {
   isLegacySearchCategory,
   LegacySearchCategory,
 } from "./_internal/commonHelpers/isLegacySearchCategory";
 import { toCollectionKey } from "./_internal/commonHelpers/toCollectionKey";
-import { expandQuery } from "./_internal/portalSearchItems";
+import {
+  applyWellKnownCollectionFilters,
+  applyWellKnownItemPredicates,
+  expandPredicates,
+  expandQuery,
+} from "./_internal/portalSearchItems";
 
 /**
  * Well known APIs
@@ -369,3 +376,38 @@ export function addDefaultItemSearchPredicates(query: IQuery): IQuery {
   queryWithDefaultItemPredicates.filters.push(defaultPredicates);
   return queryWithDefaultItemPredicates;
 }
+
+/**
+ * Returns the size in kilobytes of a query string or a SearchQueryBuilder.
+ * This is used to later determine if a query is too large or almost too large to be sent to the server.
+ * @param query
+ * @returns
+ */
+export function getKilobyteSizeOfQuery(
+  query: string | SearchQueryBuilder
+): Kilobyte {
+  // convert query to string if it isn't already
+  const queryString = typeof query === "string" ? query : query.toParam();
+
+  // get the size of the query string using the TextEncoder api
+  const encoder = new TextEncoder();
+  const encodedString = encoder.encode(queryString);
+  const sizeInBytes = encodedString.length;
+  const sizeInKB = sizeInBytes / 1024; // Convert bytes to kilobytes
+  return sizeInKB;
+}
+
+/**
+ * Expand an item IQuery for portal by applying well-known filters and predicates,
+ * and then expanding all the predicates into IMatchOption objects.
+ * @param query `IQuery` to expand
+ * @returns IQuery
+ */
+export function expandPortalQuery(query: IQuery): IQuery {
+  let updatedQuery = applyWellKnownCollectionFilters(query);
+  // Expand well-known filterGroups
+  // TODO: Should we remove this with the whole idea of collections?
+  updatedQuery = applyWellKnownItemPredicates(updatedQuery);
+  // Expand the individual predicates in each filter
+  return expandPredicates(updatedQuery);
+}

packages/common/test/search/utils.test.ts

@@ -1,6 +1,15 @@
-import { IGroup, ISearchOptions, IUser } from "@esri/arcgis-rest-portal";
+import {
+  IGroup,
+  ISearchOptions,
+  IUser,
+  SearchQueryBuilder,
+} from "@esri/arcgis-rest-portal";
 import { IHubSite, IQuery, ISearchResponse } from "../../src";
-import { IHubSearchResult, IRelativeDate } from "../../src/search";
+import {
+  IHubSearchResult,
+  IRelativeDate,
+  serializeQueryForPortal,
+} from "../../src/search";
 import {
   expandApis,
   getUserThumbnailUrl,
@@ -11,6 +20,7 @@ import {
   migrateToCollectionKey,
   getResultSiteRelativeLink,
   getGroupPredicate,
+  getKilobyteSizeOfQuery,
 } from "../../src/search/utils";
 import { MOCK_AUTH } from "../mocks/mock-auth";
 import { mockUserSession } from "../test-helpers/fake-user-session";
@@ -476,4 +486,74 @@ describe("Search Utils:", () => {
       });
     });
   });
+
+  describe("getKilobyteSizeOfQuery", () => {
+    // These tests create a blob
+    it("returns the size of the query in kilobytes", () => {
+      const query = {
+        targetEntity: "item",
+        filters: [
+          {
+            predicates: [
+              {
+                type: "Web Map",
+                typekeywords: { any: ["my|web|map"] },
+              },
+            ],
+          },
+        ],
+      } as IQuery;
+      const stringQuery = serializeQueryForPortal(query).q;
+      const size = 0.044921875;
+      const chk = getKilobyteSizeOfQuery(stringQuery);
+      expect(chk).toEqual(size);
+    });
+
+    it("returns 0 if the query is empty", () => {
+      const queryString = "";
+      const size = 0;
+      const chk = getKilobyteSizeOfQuery(queryString);
+      expect(chk).toEqual(size);
+    });
+
+    it("handles special characters in the query", () => {
+      const query = {
+        targetEntity: "item",
+        filters: [
+          {
+            predicates: [
+              {
+                type: "Web Map",
+                title: "🚀🚀🚀",
+              },
+            ],
+          },
+        ],
+      } as IQuery;
+      const stringQuery = serializeQueryForPortal(query).q;
+      const size = 0.0400390625;
+      const chk = getKilobyteSizeOfQuery(stringQuery);
+      expect(chk).toEqual(size);
+    });
+
+    it("handles a SearchQueryBuilder object", () => {
+      const query = new SearchQueryBuilder()
+        .match("Patrick")
+        .in("owner")
+        .and()
+        .startGroup()
+        .match("Web Mapping Application")
+        .in("type")
+        .or()
+        .match("Mobile Application")
+        .in("type")
+        .or()
+        .match("Application")
+        .in("type")
+        .endGroup();
+      const size = 0.0966796875;
+      const chk = getKilobyteSizeOfQuery(query);
+      expect(chk).toEqual(size);
+    });
+  });
 });