Fix filter loading issues by searching over more rows and caching

packages/app/src/components/DBSearchPageFilters.tsx

@@ -18,7 +18,12 @@ import {
 } from '@mantine/core';
 import { IconSearch } from '@tabler/icons-react';
 
-import { useAllFields, useGetKeyValues } from '@/hooks/useMetadata';
+import { useExplainQuery } from '@/hooks/useExplainQuery';
+import {
+  MAX_ROWS_TO_READ,
+  useAllFields,
+  useGetKeyValues,
+} from '@/hooks/useMetadata';
 import useResizable from '@/hooks/useResizable';
 import { FilterStateHook, usePinnedFilters } from '@/searchFilters';
 import { mergePath } from '@/utils';
@@ -343,6 +348,9 @@ export const DBSearchPageFilters = ({
   );
   const { width, startResize } = useResizable(16, 'left');
 
+  const { data: countData } = useExplainQuery(chartConfig);
+  const numRows = countData?.[0]?.rows;
+
   const { data, isLoading } = useAllFields({
     databaseName: chartConfig.from.databaseName,
     tableName: chartConfig.from.tableName,
@@ -389,20 +397,32 @@ export const DBSearchPageFilters = ({
 
   useEffect(() => {
     if (!isLive) {
-      setDateRange(chartConfig.dateRange);
+      setDateRange(() => {
+        setDisableRowLimit(() => false);
+        return chartConfig.dateRange;
+      });
     }
   }, [chartConfig.dateRange, isLive]);
 
   const showRefreshButton = isLive && dateRange !== chartConfig.dateRange;
 
+  const [disableRowLimit, setDisableRowLimit] = useState(false);
+  const keyLimit = 100;
   const {
     data: facets,
     isLoading: isFacetsLoading,
     isFetching: isFacetsFetching,
   } = useGetKeyValues({
     chartConfigs: { ...chartConfig, dateRange },
+    limit: keyLimit,
     keys: datum,
+    disableRowLimit,
   });
+  useEffect(() => {
+    if (numRows > MAX_ROWS_TO_READ && facets && facets.length < keyLimit) {
+      setDisableRowLimit(() => true);
+    }
+  }, [facets]);
 
   const shownFacets = useMemo(() => {
     const _facets: { key: string; value: string[] }[] = [];

packages/app/src/hooks/useMetadata.tsx

@@ -1,6 +1,7 @@
 import objectHash from 'object-hash';
 import { ColumnMeta } from '@hyperdx/common-utils/dist/clickhouse';
 import {
+  DEFAULT_SAMPLE_SIZE,
   Field,
   TableConnection,
   TableMetadata,
@@ -15,6 +16,8 @@ import {
 import { getMetadata } from '@/metadata';
 import { toArray } from '@/utils';
 
+export const MAX_ROWS_TO_READ = DEFAULT_SAMPLE_SIZE;
+
 export function useColumns(
   {
     databaseName,
@@ -116,6 +119,7 @@ export function useGetKeyValues(
       'useMetadata.useGetKeyValues',
       ...chartConfigsArr.map(cc => ({ ...cc })),
       ...keys,
+      disableRowLimit,
     ],
     queryFn: async () =>
       (

packages/common-utils/src/metadata.ts

@@ -11,7 +11,7 @@ import {
 import { renderChartConfig } from '@/renderChartConfig';
 import type { ChartConfig, ChartConfigWithDateRange, TSource } from '@/types';
 
-const DEFAULT_SAMPLE_SIZE = 1e6;
+export const DEFAULT_SAMPLE_SIZE = 1e6;
 
 export class MetadataCache {
   private cache = new Map<string, any>();
@@ -438,36 +438,41 @@ export class Metadata {
     limit?: number;
     disableRowLimit?: boolean;
   }) {
-    const sql = await renderChartConfig(
-      {
-        ...chartConfig,
-        select: keys
-          .map((k, i) => `groupUniqArray(${limit})(${k}) AS param${i}`)
-          .join(', '),
+    return this.cache.getOrFetch(
+      `${chartConfig.from.databaseName}.${keys.join(',')}.${chartConfig.dateRange.toString()}.${disableRowLimit}.values`,
+      async () => {
+        const sql = await renderChartConfig(
+          {
+            ...chartConfig,
+            select: keys
+              .map((k, i) => `groupUniqArray(${limit})(${k}) AS param${i}`)
+              .join(', '),
+          },
+          this,
+        );
+
+        const json = await this.clickhouseClient
+          .query<'JSON'>({
+            query: sql.sql,
+            query_params: sql.params,
+            connectionId: chartConfig.connection,
+            clickhouse_settings: !disableRowLimit
+              ? {
+                  max_rows_to_read: DEFAULT_SAMPLE_SIZE,
+                  read_overflow_mode: 'break',
+                }
+              : undefined,
+          })
+          .then(res => res.json<any>());
+
+        // TODO: Fix type issues mentioned in HDX-1548. value is not acually a
+        // string[], sometimes it's { [key: string]: string; }
+        return Object.entries(json.data[0]).map(([key, value]) => ({
+          key: keys[parseInt(key.replace('param', ''))],
+          value: (value as string[])?.filter(Boolean), // remove nulls
+        }));
       },
-      this,
     );
-
-    const json = await this.clickhouseClient
-      .query<'JSON'>({
-        query: sql.sql,
-        query_params: sql.params,
-        connectionId: chartConfig.connection,
-        clickhouse_settings: !disableRowLimit
-          ? {
-              max_rows_to_read: DEFAULT_SAMPLE_SIZE,
-              read_overflow_mode: 'break',
-            }
-          : undefined,
-      })
-      .then(res => res.json<any>());
-
-    // TODO: Fix type issues mentioned in HDX-1548. value is not acually a
-    // string[], sometimes it's { [key: string]: string; }
-    return Object.entries(json.data[0]).map(([key, value]) => ({
-      key: keys[parseInt(key.replace('param', ''))],
-      value: (value as string[])?.filter(Boolean), // remove nulls
-    }));
   }
 }