feat: Improve search and services dashboard filters (#1445)

Closes HDX-2977
Closes HDX-2602

# Summary

This PR makes a few changes to improve the filter experience for users with large numbers of facet values.

## On the search page:

1. The limit for facet values upon clicking Load More is now 10k, up from 200. This limit is applied when Load More is explicitly clicked for a single filter key, the default limit on page load remains the same.
2. Because 10k values is too many to display without serious render lag (and 10k values is more than anyone wants to scroll through) we now impose a limit of 50 values displayed with a message encouraging users to search for values if some might be hiding. 
3. When a user searches for a filter value, we now automatically load more, as presumably the value they're searching for is not already being displayed in the list
4. Filter values are sorted alphabetically when searching

### Bug Fix

Previously, when a user selected `Load More` for a filter and then switched to a different source, all values from `Load more` would be displayed as values for the second source. This has been fixed, and the Loaded More values are cleared when switching sources.

### Demo

https://github.com/user-attachments/assets/381a6366-25d9-401c-9310-fede75e9a793

## On the services dashboard

1. ServiceNames are now queried from the selected time range, to avoid poor performance or timeouts on large data volumes
2. ServiceNames are now sorted alphabetically in the dropdown
3. We now show up to 10k service names, to match the search page filter value limit

## Future improvements

Ideally, when a user searches for a filter value, we'd dispatch a new query searching for potentially matching values. This would ensure that users could find values outside of the new 10k value limit.

Author: pulpdrew

.changeset/slimy-ladybugs-attend.md

@@ -0,0 +1,5 @@
+---
+"@hyperdx/app": patch
+---
+
+feat: Improve filter search

packages/app/src/ServicesDashboardPage.tsx

@@ -110,10 +110,12 @@ function getScopedFilters({
 function ServiceSelectControlled({
   sourceId,
   onCreate,
+  dateRange,
   ...props
 }: {
   sourceId?: string;
   size?: string;
+  dateRange: [Date, Date];
   onCreate?: () => void;
 } & UseControllerProps<any>) {
   const { data: source } = useSource({ id: sourceId });
@@ -134,7 +136,8 @@ function ServiceSelectControlled({
     ],
     where: `${expressions?.service} IS NOT NULL`,
     whereLanguage: 'sql' as const,
-    limit: { limit: 200 },
+    limit: { limit: 10000 },
+    dateRange,
   };
 
   const { data, isLoading, isError } = useQueriedChartConfig(queriedConfig, {
@@ -145,7 +148,12 @@ function ServiceSelectControlled({
 
   const values = useMemo(() => {
     const services =
-      data?.data?.map((d: any) => d.service).filter(Boolean) || [];
+      data?.data
+        ?.map((d: any) => d.service)
+        .filter(Boolean)
+        .sort((a, b) =>
+          a.localeCompare(b, undefined, { sensitivity: 'base' }),
+        ) || [];
     return [
       {
         value: '',
@@ -165,6 +173,7 @@ function ServiceSelectControlled({
       placeholder="All Services"
       maxDropdownHeight={280}
       onCreate={onCreate}
+      nothingFoundMessage={isLoading ? 'Loading more...' : 'No matches found'}
     />
   );
 }
@@ -1360,6 +1369,7 @@ function ServicesDashboardPage() {
               sourceId={sourceId}
               control={control}
               name="service"
+              dateRange={searchedTimeRange}
             />
             <WhereLanguageControlled
               name="whereLanguage"

packages/app/src/components/DBSearchPageFilters.tsx

@@ -50,8 +50,20 @@ import { groupFacetsByBaseName } from './DBSearchPageFilters/utils';
 import resizeStyles from '../../styles/ResizablePanel.module.scss';
 import classes from '../../styles/SearchPage.module.scss';
 
+/* The initial number of values per filter to load */
+const INITIAL_LOAD_LIMIT = 20;
+
+/* The maximum number of values per filter to load when "Load More" is clicked */
+const LOAD_MORE_LOAD_LIMIT = 10000;
+
+/* The initial number of values per filter to render */
+const INITIAL_MAX_VALUES_DISPLAYED = 10;
+
+/* The maximum number of values per filter to render at once after loading more */
+const SHOW_MORE_MAX_VALUES_DISPLAYED = 50;
+
 // This function will clean json string attributes specifically. It will turn a string like
-// 'toString(ResourceAttributes.`hdx`.`sdk`.`version`)' into 'ResourceAttributes.hdx.sdk.verion'.
+// 'toString(ResourceAttributes.`hdx`.`sdk`.`version`)' into 'ResourceAttributes.hdx.sdk.version'.
 export function cleanedFacetName(key: string): string {
   if (key.startsWith('toString')) {
     return key
@@ -314,8 +326,6 @@ export type FilterGroupProps = {
   distributionKey?: string; // Optional key to use for distribution queries, defaults to name
 };
 
-const MAX_FILTER_GROUP_ITEMS = 10;
-
 export const FilterGroup = ({
   name,
   options,
@@ -376,6 +386,17 @@ export const FilterGroup = ({
     }
   }, [isDefaultExpanded]);
 
+  const handleSetSearch = useCallback(
+    (value: string) => {
+      setSearch(value);
+
+      if (value && !hasLoadedMore) {
+        onLoadMore(name);
+      }
+    },
+    [hasLoadedMore, name, onLoadMore],
+  );
+
   const {
     data: distributionData,
     isFetching: isFetchingDistribution,
@@ -403,11 +424,12 @@ export const FilterGroup = ({
     }
   }, [distributionError]);
 
-  const totalFiltersSize =
+  const totalAppliedFiltersSize =
     selectedValues.included.size +
     selectedValues.excluded.size +
     (hasRange ? 1 : 0);
 
+  // Loaded options + any selected options that aren't in the loaded list
   const augmentedOptions = useMemo(() => {
     const selectedSet = new Set([
       ...selectedValues.included,
@@ -421,18 +443,28 @@ export const FilterGroup = ({
     ];
   }, [options, selectedValues]);
 
-  const displayedOptions = useMemo(() => {
+  const displayedItemLimit = shouldShowMore
+    ? SHOW_MORE_MAX_VALUES_DISPLAYED
+    : INITIAL_MAX_VALUES_DISPLAYED;
+
+  // Options matching search, sorted appropriately
+  const sortedMatchingOptions = useMemo(() => {
+    // When searching, sort alphabetically
     if (search) {
-      return augmentedOptions.filter(option => {
-        return (
-          option.value &&
-          option.value.toLowerCase().includes(search.toLowerCase())
+      return augmentedOptions
+        .filter(option => {
+          return (
+            option.value &&
+            option.value.toLowerCase().includes(search.toLowerCase())
+          );
+        })
+        .toSorted((a, b) =>
+          a.value.localeCompare(b.value, undefined, { numeric: true }),
         );
-      });
     }
 
-    // General Sorting of List
-    augmentedOptions.sort((a, b) => {
+    // When not searching, sort by pinned, selected, distribution, then alphabetically
+    return augmentedOptions.toSorted((a, b) => {
       const aPinned = isPinned(a.value);
       const aIncluded = selectedValues.included.has(a.value);
       const aExcluded = selectedValues.excluded.has(a.value);
@@ -462,24 +494,22 @@ export const FilterGroup = ({
       // Finally sort alphabetically/numerically
       return a.value.localeCompare(b.value, undefined, { numeric: true });
     });
-
-    // If expanded or small list, return everything
-    if (shouldShowMore || augmentedOptions.length <= MAX_FILTER_GROUP_ITEMS) {
-      return augmentedOptions;
-    }
-    // Return the subset of items
-    const pageSize = Math.max(MAX_FILTER_GROUP_ITEMS, totalFiltersSize);
-    return augmentedOptions.slice(0, pageSize);
   }, [
     search,
-    shouldShowMore,
-    isPinned,
     augmentedOptions,
-    selectedValues,
-    totalFiltersSize,
+    isPinned,
+    selectedValues.included,
+    selectedValues.excluded,
     distributionData,
   ]);
 
+  // The subset of options to be displayed
+  const displayedOptions = useMemo(() => {
+    return sortedMatchingOptions.length <= displayedItemLimit
+      ? sortedMatchingOptions
+      : sortedMatchingOptions.slice(0, displayedItemLimit);
+  }, [sortedMatchingOptions, displayedItemLimit]);
+
   // Simple highlight animation when checkbox is checked
   const handleChange = useCallback(
     (value: string) => {
@@ -501,10 +531,14 @@ export const FilterGroup = ({
     },
     [onChange, selectedValues],
   );
+
+  const isLimitingDisplayedItems =
+    sortedMatchingOptions.length > displayedOptions.length;
+
   const showShowMoreButton =
     !search &&
-    augmentedOptions.length > MAX_FILTER_GROUP_ITEMS &&
-    totalFiltersSize < augmentedOptions.length;
+    augmentedOptions.length > INITIAL_MAX_VALUES_DISPLAYED &&
+    totalAppliedFiltersSize < augmentedOptions.length;
 
   return (
     <Accordion
@@ -581,7 +615,7 @@ export const FilterGroup = ({
                   )}
                 </>
               )}
-              {totalFiltersSize > 0 && (
+              {totalAppliedFiltersSize > 0 && (
                 <TextButton
                   label="Clear"
                   onClick={() => {
@@ -616,7 +650,7 @@ export const FilterGroup = ({
                       value={search}
                       data-testid={`filter-search-${name}`}
                       onChange={(event: React.ChangeEvent<HTMLInputElement>) =>
-                        setSearch(event.currentTarget.value)
+                        handleSetSearch(event.currentTarget.value)
                       }
                       rightSectionWidth={20}
                       rightSection={<IconSearch size={12} stroke={2} />}
@@ -669,6 +703,19 @@ export const FilterGroup = ({
                     </Text>
                   </Group>
                 ) : null}
+                {isLimitingDisplayedItems && (shouldShowMore || search) && (
+                  <Text size="xxs" ms={28} fs="italic">
+                    Search to see more
+                  </Text>
+                )}
+                {loadMoreLoading && (
+                  <Group m={6} gap="xs">
+                    <Loader size={12} color="gray" />
+                    <Text c="dimmed" size="xs">
+                      Loading more...
+                    </Text>
+                  </Group>
+                )}
                 {showShowMoreButton && (
                   <div className="d-flex m-1">
                     <TextButton
@@ -696,26 +743,18 @@ export const FilterGroup = ({
                 {onLoadMore &&
                   !showShowMoreButton &&
                   !shouldShowMore &&
-                  !hasLoadedMore && (
+                  !hasLoadedMore &&
+                  !loadMoreLoading && (
                     <div className="d-flex m-1">
-                      {loadMoreLoading ? (
-                        <Group m={6} gap="xs">
-                          <Loader size={12} color="gray" />
-                          <Text c="dimmed" size="xs">
-                            Loading more...
-                          </Text>
-                        </Group>
-                      ) : (
-                        <TextButton
-                          display={hasLoadedMore ? 'none' : undefined}
-                          label={
-                            <>
-                              <span className="bi-chevron-right" /> Load more
-                            </>
-                          }
-                          onClick={() => onLoadMore(name)}
-                        />
-                      )}
+                      <TextButton
+                        display={hasLoadedMore ? 'none' : undefined}
+                        label={
+                          <>
+                            <span className="bi-chevron-right" /> Load more
+                          </>
+                        }
+                        onClick={() => onLoadMore(name)}
+                      />
                     </div>
                   )}
               </Stack>
@@ -845,16 +884,20 @@ const DBSearchPageFiltersComponent = ({
     }
   }, [chartConfig.dateRange, isLive]);
 
+  // Clear extra facets (from "load more") when switching sources
+  useEffect(() => {
+    setExtraFacets({});
+  }, [sourceId]);
+
   const showRefreshButton = isLive && dateRange !== chartConfig.dateRange;
 
-  const keyLimit = 20;
   const {
     data: facets,
     isLoading: isFacetsLoading,
     isFetching: isFacetsFetching,
   } = useGetKeyValues({
     chartConfig: { ...chartConfig, dateRange },
-    limit: keyLimit,
+    limit: INITIAL_LOAD_LIMIT,
     keys: keysToFetch,
   });
 
@@ -894,7 +937,7 @@ const DBSearchPageFiltersComponent = ({
             dateRange,
           },
           keys: [key],
-          limit: 200,
+          limit: LOAD_MORE_LOAD_LIMIT,
           disableRowLimit: true,
         });
         const newValues = newKeyVals[0].value;