Improve Search Filter Parsing (#1426)

brandon-pereira · web-flow · commit c60e646ee231 · 2025-12-04T19:18:20.000Z
Adds better support for nested Requested Throughput deeplinking, and adds better support for passing string filters which contain JSON | Before | After | |---|---| | <img width="568" height="948" alt="Screenshot 2025-12-02 at 10 14 11 AM" src="https://github.com/user-attachments/assets/396785b0-cfa3-4b54-abf4-fe4d6b1aa9b2" /> | <img width="538" height="1166" alt="Screenshot 2025-12-02 at 10 14 41 AM" src="https://github.com/user-attachments/assets/efe384f9-c615-4ca3-b719-559c9b668874" /> | | <img width="538" height="864" alt="Screenshot 2025-12-02 at 10 13 52 AM" src="https://github.com/user-attachments/assets/375990db-45d7-45cd-a6de-d4eedf09b5bd" /> | <img width="554" height="784" alt="Screenshot 2025-12-02 at 10 13 45 AM" src="https://github.com/user-attachments/assets/9bd05162-1bc9-425d-9105-8a8626863278" /> Fixes HDX-2932, HDX-2933
diff --git a/.changeset/beige-eyes-tap.md b/.changeset/beige-eyes-tap.md
@@ -0,0 +1,5 @@
+---
+"@hyperdx/app": patch
+---
+
+Improve how filters are parsed on the search page
diff --git a/packages/app/src/__tests__/searchFilters.test.ts b/packages/app/src/__tests__/searchFilters.test.ts
@@ -118,6 +118,163 @@ describe('searchFilters', () => {
       const result = parseQuery([{ type: 'lucene', condition: `app:*` }]);
       expect(result.filters).toEqual({});
     });
+
+    it('extracts IN clauses from complex conditions with AND operator', () => {
+      const result = parseQuery([
+        {
+          type: 'sql',
+          condition: `SpanName = 'flagd.evaluation.v1.Service/EventStream' AND SpanKind IN ('Server', 'SPAN_KIND_SERVER')`,
+        },
+      ]);
+      expect(result.filters).toEqual({
+        SpanKind: {
+          included: new Set(['Server', 'SPAN_KIND_SERVER']),
+          excluded: new Set(),
+        },
+      });
+    });
+
+    it('skips conditions with OR operator (not supported)', () => {
+      const result = parseQuery([
+        {
+          type: 'sql',
+          condition: `level IN ('error') OR severity IN ('high')`,
+        },
+      ]);
+      // OR is not supported, so it just tries to parse as-is and should fail cleanly
+      expect(result.filters).toEqual({});
+    });
+
+    it('skips conditions with only equality operators', () => {
+      const result = parseQuery([
+        {
+          type: 'sql',
+          condition: `status_code = 200`,
+        },
+      ]);
+      expect(result.filters).toEqual({});
+    });
+
+    it('skips conditions with only comparison operators', () => {
+      const result = parseQuery([
+        {
+          type: 'sql',
+          condition: `duration > 1000`,
+        },
+      ]);
+      expect(result.filters).toEqual({});
+    });
+
+    it('parses simple IN conditions alongside extracting from complex conditions', () => {
+      const result = parseQuery([
+        { type: 'sql', condition: `service IN ('app', 'api')` },
+        {
+          type: 'sql',
+          condition: `SpanName = 'test' AND SpanKind IN ('Server')`,
+        },
+        { type: 'sql', condition: `level IN ('error')` },
+      ]);
+      expect(result.filters).toEqual({
+        service: { included: new Set(['app', 'api']), excluded: new Set() },
+        SpanKind: { included: new Set(['Server']), excluded: new Set() },
+        level: { included: new Set(['error']), excluded: new Set() },
+      });
+    });
+
+    it('handles multiple IN clauses with AND', () => {
+      const result = parseQuery([
+        {
+          type: 'sql',
+          condition: `service IN ('app') AND level IN ('error', 'warn')`,
+        },
+      ]);
+      expect(result.filters).toEqual({
+        service: { included: new Set(['app']), excluded: new Set() },
+        level: { included: new Set(['error', 'warn']), excluded: new Set() },
+      });
+    });
+
+    it('extracts NOT IN clauses from complex conditions', () => {
+      const result = parseQuery([
+        {
+          type: 'sql',
+          condition: `status = 'active' AND level NOT IN ('debug')`,
+        },
+      ]);
+      expect(result.filters).toEqual({
+        level: { included: new Set(), excluded: new Set(['debug']) },
+      });
+    });
+
+    it('handles string values with special characters in AND conditions', () => {
+      const result = parseQuery([
+        {
+          type: 'sql',
+          condition: `SpanName = 'flagd.evaluation.v1.Service/EventStream' AND SpanKind IN ('Server', 'SPAN_KIND_SERVER')`,
+        },
+      ]);
+      expect(result.filters).toEqual({
+        SpanKind: {
+          included: new Set(['Server', 'SPAN_KIND_SERVER']),
+          excluded: new Set(),
+        },
+      });
+    });
+
+    it('handles JSON values with commas and special characters', () => {
+      const result = parseQuery([
+        {
+          type: 'sql',
+          condition: `Body IN ('{"orderId": "123", "total": 100}')`,
+        },
+      ]);
+      expect(result.filters).toEqual({
+        Body: {
+          included: new Set(['{"orderId": "123", "total": 100}']),
+          excluded: new Set(),
+        },
+      });
+    });
+
+    it('handles complex multi-line JSON values', () => {
+      const result = parseQuery([
+        {
+          type: 'sql',
+          condition: `Body IN ('Order details: { "orderId": "7b54ad99", "items": [{"id": 1}, {"id": 2}] }')`,
+        },
+      ]);
+      expect(result.filters).toEqual({
+        Body: {
+          included: new Set([
+            'Order details: { "orderId": "7b54ad99", "items": [{"id": 1}, {"id": 2}] }',
+          ]),
+          excluded: new Set(),
+        },
+      });
+    });
+
+    it('handles multiple simple values alongside single complex JSON value', () => {
+      const result = parseQuery([
+        {
+          type: 'sql',
+          condition: `status IN ('active', 'pending')`,
+        },
+        {
+          type: 'sql',
+          condition: `data IN ('{"key": "value", "nested": {"a": 1}}')`,
+        },
+      ]);
+      expect(result.filters).toEqual({
+        status: {
+          included: new Set(['active', 'pending']),
+          excluded: new Set(),
+        },
+        data: {
+          included: new Set(['{"key": "value", "nested": {"a": 1}}']),
+          excluded: new Set(),
+        },
+      });
+    });
   });
 
   describe('areFiltersEqual', () => {
diff --git a/packages/app/src/components/DBTable/DBRowTableFieldWithPopover.tsx b/packages/app/src/components/DBTable/DBRowTableFieldWithPopover.tsx
@@ -49,12 +49,7 @@ export const DBRowTableFieldWithPopover = ({
   const { onPropertyAddClick } = useContext(RowSidePanelContext);
 
   // Check if we have both the column name and filter function available
-  // Only show filter for ServiceName and SeverityText
-  const canFilter =
-    columnName &&
-    (columnName === 'ServiceName' || columnName === 'SeverityText') &&
-    onPropertyAddClick &&
-    cellValue != null;
+  const canFilter = columnName && onPropertyAddClick && cellValue != null;
 
   const handleMouseEnter = () => {
     if (hoverDisabled) return;
diff --git a/packages/app/src/components/ServiceDashboardEndpointSidePanel.tsx b/packages/app/src/components/ServiceDashboardEndpointSidePanel.tsx
@@ -49,13 +49,13 @@ export default function ServiceDashboardEndpointSidePanel({
     const filters: Filter[] = [
       {
         type: 'sql',
-        condition: `${expressions.spanName} = '${endpoint}' AND ${expressions.isSpanKindServer}`,
+        condition: `${expressions.spanName} IN ('${endpoint}') AND ${expressions.isSpanKindServer}`,
       },
     ];
     if (service) {
       filters.push({
         type: 'sql',
-        condition: `${expressions.service} = '${service}'`,
+        condition: `${expressions.service} IN ('${service}')`,
       });
     }
     return filters;
diff --git a/packages/app/src/searchFilters.tsx b/packages/app/src/searchFilters.tsx
@@ -84,6 +84,135 @@ export const areFiltersEqual = (a: FilterState, b: FilterState) => {
   return true;
 };
 
+// Helper function to split on commas while respecting quoted strings
+function splitValuesOnComma(valuesStr: string): string[] {
+  const values: string[] = [];
+  let currentValue = '';
+  let inString = false;
+
+  for (let i = 0; i < valuesStr.length; i++) {
+    const char = valuesStr[i];
+
+    if (char === "'" && (i === 0 || valuesStr[i - 1] !== '\\')) {
+      inString = !inString;
+      currentValue += char;
+      continue;
+    }
+
+    if (!inString && char === ',') {
+      if (currentValue.trim()) {
+        // Remove surrounding quotes if present
+        const trimmed = currentValue.trim();
+        const unquoted =
+          trimmed.startsWith("'") && trimmed.endsWith("'")
+            ? trimmed.slice(1, -1)
+            : trimmed;
+        values.push(unquoted);
+      }
+      currentValue = '';
+      continue;
+    }
+
+    currentValue += char;
+  }
+
+  // Add the last value
+  if (currentValue.trim()) {
+    const trimmed = currentValue.trim();
+    const unquoted =
+      trimmed.startsWith("'") && trimmed.endsWith("'")
+        ? trimmed.slice(1, -1)
+        : trimmed;
+    values.push(unquoted);
+  }
+
+  return values;
+}
+
+// Helper function to extract simple IN/NOT IN clauses from a condition
+// This handles both simple conditions and compound conditions with AND
+function extractInClauses(condition: string): Array<{
+  key: string;
+  values: string[];
+  isExclude: boolean;
+}> {
+  const results: Array<{
+    key: string;
+    values: string[];
+    isExclude: boolean;
+  }> = [];
+
+  // Split on ' AND ' while respecting quoted strings
+  const parts: string[] = [];
+  let currentPart = '';
+  let inString = false;
+
+  for (let i = 0; i < condition.length; i++) {
+    const char = condition[i];
+
+    if (char === "'" && (i === 0 || condition[i - 1] !== '\\')) {
+      inString = !inString;
+      currentPart += char;
+      continue;
+    }
+
+    if (!inString && condition.slice(i, i + 5).toUpperCase() === ' AND ') {
+      if (currentPart.trim()) {
+        parts.push(currentPart.trim());
+      }
+      currentPart = '';
+      i += 4; // Skip past ' AND '
+      continue;
+    }
+
+    currentPart += char;
+  }
+
+  if (currentPart.trim()) {
+    parts.push(currentPart.trim());
+  }
+
+  // Process each part to extract IN/NOT IN clauses
+  for (const part of parts) {
+    // Skip parts that contain OR (not supported) or comparison operators
+    if (
+      part.toUpperCase().includes(' OR ') ||
+      part.includes('=') ||
+      part.includes('<') ||
+      part.includes('>')
+    ) {
+      continue;
+    }
+
+    const isExclude = part.includes('NOT IN');
+
+    // Check if this is an IN clause
+    if (part.includes(' IN ') || part.includes(' NOT IN ')) {
+      const [key, values] = part.split(isExclude ? ' NOT IN ' : ' IN ');
+
+      if (key && values) {
+        const keyStr = key.trim();
+        // Remove outer parentheses and split on commas while respecting quotes
+        const trimmedValues = values.trim();
+        const withoutParens =
+          trimmedValues.startsWith('(') && trimmedValues.endsWith(')')
+            ? trimmedValues.slice(1, -1)
+            : trimmedValues;
+
+        const valuesArray = splitValuesOnComma(withoutParens);
+
+        results.push({
+          key: keyStr,
+          values: valuesArray,
+          isExclude,
+        });
+      }
+    }
+  }
+
+  return results;
+}
+
 export const parseQuery = (
   q: Filter[],
 ): {
@@ -125,35 +254,23 @@ export const parseQuery = (
       }
     }
 
-    // Handle IN/NOT IN conditions
-    const isExclude = filter.condition.includes('NOT IN');
-    const [key, values] = filter.condition.split(
-      isExclude ? ' NOT IN ' : ' IN ',
-    );
-
-    // Skip if key or values is not present
-    if (!key || !values) {
-      continue;
-    }
-
-    const keyStr = key.trim();
-    const valuesStr = values
-      .replace('(', '')
-      .replace(')', '')
-      .split(',')
-      .map(v => v.trim().replace(/'/g, ''));
+    // Extract all simple IN/NOT IN clauses from the condition
+    // This handles both simple conditions and compound conditions with AND/OR
+    const inClauses = extractInClauses(filter.condition);
 
-    if (!state.has(keyStr)) {
-      state.set(keyStr, { included: new Set(), excluded: new Set() });
-    }
-    const sets = state.get(keyStr)!;
-    valuesStr.forEach(v => {
-      if (isExclude) {
-        sets.excluded.add(v);
-      } else {
-        sets.included.add(v);
+    for (const clause of inClauses) {
+      if (!state.has(clause.key)) {
+        state.set(clause.key, { included: new Set(), excluded: new Set() });
       }
-    });
+      const sets = state.get(clause.key)!;
+      clause.values.forEach(v => {
+        if (clause.isExclude) {
+          sets.excluded.add(v);
+        } else {
+          sets.included.add(v);
+        }
+      });
+    }
   }
   return { filters: Object.fromEntries(state) };
 };

-Original file line number
+Diff line change
@@ @@ -0,0 +1,5 @@ @@
 +---
 +"@hyperdx/app": patch
 +---
++
 +Improve how filters are parsed on the search page