diff --git a/src/components/common/editor-with-types.tsx b/src/components/common/editor-with-types.tsx index d95bd72..5228b7f 100644 --- a/src/components/common/editor-with-types.tsx +++ b/src/components/common/editor-with-types.tsx @@ -171,7 +171,7 @@ const MonacoEditorWithTypes = ({ definitionLinkOpensInPeek: false, contextmenu: false, }} - className="[&_.current-line]:!border-none [&_.current-line]:!bg-emerald-50 [&_.monaco-editor-background]:!bg-transparent [&_.monaco-editor]:!bg-transparent [&_[role='presentation']]:!bg-transparent" + className="[&_.current-line]:!border-none [&_.focused_.current-line]:!bg-emerald-50 [&_.monaco-editor-background]:!bg-transparent [&_.monaco-editor]:!bg-transparent [&_[role='presentation']]:!bg-transparent" /> ) diff --git a/src/components/databrowser/components/databrowser-instance.tsx b/src/components/databrowser/components/databrowser-instance.tsx index 6f30dc6..acb0214 100644 --- a/src/components/databrowser/components/databrowser-instance.tsx +++ b/src/components/databrowser/components/databrowser-instance.tsx @@ -16,6 +16,7 @@ import { Header } from "./header" import { HeaderError } from "./header-error" import { QueryBuilder } from "./query-builder" import { QueryBuilderError } from "./query-builder-error" +import { WizardButton } from "./query-wizard/wizard-button" import { SearchEmptyState } from "./search-empty-state" import { Sidebar } from "./sidebar" import { UIQueryBuilder } from "./ui-query-builder" @@ -51,7 +52,8 @@ const QueryBuilderContent = () => { return (
-
+
+ { ) : ( bytes(size, { unitSeparator: " ", + decimalPlaces: 1, }) )} @@ -59,7 +60,7 @@ export const HeaderTTLBadge = ({ dataKey }: { dataKey: string }) => { } export const Badge = ({ children, label }: { children: React.ReactNode; label: string }) => ( -
+
{label} {children}
diff --git a/src/components/databrowser/components/header/index.tsx b/src/components/databrowser/components/header/index.tsx index 18ba062..a61230a 100644 --- a/src/components/databrowser/components/header/index.tsx +++ b/src/components/databrowser/components/header/index.tsx @@ -5,25 +5,22 @@ import { IconCircleCheck, IconCirclePlus, IconLoader2, - IconPlus, IconSearch, - IconSparkles, } from "@tabler/icons-react" import { cn } from "@/lib/utils" -import { Button } from "@/components/ui/button" import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover" import { Segmented } from "@/components/ui/segmented" -import { SimpleTooltip } from "@/components/ui/tooltip" import type { TabType } from "../.." import { useDebounce } from "../../hooks/use-debounce" import { useFetchSearchIndexes } from "../../hooks/use-fetch-search-indexes" import { AddKeyModal } from "../add-key-modal" -import { QueryWizardPopover } from "../query-wizard/query-wizard-popover" -import { useQueryWizardFn } from "../query-wizard/use-query-wizard" +import { AddIndexKeyButton } from "../search/add-index-key-modal" import { CreateIndexModal } from "../search/create-index-modal" import { EditIndexModal } from "../search/edit-index-modal" +import { ExportResultsButton } from "../search/export-results-button" +import { IndexActionsMenu } from "../search/index-actions-menu" import { RefreshButton } from "./refresh-button" import { SearchInput } from "./search-input" import { DataTypeSelector } from "./type-selector" @@ -43,14 +40,7 @@ export const Header = ({ tabType, allowSearch }: { tabType: TabType; allowSearch }, { key: "values", - label: ( -
- Search -
- NEW -
-
- ), + label: "Search", }, ]} value={isValuesSearchSelected ? "values" : "keys"} @@ -63,7 +53,10 @@ export const Header = ({ tabType, allowSearch }: { tabType: TabType; allowSearch /> )} {isValuesSearchSelected ? ( - + <> + + + ) : ( <> @@ -74,9 +67,9 @@ export const Header = ({ tabType, allowSearch }: { tabType: TabType; allowSearch {/* Actions */}
- {isValuesSearchSelected && } - {isValuesSearchSelected ? : } + {isValuesSearchSelected && } + {isValuesSearchSelected ? : }
) @@ -165,15 +158,19 @@ const IndexSelector = () => { {indexes?.map((idx) => (
+ {/* Full-row overlay so the whole option (including padding) is clickable */} +
@@ -239,51 +237,3 @@ const CreateIndexButton = () => { ) } - -const AddIndexButton = () => { - const [open, setOpen] = useState(false) - - return ( - <> - - - - - - ) -} - -const WizardButton = () => { - const queryWizard = useQueryWizardFn() - const [open, setOpen] = useState(false) - - if (!queryWizard) return null - - return ( - - - - - - - - setOpen(false)} /> - - - ) -} diff --git a/src/components/databrowser/components/query-wizard/query-wizard-popover.tsx b/src/components/databrowser/components/query-wizard/query-wizard-popover.tsx index ea97e12..8f5aea8 100644 --- a/src/components/databrowser/components/query-wizard/query-wizard-popover.tsx +++ b/src/components/databrowser/components/query-wizard/query-wizard-popover.tsx @@ -6,7 +6,7 @@ import { IconChevronRight } from "@tabler/icons-react" import { useMutation } from "@tanstack/react-query" import { scanKeys } from "@/lib/scan-keys" -import { toJsLiteral } from "@/lib/utils" +import { parseJSObjectLiteral, toJsLiteral } from "@/lib/utils" import { Label } from "@/components/ui/label" import { Spinner } from "@/components/ui/spinner" @@ -81,7 +81,13 @@ export const QueryWizardPopover = ({ onClose }: { onClose?: () => void }) => { sampleData: samples, }) - const queryString = toJsLiteral(result.query) + // Format/beautify the generated query with the same formatter used elsewhere. + // Guard against the model returning an already-stringified query object. + const generated = + typeof result.query === "string" + ? (parseJSObjectLiteral(result.query) ?? result.query) + : result.query + const queryString = toJsLiteral(generated) setValuesSearchQuery(queryString) setQueryBuilderMode("code") diff --git a/src/components/databrowser/components/query-wizard/wizard-button.tsx b/src/components/databrowser/components/query-wizard/wizard-button.tsx new file mode 100644 index 0000000..3ff3ade --- /dev/null +++ b/src/components/databrowser/components/query-wizard/wizard-button.tsx @@ -0,0 +1,35 @@ +import { useState } from "react" +import { IconSparkles } from "@tabler/icons-react" + +import { Button } from "@/components/ui/button" +import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover" +import { SimpleTooltip } from "@/components/ui/tooltip" + +import { QueryWizardPopover } from "./query-wizard-popover" +import { useQueryWizardFn } from "./use-query-wizard" + +export const WizardButton = () => { + const queryWizard = useQueryWizardFn() + const [open, setOpen] = useState(false) + + if (!queryWizard) return null + + return ( + + + + + + + + setOpen(false)} /> + + + ) +} diff --git a/src/components/databrowser/components/search/add-index-key-modal.tsx b/src/components/databrowser/components/search/add-index-key-modal.tsx new file mode 100644 index 0000000..96500a9 --- /dev/null +++ b/src/components/databrowser/components/search/add-index-key-modal.tsx @@ -0,0 +1,126 @@ +import { useState } from "react" +import { useTab } from "@/tab-provider" +import type { DataType } from "@/types" +import { DialogDescription } from "@radix-ui/react-dialog" +import { IconPlus } from "@tabler/icons-react" + +import { Button } from "@/components/ui/button" +import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/ui/dialog" +import { Input } from "@/components/ui/input" +import { Label } from "@/components/ui/label" +import { Spinner } from "@/components/ui/spinner" +import { SimpleTooltip } from "@/components/ui/tooltip" +import { toast } from "@/components/ui/use-toast" +import { TypeTag } from "@/components/databrowser/components/type-tag" +import { useAddKey } from "@/components/databrowser/hooks/use-add-key" +import { useFetchSearchIndex } from "@/components/databrowser/hooks/use-fetch-search-index" + +export function AddIndexKeyButton() { + const { valuesSearch, setSelectedKey } = useTab() + const indexName = valuesSearch.index + + const [open, setOpen] = useState(false) + const [suffix, setSuffix] = useState("") + const [error, setError] = useState() + + const { data: indexDetails } = useFetchSearchIndex(indexName) + const dataType = (indexDetails?.dataType ?? "string") as DataType + const prefix = indexDetails?.prefixes?.[0] ?? "" + + const { mutateAsync: addKey, isPending } = useAddKey() + + const handleSubmit = async (e: React.FormEvent) => { + e.preventDefault() + setError(undefined) + const trimmedSuffix = suffix.trim() + if (!trimmedSuffix) { + setError("Please enter a key name") + return + } + const key = `${prefix}${trimmedSuffix}` + try { + await addKey({ key, type: dataType }) + setSelectedKey(key) + setOpen(false) + toast({ description: `Key "${key}" created` }) + } catch (error_) { + setError(error_ instanceof Error ? error_.message : "Failed to create key") + } + } + + return ( + { + if (next) { + setSuffix("") + setError(undefined) + } + setOpen(next) + }} + > + + + + + + + Add Key to Index + +
+ Add a new key to the selected search index +
+ +
+
+ Data type + +
+ +
+ +
+ {prefix && ( + + {prefix} + + )} + setSuffix(e.target.value)} + placeholder="123" + className={prefix ? "h-8 rounded-l-none" : "h-8"} + /> +
+

+ The key is prefixed to match the index and created with its data type. +

+
+ + {error &&

{error}

} + +
+ + +
+
+ + + ) +} diff --git a/src/components/databrowser/components/search/export-results-button.tsx b/src/components/databrowser/components/search/export-results-button.tsx new file mode 100644 index 0000000..2e86b08 --- /dev/null +++ b/src/components/databrowser/components/search/export-results-button.tsx @@ -0,0 +1,267 @@ +import { useRef, useState } from "react" +import { useRedis } from "@/redis-context" +import { useTab } from "@/tab-provider" +import { IconDownload } from "@tabler/icons-react" +import bytesLib from "bytes" + +import type { ExportFormat } from "@/lib/export-search-results" +import { + buildExportFilename, + downloadTextFile, + estimateExport, + estimateExportBytes, + exportSearchResults, +} from "@/lib/export-search-results" +import { parseJSObjectLiteral } from "@/lib/utils" +import { Button } from "@/components/ui/button" +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog" +import { Spinner } from "@/components/ui/spinner" +import { SimpleTooltip } from "@/components/ui/tooltip" +import { toast } from "@/components/ui/use-toast" +import { useFetchSearchIndex } from "@/components/databrowser/hooks/use-fetch-search-index" + +type Status = "estimating" | "idle" | "exporting" | "done" | "error" + +type SampleDoc = { key: string; score: number; data: unknown } + +const formatResults = (n: number) => `${n.toLocaleString()} ${n === 1 ? "result" : "results"}` + +export const ExportResultsButton = () => { + const { valuesSearch } = useTab() + const indexName = valuesSearch.index + const { redis, redisNoPipeline } = useRedis() + const { data: indexDetails } = useFetchSearchIndex(indexName) + const dataType = (indexDetails?.dataType ?? "string") as "string" | "hash" | "json" + + const [open, setOpen] = useState(false) + const [format, setFormat] = useState("json") + const [status, setStatus] = useState("idle") + const [progress, setProgress] = useState<{ count: number; bytes: number }>({ count: 0, bytes: 0 }) + const [estimate, setEstimate] = useState<{ count: number; sampleDocs: SampleDoc[] }>() + const [errorMsg, setErrorMsg] = useState() + + const exportControllerRef = useRef() + const estimateControllerRef = useRef() + + const isExporting = status === "exporting" + const estimatedBytes = estimate + ? estimateExportBytes(estimate.count, estimate.sampleDocs, format) + : undefined + + const getFilter = () => parseJSObjectLiteral>(valuesSearch.query) ?? {} + + const runEstimate = async () => { + if (!indexName || !indexDetails) return + + estimateControllerRef.current?.abort() + const controller = new AbortController() + estimateControllerRef.current = controller + + setStatus("estimating") + try { + const result = await estimateExport({ + searchRedis: redisNoPipeline, + readRedis: redis, + indexName, + filter: getFilter(), + dataType, + signal: controller.signal, + }) + if (controller.signal.aborted) return + setEstimate(result) + setStatus("idle") + } catch { + // Estimation is best-effort; allow exporting without it. + if (!controller.signal.aborted) setStatus("idle") + } + } + + const handleOpenChange = (next: boolean) => { + if (isExporting) return // don't allow closing mid-export + if (next) { + // Reset any leftover state from a previous run, then estimate. + setStatus("idle") + setProgress({ count: 0, bytes: 0 }) + setErrorMsg(undefined) + setEstimate(undefined) + setOpen(true) + void runEstimate() + } else { + estimateControllerRef.current?.abort() + setOpen(false) + } + } + + const handleExport = async () => { + if (!indexName) return + + // A still-running estimate would otherwise resolve and stomp the exporting state. + estimateControllerRef.current?.abort() + + setStatus("exporting") + setErrorMsg(undefined) + setProgress({ count: 0, bytes: 0 }) + + const controller = new AbortController() + exportControllerRef.current = controller + + try { + const { content, count, bytes } = await exportSearchResults({ + searchRedis: redisNoPipeline, + readRedis: redis, + indexName, + filter: getFilter(), + dataType, + format, + signal: controller.signal, + onProgress: (p) => setProgress(p), + }) + + downloadTextFile( + content, + buildExportFilename(indexName, format), + format === "csv" ? "text/csv;charset=utf-8" : "application/json" + ) + + setProgress({ count, bytes }) + setStatus("done") + toast({ description: `Exported ${formatResults(count)}` }) + } catch (error) { + if (error instanceof Error && error.name === "AbortError") { + setStatus("idle") + return + } + setStatus("error") + setErrorMsg(error instanceof Error ? error.message : String(error)) + } + } + + return ( + <> + + + + + + { + if (isExporting) e.preventDefault() + }} + onInteractOutside={(e) => { + if (isExporting) e.preventDefault() + }} + > + + Export results + + Exports every result matching the current query from{" "} + {indexName}, + including each document's key, score, and values. + + + +
+
+ Format +
+ {(["json", "csv"] as const).map((f) => ( + + ))} +
+
+ + {status === "estimating" && ( +
+ + Estimating size… +
+ )} + + {status === "idle" && estimate && estimate.count > 0 && ( +
+ Estimated output: ~{formatResults(estimate.count)} + {estimatedBytes ? ` · ~${bytesLib(estimatedBytes)}` : ""} +
+ )} + + {status === "idle" && estimate?.count === 0 && ( +
This index has no results to export.
+ )} + + {(isExporting || status === "done") && ( +
+ {isExporting && } + + {formatResults(progress.count)} {isExporting ? "exported so far" : "exported"} + {progress.bytes > 0 && ` · ${bytesLib(progress.bytes)}`} + +
+ )} + + {status === "error" && errorMsg && ( +
{errorMsg}
+ )} +
+ + + + + + + + + ) +} diff --git a/src/components/databrowser/components/search/index-actions-menu.tsx b/src/components/databrowser/components/search/index-actions-menu.tsx new file mode 100644 index 0000000..fcfbd4d --- /dev/null +++ b/src/components/databrowser/components/search/index-actions-menu.tsx @@ -0,0 +1,117 @@ +import { useState } from "react" +import { useTab } from "@/tab-provider" +import { IconCopy, IconDotsVertical, IconEdit, IconTrash } from "@tabler/icons-react" + +import { Button } from "@/components/ui/button" +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog" +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuSeparator, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu" +import { Spinner } from "@/components/ui/spinner" +import { toast } from "@/components/ui/use-toast" + +import { useDropSearchIndex } from "../../hooks/use-drop-search-index" +import { EditIndexModal } from "./edit-index-modal" + +export const IndexActionsMenu = () => { + const { valuesSearch, setValuesSearchIndex } = useTab() + const indexName = valuesSearch.index + + const [editOpen, setEditOpen] = useState(false) + const [deleteOpen, setDeleteOpen] = useState(false) + + const dropIndex = useDropSearchIndex() + + return ( + <> + + + + + + { + navigator.clipboard.writeText(indexName) + toast({ description: "Index key copied to clipboard" }) + }} + > + + Copy Key + + setEditOpen(true)}> + + Edit Index... + + + e.preventDefault()} + onClick={() => setDeleteOpen(true)} + > + + Delete Index... + + + + + + + + + + Delete Index + + Are you sure you want to delete index{" "} + "{indexName}"? Deleting + the index does not delete the underlying keys. This action cannot be undone. + + + + + + + + + + ) +} diff --git a/src/components/databrowser/components/search/schema-parser.test.ts b/src/components/databrowser/components/search/schema-parser.test.ts index 2e89a4e..2d744fe 100644 --- a/src/components/databrowser/components/search/schema-parser.test.ts +++ b/src/components/databrowser/components/search/schema-parser.test.ts @@ -767,7 +767,7 @@ describe("Schema Parser - Malformed Inputs", () => { } }) - test("handles field without value", () => { + test("errors on field without value", () => { const input = ` const schema: Schema = s.object({ name, @@ -775,10 +775,11 @@ describe("Schema Parser - Malformed Inputs", () => { }) ` const result = parseSchemaFromEditorValue(input) - // Should skip invalid entries and parse valid ones - expect(result.success).toBe(true) - if (result.success) { - expect(result.schema.age).toEqual({ type: "F64", fast: true }) + // A malformed field (missing value) must surface an error instead of being + // silently dropped and committing an incomplete schema. + expect(result.success).toBe(false) + if (!result.success) { + expect(result.error).toContain("name") } }) diff --git a/src/components/databrowser/components/search/schema-parser.ts b/src/components/databrowser/components/search/schema-parser.ts index 74fce65..cd0b126 100644 --- a/src/components/databrowser/components/search/schema-parser.ts +++ b/src/components/databrowser/components/search/schema-parser.ts @@ -115,7 +115,9 @@ function parseFields(content: string, prefix: string, flatSchema: FlatSchema): v for (const entry of entries) { const colonIndex = findColonIndex(entry) - if (colonIndex === -1) continue + if (colonIndex === -1) { + throw new Error(`Malformed field "${entry}": expected "fieldName: s.string()" syntax`) + } // Extract field name (handle quoted names) let fieldName = entry.slice(0, colonIndex).trim() @@ -146,10 +148,7 @@ function parseFields(content: string, prefix: string, flatSchema: FlatSchema): v } // Parse the field builder (s.string(), s.number(), etc.) - const fieldValue = parseFieldBuilder(valueStr, fullKey) - if (fieldValue) { - flatSchema[fullKey] = fieldValue - } + flatSchema[fullKey] = parseFieldBuilder(valueStr, fullKey) } } @@ -278,7 +277,7 @@ function extractFromValue(str: string): string | undefined { * Parse a field builder like s.string().noTokenize() * Throws for unrecognized field types with field name context */ -function parseFieldBuilder(str: string, fieldName: string): FieldValue | undefined { +function parseFieldBuilder(str: string, fieldName: string): FieldValue { str = str.trim().replace(/,\s*$/, "") // s.string() @@ -363,5 +362,9 @@ function parseFieldBuilder(str: string, fieldName: string): FieldValue | undefin throw new Error(`Unknown field type "s.${typeName}()" for field "${fieldName}"`) } - return undefined + // Anything that is not a recognized builder is malformed — surface it instead + // of silently dropping the field and committing an incomplete schema. + throw new Error( + `Invalid definition for field "${fieldName}": expected a schema builder like s.string()` + ) } diff --git a/src/components/databrowser/components/ui-query-builder/query-condition.tsx b/src/components/databrowser/components/ui-query-builder/query-condition.tsx index 75a00cf..5271bf7 100644 --- a/src/components/databrowser/components/ui-query-builder/query-condition.tsx +++ b/src/components/databrowser/components/ui-query-builder/query-condition.tsx @@ -103,6 +103,15 @@ export const QueryCondition = ({ const valueTypeError = getValueTypeError() + // $regex is applied per-token, so a multi-word pattern can never match. Warn the + // user instead of silently running a query that returns nothing. + const regexMultiWordError = + condition.operator === "regex" && + typeof condition.value === "string" && + condition.value.trim().includes(" ") + ? "$regex matches a single token and can't span multiple words — use the phrase operator instead." + : undefined + // Normalize values based on field type changes useEffect(() => { if (currentFieldType === "boolean") { @@ -482,17 +491,19 @@ export const QueryCondition = ({ ) : ( - handleValueChange(e.target.value)} - onBlur={handleValueBlur} - onFocus={handleValueFocus} - placeholder={condition.operator === "in" ? "value1, value2, ..." : "value"} - className={`h-[26px] min-w-0 flex-1 rounded-none rounded-r-md border border-zinc-200 bg-white px-2 text-sm transition-colors focus:border-zinc-400 focus:outline-none ${ - valueTypeError ? "text-red-500" : "" - }`} - /> + + handleValueChange(e.target.value)} + onBlur={handleValueBlur} + onFocus={handleValueFocus} + placeholder={condition.operator === "in" ? "value1, value2, ..." : "value"} + className={`h-[26px] min-w-0 flex-1 rounded-none rounded-r-md border border-zinc-200 bg-white px-2 text-sm transition-colors focus:border-zinc-400 focus:outline-none ${ + valueTypeError || regexMultiWordError ? "text-red-500" : "" + }`} + /> + )}
@@ -558,6 +569,21 @@ export const QueryCondition = ({ )} + {currentFieldInfo?.noTokenize && ( + + + no-tokenize + + + )} + {currentFieldInfo?.noStem && ( + + + no-stem + + + )} + {node.not && } {node.boost !== undefined && } diff --git a/src/components/databrowser/components/ui-query-builder/types.ts b/src/components/databrowser/components/ui-query-builder/types.ts index b826cd4..6bbc435 100644 --- a/src/components/databrowser/components/ui-query-builder/types.ts +++ b/src/components/databrowser/components/ui-query-builder/types.ts @@ -17,6 +17,8 @@ export type FieldType = "string" | "number" | "boolean" | "date" | "unknown" export type FieldInfo = { name: string type: FieldType + noTokenize?: boolean + noStem?: boolean } export const STRING_OPERATORS: FieldOperator[] = ["smart", "eq", "in", "phrase", "regex", "fuzzy"] diff --git a/src/components/databrowser/components/ui-query-builder/ui-query-builder.tsx b/src/components/databrowser/components/ui-query-builder/ui-query-builder.tsx index a07eb5d..7aed3cf 100644 --- a/src/components/databrowser/components/ui-query-builder/ui-query-builder.tsx +++ b/src/components/databrowser/components/ui-query-builder/ui-query-builder.tsx @@ -164,5 +164,7 @@ const extractFieldInfo = (schema: NonNullable["schema"]): FieldInfo return Object.entries(schema).map(([fieldPath, fieldInfo]) => ({ name: fieldPath, type: getFieldType(fieldInfo.type), + noTokenize: fieldInfo.noTokenize, + noStem: fieldInfo.noStem, })) } diff --git a/src/components/databrowser/hooks/use-drop-search-index.ts b/src/components/databrowser/hooks/use-drop-search-index.ts new file mode 100644 index 0000000..71c8b0f --- /dev/null +++ b/src/components/databrowser/hooks/use-drop-search-index.ts @@ -0,0 +1,19 @@ +import { useRedis } from "@/redis-context" +import { useMutation } from "@tanstack/react-query" + +import { queryClient } from "@/lib/clients" + +import { FETCH_SEARCH_INDEXES_QUERY_KEY } from "./use-fetch-search-indexes" + +export const useDropSearchIndex = () => { + const { redisNoPipeline: redis } = useRedis() + + return useMutation({ + mutationFn: async (indexName: string) => { + await redis.search.index({ name: indexName }).drop() + }, + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: [FETCH_SEARCH_INDEXES_QUERY_KEY] }) + }, + }) +} diff --git a/src/components/databrowser/index.tsx b/src/components/databrowser/index.tsx index d315dfb..4ad3a57 100644 --- a/src/components/databrowser/index.tsx +++ b/src/components/databrowser/index.tsx @@ -139,7 +139,14 @@ export const RedisBrowser = ({ const credentials = useMemo(() => ({ token, url }), [token, url]) const rootRef = useRef(null) + // Track the url the queries were last reset for so we only reset when the + // database actually changes. Resetting on the initial mount would refetch the + // keys query that has just started fetching, causing a duplicate first SCAN. + const lastResetUrl = useRef(credentials.url) + useEffect(() => { + if (lastResetUrl.current === credentials.url) return + lastResetUrl.current = credentials.url queryClient.resetQueries() }, [credentials.url]) diff --git a/src/lib/export-search-results.ts b/src/lib/export-search-results.ts new file mode 100644 index 0000000..ba9d8c4 --- /dev/null +++ b/src/lib/export-search-results.ts @@ -0,0 +1,226 @@ +import type { Redis } from "@upstash/redis" + +export type ExportFormat = "csv" | "json" + +const PAGE_SIZE = 1000 +const SAMPLE_SIZE = 50 + +type IndexDataType = "string" | "hash" | "json" + +type SearchDoc = { key: string; score: number; data: unknown } + +// Read a single document's value the same way the app reads values elsewhere. +const readDocumentValue = async ( + redis: Redis, + key: string, + dataType: IndexDataType +): Promise => { + if (dataType === "json") return await redis.json.get(key) + if (dataType === "hash") { + const raw = (await redis.hgetall(key)) as unknown + // hgetall may come back as a flat [field, value, ...] array; normalize to an object. + return Array.isArray(raw) + ? raw.reduce>((obj, value, i, arr) => { + if (i % 2 === 0) obj[String(value)] = arr[i + 1] + return obj + }, {}) + : raw + } + return await redis.get(key) +} + +export const exportSearchResults = async (opts: { + searchRedis: Redis // non-pipelined client used for the search query + readRedis: Redis // pipelined client used to batch the per-key value reads + indexName: string + filter: Record + dataType: IndexDataType + format: ExportFormat + signal?: AbortSignal + onProgress?: (p: { count: number; bytes: number }) => void +}): Promise<{ content: string; count: number; bytes: number }> => { + const { searchRedis, readRedis, indexName, filter, dataType, format, signal, onProgress } = opts + + const index = searchRedis.search.index({ name: indexName }) + const docs: SearchDoc[] = [] + let offset = 0 + let bytes = 0 + + for (;;) { + if (signal?.aborted) throw new DOMException("Aborted", "AbortError") + + // `select: {}` -> NOCONTENT -> returns just { key, score } for each match (the proven path). + const rows = (await index.query({ + filter, + limit: PAGE_SIZE, + offset, + select: {}, + })) as { key: string; score: number }[] + + if (rows.length === 0) break + + const pageDocs = await Promise.all( + rows.map(async ({ key, score }) => ({ + key, + score, + data: await readDocumentValue(readRedis, key, dataType), + })) + ) + + docs.push(...pageDocs) + offset += rows.length + bytes += new Blob([JSON.stringify(pageDocs)]).size + + onProgress?.({ count: docs.length, bytes }) + + if (rows.length < PAGE_SIZE) break + } + + const content = format === "csv" ? serializeCsv(docs) : serializeJson(docs) + + // Report the actual serialized output size (the running `bytes` above is a raw + // approximation used only for live progress and won't match the final file). + return { content, count: docs.length, bytes: new Blob([content]).size } +} + +/** + * Cheaply estimate an export before running it: the exact total match count via a + * COUNT query, plus a small sample of documents (with values) used to extrapolate + * the output file size. + */ +export const estimateExport = async (opts: { + searchRedis: Redis + readRedis: Redis + indexName: string + filter: Record + dataType: IndexDataType + signal?: AbortSignal +}): Promise<{ count: number; sampleDocs: SearchDoc[] }> => { + const { searchRedis, readRedis, indexName, filter, dataType, signal } = opts + const index = searchRedis.search.index({ name: indexName }) + + const { count } = await index.count({ filter }) + + if (signal?.aborted) throw new DOMException("Aborted", "AbortError") + + const rows = (await index.query({ + filter, + limit: SAMPLE_SIZE, + offset: 0, + select: {}, + })) as { key: string; score: number }[] + + const sampleDocs = await Promise.all( + rows.map(async ({ key, score }) => ({ + key, + score, + data: await readDocumentValue(readRedis, key, dataType), + })) + ) + + return { count, sampleDocs } +} + +/** + * Extrapolate the output file size for a format from the average serialized size + * of the sampled documents across the full result count. Rough by design. + */ +export const estimateExportBytes = ( + count: number, + sampleDocs: SearchDoc[], + format: ExportFormat +): number => { + if (sampleDocs.length === 0 || count === 0) return 0 + const sampleContent = format === "csv" ? serializeCsv(sampleDocs) : serializeJson(sampleDocs) + const perRow = new Blob([sampleContent]).size / sampleDocs.length + return Math.round(perRow * count) +} + +const serializeJson = (docs: SearchDoc[]): string => { + const rows = docs.map((d) => ({ + key: d.key, + ...(d.data && typeof d.data === "object" + ? (d.data as Record) + : { value: d.data }), + })) + return JSON.stringify(rows, undefined, 2) +} + +const flatten = ( + value: unknown, + prefix: string, + out: Record +): Record => { + if (value && typeof value === "object" && !Array.isArray(value)) { + for (const [k, v] of Object.entries(value as Record)) { + flatten(v, prefix ? `${prefix}.${k}` : k, out) + } + } else { + out[prefix] = value + } + return out +} + +const escapeCsvCell = (value: unknown): string => { + let str: string + if (value === undefined || value === null) { + str = "" + } else if (typeof value === "object") { + str = JSON.stringify(value) + } else { + str = String(value) + } + + if (/[\n\r",]/.test(str)) { + return `"${str.replaceAll('"', '""')}"` + } + return str +} + +const serializeCsv = (docs: SearchDoc[]): string => { + const flatRows = docs.map((d) => { + const out: Record = {} + if (d.data && typeof d.data === "object") { + flatten(d.data, "", out) + } else { + out.value = d.data + } + return out + }) + + const dataColumns = new Set() + for (const row of flatRows) { + for (const col of Object.keys(row)) dataColumns.add(col) + } + const columns = ["key", ...[...dataColumns].sort()] + + const lines: string[] = [] + lines.push(columns.map((c) => escapeCsvCell(c)).join(",")) + + for (const [i, doc] of docs.entries()) { + const row = flatRows[i] + const cells = columns.map((col) => + col === "key" ? escapeCsvCell(doc.key) : escapeCsvCell(row[col]) + ) + lines.push(cells.join(",")) + } + + return lines.join("\n") +} + +export const downloadTextFile = (content: string, filename: string, mimeType: string): void => { + const blob = new Blob([content], { type: mimeType }) + const url = URL.createObjectURL(blob) + const a = document.createElement("a") + a.href = url + a.download = filename + document.body.append(a) + a.click() + a.remove() + URL.revokeObjectURL(url) +} + +export const buildExportFilename = (indexName: string, format: ExportFormat): string => { + const safe = indexName.replaceAll(/[^\w.-]/g, "_") + return `${safe}-results.${format}` +}