Skip to content

Commit f78675f

Browse files
ericallamericallam
committed
fix: replace SQL string interpolation with parameterized promptVersions filter
1 parent a2adc21 commit f78675f

File tree

3 files changed

+31
-27
lines changed

3 files changed

+31
-27
lines changed

‎apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.prompts.$promptSlug/route.tsx‎

Lines changed: 20 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1530,9 +1530,6 @@ function MetricsTab({
15301530
const operations = filterValues("operations").filter((v) => v !== "");
15311531
const providers = filterValues("providers").filter((v) => v !== "");
15321532

1533-
const versionFilter =
1534-
versionFilters.length > 0 ? ` AND prompt_version IN (${versionFilters.join(",")})` : "";
1535-
15361533
const widgetProps = {
15371534
organizationId,
15381535
projectId,
@@ -1542,6 +1539,7 @@ function MetricsTab({
15421539
from,
15431540
to,
15441541
promptSlugs: [prompt.slug],
1542+
promptVersions: versionFilters.length > 0 ? versionFilters : undefined,
15451543
responseModels: models.length > 0 ? models : undefined,
15461544
operations: operations.length > 0 ? operations : undefined,
15471545
providers: providers.length > 0 ? providers : undefined,
@@ -1555,7 +1553,7 @@ function MetricsTab({
15551553
<MetricWidget
15561554
widgetKey={`prompt-${prompt.slug}-generations`}
15571555
title="Total"
1558-
query={`SELECT count() AS generations FROM llm_metrics WHERE 1=1${versionFilter}`}
1556+
query={`SELECT count() AS generations FROM llm_metrics WHERE 1=1`}
15591557
config={{
15601558
type: "bignumber",
15611559
column: "generations",
@@ -1569,7 +1567,7 @@ function MetricsTab({
15691567
<MetricWidget
15701568
widgetKey={`prompt-${prompt.slug}-tokens`}
15711569
title="Avg input tokens"
1572-
query={`SELECT round(avg(input_tokens)) AS avg_input FROM llm_metrics WHERE 1=1${versionFilter}`}
1570+
query={`SELECT round(avg(input_tokens)) AS avg_input FROM llm_metrics WHERE 1=1`}
15731571
config={{
15741572
type: "bignumber",
15751573
column: "avg_input",
@@ -1583,7 +1581,7 @@ function MetricsTab({
15831581
<MetricWidget
15841582
widgetKey={`prompt-${prompt.slug}-cost`}
15851583
title="Avg input cost"
1586-
query={`SELECT avg(input_cost) AS avg_cost FROM llm_metrics WHERE 1=1${versionFilter}`}
1584+
query={`SELECT avg(input_cost) AS avg_cost FROM llm_metrics WHERE 1=1`}
15871585
config={{
15881586
type: "bignumber",
15891587
column: "avg_cost",
@@ -1597,7 +1595,7 @@ function MetricsTab({
15971595
<MetricWidget
15981596
widgetKey={`prompt-${prompt.slug}-latency`}
15991597
title="Avg latency"
1600-
query={`SELECT round(avg(duration) / 1000000, 1) AS avg_ms FROM llm_metrics WHERE 1=1${versionFilter}`}
1598+
query={`SELECT round(avg(duration) / 1000000, 1) AS avg_ms FROM llm_metrics WHERE 1=1`}
16011599
config={{
16021600
type: "bignumber",
16031601
column: "avg_ms",
@@ -1651,9 +1649,6 @@ function VersionPerformanceSection({
16511649
const operations = filterValues("operations").filter((v) => v !== "");
16521650
const providers = filterValues("providers").filter((v) => v !== "");
16531651

1654-
const versionFilter =
1655-
versionFilters.length > 0 ? ` AND prompt_version IN (${versionFilters.join(",")})` : "";
1656-
16571652
const widgetProps = {
16581653
organizationId,
16591654
projectId,
@@ -1663,23 +1658,21 @@ function VersionPerformanceSection({
16631658
from,
16641659
to,
16651660
promptSlugs: [promptSlug],
1661+
promptVersions: versionFilters.length > 0 ? versionFilters : undefined,
16661662
responseModels: models.length > 0 ? models : undefined,
16671663
operations: operations.length > 0 ? operations : undefined,
16681664
providers: providers.length > 0 ? providers : undefined,
16691665
};
16701666

16711667
return (
16721668
<div className="space-y-3">
1673-
<div className="flex items-center gap-2">
1674-
</div>
1675-
16761669
<div className="grid grid-cols-2 gap-3">
16771670
{/* Row 1: Latency + TTFC */}
16781671
<div className="h-96">
16791672
<MetricWidget
1680-
widgetKey={`prompt-${promptSlug}-perf-latency-${versionFilters.join(",")}`}
1673+
widgetKey={`prompt-${promptSlug}-perf-latency`}
16811674
title="Latency p50 / p95"
1682-
query={`SELECT timeBucket(), round(quantile(0.5)(duration) / 1000000, 1) AS p50, round(quantile(0.95)(duration) / 1000000, 1) AS p95 FROM llm_metrics WHERE 1=1${versionFilter} GROUP BY timeBucket ORDER BY timeBucket`}
1675+
query={`SELECT timeBucket(), round(quantile(0.5)(duration) / 1000000, 1) AS p50, round(quantile(0.95)(duration) / 1000000, 1) AS p95 FROM llm_metrics WHERE 1=1 GROUP BY timeBucket ORDER BY timeBucket`}
16831676
config={{
16841677
type: "chart",
16851678
chartType: "line",
@@ -1696,9 +1689,9 @@ function VersionPerformanceSection({
16961689
</div>
16971690
<div className="h-96">
16981691
<MetricWidget
1699-
widgetKey={`prompt-${promptSlug}-perf-ttfc-${versionFilters.join(",")}`}
1692+
widgetKey={`prompt-${promptSlug}-perf-ttfc`}
17001693
title="TTFC p50 / p95"
1701-
query={`SELECT timeBucket(), round(quantile(0.5)(ms_to_first_chunk), 1) AS p50, round(quantile(0.95)(ms_to_first_chunk), 1) AS p95 FROM llm_metrics WHERE ms_to_first_chunk > 0${versionFilter} GROUP BY timeBucket ORDER BY timeBucket`}
1694+
query={`SELECT timeBucket(), round(quantile(0.5)(ms_to_first_chunk), 1) AS p50, round(quantile(0.95)(ms_to_first_chunk), 1) AS p95 FROM llm_metrics WHERE ms_to_first_chunk > 0 GROUP BY timeBucket ORDER BY timeBucket`}
17021695
config={{
17031696
type: "chart",
17041697
chartType: "line",
@@ -1716,9 +1709,9 @@ function VersionPerformanceSection({
17161709
{/* Row 2: Input tokens + Input cost */}
17171710
<div className="h-96">
17181711
<MetricWidget
1719-
widgetKey={`prompt-${promptSlug}-perf-input-tokens-${versionFilters.join(",")}`}
1712+
widgetKey={`prompt-${promptSlug}-perf-input-tokens`}
17201713
title="Input tokens p50 / p95"
1721-
query={`SELECT timeBucket(), round(quantile(0.5)(input_tokens)) AS p50, round(quantile(0.95)(input_tokens)) AS p95 FROM llm_metrics WHERE 1=1${versionFilter} GROUP BY timeBucket ORDER BY timeBucket`}
1714+
query={`SELECT timeBucket(), round(quantile(0.5)(input_tokens)) AS p50, round(quantile(0.95)(input_tokens)) AS p95 FROM llm_metrics WHERE 1=1 GROUP BY timeBucket ORDER BY timeBucket`}
17221715
config={{
17231716
type: "chart",
17241717
chartType: "line",
@@ -1735,9 +1728,9 @@ function VersionPerformanceSection({
17351728
</div>
17361729
<div className="h-96">
17371730
<MetricWidget
1738-
widgetKey={`prompt-${promptSlug}-perf-input-cost-${versionFilters.join(",")}`}
1731+
widgetKey={`prompt-${promptSlug}-perf-input-cost`}
17391732
title="Input cost per 1k tokens (p50 / p95)"
1740-
query={`SELECT timeBucket(), prettyFormat(quantile(0.5)(input_cost / input_tokens * 1000), 'costInDollars') AS p50, prettyFormat(quantile(0.95)(input_cost / input_tokens * 1000), 'costInDollars') AS p95 FROM llm_metrics WHERE input_tokens > 0${versionFilter} GROUP BY timeBucket ORDER BY timeBucket`}
1733+
query={`SELECT timeBucket(), prettyFormat(quantile(0.5)(input_cost / input_tokens * 1000), 'costInDollars') AS p50, prettyFormat(quantile(0.95)(input_cost / input_tokens * 1000), 'costInDollars') AS p95 FROM llm_metrics WHERE input_tokens > 0 GROUP BY timeBucket ORDER BY timeBucket`}
17411734
config={{
17421735
type: "chart",
17431736
chartType: "line",
@@ -1755,9 +1748,9 @@ function VersionPerformanceSection({
17551748
{/* Row 3: Output tokens + Output cost */}
17561749
<div className="h-96">
17571750
<MetricWidget
1758-
widgetKey={`prompt-${promptSlug}-perf-output-tokens-${versionFilters.join(",")}`}
1751+
widgetKey={`prompt-${promptSlug}-perf-output-tokens`}
17591752
title="Output tokens p50 / p95"
1760-
query={`SELECT timeBucket(), round(quantile(0.5)(output_tokens)) AS p50, round(quantile(0.95)(output_tokens)) AS p95 FROM llm_metrics WHERE 1=1${versionFilter} GROUP BY timeBucket ORDER BY timeBucket`}
1753+
query={`SELECT timeBucket(), round(quantile(0.5)(output_tokens)) AS p50, round(quantile(0.95)(output_tokens)) AS p95 FROM llm_metrics WHERE 1=1 GROUP BY timeBucket ORDER BY timeBucket`}
17611754
config={{
17621755
type: "chart",
17631756
chartType: "line",
@@ -1774,9 +1767,9 @@ function VersionPerformanceSection({
17741767
</div>
17751768
<div className="h-96">
17761769
<MetricWidget
1777-
widgetKey={`prompt-${promptSlug}-perf-output-cost-${versionFilters.join(",")}`}
1770+
widgetKey={`prompt-${promptSlug}-perf-output-cost`}
17781771
title="Output cost per 1k tokens (p50 / p95)"
1779-
query={`SELECT timeBucket(), prettyFormat(quantile(0.5)(output_cost / output_tokens * 1000), 'costInDollars') AS p50, prettyFormat(quantile(0.95)(output_cost / output_tokens * 1000), 'costInDollars') AS p95 FROM llm_metrics WHERE output_tokens > 0${versionFilter} GROUP BY timeBucket ORDER BY timeBucket`}
1772+
query={`SELECT timeBucket(), prettyFormat(quantile(0.5)(output_cost / output_tokens * 1000), 'costInDollars') AS p50, prettyFormat(quantile(0.95)(output_cost / output_tokens * 1000), 'costInDollars') AS p95 FROM llm_metrics WHERE output_tokens > 0 GROUP BY timeBucket ORDER BY timeBucket`}
17801773
config={{
17811774
type: "chart",
17821775
chartType: "line",
@@ -1795,9 +1788,9 @@ function VersionPerformanceSection({
17951788

17961789
<div className="h-48">
17971790
<MetricWidget
1798-
widgetKey={`prompt-${promptSlug}-perf-versions-table-${versionFilters.join(",")}`}
1791+
widgetKey={`prompt-${promptSlug}-perf-versions-table`}
17991792
title="Version summary"
1800-
query={`SELECT prompt_version, count() AS calls, round(avg(input_tokens)) AS avg_input_tokens, round(avg(output_tokens)) AS avg_output_tokens, prettyFormat(avg(total_cost), 'costInDollars') AS avg_total_cost, round(quantile(0.5)(duration) / 1000000, 1) AS p50_latency_ms, round(quantile(0.95)(duration) / 1000000, 1) AS p95_latency_ms FROM llm_metrics WHERE 1=1${versionFilter} GROUP BY prompt_version ORDER BY prompt_version DESC`}
1793+
query={`SELECT prompt_version, count() AS calls, round(avg(input_tokens)) AS avg_input_tokens, round(avg(output_tokens)) AS avg_output_tokens, prettyFormat(avg(total_cost), 'costInDollars') AS avg_total_cost, round(quantile(0.5)(duration) / 1000000, 1) AS p50_latency_ms, round(quantile(0.95)(duration) / 1000000, 1) AS p95_latency_ms FROM llm_metrics WHERE 1=1 GROUP BY prompt_version ORDER BY prompt_version DESC`}
18011794
config={{ type: "table", prettyFormatting: true, sorting: [] }}
18021795
{...widgetProps}
18031796
/>

‎apps/webapp/app/routes/resources.metric.tsx‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ const MetricWidgetQuery = z.object({
4646
queues: z.array(z.string()).optional(),
4747
responseModels: z.array(z.string()).optional(),
4848
promptSlugs: z.array(z.string()).optional(),
49+
promptVersions: z.array(z.number()).optional(),
4950
operations: z.array(z.string()).optional(),
5051
providers: z.array(z.string()).optional(),
5152
tags: z.array(z.string()).optional(),
@@ -80,6 +81,7 @@ export const action = async ({ request }: ActionFunctionArgs) => {
8081
queues,
8182
responseModels,
8283
promptSlugs,
84+
promptVersions,
8385
operations,
8486
providers,
8587
tags,
@@ -117,6 +119,7 @@ export const action = async ({ request }: ActionFunctionArgs) => {
117119
queues,
118120
responseModels,
119121
promptSlugs,
122+
promptVersions,
120123
operations,
121124
providers,
122125
// Set higher concurrency if many widgets are on screen at once
@@ -274,6 +277,7 @@ export function MetricWidget({
274277
JSON.stringify(props.queues),
275278
JSON.stringify(props.responseModels),
276279
JSON.stringify(props.promptSlugs),
280+
JSON.stringify(props.promptVersions),
277281
JSON.stringify(props.operations),
278282
JSON.stringify(props.providers),
279283
]);

‎apps/webapp/app/services/queryService.server.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,8 @@ export type ExecuteQueryOptions<TOut extends z.ZodSchema> = Omit<
7777
responseModels?: string[];
7878
/** Filter to specific prompt slugs */
7979
promptSlugs?: string[];
80+
/** Filter to specific prompt versions */
81+
promptVersions?: number[];
8082
/** Filter to specific operations (e.g. ai.generateText.doGenerate) */
8183
operations?: string[];
8284
/** Filter to specific providers (e.g. openai.responses) */
@@ -137,6 +139,7 @@ export async function executeQuery<TOut extends z.ZodSchema>(
137139
queues,
138140
responseModels,
139141
promptSlugs,
142+
promptVersions,
140143
operations,
141144
providers,
142145
history,
@@ -228,6 +231,10 @@ export async function executeQuery<TOut extends z.ZodSchema>(
228231
: undefined,
229232
prompt_slug:
230233
promptSlugs && promptSlugs.length > 0 ? { op: "in", values: promptSlugs } : undefined,
234+
prompt_version:
235+
promptVersions && promptVersions.length > 0
236+
? { op: "in", values: promptVersions }
237+
: undefined,
231238
operation_id:
232239
operations && operations.length > 0 ? { op: "in", values: operations } : undefined,
233240
gen_ai_system:

0 commit comments

Comments
 (0)