Skip to content

Commit 5690f98

Browse files
xzrderekxzrderek
committed
Merge branch 'main' into derekx/gha-rollout-processor
2 parents 6482c64 + 1e868cd commit 5690f98

File tree

10 files changed

+207
-83
lines changed

10 files changed

+207
-83
lines changed

‎.github/workflows/rollout.yml‎

Lines changed: 17 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,27 @@
11
name: Eval Protocol Rollout
22

3-
run-name: rollout:${{ fromJSON(inputs.metadata).rollout_id }}
3+
run-name: rollout:${{ inputs.rollout_id }}
44

55
on:
66
workflow_dispatch:
77
inputs:
88
model:
9-
description: 'Model to use'
9+
description: 'Model to use for the rollout'
1010
required: true
1111
type: string
12-
metadata:
13-
description: 'JSON serialized metadata object'
12+
rollout_id:
13+
description: 'Rollout ID for tracking'
1414
required: true
1515
type: string
16-
model_base_url:
17-
description: 'Base URL for the model API'
16+
prompt:
17+
description: 'User prompt for the rollout'
1818
required: true
1919
type: string
2020

2121
jobs:
2222
rollout:
2323
runs-on: ubuntu-latest
24+
name: rollout-${{ inputs.rollout_id }}
2425

2526
steps:
2627
- name: Checkout code
@@ -42,5 +43,13 @@ jobs:
4243
run: |
4344
python tests/github_actions/rollout_worker.py \
4445
--model "${{ inputs.model }}" \
45-
--metadata '${{ inputs.metadata }}' \
46-
--model-base-url "${{ inputs.model_base_url }}"
46+
--rollout-id "${{ inputs.rollout_id }}" \
47+
--prompt "${{ inputs.prompt }}"
48+
49+
- name: Upload rollout trace
50+
uses: actions/upload-artifact@v4
51+
if: always() # Upload even if the rollout failed
52+
with:
53+
name: rollout-trace-${{ inputs.rollout_id }}
54+
path: rollout_trace_${{ inputs.rollout_id }}.json
55+
retention-days: 7

‎vite-app/dist/assets/index-BnDJont9.css‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

vite-app/dist/assets/index-Cimg0aE8.js renamed to vite-app/dist/assets/index-Cu9t0G5i.js

Lines changed: 29 additions & 28 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

vite-app/dist/assets/index-Cimg0aE8.js.map renamed to vite-app/dist/assets/index-Cu9t0G5i.js.map

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎vite-app/dist/assets/index-kjowEXPL.css‎

Lines changed: 0 additions & 1 deletion
This file was deleted.

‎vite-app/dist/index.html‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@
55
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
66
<title>EP | Log Viewer</title>
77
<link rel="icon" href="/assets/favicon-BkAAWQga.png" />
8-
<script type="module" crossorigin src="/assets/index-Cimg0aE8.js"></script>
9-
<link rel="stylesheet" crossorigin href="/assets/index-kjowEXPL.css">
8+
<script type="module" crossorigin src="/assets/index-Cu9t0G5i.js"></script>
9+
<link rel="stylesheet" crossorigin href="/assets/index-BnDJont9.css">
1010
</head>
1111
<body>
1212
<div id="root"></div>

‎vite-app/src/components/EvaluationRow.tsx‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -432,6 +432,16 @@ export const EvaluationRow = observer(
432432
/>
433433
</TableCell>
434434

435+
{/* Model */}
436+
<TableCell className="py-3 text-xs">
437+
<RowModel model={row.input_metadata.completion_params.model} />
438+
</TableCell>
439+
440+
{/* Score */}
441+
<TableCell className="py-3 text-xs">
442+
<RowScore score={row.evaluation_result?.score} />
443+
</TableCell>
444+
435445
{/* Invocation ID */}
436446
<TableCell className="py-3 text-xs">
437447
<InvocationId
@@ -460,16 +470,6 @@ export const EvaluationRow = observer(
460470
<TableCell className="py-3 text-xs">
461471
<RolloutId rolloutId={row.execution_metadata?.rollout_id} />
462472
</TableCell>
463-
464-
{/* Model */}
465-
<TableCell className="py-3 text-xs">
466-
<RowModel model={row.input_metadata.completion_params.model} />
467-
</TableCell>
468-
469-
{/* Score */}
470-
<TableCell className="py-3 text-xs">
471-
<RowScore score={row.evaluation_result?.score} />
472-
</TableCell>
473473
</TableRowInteractive>
474474

475475
{/* Expanded Content Row */}

‎vite-app/src/components/EvaluationTable.tsx‎

Lines changed: 49 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ import Select from "./Select";
66
import FilterSelector from "./FilterSelector";
77
import {
88
TableHeader,
9-
TableHead,
109
TableBody as TableBodyBase,
1110
SortableTableHeader,
1211
} from "./TableContainer";
@@ -56,6 +55,28 @@ export const EvaluationTable = observer(() => {
5655
state.handleSortFieldClick(field);
5756
};
5857

58+
const handleExportFilteredRows = () => {
59+
const rows = state.filteredOriginalDataset;
60+
61+
if (rows.length === 0) {
62+
return;
63+
}
64+
65+
const jsonlContent = rows.map((row) => JSON.stringify(row)).join("\n");
66+
const blob = new Blob([jsonlContent], {
67+
type: "application/x-ndjson",
68+
});
69+
const url = URL.createObjectURL(blob);
70+
const timestamp = new Date().toISOString().replace(/[.:]/g, "-");
71+
const link = document.createElement("a");
72+
link.href = url;
73+
link.download = `evaluation-rows-${timestamp}.jsonl`;
74+
document.body.appendChild(link);
75+
link.click();
76+
document.body.removeChild(link);
77+
URL.revokeObjectURL(url);
78+
};
79+
5980
return (
6081
<div className="bg-white border border-gray-200">
6182
{/* Filter Controls */}
@@ -105,6 +126,14 @@ export const EvaluationTable = observer(() => {
105126
<option value={100}>100</option>
106127
<option value={200}>200</option>
107128
</Select>
129+
<Button
130+
onClick={handleExportFilteredRows}
131+
size="sm"
132+
variant="primary"
133+
disabled={totalRows === 0}
134+
>
135+
Export JSONL
136+
</Button>
108137
</div>
109138
</div>
110139
<div className="flex items-center gap-2">
@@ -159,11 +188,11 @@ export const EvaluationTable = observer(() => {
159188
</Button>
160189
</div>
161190
) : (
162-
<div className="overflow-x-auto">
163-
<table className="w-full min-w-max">
191+
<div className="max-h-[calc(100vh-80px)] overflow-auto">
192+
<table className="text-nowrap">
164193
{/* Table Header */}
165-
<TableHead>
166-
<tr>
194+
<thead>
195+
<tr className="bg-gray-50 sticky top-0 z-10">
167196
<TableHeader className="w-8">&nbsp;</TableHeader>
168197
<SortableTableHeader
169198
sortField="created_at"
@@ -198,63 +227,63 @@ export const EvaluationTable = observer(() => {
198227
Rollout Status
199228
</SortableTableHeader>
200229
<SortableTableHeader
201-
sortField="$.execution_metadata.invocation_id"
230+
sortField="$.input_metadata.completion_params.model"
202231
currentSortField={state.sortField}
203232
currentSortDirection={state.sortDirection}
204233
onSort={handleSort}
205234
>
206-
Invocation ID
235+
Model
207236
</SortableTableHeader>
208237
<SortableTableHeader
209-
sortField="$.execution_metadata.experiment_id"
238+
sortField="$.evaluation_result.score"
210239
currentSortField={state.sortField}
211240
currentSortDirection={state.sortDirection}
212241
onSort={handleSort}
213242
>
214-
Experiment ID
243+
Score
215244
</SortableTableHeader>
216245
<SortableTableHeader
217-
sortField="$.execution_metadata.run_id"
246+
sortField="$.execution_metadata.invocation_id"
218247
currentSortField={state.sortField}
219248
currentSortDirection={state.sortDirection}
220249
onSort={handleSort}
221250
>
222-
Run ID
251+
Invocation ID
223252
</SortableTableHeader>
224253
<SortableTableHeader
225-
sortField="$.input_metadata.row_id"
254+
sortField="$.execution_metadata.experiment_id"
226255
currentSortField={state.sortField}
227256
currentSortDirection={state.sortDirection}
228257
onSort={handleSort}
229258
>
230-
Row ID
259+
Experiment ID
231260
</SortableTableHeader>
232261
<SortableTableHeader
233-
sortField="$.execution_metadata.rollout_id"
262+
sortField="$.execution_metadata.run_id"
234263
currentSortField={state.sortField}
235264
currentSortDirection={state.sortDirection}
236265
onSort={handleSort}
237266
>
238-
Rollout ID
267+
Run ID
239268
</SortableTableHeader>
240269
<SortableTableHeader
241-
sortField="$.input_metadata.completion_params.model"
270+
sortField="$.input_metadata.row_id"
242271
currentSortField={state.sortField}
243272
currentSortDirection={state.sortDirection}
244273
onSort={handleSort}
245274
>
246-
Model
275+
Row ID
247276
</SortableTableHeader>
248277
<SortableTableHeader
249-
sortField="$.evaluation_result.score"
278+
sortField="$.execution_metadata.rollout_id"
250279
currentSortField={state.sortField}
251280
currentSortDirection={state.sortDirection}
252281
onSort={handleSort}
253282
>
254-
Score
283+
Rollout ID
255284
</SortableTableHeader>
256285
</tr>
257-
</TableHead>
286+
</thead>
258287

259288
{/* Table Body */}
260289
<TableBody

‎vite-app/src/components/LogsSection.tsx‎

Lines changed: 56 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,10 @@ import { getApiUrl } from "../config";
1010
import Select from "./Select";
1111
import Button from "./Button";
1212

13+
const haveLogsChanged = (prevLogs: LogEntry[], nextLogs: LogEntry[]) => {
14+
return prevLogs.length !== nextLogs.length;
15+
};
16+
1317
interface LogsSectionProps {
1418
rolloutId?: string;
1519
inputMetadata?: InputMetadata;
@@ -21,6 +25,8 @@ export const LogsSection = observer(({ rolloutId, inputMetadata }: LogsSectionPr
2125
const [error, setError] = useState<string | null>(null);
2226
const [selectedLevel, setSelectedLevel] = useState<string>("");
2327
const scrollContainerRef = useRef<HTMLDivElement | null>(null);
28+
const isAtBottomRef = useRef(true);
29+
const shouldAutoScrollRef = useRef(false);
2430

2531
const fetchLogs = async (isInitialLoad = false) => {
2632
if (!rolloutId) return;
@@ -95,7 +101,19 @@ export const LogsSection = observer(({ rolloutId, inputMetadata }: LogsSectionPr
95101
const data: LogsResponse = LogsResponseSchema.parse(
96102
await response.json()
97103
);
98-
setLogs(data.logs);
104+
setLogs((prevLogs) => {
105+
const hasChanges = haveLogsChanged(prevLogs, data.logs);
106+
107+
if (!hasChanges) {
108+
return prevLogs;
109+
}
110+
111+
if (isAtBottomRef.current) {
112+
shouldAutoScrollRef.current = true;
113+
}
114+
115+
return data.logs;
116+
});
99117
} catch (err) {
100118
if (err instanceof Error && err.message.includes("Unexpected token")) {
101119
setError(
@@ -117,11 +135,43 @@ export const LogsSection = observer(({ rolloutId, inputMetadata }: LogsSectionPr
117135
}
118136
}, [rolloutId, selectedLevel]);
119137

120-
// Auto-scroll to bottom whenever logs update
121138
useEffect(() => {
122139
const el = scrollContainerRef.current;
123-
if (!el) return;
140+
141+
if (!el) {
142+
isAtBottomRef.current = true;
143+
return;
144+
}
145+
146+
const handleScroll = () => {
147+
const distanceFromBottom =
148+
el.scrollHeight - el.scrollTop - el.clientHeight;
149+
isAtBottomRef.current = distanceFromBottom <= 8;
150+
};
151+
152+
el.addEventListener("scroll", handleScroll);
153+
handleScroll();
154+
155+
return () => {
156+
el.removeEventListener("scroll", handleScroll);
157+
};
158+
}, [logs.length]);
159+
160+
// Auto-scroll to bottom when new logs arrive and user was already at bottom
161+
useEffect(() => {
162+
if (!shouldAutoScrollRef.current) {
163+
return;
164+
}
165+
166+
const el = scrollContainerRef.current;
167+
if (!el) {
168+
shouldAutoScrollRef.current = false;
169+
return;
170+
}
171+
124172
el.scrollTo({ top: el.scrollHeight, behavior: "smooth" });
173+
shouldAutoScrollRef.current = false;
174+
isAtBottomRef.current = true;
125175
}, [logs]);
126176

127177
if (!rolloutId) {
@@ -189,13 +239,13 @@ export const LogsSection = observer(({ rolloutId, inputMetadata }: LogsSectionPr
189239
{logs.length > 0 && (
190240
<div
191241
ref={scrollContainerRef}
192-
className="max-h-[800px] min-h-4 overflow-auto border border-gray-200"
242+
className="max-h-[800px] min-h-4 overflow-auto border border-gray-200 bg-white"
193243
>
194-
<div>
244+
<div className="min-w-max">
195245
{logs.map((log, index) => (
196246
<div
197247
key={index}
198-
className={`text-xs px-3 py-1 border-b border-gray-200 last:border-b-0 ${
248+
className={`w-full text-xs px-3 py-1 border-b border-gray-200 last:border-b-0 ${
199249
index % 2 === 0 ? "bg-white" : "bg-gray-50"
200250
}`}
201251
>

0 commit comments

Comments
 (0)