Skip to content

Commit 467fffb

Browse files
authored
add logger to typescript package (#332)
* add logger to typescript package * address comment * add code to surface errors * update test and revert * fixing the dep
1 parent a876856 commit 467fffb

File tree

11 files changed

+845
-128
lines changed

11 files changed

+845
-128
lines changed

eval_protocol/quickstart/svg_agent/vercel_svg_server_ts/src/models/exceptions.ts

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -116,8 +116,10 @@ export class UnauthenticatedError extends EvalProtocolError {
116116
}
117117
}
118118

119+
type EvalProtocolErrorConstructor = new (message?: string) => EvalProtocolError;
120+
119121
// Mapping from status codes to exception classes
120-
const STATUS_CODE_TO_EXCEPTION = new Map<StatusCode, typeof EvalProtocolError | null>([
122+
const STATUS_CODE_TO_EXCEPTION = new Map<StatusCode, EvalProtocolErrorConstructor | null>([
121123
[StatusCode.OK, null],
122124
[StatusCode.CANCELLED, CancelledError],
123125
[StatusCode.UNKNOWN, UnknownError],
@@ -148,7 +150,7 @@ export function exceptionForStatusCode(code: StatusCode, message: string = ''):
148150
if (!exceptionClass) {
149151
return null;
150152
}
151-
return new exceptionClass(message, code);
153+
return new exceptionClass(message);
152154
}
153155

154156
/**

tests/test_ep_upload_e2e.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -410,7 +410,7 @@ async def test_quickstart_eval(row: EvaluationRow) -> EvaluationRow:
410410

411411
test_project_dir, test_file_path = create_test_project_with_evaluation_test(
412412
test_content,
413-
"quickstart.py", # Non test_* filename
413+
"ep_upload_non_test_prefixed_eval.py", # Non test_* filename
414414
)
415415

416416
original_cwd = os.getcwd()
@@ -423,7 +423,8 @@ async def test_quickstart_eval(row: EvaluationRow) -> EvaluationRow:
423423

424424
assert len(discovered_tests) == 1
425425
assert "test_quickstart_eval" in discovered_tests[0].qualname
426-
assert "quickstart.py" in discovered_tests[0].file_path
426+
# Verify we discovered a non-test-prefixed file (our unique filename)
427+
assert "ep_upload_non_test_prefixed_eval.py" in discovered_tests[0].file_path
427428

428429
finally:
429430
os.chdir(original_cwd)

typescript/index.ts

Lines changed: 6 additions & 113 deletions
Original file line numberDiff line numberDiff line change
@@ -1,113 +1,6 @@
1-
import z from "zod";
2-
import type { ChatCompletionCreateParamsNonStreaming } from "openai/resources/chat/completions/completions";
3-
4-
// Zod schemas for validation
5-
const roleSchema = z.enum(["system", "user", "assistant"]);
6-
const messageSchema = z.union([
7-
z.object({
8-
role: roleSchema,
9-
content: z.string(),
10-
}),
11-
z.object({
12-
role: z.literal("tool"),
13-
content: z.string(),
14-
tool_call_id: z.string(),
15-
}),
16-
]);
17-
18-
const functionDefinitionSchema = z
19-
.object({
20-
name: z.string().regex(/^[a-zA-Z0-9_-]{1,64}$/),
21-
description: z.string().optional(),
22-
// JSON Schema object; allow arbitrary keys
23-
parameters: z.object({}).loose().optional(),
24-
})
25-
.loose();
26-
27-
const toolSchema = z.object({
28-
type: z.literal("function"),
29-
function: functionDefinitionSchema,
30-
});
31-
32-
const metadataSchema = z
33-
.object({
34-
invocation_id: z.string(),
35-
experiment_id: z.string(),
36-
rollout_id: z.string(),
37-
run_id: z.string(),
38-
row_id: z.string(),
39-
})
40-
.loose();
41-
42-
export const initRequestSchema = z.object({
43-
completion_params: z.record(z.string(), z.any()).describe("Completion parameters including model and optional model_kwargs, temperature, etc."),
44-
messages: z.array(messageSchema).optional(),
45-
tools: z.array(toolSchema).optional().nullable(),
46-
metadata: metadataSchema,
47-
model_base_url: z.string().optional().nullable(),
48-
});
49-
50-
export const statusInfoSchema = z.record(z.string(), z.any());
51-
52-
export const statusResponseSchema = z.object({
53-
terminated: z.boolean(),
54-
info: statusInfoSchema.optional(),
55-
});
56-
57-
// Infer types from schemas
58-
export type Message = z.infer<typeof messageSchema>;
59-
export type FunctionDefinition = z.infer<typeof functionDefinitionSchema>;
60-
export type Tool = z.infer<typeof toolSchema>;
61-
export type Metadata = z.infer<typeof metadataSchema>;
62-
export type InitRequest = z.infer<typeof initRequestSchema>;
63-
export type StatusInfo = z.infer<typeof statusInfoSchema>;
64-
export type StatusResponse = z.infer<typeof statusResponseSchema>;
65-
66-
export function initRequestToCompletionParams(
67-
initRequest: InitRequest
68-
): ChatCompletionCreateParamsNonStreaming {
69-
const model = initRequest.completion_params?.['model'];
70-
if (!model) {
71-
throw new Error("model is required in completion_params");
72-
}
73-
74-
const toolsToOpenAI = initRequest.tools?.map((tool) => ({
75-
type: "function" as const,
76-
function: tool.function.description
77-
? {
78-
name: tool.function.name,
79-
description: tool.function.description,
80-
parameters: tool.function.parameters || {},
81-
}
82-
: {
83-
name: tool.function.name,
84-
parameters: tool.function.parameters || {},
85-
},
86-
}));
87-
88-
if (!initRequest.messages) {
89-
throw new Error("messages is required");
90-
}
91-
92-
// Spread completion_params directly (model, temperature, max_tokens, etc.)
93-
const { model: _, ...otherParams } = initRequest.completion_params || {};
94-
95-
const completionParams: ChatCompletionCreateParamsNonStreaming = {
96-
model: model,
97-
messages: initRequest.messages,
98-
...(toolsToOpenAI && { tools: toolsToOpenAI }),
99-
...otherParams // Spreads temperature, max_tokens, etc.
100-
};
101-
102-
return completionParams;
103-
}
104-
105-
export function createLangfuseConfigTags(initRequest: InitRequest): string[] {
106-
return [
107-
`invocation_id:${initRequest.metadata.invocation_id}`,
108-
`experiment_id:${initRequest.metadata.experiment_id}`,
109-
`rollout_id:${initRequest.metadata.rollout_id}`,
110-
`run_id:${initRequest.metadata.run_id}`,
111-
`row_id:${initRequest.metadata.row_id}`,
112-
];
113-
}
1+
export * from "./models/types.js";
2+
export * from "./models/status.js";
3+
export * from "./models/exceptions.js";
4+
export * from "./logging/fireworks-transport.js";
5+
export * from "./logging/logger.js";
6+
export * from "./logging/fireworks-vercel.js";
Lines changed: 223 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,223 @@
1+
/**
2+
* Winston transport that sends logs to Fireworks tracing gateway.
3+
*/
4+
5+
import Transport from 'winston-transport';
6+
import type { TransformableInfo } from 'logform';
7+
const LEVEL = Symbol.for('level');
8+
9+
interface FireworksLogInfo extends TransformableInfo {
10+
rollout_id?: string;
11+
experiment_id?: string;
12+
run_id?: string;
13+
rollout_ids?: string[];
14+
status?: any;
15+
program?: string;
16+
logger_name?: string;
17+
[key: string]: any;
18+
}
19+
20+
interface StatusInfo {
21+
code?: number;
22+
message?: string;
23+
details?: any[];
24+
}
25+
26+
interface FireworksPayload {
27+
program: string;
28+
status?: StatusInfo | null;
29+
message: string;
30+
tags: string[];
31+
extras: {
32+
logger_name: string;
33+
level: string;
34+
timestamp: string;
35+
};
36+
}
37+
38+
export class FireworksTransport extends Transport {
39+
private gatewayBaseUrl: string;
40+
private rolloutIdEnv: string;
41+
private apiKey?: string;
42+
private waitUntil?: (promise: Promise<any>) => void;
43+
44+
constructor(opts: {
45+
gatewayBaseUrl?: string;
46+
rolloutIdEnv?: string;
47+
waitUntil?: (promise: Promise<any>) => void;
48+
} = {}) {
49+
super();
50+
51+
this.gatewayBaseUrl =
52+
opts.gatewayBaseUrl ||
53+
process.env.FW_TRACING_GATEWAY_BASE_URL ||
54+
'https://tracing.fireworks.ai';
55+
56+
this.rolloutIdEnv = opts.rolloutIdEnv || 'EP_ROLLOUT_ID';
57+
this.apiKey = process.env.FIREWORKS_API_KEY;
58+
this.waitUntil = opts.waitUntil;
59+
}
60+
61+
log(info: FireworksLogInfo, callback: () => void) {
62+
setImmediate(() => {
63+
this.emit('logged', info);
64+
});
65+
66+
const sendPromise = this.sendToFireworks(info).catch((error) => {
67+
this.emit('error', error);
68+
});
69+
70+
// Use waitUntil for ALL logs when available so Fireworks logging
71+
// can complete even after the HTTP response is sent.
72+
if (this.waitUntil) {
73+
this.waitUntil(sendPromise);
74+
}
75+
76+
callback();
77+
}
78+
79+
private async sendToFireworks(info: FireworksLogInfo): Promise<void> {
80+
if (!this.gatewayBaseUrl) {
81+
return;
82+
}
83+
84+
const rolloutId = this.getRolloutId(info);
85+
if (!rolloutId) {
86+
return;
87+
}
88+
89+
const payload = this.buildPayload(info, rolloutId);
90+
const baseUrl = this.gatewayBaseUrl.replace(/\/$/, '');
91+
const url = `${baseUrl}/logs`;
92+
93+
// Debug logging
94+
if (process.env.EP_DEBUG === 'true') {
95+
const tagsLen = Array.isArray(payload.tags) ? payload.tags.length : 0;
96+
const msgPreview = typeof payload.message === 'string'
97+
? payload.message.substring(0, 80)
98+
: payload.message;
99+
const payloadSize = JSON.stringify(payload).length;
100+
const hasStatus = !!payload.status;
101+
console.log(`[FW_LOG] POST ${url} rollout_id=${rolloutId} tags=${tagsLen} msg=${msgPreview} size=${payloadSize} hasStatus=${hasStatus}`);
102+
}
103+
104+
try {
105+
const headers: HeadersInit = {
106+
'Content-Type': 'application/json',
107+
'User-Agent': 'winston-fireworks-transport/1.0.0',
108+
};
109+
110+
if (this.apiKey) {
111+
headers['Authorization'] = `Bearer ${this.apiKey}`;
112+
}
113+
114+
const response = await fetch(url, {
115+
method: 'POST',
116+
headers,
117+
body: JSON.stringify(payload),
118+
// No timeout signal for compatibility
119+
});
120+
121+
if (process.env.EP_DEBUG === 'true') {
122+
console.log(`[FW_LOG] resp=${response.status} for rollout_id=${rolloutId}`);
123+
}
124+
125+
// Fallback to /v1/logs if /logs is not found
126+
if (response.status === 404) {
127+
const altUrl = `${baseUrl}/v1/logs`;
128+
129+
if (process.env.EP_DEBUG === 'true') {
130+
const tagsLen = Array.isArray(payload.tags) ? payload.tags.length : 0;
131+
console.log(`[FW_LOG] RETRY POST ${altUrl} rollout_id=${rolloutId} tags=${tagsLen}`);
132+
}
133+
134+
const retryResponse = await fetch(altUrl, {
135+
method: 'POST',
136+
headers,
137+
body: JSON.stringify(payload),
138+
// No timeout signal for compatibility
139+
});
140+
141+
if (process.env.EP_DEBUG === 'true') {
142+
console.log(`[FW_LOG] retry resp=${retryResponse.status}`);
143+
}
144+
}
145+
146+
} catch (error: any) {
147+
// Silently handle errors - logging should not break the application
148+
if (process.env.EP_DEBUG === 'true') {
149+
console.error(`[FW_LOG] Error sending to Fireworks:`, error.message);
150+
console.error(`[FW_LOG] Payload was:`, JSON.stringify(payload, null, 2));
151+
}
152+
}
153+
}
154+
155+
private getRolloutId(info: FireworksLogInfo): string | null {
156+
// Check if rollout_id is in the log info
157+
if (info.rollout_id && typeof info.rollout_id === 'string') {
158+
return info.rollout_id;
159+
}
160+
161+
// Fallback to environment variable
162+
return process.env[this.rolloutIdEnv] || null;
163+
}
164+
165+
private getStatusInfo(info: FireworksLogInfo): StatusInfo | null {
166+
if (!info.status) {
167+
return null;
168+
}
169+
170+
const status = info.status;
171+
172+
// Handle Status class instances (with code and message properties)
173+
if (typeof status === 'object' && status !== null && 'code' in status && 'message' in status) {
174+
return {
175+
code: typeof status.code === 'number' ? status.code : undefined,
176+
message: typeof status.message === 'string' ? status.message : undefined,
177+
details: Array.isArray(status.details) ? status.details : [],
178+
};
179+
}
180+
181+
return null;
182+
}
183+
184+
private buildPayload(info: FireworksLogInfo, rolloutId: string): FireworksPayload {
185+
const timestamp = new Date().toISOString();
186+
// Ensure message is always a string for Fireworks payload
187+
const message: string = typeof info.message === 'string' ? info.message : '';
188+
const level = (info as any)[LEVEL] || info.level || 'info';
189+
190+
const tags: string[] = [`rollout_id:${rolloutId}`];
191+
192+
// Optional additional tags
193+
if (info.experiment_id && typeof info.experiment_id === 'string') {
194+
tags.push(`experiment_id:${info.experiment_id}`);
195+
}
196+
if (info.run_id && typeof info.run_id === 'string') {
197+
tags.push(`run_id:${info.run_id}`);
198+
}
199+
200+
// Groupwise list of rollout_ids
201+
if (Array.isArray(info.rollout_ids)) {
202+
for (const rid of info.rollout_ids) {
203+
if (typeof rid === 'string') {
204+
tags.push(`rollout_id:${rid}`);
205+
}
206+
}
207+
}
208+
209+
const program = (typeof info.program === 'string' ? info.program : null) || 'eval_protocol';
210+
211+
return {
212+
program,
213+
status: this.getStatusInfo(info),
214+
message,
215+
tags,
216+
extras: {
217+
logger_name: info.logger_name || 'winston',
218+
level: level.toUpperCase(),
219+
timestamp,
220+
},
221+
};
222+
}
223+
}

0 commit comments

Comments
 (0)