From 7b7d30cf96d1212e739d0c60da05e8612689559e Mon Sep 17 00:00:00 2001 From: Derek Xu Date: Sat, 15 Nov 2025 01:20:38 -0800 Subject: [PATCH 1/5] add logger to typescript package --- typescript/index.ts | 119 +---------- typescript/logging/fireworks-transport.ts | 223 +++++++++++++++++++++ typescript/logging/fireworks-vercel.ts | 21 ++ typescript/logging/logger.ts | 44 ++++ typescript/models/exceptions.ts | 213 ++++++++++++++++++++ typescript/models/status.ts | 234 ++++++++++++++++++++++ typescript/models/types.ts | 67 +++++++ typescript/package.json | 23 ++- typescript/tsconfig.json | 9 +- 9 files changed, 831 insertions(+), 122 deletions(-) create mode 100644 typescript/logging/fireworks-transport.ts create mode 100644 typescript/logging/fireworks-vercel.ts create mode 100644 typescript/logging/logger.ts create mode 100644 typescript/models/exceptions.ts create mode 100644 typescript/models/status.ts create mode 100644 typescript/models/types.ts diff --git a/typescript/index.ts b/typescript/index.ts index 19fdb95b..a5067b37 100644 --- a/typescript/index.ts +++ b/typescript/index.ts @@ -1,113 +1,6 @@ -import z from "zod"; -import type { ChatCompletionCreateParamsNonStreaming } from "openai/resources/chat/completions/completions"; - -// Zod schemas for validation -const roleSchema = z.enum(["system", "user", "assistant"]); -const messageSchema = z.union([ - z.object({ - role: roleSchema, - content: z.string(), - }), - z.object({ - role: z.literal("tool"), - content: z.string(), - tool_call_id: z.string(), - }), -]); - -const functionDefinitionSchema = z - .object({ - name: z.string().regex(/^[a-zA-Z0-9_-]{1,64}$/), - description: z.string().optional(), - // JSON Schema object; allow arbitrary keys - parameters: z.object({}).loose().optional(), - }) - .loose(); - -const toolSchema = z.object({ - type: z.literal("function"), - function: functionDefinitionSchema, -}); - -const metadataSchema = z - .object({ - invocation_id: z.string(), - experiment_id: z.string(), - rollout_id: z.string(), - run_id: z.string(), - row_id: z.string(), - }) - .loose(); - -export const initRequestSchema = z.object({ - completion_params: z.record(z.string(), z.any()).describe("Completion parameters including model and optional model_kwargs, temperature, etc."), - messages: z.array(messageSchema).optional(), - tools: z.array(toolSchema).optional().nullable(), - metadata: metadataSchema, - model_base_url: z.string().optional().nullable(), -}); - -export const statusInfoSchema = z.record(z.string(), z.any()); - -export const statusResponseSchema = z.object({ - terminated: z.boolean(), - info: statusInfoSchema.optional(), -}); - -// Infer types from schemas -export type Message = z.infer; -export type FunctionDefinition = z.infer; -export type Tool = z.infer; -export type Metadata = z.infer; -export type InitRequest = z.infer; -export type StatusInfo = z.infer; -export type StatusResponse = z.infer; - -export function initRequestToCompletionParams( - initRequest: InitRequest -): ChatCompletionCreateParamsNonStreaming { - const model = initRequest.completion_params?.['model']; - if (!model) { - throw new Error("model is required in completion_params"); - } - - const toolsToOpenAI = initRequest.tools?.map((tool) => ({ - type: "function" as const, - function: tool.function.description - ? { - name: tool.function.name, - description: tool.function.description, - parameters: tool.function.parameters || {}, - } - : { - name: tool.function.name, - parameters: tool.function.parameters || {}, - }, - })); - - if (!initRequest.messages) { - throw new Error("messages is required"); - } - - // Spread completion_params directly (model, temperature, max_tokens, etc.) - const { model: _, ...otherParams } = initRequest.completion_params || {}; - - const completionParams: ChatCompletionCreateParamsNonStreaming = { - model: model, - messages: initRequest.messages, - ...(toolsToOpenAI && { tools: toolsToOpenAI }), - ...otherParams // Spreads temperature, max_tokens, etc. - }; - - return completionParams; -} - -export function createLangfuseConfigTags(initRequest: InitRequest): string[] { - return [ - `invocation_id:${initRequest.metadata.invocation_id}`, - `experiment_id:${initRequest.metadata.experiment_id}`, - `rollout_id:${initRequest.metadata.rollout_id}`, - `run_id:${initRequest.metadata.run_id}`, - `row_id:${initRequest.metadata.row_id}`, - ]; -} +export * from "./models/types.js"; +export * from "./models/status.js"; +export * from "./models/exceptions.js"; +export * from "./logging/fireworks-transport.js"; +export * from "./logging/logger.js"; +export * from "./logging/fireworks-vercel.js"; diff --git a/typescript/logging/fireworks-transport.ts b/typescript/logging/fireworks-transport.ts new file mode 100644 index 00000000..450cd01f --- /dev/null +++ b/typescript/logging/fireworks-transport.ts @@ -0,0 +1,223 @@ +/** + * Winston transport that sends logs to Fireworks tracing gateway. + */ + +import Transport from 'winston-transport'; +import type { TransformableInfo } from 'logform'; +const LEVEL = Symbol.for('level'); + +interface FireworksLogInfo extends TransformableInfo { + rollout_id?: string; + experiment_id?: string; + run_id?: string; + rollout_ids?: string[]; + status?: any; + program?: string; + logger_name?: string; + [key: string]: any; +} + +interface StatusInfo { + code?: number; + message?: string; + details?: any[]; +} + +interface FireworksPayload { + program: string; + status?: StatusInfo | null; + message: string; + tags: string[]; + extras: { + logger_name: string; + level: string; + timestamp: string; + }; +} + +export class FireworksTransport extends Transport { + private gatewayBaseUrl: string; + private rolloutIdEnv: string; + private apiKey?: string; + private waitUntil?: (promise: Promise) => void; + + constructor(opts: { + gatewayBaseUrl?: string; + rolloutIdEnv?: string; + waitUntil?: (promise: Promise) => void; + } = {}) { + super(); + + this.gatewayBaseUrl = + opts.gatewayBaseUrl || + process.env.FW_TRACING_GATEWAY_BASE_URL || + 'https://tracing.fireworks.ai'; + + this.rolloutIdEnv = opts.rolloutIdEnv || 'EP_ROLLOUT_ID'; + this.apiKey = process.env.FIREWORKS_API_KEY; + this.waitUntil = opts.waitUntil; + } + + log(info: FireworksLogInfo, callback: () => void) { + setImmediate(() => { + this.emit('logged', info); + }); + + const sendPromise = this.sendToFireworks(info).catch((error) => { + this.emit('error', error); + }); + + // Use waitUntil for ALL logs when available so Fireworks logging + // can complete even after the HTTP response is sent. + if (this.waitUntil) { + this.waitUntil(sendPromise); + } + + callback(); + } + + private async sendToFireworks(info: FireworksLogInfo): Promise { + if (!this.gatewayBaseUrl) { + return; + } + + const rolloutId = this.getRolloutId(info); + if (!rolloutId) { + return; + } + + const payload = this.buildPayload(info, rolloutId); + const baseUrl = this.gatewayBaseUrl.replace(/\/$/, ''); + const url = `${baseUrl}/logs`; + + // Debug logging + if (process.env.EP_DEBUG === 'true') { + const tagsLen = Array.isArray(payload.tags) ? payload.tags.length : 0; + const msgPreview = typeof payload.message === 'string' + ? payload.message.substring(0, 80) + : payload.message; + const payloadSize = JSON.stringify(payload).length; + const hasStatus = !!payload.status; + console.log(`[FW_LOG] POST ${url} rollout_id=${rolloutId} tags=${tagsLen} msg=${msgPreview} size=${payloadSize} hasStatus=${hasStatus}`); + } + + try { + const headers: HeadersInit = { + 'Content-Type': 'application/json', + 'User-Agent': 'winston-fireworks-transport/1.0.0', + }; + + if (this.apiKey) { + headers['Authorization'] = `Bearer ${this.apiKey}`; + } + + const response = await fetch(url, { + method: 'POST', + headers, + body: JSON.stringify(payload), + // No timeout signal for compatibility + }); + + if (process.env.EP_DEBUG === 'true') { + console.log(`[FW_LOG] resp=${response.status} for rollout_id=${rolloutId}`); + } + + // Fallback to /v1/logs if /logs is not found + if (response.status === 404) { + const altUrl = `${baseUrl}/v1/logs`; + + if (process.env.EP_DEBUG === 'true') { + const tagsLen = Array.isArray(payload.tags) ? payload.tags.length : 0; + console.log(`[FW_LOG] RETRY POST ${altUrl} rollout_id=${rolloutId} tags=${tagsLen}`); + } + + const retryResponse = await fetch(altUrl, { + method: 'POST', + headers, + body: JSON.stringify(payload), + // No timeout signal for compatibility + }); + + if (process.env.EP_DEBUG === 'true') { + console.log(`[FW_LOG] retry resp=${retryResponse.status}`); + } + } + + } catch (error: any) { + // Silently handle errors - logging should not break the application + if (process.env.EP_DEBUG === 'true') { + console.error(`[FW_LOG] Error sending to Fireworks:`, error.message); + console.error(`[FW_LOG] Payload was:`, JSON.stringify(payload, null, 2)); + } + } + } + + private getRolloutId(info: FireworksLogInfo): string | null { + // Check if rollout_id is in the log info + if (info.rollout_id && typeof info.rollout_id === 'string') { + return info.rollout_id; + } + + // Fallback to environment variable + return process.env[this.rolloutIdEnv] || null; + } + + private getStatusInfo(info: FireworksLogInfo): StatusInfo | null { + if (!info.status) { + return null; + } + + const status = info.status; + + // Handle Status class instances (with code and message properties) + if (typeof status === 'object' && status !== null && 'code' in status && 'message' in status) { + return { + code: typeof status.code === 'number' ? status.code : undefined, + message: typeof status.message === 'string' ? status.message : undefined, + details: Array.isArray(status.details) ? status.details : [], + }; + } + + return null; + } + + private buildPayload(info: FireworksLogInfo, rolloutId: string): FireworksPayload { + const timestamp = new Date().toISOString(); + // Ensure message is always a string for Fireworks payload + const message: string = typeof info.message === 'string' ? info.message : ''; + const level = (info as any)[LEVEL] || info.level || 'info'; + + const tags: string[] = [`rollout_id:${rolloutId}`]; + + // Optional additional tags + if (info.experiment_id && typeof info.experiment_id === 'string') { + tags.push(`experiment_id:${info.experiment_id}`); + } + if (info.run_id && typeof info.run_id === 'string') { + tags.push(`run_id:${info.run_id}`); + } + + // Groupwise list of rollout_ids + if (Array.isArray(info.rollout_ids)) { + for (const rid of info.rollout_ids) { + if (typeof rid === 'string') { + tags.push(`rollout_id:${rid}`); + } + } + } + + const program = (typeof info.program === 'string' ? info.program : null) || 'eval_protocol'; + + return { + program, + status: this.getStatusInfo(info), + message, + tags, + extras: { + logger_name: info.logger_name || 'winston', + level: level.toUpperCase(), + timestamp, + }, + }; + } +} diff --git a/typescript/logging/fireworks-vercel.ts b/typescript/logging/fireworks-vercel.ts new file mode 100644 index 00000000..6cb635b5 --- /dev/null +++ b/typescript/logging/fireworks-vercel.ts @@ -0,0 +1,21 @@ +/** + * Vercel-specific helper to wire Fireworks logging into serverless handlers. + * + * Usage: + * export default withFireworksLogging(async (req, res) => { ... }) + */ + +import type { VercelRequest, VercelResponse } from '@vercel/node'; +import { waitUntil } from '@vercel/functions'; +import { setWaitUntil } from './logger.js'; + +export type VercelHandler = (req: VercelRequest, res: VercelResponse) => T | Promise; + +export function withFireworksLogging(handler: VercelHandler): VercelHandler { + return async (req, res) => { + // Hook up Vercel waitUntil for this invocation so logging + // can flush to Fireworks even after the HTTP response is sent. + setWaitUntil(waitUntil); + return handler(req, res); + }; +} diff --git a/typescript/logging/logger.ts b/typescript/logging/logger.ts new file mode 100644 index 00000000..3ab226c3 --- /dev/null +++ b/typescript/logging/logger.ts @@ -0,0 +1,44 @@ +/** + * Winston logger configuration with Fireworks tracing transport. + */ + +import winston from 'winston'; +import { FireworksTransport } from './fireworks-transport.js'; + +// Global reference to waitUntil function +let globalWaitUntil: ((promise: Promise) => void) | undefined; + +// Set waitUntil function (called from Vercel handler or host) +export function setWaitUntil(waitUntil: (promise: Promise) => void) { + globalWaitUntil = waitUntil; +} + +export const logger = winston.createLogger({ + level: 'info', + format: winston.format.combine( + winston.format.timestamp(), + winston.format.errors({ stack: true }), + winston.format.json() + ), + transports: [ + new winston.transports.Console({ + format: winston.format.combine( + winston.format.colorize(), + winston.format.simple() + ) + }), + new FireworksTransport({ + waitUntil: (promise: Promise) => globalWaitUntil?.(promise) + }) + ] +}); + +/** + * Create a child logger with rollout_id context. + */ +export function createRolloutLogger(rolloutId: string, name: string = 'init'): winston.Logger { + return logger.child({ + rollout_id: rolloutId, + logger_name: `${name}.${rolloutId}` + }); +} diff --git a/typescript/models/exceptions.ts b/typescript/models/exceptions.ts new file mode 100644 index 00000000..f163ad5a --- /dev/null +++ b/typescript/models/exceptions.ts @@ -0,0 +1,213 @@ +import OpenAI from 'openai'; +import { Status, StatusCode } from './status.js'; + +export class EvalProtocolError extends Error { + statusCode: StatusCode; + + constructor(message: string, statusCode: StatusCode) { + super(message); + this.name = this.constructor.name; + this.statusCode = statusCode; + } +} + +export class CancelledError extends EvalProtocolError { + constructor(message: string = '') { + super(message, StatusCode.CANCELLED); + } +} + +export class UnknownError extends EvalProtocolError { + constructor(message: string = '') { + super(message, StatusCode.UNKNOWN); + } +} + +export class InvalidArgumentError extends EvalProtocolError { + constructor(message: string = '') { + super(message, StatusCode.INVALID_ARGUMENT); + } +} + +export class DeadlineExceededError extends EvalProtocolError { + constructor(message: string = '') { + super(message, StatusCode.DEADLINE_EXCEEDED); + } +} + +export class NotFoundError extends EvalProtocolError { + constructor(message: string = '') { + super(message, StatusCode.NOT_FOUND); + } +} + +export class AlreadyExistsError extends EvalProtocolError { + constructor(message: string = '') { + super(message, StatusCode.ALREADY_EXISTS); + } +} + +export class PermissionDeniedError extends EvalProtocolError { + constructor(message: string = '') { + super(message, StatusCode.PERMISSION_DENIED); + } +} + +export class ResourceExhaustedError extends EvalProtocolError { + constructor(message: string = '') { + super(message, StatusCode.RESOURCE_EXHAUSTED); + } +} + +export class FailedPreconditionError extends EvalProtocolError { + constructor(message: string = '') { + super(message, StatusCode.FAILED_PRECONDITION); + } +} + +export class AbortedError extends EvalProtocolError { + constructor(message: string = '') { + super(message, StatusCode.ABORTED); + } +} + +export class OutOfRangeError extends EvalProtocolError { + constructor(message: string = '') { + super(message, StatusCode.OUT_OF_RANGE); + } +} + +export class UnimplementedError extends EvalProtocolError { + constructor(message: string = '') { + super(message, StatusCode.UNIMPLEMENTED); + } +} + +export class InternalError extends EvalProtocolError { + constructor(message: string = '') { + super(message, StatusCode.INTERNAL); + } +} + +export class UnavailableError extends EvalProtocolError { + constructor(message: string = '') { + super(message, StatusCode.UNAVAILABLE); + } +} + +export class DataLossError extends EvalProtocolError { + constructor(message: string = '') { + super(message, StatusCode.DATA_LOSS); + } +} + +export class UnauthenticatedError extends EvalProtocolError { + constructor(message: string = '') { + super(message, StatusCode.UNAUTHENTICATED); + } +} + +const STATUS_CODE_TO_EXCEPTION = new Map([ + [StatusCode.OK, null], + [StatusCode.CANCELLED, CancelledError], + [StatusCode.UNKNOWN, UnknownError], + [StatusCode.INVALID_ARGUMENT, InvalidArgumentError], + [StatusCode.DEADLINE_EXCEEDED, DeadlineExceededError], + [StatusCode.NOT_FOUND, NotFoundError], + [StatusCode.ALREADY_EXISTS, AlreadyExistsError], + [StatusCode.PERMISSION_DENIED, PermissionDeniedError], + [StatusCode.RESOURCE_EXHAUSTED, ResourceExhaustedError], + [StatusCode.FAILED_PRECONDITION, FailedPreconditionError], + [StatusCode.ABORTED, AbortedError], + [StatusCode.OUT_OF_RANGE, OutOfRangeError], + [StatusCode.UNIMPLEMENTED, UnimplementedError], + [StatusCode.INTERNAL, InternalError], + [StatusCode.UNAVAILABLE, UnavailableError], + [StatusCode.DATA_LOSS, DataLossError], + [StatusCode.UNAUTHENTICATED, UnauthenticatedError], + [StatusCode.FINISHED, null], + [StatusCode.RUNNING, null], + [StatusCode.SCORE_INVALID, null], +]); + +export function exceptionForStatusCode(code: StatusCode, message: string = ''): EvalProtocolError | null { + const exceptionClass = STATUS_CODE_TO_EXCEPTION.get(code); + if (!exceptionClass) { + return null; + } + return new exceptionClass(message, code); +} + +export function mapOpenAIErrorToStatus(error: any): Status { + const errorMessage = error.message || String(error); + + if (error instanceof OpenAI.AuthenticationError) { + return Status.rolloutPermissionDeniedError(errorMessage); + } + + if (error instanceof OpenAI.PermissionDeniedError) { + return Status.rolloutPermissionDeniedError(errorMessage); + } + + if (error instanceof OpenAI.NotFoundError) { + return Status.rolloutNotFoundError(errorMessage); + } + + if (error instanceof OpenAI.RateLimitError) { + return Status.rolloutResourceExhaustedError(errorMessage); + } + + if (error instanceof OpenAI.BadRequestError) { + return Status.rolloutInvalidArgumentError(errorMessage); + } + + if (error instanceof OpenAI.InternalServerError) { + return Status.rolloutInternalError(errorMessage); + } + + if (error instanceof OpenAI.UnprocessableEntityError) { + return Status.rolloutInvalidArgumentError(errorMessage); + } + + if (error instanceof OpenAI.APIConnectionTimeoutError) { + return Status.rolloutDeadlineExceededError(errorMessage); + } + + if (error instanceof OpenAI.ConflictError) { + return Status.rolloutAlreadyExistsError(errorMessage); + } + + if (error.code === 'ECONNRESET' || error.code === 'ENOTFOUND' || error.code === 'ETIMEDOUT') { + return Status.rolloutUnavailableError(errorMessage); + } + + return Status.rolloutInternalError(errorMessage); +} + +export function isRetryableError(error: any): boolean { + if (error instanceof OpenAI.RateLimitError || + error instanceof OpenAI.InternalServerError || + error instanceof OpenAI.APIConnectionTimeoutError || + error instanceof OpenAI.APIConnectionError) { + return true; + } + + if (error.code === 'ECONNRESET' || + error.code === 'ENOTFOUND' || + error.code === 'ETIMEDOUT' || + error.code === 'ECONNREFUSED') { + return true; + } + + if (error instanceof UnknownError || + error instanceof DeadlineExceededError || + error instanceof NotFoundError || + error instanceof PermissionDeniedError || + error instanceof UnavailableError || + error instanceof UnauthenticatedError || + error instanceof ResourceExhaustedError) { + return true; + } + + return false; +} diff --git a/typescript/models/status.ts b/typescript/models/status.ts new file mode 100644 index 00000000..323548ec --- /dev/null +++ b/typescript/models/status.ts @@ -0,0 +1,234 @@ +export enum StatusCode { + // Standard gRPC codes + OK = 0, + CANCELLED = 1, + UNKNOWN = 2, + INVALID_ARGUMENT = 3, + DEADLINE_EXCEEDED = 4, + NOT_FOUND = 5, + ALREADY_EXISTS = 6, + PERMISSION_DENIED = 7, + RESOURCE_EXHAUSTED = 8, + FAILED_PRECONDITION = 9, + ABORTED = 10, + OUT_OF_RANGE = 11, + UNIMPLEMENTED = 12, + INTERNAL = 13, + UNAVAILABLE = 14, + DATA_LOSS = 15, + UNAUTHENTICATED = 16, + + // Custom codes for EP (using higher numbers to avoid conflicts) + FINISHED = 100, + RUNNING = 101, + SCORE_INVALID = 102 +} + +export interface ErrorInfo { + '@type': string; + reason?: string; + domain?: string; + metadata?: Record; +} + +export interface StatusDetails { + code: StatusCode; + message: string; + details: ErrorInfo[]; +} + +export class Status { + code: StatusCode; + message: string; + details: ErrorInfo[]; + + constructor(code: StatusCode, message: string, details: ErrorInfo[] = []) { + this.code = code; + this.message = message; + this.details = details; + } + + private static _buildDetailsWithExtraInfo(extraInfo?: Record): ErrorInfo[] { + if (!extraInfo) { + return []; + } + return [{ + '@type': 'type.googleapis.com/google.rpc.ErrorInfo', + reason: 'EXTRA_INFO', + domain: 'eval-protocol.com', + metadata: extraInfo + }]; + } + + static ok(): Status { + return new Status(StatusCode.OK, 'Success'); + } + + static rolloutRunning(): Status { + return new Status(StatusCode.RUNNING, 'Rollout is running'); + } + + static evalRunning(): Status { + return new Status(StatusCode.RUNNING, 'Evaluation is running'); + } + + static rolloutFinished(): Status { + return new Status(StatusCode.FINISHED, 'Rollout finished'); + } + + static evalFinished(): Status { + return new Status(StatusCode.FINISHED, 'Evaluation finished'); + } + + static rolloutCancelledError(errorMessage: string, extraInfo?: Record): Status { + return Status.cancelledError(errorMessage, Status._buildDetailsWithExtraInfo(extraInfo)); + } + + static cancelledError(errorMessage: string, details: ErrorInfo[] = []): Status { + return new Status(StatusCode.CANCELLED, errorMessage, details); + } + + static rolloutUnknownError(errorMessage: string, extraInfo?: Record): Status { + return Status.unknownError(errorMessage, Status._buildDetailsWithExtraInfo(extraInfo)); + } + + static unknownError(errorMessage: string, details: ErrorInfo[] = []): Status { + return new Status(StatusCode.UNKNOWN, errorMessage, details); + } + + static rolloutInvalidArgumentError(errorMessage: string, extraInfo?: Record): Status { + return Status.invalidArgumentError(errorMessage, Status._buildDetailsWithExtraInfo(extraInfo)); + } + + static invalidArgumentError(errorMessage: string, details: ErrorInfo[] = []): Status { + return new Status(StatusCode.INVALID_ARGUMENT, errorMessage, details); + } + + static rolloutDeadlineExceededError(errorMessage: string, extraInfo?: Record): Status { + return Status.deadlineExceededError(errorMessage, Status._buildDetailsWithExtraInfo(extraInfo)); + } + + static deadlineExceededError(errorMessage: string, details: ErrorInfo[] = []): Status { + return new Status(StatusCode.DEADLINE_EXCEEDED, errorMessage, details); + } + + static rolloutNotFoundError(errorMessage: string, extraInfo?: Record): Status { + return Status.notFoundError(errorMessage, Status._buildDetailsWithExtraInfo(extraInfo)); + } + + static notFoundError(errorMessage: string, details: ErrorInfo[] = []): Status { + return new Status(StatusCode.NOT_FOUND, errorMessage, details); + } + + static rolloutAlreadyExistsError(errorMessage: string, extraInfo?: Record): Status { + return Status.alreadyExistsError(errorMessage, Status._buildDetailsWithExtraInfo(extraInfo)); + } + + static alreadyExistsError(errorMessage: string, details: ErrorInfo[] = []): Status { + return new Status(StatusCode.ALREADY_EXISTS, errorMessage, details); + } + + static rolloutPermissionDeniedError(errorMessage: string, extraInfo?: Record): Status { + return Status.permissionDeniedError(errorMessage, Status._buildDetailsWithExtraInfo(extraInfo)); + } + + static permissionDeniedError(errorMessage: string, details: ErrorInfo[] = []): Status { + return new Status(StatusCode.PERMISSION_DENIED, errorMessage, details); + } + + static rolloutResourceExhaustedError(errorMessage: string, extraInfo?: Record): Status { + return Status.resourceExhaustedError(errorMessage, Status._buildDetailsWithExtraInfo(extraInfo)); + } + + static resourceExhaustedError(errorMessage: string, details: ErrorInfo[] = []): Status { + return new Status(StatusCode.RESOURCE_EXHAUSTED, errorMessage, details); + } + + static rolloutFailedPreconditionError(errorMessage: string, extraInfo?: Record): Status { + return Status.failedPreconditionError(errorMessage, Status._buildDetailsWithExtraInfo(extraInfo)); + } + + static failedPreconditionError(errorMessage: string, details: ErrorInfo[] = []): Status { + return new Status(StatusCode.FAILED_PRECONDITION, errorMessage, details); + } + + static rolloutAbortedError(errorMessage: string, extraInfo?: Record): Status { + return Status.abortedError(errorMessage, Status._buildDetailsWithExtraInfo(extraInfo)); + } + + static abortedError(errorMessage: string, details: ErrorInfo[] = []): Status { + return new Status(StatusCode.ABORTED, errorMessage, details); + } + + static rolloutOutOfRangeError(errorMessage: string, extraInfo?: Record): Status { + return Status.outOfRangeError(errorMessage, Status._buildDetailsWithExtraInfo(extraInfo)); + } + + static outOfRangeError(errorMessage: string, details: ErrorInfo[] = []): Status { + return new Status(StatusCode.OUT_OF_RANGE, errorMessage, details); + } + + static rolloutUnimplementedError(errorMessage: string, extraInfo?: Record): Status { + return Status.unimplementedError(errorMessage, Status._buildDetailsWithExtraInfo(extraInfo)); + } + + static unimplementedError(errorMessage: string, details: ErrorInfo[] = []): Status { + return new Status(StatusCode.UNIMPLEMENTED, errorMessage, details); + } + + static rolloutInternalError(errorMessage: string, extraInfo?: Record): Status { + return Status.internalError(errorMessage, Status._buildDetailsWithExtraInfo(extraInfo)); + } + + static internalError(errorMessage: string, details: ErrorInfo[] = []): Status { + return new Status(StatusCode.INTERNAL, errorMessage, details); + } + + static rolloutError(errorMessage: string, extraInfo?: Record): Status { + return Status.internalError(errorMessage, Status._buildDetailsWithExtraInfo(extraInfo)); + } + + static error(errorMessage: string, details: ErrorInfo[] = []): Status { + return new Status(StatusCode.INTERNAL, errorMessage, details); + } + + static rolloutUnavailableError(errorMessage: string, extraInfo?: Record): Status { + return Status.unavailableError(errorMessage, Status._buildDetailsWithExtraInfo(extraInfo)); + } + + static unavailableError(errorMessage: string, details: ErrorInfo[] = []): Status { + return new Status(StatusCode.UNAVAILABLE, errorMessage, details); + } + + static rolloutDataLossError(errorMessage: string, extraInfo?: Record): Status { + return Status.dataLossError(errorMessage, Status._buildDetailsWithExtraInfo(extraInfo)); + } + + static dataLossError(errorMessage: string, details: ErrorInfo[] = []): Status { + return new Status(StatusCode.DATA_LOSS, errorMessage, details); + } + + static rolloutUnauthenticatedError(errorMessage: string, extraInfo?: Record): Status { + return Status.unauthenticatedError(errorMessage, Status._buildDetailsWithExtraInfo(extraInfo)); + } + + static unauthenticatedError(errorMessage: string, details: ErrorInfo[] = []): Status { + return new Status(StatusCode.UNAUTHENTICATED, errorMessage, details); + } + + static scoreInvalid(message: string = 'Score is invalid', details: ErrorInfo[] = []): Status { + return new Status(StatusCode.SCORE_INVALID, message, details); + } + + toJSON(): StatusDetails { + return { + code: this.code, + message: this.message, + details: this.details + }; + } + + static fromJSON(json: StatusDetails): Status { + return new Status(json.code, json.message, json.details); + } +} diff --git a/typescript/models/types.ts b/typescript/models/types.ts new file mode 100644 index 00000000..4cf5f5e8 --- /dev/null +++ b/typescript/models/types.ts @@ -0,0 +1,67 @@ +import z from "zod"; + +// Shared protocol schemas/types for Eval Protocol TypeScript support. + +export const roleSchema = z.enum(["system", "user", "assistant"]); + +export const messageSchema = z.union([ + z.object({ + role: roleSchema, + content: z.string(), + }), + z.object({ + role: z.literal("tool"), + content: z.string(), + tool_call_id: z.string(), + }), +]); + +export const functionDefinitionSchema = z + .object({ + name: z.string().regex(/^[a-zA-Z0-9_-]{1,64}$/), + description: z.string().optional(), + // JSON Schema object; allow arbitrary keys + parameters: z.object({}).catchall(z.any()).optional(), + }) + .catchall(z.any()); + +export const toolSchema = z.object({ + type: z.literal("function"), + function: functionDefinitionSchema, +}); + +export const metadataSchema = z + .object({ + invocation_id: z.string(), + experiment_id: z.string(), + rollout_id: z.string(), + run_id: z.string(), + row_id: z.string(), + }) + .catchall(z.any()); + +export const initRequestSchema = z.object({ + completion_params: z + .record(z.string(), z.any()) + .describe("Completion parameters including model and optional model_kwargs, temperature, etc."), + messages: z.array(messageSchema).optional(), + tools: z.array(toolSchema).optional().nullable(), + metadata: metadataSchema, + model_base_url: z.string().optional().nullable(), + api_key: z.string().optional().nullable(), +}); + +export const statusInfoSchema = z.record(z.string(), z.any()); + +export const statusResponseSchema = z.object({ + terminated: z.boolean(), + info: statusInfoSchema.optional(), +}); + +export type Message = z.infer; +export type FunctionDefinition = z.infer; +export type Tool = z.infer; +export type Metadata = z.infer; +export type InitRequest = z.infer; +export type StatusInfo = z.infer; +export type StatusResponse = z.infer; diff --git a/typescript/package.json b/typescript/package.json index 8fed21df..b94ada50 100644 --- a/typescript/package.json +++ b/typescript/package.json @@ -1,16 +1,29 @@ { "name": "eval-protocol", - "module": "index.ts", "type": "module", - "version": "0.1.4", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "exports": { + ".": { + "import": "./dist/index.js", + "types": "./dist/index.d.ts" + } + }, + "files": ["dist"], + "version": "0.1.7", "devDependencies": { - "@types/bun": "latest", - "openai": "^5.23.0" + "@types/bun": "latest" }, "peerDependencies": { "typescript": "^5" }, "dependencies": { - "zod": "^4.1.11" + "@vercel/functions": "^1.4.0", + "@vercel/node": "^3.0.21", + "openai": "^4.104.0", + "winston": "^3.11.0", + "winston-transport": "^4.6.0", + "logform": "^2.6.0", + "zod": "^3.22.4" } } diff --git a/typescript/tsconfig.json b/typescript/tsconfig.json index bfa0fead..cdc585b1 100644 --- a/typescript/tsconfig.json +++ b/typescript/tsconfig.json @@ -1,7 +1,7 @@ { "compilerOptions": { // Environment setup & latest features - "lib": ["ESNext"], + "lib": ["ESNext", "DOM"], "target": "ESNext", "module": "Preserve", "moduleDetection": "force", @@ -10,16 +10,17 @@ // Bundler mode "moduleResolution": "bundler", - "allowImportingTsExtensions": true, "verbatimModuleSyntax": true, - "noEmit": true, + "noEmit": false, + "declaration": true, + "emitDeclarationOnly": false, + "outDir": "dist", // Best practices "strict": true, "skipLibCheck": true, "noFallthroughCasesInSwitch": true, "noUncheckedIndexedAccess": true, - "noImplicitOverride": true, // Some stricter flags (disabled by default) "noUnusedLocals": false, From 6cdc16d8481704d22e6a6c34151df63cd9df3de2 Mon Sep 17 00:00:00 2001 From: Derek Xu Date: Mon, 17 Nov 2025 00:57:54 -0800 Subject: [PATCH 2/5] address comment --- .../vercel_svg_server_ts/src/models/exceptions.ts | 6 ++++-- typescript/models/exceptions.ts | 8 +++++--- typescript/package.json | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/eval_protocol/quickstart/svg_agent/vercel_svg_server_ts/src/models/exceptions.ts b/eval_protocol/quickstart/svg_agent/vercel_svg_server_ts/src/models/exceptions.ts index f5f5f8c8..6b310def 100644 --- a/eval_protocol/quickstart/svg_agent/vercel_svg_server_ts/src/models/exceptions.ts +++ b/eval_protocol/quickstart/svg_agent/vercel_svg_server_ts/src/models/exceptions.ts @@ -116,8 +116,10 @@ export class UnauthenticatedError extends EvalProtocolError { } } +type EvalProtocolErrorConstructor = new (message?: string) => EvalProtocolError; + // Mapping from status codes to exception classes -const STATUS_CODE_TO_EXCEPTION = new Map([ +const STATUS_CODE_TO_EXCEPTION = new Map([ [StatusCode.OK, null], [StatusCode.CANCELLED, CancelledError], [StatusCode.UNKNOWN, UnknownError], @@ -148,7 +150,7 @@ export function exceptionForStatusCode(code: StatusCode, message: string = ''): if (!exceptionClass) { return null; } - return new exceptionClass(message, code); + return new exceptionClass(message); } /** diff --git a/typescript/models/exceptions.ts b/typescript/models/exceptions.ts index f163ad5a..4d6e3b1b 100644 --- a/typescript/models/exceptions.ts +++ b/typescript/models/exceptions.ts @@ -103,11 +103,13 @@ export class DataLossError extends EvalProtocolError { export class UnauthenticatedError extends EvalProtocolError { constructor(message: string = '') { - super(message, StatusCode.UNAUTHENTICATED); + super(message, StatusCode.UNAUTHENTICATED); } } -const STATUS_CODE_TO_EXCEPTION = new Map([ +type EvalProtocolErrorConstructor = new (message?: string) => EvalProtocolError; + +const STATUS_CODE_TO_EXCEPTION = new Map([ [StatusCode.OK, null], [StatusCode.CANCELLED, CancelledError], [StatusCode.UNKNOWN, UnknownError], @@ -135,7 +137,7 @@ export function exceptionForStatusCode(code: StatusCode, message: string = ''): if (!exceptionClass) { return null; } - return new exceptionClass(message, code); + return new exceptionClass(message); } export function mapOpenAIErrorToStatus(error: any): Status { diff --git a/typescript/package.json b/typescript/package.json index b94ada50..875a9d64 100644 --- a/typescript/package.json +++ b/typescript/package.json @@ -10,7 +10,7 @@ } }, "files": ["dist"], - "version": "0.1.7", + "version": "0.1.8", "devDependencies": { "@types/bun": "latest" }, From 2075e622b1b0a70136d6767350d863a570c55926 Mon Sep 17 00:00:00 2001 From: Derek Xu Date: Mon, 17 Nov 2025 01:21:15 -0800 Subject: [PATCH 3/5] add code to surface errors --- eval_protocol/cli_commands/upload.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/eval_protocol/cli_commands/upload.py b/eval_protocol/cli_commands/upload.py index 8c6e7baf..23b68596 100644 --- a/eval_protocol/cli_commands/upload.py +++ b/eval_protocol/cli_commands/upload.py @@ -186,14 +186,30 @@ def pytest_collection_modifyitems(self, items): ] try: - # Suppress pytest output + # Run pytest collection with output captured so we can surface errors in CI import io import contextlib - with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): - pytest.main(args, plugins=[plugin]) - except Exception: - # If pytest collection fails, fall back to empty list + stdout = io.StringIO() + stderr = io.StringIO() + with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + exit_code = pytest.main(args, plugins=[plugin]) + + if exit_code != 0: + # In CI this helps diagnose why discovery returned an empty list + sys.stderr.write(f"[ep upload] pytest collection failed with exit code {exit_code} for root {abs_root}\n") + out = stdout.getvalue() + err = stderr.getvalue() + if out: + sys.stderr.write("[ep upload] pytest collection stdout:\n") + sys.stderr.write(out + "\n") + if err: + sys.stderr.write("[ep upload] pytest collection stderr:\n") + sys.stderr.write(err + "\n") + return [] + except Exception as e: + # Log the exception instead of silently swallowing it + sys.stderr.write(f"[ep upload] pytest collection raised an exception for root {abs_root}: {e!r}\n") return [] # Process collected items From b9ff0d81e3351df01ced3828255ee36a11d0d29d Mon Sep 17 00:00:00 2001 From: Derek Xu Date: Mon, 17 Nov 2025 01:39:16 -0800 Subject: [PATCH 4/5] update test and revert --- eval_protocol/cli_commands/upload.py | 26 +++++--------------------- tests/test_ep_upload_e2e.py | 5 +++-- 2 files changed, 8 insertions(+), 23 deletions(-) diff --git a/eval_protocol/cli_commands/upload.py b/eval_protocol/cli_commands/upload.py index 23b68596..8c6e7baf 100644 --- a/eval_protocol/cli_commands/upload.py +++ b/eval_protocol/cli_commands/upload.py @@ -186,30 +186,14 @@ def pytest_collection_modifyitems(self, items): ] try: - # Run pytest collection with output captured so we can surface errors in CI + # Suppress pytest output import io import contextlib - stdout = io.StringIO() - stderr = io.StringIO() - with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): - exit_code = pytest.main(args, plugins=[plugin]) - - if exit_code != 0: - # In CI this helps diagnose why discovery returned an empty list - sys.stderr.write(f"[ep upload] pytest collection failed with exit code {exit_code} for root {abs_root}\n") - out = stdout.getvalue() - err = stderr.getvalue() - if out: - sys.stderr.write("[ep upload] pytest collection stdout:\n") - sys.stderr.write(out + "\n") - if err: - sys.stderr.write("[ep upload] pytest collection stderr:\n") - sys.stderr.write(err + "\n") - return [] - except Exception as e: - # Log the exception instead of silently swallowing it - sys.stderr.write(f"[ep upload] pytest collection raised an exception for root {abs_root}: {e!r}\n") + with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + pytest.main(args, plugins=[plugin]) + except Exception: + # If pytest collection fails, fall back to empty list return [] # Process collected items diff --git a/tests/test_ep_upload_e2e.py b/tests/test_ep_upload_e2e.py index f7986e5d..a1eba1d4 100644 --- a/tests/test_ep_upload_e2e.py +++ b/tests/test_ep_upload_e2e.py @@ -410,7 +410,7 @@ async def test_quickstart_eval(row: EvaluationRow) -> EvaluationRow: test_project_dir, test_file_path = create_test_project_with_evaluation_test( test_content, - "quickstart.py", # Non test_* filename + "ep_upload_non_test_prefixed_eval.py", # Non test_* filename ) original_cwd = os.getcwd() @@ -423,7 +423,8 @@ async def test_quickstart_eval(row: EvaluationRow) -> EvaluationRow: assert len(discovered_tests) == 1 assert "test_quickstart_eval" in discovered_tests[0].qualname - assert "quickstart.py" in discovered_tests[0].file_path + # Verify we discovered a non-test-prefixed file (our unique filename) + assert "ep_upload_non_test_prefixed_eval.py" in discovered_tests[0].file_path finally: os.chdir(original_cwd) From f58ef5c34a4f2aeeff9764581cafc525cd6cc20c Mon Sep 17 00:00:00 2001 From: Derek Xu Date: Mon, 17 Nov 2025 12:08:53 -0800 Subject: [PATCH 5/5] fixing the dep --- typescript/package.json | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/typescript/package.json b/typescript/package.json index 875a9d64..90a62edb 100644 --- a/typescript/package.json +++ b/typescript/package.json @@ -10,20 +10,23 @@ } }, "files": ["dist"], - "version": "0.1.8", - "devDependencies": { - "@types/bun": "latest" - }, + "version": "0.1.10", "peerDependencies": { - "typescript": "^5" - }, - "dependencies": { + "typescript": "^5", "@vercel/functions": "^1.4.0", "@vercel/node": "^3.0.21", - "openai": "^4.104.0", + "openai": "^4.104.0" + }, + "dependencies": { "winston": "^3.11.0", "winston-transport": "^4.6.0", "logform": "^2.6.0", "zod": "^3.22.4" + }, + "devDependencies": { + "@types/bun": "latest", + "@vercel/functions": "^1.4.0", + "@vercel/node": "^3.0.21", + "openai": "^4.104.0" } }