Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
90 changes: 85 additions & 5 deletions packages/core/script/generate-venice.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,42 @@ import { ModelFamilyValues } from "../src/family.js";
const API_ENDPOINT = "https://api.venice.ai/api/v1/models?type=text";

// Zod schemas for API response validation
const ReasoningEffort = z.enum(["none", "minimal", "low", "medium", "high", "xhigh", "max"]);
type ReasoningOption = {
type: "effort";
values: Array<z.infer<typeof ReasoningEffort>>;
};

const effort = (...values: ReasoningOption["values"]): ReasoningOption[] => [{ type: "effort", values }];

// Venice documents these model-specific values even where /models is stale or incomplete.
// Source: https://docs.venice.ai/guides/features/reasoning-models
export const REASONING_OVERRIDES: Record<string, ReasoningOption[]> = {
"claude-opus-4-5": effort("low", "medium", "high"),
"claude-opus-4-6": effort("low", "medium", "high", "max"),
"claude-opus-4-6-fast": effort("low", "medium", "high", "max"),
"claude-sonnet-4-5": effort("low", "medium", "high"),
"claude-sonnet-4-6": effort("low", "medium", "high"),
"gemini-3-flash-preview": effort("minimal", "low", "medium", "high"),
"kimi-k2-5": effort("low", "medium", "high"),
"openai-gpt-52": effort("none", "low", "medium", "high", "xhigh"),
"openai-gpt-52-codex": effort("low", "medium", "high", "xhigh"),
"openai-gpt-53-codex": effort("low", "medium", "high", "xhigh"),
"qwen3-5-35b-a3b": effort("low", "medium", "high"),
"zai-org-glm-5-1": [],

// Provisional until funded Venice probes can confirm that its proxy preserves
// OpenAI's current controls. Sources:
// https://developers.openai.com/api/docs/models/gpt-5.4
// https://developers.openai.com/api/docs/models/gpt-5.4-pro
// https://developers.openai.com/api/docs/guides/reasoning
"openai-gpt-54": effort("none", "low", "medium", "high", "xhigh"),
"openai-gpt-54-mini": effort("none", "low", "medium", "high", "xhigh"),
"openai-gpt-54-pro": effort("medium", "high", "xhigh"),
"openai-gpt-55": effort("none", "low", "medium", "high", "xhigh"),
"openai-gpt-55-pro": effort("medium", "high", "xhigh"),
};

const Capabilities = z
.object({
optimizedForCode: z.boolean().optional(),
Expand All @@ -17,6 +53,9 @@ const Capabilities = z
supportsFunctionCalling: z.boolean().optional(),
supportsLogProbs: z.boolean().optional(),
supportsReasoning: z.boolean().optional(),
supportsReasoningEffort: z.boolean().optional(),
reasoningEffortOptions: z.array(ReasoningEffort).optional(),
defaultReasoningEffort: ReasoningEffort.optional(),
supportsResponseSchema: z.boolean().optional(),
supportsVideoInput: z.boolean().optional(),
supportsVision: z.boolean().optional(),
Expand Down Expand Up @@ -142,6 +181,7 @@ interface ExistingModel {
family?: string;
attachment?: boolean;
reasoning?: boolean;
reasoning_options?: ReasoningOption[];
tool_call?: boolean;
structured_output?: boolean;
temperature?: boolean;
Expand Down Expand Up @@ -235,6 +275,7 @@ interface MergedModel {
family?: string;
attachment: boolean;
reasoning: boolean;
reasoning_options?: ReasoningOption[];
tool_call: boolean;
structured_output?: boolean;
temperature: boolean;
Expand Down Expand Up @@ -267,9 +308,10 @@ interface MergedModel {
};
}

function mergeModel(
export function mergeModel(
apiModel: z.infer<typeof VeniceModel>,
existing: ExistingModel | null,
reportDiscrepancy: (message: string) => void = console.warn,
): MergedModel {
const spec = apiModel.model_spec;
const caps = spec.capabilities;
Expand Down Expand Up @@ -309,6 +351,31 @@ function mergeModel(
},
};

const override = REASONING_OVERRIDES[apiModel.id];
const catalogOptions = caps.supportsReasoningEffort === true && caps.reasoningEffortOptions !== undefined
? effort(...caps.reasoningEffortOptions)
: undefined;
const curatedOptions = existing?.reasoning_options;
const selectedOptions = override ?? curatedOptions ?? (existing === null ? catalogOptions : undefined);

if (selectedOptions !== undefined) {
merged.reasoning_options = selectedOptions;
}

const catalogClaim = catalogOptions ?? (caps.supportsReasoningEffort === false ? [] : undefined);
if (override !== undefined && JSON.stringify(override) !== JSON.stringify(curatedOptions) && curatedOptions !== undefined) {
reportDiscrepancy(`${apiModel.id}: documented override replaces curated reasoning_options`);
}
if (
selectedOptions !== undefined &&
catalogClaim !== undefined &&
JSON.stringify(selectedOptions) !== JSON.stringify(catalogClaim)
) {
reportDiscrepancy(
`${apiModel.id}: preserving ${override !== undefined ? "documented override" : "curated reasoning_options"} despite catalog ${caps.supportsReasoningEffort === false ? "supportsReasoningEffort=false" : "option mismatch"}`,
);
}

// structured_output only if true
if (caps.supportsResponseSchema === true) {
merged.structured_output = true;
Expand Down Expand Up @@ -352,7 +419,7 @@ function mergeModel(
return merged;
}

function formatToml(model: MergedModel): string {
export function formatToml(model: MergedModel): string {
const lines: string[] = [];

// Basic fields
Expand All @@ -362,6 +429,9 @@ function formatToml(model: MergedModel): string {
}
lines.push(`attachment = ${model.attachment}`);
lines.push(`reasoning = ${model.reasoning}`);
if (model.reasoning_options?.length === 0) {
lines.push("reasoning_options = []");
}
lines.push(`tool_call = ${model.tool_call}`);
if (model.structured_output !== undefined) {
lines.push(`structured_output = ${model.structured_output}`);
Expand All @@ -377,6 +447,13 @@ function formatToml(model: MergedModel): string {
lines.push(`status = "${model.status}"`);
}

for (const option of model.reasoning_options ?? []) {
lines.push("");
lines.push("[[reasoning_options]]");
lines.push(`type = "${option.type}"`);
lines.push(`values = [${option.values.map((value) => `"${value}"`).join(", ")}]`);
}

// Interleaved section (if present)
if (model.interleaved !== undefined) {
lines.push("");
Expand Down Expand Up @@ -437,7 +514,7 @@ interface Changes {
newValue: string;
}

function detectChanges(
export function detectChanges(
existing: ExistingModel | null,
merged: MergedModel,
): Changes[] {
Expand All @@ -459,7 +536,7 @@ function detectChanges(

const formatValue = (val: unknown): string => {
if (typeof val === "number") return formatNumber(val);
if (Array.isArray(val)) return `[${val.join(", ")}]`;
if (Array.isArray(val)) return JSON.stringify(val);
if (val === undefined) return "(none)";
return String(val);
};
Expand All @@ -468,6 +545,7 @@ function detectChanges(
compare("family", existing.family, merged.family);
compare("attachment", existing.attachment, merged.attachment);
compare("reasoning", existing.reasoning, merged.reasoning);
compare("reasoning_options", existing.reasoning_options, merged.reasoning_options);
compare("tool_call", existing.tool_call, merged.tool_call);
compare("structured_output", existing.structured_output, merged.structured_output);
compare("open_weights", existing.open_weights, merged.open_weights);
Expand Down Expand Up @@ -650,4 +728,6 @@ async function main() {
}
}

await main();
if (import.meta.main) {
await main();
}
121 changes: 121 additions & 0 deletions packages/core/test/venice-generator.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
import { expect, test } from "bun:test";

import {
detectChanges,
formatToml,
mergeModel,
REASONING_OVERRIDES,
} from "../script/generate-venice.js";

type Effort = "none" | "minimal" | "low" | "medium" | "high" | "xhigh" | "max";
const options = (...values: Effort[]) => [{ type: "effort" as const, values }];

function model(id: string, capabilities: Record<string, unknown>) {
return {
created: 1_700_000_000,
id,
model_spec: {
availableContextTokens: 128_000,
maxCompletionTokens: 32_000,
capabilities: { supportsReasoning: true, ...capabilities },
name: "Test Model",
},
object: "model",
owned_by: "venice.ai",
type: "text",
};
}

test("curated options survive false and stale Venice catalog metadata", () => {
const claude = options("low", "medium", "high", "max");
const codex = options("low", "medium", "high", "xhigh");
const discrepancies: string[] = [];

expect(mergeModel(model("claude-opus-4-7", {
supportsReasoningEffort: false,
}), { reasoning_options: claude }, discrepancies.push.bind(discrepancies)).reasoning_options).toEqual(claude);
expect(mergeModel(model("openai-gpt-56-codex", {
supportsReasoningEffort: true,
reasoningEffortOptions: ["none", "low"],
}), { reasoning_options: codex }, discrepancies.push.bind(discrepancies)).reasoning_options).toEqual(codex);
expect(discrepancies).toHaveLength(2);
});

test("documented override beats stale curated and catalog options", () => {
const discrepancies: string[] = [];
const merged = mergeModel(model("openai-gpt-52", {
supportsReasoningEffort: true,
reasoningEffortOptions: ["minimal", "low", "high"],
}), { reasoning_options: options("low", "high") }, discrepancies.push.bind(discrepancies));

expect(merged.reasoning_options).toEqual(options("none", "low", "medium", "high", "xhigh"));
expect(discrepancies).toHaveLength(2);
});

test("catalog fills only an uncurated new model", () => {
const merged = mergeModel(model("new-reasoner", {
supportsReasoningEffort: true,
reasoningEffortOptions: ["low", "high"],
}), null);

expect(merged.reasoning_options).toEqual(options("low", "high"));
});

test("catalog does not fill an unresolved existing model", () => {
const existing = { reasoning: true };
const merged = mergeModel(model("existing-unresolved-reasoner", {
supportsReasoningEffort: true,
reasoningEffortOptions: ["low", "high"],
}), existing);

expect(merged.reasoning_options).toBeUndefined();
expect(detectChanges(existing, merged).find((change) => change.field === "reasoning_options")).toBeUndefined();
});

test("catalog false without curated evidence leaves options undefined", () => {
const merged = mergeModel(model("unknown-fixed-reasoner", {
supportsReasoningEffort: false,
}), null);

expect(merged.reasoning_options).toBeUndefined();
expect(formatToml(merged)).not.toContain("reasoning_options");
});

test("explicit curated empty options remain stable", () => {
const existing = { reasoning_options: [] };
const merged = mergeModel(model("curated-fixed-reasoner", {
supportsReasoningEffort: false,
}), existing);

expect(merged.reasoning_options).toEqual([]);
expect(detectChanges({ ...existing, reasoning: true }, merged).find((change) => change.field === "reasoning_options")).toBeUndefined();
expect(formatToml(merged)).toContain("reasoning = true\nreasoning_options = []");
});

test("formatter emits nonempty options using model TOML convention", () => {
const merged = mergeModel(model("new-reasoner", {
supportsReasoningEffort: true,
reasoningEffortOptions: ["none", "high"],
}), null);

expect(formatToml(merged)).toContain(
'[[reasoning_options]]\ntype = "effort"\nvalues = ["none", "high"]',
);
});

test("official correction fixtures remain exact", () => {
const expected = {
"claude-opus-4-6": options("low", "medium", "high", "max"),
"openai-gpt-52": options("none", "low", "medium", "high", "xhigh"),
"openai-gpt-52-codex": options("low", "medium", "high", "xhigh"),
"openai-gpt-54-pro": options("medium", "high", "xhigh"),
"gemini-3-flash-preview": options("minimal", "low", "medium", "high"),
"kimi-k2-5": options("low", "medium", "high"),
"qwen3-5-35b-a3b": options("low", "medium", "high"),
"zai-org-glm-5-1": [],
};

for (const [id, reasoningOptions] of Object.entries(expected)) {
expect(REASONING_OVERRIDES[id]).toEqual(reasoningOptions);
}
});
1 change: 1 addition & 0 deletions providers/venice/models/aion-labs-aion-2-0.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ name = "Aion 2.0"
family = "o"
attachment = false
reasoning = true
reasoning_options = [{ type = "effort", values = ["low", "medium", "high"] }]
tool_call = false
temperature = true
release_date = "2026-03-24"
Expand Down
1 change: 1 addition & 0 deletions providers/venice/models/arcee-trinity-large-thinking.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ name = "Trinity Large Thinking"
family = "trinity"
attachment = false
reasoning = true
reasoning_options = [{ type = "effort", values = ["low", "medium", "high"] }]
tool_call = true
structured_output = true
temperature = true
Expand Down
1 change: 1 addition & 0 deletions providers/venice/models/claude-opus-4-5.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ name = "Claude Opus 4.5"
family = "claude-opus"
attachment = true
reasoning = true
reasoning_options = [{ type = "effort", values = ["low", "medium", "high"] }]
tool_call = true
structured_output = true
temperature = true
Expand Down
1 change: 1 addition & 0 deletions providers/venice/models/claude-opus-4-6-fast.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ name = "Claude Opus 4.6 Fast"
family = "claude-opus"
attachment = true
reasoning = true
reasoning_options = [{ type = "effort", values = ["low", "medium", "high", "max"] }]
tool_call = true
structured_output = true
temperature = true
Expand Down
1 change: 1 addition & 0 deletions providers/venice/models/claude-opus-4-6.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ name = "Claude Opus 4.6"
family = "claude-opus"
attachment = true
reasoning = true
reasoning_options = [{ type = "effort", values = ["low", "medium", "high", "max"] }]
tool_call = true
structured_output = true
temperature = true
Expand Down
1 change: 1 addition & 0 deletions providers/venice/models/claude-sonnet-4-5.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ name = "Claude Sonnet 4.5"
family = "claude-sonnet"
attachment = true
reasoning = true
reasoning_options = [{ type = "effort", values = ["low", "medium", "high"] }]
tool_call = true
structured_output = true
temperature = true
Expand Down
1 change: 1 addition & 0 deletions providers/venice/models/claude-sonnet-4-6.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ name = "Claude Sonnet 4.6"
family = "claude-sonnet"
attachment = true
reasoning = true
reasoning_options = [{ type = "effort", values = ["low", "medium", "high"] }]
tool_call = true
structured_output = true
temperature = true
Expand Down
1 change: 1 addition & 0 deletions providers/venice/models/deepseek-v3.2.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ name = "DeepSeek V3.2"
family = "deepseek"
attachment = false
reasoning = true
reasoning_options = [{ type = "effort", values = ["none", "low", "medium", "high"] }]
tool_call = true
structured_output = true
temperature = true
Expand Down
1 change: 1 addition & 0 deletions providers/venice/models/gemini-3-1-pro-preview.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ name = "Gemini 3.1 Pro Preview"
family = "gemini-pro"
attachment = true
reasoning = true
reasoning_options = [{ type = "effort", values = ["low", "medium", "high"] }]
tool_call = true
structured_output = true
temperature = true
Expand Down
1 change: 1 addition & 0 deletions providers/venice/models/gemini-3-5-flash.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ name = "Gemini 3.5 Flash"
family = "gemini-flash"
attachment = true
reasoning = true
reasoning_options = [{ type = "effort", values = ["low", "medium", "high"] }]
tool_call = true
structured_output = true
temperature = true
Expand Down
1 change: 1 addition & 0 deletions providers/venice/models/gemini-3-flash-preview.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ name = "Gemini 3 Flash Preview"
family = "gemini-flash"
attachment = true
reasoning = true
reasoning_options = [{ type = "effort", values = ["minimal", "low", "medium", "high"] }]
tool_call = true
structured_output = true
temperature = true
Expand Down
1 change: 1 addition & 0 deletions providers/venice/models/google-gemma-4-26b-a4b-it.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ name = "Google Gemma 4 26B A4B Instruct"
family = "gemma"
attachment = true
reasoning = true
reasoning_options = [{ type = "effort", values = ["none", "low", "medium", "high"] }]
tool_call = true
structured_output = true
temperature = true
Expand Down
Loading
Loading