diff --git a/packages/agent/src/agent.ts b/packages/agent/src/agent.ts index e39b29f08..6a611aeb0 100644 --- a/packages/agent/src/agent.ts +++ b/packages/agent/src/agent.ts @@ -17,7 +17,6 @@ import type { import type { DataSource, DataSourceFactory } from '@forestadmin/datasource-toolkit'; import type { ForestSchema } from '@forestadmin/forestadmin-client'; -import { isModelSupportingTools } from '@forestadmin/ai-proxy'; import { DataSourceCustomizer } from '@forestadmin/datasource-customizer'; import bodyParser from '@koa/bodyparser'; import cors from '@koa/cors'; @@ -246,13 +245,6 @@ export default class Agent extends FrameworkMounter ); } - if (!isModelSupportingTools(configuration.model)) { - throw new Error( - `Model '${configuration.model}' does not support function calling (tools). ` + - 'Please use a compatible model like gpt-4o, gpt-4o-mini, or gpt-4-turbo.', - ); - } - this.options.logger( 'Warn', `AI configuration added with model '${configuration.model}'. ` + diff --git a/packages/agent/test/agent.test.ts b/packages/agent/test/agent.test.ts index dadc9d566..5874f2686 100644 --- a/packages/agent/test/agent.test.ts +++ b/packages/agent/test/agent.test.ts @@ -439,19 +439,22 @@ describe('Agent', () => { ).toThrow('addAi can only be called once. Multiple AI configurations are not supported yet.'); }); - test('should throw an error when model does not support tools', () => { + test('should throw an error on start when model does not support tools', async () => { + // Use the real makeRoutes to trigger validation in AiProxyRouter + const realMakeRoutes = jest.requireActual('../src/routes').default; + mockMakeRoutes.mockImplementation(realMakeRoutes); + const agent = new Agent(options); - expect(() => - agent.addAi({ - name: 'gpt4-base', - provider: 'openai', - apiKey: 'test-key', - model: 'gpt-4', - }), - ).toThrow( - "Model 'gpt-4' does not support function calling (tools). " + - 'Please use a compatible model like gpt-4o, gpt-4o-mini, or gpt-4-turbo.', + agent.addAi({ + name: 'gpt4-base', + provider: 'openai', + apiKey: 'test-key', + model: 'gpt-4', + }); + + await expect(agent.start()).rejects.toThrow( + "Model 'gpt-4' does not support tools. Please use a model that supports function calling.", ); }); diff --git a/packages/ai-proxy/package.json b/packages/ai-proxy/package.json index 6bf0c085d..e7e32f84f 100644 --- a/packages/ai-proxy/package.json +++ b/packages/ai-proxy/package.json @@ -17,7 +17,7 @@ "@langchain/core": "1.1.15", "@langchain/langgraph": "^1.1.0", "@langchain/mcp-adapters": "1.1.1", - "@langchain/openai": "1.2.2", + "@langchain/openai": "1.2.5", "zod": "^4.3.5" }, "devDependencies": { diff --git a/packages/ai-proxy/src/errors.ts b/packages/ai-proxy/src/errors.ts index 2e2c15609..fc1e5e250 100644 --- a/packages/ai-proxy/src/errors.ts +++ b/packages/ai-proxy/src/errors.ts @@ -32,6 +32,15 @@ export class AIBadRequestError extends AIError { } } +export class AIModelNotSupportedError extends AIBadRequestError { + constructor(model: string) { + super( + `Model '${model}' does not support tools. Please use a model that supports function calling.`, + ); + this.name = 'AIModelNotSupportedError'; + } +} + export class AINotFoundError extends AIError { constructor(message: string) { super(message, 404); diff --git a/packages/ai-proxy/src/index.ts b/packages/ai-proxy/src/index.ts index 75948196d..bee805b44 100644 --- a/packages/ai-proxy/src/index.ts +++ b/packages/ai-proxy/src/index.ts @@ -7,7 +7,6 @@ export * from './remote-tools'; export * from './router'; export * from './mcp-client'; export * from './oauth-token-injector'; - export * from './errors'; export function validMcpConfigurationOrThrow(mcpConfig: McpConfiguration) { diff --git a/packages/ai-proxy/src/provider-dispatcher.ts b/packages/ai-proxy/src/provider-dispatcher.ts index b83eb80b3..ac324d01f 100644 --- a/packages/ai-proxy/src/provider-dispatcher.ts +++ b/packages/ai-proxy/src/provider-dispatcher.ts @@ -21,41 +21,6 @@ export type { } from './provider'; export type { DispatchBody } from './schemas/route'; -/** - * OpenAI model prefixes that do NOT support function calling (tools). - * Unknown models are allowed. - * @see https://platform.openai.com/docs/guides/function-calling - */ -const OPENAI_MODELS_WITHOUT_TOOLS_SUPPORT = [ - 'gpt-4', - 'gpt-3.5-turbo', - 'gpt-3.5', - 'text-davinci', - 'davinci', - 'curie', - 'babbage', - 'ada', -]; - -/** - * Exceptions to the unsupported list - these models DO support tools - * even though they start with an unsupported prefix. - */ -const OPENAI_MODELS_EXCEPTIONS = ['gpt-4-turbo', 'gpt-4o', 'gpt-4.1']; - -export function isModelSupportingTools(model: string): boolean { - const isException = OPENAI_MODELS_EXCEPTIONS.some( - exception => model === exception || model.startsWith(`${exception}-`), - ); - if (isException) return true; - - const isKnownUnsupported = OPENAI_MODELS_WITHOUT_TOOLS_SUPPORT.some( - unsupported => model === unsupported || model.startsWith(`${unsupported}-`), - ); - - return !isKnownUnsupported; -} - export class ProviderDispatcher { private readonly chatModel: ChatOpenAI | null = null; diff --git a/packages/ai-proxy/src/router.ts b/packages/ai-proxy/src/router.ts index 63115a0da..5812df10b 100644 --- a/packages/ai-proxy/src/router.ts +++ b/packages/ai-proxy/src/router.ts @@ -5,10 +5,12 @@ import type { RouteArgs } from './schemas/route'; import type { Logger } from '@forestadmin/datasource-toolkit'; import type { z } from 'zod'; -import { AIBadRequestError, ProviderDispatcher } from './index'; +import { AIBadRequestError, AIModelNotSupportedError } from './errors'; import McpClient from './mcp-client'; +import { ProviderDispatcher } from './provider-dispatcher'; import { RemoteTools } from './remote-tools'; import { routeArgsSchema } from './schemas/route'; +import isModelSupportingTools from './supported-models'; export type { AiQueryArgs, @@ -37,6 +39,16 @@ export class Router { this.aiConfigurations = params?.aiConfigurations ?? []; this.localToolsApiKeys = params?.localToolsApiKeys; this.logger = params?.logger; + + this.validateConfigurations(); + } + + private validateConfigurations(): void { + for (const config of this.aiConfigurations) { + if (!isModelSupportingTools(config.model)) { + throw new AIModelNotSupportedError(config.model); + } + } } /** diff --git a/packages/ai-proxy/src/supported-models.ts b/packages/ai-proxy/src/supported-models.ts new file mode 100644 index 000000000..37647b94e --- /dev/null +++ b/packages/ai-proxy/src/supported-models.ts @@ -0,0 +1,88 @@ +/** + * OpenAI model prefixes that do NOT support tool calls via the chat completions API. + * + * Uses prefix matching: model === prefix OR model.startsWith(prefix + '-') + * + * Unknown models are allowed by default. + * If a model fails the integration test, add it here. + * + * @see https://platform.openai.com/docs/guides/function-calling + */ +const UNSUPPORTED_MODEL_PREFIXES = [ + // Legacy models + 'gpt-4', // Base gpt-4 doesn't honor tool_choice: required + 'text-davinci', + 'davinci', + 'curie', + 'babbage', + 'ada', + // O-series reasoning models - don't support parallel_tool_calls + 'o1', + 'o3', + 'o4', + // Non-chat model families + 'dall-e', + 'whisper', + 'tts', + 'text-embedding', + 'omni-moderation', + 'chatgpt', // chatgpt-4o-latest, chatgpt-image-latest + 'computer-use', // computer-use-preview + 'gpt-image', // gpt-image-1, gpt-image-1.5 + 'gpt-realtime', // gpt-realtime, gpt-realtime-mini + 'gpt-audio', // gpt-audio + 'sora', // sora-2, sora-2-pro + 'codex', // codex-mini-latest +]; + +/** + * OpenAI model patterns that do NOT support tool calls. + * Uses contains matching: model.includes(pattern) + */ +const UNSUPPORTED_MODEL_PATTERNS = [ + // Non-chat model variants (can appear in the middle of model names) + '-realtime', + '-audio', + '-transcribe', + '-tts', + '-search', + '-codex', + '-instruct', + // Models that only support v1/responses, not v1/chat/completions + '-pro', + '-deep-research', +]; + +/** + * Models that DO support tool calls even though they match an unsupported prefix. + * These override the UNSUPPORTED_MODEL_PREFIXES list. + */ +const SUPPORTED_MODEL_OVERRIDES = ['gpt-4-turbo', 'gpt-4o', 'gpt-4.1']; + +/** + * Checks if a model is compatible with Forest Admin AI. + * + * Supported models must handle tool calls and the parallel_tool_calls parameter. + */ +export default function isModelSupportingTools(model: string): boolean { + // Check pattern matches first (contains) - these NEVER support tools + const matchesUnsupportedPattern = UNSUPPORTED_MODEL_PATTERNS.some(pattern => + model.includes(pattern), + ); + if (matchesUnsupportedPattern) return false; + + // Check unsupported prefixes + const matchesUnsupportedPrefix = UNSUPPORTED_MODEL_PREFIXES.some( + prefix => model === prefix || model.startsWith(`${prefix}-`), + ); + + // Check if model is in the supported overrides list + const isSupportedOverride = SUPPORTED_MODEL_OVERRIDES.some( + override => model === override || model.startsWith(`${override}-`), + ); + + // If it matches an unsupported prefix but is not in overrides, reject it + if (matchesUnsupportedPrefix && !isSupportedOverride) return false; + + return true; +} diff --git a/packages/ai-proxy/test/llm.integration.test.ts b/packages/ai-proxy/test/llm.integration.test.ts index 238d22a53..a3f65d55b 100644 --- a/packages/ai-proxy/test/llm.integration.test.ts +++ b/packages/ai-proxy/test/llm.integration.test.ts @@ -11,14 +11,42 @@ import type { Server } from 'http'; // eslint-disable-next-line import/extensions import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import OpenAI from 'openai'; import { z } from 'zod'; import { Router } from '../src'; import runMcpServer from '../src/examples/simple-mcp-server'; +import isModelSupportingTools from '../src/supported-models'; const { OPENAI_API_KEY } = process.env; const describeWithOpenAI = OPENAI_API_KEY ? describe : describe.skip; +/** + * Fetches available models from OpenAI API. + * Returns all models that pass `isModelSupportingTools`. + * + * If a model fails the integration test, update the blacklist in supported-models.ts. + */ +async function fetchChatModelsFromOpenAI(): Promise { + const openai = new OpenAI({ apiKey: OPENAI_API_KEY }); + + let models; + try { + models = await openai.models.list(); + } catch (error) { + throw new Error( + `Failed to fetch models from OpenAI API. ` + + `Ensure OPENAI_API_KEY is valid and network is available. ` + + `Original error: ${error}`, + ); + } + + return models.data + .map(m => m.id) + .filter(id => isModelSupportingTools(id)) + .sort(); +} + describeWithOpenAI('OpenAI Integration (real API)', () => { const router = new Router({ aiConfigurations: [ @@ -688,4 +716,89 @@ describeWithOpenAI('OpenAI Integration (real API)', () => { }, 10000); }); }); + + describe('Model tool support verification', () => { + let modelsToTest: string[]; + + beforeAll(async () => { + modelsToTest = await fetchChatModelsFromOpenAI(); + }); + + it('should have found chat models from OpenAI API', () => { + expect(modelsToTest.length).toBeGreaterThan(0); + // eslint-disable-next-line no-console + console.log(`Testing ${modelsToTest.length} models:`, modelsToTest); + }); + + it('all chat models should support tool calls', async () => { + const results: { model: string; success: boolean; error?: string }[] = []; + + for (const model of modelsToTest) { + const modelRouter = new Router({ + aiConfigurations: [{ name: 'test', provider: 'openai', model, apiKey: OPENAI_API_KEY }], + }); + + try { + const response = (await modelRouter.route({ + route: 'ai-query', + body: { + messages: [{ role: 'user', content: 'What is 2+2?' }], + tools: [ + { + type: 'function', + function: { + name: 'calculate', + description: 'Calculate a math expression', + parameters: { type: 'object', properties: { result: { type: 'number' } } }, + }, + }, + ], + tool_choice: 'required', + parallel_tool_calls: false, + }, + })) as ChatCompletionResponse; + + const success = + response.choices[0].finish_reason === 'tool_calls' && + response.choices[0].message.tool_calls !== undefined; + + results.push({ model, success }); + } catch (error) { + const errorMessage = String(error); + + // Infrastructure errors should fail the test immediately + const isInfrastructureError = + errorMessage.includes('rate limit') || + errorMessage.includes('429') || + errorMessage.includes('401') || + errorMessage.includes('Authentication') || + errorMessage.includes('ECONNREFUSED') || + errorMessage.includes('ETIMEDOUT') || + errorMessage.includes('getaddrinfo'); + + if (isInfrastructureError) { + throw new Error(`Infrastructure error testing model ${model}: ${errorMessage}`); + } + + results.push({ model, success: false, error: errorMessage }); + } + } + + const failures = results.filter(r => !r.success); + if (failures.length > 0) { + const failedModelNames = failures.map(f => f.model).join(', '); + // eslint-disable-next-line no-console + console.error( + `\n❌ ${failures.length} model(s) failed: ${failedModelNames}\n\n` + + `To fix this, add the failing model(s) to the blacklist in:\n` + + ` packages/ai-proxy/src/supported-models.ts\n\n` + + `Add to UNSUPPORTED_MODEL_PREFIXES (for prefix match)\n` + + `or UNSUPPORTED_MODEL_PATTERNS (for contains match)\n`, + failures, + ); + } + + expect(failures).toEqual([]); + }, 300000); // 5 minutes for all models + }); }); diff --git a/packages/ai-proxy/test/provider-dispatcher.test.ts b/packages/ai-proxy/test/provider-dispatcher.test.ts index e75b96ff4..b1b8afb97 100644 --- a/packages/ai-proxy/test/provider-dispatcher.test.ts +++ b/packages/ai-proxy/test/provider-dispatcher.test.ts @@ -2,7 +2,7 @@ import type { DispatchBody } from '../src'; import { convertToOpenAIFunction } from '@langchain/core/utils/function_calling'; -import { AINotConfiguredError, isModelSupportingTools, ProviderDispatcher, RemoteTools } from '../src'; +import { AINotConfiguredError, ProviderDispatcher, RemoteTools } from '../src'; // Mock raw OpenAI response (returned via __includeRawResponse: true) const mockOpenAIResponse = { @@ -297,52 +297,3 @@ describe('ProviderDispatcher', () => { }); }); }); - -describe('isModelSupportingTools', () => { - describe('should return true for supported and unknown models', () => { - const supportedModels = [ - 'gpt-4o', - 'gpt-4o-mini', - 'gpt-4o-2024-08-06', - 'gpt-4-turbo', - 'gpt-4-turbo-2024-04-09', - 'gpt-4.1', - 'gpt-4.1-mini', - 'gpt-4.1-nano', - 'gpt-5', - 'gpt-5-mini', - 'gpt-5-nano', - 'gpt-5.2', - 'o1', - 'o3', - 'o3-mini', - 'o3-pro', - 'o4-mini', - 'unknown-model', - 'future-gpt-model', - ]; - - it.each(supportedModels)('%s', model => { - expect(isModelSupportingTools(model)).toBe(true); - }); - }); - - describe('should return false for known unsupported models', () => { - const unsupportedModels = [ - 'gpt-4', - 'gpt-4-0613', - 'gpt-3.5-turbo', - 'gpt-3.5-turbo-0125', - 'gpt-3.5', - 'text-davinci-003', - 'davinci', - 'curie', - 'babbage', - 'ada', - ]; - - it.each(unsupportedModels)('%s', model => { - expect(isModelSupportingTools(model)).toBe(false); - }); - }); -}); diff --git a/packages/ai-proxy/test/router.test.ts b/packages/ai-proxy/test/router.test.ts index 4a2a6cba9..82abb1dbf 100644 --- a/packages/ai-proxy/test/router.test.ts +++ b/packages/ai-proxy/test/router.test.ts @@ -1,7 +1,7 @@ import type { DispatchBody, InvokeRemoteToolArgs, Route } from '../src'; import type { Logger } from '@forestadmin/datasource-toolkit'; -import { AIBadRequestError, Router } from '../src'; +import { AIBadRequestError, AIModelNotSupportedError, Router } from '../src'; import McpClient from '../src/mcp-client'; const invokeToolMock = jest.fn(); @@ -77,23 +77,23 @@ describe('route', () => { apiKey: 'dev', model: 'gpt-4o', }; - const gpt3Config = { - name: 'gpt3', + const gpt4MiniConfig = { + name: 'gpt4mini', provider: 'openai' as const, apiKey: 'dev', - model: 'gpt-3.5-turbo', + model: 'gpt-4o-mini', }; const router = new Router({ - aiConfigurations: [gpt4Config, gpt3Config], + aiConfigurations: [gpt4Config, gpt4MiniConfig], }); await router.route({ route: 'ai-query', - query: { 'ai-name': 'gpt3' }, + query: { 'ai-name': 'gpt4mini' }, body: { tools: [], tool_choice: 'required', messages: [] } as unknown as DispatchBody, }); - expect(ProviderDispatcherMock).toHaveBeenCalledWith(gpt3Config, expect.anything()); + expect(ProviderDispatcherMock).toHaveBeenCalledWith(gpt4MiniConfig, expect.anything()); }); it('uses first configuration when ai-name query is not provided', async () => { @@ -103,14 +103,14 @@ describe('route', () => { apiKey: 'dev', model: 'gpt-4o', }; - const gpt3Config = { - name: 'gpt3', + const gpt4MiniConfig = { + name: 'gpt4mini', provider: 'openai' as const, apiKey: 'dev', - model: 'gpt-3.5-turbo', + model: 'gpt-4o-mini', }; const router = new Router({ - aiConfigurations: [gpt4Config, gpt3Config], + aiConfigurations: [gpt4Config, gpt4MiniConfig], }); await router.route({ @@ -386,4 +386,50 @@ describe('route', () => { ); }); }); + + describe('Model validation', () => { + it('throws AIModelNotSupportedError when model does not support tools', () => { + expect( + () => + new Router({ + aiConfigurations: [ + { + name: 'test', + provider: 'openai', + apiKey: 'dev', + model: 'gpt-4', // Known unsupported model + }, + ], + }), + ).toThrow(AIModelNotSupportedError); + }); + + it('throws with helpful error message including model name', () => { + expect( + () => + new Router({ + aiConfigurations: [ + { + name: 'test', + provider: 'openai', + apiKey: 'dev', + model: 'text-davinci-003', + }, + ], + }), + ).toThrow("Model 'text-davinci-003' does not support tools"); + }); + + it('validates all configurations, not just the first', () => { + expect( + () => + new Router({ + aiConfigurations: [ + { name: 'valid', provider: 'openai', apiKey: 'dev', model: 'gpt-4o' }, + { name: 'invalid', provider: 'openai', apiKey: 'dev', model: 'gpt-4' }, + ], + }), + ).toThrow("Model 'gpt-4' does not support tools"); + }); + }); }); diff --git a/packages/ai-proxy/test/supported-models.test.ts b/packages/ai-proxy/test/supported-models.test.ts new file mode 100644 index 000000000..7c5297b9b --- /dev/null +++ b/packages/ai-proxy/test/supported-models.test.ts @@ -0,0 +1,15 @@ +import isModelSupportingTools from '../src/supported-models'; + +describe('isModelSupportingTools', () => { + it('should return true for a known supported model', () => { + expect(isModelSupportingTools('gpt-4o')).toBe(true); + }); + + it('should return true for an unknown model (allowed by default)', () => { + expect(isModelSupportingTools('unknown-future-model')).toBe(true); + }); + + it('should return false for a blacklisted model', () => { + expect(isModelSupportingTools('gpt-4')).toBe(false); + }); +}); diff --git a/yarn.lock b/yarn.lock index bba1ac377..434e55cc6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2499,6 +2499,15 @@ openai "^6.10.0" zod "^3.25.76 || ^4" +"@langchain/openai@1.2.5": + version "1.2.5" + resolved "https://registry.yarnpkg.com/@langchain/openai/-/openai-1.2.5.tgz#e99ebdc68224b895ef7c7f5cf47bc199388291ea" + integrity sha512-AtnzS0j8Kv7IIdXywp/N27ytsMIbmncvFckiaWyOGibgqQYZyyR3rFC6P4z2x4/yoMgU7nXcd8dTNf+mABzLUw== + dependencies: + js-tiktoken "^1.0.12" + openai "^6.18.0" + zod "^3.25.76 || ^4" + "@langchain/textsplitters@1.0.1": version "1.0.1" resolved "https://registry.yarnpkg.com/@langchain/textsplitters/-/textsplitters-1.0.1.tgz#292f9c93239178c248b3338acf7b68aa47aa9830" @@ -14112,6 +14121,11 @@ openai@^6.10.0: resolved "https://registry.yarnpkg.com/openai/-/openai-6.16.0.tgz#53661d9f6307dc7523e89637ff7c6ccd0be16c67" integrity sha512-fZ1uBqjFUjXzbGc35fFtYKEOxd20kd9fDpFeqWtsOZWiubY8CZ1NAlXHW3iathaFvqmNtCWMIsosCuyeI7Joxg== +openai@^6.18.0: + version "6.18.0" + resolved "https://registry.yarnpkg.com/openai/-/openai-6.18.0.tgz#bd6c0bdb1aebf93375d324de51756280f7e85c6f" + integrity sha512-odLRYyz9rlzz6g8gKn61RM2oP5UUm428sE2zOxZqS9MzVfD5/XW8UoEjpnRkzTuScXP7ZbP/m7fC+bl8jCOZZw== + openapi-types@^12.1.3: version "12.1.3" resolved "https://registry.yarnpkg.com/openapi-types/-/openapi-types-12.1.3.tgz#471995eb26c4b97b7bd356aacf7b91b73e777dd3"