fix(ai-proxy): validate model tool support at Router init (fail fast)

packages/agent/src/agent.ts

@@ -17,7 +17,6 @@ import type {
 import type { DataSource, DataSourceFactory } from '@forestadmin/datasource-toolkit';
 import type { ForestSchema } from '@forestadmin/forestadmin-client';
 
-import { isModelSupportingTools } from '@forestadmin/ai-proxy';
 import { DataSourceCustomizer } from '@forestadmin/datasource-customizer';
 import bodyParser from '@koa/bodyparser';
 import cors from '@koa/cors';
@@ -246,13 +245,6 @@ export default class Agent<S extends TSchema = TSchema> extends FrameworkMounter
       );
     }
 
-    if (!isModelSupportingTools(configuration.model)) {
-      throw new Error(
-        `Model '${configuration.model}' does not support function calling (tools). ` +
-          'Please use a compatible model like gpt-4o, gpt-4o-mini, or gpt-4-turbo.',
-      );
-    }
-
     this.options.logger(
       'Warn',
       `AI configuration added with model '${configuration.model}'. ` +

packages/agent/test/agent.test.ts

@@ -439,19 +439,22 @@ describe('Agent', () => {
       ).toThrow('addAi can only be called once. Multiple AI configurations are not supported yet.');
     });
 
-    test('should throw an error when model does not support tools', () => {
+    test('should throw an error on start when model does not support tools', async () => {
+      // Use the real makeRoutes to trigger validation in AiProxyRouter
+      const realMakeRoutes = jest.requireActual('../src/routes').default;
+      mockMakeRoutes.mockImplementation(realMakeRoutes);
+
       const agent = new Agent(options);
 
-      expect(() =>
-        agent.addAi({
-          name: 'gpt4-base',
-          provider: 'openai',
-          apiKey: 'test-key',
-          model: 'gpt-4',
-        }),
-      ).toThrow(
-        "Model 'gpt-4' does not support function calling (tools). " +
-          'Please use a compatible model like gpt-4o, gpt-4o-mini, or gpt-4-turbo.',
+      agent.addAi({
+        name: 'gpt4-base',
+        provider: 'openai',
+        apiKey: 'test-key',
+        model: 'gpt-4',
+      });
+
+      await expect(agent.start()).rejects.toThrow(
+        "Model 'gpt-4' does not support tools. Please use a model that supports function calling.",
       );
     });
 

packages/ai-proxy/package.json

@@ -17,7 +17,7 @@
     "@langchain/core": "1.1.15",
     "@langchain/langgraph": "^1.1.0",
     "@langchain/mcp-adapters": "1.1.1",
-    "@langchain/openai": "1.2.2",
+    "@langchain/openai": "1.2.5",
     "zod": "^4.3.5"
   },
   "devDependencies": {

packages/ai-proxy/src/errors.ts

@@ -32,6 +32,15 @@ export class AIBadRequestError extends AIError {
   }
 }
 
+export class AIModelNotSupportedError extends AIBadRequestError {
+  constructor(model: string) {
+    super(
+      `Model '${model}' does not support tools. Please use a model that supports function calling.`,
+    );
+    this.name = 'AIModelNotSupportedError';
+  }
+}
+
 export class AINotFoundError extends AIError {
   constructor(message: string) {
     super(message, 404);

packages/ai-proxy/src/index.ts

@@ -7,7 +7,6 @@ export * from './remote-tools';
 export * from './router';
 export * from './mcp-client';
 export * from './oauth-token-injector';
-
 export * from './errors';
 
 export function validMcpConfigurationOrThrow(mcpConfig: McpConfiguration) {

packages/ai-proxy/src/provider-dispatcher.ts

@@ -21,41 +21,6 @@ export type {
 } from './provider';
 export type { DispatchBody } from './schemas/route';
 
-/**
- * OpenAI model prefixes that do NOT support function calling (tools).
- * Unknown models are allowed.
- * @see https://platform.openai.com/docs/guides/function-calling
- */
-const OPENAI_MODELS_WITHOUT_TOOLS_SUPPORT = [
-  'gpt-4',
-  'gpt-3.5-turbo',
-  'gpt-3.5',
-  'text-davinci',
-  'davinci',
-  'curie',
-  'babbage',
-  'ada',
-];
-
-/**
- * Exceptions to the unsupported list - these models DO support tools
- * even though they start with an unsupported prefix.
- */
-const OPENAI_MODELS_EXCEPTIONS = ['gpt-4-turbo', 'gpt-4o', 'gpt-4.1'];
-
-export function isModelSupportingTools(model: string): boolean {
-  const isException = OPENAI_MODELS_EXCEPTIONS.some(
-    exception => model === exception || model.startsWith(`${exception}-`),
-  );
-  if (isException) return true;
-
-  const isKnownUnsupported = OPENAI_MODELS_WITHOUT_TOOLS_SUPPORT.some(
-    unsupported => model === unsupported || model.startsWith(`${unsupported}-`),
-  );
-
-  return !isKnownUnsupported;
-}
-
 export class ProviderDispatcher {
   private readonly chatModel: ChatOpenAI | null = null;
 

packages/ai-proxy/src/router.ts

@@ -5,10 +5,12 @@ import type { RouteArgs } from './schemas/route';
 import type { Logger } from '@forestadmin/datasource-toolkit';
 import type { z } from 'zod';
 
-import { AIBadRequestError, ProviderDispatcher } from './index';
+import { AIBadRequestError, AIModelNotSupportedError } from './errors';
 import McpClient from './mcp-client';
+import { ProviderDispatcher } from './provider-dispatcher';
 import { RemoteTools } from './remote-tools';
 import { routeArgsSchema } from './schemas/route';
+import isModelSupportingTools from './supported-models';
 
 export type {
   AiQueryArgs,
@@ -37,6 +39,16 @@ export class Router {
     this.aiConfigurations = params?.aiConfigurations ?? [];
     this.localToolsApiKeys = params?.localToolsApiKeys;
     this.logger = params?.logger;
+
+    this.validateConfigurations();
+  }
+
+  private validateConfigurations(): void {
+    for (const config of this.aiConfigurations) {
+      if (!isModelSupportingTools(config.model)) {
+        throw new AIModelNotSupportedError(config.model);
+      }
+    }
   }
 
   /**

packages/ai-proxy/src/supported-models.ts

@@ -0,0 +1,88 @@
+/**
+ * OpenAI model prefixes that do NOT support tool calls via the chat completions API.
+ *
+ * Uses prefix matching: model === prefix OR model.startsWith(prefix + '-')
+ *
+ * Unknown models are allowed by default.
+ * If a model fails the integration test, add it here.
+ *
+ * @see https://platform.openai.com/docs/guides/function-calling
+ */
+const UNSUPPORTED_MODEL_PREFIXES = [
+  // Legacy models
+  'gpt-4', // Base gpt-4 doesn't honor tool_choice: required
+  'text-davinci',
+  'davinci',
+  'curie',
+  'babbage',
+  'ada',
+  // O-series reasoning models - don't support parallel_tool_calls
+  'o1',
+  'o3',
+  'o4',
+  // Non-chat model families
+  'dall-e',
+  'whisper',
+  'tts',
+  'text-embedding',
+  'omni-moderation',
+  'chatgpt', // chatgpt-4o-latest, chatgpt-image-latest
+  'computer-use', // computer-use-preview
+  'gpt-image', // gpt-image-1, gpt-image-1.5
+  'gpt-realtime', // gpt-realtime, gpt-realtime-mini
+  'gpt-audio', // gpt-audio
+  'sora', // sora-2, sora-2-pro
+  'codex', // codex-mini-latest
+];
+
+/**
+ * OpenAI model patterns that do NOT support tool calls.
+ * Uses contains matching: model.includes(pattern)
+ */
+const UNSUPPORTED_MODEL_PATTERNS = [
+  // Non-chat model variants (can appear in the middle of model names)
+  '-realtime',
+  '-audio',
+  '-transcribe',
+  '-tts',
+  '-search',
+  '-codex',
+  '-instruct',
+  // Models that only support v1/responses, not v1/chat/completions
+  '-pro',
+  '-deep-research',
+];
+
+/**
+ * Models that DO support tool calls even though they match an unsupported prefix.
+ * These override the UNSUPPORTED_MODEL_PREFIXES list.
+ */
+const SUPPORTED_MODEL_OVERRIDES = ['gpt-4-turbo', 'gpt-4o', 'gpt-4.1'];
+
+/**
+ * Checks if a model is compatible with Forest Admin AI.
+ *
+ * Supported models must handle tool calls and the parallel_tool_calls parameter.
+ */
+export default function isModelSupportingTools(model: string): boolean {
+  // Check pattern matches first (contains) - these NEVER support tools
+  const matchesUnsupportedPattern = UNSUPPORTED_MODEL_PATTERNS.some(pattern =>
+    model.includes(pattern),
+  );
+  if (matchesUnsupportedPattern) return false;
+
+  // Check unsupported prefixes
+  const matchesUnsupportedPrefix = UNSUPPORTED_MODEL_PREFIXES.some(
+    prefix => model === prefix || model.startsWith(`${prefix}-`),
+  );
+
+  // Check if model is in the supported overrides list
+  const isSupportedOverride = SUPPORTED_MODEL_OVERRIDES.some(
+    override => model === override || model.startsWith(`${override}-`),
+  );
+
+  // If it matches an unsupported prefix but is not in overrides, reject it
+  if (matchesUnsupportedPrefix && !isSupportedOverride) return false;
+
+  return true;
+}

packages/ai-proxy/test/llm.integration.test.ts

@@ -11,14 +11,42 @@ import type { Server } from 'http';
 
 // eslint-disable-next-line import/extensions
 import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import OpenAI from 'openai';
 import { z } from 'zod';
 
 import { Router } from '../src';
 import runMcpServer from '../src/examples/simple-mcp-server';
+import isModelSupportingTools from '../src/supported-models';
 
 const { OPENAI_API_KEY } = process.env;
 const describeWithOpenAI = OPENAI_API_KEY ? describe : describe.skip;
 
+/**
+ * Fetches available models from OpenAI API.
+ * Returns all models that pass `isModelSupportingTools`.
+ *
+ * If a model fails the integration test, update the blacklist in supported-models.ts.
+ */
+async function fetchChatModelsFromOpenAI(): Promise<string[]> {
+  const openai = new OpenAI({ apiKey: OPENAI_API_KEY });
+
+  let models;
+  try {
+    models = await openai.models.list();
+  } catch (error) {
+    throw new Error(
+      `Failed to fetch models from OpenAI API. ` +
+        `Ensure OPENAI_API_KEY is valid and network is available. ` +
+        `Original error: ${error}`,
+    );
+  }
+
+  return models.data
+    .map(m => m.id)
+    .filter(id => isModelSupportingTools(id))
+    .sort();
+}
+
 describeWithOpenAI('OpenAI Integration (real API)', () => {
   const router = new Router({
     aiConfigurations: [
@@ -688,4 +716,89 @@ describeWithOpenAI('OpenAI Integration (real API)', () => {
       }, 10000);
     });
   });
+
+  describe('Model tool support verification', () => {
+    let modelsToTest: string[];
+
+    beforeAll(async () => {
+      modelsToTest = await fetchChatModelsFromOpenAI();
+    });
+
+    it('should have found chat models from OpenAI API', () => {
+      expect(modelsToTest.length).toBeGreaterThan(0);
+      // eslint-disable-next-line no-console
+      console.log(`Testing ${modelsToTest.length} models:`, modelsToTest);
+    });
+
+    it('all chat models should support tool calls', async () => {
+      const results: { model: string; success: boolean; error?: string }[] = [];
+
+      for (const model of modelsToTest) {
+        const modelRouter = new Router({
+          aiConfigurations: [{ name: 'test', provider: 'openai', model, apiKey: OPENAI_API_KEY }],
+        });
+
+        try {
+          const response = (await modelRouter.route({
+            route: 'ai-query',
+            body: {
+              messages: [{ role: 'user', content: 'What is 2+2?' }],
+              tools: [
+                {
+                  type: 'function',
+                  function: {
+                    name: 'calculate',
+                    description: 'Calculate a math expression',
+                    parameters: { type: 'object', properties: { result: { type: 'number' } } },
+                  },
+                },
+              ],
+              tool_choice: 'required',
+              parallel_tool_calls: false,
+            },
+          })) as ChatCompletionResponse;
+
+          const success =
+            response.choices[0].finish_reason === 'tool_calls' &&
+            response.choices[0].message.tool_calls !== undefined;
+
+          results.push({ model, success });
+        } catch (error) {
+          const errorMessage = String(error);
+
+          // Infrastructure errors should fail the test immediately
+          const isInfrastructureError =
+            errorMessage.includes('rate limit') ||
+            errorMessage.includes('429') ||
+            errorMessage.includes('401') ||
+            errorMessage.includes('Authentication') ||
+            errorMessage.includes('ECONNREFUSED') ||
+            errorMessage.includes('ETIMEDOUT') ||
+            errorMessage.includes('getaddrinfo');
+
+          if (isInfrastructureError) {
+            throw new Error(`Infrastructure error testing model ${model}: ${errorMessage}`);
+          }
+
+          results.push({ model, success: false, error: errorMessage });
+        }
+      }
+
+      const failures = results.filter(r => !r.success);
+      if (failures.length > 0) {
+        const failedModelNames = failures.map(f => f.model).join(', ');
+        // eslint-disable-next-line no-console
+        console.error(
+          `\n❌ ${failures.length} model(s) failed: ${failedModelNames}\n\n` +
+            `To fix this, add the failing model(s) to the blacklist in:\n` +
+            `  packages/ai-proxy/src/supported-models.ts\n\n` +
+            `Add to UNSUPPORTED_MODEL_PREFIXES (for prefix match)\n` +
+            `or UNSUPPORTED_MODEL_PATTERNS (for contains match)\n`,
+          failures,
+        );
+      }
+
+      expect(failures).toEqual([]);
+    }, 300000); // 5 minutes for all models
+  });
 });