From aa50ba927162bd1be27e1281d9a48007d34b5681 Mon Sep 17 00:00:00 2001 From: Jesse Turner Date: Mon, 18 May 2026 17:15:29 +0000 Subject: [PATCH 1/2] fix(invoke): preserve model inference params when overriding model at harness invoke time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When --model-id was passed to `agentcore invoke --harness`, the CLI constructed a new model config override that dropped maxTokens, temperature, and topP from the harness spec's model configuration. Since the server treats invoke-time model overrides as a full replacement of the deployed model config, this silently removed the model's token limit — allowing unbounded output. Also expands the HarnessModelConfiguration type to properly declare inference parameters (temperature, topP, topK, maxTokens) instead of relying on spread operators to bypass the narrow type at deploy time. --- src/cli/aws/agentcore-harness.ts | 30 +++++++++++++++++++++--- src/cli/commands/invoke/action.ts | 38 ++++++++++++++++++++++++++++--- 2 files changed, 62 insertions(+), 6 deletions(-) diff --git a/src/cli/aws/agentcore-harness.ts b/src/cli/aws/agentcore-harness.ts index cca371fbc..eed7a5837 100644 --- a/src/cli/aws/agentcore-harness.ts +++ b/src/cli/aws/agentcore-harness.ts @@ -17,10 +17,34 @@ import { randomUUID } from 'node:crypto'; export type HarnessStatus = 'CREATING' | 'READY' | 'UPDATING' | 'DELETING' | 'DELETED' | 'FAILED'; +export interface BedrockModelConfig { + modelId: string; + temperature?: number; + topP?: number; + maxTokens?: number; +} + +export interface OpenAiModelConfig { + modelId: string; + apiKeyArn?: string; + temperature?: number; + topP?: number; + maxTokens?: number; +} + +export interface GeminiModelConfig { + modelId: string; + apiKeyArn?: string; + temperature?: number; + topP?: number; + topK?: number; + maxTokens?: number; +} + export interface HarnessModelConfiguration { - bedrockModelConfig?: { modelId: string }; - openAiModelConfig?: { modelId: string; apiKeyArn?: string }; - geminiModelConfig?: { modelId: string; apiKeyArn?: string }; + bedrockModelConfig?: BedrockModelConfig; + openAiModelConfig?: OpenAiModelConfig; + geminiModelConfig?: GeminiModelConfig; } export type HarnessSystemPrompt = { text: string }[]; diff --git a/src/cli/commands/invoke/action.ts b/src/cli/commands/invoke/action.ts index 4c5501b6f..f30e39d76 100644 --- a/src/cli/commands/invoke/action.ts +++ b/src/cli/commands/invoke/action.ts @@ -516,6 +516,10 @@ interface HarnessModel { provider?: string; modelId?: string; apiKeyArn?: string; + temperature?: number; + topP?: number; + topK?: number; + maxTokens?: number; } function buildHarnessBaseOpts( @@ -527,15 +531,43 @@ function buildHarnessBaseOpts( const provider = options.modelProvider ?? harnessSpec?.provider; const modelId = options.modelId ?? harnessSpec?.modelId ?? ''; const apiKeyArn = options.apiKeyArn ?? harnessSpec?.apiKeyArn; + const temperature = harnessSpec?.temperature; + const topP = harnessSpec?.topP; + const topK = harnessSpec?.topK; + const modelMaxTokens = harnessSpec?.maxTokens; switch (provider) { case 'open_ai': - baseOpts.model = { openAiModelConfig: { modelId, ...(apiKeyArn && { apiKeyArn }) } }; + baseOpts.model = { + openAiModelConfig: { + modelId, + ...(apiKeyArn && { apiKeyArn }), + ...(temperature !== undefined && { temperature }), + ...(topP !== undefined && { topP }), + ...(modelMaxTokens !== undefined && { maxTokens: modelMaxTokens }), + }, + }; break; case 'gemini': - baseOpts.model = { geminiModelConfig: { modelId, ...(apiKeyArn && { apiKeyArn }) } }; + baseOpts.model = { + geminiModelConfig: { + modelId, + ...(apiKeyArn && { apiKeyArn }), + ...(temperature !== undefined && { temperature }), + ...(topP !== undefined && { topP }), + ...(topK !== undefined && { topK }), + ...(modelMaxTokens !== undefined && { maxTokens: modelMaxTokens }), + }, + }; break; default: - baseOpts.model = { bedrockModelConfig: { modelId } }; + baseOpts.model = { + bedrockModelConfig: { + modelId, + ...(temperature !== undefined && { temperature }), + ...(topP !== undefined && { topP }), + ...(modelMaxTokens !== undefined && { maxTokens: modelMaxTokens }), + }, + }; break; } } From 36a88c15bed317db8a2e3b25a3fc765d272f17d3 Mon Sep 17 00:00:00 2001 From: Jesse Turner Date: Mon, 18 May 2026 18:38:49 +0000 Subject: [PATCH 2/2] =?UTF-8?q?fix:=20address=20review=20=E2=80=94=20reuse?= =?UTF-8?q?=20HarnessModel=20type,=20add=20unit=20tests=20for=20buildHarne?= =?UTF-8?q?ssBaseOpts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Replace local HarnessModel interface with imported schema type to prevent drift when new inference params are added - Export buildHarnessBaseOpts for testability - Add 8 unit tests covering all provider branches, param preservation, param omission, CLI precedence, and execution limits --- .../__tests__/build-harness-base-opts.test.ts | 154 ++++++++++++++++++ src/cli/commands/invoke/action.ts | 16 +- src/schema/schemas/agentcore-project.ts | 2 +- 3 files changed, 158 insertions(+), 14 deletions(-) create mode 100644 src/cli/commands/invoke/__tests__/build-harness-base-opts.test.ts diff --git a/src/cli/commands/invoke/__tests__/build-harness-base-opts.test.ts b/src/cli/commands/invoke/__tests__/build-harness-base-opts.test.ts new file mode 100644 index 000000000..19b6ad1fc --- /dev/null +++ b/src/cli/commands/invoke/__tests__/build-harness-base-opts.test.ts @@ -0,0 +1,154 @@ +import { buildHarnessBaseOpts } from '../action.js'; +import type { InvokeOptions } from '../types.js'; +import { describe, expect, it } from 'vitest'; + +describe('buildHarnessBaseOpts', () => { + describe('preserves model inference params from harness spec when overriding model', () => { + it('bedrock: includes temperature, topP, and maxTokens', () => { + const options: InvokeOptions = { modelId: 'anthropic.claude-v3' }; + const harnessSpec = { + provider: 'bedrock' as const, + modelId: 'anthropic.claude-v2', + temperature: 0.7, + topP: 0.9, + maxTokens: 500, + }; + + const result = buildHarnessBaseOpts(options, harnessSpec); + + expect(result.model).toEqual({ + bedrockModelConfig: { + modelId: 'anthropic.claude-v3', + temperature: 0.7, + topP: 0.9, + maxTokens: 500, + }, + }); + }); + + it('open_ai: includes temperature, topP, maxTokens, and apiKeyArn', () => { + const options: InvokeOptions = { modelId: 'gpt-5' }; + const harnessSpec = { + provider: 'open_ai' as const, + modelId: 'gpt-4', + apiKeyArn: 'arn:aws:secretsmanager:us-east-1:123:secret:key', + temperature: 0.5, + topP: 0.8, + maxTokens: 2048, + }; + + const result = buildHarnessBaseOpts(options, harnessSpec); + + expect(result.model).toEqual({ + openAiModelConfig: { + modelId: 'gpt-5', + apiKeyArn: 'arn:aws:secretsmanager:us-east-1:123:secret:key', + temperature: 0.5, + topP: 0.8, + maxTokens: 2048, + }, + }); + }); + + it('gemini: includes temperature, topP, topK, maxTokens, and apiKeyArn', () => { + const options: InvokeOptions = { modelId: 'gemini-2.5-pro' }; + const harnessSpec = { + provider: 'gemini' as const, + modelId: 'gemini-2.5-flash', + apiKeyArn: 'arn:aws:secretsmanager:us-east-1:123:secret:gemini', + temperature: 0.3, + topP: 0.95, + topK: 0.5, + maxTokens: 1024, + }; + + const result = buildHarnessBaseOpts(options, harnessSpec); + + expect(result.model).toEqual({ + geminiModelConfig: { + modelId: 'gemini-2.5-pro', + apiKeyArn: 'arn:aws:secretsmanager:us-east-1:123:secret:gemini', + temperature: 0.3, + topP: 0.95, + topK: 0.5, + maxTokens: 1024, + }, + }); + }); + }); + + describe('omits undefined inference params', () => { + it('bedrock: only includes modelId when no inference params set', () => { + const options: InvokeOptions = { modelId: 'anthropic.claude-v3' }; + const harnessSpec = { provider: 'bedrock' as const, modelId: 'anthropic.claude-v2' }; + + const result = buildHarnessBaseOpts(options, harnessSpec); + + expect(result.model).toEqual({ + bedrockModelConfig: { modelId: 'anthropic.claude-v3' }, + }); + }); + + it('open_ai: omits apiKeyArn and inference params when not set', () => { + const options: InvokeOptions = { modelId: 'gpt-5', modelProvider: 'open_ai' }; + const harnessSpec = { provider: 'open_ai' as const, modelId: 'gpt-4' }; + + const result = buildHarnessBaseOpts(options, harnessSpec); + + expect(result.model).toEqual({ + openAiModelConfig: { modelId: 'gpt-5' }, + }); + }); + }); + + describe('CLI options take precedence for apiKeyArn', () => { + it('uses CLI apiKeyArn over harness spec', () => { + const options: InvokeOptions = { + modelId: 'gpt-5', + apiKeyArn: 'arn:aws:secretsmanager:us-east-1:123:secret:cli-key', + }; + const harnessSpec = { + provider: 'open_ai' as const, + modelId: 'gpt-4', + apiKeyArn: 'arn:aws:secretsmanager:us-east-1:123:secret:spec-key', + maxTokens: 1000, + }; + + const result = buildHarnessBaseOpts(options, harnessSpec); + + expect(result.model!.openAiModelConfig!.apiKeyArn).toBe('arn:aws:secretsmanager:us-east-1:123:secret:cli-key'); + expect(result.model!.openAiModelConfig!.maxTokens).toBe(1000); + }); + }); + + describe('does not set model when no model override options provided', () => { + it('returns empty opts when no model-related options are set', () => { + const options: InvokeOptions = {}; + const harnessSpec = { + provider: 'bedrock' as const, + modelId: 'anthropic.claude-v2', + maxTokens: 500, + }; + + const result = buildHarnessBaseOpts(options, harnessSpec); + + expect(result.model).toBeUndefined(); + }); + }); + + describe('harness-level execution limits', () => { + it('forwards maxTokens, maxIterations, and timeoutSeconds from CLI options', () => { + const options: InvokeOptions = { + maxTokens: 100, + maxIterations: 10, + harnessTimeout: 30, + }; + + const result = buildHarnessBaseOpts(options); + + expect(result.maxTokens).toBe(100); + expect(result.maxIterations).toBe(10); + expect(result.timeoutSeconds).toBe(30); + }); + }); +}); diff --git a/src/cli/commands/invoke/action.ts b/src/cli/commands/invoke/action.ts index f30e39d76..31a2874a0 100644 --- a/src/cli/commands/invoke/action.ts +++ b/src/cli/commands/invoke/action.ts @@ -1,5 +1,5 @@ import { ConfigIO } from '../../../lib'; -import type { AgentCoreProjectSpec, AwsDeploymentTargets, DeployedState } from '../../../schema'; +import type { AgentCoreProjectSpec, AwsDeploymentTargets, DeployedState, HarnessModel } from '../../../schema'; import { buildAguiRunInput, executeBashCommand, @@ -512,19 +512,9 @@ export async function handleInvoke(context: InvokeContext, options: InvokeOption // Shared Harness Helpers // ============================================================================ -interface HarnessModel { - provider?: string; - modelId?: string; - apiKeyArn?: string; - temperature?: number; - topP?: number; - topK?: number; - maxTokens?: number; -} - -function buildHarnessBaseOpts( +export function buildHarnessBaseOpts( options: InvokeOptions, - harnessSpec?: HarnessModel + harnessSpec?: Partial ): Partial { const baseOpts: Partial = {}; if (options.modelId || options.modelProvider || options.apiKeyArn) { diff --git a/src/schema/schemas/agentcore-project.ts b/src/schema/schemas/agentcore-project.ts index 903db1f19..35a112fca 100644 --- a/src/schema/schemas/agentcore-project.ts +++ b/src/schema/schemas/agentcore-project.ts @@ -70,7 +70,7 @@ export type { ABTestMode, TargetRef, GatewayFilter, PerVariantOnlineEvaluationCo export { ABTestModeSchema, TargetRefSchema, GatewayFilterSchema } from './primitives/ab-test'; export type { HttpGatewayTarget } from './primitives/http-gateway'; export { HttpGatewayTargetSchema } from './primitives/http-gateway'; -export type { HarnessGatewayOutboundAuth, HarnessSpec, HarnessModelProvider } from './primitives/harness'; +export type { HarnessGatewayOutboundAuth, HarnessModel, HarnessSpec, HarnessModelProvider } from './primitives/harness'; export { GatewayOAuthGrantTypeSchema, HarnessGatewayOutboundAuthSchema,