diff --git a/src/cli/aws/agentcore-harness.ts b/src/cli/aws/agentcore-harness.ts index cca371fbc..eed7a5837 100644 --- a/src/cli/aws/agentcore-harness.ts +++ b/src/cli/aws/agentcore-harness.ts @@ -17,10 +17,34 @@ import { randomUUID } from 'node:crypto'; export type HarnessStatus = 'CREATING' | 'READY' | 'UPDATING' | 'DELETING' | 'DELETED' | 'FAILED'; +export interface BedrockModelConfig { + modelId: string; + temperature?: number; + topP?: number; + maxTokens?: number; +} + +export interface OpenAiModelConfig { + modelId: string; + apiKeyArn?: string; + temperature?: number; + topP?: number; + maxTokens?: number; +} + +export interface GeminiModelConfig { + modelId: string; + apiKeyArn?: string; + temperature?: number; + topP?: number; + topK?: number; + maxTokens?: number; +} + export interface HarnessModelConfiguration { - bedrockModelConfig?: { modelId: string }; - openAiModelConfig?: { modelId: string; apiKeyArn?: string }; - geminiModelConfig?: { modelId: string; apiKeyArn?: string }; + bedrockModelConfig?: BedrockModelConfig; + openAiModelConfig?: OpenAiModelConfig; + geminiModelConfig?: GeminiModelConfig; } export type HarnessSystemPrompt = { text: string }[]; diff --git a/src/cli/commands/invoke/__tests__/build-harness-base-opts.test.ts b/src/cli/commands/invoke/__tests__/build-harness-base-opts.test.ts new file mode 100644 index 000000000..19b6ad1fc --- /dev/null +++ b/src/cli/commands/invoke/__tests__/build-harness-base-opts.test.ts @@ -0,0 +1,154 @@ +import { buildHarnessBaseOpts } from '../action.js'; +import type { InvokeOptions } from '../types.js'; +import { describe, expect, it } from 'vitest'; + +describe('buildHarnessBaseOpts', () => { + describe('preserves model inference params from harness spec when overriding model', () => { + it('bedrock: includes temperature, topP, and maxTokens', () => { + const options: InvokeOptions = { modelId: 'anthropic.claude-v3' }; + const harnessSpec = { + provider: 'bedrock' as const, + modelId: 'anthropic.claude-v2', + temperature: 0.7, + topP: 0.9, + maxTokens: 500, + }; + + const result = buildHarnessBaseOpts(options, harnessSpec); + + expect(result.model).toEqual({ + bedrockModelConfig: { + modelId: 'anthropic.claude-v3', + temperature: 0.7, + topP: 0.9, + maxTokens: 500, + }, + }); + }); + + it('open_ai: includes temperature, topP, maxTokens, and apiKeyArn', () => { + const options: InvokeOptions = { modelId: 'gpt-5' }; + const harnessSpec = { + provider: 'open_ai' as const, + modelId: 'gpt-4', + apiKeyArn: 'arn:aws:secretsmanager:us-east-1:123:secret:key', + temperature: 0.5, + topP: 0.8, + maxTokens: 2048, + }; + + const result = buildHarnessBaseOpts(options, harnessSpec); + + expect(result.model).toEqual({ + openAiModelConfig: { + modelId: 'gpt-5', + apiKeyArn: 'arn:aws:secretsmanager:us-east-1:123:secret:key', + temperature: 0.5, + topP: 0.8, + maxTokens: 2048, + }, + }); + }); + + it('gemini: includes temperature, topP, topK, maxTokens, and apiKeyArn', () => { + const options: InvokeOptions = { modelId: 'gemini-2.5-pro' }; + const harnessSpec = { + provider: 'gemini' as const, + modelId: 'gemini-2.5-flash', + apiKeyArn: 'arn:aws:secretsmanager:us-east-1:123:secret:gemini', + temperature: 0.3, + topP: 0.95, + topK: 0.5, + maxTokens: 1024, + }; + + const result = buildHarnessBaseOpts(options, harnessSpec); + + expect(result.model).toEqual({ + geminiModelConfig: { + modelId: 'gemini-2.5-pro', + apiKeyArn: 'arn:aws:secretsmanager:us-east-1:123:secret:gemini', + temperature: 0.3, + topP: 0.95, + topK: 0.5, + maxTokens: 1024, + }, + }); + }); + }); + + describe('omits undefined inference params', () => { + it('bedrock: only includes modelId when no inference params set', () => { + const options: InvokeOptions = { modelId: 'anthropic.claude-v3' }; + const harnessSpec = { provider: 'bedrock' as const, modelId: 'anthropic.claude-v2' }; + + const result = buildHarnessBaseOpts(options, harnessSpec); + + expect(result.model).toEqual({ + bedrockModelConfig: { modelId: 'anthropic.claude-v3' }, + }); + }); + + it('open_ai: omits apiKeyArn and inference params when not set', () => { + const options: InvokeOptions = { modelId: 'gpt-5', modelProvider: 'open_ai' }; + const harnessSpec = { provider: 'open_ai' as const, modelId: 'gpt-4' }; + + const result = buildHarnessBaseOpts(options, harnessSpec); + + expect(result.model).toEqual({ + openAiModelConfig: { modelId: 'gpt-5' }, + }); + }); + }); + + describe('CLI options take precedence for apiKeyArn', () => { + it('uses CLI apiKeyArn over harness spec', () => { + const options: InvokeOptions = { + modelId: 'gpt-5', + apiKeyArn: 'arn:aws:secretsmanager:us-east-1:123:secret:cli-key', + }; + const harnessSpec = { + provider: 'open_ai' as const, + modelId: 'gpt-4', + apiKeyArn: 'arn:aws:secretsmanager:us-east-1:123:secret:spec-key', + maxTokens: 1000, + }; + + const result = buildHarnessBaseOpts(options, harnessSpec); + + expect(result.model!.openAiModelConfig!.apiKeyArn).toBe('arn:aws:secretsmanager:us-east-1:123:secret:cli-key'); + expect(result.model!.openAiModelConfig!.maxTokens).toBe(1000); + }); + }); + + describe('does not set model when no model override options provided', () => { + it('returns empty opts when no model-related options are set', () => { + const options: InvokeOptions = {}; + const harnessSpec = { + provider: 'bedrock' as const, + modelId: 'anthropic.claude-v2', + maxTokens: 500, + }; + + const result = buildHarnessBaseOpts(options, harnessSpec); + + expect(result.model).toBeUndefined(); + }); + }); + + describe('harness-level execution limits', () => { + it('forwards maxTokens, maxIterations, and timeoutSeconds from CLI options', () => { + const options: InvokeOptions = { + maxTokens: 100, + maxIterations: 10, + harnessTimeout: 30, + }; + + const result = buildHarnessBaseOpts(options); + + expect(result.maxTokens).toBe(100); + expect(result.maxIterations).toBe(10); + expect(result.timeoutSeconds).toBe(30); + }); + }); +}); diff --git a/src/cli/commands/invoke/action.ts b/src/cli/commands/invoke/action.ts index 4c5501b6f..31a2874a0 100644 --- a/src/cli/commands/invoke/action.ts +++ b/src/cli/commands/invoke/action.ts @@ -1,5 +1,5 @@ import { ConfigIO } from '../../../lib'; -import type { AgentCoreProjectSpec, AwsDeploymentTargets, DeployedState } from '../../../schema'; +import type { AgentCoreProjectSpec, AwsDeploymentTargets, DeployedState, HarnessModel } from '../../../schema'; import { buildAguiRunInput, executeBashCommand, @@ -512,30 +512,52 @@ export async function handleInvoke(context: InvokeContext, options: InvokeOption // Shared Harness Helpers // ============================================================================ -interface HarnessModel { - provider?: string; - modelId?: string; - apiKeyArn?: string; -} - -function buildHarnessBaseOpts( +export function buildHarnessBaseOpts( options: InvokeOptions, - harnessSpec?: HarnessModel + harnessSpec?: Partial ): Partial { const baseOpts: Partial = {}; if (options.modelId || options.modelProvider || options.apiKeyArn) { const provider = options.modelProvider ?? harnessSpec?.provider; const modelId = options.modelId ?? harnessSpec?.modelId ?? ''; const apiKeyArn = options.apiKeyArn ?? harnessSpec?.apiKeyArn; + const temperature = harnessSpec?.temperature; + const topP = harnessSpec?.topP; + const topK = harnessSpec?.topK; + const modelMaxTokens = harnessSpec?.maxTokens; switch (provider) { case 'open_ai': - baseOpts.model = { openAiModelConfig: { modelId, ...(apiKeyArn && { apiKeyArn }) } }; + baseOpts.model = { + openAiModelConfig: { + modelId, + ...(apiKeyArn && { apiKeyArn }), + ...(temperature !== undefined && { temperature }), + ...(topP !== undefined && { topP }), + ...(modelMaxTokens !== undefined && { maxTokens: modelMaxTokens }), + }, + }; break; case 'gemini': - baseOpts.model = { geminiModelConfig: { modelId, ...(apiKeyArn && { apiKeyArn }) } }; + baseOpts.model = { + geminiModelConfig: { + modelId, + ...(apiKeyArn && { apiKeyArn }), + ...(temperature !== undefined && { temperature }), + ...(topP !== undefined && { topP }), + ...(topK !== undefined && { topK }), + ...(modelMaxTokens !== undefined && { maxTokens: modelMaxTokens }), + }, + }; break; default: - baseOpts.model = { bedrockModelConfig: { modelId } }; + baseOpts.model = { + bedrockModelConfig: { + modelId, + ...(temperature !== undefined && { temperature }), + ...(topP !== undefined && { topP }), + ...(modelMaxTokens !== undefined && { maxTokens: modelMaxTokens }), + }, + }; break; } } diff --git a/src/schema/schemas/agentcore-project.ts b/src/schema/schemas/agentcore-project.ts index 903db1f19..35a112fca 100644 --- a/src/schema/schemas/agentcore-project.ts +++ b/src/schema/schemas/agentcore-project.ts @@ -70,7 +70,7 @@ export type { ABTestMode, TargetRef, GatewayFilter, PerVariantOnlineEvaluationCo export { ABTestModeSchema, TargetRefSchema, GatewayFilterSchema } from './primitives/ab-test'; export type { HttpGatewayTarget } from './primitives/http-gateway'; export { HttpGatewayTargetSchema } from './primitives/http-gateway'; -export type { HarnessGatewayOutboundAuth, HarnessSpec, HarnessModelProvider } from './primitives/harness'; +export type { HarnessGatewayOutboundAuth, HarnessModel, HarnessSpec, HarnessModelProvider } from './primitives/harness'; export { GatewayOAuthGrantTypeSchema, HarnessGatewayOutboundAuthSchema,