From d842ce15d96662714cb8872f184f46acb6cb0edd Mon Sep 17 00:00:00 2001 From: Patrick Bareiss Date: Tue, 26 May 2026 19:50:25 +0200 Subject: [PATCH 1/2] Add batched and extra params settings for the extension --- package.json | 116 ++++++++++++++------- src/requirements/requirementsOperations.ts | 3 +- src/requirements/requirementsUtils.ts | 27 ++++- 3 files changed, 105 insertions(+), 41 deletions(-) diff --git a/package.json b/package.json index 34c9dc07..19993c86 100644 --- a/package.json +++ b/package.json @@ -492,33 +492,39 @@ "default": 2, "description": "Number of retries test generation. WARNING: High values (over 3) may lead to increased costs and longer processing times." }, - "vectorcastTestExplorer.reqs2x.reorder": { + "vectorcastTestExplorer.reqs2x.batched": { "type": "boolean", "order": 7, "default": true, + "description": "Generate tests in batches (multiple requirements per LLM call). Disable for one-request-per-requirement (slower, simpler retries)." + }, + "vectorcastTestExplorer.reqs2x.reorder": { + "type": "boolean", + "order": 8, + "default": true, "description": "Reorder test cases exported to file (disable for easier investigation of incremental results)" }, "vectorcastTestExplorer.reqs2x.noTestExamples": { "type": "boolean", - "order": 8, + "order": 9, "default": false, "description": "Do not provide additional test examples to the LLM during test generation" }, "vectorcastTestExplorer.reqs2x.functionDefinitions": { "type": "boolean", - "order": 9, + "order": 10, "default": true, "description": "Supply function definitions to the language model during test generation. Disable for blackbox-style testing." }, "vectorcastTestExplorer.reqs2x.enableUutStubbing": { "type": "boolean", - "order": 10, + "order": 11, "default": true, "description": "Enable UUT stubbing during requirements-based test generation" }, "vectorcastTestExplorer.reqs2x.generationLanguage": { "type": "string", - "order": 11, + "order": 12, "default": "en", "enum": [ "af", "sq", "am", "ar", "hy", "as", "az", "be", "bn", "bs", "bg", "my", "ch", "hr", "cs", "da", "nl", "en", "et", "tl", "fi", "fr", "ka", "de", "gu", "ha", "he", "hi", "hu", "is", "ig", "id", "ga", "it", "jp", "kn", "kk", "km", "ko", "ky", "lo", "lv", "lt", "mk", "ms", "ml", "mt", "mn", "me", "ne", "no", "or", "fa", "pl", "pt", "pa", "ro", "ru", "sr", "si", "sk", "sl", "es", "sw", "sv", "tg", "ta", "te", "th", "tr", "uk", "ur", "uz", "vi", "cy", "xh", "yo", "zu" @@ -530,13 +536,13 @@ }, "vectorcastTestExplorer.reqs2x.modelCompatibilityMode": { "type": "boolean", - "order": 12, + "order": 13, "default": false, "description": "Enable model compatibility mode (runs Reqs2X tools without structured outputs which enables compatibility with more providers)." }, "vectorcastTestExplorer.reqs2x.provider": { "type": "string", - "order": 13, + "order": 14, "default": "openai", "enum": ["openai", "azure_openai", "anthropic", "litellm", "azure_apim", "openai_at"], "enumDescriptions": ["OpenAI", "Azure OpenAI", "Anthropic", "LiteLLM", "Azure APIM (Advanced)", "OpenAI Access Token (Advanced)"], @@ -544,171 +550,211 @@ }, "vectorcastTestExplorer.reqs2x.openai.apiKey": { "type": "string", - "order": 14, + "order": 15, "default": "", "description": "OpenAI API key" }, "vectorcastTestExplorer.reqs2x.openai.modelName": { "type": "string", - "order": 15, + "order": 16, "default": "", "description": "OpenAI model name" }, "vectorcastTestExplorer.reqs2x.openai.reasoningModelName": { "type": "string", - "order": 16, + "order": 17, "default": "", "description": "Optional OpenAI reasoning model name used for specialized reasoning subtasks." }, "vectorcastTestExplorer.reqs2x.openai.baseUrl": { "type": "string", - "order": 17, + "order": 18, "default": "", "description": "OpenAI base URL (optional, set this to use OpenAI-compatible providers such as Ollama, vLLM or Bedrock)" }, "vectorcastTestExplorer.reqs2x.azure.baseUrl": { "type": "string", - "order": 18, + "order": 19, "default": "", "description": "Azure OpenAI endpoint/base URL, e.g., https://my-example-instance.openai.azure.com" }, "vectorcastTestExplorer.reqs2x.azure.apiKey": { "type": "string", - "order": 19, + "order": 20, "default": "", "description": "Azure OpenAI API key" }, "vectorcastTestExplorer.reqs2x.azure.deployment": { "type": "string", - "order": 20, + "order": 21, "default": "", "description": "Azure OpenAI deployment name, e.g., my-custom-gpt-4o-deployment" }, "vectorcastTestExplorer.reqs2x.azure.modelName": { "type": "string", - "order": 21, + "order": 22, "default": "", "description": "Azure OpenAI model name, e.g., gpt-4o" }, "vectorcastTestExplorer.reqs2x.azure.reasoningModelName": { "type": "string", - "order": 22, + "order": 23, "default": "", "description": "Optional Azure OpenAI reasoning model name used for specialized reasoning subtasks." }, "vectorcastTestExplorer.reqs2x.azure.reasoningDeployment": { "type": "string", - "order": 23, + "order": 24, "default": "", "description": "Azure OpenAI reasoning deployment name (required if Azure reasoning model name is provided)." }, "vectorcastTestExplorer.reqs2x.azure.apiVersion": { "type": "string", - "order": 24, + "order": 25, "default": "2024-12-01-preview", "description": "Azure OpenAI API version" }, "vectorcastTestExplorer.reqs2x.anthropic.apiKey": { "type": "string", - "order": 25, + "order": 26, "default": "", "description": "Anthropic API key" }, "vectorcastTestExplorer.reqs2x.anthropic.modelName": { "type": "string", - "order": 26, + "order": 27, "default": "", "description": "Anthropic model name" }, "vectorcastTestExplorer.reqs2x.anthropic.reasoningModelName": { "type": "string", - "order": 27, + "order": 28, "default": "", "description": "Optional Anthropic reasoning model name used for specialized reasoning subtasks." }, "vectorcastTestExplorer.reqs2x.litellm.modelName": { "type": "string", - "order": 28, + "order": 29, "default": "", "description": "LiteLLM model name (prefix with provider used, e.g., openai/gpt-4o or azure/o3-mini)" }, "vectorcastTestExplorer.reqs2x.litellm.reasoningModelName": { "type": "string", - "order": 29, + "order": 30, "default": "", "description": "Optional LiteLLM reasoning model name used for specialized reasoning subtasks." }, "vectorcastTestExplorer.reqs2x.litellm.providerEnvVars": { "type": "string", - "order": 30, + "order": 31, "default": "", "description": "Environment variables to set when running LiteLLM, e.g., OPENAI_API_KEY=xxxx. Multiple variables can be separated by commas, e.g., OPENAI_API_KEY=xxxx,ANOTHER_ENV_VAR=yyyy. For a list of variables to set for a specific LiteLLM-compatible provider, see https://docs.litellm.ai/docs/providers." }, "vectorcastTestExplorer.reqs2x.azureApim.subscriptionKey": { "type": "string", - "order": 31, + "order": 32, "default": "", "description": "[Advanced] Azure APIM subscription key (sent as the Ocp-Apim-Subscription-Key header)" }, "vectorcastTestExplorer.reqs2x.azureApim.baseUrl": { "type": "string", - "order": 32, + "order": 33, "default": "", "description": "[Advanced] Azure APIM gateway base URL, e.g., https://some-domain.azure-api.net/v1" }, "vectorcastTestExplorer.reqs2x.azureApim.modelName": { "type": "string", - "order": 33, + "order": 34, "default": "", "description": "[Advanced] Model name for the APIM-proxied LLM, e.g., gpt-4.1" }, "vectorcastTestExplorer.reqs2x.azureApim.apiKey": { "type": "string", - "order": 34, + "order": 35, "default": "", "description": "[Advanced] Optional downstream Azure OpenAI API key (forwarded when the APIM policy requires it)" }, "vectorcastTestExplorer.reqs2x.azureApim.reasoningModelName": { "type": "string", - "order": 35, + "order": 36, "default": "", "description": "[Advanced] Optional reasoning model name used for specialized reasoning subtasks (Azure APIM)." }, "vectorcastTestExplorer.reqs2x.openaiAt.modelName": { "type": "string", - "order": 36, + "order": 37, "default": "", "description": "[Advanced] OpenAI Access Token model name" }, "vectorcastTestExplorer.reqs2x.openaiAt.modelUrl": { "type": "string", - "order": 37, + "order": 38, "default": "", "description": "[Advanced] OpenAI Access Token model URL" }, "vectorcastTestExplorer.reqs2x.openaiAt.authUrl": { "type": "string", - "order": 38, + "order": 39, "default": "", "description": "[Advanced] OpenAI Access Token authentication URL" }, "vectorcastTestExplorer.reqs2x.openaiAt.appKey": { "type": "string", - "order": 39, + "order": 40, "default": "", "description": "[Advanced] OpenAI Access Token application key" }, "vectorcastTestExplorer.reqs2x.openaiAt.appSecret": { "type": "string", - "order": 40, + "order": 41, "default": "", "description": "[Advanced] OpenAI Access Token application secret" }, "vectorcastTestExplorer.reqs2x.openaiAt.reasoningModelName": { "type": "string", - "order": 41, + "order": 42, "default": "", "description": "[Advanced] Optional reasoning model name used for specialized reasoning subtasks (OpenAI Access Token)." + }, + "vectorcastTestExplorer.reqs2x.extraModelParamsTemplate": { + "type": "string", + "order": 43, + "default": "", + "enum": ["", "vllm-thinking-off", "gemini-reasoning-minimal", "gpt5-reasoning-minimal"], + "enumDescriptions": [ + "None", + "vLLM-hosted reasoning models: disable chat-template thinking flag (Qwen3, GLM-4.6, DeepSeek-V3.1, IBM Granite 3.2)", + "Google Gemini via OpenAI-compatible endpoint: reasoning_effort=minimal", + "OpenAI gpt-5: reasoning_effort=minimal" + ], + "description": "[Advanced] Preconfigured extra model parameter template for the main role. See Reqs2x CLI docs for details." + }, + "vectorcastTestExplorer.reqs2x.extraModelParamsJson": { + "type": "string", + "order": 44, + "default": "", + "editPresentation": "multilineText", + "description": "[Advanced] Custom JSON blob deep-merged on top of the template (or used standalone), e.g., {\"temperature\": 0.6, \"max_tokens\": 16000}. Passed straight to the SDK — invalid keys fail at request time." + }, + "vectorcastTestExplorer.reqs2x.reasoningExtraModelParamsTemplate": { + "type": "string", + "order": 45, + "default": "", + "enum": ["", "vllm-thinking-off", "gemini-reasoning-minimal", "gpt5-reasoning-minimal"], + "enumDescriptions": [ + "None", + "vLLM-hosted reasoning models: disable chat-template thinking flag (Qwen3, GLM-4.6, DeepSeek-V3.1, IBM Granite 3.2)", + "Google Gemini via OpenAI-compatible endpoint: reasoning_effort=minimal", + "OpenAI gpt-5: reasoning_effort=minimal" + ], + "description": "[Advanced] Extra model parameter template for the reasoning role (overrides the main-role template for reasoning subtasks)." + }, + "vectorcastTestExplorer.reqs2x.reasoningExtraModelParamsJson": { + "type": "string", + "order": 46, + "default": "", + "editPresentation": "multilineText", + "description": "[Advanced] Custom JSON for the reasoning role. Same semantics as extraModelParamsJson." } } } diff --git a/src/requirements/requirementsOperations.ts b/src/requirements/requirementsOperations.ts index 91451ecb..16584ee5 100644 --- a/src/requirements/requirementsOperations.ts +++ b/src/requirements/requirementsOperations.ts @@ -380,6 +380,7 @@ export async function generateTestsFromRequirements( const reorder = config.get("reorder", true); const funcDefs = config.get("functionDefinitions", true); const allowUUTStubs = config.get("enableUutStubbing", true); + const batched = config.get("batched", true); const retries = config.get("retries", 2); if (retries < 1) { @@ -402,7 +403,7 @@ export async function generateTestsFromRequirements( tstPath, "--retries", retries.toString(), - "--batched", + batched ? "--batched" : "--no-batched", ...(decomposeRequirements ? [] : ["--no-requirement-decomposition"]), ...(noTestExamples ? ["--no-test-examples"] : []), ...(reorder ? [] : ["--no-reorder"]), diff --git a/src/requirements/requirementsUtils.ts b/src/requirements/requirementsUtils.ts index 7eb8d682..3bef68df 100644 --- a/src/requirements/requirementsUtils.ts +++ b/src/requirements/requirementsUtils.ts @@ -329,6 +329,21 @@ export interface LLMProviderSettingsResult { missing: string[]; } +function applyExtraModelParams( + config: vscode.WorkspaceConfiguration, + processEnv: Record +): void { + const tmpl = config.get("extraModelParamsTemplate", ""); + if (tmpl) processEnv.VCAST_REQS2X_EXTRA_MODEL_PARAMS_TEMPLATE = tmpl; + const json = config.get("extraModelParamsJson", ""); + if (json) processEnv.VCAST_REQS2X_EXTRA_MODEL_PARAMS_JSON = json; + const rTmpl = config.get("reasoningExtraModelParamsTemplate", ""); + if (rTmpl) + processEnv.VCAST_REQS2X_REASONING_EXTRA_MODEL_PARAMS_TEMPLATE = rTmpl; + const rJson = config.get("reasoningExtraModelParamsJson", ""); + if (rJson) processEnv.VCAST_REQS2X_REASONING_EXTRA_MODEL_PARAMS_JSON = rJson; +} + export function isLLMProviderEnvironmentUsable(): Promise<{ usable: boolean; problem: string | null; @@ -340,13 +355,13 @@ export function isLLMProviderEnvironmentUsable(): Promise<{ if (v) processEnv[k] = v; } - if ( - vscode.workspace - .getConfiguration("vectorcastTestExplorer.reqs2x") - .get("modelCompatibilityMode", false) - ) { + const reqs2xConfig = vscode.workspace.getConfiguration( + "vectorcastTestExplorer.reqs2x" + ); + if (reqs2xConfig.get("modelCompatibilityMode", false)) { processEnv.VCAST_REQS2X_MODEL_COMPATIBILITY_MODE = "1"; } + applyExtraModelParams(reqs2xConfig, processEnv); const proc = spawn(LLM2CHECK_EXECUTABLE_PATH, ["--json"], { env: processEnv, @@ -615,6 +630,8 @@ export async function createProcessEnvironment(): Promise { processEnv.VCAST_REQS2X_MODEL_COMPATIBILITY_MODE = "1"; } + applyExtraModelParams(config, processEnv); + // Return the constructed environment return processEnv; } From a8622db53e69486e516fc7bd42ff634a26a2c2a2 Mon Sep 17 00:00:00 2001 From: Patrick Bareiss Date: Tue, 26 May 2026 20:27:53 +0200 Subject: [PATCH 2/2] Add source file encoding setting --- package.json | 74 +++++++++++++++------------ src/requirements/requirementsUtils.ts | 5 ++ 2 files changed, 45 insertions(+), 34 deletions(-) diff --git a/package.json b/package.json index 19993c86..a021b8b5 100644 --- a/package.json +++ b/package.json @@ -534,15 +534,21 @@ ], "description": "Generation language for requirements and test generation responses" }, + "vectorcastTestExplorer.reqs2x.sourceFileEncoding": { + "type": "string", + "order": 13, + "default": "", + "description": "Encoding used by Reqs2X to read source files and existing test scripts (e.g., cp1252, gbk). If empty, Reqs2X tries UTF-8 first and falls back to automatic charset detection." + }, "vectorcastTestExplorer.reqs2x.modelCompatibilityMode": { "type": "boolean", - "order": 13, + "order": 14, "default": false, "description": "Enable model compatibility mode (runs Reqs2X tools without structured outputs which enables compatibility with more providers)." }, "vectorcastTestExplorer.reqs2x.provider": { "type": "string", - "order": 14, + "order": 15, "default": "openai", "enum": ["openai", "azure_openai", "anthropic", "litellm", "azure_apim", "openai_at"], "enumDescriptions": ["OpenAI", "Azure OpenAI", "Anthropic", "LiteLLM", "Azure APIM (Advanced)", "OpenAI Access Token (Advanced)"], @@ -550,175 +556,175 @@ }, "vectorcastTestExplorer.reqs2x.openai.apiKey": { "type": "string", - "order": 15, + "order": 16, "default": "", "description": "OpenAI API key" }, "vectorcastTestExplorer.reqs2x.openai.modelName": { "type": "string", - "order": 16, + "order": 17, "default": "", "description": "OpenAI model name" }, "vectorcastTestExplorer.reqs2x.openai.reasoningModelName": { "type": "string", - "order": 17, + "order": 18, "default": "", "description": "Optional OpenAI reasoning model name used for specialized reasoning subtasks." }, "vectorcastTestExplorer.reqs2x.openai.baseUrl": { "type": "string", - "order": 18, + "order": 19, "default": "", "description": "OpenAI base URL (optional, set this to use OpenAI-compatible providers such as Ollama, vLLM or Bedrock)" }, "vectorcastTestExplorer.reqs2x.azure.baseUrl": { "type": "string", - "order": 19, + "order": 20, "default": "", "description": "Azure OpenAI endpoint/base URL, e.g., https://my-example-instance.openai.azure.com" }, "vectorcastTestExplorer.reqs2x.azure.apiKey": { "type": "string", - "order": 20, + "order": 21, "default": "", "description": "Azure OpenAI API key" }, "vectorcastTestExplorer.reqs2x.azure.deployment": { "type": "string", - "order": 21, + "order": 22, "default": "", "description": "Azure OpenAI deployment name, e.g., my-custom-gpt-4o-deployment" }, "vectorcastTestExplorer.reqs2x.azure.modelName": { "type": "string", - "order": 22, + "order": 23, "default": "", "description": "Azure OpenAI model name, e.g., gpt-4o" }, "vectorcastTestExplorer.reqs2x.azure.reasoningModelName": { "type": "string", - "order": 23, + "order": 24, "default": "", "description": "Optional Azure OpenAI reasoning model name used for specialized reasoning subtasks." }, "vectorcastTestExplorer.reqs2x.azure.reasoningDeployment": { "type": "string", - "order": 24, + "order": 25, "default": "", "description": "Azure OpenAI reasoning deployment name (required if Azure reasoning model name is provided)." }, "vectorcastTestExplorer.reqs2x.azure.apiVersion": { "type": "string", - "order": 25, + "order": 26, "default": "2024-12-01-preview", "description": "Azure OpenAI API version" }, "vectorcastTestExplorer.reqs2x.anthropic.apiKey": { "type": "string", - "order": 26, + "order": 27, "default": "", "description": "Anthropic API key" }, "vectorcastTestExplorer.reqs2x.anthropic.modelName": { "type": "string", - "order": 27, + "order": 28, "default": "", "description": "Anthropic model name" }, "vectorcastTestExplorer.reqs2x.anthropic.reasoningModelName": { "type": "string", - "order": 28, + "order": 29, "default": "", "description": "Optional Anthropic reasoning model name used for specialized reasoning subtasks." }, "vectorcastTestExplorer.reqs2x.litellm.modelName": { "type": "string", - "order": 29, + "order": 30, "default": "", "description": "LiteLLM model name (prefix with provider used, e.g., openai/gpt-4o or azure/o3-mini)" }, "vectorcastTestExplorer.reqs2x.litellm.reasoningModelName": { "type": "string", - "order": 30, + "order": 31, "default": "", "description": "Optional LiteLLM reasoning model name used for specialized reasoning subtasks." }, "vectorcastTestExplorer.reqs2x.litellm.providerEnvVars": { "type": "string", - "order": 31, + "order": 32, "default": "", "description": "Environment variables to set when running LiteLLM, e.g., OPENAI_API_KEY=xxxx. Multiple variables can be separated by commas, e.g., OPENAI_API_KEY=xxxx,ANOTHER_ENV_VAR=yyyy. For a list of variables to set for a specific LiteLLM-compatible provider, see https://docs.litellm.ai/docs/providers." }, "vectorcastTestExplorer.reqs2x.azureApim.subscriptionKey": { "type": "string", - "order": 32, + "order": 33, "default": "", "description": "[Advanced] Azure APIM subscription key (sent as the Ocp-Apim-Subscription-Key header)" }, "vectorcastTestExplorer.reqs2x.azureApim.baseUrl": { "type": "string", - "order": 33, + "order": 34, "default": "", "description": "[Advanced] Azure APIM gateway base URL, e.g., https://some-domain.azure-api.net/v1" }, "vectorcastTestExplorer.reqs2x.azureApim.modelName": { "type": "string", - "order": 34, + "order": 35, "default": "", "description": "[Advanced] Model name for the APIM-proxied LLM, e.g., gpt-4.1" }, "vectorcastTestExplorer.reqs2x.azureApim.apiKey": { "type": "string", - "order": 35, + "order": 36, "default": "", "description": "[Advanced] Optional downstream Azure OpenAI API key (forwarded when the APIM policy requires it)" }, "vectorcastTestExplorer.reqs2x.azureApim.reasoningModelName": { "type": "string", - "order": 36, + "order": 37, "default": "", "description": "[Advanced] Optional reasoning model name used for specialized reasoning subtasks (Azure APIM)." }, "vectorcastTestExplorer.reqs2x.openaiAt.modelName": { "type": "string", - "order": 37, + "order": 38, "default": "", "description": "[Advanced] OpenAI Access Token model name" }, "vectorcastTestExplorer.reqs2x.openaiAt.modelUrl": { "type": "string", - "order": 38, + "order": 39, "default": "", "description": "[Advanced] OpenAI Access Token model URL" }, "vectorcastTestExplorer.reqs2x.openaiAt.authUrl": { "type": "string", - "order": 39, + "order": 40, "default": "", "description": "[Advanced] OpenAI Access Token authentication URL" }, "vectorcastTestExplorer.reqs2x.openaiAt.appKey": { "type": "string", - "order": 40, + "order": 41, "default": "", "description": "[Advanced] OpenAI Access Token application key" }, "vectorcastTestExplorer.reqs2x.openaiAt.appSecret": { "type": "string", - "order": 41, + "order": 42, "default": "", "description": "[Advanced] OpenAI Access Token application secret" }, "vectorcastTestExplorer.reqs2x.openaiAt.reasoningModelName": { "type": "string", - "order": 42, + "order": 43, "default": "", "description": "[Advanced] Optional reasoning model name used for specialized reasoning subtasks (OpenAI Access Token)." }, "vectorcastTestExplorer.reqs2x.extraModelParamsTemplate": { "type": "string", - "order": 43, + "order": 44, "default": "", "enum": ["", "vllm-thinking-off", "gemini-reasoning-minimal", "gpt5-reasoning-minimal"], "enumDescriptions": [ @@ -731,14 +737,14 @@ }, "vectorcastTestExplorer.reqs2x.extraModelParamsJson": { "type": "string", - "order": 44, + "order": 45, "default": "", "editPresentation": "multilineText", "description": "[Advanced] Custom JSON blob deep-merged on top of the template (or used standalone), e.g., {\"temperature\": 0.6, \"max_tokens\": 16000}. Passed straight to the SDK — invalid keys fail at request time." }, "vectorcastTestExplorer.reqs2x.reasoningExtraModelParamsTemplate": { "type": "string", - "order": 45, + "order": 46, "default": "", "enum": ["", "vllm-thinking-off", "gemini-reasoning-minimal", "gpt5-reasoning-minimal"], "enumDescriptions": [ @@ -751,7 +757,7 @@ }, "vectorcastTestExplorer.reqs2x.reasoningExtraModelParamsJson": { "type": "string", - "order": 46, + "order": 47, "default": "", "editPresentation": "multilineText", "description": "[Advanced] Custom JSON for the reasoning role. Same semantics as extraModelParamsJson." diff --git a/src/requirements/requirementsUtils.ts b/src/requirements/requirementsUtils.ts index 3bef68df..766f5752 100644 --- a/src/requirements/requirementsUtils.ts +++ b/src/requirements/requirementsUtils.ts @@ -632,6 +632,11 @@ export async function createProcessEnvironment(): Promise { applyExtraModelParams(config, processEnv); + const sourceFileEncoding = config.get("sourceFileEncoding", ""); + if (sourceFileEncoding) { + processEnv.VCAST_REQS2X_SOURCE_FILE_ENCODING = sourceFileEncoding; + } + // Return the constructed environment return processEnv; }