From 747362ecbaefe2b2aa57158526dc3f819c4c8487 Mon Sep 17 00:00:00 2001 From: aliciapaz Date: Fri, 13 Mar 2026 17:22:20 -0600 Subject: [PATCH 1/2] Support multiple AI providers via OpenAI-compatible API Switch from OpenAI's proprietary Responses API to the standard Chat Completions API so any OpenAI-compatible provider (Minimax, Ollama, Groq, Together AI, etc.) can be used as the AI backend. - Replace client.responses.create() with client.chat.completions.create() - Add AI_API_KEY env var (with OPENAI_API_KEY fallback for existing users) - Add AI_BASE_URL env var to point at any compatible endpoint - Guard against null response content with a clear error message - Update README, gemspec, and generator instructions Closes #3 Co-Authored-By: Claude Opus 4.6 (1M context) --- README.md | 26 +++++++++++++------ agent_e2e.gemspec | 5 ++-- .../agent_e2e/install/install_generator.rb | 10 ++++--- .../agent_e2e/install/templates/ai.js | 12 +++++++-- .../agent_e2e/install/templates/config.js | 15 +++++++++-- 5 files changed, 51 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 23330ec..f819775 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Agent E2E -AI-powered end-to-end testing for Rails applications. An OpenAI agent drives a real Chromium browser via Playwright to execute natural-language test cases against your running app. +AI-powered end-to-end testing for Rails applications. An AI agent drives a real Chromium browser via Playwright to execute natural-language test cases against your running app. Works with OpenAI, Minimax, Ollama, and any OpenAI-compatible API. Write tests like plain English: @@ -25,7 +25,7 @@ The agent reads the page, decides what to click/fill/navigate, and reports pass/ - **Ruby** >= 3.1, **Rails** >= 7.0 - **Node.js** >= 18 -- **OpenAI API key** (GPT-4o or newer recommended) +- **AI API key** from any OpenAI-compatible provider (OpenAI, Minimax, Ollama, etc.) ## Installation @@ -60,12 +60,21 @@ This will: ### Step 3: Set up your environment -Add your OpenAI API key to your `.env` file in the Rails root: +Add your AI provider API key to your `.env` file in the Rails root: ```env -OPENAI_API_KEY=sk-proj-... +AI_API_KEY=sk-proj-... ``` +For non-OpenAI providers, also set the base URL and model: + +```env +AI_API_KEY=your-api-key +AI_BASE_URL=https://api.minimax.chat/v1 +AI_MODEL=your-model-name +``` + +> **Note:** `OPENAI_API_KEY` is still supported as a fallback for existing setups. > **Important:** Make sure `.env` is in your `.gitignore` (Rails apps typically ignore `/.env*` by default). ### Step 4: Create a QA test user @@ -154,8 +163,9 @@ All configuration is via environment variables. Set them in `.env` or pass them | Variable | Default | Description | |---|---|---| -| `OPENAI_API_KEY` | *(required)* | Your OpenAI API key | -| `AI_MODEL` | `gpt-5.1` | OpenAI model to use for the agent | +| `AI_API_KEY` | *(required)* | API key for your AI provider (`OPENAI_API_KEY` also supported) | +| `AI_BASE_URL` | `https://api.openai.com/v1` | Base URL for the AI API (change for Minimax, Ollama, etc.) | +| `AI_MODEL` | `gpt-4o` | Model to use for the agent | | `BASE_URL` | `http://localhost:3000` | Base URL of the app (overridden to port 3001 by `bin/e2e`) | | `MAX_STEPS` | `25` | Maximum steps per test before timeout | | `ACTION_TIMEOUT` | `8000` | Timeout in ms for each browser action | @@ -166,8 +176,8 @@ All configuration is via environment variables. Set them in `.env` or pass them ## How it works 1. The agent reads the current page (visible text + interactive controls) -2. Sends a snapshot to OpenAI with the test goal and action history -3. OpenAI returns the next action (click, fill, navigate, etc.) +2. Sends a snapshot to the AI provider with the test goal and action history +3. The AI returns the next action (click, fill, navigate, etc.) 4. The agent executes the action via Playwright 5. Repeats until the goal is done, fails, or hits the step limit 6. Loop detection aborts tests that get stuck cycling the same actions diff --git a/agent_e2e.gemspec b/agent_e2e.gemspec index 961829f..f0e75cd 100644 --- a/agent_e2e.gemspec +++ b/agent_e2e.gemspec @@ -8,9 +8,10 @@ Gem::Specification.new do |spec| spec.authors = ["Your Name"] spec.email = ["your@email.com"] - spec.summary = "AI-powered E2E testing for Rails using Playwright and OpenAI" + spec.summary = "AI-powered E2E testing for Rails using Playwright and any OpenAI-compatible API" spec.description = "Sets up an AI QA agent that drives a real browser with Playwright, " \ - "guided by OpenAI, to run natural-language E2E tests against your Rails app. " \ + "guided by any OpenAI-compatible API (OpenAI, Minimax, Ollama, etc.), " \ + "to run natural-language E2E tests against your Rails app. " \ "Includes letter_opener_web for email testing." spec.homepage = "https://github.com/your-org/agent_e2e" spec.license = "MIT" diff --git a/lib/generators/agent_e2e/install/install_generator.rb b/lib/generators/agent_e2e/install/install_generator.rb index 8f9559e..a080cf6 100644 --- a/lib/generators/agent_e2e/install/install_generator.rb +++ b/lib/generators/agent_e2e/install/install_generator.rb @@ -7,7 +7,7 @@ module Generators class InstallGenerator < Rails::Generators::Base source_root File.expand_path("templates", __dir__) - desc "Sets up AI-powered E2E testing with Playwright, OpenAI, and letter_opener_web" + desc "Sets up AI-powered E2E testing with Playwright, an AI provider (OpenAI-compatible), and letter_opener_web" def copy_agent_test_files say "Creating agent-tests/ directory...", :green @@ -114,8 +114,12 @@ def print_next_steps say "" say "Next steps:", :yellow say "" - say " 1. Add your OpenAI API key to .env:" - say " OPENAI_API_KEY=sk-..." + say " 1. Add your AI provider API key to .env:" + say " AI_API_KEY=sk-..." + say "" + say " For non-OpenAI providers, also set:" + say " AI_BASE_URL=https://api.your-provider.com/v1" + say " AI_MODEL=your-model-name" say "" say " 2. Add data-testid attributes to key UI elements:" say ' ' diff --git a/lib/generators/agent_e2e/install/templates/ai.js b/lib/generators/agent_e2e/install/templates/ai.js index be23a66..f59fcd5 100644 --- a/lib/generators/agent_e2e/install/templates/ai.js +++ b/lib/generators/agent_e2e/install/templates/ai.js @@ -97,9 +97,17 @@ export async function decideNextAction({ goal, snapshot, history, previousUrl }) Return ONLY valid JSON, nothing else.`; const resp = await callWithRetry(() => - client.responses.create({ model: MODEL, input: prompt }) + client.chat.completions.create({ + model: MODEL, + messages: [{ role: "user", content: prompt }], + }) ); - const raw = resp.output_text.trim().replace(/^```json\n?/, "").replace(/\n?```$/, ""); + const content = resp.choices[0]?.message?.content; + if (!content) { + throw new Error(`AI returned empty response (finish_reason: ${resp.choices[0]?.finish_reason})`); + } + + const raw = content.trim().replace(/^```json\n?/, "").replace(/\n?```$/, ""); return JSON.parse(raw); } diff --git a/lib/generators/agent_e2e/install/templates/config.js b/lib/generators/agent_e2e/install/templates/config.js index 6218c98..23d6672 100644 --- a/lib/generators/agent_e2e/install/templates/config.js +++ b/lib/generators/agent_e2e/install/templates/config.js @@ -6,10 +6,21 @@ import OpenAI from "openai"; const __dirname = dirname(fileURLToPath(import.meta.url)); dotenv.config({ path: resolve(__dirname, "../.env") }); -export const client = new OpenAI({ apiKey: process.env.OPENAI_API_KEY }); +const apiKey = process.env.AI_API_KEY || process.env.OPENAI_API_KEY; +if (!apiKey) { + console.error("Error: Set AI_API_KEY (or OPENAI_API_KEY) in your .env file."); + process.exit(1); +} + +const clientOptions = { apiKey }; +if (process.env.AI_BASE_URL) { + clientOptions.baseURL = process.env.AI_BASE_URL; +} + +export const client = new OpenAI(clientOptions); export const BASE_URL = process.env.BASE_URL || "http://localhost:3000"; export const MAX_STEPS = parseInt(process.env.MAX_STEPS || "25", 10); -export const MODEL = process.env.AI_MODEL || "gpt-5.1"; +export const MODEL = process.env.AI_MODEL || "gpt-4o"; export const ACTION_TIMEOUT = parseInt(process.env.ACTION_TIMEOUT || "8000", 10); export const TESTS_DIR = __dirname; From 53a78270a2fa75d18f2b017c050d8411579977b9 Mon Sep 17 00:00:00 2001 From: aliciapaz Date: Wed, 18 Mar 2026 19:21:30 -0600 Subject: [PATCH 2/2] Harden provider compatibility for AI-driven actions --- .../agent_e2e/install/templates/ai.js | 75 ++++++++++++++++--- .../agent_e2e/install/templates/browser.js | 4 + .../agent_e2e/install/templates/config.js | 5 +- 3 files changed, 72 insertions(+), 12 deletions(-) diff --git a/lib/generators/agent_e2e/install/templates/ai.js b/lib/generators/agent_e2e/install/templates/ai.js index f59fcd5..9ec4c4d 100644 --- a/lib/generators/agent_e2e/install/templates/ai.js +++ b/lib/generators/agent_e2e/install/templates/ai.js @@ -65,6 +65,7 @@ export async function decideNextAction({ goal, snapshot, history, previousUrl }) ${snapshot.controls.map(c => { let desc = `- ${c.tag} label="${c.label}"`; if (c.testid) desc += ` testid="${c.testid}"`; + if (c.name) desc += ` name="${c.name}"`; if (c.type) desc += ` type="${c.type}"`; if (c.role) desc += ` role="${c.role}"`; if (c.href) desc += ` href="${c.href}"`; @@ -85,6 +86,7 @@ export async function decideNextAction({ goal, snapshot, history, previousUrl }) {"type":"key_press","key":"Enter|Tab|Escape|..."} {"type":"select","testid":"...","value":"option text"} {"type":"select","label":"...","value":"option text"} + {"type":"select","name":"...","value":"option text"} {"type":"assert_text","text":"..."} {"type":"done","reason":"..."} {"type":"fail","reason":"..."} @@ -96,18 +98,69 @@ export async function decideNextAction({ goal, snapshot, history, previousUrl }) Return ONLY valid JSON, nothing else.`; - const resp = await callWithRetry(() => - client.chat.completions.create({ - model: MODEL, - messages: [{ role: "user", content: prompt }], - }) - ); + const parseAction = (content) => { + const raw = content.trim().replace(/^```json\n?/, "").replace(/\n?```$/, ""); - const content = resp.choices[0]?.message?.content; - if (!content) { - throw new Error(`AI returned empty response (finish_reason: ${resp.choices[0]?.finish_reason})`); + try { + return JSON.parse(raw); + } catch (_) { + let start = -1; + let depth = 0; + let inString = false; + let escaped = false; + + for (let i = 0; i < raw.length; i++) { + const ch = raw[i]; + if (escaped) { + escaped = false; + continue; + } + if (ch === "\\") { + escaped = true; + continue; + } + if (ch === '"') { + inString = !inString; + continue; + } + if (inString) continue; + if (ch === "{") { + if (depth === 0) start = i; + depth += 1; + } + if (ch === "}") { + depth -= 1; + if (depth === 0 && start >= 0) { + return JSON.parse(raw.slice(start, i + 1)); + } + } + } + + throw new Error("AI response did not contain valid JSON action payload"); + } + }; + + let parseError = null; + for (let attempt = 0; attempt < 2; attempt++) { + const currentPrompt = attempt === 0 ? prompt : `${prompt}\n\nYour last response was not valid JSON. Return exactly one valid JSON object and no other text.`; + const resp = await callWithRetry(() => + client.chat.completions.create({ + model: MODEL, + messages: [{ role: "user", content: currentPrompt }], + }) + ); + + const content = resp.choices[0]?.message?.content; + if (!content) { + throw new Error(`AI returned empty response (finish_reason: ${resp.choices[0]?.finish_reason})`); + } + + try { + return parseAction(content); + } catch (error) { + parseError = error; + } } - const raw = content.trim().replace(/^```json\n?/, "").replace(/\n?```$/, ""); - return JSON.parse(raw); + throw parseError || new Error("AI response could not be parsed"); } diff --git a/lib/generators/agent_e2e/install/templates/browser.js b/lib/generators/agent_e2e/install/templates/browser.js index a438a3b..5685348 100644 --- a/lib/generators/agent_e2e/install/templates/browser.js +++ b/lib/generators/agent_e2e/install/templates/browser.js @@ -39,6 +39,7 @@ const extractControls = () => { tag: el.tagName.toLowerCase(), label: labelFor(el), testid: el.getAttribute("data-testid"), + name: el.getAttribute("name"), type: el.getAttribute("type"), href: el.getAttribute("href"), role: el.getAttribute("role"), @@ -139,6 +140,9 @@ export async function executeAction(page, action, timeout) { } else if (action.label) { const el = await findInFrames(page, (ctx) => ctx.getByLabel(action.label)); await el.selectOption({ label: val }, { timeout }); + } else if (action.name) { + const el = await findInFrames(page, (ctx) => ctx.locator(`select[name="${action.name}"]`)); + await el.selectOption({ label: val }, { timeout }); } } else if (action.type === "assert_text" && action.text) { const el = await findInFrames(page, (ctx) => ctx.locator(`text=${action.text}`)); diff --git a/lib/generators/agent_e2e/install/templates/config.js b/lib/generators/agent_e2e/install/templates/config.js index 23d6672..a74b81a 100644 --- a/lib/generators/agent_e2e/install/templates/config.js +++ b/lib/generators/agent_e2e/install/templates/config.js @@ -14,7 +14,10 @@ if (!apiKey) { const clientOptions = { apiKey }; if (process.env.AI_BASE_URL) { - clientOptions.baseURL = process.env.AI_BASE_URL; + let baseURL = process.env.AI_BASE_URL.trim(); + baseURL = baseURL.replace(/\/$/, ""); + baseURL = baseURL.replace(/\/chat\/completions$/, ""); + clientOptions.baseURL = baseURL; } export const client = new OpenAI(clientOptions);