Latest updated for semi-automated eval builder with JS verification

Author: olesho

evals/eval_builder_snapshots.py

@@ -0,0 +1,31 @@
+id: action-agent-accordion-001
+name: Expand Accordion Section
+description: Test clicking to expand an accordion panel
+enabled: true
+target:
+  url: https://jqueryui.com/accordion/
+  wait_for: networkidle
+  wait_timeout: 5000
+tool: action_agent
+timeout: 60000
+input:
+  objective: Click to expand the "Section 2" accordion panel
+  reasoning: Testing accordion expand/collapse interaction
+validation:
+  type: js-eval
+  js-eval:
+    script: verify.js
+    expected_result: true
+    timeout: 5000
+metadata:
+  tags:
+  - action
+  - accordion
+  - expand
+  - collapse
+  - ui
+  priority: high
+  timeout: 60000
+  retries: 2
+  flaky: false
+  owner: devtools-team

evals/native/data/js-verifier/action/accordion/task.yaml

@@ -0,0 +1,28 @@
+// Validation: Check if "Section 2" accordion panel is expanded
+// The objective is to verify that Section 2's content is visible (expanded)
+// The accordion is inside an iframe with class="demo-frame"
+
+// Wrap in an IIFE to avoid variable conflicts
+(() => {
+  // Find the iframe and access its content document
+  const iframe = document.querySelector('iframe.demo-frame');
+  const iframeDoc = iframe && iframe.contentDocument;
+
+  // Find all h3 headers in the accordion
+  const headers = iframeDoc && iframeDoc.querySelectorAll('#accordion h3');
+
+  // Find the Section 2 header
+  let section2Header = null;
+  if (headers) {
+    for (const header of headers) {
+      if (header.textContent.trim() === 'Section 2') {
+        section2Header = header;
+        break;
+      }
+    }
+  }
+
+  // Check if Section 2 header exists and has aria-expanded="true"
+  // This indicates the accordion panel is expanded
+  return section2Header && section2Header.getAttribute('aria-expanded') === 'true';
+})()

evals/native/data/js-verifier/action/accordion/verify.js

@@ -0,0 +1,33 @@
+id: action-agent-autocomplete-001
+name: Use Autocomplete Search
+description: Test typing in autocomplete field and selecting from suggestions
+enabled: true
+target:
+  url: https://jqueryui.com/autocomplete/
+  wait_for: networkidle
+  wait_timeout: 5000
+tool: action_agent
+timeout: 60000
+input:
+  objective: Type "Java" in the autocomplete field and select "JavaScript" from suggestions
+  reasoning: Testing autocomplete/typeahead interaction patterns
+validation:
+  type: js-eval
+  js-eval:
+    script: '// validation script
+
+      '
+    expected_result: true
+    timeout: 5000
+metadata:
+  tags:
+  - action
+  - autocomplete
+  - typeahead
+  - search
+  - suggestions
+  priority: high
+  timeout: 60000
+  retries: 2
+  flaky: false
+  owner: devtools-team

evals/native/data/js-verifier/action/autocomplete/task.yaml

@@ -0,0 +1,34 @@
+// Validation: Check if "JavaScript" was successfully entered/selected in the autocomplete
+// Objective: Type "Java" and select "JavaScript" from the autocomplete suggestions
+//
+// Since the autocomplete demo may be in an iframe or the main document,
+// we check both locations
+
+(() => {
+  // Helper function to check for the input value
+  const checkInput = (doc) => {
+    const input = doc.querySelector('#tags');
+    return input && input.value.trim() === 'JavaScript';
+  };
+
+  // Check main document first
+  if (checkInput(document)) {
+    return true;
+  }
+
+  // Check all iframes
+  const iframes = document.querySelectorAll('iframe');
+  for (const iframe of iframes) {
+    try {
+      if (iframe.contentDocument && checkInput(iframe.contentDocument)) {
+        return true;
+      }
+    } catch (e) {
+      // Cross-origin iframe, skip it
+      continue;
+    }
+  }
+
+  // Not found or value doesn't match
+  return false;
+})()

evals/native/data/js-verifier/action/autocomplete/verify.js

@@ -0,0 +1,31 @@
+id: action-agent-checkbox-001
+name: Toggle Newsletter Checkbox
+description: Test clicking checkbox elements for form options
+enabled: true
+target:
+  url: https://www.w3schools.com/html/tryit.asp?filename=tryhtml_checkbox
+  wait_for: networkidle
+  wait_timeout: 5000
+tool: action_agent
+timeout: 45000
+input:
+  objective: Click the checkbox labeled "I have a bike" to check it
+  reasoning: Testing interaction with checkbox form elements
+validation:
+  type: js-eval
+  js-eval:
+    script: verify.js
+    expected_result: true
+    timeout: 5000
+metadata:
+  tags:
+  - action
+  - checkbox
+  - form
+  - w3schools
+  - input
+  priority: high
+  timeout: 60000
+  retries: 2
+  flaky: false
+  owner: devtools-team

evals/native/data/js-verifier/action/checkbox-001/task.yaml

@@ -0,0 +1,8 @@
+// Check if the "I have a bike" checkbox is checked
+// The checkbox is inside the iframeResult iframe (W3Schools Try It editor)
+(() => {
+  const iframe = document.getElementById('iframeResult');
+  if (!iframe || !iframe.contentDocument) return false;
+  const checkbox = iframe.contentDocument.querySelector('input[type="checkbox"][value="Bike"]');
+  return checkbox ? checkbox.checked === true : false;
+})()

evals/native/data/js-verifier/action/checkbox-001/verify.js

@@ -0,0 +1,32 @@
+id: action-agent-checkbox-002
+name: Check Extra Cheese Checkbox
+description: Test checking a specific checkbox using the check method
+enabled: true
+target:
+  url: https://httpbin.org/forms/post
+  wait_for: networkidle
+  wait_timeout: 5000
+tool: action_agent
+timeout: 45000
+input:
+  objective: Find and check the "Extra Cheese" checkbox in the Pizza Toppings section
+  reasoning: Testing checkbox interaction functionality using check method
+  hint: Look for the Extra Cheese checkbox and use the check method to select it
+validation:
+  type: js-eval
+  js-eval:
+    script: verify.js
+    expected_result: true
+    timeout: 5000
+metadata:
+  tags:
+  - action
+  - checkbox
+  - check
+  - form
+  - httpbin
+  priority: high
+  timeout: 45000
+  retries: 2
+  flaky: false
+  owner: devtools-team

evals/native/data/js-verifier/action/checkbox-002/task.yaml

@@ -0,0 +1,6 @@
+// Check if the "Extra Cheese" checkbox is checked
+// Using IIFE to avoid variable redeclaration errors
+(() => {
+  const cheeseCheckbox = document.querySelector('input[type="checkbox"][name="topping"][value="cheese"]');
+  return cheeseCheckbox && cheeseCheckbox.checked === true;
+})()