Merge pull request #97 from SentienceAPI/optimize_llm2

add elements to step_end in Trace

Author: SentienceDEV

src/agent.ts

@@ -324,14 +324,15 @@ export class SentienceAgent {
           const postUrl = this.browser.getPage()?.url() || null;
 
           // Build step_end event using TraceEventBuilder
+          // Use snapWithDiff to include elements with diff_status in pre field
           const stepEndData = TraceEventBuilder.buildStepEndData({
             stepId,
             stepIndex: this.stepCount,
             goal,
             attempt,
             preUrl,
             postUrl,
-            snapshot: snap,
+            snapshot: snapWithDiff,
             llmResponse,
             result,
           });

src/tracing/types.ts

@@ -52,6 +52,7 @@ export interface TraceElement {
 export interface SnapshotInfo {
   url?: string;
   snapshot_digest?: string;
+  elements?: TraceElement[]; // Include elements with diff_status for diff overlay support
 }
 
 /**

src/utils/trace-event-builder.ts

@@ -177,6 +177,41 @@ export class TraceEventBuilder {
     const execData = this.buildExecutionData(result, snapshot);
     const verifyData = this.buildVerifyData(result, snapshot);
 
+    // Build elements data for pre field (include diff_status from snapshot)
+    // Normalize importance values to importance_score (0-1 range) per snapshot
+    const importanceValues = snapshot.elements.map(el => el.importance);
+    const minImportance = importanceValues.length > 0 ? Math.min(...importanceValues) : 0;
+    const maxImportance = importanceValues.length > 0 ? Math.max(...importanceValues) : 0;
+    const importanceRange = maxImportance - minImportance;
+
+    const preElements: TraceElement[] = snapshot.elements.map(el => {
+      // Compute normalized importance_score
+      let importanceScore: number;
+      if (importanceRange > 0) {
+        importanceScore = (el.importance - minImportance) / importanceRange;
+      } else {
+        importanceScore = 0.5;
+      }
+
+      return {
+        id: el.id,
+        role: el.role,
+        text: el.text,
+        bbox: el.bbox,
+        importance: el.importance,
+        importance_score: importanceScore,
+        visual_cues: el.visual_cues,
+        in_viewport: el.in_viewport,
+        is_occluded: el.is_occluded,
+        z_index: el.z_index,
+        rerank_index: el.rerank_index,
+        heuristic_index: el.heuristic_index,
+        ml_probability: el.ml_probability,
+        ml_score: el.ml_score,
+        diff_status: el.diff_status,
+      };
+    });
+
     return {
       v: 1,
       step_id: stepId,
@@ -186,6 +221,7 @@ export class TraceEventBuilder {
       pre: {
         url: preUrl,
         snapshot_digest: snapshotDigest,
+        elements: preElements, // Add elements array with diff_status
       },
       llm: llmData,
       exec: execData,