Merge pull request #46 from SentienceAPI/improve_agent3

SentienceDEV · SentienceDEV · commit e88dd35e6e30 · 2025-12-24T16:06:08.000-08:00
optimize agent.ts and stealth mode
diff --git a/package-lock.json b/package-lock.json
diff --git a/src/agent.ts b/src/agent.ts
@@ -152,8 +152,17 @@ export class SentienceAgent {
           throw new Error(`Snapshot failed: ${snap.error}`);
         }
 
+        // Apply element filtering based on goal
+        const filteredElements = this.filterElements(snap, goal);
+
+        // Create filtered snapshot
+        const filteredSnap: Snapshot = {
+          ...snap,
+          elements: filteredElements
+        };
+
         // 2. GROUND: Format elements for LLM context
-        const context = this.buildContext(snap, goal);
+        const context = this.buildContext(filteredSnap, goal);
 
         // 3. THINK: Query LLM for next action
         const llmResponse = await this.queryLLM(context, goal);
@@ -169,7 +178,7 @@ export class SentienceAgent {
         const actionStr = llmResponse.content.trim();
 
         // 4. EXECUTE: Parse and run action
-        const result = await this.executeAction(actionStr, snap);
+        const result = await this.executeAction(actionStr, filteredSnap);
 
         const durationMs = Date.now() - startTime;
         result.durationMs = durationMs;
@@ -217,14 +226,78 @@ export class SentienceAgent {
     throw new Error('Unexpected: loop should have returned or thrown');
   }
 
+  /**
+   * Filter elements from snapshot based on goal context.
+   * Applies goal-based keyword matching to boost relevant elements and filters out irrelevant ones.
+   */
+  private filterElements(snap: Snapshot, goal: string): Element[] {
+    let elements = snap.elements;
+
+    // If no goal provided, return all elements (up to limit)
+    if (!goal) {
+      return elements.slice(0, this.snapshotLimit);
+    }
+
+    const goalLower = goal.toLowerCase();
+
+    // Extract keywords from goal
+    const keywords = this.extractKeywords(goalLower);
+
+    // Boost elements matching goal keywords
+    const scoredElements: Array<[number, Element]> = [];
+    for (const el of elements) {
+      let score = el.importance;
+
+      // Boost if element text matches goal
+      if (el.text && keywords.some(kw => el.text!.toLowerCase().includes(kw))) {
+        score += 0.3;
+      }
+
+      // Boost if role matches goal intent
+      if (goalLower.includes('click') && el.visual_cues.is_clickable) {
+        score += 0.2;
+      }
+      if (goalLower.includes('type') && (el.role === 'textbox' || el.role === 'searchbox')) {
+        score += 0.2;
+      }
+      if (goalLower.includes('search')) {
+        // Filter out non-interactive elements for search tasks
+        if ((el.role === 'link' || el.role === 'img') && !el.visual_cues.is_primary) {
+          score -= 0.5;
+        }
+      }
+
+      scoredElements.push([score, el]);
+    }
+
+    // Re-sort by boosted score
+    scoredElements.sort((a, b) => b[0] - a[0]);
+    elements = scoredElements.map(([, el]) => el);
+
+    return elements.slice(0, this.snapshotLimit);
+  }
+
+  /**
+   * Extract meaningful keywords from goal text
+   */
+  private extractKeywords(text: string): string[] {
+    const stopwords = new Set([
+      'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for',
+      'of', 'with', 'by', 'from', 'as', 'is', 'was'
+    ]);
+    const words = text.split(/\s+/);
+    return words.filter(w => !stopwords.has(w) && w.length > 2);
+  }
+
   /**
    * Convert snapshot elements to token-efficient prompt string
    * Format: [ID] <role> "text" {cues} @ (x,y) (Imp:score)
+   * Note: elements are already filtered by filterElements() in act()
    */
   private buildContext(snap: Snapshot, goal: string): string {
     const lines: string[] = [];
 
-    for (const el of snap.elements.slice(0, this.snapshotLimit)) {
+    for (const el of snap.elements) {
       // Extract visual cues
       const cues: string[] = [];
       if (el.visual_cues.is_primary) cues.push('PRIMARY');
diff --git a/src/browser.ts b/src/browser.ts
@@ -104,10 +104,251 @@ export class SentienceBrowser {
     });
 
     this.page = this.context.pages()[0] || await this.context.newPage();
+    
+    // Apply context-level stealth patches (runs on every new page)
+    await this.context.addInitScript(() => {
+        // Early webdriver hiding - runs before any page script
+        // Use multiple strategies to completely hide webdriver
+        
+        // Strategy 1: Try to delete it first
+        try {
+            delete (navigator as any).webdriver;
+        } catch (e) {
+            // Property might not be deletable
+        }
+        
+        // Strategy 2: Redefine to return undefined and hide from enumeration
+        Object.defineProperty(navigator, 'webdriver', {
+            get: () => undefined,
+            configurable: true,
+            enumerable: false,
+            writable: false
+        });
+        
+        // Strategy 3: Override 'in' operator check
+        const originalHasOwnProperty = Object.prototype.hasOwnProperty;
+        Object.prototype.hasOwnProperty = function(prop: string | number | symbol) {
+            if (this === navigator && (prop === 'webdriver' || prop === 'Webdriver')) {
+                return false;
+            }
+            return originalHasOwnProperty.call(this, prop);
+        };
+    });
 
-    // 5. Apply Stealth (Basic)
+    // 5. Apply Comprehensive Stealth Patches
+    // Use both CDP (earlier) and addInitScript (backup) for maximum coverage
+    
+    // Strategy A: Use CDP to inject at the earliest possible moment
+    const client = await this.page.context().newCDPSession(this.page);
+    await client.send('Page.addScriptToEvaluateOnNewDocument', {
+      source: `
+        // Aggressive webdriver hiding - must run before ANY page script
+        Object.defineProperty(navigator, 'webdriver', {
+          get: () => undefined,
+          configurable: true,
+          enumerable: false
+        });
+        
+        // Override Object.getOwnPropertyDescriptor
+        const originalGetOwnPropertyDescriptor = Object.getOwnPropertyDescriptor;
+        Object.getOwnPropertyDescriptor = function(obj, prop) {
+          if (obj === navigator && (prop === 'webdriver' || prop === 'Webdriver')) {
+            return undefined;
+          }
+          return originalGetOwnPropertyDescriptor(obj, prop);
+        };
+        
+        // Override Object.keys
+        const originalKeys = Object.keys;
+        Object.keys = function(obj) {
+          const keys = originalKeys(obj);
+          if (obj === navigator) {
+            return keys.filter(k => k !== 'webdriver' && k !== 'Webdriver');
+          }
+          return keys;
+        };
+        
+        // Override Object.getOwnPropertyNames
+        const originalGetOwnPropertyNames = Object.getOwnPropertyNames;
+        Object.getOwnPropertyNames = function(obj) {
+          const names = originalGetOwnPropertyNames(obj);
+          if (obj === navigator) {
+            return names.filter(n => n !== 'webdriver' && n !== 'Webdriver');
+          }
+          return names;
+        };
+        
+        // Override 'in' operator check
+        const originalHasOwnProperty = Object.prototype.hasOwnProperty;
+        Object.prototype.hasOwnProperty = function(prop) {
+          if (this === navigator && (prop === 'webdriver' || prop === 'Webdriver')) {
+            return false;
+          }
+          return originalHasOwnProperty.call(this, prop);
+        };
+      `
+    });
+    
+    // Strategy B: Also use addInitScript as backup (runs after CDP but before page scripts)
     await this.page.addInitScript(() => {
-        Object.defineProperty(navigator, 'webdriver', { get: () => false });
+        // 1. Hide navigator.webdriver (comprehensive approach for advanced detection)
+        // Advanced detection checks for property descriptor, so we need multiple strategies
+        try {
+            // Strategy 1: Try to delete the property
+            delete (navigator as any).webdriver;
+        } catch (e) {
+            // Property might not be deletable, continue with redefine
+        }
+        
+        // Strategy 2: Redefine to return undefined (better than false)
+        // Also set enumerable: false to hide from Object.keys() checks
+        Object.defineProperty(navigator, 'webdriver', {
+            get: () => undefined,
+            configurable: true,
+            enumerable: false
+        });
+        
+        // Strategy 3: Override Object.getOwnPropertyDescriptor only for navigator.webdriver
+        // This prevents advanced detection that checks the property descriptor
+        const originalGetOwnPropertyDescriptor = Object.getOwnPropertyDescriptor;
+        Object.getOwnPropertyDescriptor = function(obj: any, prop: string | symbol) {
+            if (obj === navigator && (prop === 'webdriver' || prop === 'Webdriver')) {
+                return undefined;
+            }
+            return originalGetOwnPropertyDescriptor(obj, prop);
+        } as any;
+        
+        // Strategy 4: Hide from Object.keys() and Object.getOwnPropertyNames()
+        const originalKeys = Object.keys;
+        Object.keys = function(obj: any) {
+            const keys = originalKeys(obj);
+            if (obj === navigator) {
+                return keys.filter(k => k !== 'webdriver' && k !== 'Webdriver');
+            }
+            return keys;
+        } as any;
+        
+        // Strategy 5: Hide from Object.getOwnPropertyNames()
+        const originalGetOwnPropertyNames = Object.getOwnPropertyNames;
+        Object.getOwnPropertyNames = function(obj: any) {
+            const names = originalGetOwnPropertyNames(obj);
+            if (obj === navigator) {
+                return names.filter(n => n !== 'webdriver' && n !== 'Webdriver');
+            }
+            return names;
+        } as any;
+        
+        // Strategy 6: Override hasOwnProperty to hide from 'in' operator checks
+        const originalHasOwnProperty = Object.prototype.hasOwnProperty;
+        Object.prototype.hasOwnProperty = function(prop: string | number | symbol) {
+            if (this === navigator && (prop === 'webdriver' || prop === 'Webdriver')) {
+                return false;
+            }
+            return originalHasOwnProperty.call(this, prop);
+        };
+
+        // 2. Inject window.chrome object (required for Chrome detection)
+        if (typeof (window as any).chrome === 'undefined') {
+            (window as any).chrome = {
+                runtime: {},
+                loadTimes: function() {},
+                csi: function() {},
+                app: {}
+            };
+        }
+
+        // 3. Patch navigator.plugins (should have length > 0)
+        // Only patch if plugins array is empty (headless mode issue)
+        const originalPlugins = navigator.plugins;
+        if (originalPlugins.length === 0) {
+            // Create a PluginArray-like object with minimal plugins
+            const fakePlugins = [
+                { 
+                    name: 'Chrome PDF Plugin', 
+                    filename: 'internal-pdf-viewer', 
+                    description: 'Portable Document Format',
+                    length: 1,
+                    item: function() { return null; },
+                    namedItem: function() { return null; }
+                },
+                { 
+                    name: 'Chrome PDF Viewer', 
+                    filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', 
+                    description: '',
+                    length: 0,
+                    item: function() { return null; },
+                    namedItem: function() { return null; }
+                },
+                { 
+                    name: 'Native Client', 
+                    filename: 'internal-nacl-plugin', 
+                    description: '',
+                    length: 0,
+                    item: function() { return null; },
+                    namedItem: function() { return null; }
+                }
+            ];
+            
+            // Create PluginArray-like object (array-like but not a real array)
+            // This needs to behave like the real PluginArray for detection to pass
+            const pluginArray: any = {};
+            fakePlugins.forEach((plugin, index) => {
+                Object.defineProperty(pluginArray, index.toString(), {
+                    value: plugin,
+                    enumerable: true,
+                    configurable: true
+                });
+            });
+            
+            Object.defineProperty(pluginArray, 'length', {
+                value: fakePlugins.length,
+                enumerable: false,
+                configurable: false
+            });
+            
+            pluginArray.item = function(index: number) { 
+                return this[index] || null; 
+            };
+            pluginArray.namedItem = function(name: string) { 
+                for (let i = 0; i < this.length; i++) {
+                    if (this[i] && this[i].name === name) return this[i];
+                }
+                return null;
+            };
+            
+            // Make it iterable (for for...of loops)
+            pluginArray[Symbol.iterator] = function*() {
+                for (let i = 0; i < this.length; i++) {
+                    yield this[i];
+                }
+            };
+            
+            // Make it array-like for Array.from() and spread
+            Object.setPrototypeOf(pluginArray, Object.create(null));
+            
+            Object.defineProperty(navigator, 'plugins', {
+                get: () => pluginArray,
+                configurable: true,
+                enumerable: true
+            });
+        }
+
+        // 4. Ensure navigator.languages exists and has values
+        if (!navigator.languages || navigator.languages.length === 0) {
+            Object.defineProperty(navigator, 'languages', {
+                get: () => ['en-US', 'en'],
+                configurable: true
+            });
+        }
+
+        // 5. Patch permissions API (should exist)
+        if (!navigator.permissions) {
+            (navigator as any).permissions = {
+                query: async (parameters: PermissionDescriptor) => {
+                    return { state: 'granted', onchange: null } as PermissionStatus;
+                }
+            };
+        }
     });
     
     // Inject API Key if present
