update the project

Author: rkvalandas

Readme.md

@@ -13,25 +13,14 @@
 from langgraph.prebuilt import ToolNode, tools_condition
 
 class AgentState(TypedDict):
-    """State for the browser agent."""
     messages: Annotated[list, add_messages]
 
 async def create_agent(api_key: str):
-    """Create and return a LangGraph agent with specified tools."""
-    
-    # Initialize Groq model
-    # llm = ChatGroq(
-    #     model="qwen-qwq-32b",
-    #     temperature=0,
-    #     max_tokens=2048,
-    #     api_key=api_key
-    # )
-
-    # For standard OpenAI API
+    # Initialize Azure OpenAI
     llm = AzureChatOpenAI(
         model_name="gpt-4o",
         openai_api_key=api_key,
-        temperature=1.0,
+        temperature=0,
         api_version="2024-12-01-preview",
         azure_endpoint= os.getenv("AZURE_ENDPOINT"),
     )
@@ -42,7 +31,7 @@ async def create_agent(api_key: str):
 
     # Create the system prompt
     system_prompt = """
-You are an expert AI agent that controls a web browser with precision, robust navigation awareness, and human-like interaction. Your goal is to reliably complete user tasks by analyzing, navigating, and interacting with web pages.
+You are an expert AI agent that controls a web browser with precision, robust navigation awareness. Your goal is to reliably complete user tasks by analyzing, navigating, and interacting with web pages.
 
 ## CORE TOOLS & CAPABILITIES
 - analyze_page: Extracts all visible elements and text. Use IMMEDIATELY after any navigation, click, or popup event. This tool builds an internal map of all interactive elements with numbered IDs.
@@ -53,6 +42,7 @@ async def create_agent(api_key: str):
 - navigate: Opens a URL. Ensures protocol (https://) is added if missing. ALWAYS follow with analyze_page().
 - go_back: Goes to previous page in browser history. ALWAYS follow with analyze_page().
 - scroll: Scrolls viewport in specified direction ("down", "up", "top", "bottom"). Use when elements are off-screen, for infinite scrolling pages, or when loading more content.
+- ask_user: Requests information directly from the user. Use JSON format: ask_user('{"prompt":"What is your username?","type":"text"}') or ask_user('{"prompt":"Choose payment method","type":"choice","choices":["Credit","PayPal"]}') or ask_user('{"prompt":"Enter password","type":"password"}').
 
 ## DYNAMIC CONTENT INTERACTION
 - For infinite scroll pages: Use scroll("down") repeatedly, running analyze_page() after each scroll to capture newly loaded content.
@@ -98,11 +88,26 @@ async def create_agent(api_key: str):
 - Filters and sorting: Use these controls to narrow down large sets of results to find specific items.
 - Pagination: Look for pagination controls when dealing with multi-page content and navigate between pages as needed.
 - Browser history: Use go_back() strategically to return to previously visited pages rather than re-navigating from the start.
+
+## USER INTERACTION WITH ASK_USER TOOL
+Use the ask_user tool to collect information or decisions from users during tasks:
+
+- **Syntax**: `ask_user('{"prompt":"Question?","type":"text|password|choice","choices":["Option1","Option2"],"default":"Default"}')`
+- **Common Uses**:
+  1. Authentication: `ask_user('{"prompt":"Enter password","type":"password"}')`
+  2. Choices: `ask_user('{"prompt":"Select payment method","type":"choice","choices":["Credit","PayPal"]}')`
+  3. Confirmations: `ask_user('{"prompt":"Proceed with purchase?","type":"choice","choices":["Yes","No"]}')`
+  4. Form data: `ask_user('{"prompt":"Enter shipping address","type":"text"}')`
+  5. CAPTCHA assistance: `ask_user('{"prompt":"Please help with CAPTCHA verification"}')`
+
+Always ask for ONE piece of information at a time, use clear prompts, and choose appropriate input types.
+
+Response : [Provide the specific information requested by the user, including any data, facts, or details discovered. State "Goal completed successfully" when done.]
+
 """
 
     # Create async node for chatbot
     async def chatbot(state: AgentState):
-        """Process messages and generate a response."""
         # If no message exists, return no change to state
         if not state.get("messages", []):
             return {"messages": []}
@@ -132,13 +137,12 @@ async def chatbot(state: AgentState):
     memory = MemorySaver()
     graph = graph_builder.compile(checkpointer=memory)
 
-    # Wrap the graph with an interface similar to the original AgentExecutor
+    # Wrap the graph with an interface
     class LangGraphAgent:
         def __init__(self, graph):
             self.graph = graph
 
         async def ainvoke(self, input_text, thread_id="main"):
-            """Invoke the agent with the input text asynchronously."""
             config = {"configurable": {"thread_id": thread_id}}
 
             # Start with system message and user input
@@ -152,18 +156,17 @@ async def ainvoke(self, input_text, thread_id="main"):
             # Run the graph asynchronously
             result = await self.graph.ainvoke(state, config)
 
-            # Format the result to match the expected format
+            # Format the result
             output = result["messages"][-1].content
 
-            # Create a result similar to AgentExecutor's format
+            # Create a result
             return {
                 "input": input_text,
                 "output": output,
                 "messages": result["messages"]
             }
 
         def invoke(self, input_text, thread_id="main"):
-            """Synchronous wrapper for backwards compatibility."""
             # Get the current event loop or create a new one
             try:
                 loop = asyncio.get_event_loop()
@@ -175,7 +178,6 @@ def invoke(self, input_text, thread_id="main"):
             return loop.run_until_complete(self.ainvoke(input_text, thread_id))
 
         async def astream(self, input_text, thread_id="main"):
-            """Stream the agent's thinking process asynchronously."""
             config = {"configurable": {"thread_id": thread_id}}
 
             # Start with system message and user input
@@ -192,7 +194,6 @@ async def astream(self, input_text, thread_id="main"):
                     event["messages"][-1].pretty_print()
 
         def stream(self, input_text, thread_id="main"):
-            """Synchronous streaming wrapper for backwards compatibility."""
             # Get the current event loop or create a new one
             try:
                 loop = asyncio.get_event_loop()

agent/agent.py

@@ -175,31 +175,91 @@ async def analyze_page():
                 
                 // Find all modal/dialog/popup elements
                 function findPopups() {
-                    // Common selectors for modals/dialogs/popups
+                    // Expanded selectors for modals/dialogs/popups
                     const selectors = [
-                        '[role=dialog]',
-                        '[aria-modal="true"]',
-                        '[data-modal="true"]',
-                        '.modal',
-                        '.dialog',
-                        '.popup',
-                        '.overlay',
-                        '.pop-up',
-                        '.ant-modal', // Ant Design
-                        '.MuiDialog-root', // Material UI
+                        // ARIA roles and attributes
+                        '[role=dialog]', '[role=alertdialog]', '[role=drawer]', '[role=tooltip]', '[role=menu]',
+                        '[aria-modal="true"]', '[aria-haspopup="dialog"]', '[aria-haspopup="menu"]',
+                        
+                        // Data attributes
+                        '[data-modal="true"]', '[data-popup="true"]', '[data-dialog="true"]', '[data-overlay="true"]',
+                        
+                        // Common class patterns
+                        '.modal', '.dialog', '.popup', '.overlay', '.pop-up', '.popover', '.tooltip', '.drawer',
+                        '.toast', '.notification', '.alert-box',
+                        
+                        // Framework-specific selectors
+                        '.ant-modal', '.ant-drawer', '.ant-popover', // Ant Design
+                        '.MuiDialog-root', '.MuiDrawer-root', '.MuiPopover-root', // Material UI
                         '.ReactModal__Content', // React Modal
-                        '[class*="modal"]',
-                        '[class*="dialog"]',
-                        '[class*="popup"]',
-                        '[class*="overlay"]'
+                        '.modal-dialog', '.modal-content', '.popover', // Bootstrap
+                        '.chakra-modal', '.chakra-dialog', // Chakra UI
+                        '.ui.modal', '.ui.popup', // Semantic UI
+                        '.v-dialog', '.v-menu', // Vuetify
+                        
+                        // Generic patterns
+                        '[class*="modal"]', '[class*="dialog"]', '[class*="popup"]', '[class*="overlay"]',
+                        '[class*="drawer"]', '[class*="toast"]', '[class*="tooltip"]', '[class*="popover"]'
                     ];
-                    // Select visible popups only
+                    
+                    // Find all visible popups
                     const popups = [];
+                    
+                    // Check for elements matching our selectors
                     for (const sel of selectors) {
                         for (const el of document.querySelectorAll(sel)) {
                             if (isVisible(el)) popups.push(el);
                         }
                     }
+                    
+                    // Check for fixed/absolute positioned elements with high z-index
+                    document.querySelectorAll('div, section, aside').forEach(el => {
+                        if (!popups.includes(el) && isVisible(el)) {
+                            const style = window.getComputedStyle(el);
+                            const position = style.position;
+                            const zIndex = parseInt(style.zIndex) || 0;
+                            
+                            // Fixed/absolute with high z-index are often modals/popups
+                            if ((position === 'fixed' || position === 'absolute') && zIndex > 10) {
+                                const rect = el.getBoundingClientRect();
+                                if (rect.width > 50 && rect.height > 50) { // Reasonable size check
+                                    popups.push(el);
+                                }
+                            }
+                        }
+                    });
+                    
+                    // Check for elements near known backdrops (often indicates a modal)
+                    const backdrops = document.querySelectorAll('.modal-backdrop, .overlay, .backdrop, .dimmer, [class*="backdrop"], [class*="overlay"]');
+                    for (const backdrop of backdrops) {
+                        if (isVisible(backdrop)) {
+                            const backdropRect = backdrop.getBoundingClientRect();
+                            const viewportCenter = {
+                                x: window.innerWidth / 2,
+                                y: window.innerHeight / 2
+                            };
+                            
+                            // Look for visible centered elements - often these are modals related to backdrops
+                            document.querySelectorAll('div, section, aside').forEach(el => {
+                                if (!popups.includes(el) && isVisible(el)) {
+                                    const rect = el.getBoundingClientRect();
+                                    const elementCenter = {
+                                        x: rect.left + rect.width / 2,
+                                        y: rect.top + rect.height / 2
+                                    };
+                                    
+                                    // Is it centered, reasonable size, and contained within backdrop?
+                                    const isCentered = Math.abs(elementCenter.x - viewportCenter.x) < viewportCenter.x / 3 &&
+                                                      Math.abs(elementCenter.y - viewportCenter.y) < viewportCenter.y / 3;
+                                    
+                                    if (isCentered && rect.width > 50 && rect.height > 50) {
+                                        popups.push(el);
+                                    }
+                                }
+                            });
+                        }
+                    }
+                    
                     // Remove duplicates
                     return Array.from(new Set(popups));
                 }
@@ -335,6 +395,18 @@ async def analyze_page():
 
             # Add each item, grouping related content on the same line
             for item in page_content['content']:
+                # Skip elements where type matches display text (e.g., "[49][button]button")
+                if item.startswith('['):
+                    # Parse the element format: [id][type]text
+                    parts = item.split(']', 2)
+                    if len(parts) >= 3:
+                        element_type = parts[1][1:]  # Get the type without '['
+                        display_text = parts[2]      # Get the display text
+                        
+                        # Skip if the element type exactly matches its display text
+                        if display_text.strip() == element_type:
+                            continue
+                
                 # Start a new line for interactive elements or if current line is empty
                 if item.startswith('[') or not current_line:
                     if (current_line):  # Add the previous line if it exists

browser/analyzers/page_analyzer.py

@@ -1,7 +1,6 @@
 from playwright.async_api import async_playwright
 
 def inject_cursor_script():
-    """Returns the script to inject for cursor visualization"""
     return """
     // Create a custom cursor element
     const cursor = document.createElement('div');
@@ -17,7 +16,7 @@ def inject_cursor_script():
     cursor.style.zIndex = '999999';
     cursor.style.transition = 'left 0.1s, top 0.1s';
     
-    // Add cursor to the page when it loads
+    // Add cursor to the page
     document.addEventListener('DOMContentLoaded', function() {
         document.body.appendChild(cursor);
     });
@@ -41,7 +40,6 @@ def inject_cursor_script():
     """
 
 async def initialize_browser(options, connection_options=None):
-    """Initialize the browser by connecting to existing instance or launching a new one."""
     playwright = await async_playwright().start()
 
     # Default connection options if none provided
@@ -80,15 +78,14 @@ async def initialize_browser(options, connection_options=None):
                 raise e
 
             print("Falling back to launching a new browser instance...")
-            browser = None  # Reset for fallback path
+            browser = None
 
     # Launch a new browser if needed
     if browser is None:
         print(f"Launching new browser with options: {options}")
         browser = await playwright.chromium.launch(**options)
         page = await browser.new_page(viewport=None)
 
-    # Shared initialization regardless of connection method
     # Inject cursor visualization CSS and JavaScript
     await page.add_init_script(inject_cursor_script())
 
@@ -119,7 +116,6 @@ async def initialize_browser(options, connection_options=None):
     # Ensure cursor is created and function is available
     await page.evaluate("""
         () => {
-            // Create a custom cursor element if it doesn't exist
             if (!document.getElementById('ai-agent-cursor')) {
                 const cursor = document.createElement('div');
                 cursor.id = 'ai-agent-cursor';
@@ -136,7 +132,6 @@ async def initialize_browser(options, connection_options=None):
                 document.body.appendChild(cursor);
             }
             
-            // Define the updateAICursor function if it doesn't exist
             if (typeof window.updateAICursor !== 'function') {
                 window.updateAICursor = function(x, y) {
                     const cursor = document.getElementById('ai-agent-cursor');
@@ -157,7 +152,6 @@ async def initialize_browser(options, connection_options=None):
     return playwright, browser, page
 
 async def close_browser(playwright, browser, is_connected=False):
-    """Close the browser cleanly."""
     try:
         if is_connected:
             # If connected to existing browser, just disconnect

browser/browser_setup.py

@@ -1,6 +1,5 @@
 """
 Main browser controller that serves as the interface to all browser functionality.
-This controller delegates to specialized controllers for different aspects of browser control.
 """
 
 import asyncio
@@ -9,18 +8,20 @@
 from browser.analyzers.page_analyzer import analyze_page
 from browser.navigation.navigator import navigate, go_back
 from browser.navigation.scroll_manager import scroll
+from browser.utils.user_interaction import ask_user
 
 # Global page reference
 page = None
 
 async def initialize(browser_page):
-    """Initialize the browser controller with a browser page."""
     # Import locally to avoid circular imports
     from browser.controllers.element_controller import initialize as init_element
     from browser.controllers.keyboard_controller import initialize as init_keyboard
     from browser.analyzers.page_analyzer import initialize as init_analyzer
     from browser.navigation.navigator import initialize as init_navigator
     from browser.navigation.scroll_manager import initialize as init_scroll
+    from browser.utils.dom_helpers import initialize as init_dom_helpers
+    from browser.utils.user_interaction import initialize as init_user_interaction
 
     global page
     page = browser_page
@@ -31,28 +32,19 @@ async def initialize(browser_page):
     await init_analyzer(page)
     await init_navigator(page)
     await init_scroll(page)
+    await init_dom_helpers(page)
+    init_user_interaction()
 
     print("Browser controller initialized successfully")
 
 async def close():
-    """Close the browser cleanly."""
     try:
         await page.context.browser.close()
         return "Browser closed successfully"
     except Exception as e:
         return f"Error closing browser: {str(e)}"
 
 def get_browser_tools():
-    """
-    Return all the tools from the browser controller.
-    
-    This function returns a list of all public browser control functions
-    that are decorated with @tool. These functions can be used to control
-    the browser programmatically.
-    
-    Returns:
-        list: A list of browser control tool functions
-    """
     browser_tools = [
         analyze_page,
         click,
@@ -62,6 +54,7 @@ def get_browser_tools():
         go_back,
         navigate,
         scroll,
+        ask_user,
     ]
 
     return browser_tools