fix(cua): handle lowercase modifier keys and unify right-click in Tzafon templates

Author: mttias

pkg/templates/python/tzafon-computer-use/tools/computer.py

@@ -15,7 +15,19 @@
 
 from .base import ToolError
 
-MODIFIER_MAP = {"Control": "Ctrl", "Enter": "Return"}
+MODIFIER_MAP = {
+    "Control": "Ctrl",
+    "control": "Ctrl",
+    "ctrl": "Ctrl",
+    "Enter": "Return",
+    "enter": "Return",
+    "Escape": "Escape",
+    "esc": "Escape",
+    "Shift": "Shift",
+    "shift": "Shift",
+    "Alt": "Alt",
+    "alt": "Alt",
+}
 MODIFIER_NAMES = {"Ctrl", "Shift", "Alt", "Meta", "Super"}
 
 
@@ -65,7 +77,11 @@ async def execute(self, action: Any) -> None:
         """Map a Tzafon model action to Kernel Computer Controls."""
         t = action.type
 
-        if t == "click":
+        if t == "click" and getattr(action, "button", "left") == "right":
+            self.kernel.browsers.computer.click_mouse(
+                self.session_id, x=self._x(action), y=self._y(action), button="right",
+            )
+        elif t == "click":
             self.kernel.browsers.computer.click_mouse(
                 self.session_id, x=self._x(action), y=self._y(action),
             )
@@ -77,10 +93,6 @@ async def execute(self, action: Any) -> None:
             self.kernel.browsers.computer.click_mouse(
                 self.session_id, x=self._x(action), y=self._y(action), num_clicks=3,
             )
-        elif t == "right_click":
-            self.kernel.browsers.computer.click_mouse(
-                self.session_id, x=self._x(action), y=self._y(action), button="right",
-            )
         elif t == "type":
             self.kernel.browsers.computer.type_text(self.session_id, text=action.text)
         elif t in ("key", "keypress"):
@@ -115,16 +127,19 @@ async def execute(self, action: Any) -> None:
         elif t == "drag":
             x1 = getattr(action, "x", None)
             if x1 is None:
-                x1 = getattr(action, "x1", 0)
+                x1 = getattr(action, "x1", None)
             y1 = getattr(action, "y", None)
             if y1 is None:
-                y1 = getattr(action, "y1", 0)
+                y1 = getattr(action, "y1", None)
             x2 = getattr(action, "end_x", None)
             if x2 is None:
-                x2 = getattr(action, "x2", 0)
+                x2 = getattr(action, "x2", None)
             y2 = getattr(action, "end_y", None)
             if y2 is None:
-                y2 = getattr(action, "y2", 0)
+                y2 = getattr(action, "y2", None)
+            if any(v is None for v in (x1, y1, x2, y2)):
+                print(f"drag action missing coordinates, skipping: {action}")
+                return
             self.kernel.browsers.computer.drag_mouse(
                 self.session_id,
                 path=[

pkg/templates/typescript/tzafon-computer-use/tools/computer.ts

@@ -17,7 +17,16 @@ export class ToolError extends Error {
 
 const MODIFIER_MAP: Record<string, string> = {
   Control: 'Ctrl',
+  control: 'Ctrl',
+  ctrl: 'Ctrl',
   Enter: 'Return',
+  enter: 'Return',
+  Escape: 'Escape',
+  esc: 'Escape',
+  Shift: 'Shift',
+  shift: 'Shift',
+  Alt: 'Alt',
+  alt: 'Alt',
 };
 
 const MODIFIER_NAMES = new Set(['Ctrl', 'Shift', 'Alt', 'Meta', 'Super']);
@@ -75,6 +84,13 @@ export class ComputerTool {
   }
 
   async execute(action: any): Promise<void> {
+    if (action.type === 'click' && action.button === 'right') {
+      await this.kernel.browsers.computer.clickMouse(this.sessionId, {
+        x: this.x(action), y: this.y(action), button: 'right',
+      });
+      return;
+    }
+
     switch (action.type) {
       case 'click':
         await this.kernel.browsers.computer.clickMouse(this.sessionId, {
@@ -94,12 +110,6 @@ export class ComputerTool {
         });
         break;
 
-      case 'right_click':
-        await this.kernel.browsers.computer.clickMouse(this.sessionId, {
-          x: this.x(action), y: this.y(action), button: 'right',
-        });
-        break;
-
       case 'type':
         await this.kernel.browsers.computer.typeText(this.sessionId, {
           text: action.text,
@@ -146,20 +156,23 @@ export class ComputerTool {
         });
         break;
 
-      case 'drag':
+      case 'drag': {
+        const x1 = action.x ?? action.x1;
+        const y1 = action.y ?? action.y1;
+        const x2 = action.end_x ?? action.x2;
+        const y2 = action.end_y ?? action.y2;
+        if (x1 == null || y1 == null || x2 == null || y2 == null) {
+          console.warn('drag action missing coordinates, skipping', action);
+          break;
+        }
         await this.kernel.browsers.computer.dragMouse(this.sessionId, {
           path: [
-            [
-              clamp(action.x ?? action.x1, this.width),
-              clamp(action.y ?? action.y1, this.height),
-            ],
-            [
-              clamp(action.end_x ?? action.x2, this.width),
-              clamp(action.end_y ?? action.y2, this.height),
-            ],
+            [clamp(x1, this.width), clamp(y1, this.height)],
+            [clamp(x2, this.width), clamp(y2, this.height)],
           ],
         });
         break;
+      }
 
       case 'wait':
         await new Promise((r) => setTimeout(r, 2000));