computer_controller.py

# Control computer given action
import pyautogui
import platform
from gestures import Gesture

# Platform detection and imports for raw mouse movement
IS_WINDOWS = platform.system() == 'Windows'
IS_MAC = platform.system() == "Darwin"

if IS_MAC:
    from Quartz.CoreGraphics import (
        CGEventCreateMouseEvent,
        CGEventPost,
        kCGHIDEventTap,
        kCGEventMouseMoved
    )

if IS_WINDOWS:
    try:
        import win32api
        import win32con
        HAS_WIN32 = True
    except ImportError:
        HAS_WIN32 = False
        print("Warning: pywin32 not available, using pyautogui for mouse control")
        
def mac_move_relative(dx, dy):
    # Create relative mouse movement event
    event = CGEventCreateMouseEvent(
        None,
        kCGEventMouseMoved,
        (0, 0),  # ignored when using relative mode
        0
    )
    # Set relative movement fields
    event.setIntegerValueField_(0x0000000D, dx)  # kCGMouseEventDeltaX
    event.setIntegerValueField_(0x0000000E, dy)  # kCGMouseEventDeltaY

    CGEventPost(kCGHIDEventTap, event)

    
class ComputerController:
    def __init__(self):
        pyautogui.FAILSAFE = False
        self.left_down = False
        self.right_down = False

        # For cursor smoothing
        self.prev_x = None
        self.prev_y = None
        
        # Movement mode (True = relative for games, False = absolute for desktop)
        self.relative_mode = False
    
    def perform_action(self, gesture, palm_center, cam_w, cam_h):

        if palm_center:
            cx, cy, cz = palm_center

            # Normalize hand position (0 to 1)
            rx = cx / cam_w
            ry = cy / cam_h

            if self.relative_mode:
                # RELATIVE MODE (for games like Minecraft)
                # Convert to centered coordinates (-0.5 to 0.5)
                centered_x = rx - 0.5
                centered_y = ry - 0.5

                SENSITIVITY = 200    # Adjust for game sensitivity
                DEADZONE = 0.05     # Ignore small movements (0-1 range)
                SMOOTHING = 0.3     # Smooth the movement

                # Apply deadzone
                if abs(centered_x) < DEADZONE:
                    centered_x = 0
                if abs(centered_y) < DEADZONE:
                    centered_y = 0

                if self.prev_x is not None:
                    # Smooth the centered positions
                    centered_x = self.prev_x * (1 - SMOOTHING) + centered_x * SMOOTHING
                    centered_y = self.prev_y * (1 - SMOOTHING) + centered_y * SMOOTHING

                # Calculate relative movement for this frame
                dx = int(centered_x * SENSITIVITY)
                dy = int(centered_y * SENSITIVITY)

                # Use platform-specific raw mouse movement
                if dx != 0 or dy != 0:
                    if IS_WINDOWS and HAS_WIN32:
                        # Windows: Use win32api for true raw input
                        win32api.mouse_event(win32con.MOUSEEVENTF_MOVE, dx, dy, 0, 0)
                    elif IS_MAC:
                        # Mac: Use Quartz CoreGraphics for raw input
                        mac_move_relative(dx, dy)
                    else:
                        # Linux or fallback: Use pyautogui
                        pyautogui.moveRel(dx, dy, _pause=False)

                self.prev_x = centered_x
                self.prev_y = centered_y
            
            else:
                # ABSOLUTE MODE (for desktop use)
                # Map to screen coords with expanded range
                screen_w, screen_h = pyautogui.size()
                
                # Remap from camera range to larger virtual range, then clamp
                # Adjust EXPANSION_FACTOR: higher = less hand movement needed
                EXPANSION_FACTOR = 1.5
                
                # Center and expand the mapping
                centered_rx = (rx - 0.5) * EXPANSION_FACTOR + 0.5
                centered_ry = (ry - 0.5) * EXPANSION_FACTOR + 0.5
                
                # Clamp to screen bounds
                centered_rx = max(0, min(1, centered_rx))
                centered_ry = max(0, min(1, centered_ry))
                
                mouse_x = int(centered_rx * screen_w)
                mouse_y = int(centered_ry * screen_h)

                SMOOTHING = 0.25    # lower = smoother, higher = faster
                DEADZONE = 2        # ignore tiny jitter

                if self.prev_x is not None:
                    dx = mouse_x - self.prev_x
                    dy = mouse_y - self.prev_y

                    # Deadzone removes tiny tremors
                    if abs(dx) < DEADZONE and abs(dy) < DEADZONE:
                        return

                    # Smooth movement
                    mouse_x = int(self.prev_x * (1-SMOOTHING) + mouse_x * SMOOTHING)
                    mouse_y = int(self.prev_y * (1-SMOOTHING) + mouse_y * SMOOTHING)

                # Use absolute positioning (cross-platform)
                pyautogui.moveTo(mouse_x, mouse_y, _pause=False)

                self.prev_x = mouse_x
                self.prev_y = mouse_y

        # Hold Left Click (POINT)
        if gesture == "point":
            if not self.left_down:
                pyautogui.mouseDown(button='left')
                self.left_down = True

        # Hold Right Click (PEACE)
        if gesture == "peace":
            if not self.right_down:
                pyautogui.mouseDown(button='right')
                self.right_down = True

        # Neutral, Release Everything (FIST)
        if gesture == "fist":
            self.stop_all_actions()

        # Scroll (THUMBS_UP / THUMBS_DOWN)
        if gesture == "thumbs_up":
            pyautogui.scroll(40)
        elif gesture == "thumbs_down":
            pyautogui.scroll(-40)

        #double left click
        if gesture =="five":
            if self.left_down:
                pyautogui.mouseUp(button='left')
                self.left_down = False
            if self.right_down:
                pyautogui.mouseUp(button='right')
                self.right_down = False

            # perform a double left click
            pyautogui.click(button='left', clicks=2, interval=0.1)
    
    def toggle_mode(self):
        """Toggle between relative (game) and absolute (desktop) mouse mode"""
        self.relative_mode = not self.relative_mode
        self.prev_x = None  # reset smoothing when switching modes
        self.prev_y = None
        mode_name = "RELATIVE (Game)" if self.relative_mode else "ABSOLUTE (Desktop)"
        print(f"Switched to {mode_name} mode")
        return mode_name
    
    def stop_all_actions(self):
        self.prev_x = None  # reset smoothing

        if self.left_down:
            pyautogui.mouseUp(button='left')
            self.left_down = False

        if self.right_down:
            pyautogui.mouseUp(button='right')
            self.right_down = False