From deb3c2c96a1094bb36abcde45918198650ff9c60 Mon Sep 17 00:00:00 2001
From: leopengpong <leopengpong@gmail.com>
Date: Mon, 16 Jun 2025 20:31:02 -0700
Subject: [PATCH] rewrite doubleclick action using Quartz, fix key action when
 handling keyboard shortcuts like cmd+c

---
 main.py               | 17 ++++++++++++-----
 quartz_doubleclick.py | 39 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+), 5 deletions(-)
 create mode 100644 quartz_doubleclick.py

diff --git a/main.py b/main.py
index 19a94cc..9b2b499 100644
--- a/main.py
+++ b/main.py
@@ -6,6 +6,7 @@ import pyautogui
 import subprocess
 from io import BytesIO
 from PIL import Image
+import quartz_doubleclick
 
 ANTHROPIC_API_URL = "https://api.anthropic.com/v1/messages"
 MODEL = "claude-3-7-sonnet-20250219"
@@ -103,7 +104,8 @@ def execute_computer_tool(tool_input):
             return {"type": "text", "text": "Error: Missing coordinates for double click action"}
         
         # Perform double click
-        pyautogui.doubleClick(x, y, interval=0.2)
+        # pyautogui.doubleClick(x, y, interval=0.2) this doesn't work
+        quartz_doubleclick.double_click(x, y)
         return {"type": "text", "text": f"Double-clicked at coordinates ({x}, {y})"}
     
     elif action == "type":
@@ -114,12 +116,17 @@ def execute_computer_tool(tool_input):
     
     elif action == "key":
         # Press a key or key combination
-        key = tool_input.get("key", "")
+        text = tool_input.get("text", "")
         try:
-            pyautogui.press(key)
-            return {"type": "text", "text": f"Pressed key: {key}"}
+            if '+' in text:
+                # Handle key combinations like "command+c"
+                keys = text.replace('super', 'command').split('+')
+                pyautogui.hotkey(*keys, interval=0.05) # interval is required
+            else:
+                pyautogui.press(text)
+            return {"type": "text", "text": f"Pressed key: {text}"}
         except Exception as e:
-            return {"type": "text", "text": f"Error pressing key {key}: {str(e)}"}
+            return {"type": "text", "text": f"Error pressing key {text}: {str(e)}"}
     
     elif action == "scroll":
         # Scroll action
diff --git a/quartz_doubleclick.py b/quartz_doubleclick.py
new file mode 100644
index 0000000..db0faef
--- /dev/null
+++ b/quartz_doubleclick.py
@@ -0,0 +1,39 @@
+'''
+This script simulates a double-click at the given mouse cursor position.
+'''
+
+import Quartz
+from time import sleep
+
+def post_mouse_event(type, pos, click_state):
+    event = Quartz.CGEventCreateMouseEvent(
+        None, type, pos, Quartz.kCGMouseButtonLeft
+    )
+    Quartz.CGEventSetIntegerValueField(event, Quartz.kCGMouseEventClickState, click_state)
+    Quartz.CGEventPost(Quartz.kCGHIDEventTap, event)
+
+def double_click (x, y):
+    pos = None
+    if x is None or y is None:
+        # Get current mouse position if no coordinates are provided
+        loc = Quartz.CGEventGetLocation(Quartz.CGEventCreate(None))
+        pos = (loc.x, loc.y)
+    else:
+        # Use provided coordinates
+        pos = (x, y)
+
+    # First click
+    post_mouse_event(Quartz.kCGEventLeftMouseDown, pos, 1)
+    post_mouse_event(Quartz.kCGEventLeftMouseUp, pos, 1)
+
+    sleep(0.05)  # Short delay within double-click threshold
+
+    # Second click
+    post_mouse_event(Quartz.kCGEventLeftMouseDown, pos, 2)
+    post_mouse_event(Quartz.kCGEventLeftMouseUp, pos, 2)
+
+
+if __name__ == "__main__":
+    print('test: double clicking at cursor position in 3 seconds...')
+    sleep(3)
+    double_click()
\ No newline at end of file