rewrite doubleclick action using Quartz, fix key action when handling keyboard shortcuts like cmd+c

This commit is contained in:
leopengpong
2025-06-16 20:31:02 -07:00
parent e26b1de89c
commit deb3c2c96a
2 changed files with 51 additions and 5 deletions
+12 -5
View File
@@ -6,6 +6,7 @@ import pyautogui
import subprocess
from io import BytesIO
from PIL import Image
import quartz_doubleclick
ANTHROPIC_API_URL = "https://api.anthropic.com/v1/messages"
MODEL = "claude-3-7-sonnet-20250219"
@@ -103,7 +104,8 @@ def execute_computer_tool(tool_input):
return {"type": "text", "text": "Error: Missing coordinates for double click action"}
# Perform double click
pyautogui.doubleClick(x, y, interval=0.2)
# pyautogui.doubleClick(x, y, interval=0.2) this doesn't work
quartz_doubleclick.double_click(x, y)
return {"type": "text", "text": f"Double-clicked at coordinates ({x}, {y})"}
elif action == "type":
@@ -114,12 +116,17 @@ def execute_computer_tool(tool_input):
elif action == "key":
# Press a key or key combination
key = tool_input.get("key", "")
text = tool_input.get("text", "")
try:
pyautogui.press(key)
return {"type": "text", "text": f"Pressed key: {key}"}
if '+' in text:
# Handle key combinations like "command+c"
keys = text.replace('super', 'command').split('+')
pyautogui.hotkey(*keys, interval=0.05) # interval is required
else:
pyautogui.press(text)
return {"type": "text", "text": f"Pressed key: {text}"}
except Exception as e:
return {"type": "text", "text": f"Error pressing key {key}: {str(e)}"}
return {"type": "text", "text": f"Error pressing key {text}: {str(e)}"}
elif action == "scroll":
# Scroll action