fix text editor tool and scroll action in computer tool

main.py

@@ -6,11 +6,14 @@ import pyautogui
 import subprocess
 from io import BytesIO
 from PIL import Image
+import quartz_doubleclick
 
 ANTHROPIC_API_URL = "https://api.anthropic.com/v1/messages"
 MODEL = "claude-3-7-sonnet-20250219"
 BETA_FLAG = "computer-use-2025-01-24"
 TOOL_VERSION = "20250124"
+pyautogui.PAUSE = 0.1
+SCALING_FACTOR = 1.25
 
 HEADERS = {
     "content-type": "application/json",
@@ -48,8 +51,9 @@ def execute_computer_tool(tool_input):
         buffered = BytesIO()
         screenshot.save(buffered, format="PNG")
 
-        # Check size and resize if needed (5MB = 5242880 bytes)
-        if buffered.tell() > 4242880:
+        MAX_BINARY_SIZE = 5242880 * 3 // 4
+        img_data = None
+        if buffered.tell() > MAX_BINARY_SIZE:
             # Reset buffer
             buffered.seek(0)
             screenshot = Image.open(buffered)
@@ -58,14 +62,19 @@ def execute_computer_tool(tool_input):
             while True:
                 buffered = BytesIO()
                 new_size = (int(screenshot.width * 0.8), int(screenshot.height * 0.8))
+                print('new size:', new_size)
                 screenshot = screenshot.resize(new_size, Image.Resampling.LANCZOS)
                 screenshot.save(buffered, format="PNG", optimize=True)
-                if buffered.tell() <= 4242880:
+                buffered.seek(0)
+                img_data = buffered.read()
+                if len(img_data) <= MAX_BINARY_SIZE:
                     break
-         
+                buffered.seek(0)
+                screenshot = Image.open(buffered)
+        else:
+            img_data = buffered.getvalue()
         # Convert to base64
-        buffered.seek(0)
-        img_base64 = base64.b64encode(buffered.read()).decode('utf-8')
+        img_base64 = base64.b64encode(img_data).decode('utf-8')
         
         return {
             "type": "image",
@@ -76,10 +85,10 @@ def execute_computer_tool(tool_input):
             }
         }
     
-    elif action == "click":
+    elif action == "left_click":
         # Get coordinates
-        x = tool_input.get("coordinate_x")
-        y = tool_input.get("coordinate_y")
+        x = tool_input.get("coordinate")[0] * SCALING_FACTOR
+        y = tool_input.get("coordinate")[1] * SCALING_FACTOR
         if x is None or y is None:
             return {"type": "text", "text": "Error: Missing coordinates for click action"}
         
@@ -89,13 +98,14 @@ def execute_computer_tool(tool_input):
     
     elif action == "double_click":
         # Get coordinates
-        x = tool_input.get("coordinate_x")
-        y = tool_input.get("coordinate_y")
+        x = tool_input.get("coordinate")[0] * SCALING_FACTOR
+        y = tool_input.get("coordinate")[1] * SCALING_FACTOR
         if x is None or y is None:
             return {"type": "text", "text": "Error: Missing coordinates for double click action"}
         
         # Perform double click
-        pyautogui.doubleClick(x, y)
+        # pyautogui.doubleClick(x, y, interval=0.2) this doesn't work
+        quartz_doubleclick.double_click(x, y)
         return {"type": "text", "text": f"Double-clicked at coordinates ({x}, {y})"}
     
     elif action == "type":
@@ -106,19 +116,24 @@ def execute_computer_tool(tool_input):
     
     elif action == "key":
         # Press a key or key combination
-        key = tool_input.get("key", "")
+        text = tool_input.get("text", "")
         try:
-            pyautogui.press(key)
-            return {"type": "text", "text": f"Pressed key: {key}"}
+            if '+' in text:
+                # Handle key combinations like "command+c"
+                keys = text.replace('super', 'command').split('+')
+                pyautogui.hotkey(*keys, interval=0.05) # interval is required
+            else:
+                pyautogui.press(text)
+            return {"type": "text", "text": f"Pressed key: {text}"}
         except Exception as e:
-            return {"type": "text", "text": f"Error pressing key {key}: {str(e)}"}
+            return {"type": "text", "text": f"Error pressing key {text}: {str(e)}"}
     
     elif action == "scroll":
-        # Scroll action
-        direction = tool_input.get("direction", "down")
-        amount = tool_input.get("amount", 3)
+        # Scroll action (should we really have defaults here?)
+        direction = tool_input.get("scroll_direction", "down")
+        amount = tool_input.get("scroll_amount", 3)
         
-        scroll_amount = -amount if direction == "up" else amount
+        scroll_amount = -amount if direction == "down" else amount
         pyautogui.scroll(scroll_amount)
         return {"type": "text", "text": f"Scrolled {direction} by {amount}"}
     
@@ -150,23 +165,23 @@ def execute_bash_tool(command):
         return {"type": "text", "text": f"Error executing command: {str(e)}"}
 
 
-def execute_text_editor_tool(command, path, **kwargs):
+def execute_text_editor_tool(_command, _path, **kwargs):
     """Execute text editor actions."""
     
-    if command == "view":
+    if _command == "view":
         try:
-            with open(path, 'r') as f:
+            with open(_path, 'r') as f:
                 content = f.read()
             return {"type": "text", "text": content}
         except Exception as e:
             return {"type": "text", "text": f"Error reading file: {str(e)}"}
     
-    elif command == "str_replace":
+    elif _command == "str_replace":
         old_str = kwargs.get("old_str", "")
         new_str = kwargs.get("new_str", "")
         
         try:
-            with open(path, 'r') as f:
+            with open(_path, 'r') as f:
                 content = f.read()
             
             if old_str not in content:
@@ -176,7 +191,7 @@ def execute_text_editor_tool(command, path, **kwargs):
             new_content = content.replace(old_str, new_str)
             
             # Write back to file
-            with open(path, 'w') as f:
+            with open(_path, 'w') as f:
                 f.write(new_content)
             
             return {"type": "text", "text": "String replaced successfully"}
@@ -184,17 +199,17 @@ def execute_text_editor_tool(command, path, **kwargs):
         except Exception as e:
             return {"type": "text", "text": f"Error modifying file: {str(e)}"}
     
-    elif command == "create":
+    elif _command == "create":
         content = kwargs.get("content", "")
         try:
-            with open(path, 'w') as f:
+            with open(_path, 'w') as f:
                 f.write(content)
-            return {"type": "text", "text": f"File created: {path}"}
+            return {"type": "text", "text": f"File created: {_path}"}
         except Exception as e:
             return {"type": "text", "text": f"Error creating file: {str(e)}"}
     
     else:
-        return {"type": "text", "text": f"Unknown text editor command: {command}"}
+        return {"type": "text", "text": f"Unknown text editor command: {_command}"}
 
 
 def execute_tool(tool_name, tool_input):
@@ -231,8 +246,8 @@ def send_prompt(api_key, messages):
             {
                 "type": f"computer_{TOOL_VERSION}",
                 "name": "computer",
-                "display_width_px": 1024,
-                "display_height_px": 768,
+                "display_width_px": 2880,
+                "display_height_px": 1864,
                 "display_number": 1
             },
             {
@@ -315,6 +330,12 @@ def run():
             
             # Execute the tool
             result_content = execute_tool(tool_name, tool_input)
+            print('[TOOL RESULT]')
+            for key in result_content: 
+                if key == "source":
+                    print(f"{key}: {len(result_content[key])} chars long")
+                else:
+                    print(f"{key}: {result_content[key]}")
             
             # Add to results
             tool_results.append({

quartz_doubleclick.py

@@ -0,0 +1,39 @@
+'''
+This script simulates a double-click at the given mouse cursor position.
+'''
+
+import Quartz
+from time import sleep
+
+def post_mouse_event(type, pos, click_state):
+    event = Quartz.CGEventCreateMouseEvent(
+        None, type, pos, Quartz.kCGMouseButtonLeft
+    )
+    Quartz.CGEventSetIntegerValueField(event, Quartz.kCGMouseEventClickState, click_state)
+    Quartz.CGEventPost(Quartz.kCGHIDEventTap, event)
+
+def double_click (x, y):
+    pos = None
+    if x is None or y is None:
+        # Get current mouse position if no coordinates are provided
+        loc = Quartz.CGEventGetLocation(Quartz.CGEventCreate(None))
+        pos = (loc.x, loc.y)
+    else:
+        # Use provided coordinates
+        pos = (x, y)
+
+    # First click
+    post_mouse_event(Quartz.kCGEventLeftMouseDown, pos, 1)
+    post_mouse_event(Quartz.kCGEventLeftMouseUp, pos, 1)
+
+    sleep(0.05)  # Short delay within double-click threshold
+
+    # Second click
+    post_mouse_event(Quartz.kCGEventLeftMouseDown, pos, 2)
+    post_mouse_event(Quartz.kCGEventLeftMouseUp, pos, 2)
+
+
+if __name__ == "__main__":
+    print('test: double clicking at cursor position in 3 seconds...')
+    sleep(3)
+    double_click()