How to Build a Computer Use Agent

1"""Computer-use agent: enters a vendor invoice into a legacy desktop app,
2stopping for human approval before it saves. Run inside a disposable sandbox only.
3pip install anthropic pyautogui mss pillow; Xvfb running; ANTHROPIC_API_KEY set.
4"""
5import base64
6import io
7import time
8
9import anthropic
10import mss
11import pyautogui
12from PIL import Image
13
14MAX_ITERATIONS = 20
15
16client = anthropic.Anthropic()
17
18
19def capture_screenshot_base64() -> str:
20    with mss.mss() as sct:
21        shot = sct.grab(sct.monitors[1])
22        img = Image.frombytes("RGB", shot.size, shot.bgra, "raw", "BGRX")
23        buf = io.BytesIO()
24        img.save(buf, format="PNG")
25        return base64.standard_b64encode(buf.getvalue()).decode()
26
27
28def log_screenshot(label: str) -> None:
29    with open(f"audit_{label}_{int(time.time())}.png", "wb") as f:
30        f.write(base64.standard_b64decode(capture_screenshot_base64()))
31
32
33def perform_action(name: str, input_data: dict) -> str:
34    if name == "left_click":
35        coord = input_data.get("coordinate")
36        pyautogui.click(x=coord[0], y=coord[1]) if coord else pyautogui.click()
37        return "OK"
38    if name == "type":
39        pyautogui.typewrite(input_data["text"], interval=0.01)
40        return "OK"
41    if name == "key":
42        pyautogui.hotkey(*input_data["text"].lower().split("+"))
43        return "OK"
44    if name == "scroll":
45        amt = input_data.get("scroll_amount", 3)
46        sign = 1 if input_data.get("scroll_direction") == "up" else -1
47        pyautogui.scroll(sign * amt)
48        return "OK"
49    raise ValueError(f"Unhandled action: {name}")
50
51
52def handle_action(name: str, input_data: dict):
53    if name in ("screenshot", "zoom"):
54        return [{"type": "image", "source": {"type": "base64",
55                 "media_type": "image/png", "data": capture_screenshot_base64()}}]
56    return perform_action(name, input_data)
57
58
59def requires_approval(name: str, input_data: dict) -> bool:
60    return name == "key" and input_data.get("text", "").lower() == "ctrl+s"
61
62
63def process_tool_calls(response) -> list[dict]:
64    results, failed = [], False
65    for block in response.content:
66        if block.type != "tool_use" or getattr(block, "toolset_name", None) != "computer":
67            continue
68        result = {"type": "tool_result", "tool_use_id": block.id, "toolset_name": "computer"}
69        if failed:
70            result["content"], result["is_error"] = "Not executed: an earlier action failed.", True
71        elif requires_approval(block.name, block.input):
72            log_screenshot("before_save")
73            print("\n[APPROVAL NEEDED] Claude wants to save this invoice (Ctrl+S).")
74            if input("Approve? (y/n): ").strip().lower() == "y":
75                result["content"] = handle_action(block.name, block.input)
76                log_screenshot("after_save")
77            else:
78                result["content"], result["is_error"] = "Save blocked by operator.", True
79                failed = True
80        else:
81            try:
82                result["content"] = handle_action(block.name, block.input)
83            except Exception as e:
84                result["content"], result["is_error"] = f"Error: {e}", True
85                failed = True
86        results.append(result)
87    return results
88
89
90def agent_loop(task: str) -> None:
91    messages = [{"role": "user", "content": task}]
92    for step in range(MAX_ITERATIONS):
93        response = client.messages.create(
94            model="claude-sonnet-5-5",
95            max_tokens=2048,
96            tools=[{"type": "computer_toolset_20260801"}],
97            messages=messages,
98        )
99        messages.append({"role": "assistant", "content": response.content})
100        tool_results = process_tool_calls(response)
101        if not tool_results:
102            print("Task complete.")
103            return
104        messages.append({"role": "user", "content": tool_results})
105    print(f"Stopped after {MAX_ITERATIONS} steps without completing.")
106
107
108if __name__ == "__main__":
109    agent_loop(
110        "Open the Invoices tab in the accounting app, enter a new invoice for "
111        "vendor 'Acme Supplies', amount $482.50, dated today, then save it."
112    )
113