Automate Desktop Workflows with Claude Computer Use

$ docker run -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
    -v $HOME/.anthropic:/home/computeruse/.anthropic \
    -p 127.0.0.1:5900:5900 -p 127.0.0.1:8501:8501 \
    -p 127.0.0.1:6080:6080 -p 127.0.0.1:8080:8080 \
    -it ghcr.io/anthropics/anthropic-quickstarts:computer-use-demo-latest

The ERP client is open. Go to Reports > Weekly Summary, set the date range to last
week, click Export and choose CSV. Save the file as weekly-summary.csv in Downloads.
After every step take a screenshot and confirm the result before the next step.
Stop and ask me if a dialog appears that you did not expect.

import base64
import subprocess
import time
from pathlib import Path

import anthropic
import pyautogui

client = anthropic.Anthropic()          # reads ANTHROPIC_API_KEY
MODEL = "claude-sonnet-5-5"
# All 17 members are on by default; withhold the ones this loop does not implement.
UNUSED = ("zoom", "middle_click", "triple_click", "left_click_drag", "mouse_move",
          "left_mouse_down", "left_mouse_up", "cursor_position", "hold_key")
TOOLS = [{"type": "computer_toolset_20260801",
          "configs": {m: {"enabled": False} for m in UNUSED}}]
RISKY = ("submit", "delete", "purchase", "send", "pay", "upload")


def screenshot() -> list[dict]:
    subprocess.run(["scrot", "-o", "/tmp/screen.png"], check=True)
    data = base64.standard_b64encode(Path("/tmp/screen.png").read_bytes()).decode()
    return [{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": data}}]


def perform(name: str, a: dict):
    xy = a.get("coordinate")
    if name == "screenshot":
        return screenshot()
    if name == "left_click":
        pyautogui.click(*xy)
    elif name == "double_click":
        pyautogui.doubleClick(*xy)
    elif name == "right_click":
        pyautogui.rightClick(*xy)
    elif name == "type":
        pyautogui.write(a["text"], interval=0.02)
    elif name == "key":
        for _ in range(a.get("repeat", 1)):
            pyautogui.hotkey(*a["text"].lower().split("+"))
    elif name == "scroll":
        sign = -1 if a["scroll_direction"] == "down" else 1
        pyautogui.scroll(sign * a["scroll_amount"], *(xy or pyautogui.position()))
    elif name == "wait":
        time.sleep(a["duration"])
    else:
        raise NotImplementedError(name)   # add a member here, and remove it from UNUSED, if the task needs it
    time.sleep(0.5)
    return "OK"


def approved(name: str, a: dict, task: str) -> bool:
    """Ask a person before clicks or keys when the task involves irreversible words."""
    if name in ("left_click", "double_click", "key") and any(w in task.lower() for w in RISKY):
        return input(f"Allow {name} {a}? [y/N] ").strip().lower() == "y"
    return True


def run_agent(task: str, max_steps: int = 40) -> str:
    messages = [{"role": "user", "content": task}]
    for _ in range(max_steps):
        resp = client.messages.create(model=MODEL, max_tokens=4096, tools=TOOLS, messages=messages)
        messages.append({"role": "assistant", "content": resp.content})
        calls = [b for b in resp.content if b.type == "tool_use"]
        if not calls:
            return next((b.text for b in resp.content if b.type == "text"), "")

        results, failed = [], False
        for call in calls:                    # a turn can hold several actions: answer every one
            res = {"type": "tool_result", "tool_use_id": call.id, "toolset_name": "computer"}
            if failed:
                res.update(is_error=True, content="Not executed: an earlier computer action in this turn failed.")
            elif not approved(call.name, call.input, task):
                failed = True
                res.update(is_error=True, content="Denied by the operator. Stop and explain what you were about to do.")
            else:
                try:
                    res["content"] = perform(call.name, call.input)
                except Exception as exc:
                    failed = True
                    res.update(is_error=True, content=f"Error: {exc}")
            results.append(res)
        messages.append({"role": "user", "content": results})
    return "Stopped: max_steps reached"


if __name__ == "__main__":
    print(run_agent(
        "The ERP client is open and logged in. Export the 'Weekly Summary' report for last "
        "week as CSV into ~/Downloads/weekly-summary.csv. Take a screenshot after every step "
        "and verify it. If anything unexpected appears, stop and describe it."
    ))

$ docker cp <container>:/home/computeruse/Downloads/weekly-summary.csv ./exports/
$ python3 reformat_and_upload.py exports/weekly-summary.csv