Run a Computer-Use Agent on a Linux Desktop in the Cloud

sbx new --type default
id=sb_4943jbsept    # the id from sbx new
sbx desktop $id --no-open    # starts the desktop and prints a viewer URL
sbx exec $id "echo DISPLAY=\$DISPLAY; pgrep -a Xvfb"

DISPLAY=
569 /usr/bin/Xvfb :1 -screen 0 1920x1080x24 -noreset -nolisten tcp +extension RANDR

sbx exec $id "DISPLAY=:1 xdotool getdisplaygeometry; DISPLAY=:1 xdotool getmouselocation"

1920 1080
x:960 y:540 screen:0 window:20971523

# computer.py: the tool functions a computer-use agent calls, run inside the sandbox.
import os, subprocess, time
from PIL import Image, ImageChops

os.environ["DISPLAY"] = ":1"  # the sandbox desktop

def screenshot(path):
    subprocess.run(["ffmpeg", "-loglevel", "error", "-y", "-f", "x11grab",
                    "-video_size", "1920x1080", "-i", ":1", "-frames:v", "1", path], check=True)
    return path

def xdo(*args):
    return subprocess.run(["xdotool", *map(str, args)], check=True,
                          capture_output=True, text=True).stdout.strip()

def click(x, y):
    xdo("mousemove", x, y, "click", 1)

def type_text(text):
    xdo("type", "--delay", 20, text)

def key(combo):
    xdo("key", combo)

def active_window():
    try:
        return xdo("getactivewindow", "getwindowname")
    except subprocess.CalledProcessError:
        return None  # bare desktop: nothing focused

def changed_box(a, b):
    return ImageChops.difference(Image.open(a).convert("RGB"),
                                 Image.open(b).convert("RGB")).getbbox()

if __name__ == "__main__":
    # 1. Look
    before = screenshot("/home/user/step0.png")
    print("step0", Image.open(before).size, "active:", active_window())

    # 2. Act: open a terminal, wait for its window
    subprocess.Popen(["xfce4-terminal", "--title", "agent-term"])
    for _ in range(50):
        try:
            wid = xdo("search", "--sync", "--onlyvisible", "--name", "agent-term")
            break
        except subprocess.CalledProcessError:
            time.sleep(0.2)
    geo = xdo("getwindowgeometry", wid.splitlines()[0])
    print(geo.replace("\n", " | "))

    # 3. Act: click inside the window, type a command, press Enter
    pos = geo.split("Position: ")[1].split(" ")[0]
    x, y = map(int, pos.split(","))
    click(x + 100, y + 100)
    type_text("echo hello from the agent > ~/agent.txt; date")
    key("Return")
    time.sleep(1)

    # 4. Look again and compare
    after = screenshot("/home/user/step1.png")
    print("step1", Image.open(after).size, "active:", active_window())
    print("changed region:", changed_box(before, after))
    print("file written:", open("/home/user/agent.txt").read().strip())

sbx scp ./computer.py $id:/home/user/computer.py
sbx exec $id "python3 ~/computer.py"

step0 (1920, 1080) active: None
Window 52428803 |   Position: 10,85 (screen: 0) |   Geometry: 817x483
step1 (1920, 1080) active: agent-term
changed region: (0, 0, 1606, 564)
file written: hello from the agent

# browse.py
import time
from computer import xdo, key, type_text, screenshot, active_window, changed_box

wid = xdo("search", "--onlyvisible", "--class", "google-chrome").splitlines()[-1]
xdo("windowactivate", "--sync", wid)
key("ctrl+l")
type_text("example.com")
key("Return")
time.sleep(3)
print("active:", active_window())
print("changed region:", changed_box("/home/user/step1.png", screenshot("/home/user/step2.png")))

sbx scp ./browse.py $id:/home/user/browse.py
sbx exec $id "cd ~ && python3 browse.py"

active: Example Domain - Google Chrome
changed region: (0, 0, 1920, 1080)