How Do I Make a Long-Running Agent Resume After a Crash?

#!/usr/bin/env python3
"""A resumable worker: works through tasks.txt and checkpoints after every step."""
import json, os, time
from pathlib import Path

DIR = Path("/home/user/agent")
STATE = DIR / "state.json"
BEAT = DIR / "heartbeat"
AWAKE = Path("/home/user/.sandboxer/keep-awake")
TASKS = (DIR / "tasks.txt").read_text().split()

def write_atomic(path, text):
    tmp = path.with_name(path.name + ".tmp")
    with open(tmp, "w") as f:
        f.write(text)
        f.flush()
        os.fsync(f.fileno())
    os.replace(tmp, path)          # readers see the old file or the new one, never half
    dfd = os.open(path.parent, os.O_RDONLY)
    os.fsync(dfd)                  # make the rename itself durable
    os.close(dfd)

def save(state):
    write_atomic(STATE, json.dumps(state))

def beat():
    write_atomic(BEAT, str(int(time.time())))

def log(msg):
    print(f"pid={os.getpid()} {msg}", flush=True)

state = json.loads(STATE.read_text()) if STATE.exists() else {"done": [], "status": "running"}
if state["status"] == "done":
    log("nothing to do")
    raise SystemExit(0)

AWAKE.parent.mkdir(exist_ok=True)
AWAKE.touch()                      # keep idle pause away while there is work
log(f"start: {len(state['done'])}/{len(TASKS)} done, resuming")
for task in TASKS:
    if task in state["done"]:
        continue                   # finished before the crash: skip it
    state["current"] = task
    save(state)
    log(f"working on {task}")
    for _ in range(15):            # the real step: a model call, a test run, a download…
        time.sleep(1)
        beat()
    write_atomic(DIR / "out" / f"{task}.txt", f"result of {task}\n")
    state["done"].append(task)
    state["current"] = None
    save(state)                    # checkpoint: this task is never redone
    log(f"finished {task}")

state["status"] = "done"
save(state)
AWAKE.unlink(missing_ok=True)      # let the sandbox idle-pause now
log("all tasks done")

[Unit]
Description=Resumable agent worker
After=network-online.target
Wants=network-online.target

[Service]
User=user
WorkingDirectory=/home/user/agent
ExecStart=/usr/bin/python3 /home/user/agent/worker.py
Restart=on-failure
RestartSec=2

[Install]
WantedBy=multi-user.target

id=$(sbx new --type small --json | jq -r 'select(.event == "ready") | .id')
sbx exec $id 'mkdir -p ~/agent/out && seq -f task-%02g 1 40 > ~/agent/tasks.txt'
sbx scp worker.py $id:/home/user/agent/
sbx scp agent.service $id:/home/user/agent/
sbx exec $id 'sudo cp ~/agent/agent.service /etc/systemd/system/agent.service && sudo systemctl daemon-reload && sudo systemctl enable --now agent && systemctl is-active agent'

active
Created symlink /etc/systemd/system/multi-user.target.wants/agent.service → /etc/systemd/system/agent.service.

sbx exec $id 'sudo kill -9 $(systemctl show agent -p MainPID --value); sleep 4; systemctl show agent -p MainPID -p NRestarts'
sbx exec $id 'sudo journalctl -u agent --no-pager -o cat | tail -12'

MainPID=713
NRestarts=1

…
pid=671 working on task-03
agent.service: Main process exited, code=killed, status=9/KILL
agent.service: Failed with result 'signal'.
agent.service: Scheduled restart job, restart counter is at 1.
Started agent.service - Resumable agent worker.
pid=713 start: 2/40 done, resuming
pid=713 working on task-03

sbx resume $id
sbx info $id --json | jq -c '.sandbox | {memoryRestored, resumeReason}'
sbx exec $id 'sudo journalctl -u agent --no-pager -o short-iso | tail -3'

{"memoryRestored":true,"resumeReason":null}

11:03:44 pid=713 working on task-05
11:05:31 pid=713 finished task-05
11:05:31 pid=713 working on task-06