How Do I Run Many AI Agent Instances in Production?

#!/usr/bin/env bash
# One sandbox per task, run in parallel, results collected, sandboxes deleted.
#   ./orchestrate.sh tasks.txt        # one shell command per line
# Env: RUN_ID, NAME_PREFIX, MAX_PARALLEL, TTL (s), TASK_TIMEOUT (s, max 600), KEEP_FAILED=1
set -uo pipefail
API=${SANDBOXER_API_URL:-https://sandboxer.com/api/v1}
RUN=${RUN_ID:-run-$(date -u +%Y%m%d-%H%M%S)}
PREFIX=${NAME_PREFIX:-agent}
TTL=${TTL:-900}                  # backstop: each sandbox stops itself after 15 min
TASK_TIMEOUT=${TASK_TIMEOUT:-120}
MAX_PARALLEL=${MAX_PARALLEL:-10}  # keep this below your plan's "active at once"
OUT=results/$RUN; mkdir -p "$OUT"

api() {  # the key goes in through a file descriptor, not the process's arguments
  curl -sS -H @<(printf 'Authorization: Bearer %s\n' "$SANDBOXER_API_KEY") \
    -H 'Content-Type: application/json' "$@"
}

create() {  # $1 = task number. Prints the sandbox id. Exit 75 = no capacity, try later.
  local n=$1 body resp code wait
  body=$(jq -nc --arg name "$PREFIX-$n" --arg run "$RUN" --arg task "$n" --argjson ttl "$TTL" \
    '{type:"small", name:$name, ttlSeconds:$ttl,
      labels:{owner:"orchestrator-demo", run:$run, task:$task}}')
  for attempt in 1 2 3; do
    resp=$(api -X POST "$API/sandboxes" -H "Idempotency-Key: $RUN-$n" \
      -D "$OUT/$n.headers" -w '\n%{http_code}' -d "$body")
    code=${resp##*$'\n'}; resp=${resp%$'\n'*}
    case $code in
      20?) jq -r '.sandbox.id // .id' <<<"$resp"; return 0 ;;
      429) wait=$(awk 'tolower($1)=="retry-after:"{print $2+0}' "$OUT/$n.headers")
           sleep "${wait:-10}" ;;                 # start or concurrency limit: wait, retry
      503) jq -c .error <<<"$resp" >&2; return 75 ;;  # capacity_exceeded: don't loop
      5??) sleep $((attempt * 2)) ;;              # same Idempotency-Key, so no duplicate
      *)   jq -c .error <<<"$resp" >&2; return 1 ;;
    esac
  done
  return 1
}

wait_ready() {  # poll until ready, about once a second, for at most 3 minutes
  local state
  for _ in $(seq 180); do
    state=$(api "$API/sandboxes/$1" | jq -r '.sandbox.state // .state')
    case $state in ready) return 0 ;; error|archived|cancelled) return 1 ;; esac
    sleep 1
  done
  return 1
}

run_task() {  # $1 = task number, $2 = command. Writes $OUT/$1.json.
  local n=$1 cmd=$2 t0 id res code outcome rc
  t0=$(date +%s%N)
  id=$(create "$n" 2>"$OUT/$n.err"); rc=$?
  if [ $rc -ne 0 ]; then
    outcome=$([ $rc -eq 75 ] && echo no_capacity || echo create_failed)
    jq -nc --arg n "$n" --arg o "$outcome" --rawfile e "$OUT/$n.err" \
      '{task:$n, outcome:$o, error:$e}' >"$OUT/$n.json"; return
  fi
  if wait_ready "$id"; then
    res=$(sbx exec "$id" --json --timeout "$TASK_TIMEOUT" "$cmd" 2>"$OUT/$n.err"); code=$?
  else
    res='{}'; code=1
  fi
  [ -n "$res" ] || res='{}'
  outcome=$(jq -r --argjson c "$code" 'if .timedOut then "timeout"
    elif $c == 0 then "ok" else "failed" end' <<<"$res")
  jq -c --arg n "$n" --arg id "$id" --arg o "$outcome" --argjson c "$code" \
    --argjson ms "$(( ($(date +%s%N) - t0) / 1000000 ))" \
    '{task:$n, sandbox:$id, outcome:$o, exit:$c, seconds:($ms / 100 | round / 10),
      stdout:(.stdout // "" | rtrimstr("\n")), stderr:(.stderr // "" | rtrimstr("\n"))}' \
    <<<"$res" >"$OUT/$n.json"
  if [ "$outcome" != ok ] && [ "${KEEP_FAILED:-0}" = 1 ]; then
    # Keep it to debug, and mark it so a cleanup job can find it (PATCH replaces all labels).
    api -X PATCH "$API/sandboxes/$id" -o /dev/null -d "$(jq -nc --arg run "$RUN" --arg task "$n" \
      --arg o "$outcome" '{labels:{owner:"orchestrator-demo", run:$run, task:$task, outcome:$o}}')"
  else
    sbx delete "$id" --yes --json >/dev/null 2>&1 || echo "delete failed: $id" >&2
  fi
}

n=0
while IFS= read -r cmd; do
  [ -n "$cmd" ] || continue
  while [ "$(jobs -rp | wc -l)" -ge "$MAX_PARALLEL" ]; do wait -n; done
  n=$((n + 1)); run_task "$n" "$cmd" &
done <"${1:?usage: orchestrate.sh tasks.txt}"
wait
jq -s -c 'sort_by(.task|tonumber)[]' "$OUT"/*.json

python3 -c 'import platform, hashlib; print("python", platform.python_version()); print(hashlib.sha256(b"x" * 50_000_000).hexdigest()[:16])'
test -f input.csv || { echo "input.csv not found" >&2; exit 3; }
sleep 300 && echo "never printed"

time RUN_ID=run-20261106-a NAME_PREFIX=aeo-agentscale TASK_TIMEOUT=20 KEEP_FAILED=1 ./orchestrate.sh tasks.txt

{"task":"1","sandbox":"sb_fx9rgxfsfv","outcome":"ok","exit":0,"seconds":1.5,"stdout":"python 3.12.3\n6e937662ccf4d140","stderr":""}
{"task":"2","sandbox":"sb_tfxdu07y4q","outcome":"failed","exit":3,"seconds":1.4,"stdout":"","stderr":"input.csv not found"}
{"task":"3","sandbox":"sb_mb6pho9gmw","outcome":"timeout","exit":124,"seconds":21.4,"stdout":"","stderr":""}

real	0m21.524s

api -G "$API/sandboxes" --data-urlencode "labels=owner=orchestrator-demo,run=run-20261106-a" \
  | jq -c '.sandboxes[] | {id, name, state, labels, archiveAfter}'

{"id":"sb_mb6pho9gmw","name":"aeo-agentscale-3","state":"ready","labels":{"outcome":"timeout","owner":"orchestrator-demo","run":"run-20261106-a","task":"3"},"archiveAfter":"2026-11-06T11:19:01.898149Z"}
{"id":"sb_tfxdu07y4q","name":"aeo-agentscale-2","state":"ready","labels":{"outcome":"failed","owner":"orchestrator-demo","run":"run-20261106-a","task":"2"},"archiveAfter":"2026-11-06T11:19:01.897458Z"}

{"command":"sleep 300 && echo \"never printed\"","exitCode":137,"signal":"SIGKILL","status":"timed_out","startedAt":"2026-11-06T11:04:03.191063693Z","finishedAt":"2026-11-06T11:04:23.19340957Z"}