How Do I Run Best-of-N Agent Attempts With Parallel Forks?

# repo/test_slug.py
import pytest
from slugkit.slug import slugify

@pytest.mark.parametrize("text,want", [
    ("Hello World", "hello-world"),
    ("  Hello   World  ", "hello-world"),
    ("Hello, World!", "hello-world"),
    ("Crème Brûlée", "creme-brulee"),
    ("already-a-slug", "already-a-slug"),
    ("", ""),
    ("C++ & Rust", "c-rust"),
    ("Ünïcödé 2026", "unicode-2026"),
])
def test_slugify(text, want):
    assert slugify(text) == want

#!/bin/bash
# repo/score.sh: prints "<passed> <total>" for the current checkout.
cd ~/repo
out=$(.venv/bin/python -m pytest -q --tb=no -p no:cacheprovider 2>&1 | tail -1)
passed=$(grep -o '[0-9]* passed' <<<"$out" | cut -d' ' -f1); failed=$(grep -o '[0-9]* failed' <<<"$out" | cut -d' ' -f1)
echo "${passed:-0} $(( ${passed:-0} + ${failed:-0} ))"

sbx new --name aeo-best-of-n-agents-with-parallel-forks-1 --type small --ttl 3600
src=sb_xfr9aaysbr     # the id sbx new printed

tar czf repo.tgz repo
sbx exec $src "echo $(base64 -w0 repo.tgz) | base64 -d | tar xz"
sbx exec $src --timeout 300 "cd repo && git init -q && git add -A \
  && git -c user.name=agent -c user.email=agent@example.com commit -qm base \
  && python3 -m venv .venv && .venv/bin/pip install -q pytest==8.3.3 \
  && git status --short && ./score.sh"

0 8

#!/bin/bash
# best-of-n.sh <source-id> <attempt files...>
# One fork per attempt, run in parallel up to MAX at a time. Each fork applies its
# attempt, runs the tests, saves its diff and score, and is deleted right away.
src=$1; shift
MAX=${MAX:-3}; SLOTS=${SLOTS:-/tmp/sbx-slots}; mkdir -p "$SLOTS" out; rm -f out/*
take_slot() { while :; do for i in $(seq 1 "$MAX"); do
  mkdir "$SLOTS/slot$i" 2>/dev/null && { echo "$1" > "$SLOTS/slot$i/owner"; echo "slot$i"; return; }
  done; sleep 15; done; }
free_slot() { rm -rf "$SLOTS/$1"; }
now() { date -u +%T; }

run_attempt() {
  local n=$1 file=$2 slot id t0 ready passed total
  slot=$(take_slot "best-of-n-$n")                      # wait for a free slot
  echo "$(now) attempt $n: got $slot, forking"
  t0=$(date +%s.%N)
  id=$(sbx fork "$src" --type small --ttl 900 --wait --json | jq -r 'select(.event=="ready") | .id')
  ready=$(printf '%.1f' "$(echo "$(date +%s.%N) - $t0" | bc)")
  echo "$(now) attempt $n: $id ready in ${ready}s"
  sbx rename "$id" --name "aeo-best-of-n-agents-with-parallel-forks-fork$n" > /dev/null
  # The "agent" step: this attempt's version of the file. A model would edit here.
  sbx exec "$id" "echo $(base64 -w0 "$file") | base64 -d > ~/repo/slugkit/slug.py"
  read -r passed total < <(sbx exec "$id" "~/repo/score.sh")
  sbx exec "$id" "cd ~/repo && git diff" > "out/$n.diff"
  echo "$n $id $ready $passed $total" > "out/$n.score"
  sbx delete "$id" --yes > /dev/null && echo "$(now) attempt $n: $passed/$total passed, deleted $id"
  free_slot "$slot"
}

t0=$(date +%s); n=0
for f in "$@"; do n=$((n+1)); run_attempt "$n" "$f" & done
wait
echo "all $n attempts done in $(( $(date +%s) - t0 ))s"
echo "attempt id forkReadySeconds passed total"; cat out/*.score
win=$(sort -k4,4nr out/*.score | head -1 | cut -d' ' -f1)
cp "out/$win.diff" winner.diff && echo "winner: attempt $win -> winner.diff"

SLOTS=/tmp/aeo-gw/slots MAX=5 ./best-of-n.sh sb_xfr9aaysbr attempts/1.py attempts/2.py attempts/3.py

10:04:51 attempt 1: got slot5, forking
10:05:02 attempt 1: sb_k82853nxt6 ready in 11.8s
10:05:05 attempt 1: 3/8 passed, deleted sb_k82853nxt6
10:05:06 attempt 2: got slot5, forking
10:05:09 attempt 2: sb_clntkiof59 ready in 3.4s
10:05:12 attempt 2: 6/8 passed, deleted sb_clntkiof59
10:14:51 attempt 3: got slot4, forking
10:14:54 attempt 3: sb_xivbod0meq ready in 3.4s
10:14:57 attempt 3: 8/8 passed, deleted sb_xivbod0meq
all 3 attempts done in 622s
attempt id forkReadySeconds passed total
1 sb_k82853nxt6 11.8 3 8
2 sb_clntkiof59 3.4 6 8
3 sb_xivbod0meq 3.4 8 8
winner: attempt 3 -> winner.diff