Are Sandboxer Sandboxes a Fit for Eval and Research Teams?

#!/bin/bash
# run_task.sh <task> : apply the candidate patch, run that task's tests, write ~/out/<task>/
set -u
task=$1; out=~/out/$task; mkdir -p $out
cd ~/env/repo
git apply ../patches/$task.diff && applied=true || applied=false
git diff > $out/patch.diff
t=$(date +%s.%N)
../.venv/bin/python -m pytest -q -p no:cacheprovider --junitxml=$out/junit.xml test_$task.py > $out/pytest.txt 2>&1; rc=$?
secs=$(echo "$(date +%s.%N) - $t" | bc)
cat > $out/env.json <<J
{"cpu": "$(lscpu | sed -n 's/^Model name: *//p')", "kernel": "$(uname -r)", "python": "$(../.venv/bin/python -V | cut -d' ' -f2)", "deps": "$(../.venv/bin/pip freeze | sha256sum | cut -c1-12)", "repo": "$(git rev-parse HEAD | cut -c1-12)"}
J
echo "{\"task\": \"$task\", \"applied\": $applied, \"pytestExit\": $rc, \"testSeconds\": $secs, \"summary\": \"$(tail -1 $out/pytest.txt)\"}" > $out/result.json
tar czf ~/out/$task.tgz -C ~/out $task
cat $out/result.json

sbx --json new --name eval-1 --type small --ttl 3600 | jq -r 'select(.event=="ready") | .id'
id=sb_exupvsjls5

sbx exec $id "echo $(base64 -w0 env.tgz) | base64 -d | tar xz && chmod +x env/run_task.sh"
sbx exec $id --timeout 300 "cd env && python3 -m venv .venv && .venv/bin/pip install -q pytest==8.3.3 && .venv/bin/python -m pytest --version \
  && cd repo && git init -q && git add -A && git -c user.email=eval@example.com -c user.name=eval commit -qm task-v1 \
  && git rev-parse --short=12 HEAD"
sbx exec $id "cd env/repo && ../.venv/bin/python -m pytest -q -p no:cacheprovider --tb=no | tail -1"

pytest 8.3.3
08d894442ecd
3 failed, 1 passed in 0.01s

sbx --json snapshot $id eval-task-v1
sbx delete $id --yes

{"ok":true,"type":"snapshot.named.info","snapshot":{"name":"eval-task-v1","snapshotId":"7396f9c8-…","sourceSandboxId":"sb_exupvsjls5","status":"ready","type":"small",…}}

#!/bin/bash
# fanout.sh: one sandbox per task from the named snapshot, collect artifacts, delete.
snap=eval-task-v1
run_one() {   # $1 = task, $2 = sandbox name
  t0=$(date +%s.%N)
  id=$(sbx --json new --from $snap --name $2 --type small --ttl 900 | jq -r 'select(.event=="ready") | .id')
  t1=$(date +%s.%N)
  sbx exec $id "~/env/run_task.sh $1" > /dev/null
  sbx scp $id:/home/user/out/$1.tgz results/ > /dev/null
  t2=$(date +%s.%N)
  sbx delete $id --yes > /dev/null
  t3=$(date +%s.%N)
  printf '%s %s start=%.2fs run+collect=%.2fs delete=%.2fs total=%.2fs\n' $1 $id \
    $(echo "$t1-$t0" | bc) $(echo "$t2-$t1" | bc) $(echo "$t3-$t2" | bc) $(echo "$t3-$t0" | bc)
}
mkdir -p results
s=$(date +%s.%N)
run_one median eval-2 &
run_one clamp eval-3 &
wait
printf 'wall=%.2fs\n' $(echo "$(date +%s.%N)-$s" | bc)

/home/user/env/run_task.sh: line 10: bc: command not found
/home/user/env/run_task.sh: line 10: bc: command not found
median sb_x7plngnio6 start=3.19s run+collect=3.64s delete=2.13s total=8.96s
clamp sb_bryg2ymo9f start=3.19s run+collect=3.76s delete=2.13s total=9.09s
wall=9.10s

$ cat median/result.json
{"task": "median", "applied": true, "pytestExit": 0, "testSeconds": , "summary": "2 passed in 0.01s"}
$ cat median/env.json
{"cpu": "AMD Ryzen 9 7950X3D 16-Core Processor", "kernel": "6.18.56-sandboxer", "python": "3.12.3", "deps": "90531af91104", "repo": "08d894442ecd"}
$ cat median/patch.diff
diff --git a/calc.py b/calc.py
index e7a86df..a9f4cea 100644
--- a/calc.py
+++ b/calc.py
@@ -1,6 +1,9 @@
 def median(xs):
-    xs = list(xs)
-    return xs[len(xs) // 2]
+    xs = sorted(xs)
+    n = len(xs)
+    if n % 2:
+        return xs[n // 2]
+    return (xs[n // 2 - 1] + xs[n // 2]) / 2