How Do I Check That an Agent's Change Improved a Metric?

# search.py: rank documents for a query. This is the file the agent is asked to improve.
import re

def tokens(text):
    return re.findall(r"[a-z0-9]+", text.lower())

class Index:
    def __init__(self, docs):
        self.docs = {name: tokens(name + " " + text) for name, text in docs.items()}

    def search(self, query, k=10):
        q = set(tokens(query))
        scores = {name: sum(1 for t in toks if t in q) for name, toks in self.docs.items()}
        return sorted(scores, key=lambda n: (-scores[n], n))[:k]

# eval.py: score search.py on the labeled queries. Prints one JSON line.
import json, os, statistics, time
from search import Index

docs = json.load(open("data/docs.json"))
queries = json.load(open("data/queries.json"))   # [["read and write zip archive files", "zipfile"], …]
t = time.perf_counter(); index = Index(docs); build_ms = (time.perf_counter() - t) * 1000

rr, hit1, hit3 = [], 0, 0
for q, want in queries:
    ranked = index.search(q)
    rank = ranked.index(want) + 1 if want in ranked else None
    rr.append(1 / rank if rank else 0.0)
    hit1 += rank == 1
    hit3 += bool(rank and rank <= 3)

passes = []
for _ in range(int(os.environ.get("REPEATS", "15"))):
    t = time.perf_counter()
    for q, _want in queries:
        index.search(q)
    passes.append((time.perf_counter() - t) * 1000 / len(queries))

print(json.dumps({"mrr": round(sum(rr) / len(rr), 3), "hit1": hit1, "hit3": hit3, "n": len(queries),
                  "msPerQuery": round(statistics.median(passes), 3), "buildMs": round(build_ms, 1)}))

#!/bin/bash
# bench.sh <rounds> <label=dir>...   Run eval.py in each checkout, interleaved, for N rounds.
rounds=$1; shift
for r in $(seq 1 "$rounds"); do
  for arm in "$@"; do
    label=${arm%%=*}; dir=${arm#*=}
    (cd "$dir" && python3 eval.py) | sed "s/^{/{\"arm\":\"$label\",\"round\":$r,/"
  done
done

sbx new --name search-src --type small --ttl 3600
src=sb_imdxffhhfg       # the id sbx new printed

tar czf repo.tgz repo
sbx exec $src "echo $(base64 -w0 repo.tgz) | base64 -d | tar xz"
sbx exec $src --timeout 120 "cd repo && python3 --version && python3 build_corpus.py \
  && git init -q && git add -A && git -c user.name=agent -c user.email=agent@example.com commit -qm base \
  && git status --short && python3 eval.py"

Python 3.12.3
79 docs, 50859 chars
{"mrr": 0.44, "hit1": 10, "hit3": 16, "n": 30, "msPerQuery": 0.177, "buildMs": 1.0}

sbx stop $src --wait

id=$(sbx fork $src --name search-aa --type small --ttl 900 --wait --json \
  | jq -r 'select(.event=="ready") | .id')
sbx exec $id --timeout 120 "cd ~/repo && ./bench.sh 5 base=." | tee -a runs.jsonl
sbx delete $id --yes

{"arm":"base","round":1,"mrr": 0.44, "hit1": 10, "hit3": 16, "n": 30, "msPerQuery": 0.172, "buildMs": 1.1}
{"arm":"base","round":2,"mrr": 0.44, "hit1": 10, "hit3": 16, "n": 30, "msPerQuery": 0.178, "buildMs": 1.0}
{"arm":"base","round":3,"mrr": 0.44, "hit1": 10, "hit3": 16, "n": 30, "msPerQuery": 0.172, "buildMs": 2.7}
{"arm":"base","round":4,"mrr": 0.44, "hit1": 10, "hit3": 16, "n": 30, "msPerQuery": 0.179, "buildMs": 2.6}
{"arm":"base","round":5,"mrr": 0.44, "hit1": 10, "hit3": 16, "n": 30, "msPerQuery": 0.18, "buildMs": 2.6}

id=$(sbx fork $src --name search-ab --type small --ttl 900 --wait --json \
  | jq -r 'select(.event=="ready") | .id')
sbx exec $id "cd ~/repo && git worktree add -q ../base HEAD && git worktree add -q ../try1 HEAD \
  && echo $(base64 -w0 try1.py) | base64 -d > ../try1/search.py \
  && echo $(base64 -w0 try2.py) | base64 -d > search.py && git worktree list"
sbx exec $id --timeout 120 "cd ~/repo && ./bench.sh 5 base=../base try1=../try1 try2=." > runsB.jsonl

/home/user/repo  ecbe4c3 [master]
/home/user/base  ecbe4c3 (detached HEAD)
/home/user/try1  ecbe4c3 (detached HEAD)

{"arm":"base","round":1,"mrr": 0.44, "hit1": 10, "hit3": 16, "n": 30, "msPerQuery": 0.174, …}
{"arm":"try1","round":1,"mrr": 0.407, "hit1": 9, "hit3": 15, "n": 30, "msPerQuery": 0.067, …}
{"arm":"try2","round":1,"mrr": 0.906, "hit1": 26, "hit3": 28, "n": 30, "msPerQuery": 0.057, …}
…
{"arm":"base","round":5,"mrr": 0.44, "hit1": 10, "hit3": 16, "n": 30, "msPerQuery": 0.174, …}
{"arm":"try1","round":5,"mrr": 0.407, "hit1": 9, "hit3": 15, "n": 30, "msPerQuery": 0.067, …}
{"arm":"try2","round":5,"mrr": 0.906, "hit1": 26, "hit3": 28, "n": 30, "msPerQuery": 0.057, …}