Improving AI Agent Performance Through Automated Harness Evolution

# Illustrative sketch (not from the paper)
import torch
class Harness:
    def __init__(self): self.patches=[]

def mean_score(s): return torch.mean(torch.tensor(s, dtype=torch.float))

def stratify(tasks):
    n=len(tasks)//3
    return {'search':tasks[:n],'selection':tasks[n:2*n],'eval':tasks[2*n:]}

def propose(search): return {"patch":"cand"}  # placeholder

def validate(p): return True

def evaluate(p, sel):
    scores=[0.6,0.8]  # placeholder
    return mean_score(scores)

def accept(cur,new): return new>cur

def evolve(tasks,h):
    pool=stratify(tasks); cur=0.0
    for _ in range(5):
        p=propose(pool['search'])
        if not validate(p): continue
        new=evaluate(p,pool['selection'])
        if accept(cur,new): h.patches.append(p); cur=new
# Demo
tasks=["t1","t2","t3","t4","t5","t6"]
h=Harness()
evolve(tasks,h)

// Illustrative sketch (not from the paper)
class Harness { constructor(){ this.patches=[]; } }
const meanScore = s => s.reduce((a,b)=>a+b,0)/s.length;
function stratify(tasks){
  const n=Math.floor(tasks.length/3);
  return {search:tasks.slice(0,n), selection:tasks.slice(n,2*n), eval:tasks.slice(2*n)};
}
function propose(search){ return {patch:"cand"}; }
function validate(p){ return true; }
function evaluate(p, sel){
  const scores=[0.6,0.8]; // placeholder
  return meanScore(scores);
}
function accept(cur,newScore){ return newScore>cur; }
function evolve(tasks,h){
  const pool=stratify(tasks);
  let cur=0.0;
  for(let i=0;i<5;i++){
    const p=propose(pool.search);
    if(!validate(p)) continue;
    const newScore=evaluate(p,pool.selection);
    if(accept(cur,newScore)){ h.patches.push(p); cur=newScore; }
  }
}
// Demo
const tasks=["t1","t2","t3","t4","t5","t6"];
const h=new Harness();
evolve(tasks,h);