Improving Enterprise Agent Performance with Evolving Harnesses

# Illustrative sketch (not from the paper)
import torch

# 1. Load benchmark tasks and stratify them
tasks = load_tasks()  # placeholder
strata = stratify(tasks, by=["failure_mode", "score", "verifier_pass"])

# 2. Split into visible, hidden, and held‑out sets
visible = strata["proposer_visible"]
hidden = strata["proposer_hidden"]
held_out = strata["evaluation"]

# 3. Outer‑loop evolution
harness = load_default_harness()
for generation in range(MAX_GENERATIONS):
    # Proposer (GPT‑5.4) suggests a code delta for the harness
    patch = proposer_generate_delta(harness, visible)

    # 4. Validator checks scope, imports, and a smoke test on one task
    if not validator_check(patch, visible[0]):
        continue  # reject invalid patch

    # 5. Apply patch temporarily and test on hidden selection
    new_harness = apply_patch(harness, patch)
    hidden_score = evaluate(new_harness, hidden)
    base_score = evaluate(harness, hidden)

    # 6. Commit if mean selection score improves
    if hidden_score.mean() > base_score.mean():
        harness = new_harness  # commit
        log_success(patch, hidden_score.mean())
    else:
        log_failure(patch)

# 7. Final evaluation on held‑out tasks (no further evolution)
final_score = evaluate(harness, held_out)
print("Final held‑out score:", final_score.mean())

// Illustrative sketch (not from the paper)
const torch = require('torch-js'); // placeholder for any tensor ops

// 1. Load and stratify benchmark tasks
const tasks = loadTasks(); // placeholder
const strata = stratify(tasks, ['failure_mode', 'score', 'verifier_pass']);

// 2. Partition task pool
const visible = strata.proposer_visible;
const hidden = strata.proposer_hidden;
const heldOut = strata.evaluation;

// 3. Outer‑loop evolution
let harness = loadDefaultHarness();
for (let gen = 0; gen < MAX_GENERATIONS; gen++) {
  // Proposer (GPT‑5.4) creates a delta patch
  const patch = proposerGenerateDelta(harness, visible);

  // 4. Validator runs scope/import checks and a smoke test on one task
  if (!validatorCheck(patch, visible[0])) continue; // reject

  // 5. Apply patch temporarily and evaluate on hidden selection
  const newHarness = applyPatch(harness, patch);
  const hiddenScore = evaluate(newHarness, hidden);
  const baseScore = evaluate(harness, hidden);

  // 6. Commit if mean score improves
  if (mean(hiddenScore) > mean(baseScore)) {
    harness = newHarness; // commit
    logSuccess(patch, mean(hiddenScore));
  } else {
    logFailure(patch);
  }
}

// 7. Final held‑out evaluation (no further evolution)
const finalScore = evaluate(harness, heldOut);
console.log('Final held‑out score:', mean(finalScore));