Benchmarking AI Agents at Algorithmic Design

# Illustrative sketch (not from the paper)
import torch
import subprocess, time, os, json

# 1. Load the frozen starting model provided by the benchmark repository
def load_frozen_model(path="repo/frozen_model.pt"):
    return torch.load(path)

# 2. Compute the inexpensive proxy metric (e.g., MATH-500) on the current model
def compute_proxy_metric(model):
    # placeholder: run a quick validation loop and return a scalar
    return torch.rand(1).item()

# 3. Agent edits the training algorithm source code (as a string patch)
def apply_patch(original_path, patch_str, out_path="/tmp/modified_train.py"):
    with open(original_path, "r") as f:
        code = f.read()
    # naive patch application: replace a marker comment with the patch
    modified = code.replace("# PATCH_POINT", patch_str)
    with open(out_path, "w") as f:
        f.write(modified)
    return out_path

# 4. Execute the modified training script from scratch in a fresh container (simulated)
def run_training(script_path, timeout_hours=12):
    # In practice this would launch a Docker container; here we just subprocess call
    cmd = ["python", script_path]
    try:
        subprocess.run(cmd, timeout=timeout_hours*3600, check=True)
    except subprocess.TimeoutExpired:
        print("Training exceeded time budget")

# 5. Fixed evaluator (hidden from the agent) scores the resulting model on the final metric
def evaluate_final(model_path, metric_name="AIME"):
    # placeholder: load model and return a score between 0 and 1
    model = torch.load(model_path)
    return torch.rand(1).item()

# ---- Example orchestration (what the benchmark does) ----
model = load_frozen_model()
proxy_score = compute_proxy_metric(model)
print(f"Proxy metric: {proxy_score:.4f}")

# Agent provides a patch (as a string) during its 4‑hour window
agent_patch = "# new learning rule implementation"
script = apply_patch("repo/train_algorithm.py", agent_patch)
run_training(script)

final_score = evaluate_final("/tmp/trained_model.pt")
print(f"Final benchmark score: {final_score:.4f}")

// Illustrative sketch (not from the paper)
const fs = require('fs');
const { exec } = require('child_process');
const torch = require('torch-js'); // placeholder for PyTorch-like API

// 1. Load the frozen starting model provided by the benchmark repository
function loadFrozenModel(path = 'repo/frozen_model.pt') {
  // In practice deserialize the tensor file
  return torch.load(path);
}

// 2. Compute the inexpensive proxy metric (e.g., MATH-500) on the current model
function computeProxyMetric(model) {
  // placeholder: return a random scalar as proxy
  return Math.random();
}

// 3. Agent edits the training algorithm source code (as a string patch)
function applyPatch(originalPath, patchStr, outPath = '/tmp/modified_train.js') {
  const code = fs.readFileSync(originalPath, 'utf8');
  // naive patch: replace a marker comment with the patch string
  const modified = code.replace('// PATCH_POINT', patchStr);
  fs.writeFileSync(outPath, modified);
  return outPath;
}

// 4. Execute the modified training script from scratch in a fresh container (simulated)
function runTraining(scriptPath, timeoutHours = 12) {
  const cmd = `node ${scriptPath}`;
  const child = exec(cmd, { timeout: timeoutHours * 3600 * 1000 }, (error) => {
    if (error) {
      if (error.killed) console.log('Training exceeded time budget');
      else console.error('Training error:', error);
    }
  });
}

// 5. Fixed evaluator (hidden from the agent) scores the resulting model on the final metric
function evaluateFinal(modelPath, metricName = 'AIME') {
  // placeholder: load model and return a random score between 0 and 1
  const model = torch.load(modelPath);
  return Math.random();
}

// ---- Example orchestration (what the benchmark does) ----
const model = loadFrozenModel();
const proxyScore = computeProxyMetric(model);
console.log(`Proxy metric: ${proxyScore.toFixed(4)}`);

// Agent provides a patch during its 4‑hour window
const agentPatch = '// new learning rule implementation';
const script = applyPatch('repo/train_algorithm.js', agentPatch);
runTraining(script);

const finalScore = evaluateFinal('/tmp/trained_model.pt');
console.log(`Final benchmark score: ${finalScore.toFixed(4)}`);