Measuring Model Performance on Scientific Requests

# Illustrative sketch (not from the paper)
import torch

# 178 first‑turn scientific requests sampled from live traffic
requests = [f"req_{i}" for i in range(178)]

# Nine frontier models evaluated in identical sandboxes
models = [
    "gpt-5.6-sol",
    "claude-opus-5",
    "gpt-5.6-luna",
    "kimi-k3",
    "grok-4.5",
    "gemini-3.6-flash",
    "muse-spark-1.2",
    "gemma-4-31b-it",
    "nemotron-3-ultra-550b-a55b",
]

# Three identity‑blinded LLM judges
judges = ["gpt-5.6-sol", "qwen3.8-max", "grok-4.5"]

# Containers for per‑run scores
scores = {m: [] for m in models}

def run_model(model, request):
    """Execute a request end‑to‑end in a sandbox (placeholder)."""
    # Simulate whether an output file was produced
    output_file = bool(torch.rand(1) > 0.5)
    return {"output_file": output_file, "transcript": "…"}

def judge_output(judge, artifact):
    """Apply the eight‑dimension rubric and return a 0‑10 score (placeholder)."""
    return torch.rand(1).item() * 10

# 1,602 total runs (178 requests × 9 models)
for req in requests:
    for m in models:
        artifact = run_model(m, req)
        # Each judge scores the run independently
        run_scores = [judge_output(j, artifact) for j in judges]
        overall = sum(run_scores) / len(run_scores)
        scores[m].append(overall)

# Compute mean overall score per model (e.g., gpt-5.6-sol ≈ 8.04)
mean_scores = {m: sum(s) / len(s) for m, s in scores.items()}
print(mean_scores)

// Illustrative sketch (not from the paper)
const crypto = require('crypto'); // for pseudo‑randomness

// 178 first‑turn scientific requests sampled from live traffic
const requests = Array.from({ length: 178 }, (_, i) => `req_${i}`);

// Nine frontier models evaluated in identical sandboxes
const models = [
  'gpt-5.6-sol',
  'claude-opus-5',
  'gpt-5.6-luna',
  'kimi-k3',
  'grok-4.5',
  'gemini-3.6-flash',
  'muse-spark-1.2',
  'gemma-4-31b-it',
  'nemotron-3-ultra-550b-a55b',
];

// Three identity‑blinded LLM judges
const judges = ['gpt-5.6-sol', 'qwen3.8-max', 'grok-4.5'];

// Containers for per‑run scores
const scores = {};
models.forEach(m => (scores[m] = []));

function runModel(model, request) {
  // Placeholder sandbox execution; returns dummy artifact
  const outputFile = Math.random() > 0.5;
  return { outputFile, transcript: '…' };
}

function judgeOutput(judge, artifact) {
  // Placeholder rubric returning a 0‑10 score
  return Math.random() * 10;
}

// 1,602 total runs (178 requests × 9 models)
for (const req of requests) {
  for (const m of models) {
    const artifact = runModel(m, req);
    const runScores = judges.map(j => judgeOutput(j, artifact));
    const overall = runScores.reduce((a, b) => a + b, 0) / runScores.length;
    scores[m].push(overall);
  }
}

// Compute mean overall score per model (e.g., gpt-5.6-sol ≈ 8.04)
const meanScores = {};
for (const m of models) {
  const arr = scores[m];
  meanScores[m] = arr.reduce((a, b) => a + b, 0) / arr.length;
}
console.log(meanScores);