Why AI Agents Change Their Answers

# Illustrative sketch (not from the paper)
import random, itertools, math
import torch

# 1. Fixed snapshot configuration (prompt, model id, retrieval policy, etc.)
PROMPT = "Answer the question using retrieved evidence."
MODEL_ID = "deepseek-v4-flash"
RETRIEVAL_POLICY = "top_k=5"
EVIDENCE_DEPTH = 2

# 2. Simulated function to get two independent generator responses per question
def generate_responses(question, corpus_version):
    # In practice call the LLM with fixed seed per call to get stochasticity
    return [f"resp_{question}_{corpus_version}_{i}" for i in (0, 1)]

# 3. Compute normalized‑exact agreement (binary match) and blinded semantic agreement (placeholder)
def exact_agreement(a, b):
    return int(a == b)

def semantic_agreement(a, b):
    # Placeholder for blinded human/ML judgment; returns 0/1
    return random.choice([0, 1])

# 4. Snapshot Compatibility Audit
def audit_snapshot(questions, corpus_versions):
    results = []
    for q in questions:
        # draw responses for each corpus version
        resp = {v: generate_responses(q, v) for v in corpus_versions}
        # within‑snapshot disagreement (noise floor)
        within = [1 - exact_agreement(r1, r2) for r1, r2 in resp[corpus_versions[0]]]
        # cross‑snapshot disagreement
        cross = []
        for v1, v2 in itertools.combinations(corpus_versions, 2):
            for r1 in resp[v1]:
                for r2 in resp[v2]:
                    cross.append(1 - exact_agreement(r1, r2))
        # excess churn = cross - mean(within)
        excess = sum(cross) / len(cross) - sum(within) / len(within)
        results.append(excess)
    return results

# 5. Bootstrap confidence intervals using PyTorch
def bootstrap_ci(data, n_boot=1000, alpha=0.05):
    tensor = torch.tensor(data, dtype=torch.float32)
    boots = []
    for _ in range(n_boot):
        idx = torch.randint(0, len(tensor), (len(tensor),))
        boots.append(tensor[idx].mean().item())
    lower = torch.quantile(torch.tensor(boots), alpha/2).item()
    upper = torch.quantile(torch.tensor(boots), 1-alpha/2).item()
    return lower, upper

# Example usage (illustrative)
questions = ["Q1", "Q2", "Q3"]
corpus_versions = ["v1", "v2"]
excess_churn = audit_snapshot(questions, corpus_versions)
ci = bootstrap_ci(excess_churn)
print("Excess churn:", excess_churn)
print("95% CI:", ci)

// Illustrative sketch (not from the paper)
const crypto = require('crypto');

// 1. Fixed snapshot configuration
const PROMPT = "Answer the question using retrieved evidence.";
const MODEL_ID = "deepseek-v4-flash";
const RETRIEVAL_POLICY = "top_k=5";
const EVIDENCE_DEPTH = 2;

// 2. Simulated generator returning two independent responses per call
function generateResponses(question, corpusVersion) {
  // In practice invoke the LLM with deterministic seed per call
  return [
    `resp_${question}_${corpusVersion}_0`,
    `resp_${question}_${corpusVersion}_1`
  ];
}

// 3. Agreement metrics (exact and blinded semantic placeholder)
function exactAgreement(a, b) { return a === b ? 1 : 0; }
function semanticAgreement(a, b) { // random placeholder for blinded judgment
  return Math.random() < 0.5 ? 1 : 0;
}

// 4. Snapshot Compatibility Audit
function auditSnapshot(questions, corpusVersions) {
  const results = [];
  for (const q of questions) {
    const resp = {};
    for (const v of corpusVersions) {
      resp[v] = generateResponses(q, v);
    }
    // within‑snapshot disagreement (noise floor)
    const within = resp[corpusVersions[0]].map((r, i) => 1 - exactAgreement(r, resp[corpusVersions[0]][i]));
    // cross‑snapshot disagreement
    const cross = [];
    for (let i = 0; i < corpusVersions.length; i++) {
      for (let j = i + 1; j < corpusVersions.length; j++) {
        for (const r1 of resp[corpusVersions[i]]) {
          for (const r2 of resp[corpusVersions[j]]) {
            cross.push(1 - exactAgreement(r1, r2));
          }
        }
      }
    }
    const mean = arr => arr.reduce((a, b) => a + b, 0) / arr.length;
    const excess = mean(cross) - mean(within);
    results.push(excess);
  }
  return results;
}

// 5. Bootstrap confidence interval (simple implementation)
function bootstrapCI(data, nBoot = 1000, alpha = 0.05) {
  const boots = [];
  for (let b = 0; b < nBoot; b++) {
    const sample = [];
    for (let i = 0; i < data.length; i++) {
      const idx = Math.floor(Math.random() * data.length);
      sample.push(data[idx]);
    }
    const mean = sample.reduce((a, c) => a + c, 0) / sample.length;
    boots.push(mean);
  }
  boots.sort((a, b) => a - b);
  const lower = boots[Math.floor((alpha / 2) * nBoot)];
  const upper = boots[Math.floor((1 - alpha / 2) * nBoot)];
  return [lower, upper];
}

// Example usage (illustrative)
const questions = ['Q1', 'Q2', 'Q3'];
const corpusVersions = ['v1', 'v2'];
const excessChurn = auditSnapshot(questions, corpusVersions);
const ci = bootstrapCI(excessChurn);
console.log('Excess churn:', excessChurn);
console.log('95% CI:', ci);