A Bayesian Model for RAG Evaluation

# Illustrative sketch (not from the paper)
import torch
import pyro
import pyro.distributions as dist
from pyro.infer import MCMC, NUTS

# binary latent variables: R, A, T (0/1)
# deterministic generator success G

def model(data=None):
    # prior for retrieval success
    R = pyro.sample("R", dist.Bernoulli(0.5))
    # abstention depends on R
    p_A = torch.where(R == 1, torch.tensor(0.2), torch.tensor(0.8))
    A = pyro.sample("A", dist.Bernoulli(p_A))
    # task success depends on R and A
    p_T = torch.where((R == 1) & (A == 0), torch.tensor(0.7), torch.tensor(0.1))
    T = pyro.sample("T", dist.Bernoulli(p_T))
    # deterministic generator success
    G = ((R == 0) & (A == 1)) | ((R == 1) & (A == 0) & (T == 1))
    # optional noisy observation (e.g., LLM judge) could be added here
    if data is not None:
        pyro.sample("obs_G", dist.Bernoulli(G.float()), obs=data)
    return G

# run posterior inference with NUTS
nuts_kernel = NUTS(model)
mcmc = MCMC(nuts_kernel, num_samples=200, warmup_steps=100)
mcmc.run()
print(mcmc.get_samples())

// Illustrative sketch (not from the paper)
const torch = require('torch-js'); // placeholder for tensor ops
const { sampleBernoulli } = require('prob-utils'); // mock helper

// latent binary variables R, A, T
function model(observedG = null) {
  // prior P(R)
  const R = sampleBernoulli(0.5);
  // P(A|R)
  const pA = R === 1 ? 0.2 : 0.8;
  const A = sampleBernoulli(pA);
  // P(T|A,R)
  const pT = (R === 1 && A === 0) ? 0.7 : 0.1;
  const T = sampleBernoulli(pT);
  // deterministic generator success G
  const G = (R === 0 && A === 1) || (R === 1 && A === 0 && T === 1);
  // if we have a noisy observation, compare (placeholder)
  if (observedG !== null) {
    // likelihood could be modeled here
  }
  return { R, A, T, G };
}

// Placeholder for HMC/NUTS inference loop (implementation omitted)
function runInference() {
  const samples = [];
  for (let i = 0; i < 200; i++) {
    samples.push(model());
  }
  console.log(samples);
}

runInference();