Attributing Failure Points in Agentic RAG

# Illustrative sketch (not from the paper)
import random
import torch

def run_agent(question, hops, fault_hop=None, fault_type='struct'):
    """Simulate a ReAct‑style agent across *hops*.
    If *fault_hop* matches the current hop, inject a faulty output.
    """
    trace = []
    for h in range(1, hops + 1):
        step = f"hop{h}_output"
        if h == fault_hop:
            step = f"faulty_{fault_type}_output"
        trace.append(step)
    return trace

def coverage_diagnoser(trace):
    """Coverage‑based localization: return first hop containing 'faulty'."""
    for i, step in enumerate(trace):
        if 'faulty' in step:
            return i + 1  # hops are 1‑indexed
    return None

def frozen_counterfactual(trace, fault_hop_idx):
    """Frozen‑hop counterfactual repair: replace the faulty hop and keep downstream.
    *fault_hop_idx* is zero‑based index.
    """
    if fault_hop_idx < 0 or fault_hop_idx >= len(trace):
        return trace
    repaired = trace.copy()
    repaired[fault_hop_idx] = 'repaired_output'
    return repaired

# Example on a 3‑hop MuSiQue question
question = "Example multi‑hop QA"
original_trace = run_agent(question, hops=3)
# Inject a content fault at hop 2
faulty_trace = run_agent(question, hops=3, fault_hop=2, fault_type='content')
predicted_hop = coverage_diagnoser(faulty_trace)
repaired_trace = frozen_counterfactual(faulty_trace, (predicted_hop or 0) - 1)

print("Original trace:", original_trace)
print("Faulty trace:", faulty_trace)
print("Diagnosed fault at hop:", predicted_hop)
print("Repaired trace:", repaired_trace)

// Illustrative sketch (not from the paper)
const torch = require('torch-js'); // placeholder for any tensor ops

/** Simulate a ReAct‑style agent across hops.
 *  If faultHop matches the current hop, inject a faulty output.
 */
function runAgent(question, hops, faultHop = null, faultType = 'struct') {
  const trace = [];
  for (let h = 1; h <= hops; h++) {
    let step = `hop${h}_output`;
    if (h === faultHop) {
      step = `faulty_${faultType}_output`;
    }
    trace.push(step);
  }
  return trace;
}

/** Coverage‑based diagnoser: returns first hop containing 'faulty'. */
function coverageDiagnoser(trace) {
  for (let i = 0; i < trace.length; i++) {
    if (trace[i].includes('faulty')) {
      return i + 1; // 1‑indexed hop number
    }
  }
  return null;
}

/** Frozen‑hop counterfactual repair: replace faulty hop with a repaired token. */
function frozenCounterfactual(trace, faultHopIdx) {
  if (faultHopIdx < 0 || faultHopIdx >= trace.length) return trace;
  const repaired = [...trace];
  repaired[faultHopIdx] = 'repaired_output';
  return repaired;
}

// Example usage on a 3‑hop HotpotQA question
const question = 'Example multi‑hop QA';
const originalTrace = runAgent(question, 3);
// Inject a structural fault at hop 1
const faultyTrace = runAgent(question, 3, 1, 'struct');
const predictedHop = coverageDiagnoser(faultyTrace);
const repairedTrace = frozenCounterfactual(faultyTrace, (predictedHop || 0) - 1);

console.log('Original trace:', originalTrace);
console.log('Faulty trace:', faultyTrace);
console.log('Diagnosed fault at hop:', predictedHop);
console.log('Repaired trace:', repairedTrace);