Why Large Language Models Miss Information

# Illustrative sketch (not from the paper)
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer

# Load the primary 9B Qwen3.5 model (open-weight)
model_name = "Qwen3.5-9B"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)
model.eval()

# Core disclosure (e.g., covenant threshold) – kept constant across experiments
disclosure = "The firm must maintain a debt‑to‑EBITDA ratio below 3.0."

# Context lengths examined in the paper
context_lengths = [2000, 8000, 32000, 128000]

for ctx_len in context_lengths:
    # Generate genre‑matched filler (placeholder – actual filler is unrelated text)
    filler = "[FILLER] " * (ctx_len // 10)  # rough token approximation
    # Build two documents: with disclosure and with neutral replacement
    doc_with = filler + "\n" + disclosure
    doc_neutral = filler + "\n" + "[NEUTRAL PARAGRAPH]"
    # Tokenize
    inputs_with = tokenizer(doc_with, return_tensors="pt")
    inputs_neutral = tokenizer(doc_neutral, return_tensors="pt")
    # Get model logits for a decision prompt (e.g., "Should we sell? Yes/No")
    decision_prompt = "Based on the filing, should we sell the stock? Answer with Yes or No."
    with torch.no_grad():
        out_with = model.generate(**inputs_with, max_new_tokens=10, do_sample=False, eos_token_id=tokenizer.eos_token_id)
        out_neutral = model.generate(**inputs_neutral, max_new_tokens=10, do_sample=False, eos_token_id=tokenizer.eos_token_id)
    # Simple probability extraction (placeholder)
    prob_sell_with = out_with[0].float().mean().item()  # mock probability
    prob_sell_neutral = out_neutral[0].float().mean().item()
    influence = prob_sell_with - prob_sell_neutral
    print(f"Context {ctx_len} tokens – decision influence: {influence:.3f}")
    # Retrieval check – ask model to recall the disclosure directly
    retrieval_q = "What is the covenant threshold mentioned in the filing?"
    retrieval_input = tokenizer(doc_with + "\n" + retrieval_q, return_tensors="pt")
    retrieval_out = model.generate(**retrieval_input, max_new_tokens=20, do_sample=False)
    retrieved_text = tokenizer.decode(retrieval_out[0])
    retrieved = disclosure in retrieved_text
    print(f"Retrieval success at {ctx_len} tokens: {retrieved}")

# Targeted workflow: place a structured restatement adjacent to the decision prompt
restated = f"COVENANT: debt‑to‑EBITDA < 3.0.\n{decision_prompt}"
inputs_targeted = tokenizer(restated, return_tensors="pt")
with torch.no_grad():
    out_targeted = model.generate(**inputs_targeted, max_new_tokens=10, do_sample=False)
prob_targeted = out_targeted[0].float().mean().item()
print(f"Targeted workflow influence (128k tokens) – mock value: {prob_targeted:.3f}")

// Illustrative sketch (not from the paper)
const { AutoTokenizer, AutoModelForCausalLM } = require('@xenova/transformers');

(async () => {
  // Load the primary 9B Qwen3.5 model (open-weight)
  const modelName = 'Qwen3.5-9B';
  const tokenizer = await AutoTokenizer.from_pretrained(modelName);
  const model = await AutoModelForCausalLM.from_pretrained(modelName);

  // Core disclosure – constant across experiments
  const disclosure = 'The firm must maintain a debt‑to‑EBITDA ratio below 3.0.';

  // Context lengths examined in the paper
  const contextLengths = [2000, 8000, 32000, 128000];

  for (const ctxLen of contextLengths) {
    // Generate genre‑matched filler (placeholder – actual filler is unrelated text)
    const filler = '[FILLER] '.repeat(Math.floor(ctxLen / 10)); // rough token approximation
    const docWith = `${filler}\n${disclosure}`;
    const docNeutral = `${filler}\n[NEUTRAL PARAGRAPH]`;

    // Tokenize documents
    const inputsWith = await tokenizer(docWith, { returnTensors: 'pt' });
    const inputsNeutral = await tokenizer(docNeutral, { returnTensors: 'pt' });

    // Decision prompt
    const decisionPrompt = 'Based on the filing, should we sell the stock? Answer Yes or No.';

    // Generate model outputs (mock – actual generation API may differ)
    const outWith = await model.generate({ ...inputsWith, max_new_tokens: 10, do_sample: false });
    const outNeutral = await model.generate({ ...inputsNeutral, max_new_tokens: 10, do_sample: false });

    // Mock probability extraction (using mean of logits as placeholder)
    const probSellWith = outWith.logits.mean().dataSync()[0];
    const probSellNeutral = outNeutral.logits.mean().dataSync()[0];
    const influence = probSellWith - probSellNeutral;
    console.log(`Context ${ctxLen} tokens – decision influence: ${influence.toFixed(3)}`);

    // Retrieval check – ask model to recall the disclosure directly
    const retrievalQ = 'What is the covenant threshold mentioned in the filing?';
    const retrievalInput = await tokenizer(`${docWith}\n${retrievalQ}`, { returnTensors: 'pt' });
    const retrievalOut = await model.generate({ ...retrievalInput, max_new_tokens: 20, do_sample: false });
    const retrievedText = tokenizer.decode(retrievalOut.sequences[0]);
    const retrieved = retrievedText.includes(disclosure);
    console.log(`Retrieval success at ${ctxLen} tokens: ${retrieved}`);
  }

  // Targeted workflow: structured restatement adjacent to decision prompt
  const restated = `COVENANT: debt‑to‑EBITDA < 3.0.\nBased on the filing, should we sell the stock? Answer Yes or No.`;
  const inputsTargeted = await tokenizer(restated, { returnTensors: 'pt' });
  const outTargeted = await model.generate({ ...inputsTargeted, max_new_tokens: 10, do_sample: false });
  const probTargeted = outTargeted.logits.mean().dataSync()[0];
  console.log(`Targeted workflow influence (128k tokens) – mock value: ${probTargeted.toFixed(3)}`);
})();