Benchmarking Financial Document Question Answering Systems

# Illustrative sketch (not from the paper)
import torch, torch.nn.functional as F
embedder = torch.hub.load('e5-mistral-7b-instruct', 'model')  # mock

def encode(t): return F.normalize(embedder.encode(t), dim=-1)

queries = ["What is the revenue for Company A?"]
gold = ["Revenue increased to $5B in FY2023."]
hard = ["Revenue decreased to $3B in FY2022."]
q, p, n = encode(queries[0]), encode(gold[0]), encode(hard[0])
pos, neg = (q * p).sum(), (q * n).sum()
pairwise_acc = (pos > neg).float().item()
print("Pairwise accuracy:", pairwise_acc)
# Mock Recall@10 using BM25 placeholder
retrieved = ["..."]  # top‑k passages
recall10 = 1.0 if gold[0] in retrieved[:10] else 0.0
print("Recall@10:", recall10)

// Illustrative sketch (not from the paper)
const tf = require('@tensorflow/tfjs-node');
// Mock embedder loader (e5-mistral-7b-instruct)
const embedder = { encode: (txt) => tf.randomNormal([768]) }; // placeholder
function encode(text){ return tf.linalg.l2Normalize(embedder.encode(text), -1); }

const queries = ["What is the revenue for Company A?"];
const gold = ["Revenue increased to $5B in FY2023."];
const hard = ["Revenue decreased to $3B in FY2022."];
const q = encode(queries[0]);
const p = encode(gold[0]);
const n = encode(hard[0]);
const pos = tf.sum(tf.mul(q, p)).arraySync();
const neg = tf.sum(tf.mul(q, n)).arraySync();
const pairwiseAcc = pos > neg ? 1 : 0;
console.log("Pairwise accuracy:", pairwiseAcc);
// Mock Recall@10 with BM25 placeholder
const retrieved = ["..."]; // top‑k passages
const recall10 = retrieved.slice(0,10).includes(gold[0]) ? 1.0 : 0.0;
console.log("Recall@10:", recall10);