Structured Evidence for Accurate Image Retrieval

# Illustrative sketch (not from the paper)
import torch
from transformers import AutoTokenizer, AutoModel

def normalize_query(query):
    # Convert any modality (text or image tensor) to a textual description
    if isinstance(query, torch.Tensor):  # image input placeholder
        return "image description placeholder"
    return str(query)

def parse_evidence_frame(text):
    # Build a simple Evidence Frame with three semantic slots
    frame = {"required": [], "forbidden": [], "ignorable": []}
    for token in text.split():
        if token.startswith("must:"):
            frame["required"].append(token[5:])
        elif token.startswith("no:"):
            frame["forbidden"].append(token[3:])
        else:
            frame["ignorable"].append(token)
    return frame

def rubric_score(frame, caption):
    # Deterministic slot‑wise satisfaction/violation scoring
    score = 0
    for req in frame["required"]:
        if req in caption:
            score += 1
    for forb in frame["forbidden"]:
        if forb not in caption:
            score += 1
    denom = len(frame["required"]) + len(frame["forbidden"]) + 1e-5
    return score / denom

def evirank(query, candidate_captions):
    text = normalize_query(query)
    frame = parse_evidence_frame(text)
    # Listwise comparison: rank candidates by rubric score
    scored = [(cap, rubric_score(frame, cap)) for cap in candidate_captions]
    ranked = sorted(scored, key=lambda x: x[1], reverse=True)
    return ranked

# Example usage
query = "must:red no:cat a sunny beach"
captions = ["a red ball on a sunny beach", "a cat sitting on a red mat", "a blue sky"]
print(evirank(query, captions)[:3])

// Illustrative sketch (not from the paper)
const normalizeQuery = (query) => {
  // Convert any modality to a textual description (placeholder for images)
  if (Buffer.isBuffer(query)) {
    return "image description placeholder";
  }
  return String(query);
};

const parseEvidenceFrame = (text) => {
  // Simple Evidence Frame with required, forbidden, ignorable slots
  const frame = { required: [], forbidden: [], ignorable: [] };
  text.split(/\s+/).forEach(token => {
    if (token.startsWith('must:')) frame.required.push(token.slice(5));
    else if (token.startsWith('no:')) frame.forbidden.push(token.slice(3));
    else frame.ignorable.push(token);
  });
  return frame;
};

const rubricScore = (frame, caption) => {
  // Deterministic slot‑wise satisfaction/violation scoring
  let score = 0;
  frame.required.forEach(req => { if (caption.includes(req)) score += 1; });
  frame.forbidden.forEach(forb => { if (!caption.includes(forb)) score += 1; });
  const denom = frame.required.length + frame.forbidden.length || 1e-5;
  return score / denom;
};

const evirank = (query, candidateCaptions) => {
  const text = normalizeQuery(query);
  const frame = parseEvidenceFrame(text);
  // Listwise comparison: rank by rubric score
  const scored = candidateCaptions.map(cap => ({ cap, score: rubricScore(frame, cap) }));
  return scored.sort((a, b) => b.score - a.score);
};

// Example usage
const query = 'must:red no:cat a sunny beach';
const captions = [
  'a red ball on a sunny beach',
  'a cat sitting on a red mat',
  'a blue sky'
];
console.log(evirank(query, captions).slice(0, 3));