Evaluating Image Captions with Dense Question Answering

# Illustrative sketch (not from the paper)
import torch
from pathlib import Path

# 1. Load curated image list (346 images spanning 37 L1 / 219 L2 domains)
image_paths = list(Path('capprobe_images').glob('*.jpg'))  # placeholder

# 2. Segment each image into objects (YOLOv26‑seg) and stuff (SAM3)
seg_model = torch.hub.load('ultralytics/yolov5', 'yolov5s')  # stand‑in for YOLOv26‑seg

def segment_image(img_path):
    # returns list of masks (region_id, mask_tensor)
    # actual implementation uses YOLOv26‑seg + SAM3
    return [(i, torch.randn(1, 224, 224)) for i in range(5)]  # mock 5 regions

# 3. Generate structured metadata per region via Gemini‑3.1‑Pro

def gen_metadata(region_mask, img_path):
    # placeholder call to Gemini‑3.1‑Pro conditioned on image+mask
    return {"attributes": "..."}

# 4. Create dense MCQs (10 semantic categories) using Gemini‑3.1‑Pro & GPT‑5.5

def gen_mcqs(metadata):
    # returns list of {question, options, answer}
    return [{"q": "...", "opts": ["A","B","C","Uncertain"], "ans": "A"}]

# 5. Deduplicate questions (cross‑region & intra‑region) via embedding similarity

def dedup_questions(all_qs):
    # mock deduplication reduces count (23.2% reduction reported)
    return all_qs[:int(len(all_qs)*0.768)]

# 6. Evaluate a caption by answering MCQs using a language judge

def judge_caption(caption, questions):
    # placeholder language model answering
    correct = sum(1 for q in questions if q["ans"] == "A")  # mock count
    return {"overall_acc": correct/len(questions), "coverage": len(questions)/74}  # 74≈avg QA per image

# Main workflow (illustrative)
all_questions = []
for img_path in image_paths:
    regions = segment_image(img_path)
    for _, mask in regions:
        meta = gen_metadata(mask, img_path)
        all_questions.extend(gen_mcqs(meta))

unique_qs = dedup_questions(all_questions)
metrics = judge_caption("sample caption", unique_qs)
print(metrics)  # e.g., Overall Accuracy ~68.7%, Coverage ~73.7% as reported

// Illustrative sketch (not from the paper)
const fs = require('fs');
const path = require('path');

// 1. Load curated image list (346 images across 37 L1 / 219 L2 domains)
const imageDir = 'capprobe_images';
const imagePaths = fs.readdirSync(imageDir).filter(f => f.endsWith('.jpg')).map(f => path.join(imageDir, f));

// 2. Segment each image (YOLOv26‑seg + SAM3) – placeholder function
function segmentImage(imgPath) {
  // returns array of region objects {id, mask}
  // mock 5 regions per image
  return Array.from({length:5}, (_,i)=>({id:i, mask: Buffer.alloc(0)}));
}

// 3. Generate structured metadata per region via Gemini‑3.1‑Pro
function generateMetadata(region, imgPath) {
  // placeholder call to Gemini‑3.1‑Pro with image+mask
  return {attributes: '...'};
}

// 4. Create dense MCQs (10 semantic categories) using Gemini‑3.1‑Pro & GPT‑5.5
function generateMCQs(metadata) {
  return [{question: '...', options: ['A','B','C','Uncertain'], answer: 'A'}];
}

// 5. Deduplicate questions (cross‑region & intra‑region) via embedding similarity
function deduplicate(questions) {
  // mock 23.2% reduction as reported
  return questions.slice(0, Math.floor(questions.length * 0.768));
}

// 6. Evaluate caption by answering MCQs with a language judge
function judgeCaption(caption, questions) {
  // placeholder language model answering
  const correct = questions.filter(q => q.answer === 'A').length; // mock
  return {
    overallAcc: correct / questions.length,
    coverage: questions.length / 74 // 74 ≈ avg QA per image
  };
}

// Main workflow (illustrative)
let allQuestions = [];
for (const imgPath of imagePaths) {
  const regions = segmentImage(imgPath);
  for (const region of regions) {
    const meta = generateMetadata(region, imgPath);
    allQuestions.push(...generateMCQs(meta));
  }
}

const uniqueQs = deduplicate(allQuestions);
const metrics = judgeCaption('sample caption', uniqueQs);
console.log(metrics); // Expected Overall Accuracy ~68.7%, Coverage ~73.7% per paper