Agentic Multimodal Instruction Data Synthesis

# Illustrative sketch (not from the paper)
import torch

# Simple in‑memory memory for constraints and embeddings
memory = {
    "constraints": [],
    "inst_embeddings": [],
    "difficulty": []
}

def perception(image):
    # Analyze image, filter incompatible constraints, discover new ones
    # (placeholder logic)
    return ["color_match", "object_presence"]

def plan(constraints):
    # Sample constraints weighted by past success (mocked here)
    weights = torch.ones(len(constraints))
    sampled = torch.multinomial(weights, num_samples=2, replacement=False)
    return [constraints[i] for i in sampled]

def generate_instruction(sampled_constraints):
    # Embed instruction (dummy tensor) and select diverse candidates
    emb = torch.randn(1, 768)
    return "Describe the image with constraints: " + ", ".join(sampled_constraints), emb

def reflect(instruction, response):
    # Dual‑track verifier: tool check + LLM judge (mocked as bool)
    tool_ok = "error" not in response
    llm_ok = len(instruction) < 200
    return tool_ok and llm_ok

def update_memory(inst, emb, success):
    memory["inst_embeddings"].append(emb)
    memory["constraints"].append(inst)
    memory["difficulty"].append(0 if success else 1)

def rl_step(instruction, success):
    # Verifier contract as reward (1 for success, 0 otherwise)
    reward = 1.0 if success else 0.0
    # Placeholder policy gradient update
    return reward

# Main loop (single iteration shown)
image = torch.randn(3, 224, 224)  # placeholder image tensor
constraints = perception(image)
sampled = plan(constraints)
instruction, emb = generate_instruction(sampled)
# Mock model response
response = "Generated answer based on image"
success = reflect(instruction, response)
update_memory(instruction, emb, success)
reward = rl_step(instruction, success)
print("Reward:", reward)

// Illustrative sketch (not from the paper)
const torch = require('torch-js'); // placeholder import

// Simple in‑memory store
const memory = {
  constraints: [],
  instEmbeddings: [],
  difficulty: []
};

function perception(image) {
  // Analyze image, filter incompatible constraints, discover new ones
  // (placeholder logic)
  return ['color_match', 'object_presence'];
}

function plan(constraints) {
  // Sample constraints weighted by past success (mocked uniformly)
  const weights = torch.ones(constraints.length);
  const sampledIdx = torch.multinomial(weights, 2, false);
  return sampledIdx.map(i => constraints[i]);
}

function generateInstruction(sampled) {
  // Embed instruction (dummy tensor) and select diverse candidates
  const emb = torch.randn([1, 768]);
  const instr = `Describe the image with constraints: ${sampled.join(', ')}`;
  return { instr, emb };
}

function reflect(instruction, response) {
  // Dual‑track verifier: tool check + LLM judge (mocked)
  const toolOk = !response.includes('error');
  const llmOk = instruction.length < 200;
  return toolOk && llmOk;
}

function updateMemory(inst, emb, success) {
  memory.instEmbeddings.push(emb);
  memory.constraints.push(inst);
  memory.difficulty.push(success ? 0 : 1);
}

function rlStep(instruction, success) {
  // Verifier contract as reward
  const reward = success ? 1.0 : 0.0;
  // Placeholder policy gradient update
  return reward;
}

// Main loop (single iteration shown)
const image = torch.randn([3, 224, 224]); // placeholder image tensor
const constraints = perception(image);
const sampled = plan(constraints);
const { instr, emb } = generateInstruction(sampled);
const response = 'Generated answer based on image'; // mock model output
const success = reflect(instr, response);
updateMemory(instr, emb, success);
const reward = rlStep(instr, success);
console.log('Reward:', reward);