Benchmarking Explicit Strategy Induction in LLMs

# Illustrative sketch (not from the paper)
import torch
from typing import List, Dict

# 1. Load and preprocess BIG-Bench tasks for uniform format
def load_tasks(path: str) -> List[Dict]:
    # placeholder: read JSONL, normalize fields
    return []

# 2. Few‑shot in‑context learning baseline
def few_shot_predict(model, examples, query):
    prompt = "\n".join([ex['input'] + "\n" + ex['output'] for ex in examples]) + "\n" + query
    # model generates answer directly
    return model.generate(prompt)

# 3. Model‑assisted pre‑filtering to find strategy‑inducible tasks
def filter_strategy_tasks(tasks, selector_model) -> List[Dict]:
    inducible = []
    for t in tasks:
        # ask selector if a reusable strategy seems possible
        resp = selector_model.generate(f"Can a concise strategy solve: {t['description']}?")
        if "yes" in resp.lower():
            inducible.append(t)
    return inducible

# 4. Category‑aware strategy generation
def generate_strategy(task, generator_model) -> str:
    cat = task['category']  # e.g., Numerical, Logic, …
    prompt = f"Generate a short, step‑by‑step strategy for a {cat} task: {task['description']}"
    return generator_model.generate(prompt)

# 5. Evaluate downstream utility (performance gain over baseline)
def evaluate_strategy(task, strategy, executor_model, baseline_score):
    # execute task using the generated strategy
    answer = executor_model.generate(f"Strategy:\n{strategy}\nQuestion:\n{task['input']}")
    # placeholder correctness check
    correct = (answer.strip() == task['output'].strip())
    gain = 1 if correct else 0
    return gain > baseline_score

# 6. Multi‑dimensional evaluation (conciseness, format compliance, correctness)
def evaluate_quality(strategy: str) -> Dict:
    lines = strategy.strip().split('\n')
    conciseness = len(lines) <= 5
    format_ok = strategy.startswith("Step")  # naive format check
    return {"concise": conciseness, "format_compliant": format_ok}

# Example driver (mock models)
if __name__ == "__main__":
    tasks = load_tasks("bigbench_dir")
    inducible = filter_strategy_tasks(tasks, selector_model=torch.nn.Identity())
    for t in inducible[:3]:
        strat = generate_strategy(t, generator_model=torch.nn.Identity())
        quality = evaluate_quality(strat)
        print(t['id'], quality)

// Illustrative sketch (not from the paper)
const fs = require('fs');

// 1. Load and normalize BIG‑Bench tasks
function loadTasks(dir) {
  // placeholder: read JSON files, return array of task objects
  return [];
}

// 2. Few‑shot in‑context baseline
function fewShotPredict(model, examples, query) {
  const prompt = examples.map(e => `${e.input}\n${e.output}`).join('\n') + '\n' + query;
  return model.generate(prompt); // mock API
}

// 3. Pre‑filter for strategy‑inducible tasks using a selector LLM
function filterStrategyTasks(tasks, selector) {
  return tasks.filter(t => {
    const resp = selector.generate(`Can a concise strategy solve: ${t.description}?`);
    return /yes/i.test(resp);
  });
}

// 4. Category‑aware strategy generation
function generateStrategy(task, generator) {
  const prompt = `Generate a short, step‑by‑step strategy for a ${task.category} task: ${task.description}`;
  return generator.generate(prompt);
}

// 5. Downstream utility evaluation (gain over baseline)
function evaluateStrategy(task, strategy, executor, baselineScore) {
  const answer = executor.generate(`Strategy:\n${strategy}\nQuestion:\n${task.input}`);
  const correct = answer.trim() === task.output.trim();
  const gain = correct ? 1 : 0;
  return gain > baselineScore;
}

// 6. Quality metrics for the generated strategy
function evaluateQuality(strategy) {
  const lines = strategy.trim().split('\n');
  const concise = lines.length <= 5;
  const formatOk = strategy.startsWith('Step'); // simple check
  return { concise, formatOk };
}

// Mock driver
(function main() {
  const tasks = loadTasks('./bigbench');
  const selector = { generate: txt => 'yes' }; // placeholder model
  const generator = { generate: txt => 'Step 1: ...' };
  const executor = { generate: txt => 'answer' };

  const inducible = filterStrategyTasks(tasks, selector);
  inducible.slice(0, 3).forEach(t => {
    const strat = generateStrategy(t, generator);
    const quality = evaluateQuality(strat);
    console.log(t.id, quality);
  });
})();