Extracting Structured Knowledge from Language Models

# Illustrative sketch (not from the paper)
import json, re
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer

# Load a small instruction-tuned model (e.g., Mistral-Small-24B-Instruct-2501)
model_name = "Mistral-Small-24B-Instruct-2501"
model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16)
tokenizer = AutoTokenizer.from_pretrained(model_name)
model.eval()

def elicit_relation(entity, relation):
    """Stage 1: Relation‑aware elicitation via CoT prompting."""
    # Simple relation‑specific prompt (illustrative only)
    prompt = f"Answer the following with a step‑by‑step chain‑of‑thought.\nEntity: {entity}\nRelation: {relation}\nProvide the answer as a JSON array."
    inputs = tokenizer(prompt, return_tensors="pt")
    with torch.no_grad():
        output = model.generate(**inputs, max_new_tokens=200)
    return tokenizer.decode(output[0], skip_special_tokens=True)

def hybrid_parse(raw_text):
    """Stage 2: Deterministic JSON parsing with LLM fallback."""
    try:
        # Direct deterministic parsing
        return json.loads(raw_text)
    except json.JSONDecodeError:
        # Fallback: ask LLM to re‑format (mocked here as empty list)
        return []

def post_process(records):
    """Automated cleaning: title/noise filter, parentheses removal, case‑insensitive deduplication."""
    cleaned = []
    seen = set()
    for rec in records:
        # Remove surrounding whitespace and parentheses content
        rec = rec.strip()
        rec = re.sub(r"\s*\(.*?\)\s*", "", rec)
        # Simple title/noise filter (skip empty strings)
        if not rec:
            continue
        key = rec.lower()
        if key not in seen:
            seen.add(key)
            cleaned.append(rec)
    return cleaned

# Example usage
entity = "Apple Inc."
relation = "companyTradesAtStockExchange"
raw = elicit_relation(entity, relation)
records = hybrid_parse(raw)
final = post_process(records)
print(final)

// Illustrative sketch (not from the paper)
const fs = require('fs');
const { pipeline } = require('stream');
const { TextDecoder } = require('util');
// Placeholder for an LLM inference call (e.g., via HuggingFace Inference API)
async function elicitRelation(entity, relation) {
  const prompt = `Answer with a step-by-step chain-of-thought.\nEntity: ${entity}\nRelation: ${relation}\nProvide the answer as a JSON array.`;
  // In practice, send `prompt` to the model and get `rawOutput`
  const rawOutput = await mockLLMGenerate(prompt);
  return rawOutput;
}

async function mockLLMGenerate(prompt) {
  // Mocked response – replace with real API call
  return '["NASDAQ", "NYSE"]';
}

function hybridParse(rawText) {
  try {
    // Direct deterministic parsing
    return JSON.parse(rawText);
  } catch (e) {
    // Fallback extraction (return empty array as placeholder)
    return [];
  }
}

function postProcess(records) {
  const cleaned = [];
  const seen = new Set();
  for (let rec of records) {
    rec = rec.trim();
    // Remove parenthetical content
    rec = rec.replace(/\s*\(.*?\)\s*/g, '');
    if (!rec) continue; // title/noise filter
    const key = rec.toLowerCase();
    if (!seen.has(key)) {
      seen.add(key);
      cleaned.push(rec);
    }
  }
  return cleaned;
}

(async () => {
  const entity = 'Apple Inc.';
  const relation = 'companyTradesAtStockExchange';
  const raw = await elicitRelation(entity, relation);
  const records = hybridParse(raw);
  const final = postProcess(records);
  console.log(final);
})();