Improving AI Code Generation With Robust Testing

# Illustrative sketch (not from the paper)
import torch
from sklearn.cluster import KMeans

# 1. Generate diverse faulty implementations (placeholder)
def generate_faulty_codes(prompt, n=10):
    return [f"faulty_impl_{i}()" for i in range(n)]

# 2. Dynamic filtering based on pass/fail criteria (placeholder)
def filter_faulty(codes):
    return [c for c in codes if "fail" not in c]  # mock condition

# 3. Synthesize test cases from faulty codes (negative examples)
def synthesize_tests(faulty_codes):
    return [f"assert {c} == expected" for c in faulty_codes]

# 4. Validate tests with executable validator (placeholder)
def validate_tests(tests):
    return [t for t in tests if "assert" in t]

# 5. Compute execution vectors (mock binary pass/fail across tests)
def exec_vectors(codes, tests):
    return torch.randint(0, 2, (len(codes), len(tests)), dtype=torch.float)

# 6. Diversity-driven selection via K‑means clustering
def select_diverse(vectors, k=3):
    km = KMeans(n_clusters=k, random_state=0).fit(vectors.numpy())
    idx = []
    for cluster in range(k):
        members = (km.labels_ == cluster).nonzero()[0]
        idx.append(members[0])  # pick first member per cluster
    return idx

# 7. Stepwise dense reward based on test pass rates
def dense_reward(vectors, selected_idx):
    selected = vectors[selected_idx]
    # reward = average pass rate per step (mock)
    return selected.mean(dim=1)

# ---- Workflow ----
prompt = "solve problem X"
faulty = generate_faulty_codes(prompt)
filtered = filter_faulty(faulty)
tests = synthesize_tests(filtered)
valid_tests = validate_tests(tests)
vecs = exec_vectors(filtered, valid_tests)
selected_idx = select_diverse(vecs)
rewards = dense_reward(vecs, selected_idx)
print("Dense rewards per selected code:", rewards)

// Illustrative sketch (not from the paper)
const { KMeans } = require('ml-kmeans'); // assume a K‑means lib

// 1. Generate faulty implementations (mock)
function generateFaultyCodes(prompt, n = 10) {
  return Array.from({ length: n }, (_, i) => `faulty_impl_${i}()`);
}

// 2. Filter based on pass/fail criteria (mock)
function filterFaulty(codes) {
  return codes.filter(c => !c.includes('fail'));
}

// 3. Synthesize tests from faulty codes
function synthesizeTests(faultyCodes) {
  return faultyCodes.map(c => `assert ${c} === expected`);
}

// 4. Validate tests with executable validator (mock)
function validateTests(tests) {
  return tests.filter(t => t.includes('assert'));
}

// 5. Mock execution vectors: 1 = pass, 0 = fail
function execVectors(codes, tests) {
  return codes.map(() => tests.map(() => Math.round(Math.random())));
}

// 6. Diversity selection via K‑means clustering
function selectDiverse(vectors, k = 3) {
  const km = new KMeans(vectors, k);
  const clusters = km.clusters; // array of cluster ids per vector
  const selected = [];
  for (let i = 0; i < k; i++) {
    const idx = clusters.findIndex(c => c === i);
    if (idx !== -1) selected.push(idx);
  }
  return selected;
}

// 7. Dense reward: average pass rate per selected code
function denseReward(vectors, selectedIdx) {
  return selectedIdx.map(i => {
    const vec = vectors[i];
    const sum = vec.reduce((a, b) => a + b, 0);
    return sum / vec.length;
  });
}

// ---- Workflow ----
const prompt = 'solve problem X';
const faulty = generateFaultyCodes(prompt);
const filtered = filterFaulty(faulty);
const tests = synthesizeTests(filtered);
const validTests = validateTests(tests);
const vectors = execVectors(filtered, validTests);
const selectedIdx = selectDiverse(vectors);
const rewards = denseReward(vectors, selectedIdx);
console.log('Dense rewards per selected code:', rewards);