Recovering Compressed 4 Bit LLMs

# Illustrative sketch (not from the paper)
import torch, torch.nn.functional as F
teacher = torch.load('original.pt').eval()
compressed = apply_compression(teacher)  # structural compression
student = compressed.clone().to(torch.bfloat16)  # recovery checkpoint
opt = torch.optim.Adam(student.parameters(), lr=1e-4)
for x in loader:  # full‑precision KL distillation
    s = student(x)
    with torch.no_grad(): t = teacher(x)
    loss = F.kl_div(F.log_softmax(s, -1), F.softmax(t, -1), reduction='batchmean')
    opt.zero_grad(); loss.backward(); opt.step()
# QAH: insert STE fake‑quantizer
class QAH(nn.Module):
    def __init__(self, fp): super().__init__(); self.fp=fp
    def forward(self, x): return (self.fp(x).round().clamp(-8,7)).to(torch.int8)
qah = QAH(student).train()
opt = torch.optim.Adam(qah.parameters(), lr=1e-5)
topk_idx = torch.topk(teacher(all_inputs), k=50, dim=-1).indices  # offline cache
for epoch in range(epochs):
    for x in loader:
        logits = qah(x)[:, topk_idx]
        with torch.no_grad(): t = teacher(x)[:, topk_idx]
        loss = F.kl_div(F.log_softmax(logits, -1), F.softmax(t, -1), reduction='batchmean')
        loss *= mask_schedule(step)  # two‑phase mask (placeholder)
        opt.zero_grad(); loss.backward(); opt.step()

// Illustrative sketch (not from the paper)
const tf = require('@tensorflow/tfjs-node');
const teacher = tf.loadLayersModel('file://original/model.json');
const compressed = applyCompression(teacher); // structural compression placeholder
let student = tf.clone(compressed).cast('bfloat16'); // recovery checkpoint
const opt = tf.train.adam(1e-4);
for await (const x of dataLoader()) { // full‑precision KL distillation
  const s = student.apply(x);
  const t = tf.tidy(() => teacher.apply(x));
  const loss = tf.losses.kullbackLeiblerDiv(tf.logSoftmax(s), tf.softmax(t));
  opt.minimize(() => loss, true, student.trainableWeights);
}
// QAH: STE fake‑quantizer layer
class QAH extends tf.layers.Layer {
  constructor(fp) { super({}); this.fp = fp; }
  call(input) { return tf.tidy(() => this.fp.apply(input).round().clipByValue(-8,7).toInt()); }
}
const qah = new QAH(student);
const opt2 = tf.train.adam(1e-5);
const topkIdx = tf.tidy(() => tf.topk(teacher.predict(allInputs), 50).indices); // offline cache
for (let epoch = 0; epoch < epochs; epoch++) {
  for await (const x of dataLoader()) {
    const logits = tf.gather(qah.apply(x), topkIdx, 1);
    const t = tf.gather(tf.tidy(() => teacher.apply(x)), topkIdx, 1);
    const loss = tf.losses.kullbackLeiblerDiv(tf.logSoftmax(logits), tf.softmax(t));
    const masked = loss.mul(maskSchedule(step)); // two‑phase mask placeholder
    opt2.minimize(() => masked, true, qah.trainableWeights);
  }
}