Controlling Mixture of Experts Language Models

# Illustrative sketch (not from the paper)
import torch

def rare_moe_layer(h, R, k, H_pos, H_neg):
    # 1. Router logits and probabilities
    z = R @ h.T  # shape: (num_experts, batch)
    p = torch.softmax(z, dim=0)
    # 2. Top‑k expert selection
    topk_vals, topk_idx = torch.topk(p, k, dim=0)
    # (weighted sum of selected experts omitted for brevity)
    # 3. Contrastive means
    mu_pos = H_pos.mean(dim=0)
    mu_neg = H_neg.mean(dim=0)
    delta_mu = mu_pos - mu_neg  # Δμ_ℓ
    # 4. Perturbation estimator (AffineGaussian placeholder)
    u = delta_mu * torch.randn_like(delta_mu)  # raw perturbation
    # 5. Orthonormal basis of router row space
    Q, _ = torch.linalg.qr(R.T)  # Q spans row space of R
    # 6. Null‑space projector
    Pi_perp = torch.eye(R.shape[1]) - Q @ Q.T
    # 7. Project perturbation (router‑invisible)
    h_pert = h + Pi_perp @ u
    # 8. Router‑protected correction using stored clean state
    h0, z0 = h.clone(), z.clone()  # assume stored earlier
    h_tilde = h0 + Pi_perp @ (h_pert - h0)
    return h_tilde, z0

# Example tensors (dimensions follow model conventions)
h = torch.randn(1, 1024)          # hidden state
R = torch.randn(64, 1024)          # router weight matrix
H_pos = torch.randn(10, 1024)      # positive examples
H_neg = torch.randn(10, 1024)      # negative examples
h_tilde, z0 = rare_moe_layer(h, R, k=2, H_pos=H_pos, H_neg=H_neg)

// Illustrative sketch (not from the paper)
const tf = require('@tensorflow/tfjs-node');

function rareMoeLayer(h, R, k, Hpos, Hneg) {
  // 1. Router logits and probabilities
  const z = tf.matMul(R, h.transpose()); // [numExperts, batch]
  const p = tf.softmax(z, 0);
  // 2. Top‑k expert selection (indices only)
  const { values: topVals, indices: topIdx } = tf.topk(p, k, true);
  // 3. Contrastive means
  const muPos = Hpos.mean(0);
  const muNeg = Hneg.mean(0);
  const deltaMu = tf.sub(muPos, muNeg); // Δμ_ℓ
  // 4. Perturbation estimator (AffineGaussian placeholder)
  const u = tf.mul(deltaMu, tf.randomNormal(deltaMu.shape));
  // 5. Orthonormal basis of router row space via QR
  const qr = tf.linalg.qr(R.transpose());
  const Q = qr.q; // spans row space of R
  // 6. Null‑space projector
  const I = tf.eye(R.shape[1]);
  const PiPerp = tf.sub(I, tf.matMul(Q, Q.transpose()));
  // 7. Project perturbation (router‑invisible)
  const hPert = tf.add(h, tf.matMul(PiPerp, u));
  // 8. Router‑protected correction using stored clean state
  const h0 = h.clone(); // assume stored earlier
  const hTilde = tf.add(h0, tf.matMul(PiPerp, tf.sub(hPert, h0)));
  return { hTilde, z0: z.clone() };
}

// Example tensors (shapes follow model conventions)
const h = tf.randomNormal([1, 1024]);
const R = tf.randomNormal([64, 1024]);
const Hpos = tf.randomNormal([10, 1024]);
const Hneg = tf.randomNormal([10, 1024]);
const { hTilde, z0 } = rareMoeLayer(h, R, 2, Hpos, Hneg);