Local Distillation for Interpretable Machine Learning

# Illustrative sketch (not from the paper)
import numpy as np
from sklearn.linear_model import Lasso
from sklearn.model_selection import cross_val_score
from sklearn.metrics import mean_squared_error

# teacher: pre‑trained TabPFN (placeholder function)
teacher_predict = lambda X: np.random.rand(len(X))  # mock prediction

# data placeholders
X_train, y_train = np.random.rand(200, 10), np.random.rand(200)
X_query = np.random.rand(1, 10)

# 1. distillation strength = CV loss ratio (student/teacher)
student = Lasso(alpha=0.1)
student_cv = -cross_val_score(student, X_train, y_train, scoring='neg_mean_squared_error').mean()
teacher_cv = mean_squared_error(y_train, teacher_predict(X_train))
strength = np.sqrt(student_cv) / np.sqrt(teacher_cv)

# 2. similarity weights based on teacher prediction proximity
teacher_q = teacher_predict(X_query)[0]
weights = np.exp(-np.square(teacher_predict(X_train) - teacher_q))
weights /= weights.sum()

# 3. pseudo‑observation at query point
pseudo_y = teacher_q
X_aug = np.vstack([X_train, X_query])
y_aug = np.append(y_train, pseudo_y)
W_aug = np.append(weights, 1.0)  # give full weight to pseudo‑obs

# 4. randomized local distillation (Gaussian noise on objective)
noise = np.random.normal(scale=0.01, size=X_aug.shape[0])
W_noisy = W_aug + noise

# 5. fit weighted sparse linear model (student)
student_local = Lasso(alpha=0.1)
student_local.fit(X_aug, y_aug, sample_weight=W_noisy)

# 6. repeat randomization to get selection frequencies
freq = np.zeros(X_train.shape[1])
for _ in range(30):
    noise = np.random.normal(scale=0.01, size=X_aug.shape[0])
    W_noisy = W_aug + noise
    model = Lasso(alpha=0.1)
    model.fit(X_aug, y_aug, sample_weight=W_noisy)
    freq += (model.coef_ != 0)
freq /= 30

print('Distillation strength:', strength)
print('Feature selection frequencies:', freq)

// Illustrative sketch (not from the paper)
const tf = require('@tensorflow/tfjs-node');
const _ = require('lodash');

// mock teacher prediction function (TabPFN placeholder)
function teacherPredict(X) { return tf.randomUniform([X.shape[0], 1]); }

// synthetic data
const X_train = tf.randomUniform([200, 10]);
const y_train = tf.randomUniform([200, 1]);
const X_query = tf.randomUniform([1, 10]);

// 1. distillation strength = CV loss ratio (student/teacher) – placeholder values
const studentLoss = 0.12; // pretend CV MSE of lasso
const teacherLoss = 0.08; // MSE of teacher on train
const strength = Math.sqrt(studentLoss) / Math.sqrt(teacherLoss);

// 2. similarity weights based on teacher prediction proximity
const teacherAll = teacherPredict(X_train).reshape([200]);
const teacherQ = teacherPredict(X_query).arraySync()[0][0];
let weights = teacherAll.sub(tf.scalar(teacherQ)).square().neg().exp();
weights = weights.div(weights.sum());

// 3. pseudo‑observation at query point
const pseudoY = tf.scalar(teacherQ);
const X_aug = tf.concat([X_train, X_query], 0);
const y_aug = tf.concat([y_train, pseudoY.reshape([1,1])], 0);
let W_aug = tf.concat([weights, tf.tensor1d([1.0])]); // full weight for pseudo‑obs

// helper: fit weighted lasso (using simple gradient descent as placeholder)
async function fitLasso(X, y, sampleWeight, alpha=0.1, epochs=200) {
  const nFeatures = X.shape[1];
  let w = tf.variable(tf.zeros([nFeatures, 1]));
  const optimizer = tf.train.adam(0.05);
  for (let i=0;i<epochs;i++) {
    optimizer.minimize(() => {
      const preds = X.matMul(w);
      const diff = preds.sub(y);
      const mse = diff.square().mul(sampleWeight.reshape([-1,1])).mean();
      const l1 = w.abs().sum().mul(alpha);
      return mse.add(l1);
    });
  }
  return w;
}

// 4. randomized local distillation (Gaussian noise on weights)
async function runRandomizedFit() {
  const noise = tf.randomNormal(W_aug.shape, 0, 0.01);
  const W_noisy = W_aug.add(noise);
  const coeff = await fitLasso(X_aug, y_aug, W_noisy);
  return coeff.squeeze().arraySync();
}

(async () => {
  // single fit
  const coeff = await runRandomizedFit();
  console.log('Distillation strength:', strength);
  console.log('Local coefficients:', coeff);

  // 5. selection frequencies over many randomizations
  const reps = 30;
  let freq = Array(10).fill(0);
  for (let i=0;i<reps;i++) {
    const c = await runRandomizedFit();
    c.forEach((val, idx) => { if (Math.abs(val) > 1e-6) freq[idx] += 1; });
  }
  freq = freq.map(v => v / reps);
  console.log('Feature selection frequencies:', freq);
})();