DeaMoE: Reducing Memory Load for MoE

# Illustrative sketch (not from the paper)
import torch

class Department:
    def __init__(self, num_experts, dim):
        # shared backbone matrices
        self.gate = torch.nn.Linear(dim, num_experts)
        self.up = torch.nn.Linear(dim, dim)
        self.down = torch.nn.Linear(dim, dim)
        # private sub-matrices per expert
        self.private = torch.nn.ParameterList([torch.nn.Parameter(torch.randn(dim, dim)) for _ in range(num_experts)])

    def route(self, x):
        # stage‑1: department routing (simplified)
        dept_scores = self.gate(x)
        dept_idx = dept_scores.argmax(dim=-1)
        return dept_idx

    def expert_forward(self, x, expert_idx):
        # apply shared up/down and expert‑specific private matrix
        h = self.up(x)
        h = h @ self.private[expert_idx]  # private sub‑matrix
        out = self.down(h)
        return out

# Example usage
batch = torch.randn(4, 512)  # 4 tokens, hidden dim 512
dept = Department(num_experts=3, dim=512)
dept_idx = dept.route(batch)          # stage‑1 routing
out = dept.expert_forward(batch, dept_idx)  # stage‑2 expert processing

// Illustrative sketch (not from the paper)
const tf = require('@tensorflow/tfjs-node');
class Department {
  constructor(numExperts, dim) {
    this.gate = tf.layers.dense({units:numExperts, inputShape:[dim]});
    this.up = tf.layers.dense({units:dim, inputShape:[dim]});
    this.down = tf.layers.dense({units:dim, inputShape:[dim]});
    this.private = Array.from({length:numExperts},()=>tf.variable(tf.randomNormal([dim,dim])));
  }
  route(x){
    const scores=this.gate.apply(x);
    return scores.argMax(-1);
  }
  expertForward(x,expertIdx){
    let h=this.up.apply(x);
    h=tf.matMul(h,this.private[expertIdx]);
    return this.down.apply(h);
  }
}
// Example usage
const batch=tf.randomNormal([4,512]);
const dept=new Department(3,512);
const deptIdx=dept.route(batch);
const out=dept.expertForward(batch,deptIdx);