Precise Action Recognition Using Expert Models

# Illustrative sketch (not from the paper)
import torch, torch.nn as nn

# frozen backbones (placeholders for R(2+1)D‑34, PoseC3D, ST‑GCN++)
rgb_backbone = nn.Identity()
pose_backbone = nn.Identity()
graph_backbone = nn.Identity()

# project to shared D‑dimensional space
proj = nn.Linear(512, 256)  # example dimensions

# pairwise cross‑attention (single layer for brevity)
cross_attn = nn.MultiheadAttention(embed_dim=256, num_heads=4)

# latent sparse Mixture‑of‑Experts
class SparseMoE(nn.Module):
    def __init__(self, D, E=4):
        super().__init__()
        self.experts = nn.ModuleList([nn.Linear(D, D) for _ in range(E)])
        self.router = nn.Linear(D, E)  # produces logits per expert
    def forward(self, x):
        logits = self.router(x)                     # [B,T,E]
        gate = torch.softmax(logits, dim=-1)       # routing probabilities
        top_gate, idx = gate.max(dim=-1, keepdim=True)
        mask = torch.zeros_like(gate).scatter_(-1, idx, top_gate)
        out = sum(e(x) * mask[..., i:i+1] for i, e in enumerate(self.experts))
        return out, logits

moe = SparseMoE(256)

# load‑balancing regularizer
def load_balancing_loss(logits):
    probs = torch.softmax(logits, dim=-1)
    return (probs.mean(0) * probs.mean(0)).sum()

# full forward sketch
def forward(rgb, pose, graph):
    # extract (frozen) features and project
    f_rgb = proj(rgb_backbone(rgb)).unsqueeze(0)
    f_pose = proj(pose_backbone(pose)).unsqueeze(0)
    f_graph = proj(graph_backbone(graph)).unsqueeze(0)
    # pairwise cross‑attention (query‑key‑value from other streams)
    f_rgb = cross_attn(f_rgb, f_pose, f_pose)[0] + cross_attn(f_rgb, f_graph, f_graph)[0]
    f_pose = cross_attn(f_pose, f_rgb, f_rgb)[0] + cross_attn(f_pose, f_graph, f_graph)[0]
    f_graph = cross_attn(f_graph, f_rgb, f_rgb)[0] + cross_attn(f_graph, f_pose, f_pose)[0]
    # MoE refinement per stream
    f_rgb, l1 = moe(f_rgb.squeeze(0))
    f_pose, l2 = moe(f_pose.squeeze(0))
    f_graph, l3 = moe(f_graph.squeeze(0))
    # mean‑pool and classify
    pooled = torch.stack([f_rgb.mean(0), f_pose.mean(0), f_graph.mean(0)]).mean(0)
    logits = nn.Linear(256, 10)(pooled)  # 10‑class placeholder
    loss_bal = load_balancing_loss(torch.stack([l1, l2, l3]))
    return logits, loss_bal

// Illustrative sketch (not from the paper)
const tf = require('@tensorflow/tfjs-node');

// frozen backbones (identity placeholders for the three streams)
const rgbBackbone = x => x;
const poseBackbone = x => x;
const graphBackbone = x => x;

// projection to shared D‑dimensional space
const proj = tf.layers.dense({units: 256, inputShape: [512]}); // example dims

// simple multi‑head attention (single layer, using tf.layers)
const crossAttn = tf.layers.multiHeadAttention({numHeads: 4, keyDim: 256});

// latent sparse MoE
class SparseMoE {
  constructor(D, E = 4) {
    this.experts = Array.from({length: E}, () => tf.layers.dense({units: D, useBias: false}));
    this.router = tf.layers.dense({units: E, useBias: false}); // logits
  }
  call(x) {
    const logits = this.router.apply(x); // [B,T,E]
    const gate = tf.softmax(logits, -1);
    const topIdx = tf.argMax(gate, -1);
    const topGate = tf.max(gate, -1, true);
    const mask = tf.oneHot(topIdx, gate.shape[2]).mul(topGate);
    const outs = this.experts.map((e, i) => e.apply(x).mul(mask.slice([0,0,i], [-1,-1,1])));
    const out = tf.addN(outs);
    return [out, logits];
  }
}
const moe = new SparseMoE(256);

// load‑balancing loss
function loadBalancingLoss(logits) {
  const probs = tf.softmax(logits, -1);
  const mean = probs.mean(0);
  return tf.sum(tf.mul(mean, mean));
}

// full forward sketch
function forward(rgb, pose, graph) {
  // extract and project
  let fRgb = proj.apply(rgbBackbone(rgb)).expandDims(0);
  let fPose = proj.apply(poseBackbone(pose)).expandDims(0);
  let fGraph = proj.apply(graphBackbone(graph)).expandDims(0);
  // pairwise cross‑attention (query‑key‑value from other streams)
  fRgb = tf.add(crossAttn.apply([fRgb, fPose, fPose]), crossAttn.apply([fRgb, fGraph, fGraph]));
  fPose = tf.add(crossAttn.apply([fPose, fRgb, fRgb]), crossAttn.apply([fPose, fGraph, fGraph]));
  fGraph = tf.add(crossAttn.apply([fGraph, fRgb, fRgb]), crossAttn.apply([fGraph, fPose, fPose]));
  // MoE refinement per stream
  const [r1, l1] = moe.call(fRgb.squeeze([0]));
  const [r2, l2] = moe.call(fPose.squeeze([0]));
  const [r3, l3] = moe.call(fGraph.squeeze([0]));
  // mean‑pool and classify
  const pooled = tf.mean(tf.stack([r1.mean(0), r2.mean(0), r3.mean(0)]), 0);
  const logits = tf.layers.dense({units: 10}).apply(pooled); // 10‑class placeholder
  const lossBal = loadBalancingLoss(tf.stack([l1, l2, l3]));
  return {logits, lossBal};
}