Improving Efficiency and Expressivity in TTT

# Illustrative sketch (not from the paper)
import torch

def e2_ttt_update(chunk_grads, prev_weight, prev_momentum, decay, momentum_coeff):
    """Chunk‑wise TTT update using scalar kernels.
    chunk_grads: [L] tensor of per‑token gradient scalars
    prev_weight, prev_momentum: scalars from previous chunk
    decay, momentum_coeff: scalar hyper‑parameters
    """
    L = chunk_grads.shape[0]
    # log‑space cumulative sum → suffix products
    log_cum = torch.logcumsumexp(chunk_grads, dim=0)
    suffix_prod = torch.exp(log_cum[-1] - log_cum)
    # momentum kernel (coupled decay & momentum) in log‑space
    log_mom = torch.logcumsumexp(chunk_grads * momentum_coeff, dim=0)
    momentum_kernel = torch.exp(log_mom[-1] - log_mom)
    # weight kernel using decay
    weight_kernel = torch.exp(-decay * torch.arange(L, dtype=chunk_grads.dtype))
    # aggregate per‑token contributions into chunk‑end states
    new_weight = prev_weight * weight_kernel[-1] + (weight_kernel * chunk_grads).sum()
    new_momentum = prev_momentum * momentum_kernel[-1] + (momentum_kernel * chunk_grads).sum()
    # data‑dependent gate mixing with sliding‑window attention (placeholder)
    gate = torch.sigmoid(new_weight)
    updated_state = gate * new_weight + (1 - gate) * new_momentum
    return updated_state, new_weight, new_momentum

// Illustrative sketch (not from the paper)
const tf = require('@tensorflow/tfjs-node');

function e2TttUpdate(chunkGrads, prevWeight, prevMomentum, decay, momentumCoeff) {
  // chunkGrads: 1‑D tensor of per‑token gradient scalars
  const L = chunkGrads.shape[0];
  // log‑space cumulative sum → suffix products
  const logCum = tf.log(tf.cumsum(tf.exp(chunkGrads)));
  const suffixProd = tf.exp(tf.sub(logCum.slice([L - 1], [1]), logCum));
  // momentum kernel (coupled decay & momentum) in log‑space
  const logMom = tf.log(tf.cumsum(tf.mul(chunkGrads, momentumCoeff)));
  const momentumKernel = tf.exp(tf.sub(logMom.slice([L - 1], [1]), logMom));
  // weight kernel using decay
  const idx = tf.range(0, L, 1, 'float32');
  const weightKernel = tf.exp(tf.mul(tf.neg(decay), idx));
  // aggregate per‑token contributions
  const newWeight = tf.add(tf.mul(prevWeight, weightKernel.slice([L - 1], [1])), tf.sum(tf.mul(weightKernel, chunkGrads)));
  const newMomentum = tf.add(tf.mul(prevMomentum, momentumKernel.slice([L - 1], [1])), tf.sum(tf.mul(momentumKernel, chunkGrads)));
  // data‑dependent gate mixing with sliding‑window attention (placeholder)
  const gate = tf.sigmoid(newWeight);
  const updatedState = tf.add(tf.mul(gate, newWeight), tf.mul(tf.sub(1, gate), newMomentum));
  return { updatedState, newWeight, newMomentum };
}

module.exports = { e2TttUpdate };