Improving Multi-Reward Language Model Training

# Illustrative sketch (not from the paper)
import torch

def sat_mrpo_step(logits, rewards, reward_ranges, gamma=0.5, eps=1e-8):
    # logits: model output for a batch, shape [B, vocab]
    # rewards: dict of reward tensors, each shape [B]
    # reward_ranges: dict of (min, max) tuples for each reward
    # 1. Standardize each reward independently per query group (here batch)
    std_rewards = {}
    for name, r in rewards.items():
        r_min, r_max = reward_ranges[name]
        std = (r - r_min) / (r_max - r_min + eps)
        std_rewards[name] = std
    # 2. Compute batch-level saturation ratio s^{(k)} = mean(std_reward)
    sat_ratio = {name: torch.mean(r) for name, r in std_rewards.items()}
    # 3. Compute saturation‑aware advantage
    # Assume base_adv is the usual PPO advantage (placeholder here)
    base_adv = torch.randn_like(next(iter(std_rewards.values())))  # placeholder
    weighted_adv = 0.0
    for name, adv in std_rewards.items():
        weighted_adv += (adv ** gamma) * base_adv
    # 4. Normalize across batch
    adv_norm = (weighted_adv - weighted_adv.mean()) / (weighted_adv.std(unbiased=False) + eps)
    # 5. PPO clipped surrogate (simplified)
    old_logp = torch.log_softmax(logits.detach(), dim=-1)
    new_logp = torch.log_softmax(logits, dim=-1)
    ratio = torch.exp(new_logp - old_logp).gather(1, torch.argmax(logits, dim=1, keepdim=True))
    clip_eps = 0.2
    surrogate = torch.min(ratio * adv_norm, torch.clamp(ratio, 1-clip_eps, 1+clip_eps) * adv_norm)
    loss = -surrogate.mean()
    loss.backward()
    return loss.item()

// Illustrative sketch (not from the paper)
const tf = require('@tensorflow/tfjs-node');

function satMrpoStep(logits, rewards, rewardRanges, gamma = 0.5, eps = 1e-8) {
  // logits: Tensor [B, vocab]
  // rewards: {name: Tensor[B]}
  // rewardRanges: {name: [min, max]}
  // 1. Standardize each reward independently
  const stdRewards = {};
  for (const name in rewards) {
    const [rMin, rMax] = rewardRanges[name];
    stdRewards[name] = rewards[name].sub(rMin).div(rMax - rMin + eps);
  }
  // 2. Batch‑level saturation ratio (mean of standardized reward)
  const satRatio = {};
  for (const name in stdRewards) {
    satRatio[name] = stdRewards[name].mean();
  }
  // 3. Saturation‑aware advantage (placeholder baseAdv)
  const baseAdv = tf.randomNormal([logits.shape[0]]); // placeholder
  let weightedAdv = tf.zerosLike(baseAdv);
  for (const name in stdRewards) {
    const term = tf.pow(stdRewards[name], gamma).mul(baseAdv);
    weightedAdv = weightedAdv.add(term);
  }
  // 4. Normalize across batch
  const advMean = weightedAdv.mean();
  const advStd = tf.moments(weightedAdv).variance.sqrt().add(eps);
  const advNorm = weightedAdv.sub(advMean).div(advStd);
  // 5. Simplified PPO clipped surrogate
  const oldLogp = tf.logSoftmax(logits, -1);
  const newLogp = tf.logSoftmax(logits, -1);
  const ratio = tf.exp(newLogp.sub(oldLogp)).gather(tf.argMax(logits, 1), 1);
  const clipEps = 0.2;
  const clipped = tf.clipByValue(ratio, 1 - clipEps, 1 + clipEps);
  const surrogate = tf.minimum(ratio.mul(advNorm), clipped.mul(advNorm));
  const loss = surrogate.mean().neg();
  // In practice, call optimizer.minimize(() => loss);
  return loss;
}