Training Adaptable Agents for Live Streaming

# Illustrative sketch (not from the paper)
import torch
from torch import nn, optim

# frozen policy model (e.g., Qwen3.6-35B-A3B)
policy = torch.load('qwen3.6-35b-a3b.pt')
policy.eval()

def augment_harness(harness):
    """Apply task‑preserving perturbations to identifiers, content, schemas, prompts, hooks."""
    # simple example: rename skill keys
    harness['skills'] = {f"skill_{i}": v for i, v in enumerate(harness['skills'].values())}
    return harness

def hsa_sft(teacher_data, harness):
    """Stage 1 supervised fine‑tuning on augmented harness configurations."""
    optimizer = optim.Adam(policy.parameters(), lr=1e-5)
    loss_fn = nn.CrossEntropyLoss()
    for inputs, targets in teacher_data:
        outputs = policy(inputs)
        loss = loss_fn(outputs, targets)
        loss.backward()
        optimizer.step()
        optimizer.zero_grad()

def general_opd(general_data):
    """Stage 2 on‑policy distillation to recover instruction‑following ability."""
    # placeholder: same loop as SFT but using the current policy as teacher
    pass

def hsa_rl(env):
    """Stage 3 reinforcement learning in simulated augmented environments."""
    optimizer = optim.Adam(policy.parameters(), lr=1e-6)
    for episode in range(10):
        state = env.reset()
        done = False
        while not done:
            action = policy(state).argmax()
            next_state, reward, done, _ = env.step(action)
            # simple policy‑gradient style update (illustrative)
            logp = torch.log_softmax(policy(state), dim=-1)[action]
            loss = -reward * logp
            loss.backward()
            optimizer.step()
            optimizer.zero_grad()
            state = next_state

# Example usage (data pipelines omitted for brevity)
base_harness = {'skills': {'greet': '...'}, 'tools': {}, 'hooks': {}}
aug_harness = augment_harness(base_harness)
# teacher_data, general_data, and env would be supplied by the training pipeline

// Illustrative sketch (not from the paper)
const torch = require('torch-js'); // placeholder import

// frozen policy model (e.g., Qwen3.6-35B-A3B)
let policy = torch.load('qwen3.6-35b-a3b.pt');
policy.eval();

function augmentHarness(harness) {
  // Apply task‑preserving perturbations to identifiers, content, schemas, prompts, hooks
  const renamed = {};
  Object.values(harness.skills).forEach((v, i) => {
    renamed[`skill_${i}`] = v;
  });
  harness.skills = renamed;
  return harness;
}

function hsaSft(teacherData, harness) {
  // Stage 1 supervised fine‑tuning on augmented harness configurations
  const optimizer = new torch.optim.Adam(policy.parameters(), { lr: 1e-5 });
  const lossFn = torch.nn.CrossEntropyLoss();
  for (const [inputs, targets] of teacherData) {
    const outputs = policy.forward(inputs);
    const loss = lossFn(outputs, targets);
    loss.backward();
    optimizer.step();
    optimizer.zeroGrad();
  }
}

function generalOpd(generalData) {
  // Stage 2 on‑policy distillation (placeholder)
  // Would reuse the same loop as SFT but with policy as teacher
}

function hsaRl(env) {
  // Stage 3 reinforcement learning in simulated augmented environments
  const optimizer = new torch.optim.Adam(policy.parameters(), { lr: 1e-6 });
  for (let episode = 0; episode < 10; episode++) {
    let state = env.reset();
    let done = false;
    while (!done) {
      const logits = policy.forward(state);
      const action = logits.argmax();
      const { nextState, reward, done: stepDone } = env.step(action);
      const logProb = torch.logSoftmax(logits, -1).gather(action);
      const loss = torch.neg(torch.mul(reward, logProb));
      loss.backward();
      optimizer.step();
      optimizer.zeroGrad();
      state = nextState;
      done = stepDone;
    }
  }
}

// Example usage (data pipelines omitted for brevity)
const baseHarness = { skills: { greet: '...' }, tools: {}, hooks: {} };
const augHarness = augmentHarness(baseHarness);
// teacherData, generalData, and env would be provided by the training pipeline