Optimizing Reinforcement Learning for Autonomous Agents

# Illustrative sketch (not from the paper)
import torch
from torch import nn
from collections import deque

# Declarative rollout abstraction
class Rollout:
    def __init__(self, id, env_spec):
        self.id = id
        self.env_spec = env_spec
        self.tokens = []          # token IDs observed during execution
        self.rewards = []

# Controller that launches rollouts (K8s jobs or local processes)
class RolloutController:
    def __init__(self):
        self.active = {}
    def launch(self, rollout):
        # placeholder for async execution
        self.active[rollout.id] = rollout
    def collect(self, rollout_id):
        return self.active.pop(rollout_id, None)

# Simple token‑level prefix merge
def can_merge(prev: Rollout, cur: Rollout):
    return cur.tokens[:len(prev.tokens)] == prev.tokens

def merge(prev: Rollout, cur: Rollout):
    if can_merge(prev, cur):
        merged = Rollout(prev.id, prev.env_spec)
        merged.tokens = cur.tokens  # keep longer sequence
        merged.rewards = cur.rewards
        return merged
    return cur

# Trainer that waits for rollouts, computes rollout‑level advantage and token‑mean loss
class Trainer:
    def __init__(self, model):
        self.model = model
        self.optimizer = torch.optim.Adam(model.parameters(), lr=1e-5)
        self.buffer = deque()
    def register(self, rollout: Rollout):
        self.buffer.append(rollout)
    def step(self):
        if not self.buffer:
            return
        rollout = self.buffer.popleft()
        # rollout‑level advantage (reward sum)
        advantage = sum(rollout.rewards)
        # forward pass on token IDs
        logits = self.model(torch.tensor(rollout.tokens))
        # token‑mean loss (normalize by number of tokens)
        loss = nn.CrossEntropyLoss(reduction='mean')(logits, torch.tensor(rollout.tokens)) * advantage
        loss.backward()
        self.optimizer.step()
        self.optimizer.zero_grad()

# Mock model
class SimpleLM(nn.Module):
    def __init__(self, vocab=10000, dim=256):
        super().__init__()
        self.emb = nn.Embedding(vocab, dim)
        self.head = nn.Linear(dim, vocab)
    def forward(self, x):
        return self.head(self.emb(x))

# Example usage (collocated async RL)
controller = RolloutController()
model = SimpleLM()
trainer = Trainer(model)
# launch a few rollouts
for i in range(3):
    r = Rollout(id=i, env_spec={})
    controller.launch(r)
# simulate completion and training loop
for i in range(3):
    r = controller.collect(i)
    # mock token/reward population
    r.tokens = [1, 2, 3]
    r.rewards = [0.5]
    trainer.register(r)
    trainer.step()

// Illustrative sketch (not from the paper)
const torch = require('torch-js'); // placeholder for tensor ops

// Declarative rollout abstraction
class Rollout {
  constructor(id, envSpec) {
    this.id = id;
    this.envSpec = envSpec;
    this.tokens = []; // token IDs observed
    this.rewards = [];
  }
}

// Controller that would start K8s jobs or local processes
class RolloutController {
  constructor() {
    this.active = new Map();
  }
  launch(rollout) {
    // async execution stub
    this.active.set(rollout.id, rollout);
  }
  collect(id) {
    const r = this.active.get(id);
    this.active.delete(id);
    return r;
  }
}

// Token‑level prefix merge check
function canMerge(prev, cur) {
  const prefix = cur.tokens.slice(0, prev.tokens.length);
  return JSON.stringify(prefix) === JSON.stringify(prev.tokens);
}
function merge(prev, cur) {
  if (canMerge(prev, cur)) {
    const merged = new Rollout(prev.id, prev.envSpec);
    merged.tokens = cur.tokens; // keep longer sequence
    merged.rewards = cur.rewards;
    return merged;
  }
  return cur;
}

// Simple trainer handling rollout‑level advantage and token‑mean loss
class Trainer {
  constructor(model) {
    this.model = model;
    this.optimizer = new torch.optim.Adam(model.parameters(), { lr: 1e-5 });
    this.buffer = [];
  }
  register(rollout) {
    this.buffer.push(rollout);
  }
  step() {
    if (this.buffer.length === 0) return;
    const rollout = this.buffer.shift();
    const advantage = rollout.rewards.reduce((a, b) => a + b, 0);
    const tokensTensor = torch.tensor(rollout.tokens, 'int64');
    const logits = this.model.forward(tokensTensor);
    const lossFn = torch.nn.CrossEntropyLoss({ reduction: 'mean' });
    const loss = lossFn.forward(logits, tokensTensor).mul(advantage);
    loss.backward();
    this.optimizer.step();
    this.optimizer.zeroGrad();
  }
}

// Mock language model
class SimpleLM {
  constructor(vocab = 10000, dim = 256) {
    this.emb = new torch.nn.Embedding(vocab, dim);
    this.head = new torch.nn.Linear(dim, vocab);
  }
  forward(x) {
    return this.head.forward(this.emb.forward(x));
  }
  parameters() { return [...this.emb.parameters(), ...this.head.parameters()]; }
}

// Example collocated async RL loop
const controller = new RolloutController();
const model = new SimpleLM();
const trainer = new Trainer(model);
// launch rollouts
for (let i = 0; i < 3; i++) {
  controller.launch(new Rollout(i, {}));
}
// simulate completion and training
for (let i = 0; i < 3; i++) {
  const r = controller.collect(i);
  r.tokens = [1, 2, 3]; // mock token IDs
  r.rewards = [0.5];
  trainer.register(r);
  trainer.step();
}