Simulating Realistic Human Shopping Behavior

# Illustrative sketch (not from the paper)
import torch
import torch.nn as nn

# Backbone model (e.g., Qwen3.5-2B placeholder)
class Backbone(nn.Module):
    def forward(self, img):
        # encode screenshot to latent
        return torch.randn(1, 768)  # mock embedding

# Hierarchical memory containers
class WorkingMemory:
    def __init__(self, capacity=5):
        self.buf = []
        self.cap = capacity
    def update(self, embed):
        self.buf.append(embed)
        if len(self.buf) > self.cap:
            self.buf.pop(0)
    def get(self):
        return torch.stack(self.buf) if self.buf else torch.zeros(1, 768)

class EpisodicMemory:
    def __init__(self):
        self.records = []
    def add(self, text):
        self.records.append(text)
    def get(self):
        return " ".join(self.records)

class PreferenceMemory:
    def __init__(self):
        self.intent = None
    def distill(self, episodic_text):
        # simple heuristic to set high‑level intent
        self.intent = "category:" + episodic_text.split()[0] if episodic_text else None
    def get(self):
        return self.intent

# Policy network using backbone + memories
class RecVerseAgent(nn.Module):
    def __init__(self):
        super().__init__()
        self.backbone = Backbone()
        self.policy_head = nn.Linear(768, 10)  # 10 possible actions
        self.working = WorkingMemory()
        self.episodic = EpisodicMemory()
        self.preference = PreferenceMemory()
    def forward(self, screenshot, action_text):
        embed = self.backbone(screenshot)
        self.working.update(embed)
        self.episodic.add(action_text)
        self.preference.distill(self.episodic.get())
        # combine current embed with intent embedding (mocked)
        intent_vec = torch.randn(1, 768)
        state = embed + intent_vec
        logits = self.policy_head(state)
        return logits

# Trajectory‑level RL loop (simplified)
agent = RecVerseAgent()
optimizer = torch.optim.Adam(agent.parameters(), lr=1e-4)
for episode in range(10):  # mock episodes
    log_probs = []
    rewards = []
    for step in range(5):  # mock steps per trajectory
        screenshot = torch.randn(3, 224, 224)  # placeholder image tensor
        action_text = "click"  # placeholder textual observation
        logits = agent(screenshot, action_text)
        dist = torch.distributions.Categorical(logits=logits)
        action = dist.sample()
        log_probs.append(dist.log_prob(action))
        # mock macro and micro rewards
        macro = 1.0 if action.item() == 2 else 0.0
        micro = 0.5 if agent.preference.get() and "category" in agent.preference.get() else 0.0
        rewards.append(macro + micro)
    # trajectory‑level return
    R = sum(rewards)
    loss = -torch.stack(log_probs).sum() * R
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

// Illustrative sketch (not from the paper)
const torch = require('torch-js'); // placeholder for tensor ops

// Backbone model mock (e.g., Qwen3.5-2B)
class Backbone {
  forward(img) {
    // encode screenshot to latent vector
    return torch.randn([1, 768]); // mock embedding
  }
}

// Hierarchical memory structures
class WorkingMemory {
  constructor(capacity = 5) {
    this.buf = [];
    this.cap = capacity;
  }
  update(embed) {
    this.buf.push(embed);
    if (this.buf.length > this.cap) this.buf.shift();
  }
  get() {
    return this.buf.length ? torch.stack(this.buf) : torch.zeros([1, 768]);
  }
}

class EpisodicMemory {
  constructor() { this.records = []; }
  add(text) { this.records.push(text); }
  get() { return this.records.join(' '); }
}

class PreferenceMemory {
  constructor() { this.intent = null; }
  distill(episodicText) {
    const parts = episodicText.split(' ');
    this.intent = parts.length ? `category:${parts[0]}` : null;
  }
  get() { return this.intent; }
}

// Agent combining backbone and memories
class RecVerseAgent {
  constructor() {
    this.backbone = new Backbone();
    this.policyHead = torch.nn.Linear(768, 10); // 10 actions
    this.working = new WorkingMemory();
    this.episodic = new EpisodicMemory();
    this.preference = new PreferenceMemory();
    this.optimizer = torch.optim.Adam(this.policyHead.parameters(), 1e-4);
  }
  forward(screenshot, actionText) {
    const embed = this.backbone.forward(screenshot);
    this.working.update(embed);
    this.episodic.add(actionText);
    this.preference.distill(this.episodic.get());
    const intentVec = torch.randn([1, 768]); // mock intent embedding
    const state = embed.add(intentVec);
    return this.policyHead.forward(state);
  }
}

// Trajectory‑level RL loop (simplified)
const agent = new RecVerseAgent();
for (let episode = 0; episode < 10; episode++) {
  const logProbs = [];
  const rewards = [];
  for (let step = 0; step < 5; step++) {
    const screenshot = torch.randn([3, 224, 224]); // placeholder image
    const actionText = 'click'; // placeholder textual observation
    const logits = agent.forward(screenshot, actionText);
    const dist = torch.distributions.Categorical({ logits });
    const action = dist.sample();
    logProbs.push(dist.logProb(action));
    // mock macro and micro rewards
    const macro = action.item() === 2 ? 1.0 : 0.0;
    const micro = agent.preference.get() && agent.preference.get().includes('category') ? 0.5 : 0.0;
    rewards.push(macro + micro);
  }
  const R = rewards.reduce((a, b) => a + b, 0);
  const loss = torch.neg(torch.stack(logProbs).sum()).mul(R);
  agent.optimizer.zeroGrad();
  loss.backward();
  agent.optimizer.step();
}