Efficient Robot Control Using Latent Actions

# Illustrative sketch (not from the paper)
import torch
from torch import nn

# 1. Pre‑train tokenizer: DINOv2 encoder + SAM2 mask decoder (placeholder modules)
class Tokenizer(nn.Module):
    def __init__(self, encoder, mask_decoder):
        super().__init__()
        self.encoder = encoder          # DINOv2 backbone
        self.mask_decoder = mask_decoder  # SAM2 supervised mask head
    def forward(self, frames):
        # extract patch tokens per frame
        patch_tokens = self.encoder(frames)          # [B, T, C]
        # generate segmentation masks (unused for downstream, only for training tokenizer)
        masks = self.mask_decoder(patch_tokens)
        # discretize into latent action tokens (e.g., via k‑means – omitted)
        latent_actions = torch.argmax(patch_tokens, dim=-1)  # placeholder
        return latent_actions

# 2. LAWA – joint video, latent‑action, and action denoising experts
class LAWA(nn.Module):
    def __init__(self, video_expert, latent_expert, action_expert):
        super().__init__()
        self.video_expert = video_expert
        self.latent_expert = latent_expert
        self.action_expert = action_expert
        # structured attention mask: block future‑video tokens
        self.attn_mask = None
    def forward(self, video_tokens, latent_intent, action_chunks, future_video=None):
        # during training we may provide future_video; at inference it is None
        if future_video is not None:
            # concatenate future video tokens but mask them from action generation
            combined = torch.cat([video_tokens, future_video], dim=1)
            # apply structured mask (implementation omitted)
        else:
            combined = video_tokens
        # joint denoising via multi‑model attention (simplified)
        video_out = self.video_expert(combined)
        latent_out = self.latent_expert(latent_intent)
        action_out = self.action_expert(action_chunks, latent_out)
        return action_out

# 3. Inference – omit future‑video branch, denoise latent intentions + actions only
def inference(lawa, video_tokens, latent_intent, action_chunks):
    # future_video is omitted → faster inference
    return lawa(video_tokens, latent_intent, action_chunks, future_video=None)

// Illustrative sketch (not from the paper)
const torch = require('torch-js'); // placeholder for PyTorch‑like API

// 1. Tokenizer: DINOv2 encoder + SAM2 mask decoder (mock classes)
class Tokenizer {
  constructor(encoder, maskDecoder) {
    this.encoder = encoder; // DINOv2 backbone
    this.maskDecoder = maskDecoder; // SAM2 mask head
  }
  forward(frames) {
    // extract patch tokens per frame
    const patchTokens = this.encoder(frames); // [B, T, C]
    // generate masks (used only for tokenizer pre‑training)
    const masks = this.maskDecoder(patchTokens);
    // discretize into latent action tokens (placeholder argmax)
    const latentActions = patchTokens.argMax(-1);
    return latentActions;
  }
}

// 2. LAWA model – joint experts with structured attention mask
class LAWA {
  constructor(videoExpert, latentExpert, actionExpert) {
    this.videoExpert = videoExpert;
    this.latentExpert = latentExpert;
    this.actionExpert = actionExpert;
    this.attnMask = null; // mask that blocks future‑video tokens
  }
  forward(videoTokens, latentIntent, actionChunks, futureVideo = null) {
    let combined;
    if (futureVideo) {
      // concatenate but future tokens are masked from action generation
      combined = torch.cat([videoTokens, futureVideo], 1);
      // apply structured mask (details omitted)
    } else {
      combined = videoTokens;
    }
    const videoOut = this.videoExpert(combined);
    const latentOut = this.latentExpert(latentIntent);
    const actionOut = this.actionExpert(actionChunks, latentOut);
    return actionOut;
  }
}

// 3. Inference – skip future video branch for speed
function inference(lawa, videoTokens, latentIntent, actionChunks) {
  return lawa.forward(videoTokens, latentIntent, actionChunks, null);
}