Improving Long Horizon World Model Consistency

# Illustrative sketch (not from the paper)
import torch
from torch import nn

# 1. Motion‑aware latent conditioning: stack 9 consecutive frames
def motion_condition(frames):
    # frames: Tensor [9, C, H, W]
    return torch.mean(frames, dim=0)  # simple aggregation placeholder

# 2. Streaming 3D point‑cache rendering using ViGeo (placeholder API)
def render_spatial_memory(point_cache, camera_pose):
    # point_cache: per‑pixel 3D geometry from ViGeo
    # returns a rendered RGB‑D tensor
    return point_cache  # mock return

# 3. Causal encoding of re‑rendered memory as a continuous sequence
class CausalEncoder(nn.Module):
    def __init__(self, dim):
        super().__init__()
        self.rnn = nn.GRU(dim, dim, batch_first=True)
    def forward(self, seq):
        # seq: [T, B, dim]
        out, _ = self.rnn(seq)
        return out

# 4. Align temporal‑memory window in pixel space before VAE encoding
def align_window(memory_frames, conditioning_latent):
    # simple concatenation as alignment placeholder
    return torch.cat([memory_frames, conditioning_latent.unsqueeze(0)], dim=0)

# 5. Hard memory dropout: remove tokens (here drop random time steps)
def hard_dropout(seq, drop_prob=0.1):
    mask = torch.rand(seq.size(0)) > drop_prob
    return seq[mask]

# 6. Unified VAE encode/decode (shared across train/infer)
class SimpleVAE(nn.Module):
    def __init__(self, dim):
        super().__init__()
        self.encoder = nn.Linear(dim, dim)
        self.decoder = nn.Linear(dim, dim)
    def encode(self, x):
        return self.encoder(x)
    def decode(self, z):
        return self.decoder(z)

# 7. Remove camera AdaLN branch; viewpoint control comes from re‑rendered condition
def generate_frame(prev_latent, spatial_cond):
    # combine latent with spatial condition
    combined = prev_latent + spatial_cond.mean(dim=[1,2,3])
    return combined  # placeholder for decoder output

# Example workflow (high‑level, not runnable)
frames = torch.randn(9, 3, 64, 64)                     # nine‑frame window
cond_latent = motion_condition(frames)               # step 1
point_cache = torch.randn(3, 64, 64)                  # mock ViGeo output
spatial_mem = render_spatial_memory(point_cache, None)  # step 2
aligned = align_window(spatial_mem.unsqueeze(0), cond_latent)  # step 4
seq = hard_dropout(aligned)                          # step 5
causal_enc = CausalEncoder(dim=aligned.shape[1])
encoded = causal_enc(seq.unsqueeze(1))               # step 3
vae = SimpleVAE(dim=encoded.shape[-1])
z = vae.encode(encoded.squeeze(1))
output = vae.decode(z)                               # unified VAE
next_frame = generate_frame(output, spatial_mem)    # step 7

// Illustrative sketch (not from the paper)
const torch = require('torch-js'); // placeholder for PyTorch-like API

// 1. Motion‑aware latent conditioning: aggregate 9 frames
function motionCondition(frames) {
  // frames: Tensor [9, C, H, W]
  return torch.mean(frames, 0); // simple mean aggregation
}

// 2. Streaming 3D point‑cache rendering using ViGeo (mock function)
function renderSpatialMemory(pointCache, cameraPose) {
  // Returns rendered tensor (placeholder)
  return pointCache;
}

// 3. Causal encoder (GRU) for continuous sequence
class CausalEncoder {
  constructor(dim) {
    this.rnn = new torch.nn.GRU(dim, dim, { batchFirst: true });
  }
  forward(seq) {
    // seq: [T, B, dim]
    const [out] = this.rnn.forward(seq);
    return out;
  }
}

// 4. Align temporal‑memory window before VAE encoding
function alignWindow(memoryFrames, conditioningLatent) {
  // Concatenate along time dimension (placeholder)
  return torch.cat([memoryFrames, conditioningLatent.unsqueeze(0)], 0);
}

// 5. Hard memory dropout: drop whole tokens (time steps)
function hardDropout(seq, dropProb = 0.1) {
  const mask = torch.rand([seq.size(0)]).greater(dropProb);
  return seq.indexSelect(0, torch.where(mask)[0]);
}

// 6. Unified VAE encode/decode (shared protocol)
class SimpleVAE {
  constructor(dim) {
    this.encoder = new torch.nn.Linear(dim, dim);
    this.decoder = new torch.nn.Linear(dim, dim);
  }
  encode(x) { return this.encoder.forward(x); }
  decode(z) { return this.decoder.forward(z); }
}

// 7. Generate next frame using spatial condition (no camera AdaLN)
function generateFrame(prevLatent, spatialCond) {
  const viewCtrl = spatialCond.mean([1, 2, 3]); // aggregate spatial condition
  return prevLatent.add(viewCtrl);
}

// High‑level workflow (illustrative only)
const frames = torch.randn([9, 3, 64, 64]);               // nine‑frame window
const condLatent = motionCondition(frames);               // step 1
const pointCache = torch.randn([3, 64, 64]);               // mock ViGeo output
const spatialMem = renderSpatialMemory(pointCache, null); // step 2
const aligned = alignWindow(spatialMem.unsqueeze(0), condLatent); // step 4
const dropped = hardDropout(aligned);                     // step 5
const encoder = new CausalEncoder(aligned.shape[1]);
const encoded = encoder.forward(dropped.unsqueeze(1));   // step 3
const vae = new SimpleVAE(encoded.shape[2]);
const z = vae.encode(encoded.squeeze(1));
const decoded = vae.decode(z);                            // unified VAE
const nextFrame = generateFrame(decoded, spatialMem);    // step 7