Scaling Web Agents With Browser Sandboxes

# Illustrative sketch (not from the paper)
import os, json, requests, multiprocessing as mp
import torch.nn as nn

def load_urls(path):
    with open(path) as f:
        return [line.strip() for line in f if line.strip()]

def filter_reachable(urls):
    reachable = []
    for u in urls:
        try:
            r = requests.head(u, timeout=2)
            if r.status_code == 200:
                reachable.append(u)
        except:
            pass
    return reachable

def run_sandbox(url):
    task = proposer(url)
    traj = solver(task)
    return clean_trajectory(traj)

class Proposer(nn.Module):
    def forward(self, url):
        return {"action": "click", "target": "#start", "url": url}

proposer = Proposer()

def solver(task):
    return {"url": task.get("url", ""), "steps": [task]}

def clean_trajectory(traj):
    if not traj["steps"]:
        return None
    traj["thought"] = "executed task"
    return traj

if __name__ == "__main__":
    urls = filter_reachable(load_urls("common_crawl_urls.txt"))
    with mp.Pool(os.cpu_count()) as pool:
        results = pool.map(run_sandbox, urls[:100])  # demo subset
    with open("browserforge_corpus.json", "w") as f:
        json.dump([r for r in results if r], f)

// Illustrative sketch (not from the paper)
const fs = require('fs');
const fetch = require('node-fetch');
const os = require('os');

function loadUrls(path) {
  return fs.readFileSync(path, 'utf8')
    .split('\n')
    .map(l => l.trim())
    .filter(Boolean);
}

async function filterReachable(urls) {
  const reachable = [];
  for (const u of urls) {
    try {
      const res = await fetch(u, { method: 'HEAD', timeout: 2000 });
      if (res.ok) reachable.push(u);
    } catch (_) {}
  }
  return reachable;
}

function propose(url) {
  return { action: 'click', target: '#start', url };
}

function solve(task) {
  return { url: task.url, steps: [task] };
}

function cleanTrajectory(traj) {
  if (!traj.steps.length) return null;
  traj.thought = 'executed task';
  return traj;
}

async function runSandbox(url) {
  const task = propose(url);
  const traj = solve(task);
  return cleanTrajectory(traj);
}

(async () => {
  const urls = await filterReachable(loadUrls('common_crawl_urls.txt'));
  const concurrency = os.cpus().length;
  const cleaned = [];
  for (let i = 0; i < urls.length && i < 100; i++) {
    const result = await runSandbox(urls[i]);
    if (result) cleaned.push(result);
    if (cleaned.length % concurrency === 0) {
      // placeholder for batch handling
    }
  }
  fs.writeFileSync('browserforge_corpus.json', JSON.stringify(cleaned));
})();