Benchmarking Resource Aware LLM Tool Agents

# Illustrative sketch (not from the paper)
import json, torch

# Load tool catalog with resource footprints (CPU, MEM, GPU, NET, DISK)
with open('tool_catalog.json') as f:
    catalog = json.load(f)  # {tool_id: [cpu, mem, gpu, net, disk]}

# Load a synthetic workflow (list of tool ids and declared edges)
with open('workflow.json') as f:
    wf = json.load(f)  # {"tools": [...], "edges": [(src, dst), ...]}

# ---------- Dimension I: Logical Planning ----------
# Assume we have a model-predicted edge set `pred_edges`
# Placeholder: use ground‑truth edges as prediction for illustration
pred_edges = wf['edges']

def graph_edit_distance(true, pred):
    # simple symmetric difference size as a proxy
    return len(set(true) ^ set(pred))

def edge_f1(true, pred):
    true_set, pred_set = set(true), set(pred)
    tp = len(true_set & pred_set)
    fp = len(pred_set - true_set)
    fn = len(true_set - pred_set)
    precision = tp / (tp + fp) if tp + fp else 0
    recall = tp / (tp + fn) if tp + fn else 0
    return 2 * precision * recall / (precision + recall) if precision + recall else 0

ged = graph_edit_distance(wf['edges'], pred_edges)
edge_f1_score = edge_f1(wf['edges'], pred_edges)
print('GED:', ged, 'Edge F1:', edge_f1_score)

# ---------- Dimension II: Physical Scheduling ----------
# Convert resource footprints to torch tensors for vectorized ops
resource_matrix = torch.tensor([catalog[t] for t in wf['tools']], dtype=torch.float)
# Example capacity budget (CPU, MEM, GPU, NET, DISK)
capacity = torch.tensor([100.0, 200.0, 8.0, 1000.0, 500.0])

# Simple scheduler that respects prerequisites and caps resources per timestep
schedule = []  # list of (timestep, [tool_indices])
remaining = set(range(len(wf['tools'])))
prereq = {dst: {src for src, d in wf['edges'] if d == dst} for dst in range(len(wf['tools']))}
current_time = 0
while remaining:
    ready = [i for i in remaining if prereq.get(i, set()).issubset(set(t for _, ts in schedule for t in ts))]
    # pack as many ready tools as capacity allows
    batch = []
    usage = torch.zeros(5)
    for i in ready:
        if torch.all(usage + resource_matrix[i] <= capacity):
            batch.append(i)
            usage += resource_matrix[i]
    schedule.append((current_time, batch))
    remaining.difference_update(batch)
    current_time += 1

# Compute metrics (placeholders)
scheduling_latency = len(schedule)
# Capacity Violation Area (CVA) = 0 because we enforced capacity
cva = 0.0
# Strict Mean Resource Utilization (MRU) = average usage / capacity
mru = torch.stack([torch.sum(resource_matrix[batch], dim=0) for _, batch in schedule if batch])
strict_mru = torch.mean(mru / capacity).item()
print('Latency:', scheduling_latency, 'CVA:', cva, 'Strict MRU:', strict_mru)

# ---------- RASC baseline (resource‑aware prompt) ----------
# In practice we would prepend resource metadata to the scheduling prompt
rasc_prompt = f"Capacity:{capacity.tolist()}\nTools:{resource_matrix.tolist()}\nSchedule the workflow safely."
print('RASC prompt snippet:', rasc_prompt[:80], '...')

// Illustrative sketch (not from the paper)
const fs = require('fs');

// Load tool catalog with resource footprints (CPU, MEM, GPU, NET, DISK)
const catalog = JSON.parse(fs.readFileSync('tool_catalog.json'));
// Load a synthetic workflow (tools array and edges list)
const wf = JSON.parse(fs.readFileSync('workflow.json'));

// ---------- Dimension I: Logical Planning ----------
// Assume model‑predicted edges; use ground‑truth for illustration
const predEdges = wf.edges;

function graphEditDistance(trueEdges, predEdges) {
  const trueSet = new Set(trueEdges.map(e => e.toString()));
  const predSet = new Set(predEdges.map(e => e.toString()));
  const symDiff = new Set([...trueSet].filter(x => !predSet.has(x)).concat([...predSet].filter(x => !trueSet.has(x))));
  return symDiff.size;
}

function edgeF1(trueEdges, predEdges) {
  const trueSet = new Set(trueEdges.map(e => e.toString()));
  const predSet = new Set(predEdges.map(e => e.toString()));
  const tp = [...trueSet].filter(x => predSet.has(x)).length;
  const fp = predSet.size - tp;
  const fn = trueSet.size - tp;
  const precision = tp + fp ? tp / (tp + fp) : 0;
  const recall = tp + fn ? tp / (tp + fn) : 0;
  return precision + recall ? (2 * precision * recall) / (precision + recall) : 0;
}

const ged = graphEditDistance(wf.edges, predEdges);
const edgeF1Score = edgeF1(wf.edges, predEdges);
console.log('GED:', ged, 'Edge F1:', edgeF1Score);

// ---------- Dimension II: Physical Scheduling ----------
// Build resource matrix for tools
const resourceMatrix = wf.tools.map(tid => catalog[tid]); // [[cpu,mem,gpu,net,disk], ...]
// Example capacity budget
const capacity = [100, 200, 8, 1000, 500];

let schedule = []; // [{time:0, batch:[idx,...]}, ...]
let remaining = new Set(wf.tools.map((_, i) => i));
// prerequisite map: dst -> Set(src)
const prereq = {};
wf.edges.forEach(([src, dst]) => {
  if (!prereq[dst]) prereq[dst] = new Set();
  prereq[dst].add(src);
});
let time = 0;
while (remaining.size) {
  const ready = [...remaining].filter(i => {
    const deps = prereq[i] || new Set();
    return [...deps].every(d => schedule.some(s => s.batch.includes(d)));
  });
  const batch = [];
  const usage = [0,0,0,0,0];
  for (const i of ready) {
    const res = resourceMatrix[i];
    const fits = usage.every((u, idx) => u + res[idx] <= capacity[idx]);
    if (fits) {
      batch.push(i);
      for (let j=0;j<5;j++) usage[j] += res[j];
    }
  }
  schedule.push({time, batch});
  batch.forEach(i => remaining.delete(i));
  time += 1;
}

// Metrics (placeholders)
const schedulingLatency = schedule.length;
const cva = 0; // capacity violations prevented by the scheduler
const mruValues = schedule.filter(s=>s.batch.length).map(s=>{
  const agg = [0,0,0,0,0];
  s.batch.forEach(i=>{
    const r = resourceMatrix[i];
    for(let j=0;j<5;j++) agg[j]+=r[j];
  });
  return agg.map((u,idx)=>u/capacity[idx]);
});
const strictMru = mruValues.flat().reduce((a,b)=>a+b,0) / (mruValues.length*5);
console.log('Latency:', schedulingLatency, 'CVA:', cva, 'Strict MRU:', strictMru);

// ---------- RASC baseline (resource‑aware prompt) ----------
const rascPrompt = `Capacity:${capacity}\nTools:${JSON.stringify(resourceMatrix)}\nSchedule the workflow safely.`;
console.log('RASC prompt snippet:', rascPrompt.slice(0,80), '...');