Jointly Training AI to Create and Use Tools

# Illustrative sketch (not from the paper)
import torch
from torch import nn

# Simple shared policy that can both generate a tool (as code+JSON schema) and decide how to call it
class Policy(nn.Module):
    def forward(self, obs):
        # obs is a tuple like ('build', examples) or ('use', schema, question)
        # Returns either (tool_code, json_schema) or an action dict for tool invocation
        ...

def build_task(policy, examples):
    # Synthesize a reusable Python function and its OpenAI‑compatible JSON schema
    tool_code, schema = policy(('build', examples))
    return tool_code, schema

def use_task(policy, schema, question):
    # Ask the policy to produce a call that respects the schema, then execute it
    action = policy(('use', schema, question))
    answer = execute_tool(action, schema)  # placeholder sandbox execution
    return answer

def compute_reward(answer, correct, judge_score, format_ok):
    # Three independent axes: execution accuracy, LLM‑as‑judge quality, format consistency
    acc = 1.0 if answer == correct else 0.0
    return acc + judge_score + format_ok

def dap_update(policy, trajectories, optimizer):
    # Single backward pass over combined build and use rollouts (DAPO / GRPO variant)
    loss = compute_loss(trajectories)  # placeholder loss computation
    loss.backward()
    optimizer.step()
    optimizer.zero_grad()

# ---------------------------------------------------------------------
# Simplified training loop illustrating the workflow
policy = Policy()
optimizer = torch.optim.Adam(policy.parameters(), lr=1e-4)
tool_pool = []  # cache of verified tools

for epoch in range(5):
    # ---- Build phase ----
    tool, schema = build_task(policy, examples=[])

    # ---- Use phase ----
    answer = use_task(policy, schema, question="What is the result?")

    # ---- Reward computation ----
    reward = compute_reward(answer, correct=True, judge_score=0.8, format_ok=1)

    # ---- Policy update ----
    dap_update(policy, trajectories=[{'obs': None, 'reward': reward}], optimizer)

    # ---- Tool pool caching ----
    if reward > 0:
        tool_pool.append((tool, schema))

// Illustrative sketch (not from the paper)
const tf = require('@tensorflow/tfjs-node');

// Shared policy network (placeholder) that can emit tool code+JSON or an invocation action
class Policy {
  constructor() {
    // simple dense model; actual architecture omitted
    this.model = tf.sequential();
    this.model.add(tf.layers.dense({units: 128, inputShape: [/* obs dim */]}));
    this.model.add(tf.layers.dense({units: 64, activation: 'relu'}));
    this.model.add(tf.layers.dense({units: /* output dim */}));
    this.optimizer = tf.train.adam(1e-4);
  }

  // obs: ['build', examples] or ['use', schema, question]
  async forward(obs) {
    // Returns either {toolCode, schema} or an action object
    // Placeholder: actual generation logic not shown
    return {};
  }
}

async function buildTask(policy, examples) {
  // Synthesize a reusable tool (Python function) and its JSON schema
  const {toolCode, schema} = await policy.forward(['build', examples]);
  return {toolCode, schema};
}

async function useTask(policy, schema, question) {
  // Produce a call respecting the schema and execute it in a sandbox (mocked)
  const action = await policy.forward(['use', schema, question]);
  const answer = await executeTool(action, schema); // placeholder sandbox exec
  return answer;
}

function computeReward(answer, correct, judgeScore, formatOk) {
  const acc = answer === correct ? 1 : 0;
  return acc + judgeScore + formatOk; // three axes summed
}

async function dapUpdate(policy, trajectories) {
  // Single backward pass over combined rollouts (DAPO/GRPO variant)
  const lossFn = () => computeLoss(trajectories); // placeholder loss
  const grads = tf.variableGrads(lossFn);
  policy.optimizer.applyGradients(grads.grads);
  tf.dispose(grads);
}

// ---------------------------------------------------------------------
(async () => {
  const policy = new Policy();
  const toolPool = [];

  for (let epoch = 0; epoch < 5; epoch++) {
    // Build phase
    const {toolCode, schema} = await buildTask(policy, []);

    // Use phase
    const answer = await useTask(policy, schema, 'What is the result?');

    // Reward computation
    const reward = computeReward(answer, true, 0.8, 1);

    // Policy update
    await dapUpdate(policy, [{obs: null, reward}]);

    // Cache positive tools
    if (reward > 0) {
      toolPool.push({toolCode, schema});
    }
  }
})();