Evaluating LLMs as Mobile Personal Assistants

# Illustrative sketch (not from the paper)
import re

# Unified user profile (identity)
user_profile = {"user_id": "U123", "name": "Alice"}

# Simulated apps with simple schemas
apps = {
    "contacts": [{"id": 1, "name": "Bob", "phone": "555-1234"}],
    "calendar": [{"id": 1, "event": "Meeting", "time": "10:00"}],
}

# Retrieval tools (substring, regex, fuzzy placeholder)
def retrieve(app, query, method="substring"):
    records = apps.get(app, [])
    if method == "substring":
        return [r for r in records if query in str(r)]
    if method == "regex":
        pattern = re.compile(query)
        return [r for r in records if any(pattern.search(str(v)) for v in r.values())]
    # fuzzy not implemented in sketch
    return []

# Execution tool placeholder
def execute(action, target):
    return f"Executed {action} on {target}"

# Core evaluation loop: generate tool calls from instruction
def process_instruction(instruction):
    # naive parsing: look for app name and keyword
    if "call" in instruction.lower():
        app = "contacts"
        query = "Bob"
        retrieved = retrieve(app, query)
        if retrieved:
            return [
                {"tool": "retrieve", "app": app, "query": query},
                {"tool": "execute", "action": "call", "target": retrieved[0]["phone"]},
            ]
    return []

# Example usage
instr = "Call my friend Bob"
tool_calls = process_instruction(instr)
print(tool_calls)

// Illustrative sketch (not from the paper)
const userProfile = { userId: "U123", name: "Alice" };

// Simulated apps with simple schemas
const apps = {
  contacts: [{ id: 1, name: "Bob", phone: "555-1234" }],
  calendar: [{ id: 1, event: "Meeting", time: "10:00" }],
};

// Retrieval tools (substring, regex)
function retrieve(app, query, method = "substring") {
  const records = apps[app] || [];
  if (method === "substring") {
    return records.filter(r => JSON.stringify(r).includes(query));
  }
  if (method === "regex") {
    const regex = new RegExp(query);
    return records.filter(r => Object.values(r).some(v => regex.test(String(v))));
  }
  // fuzzy not implemented in sketch
  return [];
}

// Execution tool placeholder
function execute(action, target) {
  return `Executed ${action} on ${target}`;
}

// Core evaluation: turn instruction into tool calls
function processInstruction(instruction) {
  if (instruction.includes("Call")) {
    const app = "contacts";
    const query = "Bob";
    const retrieved = retrieve(app, query);
    if (retrieved.length) {
      return [
        { tool: "retrieve", app, query },
        { tool: "execute", action: "call", target: retrieved[0].phone },
      ];
    }
  }
  return [];
}

// Example usage
const instr = "Call my friend Bob";
const toolCalls = processInstruction(instr);
console.log(toolCalls);