Benchmarking Reliability in Scientific AI Agents

# Illustrative sketch (not from the paper)
import os
import json
import torch

# Load EarthVerse metadata (405 investigations, avg 34 files per event)
with open('earthverse_index.json') as f:
    index = json.load(f)  # contains list of investigations with file lists

def load_evidence(files):
    """Read all evidence files for one investigation"""
    data = []
    for fp in files:
        with open(fp) as f:
            data.append(f.read())
    return data

def compute_quantity(tensor_a, tensor_b):
    """Transparent calculation preserving provenance"""
    # Example: simple physics operation using PyTorch
    result = torch.mul(tensor_a, tensor_b)
    provenance = {'op': 'mul', 'inputs': [tensor_a.tolist(), tensor_b.tolist()]}
    return result, provenance

for inv in index['investigations'][:5]:  # iterate a few sample investigations
    evidence = load_evidence(inv['evidence_files'])
    # placeholder: extract numeric tensors from evidence (mocked here)
    a = torch.tensor([1.0, 2.0])
    b = torch.tensor([3.0, 4.0])
    qty, prov = compute_quantity(a, b)
    # Assemble answer unit with provenance chain
    answer = {
        'investigation_id': inv['id'],
        'calculation': qty.tolist(),
        'provenance': prov,
        'evidence_used': inv['evidence_files']
    }
    print(json.dumps(answer))

// Illustrative sketch (not from the paper)
const fs = require('fs');
const path = require('path');
const tf = require('@tensorflow/tfjs-node'); // using tfjs for tensor ops

// Load EarthVerse index (405 investigations, avg 34 files per event)
const index = JSON.parse(fs.readFileSync('earthverse_index.json', 'utf8'));

function loadEvidence(files) {
  // Read all evidence files for one investigation
  return files.map(fp => fs.readFileSync(fp, 'utf8'));
}

function computeQuantity(tensorA, tensorB) {
  // Transparent calculation preserving provenance
  const result = tf.mul(tensorA, tensorB);
  const provenance = { op: 'mul', inputs: [tensorA.arraySync(), tensorB.arraySync()] };
  return { result, provenance };
}

index.investigations.slice(0, 5).forEach(inv => {
  const evidence = loadEvidence(inv.evidence_files);
  // placeholder: create mock tensors from evidence
  const a = tf.tensor([1, 2]);
  const b = tf.tensor([3, 4]);
  const { result, provenance } = computeQuantity(a, b);
  const answer = {
    investigation_id: inv.id,
    calculation: result.arraySync(),
    provenance: provenance,
    evidence_used: inv.evidence_files
  };
  console.log(JSON.stringify(answer));
});