Testing How Well LLMs Judge Responses

# Illustrative sketch (not from the paper)
import torch
from torch import nn
class Judge(nn.Module):
  def forward(self, prompt): return torch.randint(0,2,(1,)).item()
paired = [("orig1","edit1",False),("orig2","edit2",True)]
judge = Judge()
inv_hits=sen_hits=tot_inv=tot_sen=0
for o,e,ch in paired:
  po,pe=judge(o),judge(e)
  if not ch:
    tot_inv+=1; inv_hits+= (po==pe)
  else:
    tot_sen+=1; sen_hits+= (po!=pe)
S=inv_hits/tot_inv if tot_inv else 0
R=sen_hits/tot_sen if tot_sen else 0
print(f"Invariance S={S:.3f}, Sensitivity R={R:.3f}")
surf_hits=0
for o,e,ch in paired:
  pred=1 if len(e)<=len(o) else 0
  surf_hits+= (pred==judge(e))
print(f"Surface acc={surf_hits/len(paired):.3f}")

// Illustrative sketch (not from the paper)
const torch = require('torch-js'); // placeholder for tensor ops
class Judge {
  predict(prompt) { return Math.random()<0.5?0:1; }
}
const paired = [["orig1","edit1",false],["orig2","edit2",true]];
const judge = new Judge();
let invHits=0, senHits=0, totInv=0, totSen=0;
for (const [o,e,ch] of paired) {
  const po=judge.predict(o), pe=judge.predict(e);
  if (!ch) { totInv++; invHits+= (po===pe); }
  else { totSen++; senHits+= (po!==pe); }
}
const S = totInv? invHits/totInv : 0;
const R = totSen? senHits/totSen : 0;
console.log(`Invariance S=${S.toFixed(3)}, Sensitivity R=${R.toFixed(3)}`);
let surfHits=0;
for (const [o,e,ch] of paired) {
  const pred = e.length<=o.length?1:0;
  surfHits+= (pred===judge.predict(e))?1:0;
}
console.log(`Surface acc=${(surfHits/paired.length).toFixed(3)}`);