evaluate.js

// evaluate.js
// The testing engine. Two flows:
//   • runCompare — put TWO prompts head to head: generate an answer from each for every
//     sample task, show them blind and in random order, let you pick the better, then
//     (optionally) let the LLM judge score them too, and report how often the judge agreed
//     with you. This is the original A/B method.
//   • runSingle  — score ONE prompt on its own: generate an answer per sample and have the
//     judge score each 1-5 on the rubric, then report the average.
//
// Both need a working API key (these actually call the model). Both save a full report
// (report-<timestamp>.json) so you can re-read the answers afterwards.

const fs = require('fs');
const path = require('path');

const ui = require('./ui');
const judge = require('./judge');
const config = require('./config');
const { ask, askRaw } = require('./prompt');

// One non-streaming completion from a pre-seed prompt + a user task. (No web search — we
// are testing how the PROMPT shapes answers, not the model's research.)
async function complete(client, model, system, user) {
  const msg = await client.messages.create({
    model: model, max_tokens: 4000,
    system: system, messages: [{ role: 'user', content: user }],
  });
  const text = (msg.content || []).filter(b => b.type === 'text').map(b => b.text).join('').trim();
  // Flag answers cut off at the token cap — otherwise the judge (and you) penalise an answer
  // that the harness truncated, not one the prompt actually produced. You only pay for tokens
  // generated, so the 4000 cap costs nothing extra unless an answer was genuinely being cut off.
  return msg.stop_reason === 'max_tokens' ? text + '\n\n[truncated — hit the answer length cap]' : text;
}

// Tally {A:n, B:n, tie:n} from a list of picks; format with the two labels.
function tally(list) { return list.reduce((m, v) => { if (v) m[v] = (m[v] || 0) + 1; return m; }, {}); }

// Show the two answers the way an end user would see them — rendered markdown. Layout is
// chosen in Settings:
//   • 'stacked' (default) — each answer in full width, one after the other. Most readable;
//     never mangles tables or long lines.
//   • 'side-by-side' — two aligned columns; nicer for short answers, but cramps tables and
//     long lines. Falls back to stacked when the terminal is too narrow for two columns.
function showPair(a, b, layout) {
  if (layout === 'side-by-side') {
    const cols = ui.columns(a, b, {
      head1: ui.bold(ui.magenta('Response 1')),
      head2: ui.bold(ui.green('Response 2')),
    });
    if (cols) { console.log(cols + '\n'); return; }
    console.log(ui.dim('(terminal too narrow for side-by-side — showing stacked; widen to ~90+ cols)\n'));
  }
  console.log(ui.bold(ui.magenta('── Response 1 ──')) + '\n' + ui.md(a) + '\n');
  console.log(ui.bold(ui.green('── Response 2 ──')) + '\n' + ui.md(b) + '\n');
}

// Save a report next to the program and tell the user where it went.
function saveReport(payload) {
  const out = path.join(__dirname, 'report-' + Date.now() + '.json');
  try {
    fs.writeFileSync(out, JSON.stringify(payload, null, 2));
    console.log('\n' + ui.dim('Saved ' + out));
  } catch (e) {
    // A failed write (read-only dir, disk full) must not crash the program after a finished
    // run — the summary is already printed; just report that only the saved copy failed.
    console.log('\n' + ui.red('Could not save the report file — ' + config.friendly(e)));
    console.log(ui.dim('(The results above are still valid; only the on-disk copy failed.)'));
  }
}

// ── Compare two prompts ──────────────────────────────────────────────────────────────
// opts: { candidateModel, judgeModel, promptA:{label,system}, promptB:{...}, samples,
//         useJudge, useHuman }
async function runCompare(client, opts) {
  const { candidateModel, judgeModel, promptA, promptB, samples, useJudge, useHuman, layout, jurisdiction } = opts;

  console.log('\n' + ui.bold('A = ') + promptA.label + '\n' + ui.bold('B = ') + promptB.label);
  if (jurisdiction) console.log(ui.dim('Governing law: ' + jurisdiction));
  console.log(ui.dim('Answering with: ' + candidateModel + (useJudge ? '   Judge: ' + judgeModel : '')) + '\n');

  // Generate both answers for every sample (A and B in parallel) and fix a random display order.
  const results = [];
  for (const fx of samples) {
    process.stdout.write(ui.dim('Generating ' + fx.id + ' … '));
    try {
      const [a, b] = await Promise.all([
        complete(client, candidateModel, promptA.system, fx.user),
        complete(client, candidateModel, promptB.system, fx.user),
      ]);
      results.push({ id: fx.id, user: fx.user, A: a, B: b,
        order: Math.random() < 0.5 ? ['A', 'B'] : ['B', 'A'], human: null, judge: null });
      console.log(ui.green('done'));
    } catch (e) {
      console.log(ui.red('FAILED — ' + config.friendly(e) + ' (skipped)'));
    }
  }
  if (!results.length) { console.log('\n' + ui.red('No answers were generated — nothing to evaluate.')); return; }

  // Human pass — blind: pick the better of two unlabelled responses, and optionally say why.
  if (useHuman) {
    console.log('\n' + ui.title('Blind A/B — pick the better answer, then note why (optional)'));
    for (let i = 0; i < results.length; i++) {
      const r = results[i];
      console.log('\n' + ui.band('Task ' + (i + 1) + ' of ' + results.length + ' · ' + r.id));
      console.log(ui.bold('Task: ') + r.user + '\n');
      showPair(r[r.order[0]], r[r.order[1]], layout);
      let pick = '';
      while (!['1', '2', 't', 's', 'q'].includes(pick)) {
        pick = await ask(ui.yellow('Better? [1 / 2 / t=tie / s=skip / q=quit] '));
        if (pick === null) break;   // EOF (Ctrl+D / closed stdin) — stop asking
      }
      if (pick === 'q' || pick === null) break;   // EOF ends the human pass like a quit
      if (pick === 's') continue;
      r.human = pick === 't' ? 'tie' : r.order[Number(pick) - 1];
      const note = await askRaw(ui.dim('  Note (optional, Enter to skip): '));
      if (note) r.note = note;
    }
  }

  // Judge pass — same blind pairs, scored against the rubric in judge.js.
  if (useJudge) {
    console.log('\n' + ui.title('LLM judge'));
    for (const r of results) {
      try {
        const v = await judge.judge(client, judgeModel, r.user, r[r.order[0]], r[r.order[1]]);
        // Coerce the winner: models sometimes emit an unquoted JSON number ({"winner":1}),
        // which would fail a strict === '1' string test and silently credit the OTHER prompt.
        // Anything unexpected degrades to a tie rather than a wrong, plausible-looking result.
        const w = String(v.winner);
        r.judge = w === 'tie' ? 'tie' : (w === '1' ? r.order[0] : (w === '2' ? r.order[1] : 'tie'));
        r.judgeReason = v.reason;
        // Keep the per-criterion scores too — keyed "1"/"2" matching r.order, so the saved
        // report shows WHY one prompt beat the other, not just the winner.
        r.judgeScores = v.scores;
        console.log(ui.dim(r.id + ': ') + ui.bold(r.judge) + (r.judgeReason ? ui.dim('  (' + r.judgeReason + ')') : ''));
      } catch (e) {
        r.judgeError = config.friendly(e);
        console.log(ui.dim(r.id + ': ') + ui.red('judge failed — ' + r.judgeError));
      }
    }
  }

  // Summary + agreement.
  const labels = { A: promptA.label, B: promptB.label };
  const fmt = t => [labels.A + ': ' + (t.A || 0), labels.B + ': ' + (t.B || 0), 'tie: ' + (t.tie || 0)].join('  ·  ');
  console.log('\n' + ui.title('Summary'));
  if (useHuman) console.log(ui.bold('You:   ') + fmt(tally(results.map(r => r.human))));
  if (useJudge) console.log(ui.bold('Judge: ') + fmt(tally(results.map(r => r.judge))));
  if (useHuman && useJudge) {
    const both = results.filter(r => r.human && r.judge && r.human !== 'tie' && r.judge !== 'tie');
    const agree = both.filter(r => r.human === r.judge).length;
    console.log(ui.bold('Judge vs you: ') + agree + '/' + both.length + ' agree');
  }

  // Recap of notes — the to-do list for the next revision.
  const noted = results.filter(r => r.note);
  if (noted.length) {
    console.log('\n' + ui.title('Your notes (to-do list for the next revision)'));
    noted.forEach(r => {
      const pick = r.human === 'tie' ? 'tie' : (r.human ? labels[r.human] : '—');
      console.log(ui.cyan('• ') + ui.dim(r.id + ' → ' + pick + ': ') + r.note);
    });
  }

  saveReport({ mode: 'compare', jurisdiction, candidateModel, judgeModel: useJudge ? judgeModel : null, labels, results });
}

// ── Score one prompt ─────────────────────────────────────────────────────────────────
// opts: { candidateModel, judgeModel, prompt:{label,system}, samples }
async function runSingle(client, opts) {
  const { candidateModel, judgeModel, prompt, samples, jurisdiction } = opts;
  console.log('\n' + ui.bold('Testing: ') + prompt.label);
  if (jurisdiction) console.log(ui.dim('Governing law: ' + jurisdiction));
  console.log(ui.dim('Answering with: ' + candidateModel + '   Judge: ' + judgeModel) + '\n');

  const results = [];
  for (const fx of samples) {
    process.stdout.write(ui.dim('Task ' + fx.id + ' … '));
    try {
      const answer = await complete(client, candidateModel, prompt.system, fx.user);
      const verdict = await judge.scoreOne(client, judgeModel, fx.user, answer);
      results.push({ id: fx.id, user: fx.user, answer, ...verdict });
      console.log(ui.bold(String(verdict.overall)) + '/5' + (verdict.comment ? ui.dim('  (' + verdict.comment + ')') : ''));
    } catch (e) {
      console.log(ui.red('FAILED — ' + config.friendly(e) + ' (skipped)'));
    }
  }
  if (!results.length) { console.log('\n' + ui.red('No answers were scored — nothing to report.')); return; }

  const overall = judge.average(results.map(r => r.overall));
  console.log('\n' + ui.title('Summary'));
  console.log(ui.bold('Average score: ') + overall + '/5  across ' + results.length + ' tasks');
  console.log(ui.dim('Per-criterion is saved in the report; a low score on a criterion tells you ' +
    'what to sharpen in the prompt.'));

  saveReport({ mode: 'single', jurisdiction, candidateModel, judgeModel, label: prompt.label, overall, results });
}

module.exports = { runCompare, runSingle };