fix: keep inbox sampling cohort stable

scripts/collect-metrics.js

@@ -10,6 +10,8 @@ const path = require('path');
 
 const AIBTC_API = 'https://aibtc.com/api';
 const DATA_DIR = path.join(__dirname, '..', 'data');
+const COHORT_FILE = path.join(DATA_DIR, 'sample-cohort.json');
+const DEFAULT_SAMPLE_SIZE = 20;
 
 async function fetchAllAgents() {
   console.log('Fetching agents...');
@@ -32,10 +34,68 @@ async function fetchAllAgents() {
   return agents;
 }
 
-async function sampleInboxMetrics(agents, sampleSize = 20) {
+function selectStableSample(agents, sampleSize = DEFAULT_SAMPLE_SIZE, cohortAddresses = []) {
+  const agentsByAddress = new Map(
+    agents
+      .filter(agent => agent.btcAddress)
+      .map(agent => [agent.btcAddress, agent])
+  );
+  const selectedAddresses = [];
+  const seen = new Set();
+
+  for (const address of cohortAddresses) {
+    if (agentsByAddress.has(address) && !seen.has(address)) {
+      selectedAddresses.push(address);
+      seen.add(address);
+    }
+    if (selectedAddresses.length >= sampleSize) break;
+  }
+
+  const stableAddresses = [...agentsByAddress.keys()]
+    .filter(address => !seen.has(address))
+    .sort();
+
+  for (const address of stableAddresses) {
+    selectedAddresses.push(address);
+    if (selectedAddresses.length >= sampleSize) break;
+  }
+
+  return selectedAddresses
+    .map(address => agentsByAddress.get(address))
+    .filter(Boolean);
+}
+
+function readSampleCohort() {
+  if (!fs.existsSync(COHORT_FILE)) {
+    return [];
+  }
+
+  try {
+    const contents = fs.readFileSync(COHORT_FILE, 'utf8').trim();
+    if (!contents) {
+      return [];
+    }
+
+    const cohort = JSON.parse(contents);
+    return Array.isArray(cohort.btcAddresses) ? cohort.btcAddresses : [];
+  } catch (error) {
+    console.warn(`Ignoring unreadable sample cohort: ${error.message}`);
+    return [];
+  }
+}
+
+function writeSampleCohort(sample, sampleSize) {
+  fs.writeFileSync(COHORT_FILE, JSON.stringify({
+    sampleSize,
+    updated_at: new Date().toISOString(),
+    btcAddresses: sample.map(agent => agent.btcAddress)
+  }, null, 2));
+}
+
+async function sampleInboxMetrics(agents, sampleSize = DEFAULT_SAMPLE_SIZE, cohortAddresses = []) {
   console.log(`Sampling ${sampleSize} agent inboxes...`);
-  const sample = agents.slice(0, sampleSize);
-  
+  const sample = selectStableSample(agents, sampleSize, cohortAddresses);
+
   let totalMessages = 0;
   let totalSatsReceived = 0;
   let totalSatsSent = 0;
@@ -53,13 +113,23 @@ async function sampleInboxMetrics(agents, sampleSize = 20) {
       console.error(`Failed to fetch inbox for ${agent.btcAddress}:`, err.message);
     }
   }
-
+
+  if (!sample.length) {
+    return {
+      totalMessages: 0,
+      totalSatsReceived: 0,
+      totalSatsSent: 0,
+      sample
+    };
+  }
+
   // Extrapolate to full network
   const scaleFactor = agents.length / sample.length;
   return {
     totalMessages: Math.round(totalMessages * scaleFactor),
     totalSatsReceived: Math.round(totalSatsReceived * scaleFactor),
     totalSatsSent: Math.round(totalSatsSent * scaleFactor),
+    sample
   };
 }
 
@@ -75,7 +145,8 @@ async function main() {
     const registeredCount = agents.filter(a => a.level === 1).length;
 
     // Sample inbox metrics
-    const inboxMetrics = await sampleInboxMetrics(agents);
+    const sampleCohort = readSampleCohort();
+    const inboxMetrics = await sampleInboxMetrics(agents, DEFAULT_SAMPLE_SIZE, sampleCohort);
 
     // Count new agents today
     const newAgentsCount = agents.filter(a => {
@@ -125,6 +196,7 @@ async function main() {
     // Write latest snapshot for quick access
     const latestFile = path.join(DATA_DIR, 'latest.json');
     fs.writeFileSync(latestFile, JSON.stringify(todayMetrics, null, 2));
+    writeSampleCohort(inboxMetrics.sample, DEFAULT_SAMPLE_SIZE);
 
     // Write agent list
     const agentsFile = path.join(DATA_DIR, 'agents.json');
@@ -152,4 +224,11 @@ async function main() {
   }
 }
 
-main();
+if (require.main === module) {
+  main();
+}
+
+module.exports = {
+  selectStableSample,
+  sampleInboxMetrics
+};

scripts/collect-metrics.test.js

@@ -0,0 +1,46 @@
+const assert = require('node:assert/strict');
+const { test } = require('node:test');
+const { selectStableSample } = require('./collect-metrics');
+
+function agent(btcAddress) {
+  return { btcAddress };
+}
+
+test('selectStableSample is stable when API result ordering changes', () => {
+  const agents = [
+    agent('btc-z'),
+    agent('btc-c'),
+    agent('btc-a'),
+    agent('btc-b')
+  ];
+  const reorderedAgents = [
+    agent('btc-b'),
+    agent('btc-z'),
+    agent('btc-a'),
+    agent('btc-c')
+  ];
+
+  assert.deepEqual(
+    selectStableSample(agents, 3).map(item => item.btcAddress),
+    ['btc-a', 'btc-b', 'btc-c']
+  );
+  assert.deepEqual(
+    selectStableSample(reorderedAgents, 3).map(item => item.btcAddress),
+    ['btc-a', 'btc-b', 'btc-c']
+  );
+});
+
+test('selectStableSample preserves an existing cohort and backfills missing addresses', () => {
+  const agents = [
+    agent('btc-a'),
+    agent('btc-b'),
+    agent('btc-c'),
+    agent('btc-d')
+  ];
+  const cohort = ['btc-c', 'btc-missing', 'btc-a'];
+
+  assert.deepEqual(
+    selectStableSample(agents, 3, cohort).map(item => item.btcAddress),
+    ['btc-c', 'btc-a', 'btc-b']
+  );
+});