Repository navigation
Expand file tree
/
Copy pathJustfile
More file actions
1157 lines (1115 loc) · 71.9 KB
/
Copy pathJustfile
File metadata and controls
1157 lines (1115 loc) · 71.9 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# The checks and jobs of this repository's automation. Workflows call these
# recipes rather than inline scripts, so a check runs the same here and in CI.
# Recipes see their arguments as $1, $2, ...
set positional-arguments
# Every test of the files under .github.
test: test-agent-board-install test-config test-supervisor-config test-supervisor-workflow test-poll-bounds test-poll-next test-review-gate test-forge-sync agent-board-pin
# Manual structural checks of the offline architecture-review prototype.
test-architecture-review mode="test" id="B2588":
#!/usr/bin/env node
const assert = require('node:assert/strict');
const { createHash } = require('node:crypto');
const { readFileSync } = require('node:fs');
const test = require('node:test');
const { execFileSync } = require('node:child_process');
const { mkdirSync, writeFileSync, lstatSync, readdirSync, mkdtempSync, rmSync, renameSync } = require('node:fs');
const read = name => readFileSync(`docs/architecture-review/${name}`, 'utf8');
const json = name => JSON.parse(read(name));
const references = json('references.json');
const version = references.version;
const sample = json('sample.json');
const results = json('results.json');
const sha256 = text => createHash('sha256').update(text).digest('hex');
// Manual only: no network in the default check or fixtures. Bound network
// requests so missing history cannot turn a packet audit into a crawl.
const api = (endpoint, accept = 'application/vnd.github+json') => execFileSync(
'gh', ['api', endpoint, '-H', `Accept: ${accept}`, '-H', 'X-GitHub-Api-Version: 2022-11-28'],
{ timeout: 20000, maxBuffer: 8 * 1024 * 1024 });
const text = bytes => new TextDecoder('utf-8', { fatal: true, ignoreBOM: true }).decode(bytes);
const apiJSON = endpoint => JSON.parse(text(api(endpoint)));
const save = (name, value) => writeFileSync(`docs/architecture-review/${name}`, JSON.stringify(value, null, 2) + '\n');
function annotate(diff) {
let next = null;
return diff.split('\n').map(line => {
if (line.startsWith('diff --git ')) next = null;
const hunk = /^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@/.exec(line);
if (hunk) { next = Number(hunk[1]); return line; }
if (next !== null && /^[ +\-]/.test(line)) {
const number = line[0] === '-' ? '' : String(next++);
return number.padStart(4) + ' ' + line;
}
return line;
}).join('\n');
}
function select(diff, sources, limits) {
const files = [{ name: 'diff.patch', data: Buffer.from(annotate(text(diff)), 'utf8') }];
let total = files[0].data.length;
assert.ok(limits.max_files >= 1 && total <= limits.max_bytes, 'complete annotated diff must fit');
const omitted = [];
const names = new Set(['diff.patch']);
// UTF-8 byte ordering is independent of the host locale.
sources.sort((a, b) => a.data.length - b.data.length || Buffer.compare(Buffer.from(a.path), Buffer.from(b.path)));
for (const source of sources) {
text(source.data); // No lossy decoding, BOM stripping or newline normalization.
const name = 'head/' + source.path.replaceAll('/', '__');
assert.ok(!names.has(name), `filename collision: ${source.path}`);
names.add(name);
if (files.length >= limits.max_files || total + source.data.length > limits.max_bytes) {
omitted.push({ path: source.path, revision: source.revision, bytes: source.data.length,
reason: 'packet byte/file limit; changed hunks remain in diff.patch' });
} else {
files.push({ name, path: source.path, data: source.data });
total += source.data.length;
}
}
return { files, omitted, total_bytes: total };
}
function comparePacket(packet, manifest) {
const files = manifest.files.map(expected => {
const actual = packet.files.find(f => f.name === expected.name);
return { name: expected.name, expected_bytes: expected.bytes, actual_bytes: actual?.data.length ?? null,
expected_sha256: expected.sha256, actual_sha256: actual ? sha256(actual.data) : null,
match: !!actual && actual.data.length === expected.bytes && sha256(actual.data) === expected.sha256 };
});
return { files, expected_total_bytes: manifest.total_bytes, actual_total_bytes: packet.total_bytes,
expected_file_order: manifest.files.map(f => f.name), actual_file_order: packet.files.map(f => f.name),
expected_omitted: manifest.omitted, actual_omitted: packet.omitted,
total_match: packet.total_bytes === manifest.total_bytes,
ordering_match: JSON.stringify(packet.files.map(f => f.name)) === JSON.stringify(manifest.files.map(f => f.name)),
omissions_match: JSON.stringify(packet.omitted) === JSON.stringify(manifest.omitted),
all_match: files.every(f => f.match) && packet.total_bytes === manifest.total_bytes &&
JSON.stringify(packet.files.map(f => f.name)) === JSON.stringify(manifest.files.map(f => f.name)) &&
JSON.stringify(packet.omitted) === JSON.stringify(manifest.omitted) };
}
function validateHistory(comparison, s, manifest) {
const base = s.reconstructed_base ?? s.candidate_base;
assert.equal(comparison.status, 'ahead');
assert.equal(comparison.merge_base_commit.sha, base);
assert.equal(comparison.total_commits, comparison.commits.length, 'truncated commits');
assert.ok(comparison.files.length < 300, 'possibly truncated changed files');
assert.equal(manifest.first_feedback_at, s.feedback_at);
assert.ok(comparison.commits.every(c => c.commit.committer.date <= s.feedback_at), 'post-feedback commit');
assert.deepEqual(comparison.commits.map(c => ({ sha: c.sha, committed: c.commit.committer.date })), manifest.commits);
}
// Hashes alone miss extra context (especially feedback) beside reviewed files.
// The only allowed directories are the root and head, if the manifest uses it.
function readPacket(directory, manifest) {
const names = manifest.files.map(f => f.name);
assert.equal(new Set(names).size, names.length, 'duplicate packet filename');
assert.ok(names.includes('diff.patch'), 'missing diff.patch');
for (const name of names) assert.match(name, /^(?:diff\.patch|head\/[^/]+)$/);
for (const name of names) assert.ok(!['head/.', 'head/..'].includes(name), 'unsafe packet filename');
const directories = names.some(n => n.startsWith('head/')) ? ['', 'head'] : [''];
for (const dir of directories) {
const path = dir ? `${directory}/${dir}` : directory;
assert.ok(lstatSync(path).isDirectory(), `not a real directory: ${path}`);
const expected = dir ? names.filter(n => n.startsWith('head/')).map(n => n.slice(5)) :
['diff.patch', ...(directories.includes('head') ? ['head'] : [])];
assert.deepEqual([...readdirSync(path)].sort(), expected.sort(), `unexpected packet entries: ${path}`);
}
return names.map(name => {
const path = `${directory}/${name}`;
assert.ok(lstatSync(path).isFile(), `not a regular packet file: ${path}`);
return { name, data: readFileSync(path) };
});
}
if (process.argv[2] === 'verify') {
const id = process.argv[3];
assert.ok(sample.samples.some(s => s.id === id && s.status === 'evaluated'), 'unknown evaluated sample');
const manifest = json(`runs/${id}.manifest.json`);
const files = readPacket(`/tmp/opencode/architecture-review-${id}`, manifest);
const report = comparePacket({ files, omitted: manifest.omitted,
total_bytes: files.reduce((sum, f) => sum + f.data.length, 0) }, manifest);
// Exact directory integrity and bytes, not independently reconstructed selection/omissions.
console.log(`${id}: exact packet verification ${report.all_match ? 'matches' : 'MISMATCH'}`);
if (!report.all_match) console.log(JSON.stringify(report, null, 2));
process.exit(report.all_match ? 0 : 1);
}
if (process.argv[2] === 'rebuild') {
const id = process.argv[3];
const s = sample.samples.find(s => s.id === id && s.status === 'evaluated');
assert.ok(s, 'unknown evaluated sample');
const manifest = json(`runs/${id}.manifest.json`);
const base = s.reconstructed_base ?? s.candidate_base;
const endpoint = `repos/${s.repository}/compare/${base}...${s.head}`;
const comparison = apiJSON(endpoint);
validateHistory(comparison, s, manifest);
const diff = api(endpoint, 'application/vnd.github.diff');
const sources = comparison.files.filter(f => f.status !== 'removed').map(f => {
const path = f.filename.split('/').map(encodeURIComponent).join('/');
const content = apiJSON(`repos/${s.repository}/contents/${path}?ref=${s.head}`);
assert.equal(content.type, 'file');
assert.equal(content.encoding, 'base64');
const data = Buffer.from(content.content, 'base64');
assert.equal(data.length, content.size, 'missing or truncated contents');
return { path: f.filename, revision: s.head, data };
});
const packet = select(diff, sources, manifest.limits);
const directory = `/tmp/opencode/architecture-review-${id}`;
const report = { schema: 'architecture-review.rebuild.v1', id, builder: 'Justfile test-architecture-review rebuild',
historical_identity: 'Only matching bytes are reconstructed evidence; no new historical review ran.',
manifest_sha256: sha256(read(`runs/${id}.manifest.json`)),
ancestry: { status: comparison.status, merge_base: comparison.merge_base_commit.sha, commits_match: true, pre_feedback: true },
...comparePacket(packet, manifest) };
assert.ok(report.all_match, `reconstructed packet MISMATCH: ${JSON.stringify(report)}`);
const staging = mkdtempSync(`${directory}.staging-`);
try {
if (packet.files.some(f => f.name.startsWith('head/'))) mkdirSync(`${staging}/head`);
for (const file of packet.files) writeFileSync(`${staging}/${file.name}`, file.data);
const files = readPacket(staging, manifest);
assert.ok(comparePacket({ ...packet, files }, manifest).all_match, 'staged packet MISMATCH');
// Replace the entire old tree: no stale source or scoring material survives.
// Publication is after comparison; a failed retrieval never publishes new bytes.
rmSync(directory, { recursive: true, force: true });
renameSync(staging, directory);
} finally {
rmSync(staging, { recursive: true, force: true });
}
save(`runs/${id}.rebuild.json`, report);
console.log(`${id}: exact packet matches; details in runs/${id}.rebuild.json`);
process.exit(0);
}
if (process.argv[2] === 'feedback') {
const inventory = { schema: 'architecture-review.feedback.v1', author: 'cgwalters', retrieved_at: new Date().toISOString(),
purpose: 'Scoring only; never historical reviewer input.', samples: [] };
for (const s of sample.samples.filter(s => s.status === 'evaluated')) {
const entry = { id: s.id, channels: [], units: [] };
for (const [channel, endpoint] of [
['inline', `repos/${s.repository}/pulls/${s.pr}/comments`],
['review', `repos/${s.repository}/pulls/${s.pr}/reviews`],
['pr', `repos/${s.repository}/issues/${s.pr}/comments`],
]) {
let complete = false;
const pages = [];
for (let page = 1; page <= 10; page++) {
const rows = apiJSON(`${endpoint}?per_page=100&page=${page}`);
pages.push(rows.length);
for (const r of rows.filter(r => r.user.login === 'cgwalters')) {
entry.units.push({ channel, id: r.id, url: r.html_url, body: r.body ?? '',
created_at: r.created_at ?? r.submitted_at, updated_at: r.updated_at ?? null,
commit_id: r.commit_id ?? null, in_reply_to_id: r.in_reply_to_id ?? null,
path: r.path ?? null, original_line: r.original_line ?? null, diff_hunk: r.diff_hunk ?? null });
}
if (rows.length < 100) { complete = true; break; }
}
assert.ok(complete, `pagination cap: ${endpoint}`);
entry.channels.push({ channel, endpoint, per_page: 100, page_sizes: pages, complete });
}
inventory.samples.push(entry);
}
save('feedback.json', inventory);
console.log('Archived paginated cgwalters feedback for three samples; classification is separate.');
process.exit(0);
}
assert.ok(['test', 'fixtures', 'e2e'].includes(process.argv[2]), 'expected test, fixtures, e2e, rebuild, verify or feedback');
if (process.argv[2] === 'e2e') test('offline canned-API rebuild/verify dispatch', () => {
const fs = require('node:fs');
const vm = require('node:vm');
const sandbox = fs.mkdtempSync('/tmp/opencode/architecture-review-test-');
const source = /^test-architecture-review .*:\n((?: .*\n|\n)+)/m.exec(fs.readFileSync('Justfile', 'utf8'))[1]
.replace(/^ /gm, '');
const id = 'B2588';
const s = sample.samples.find(s => s.id === id);
const base = s.reconstructed_base ?? s.candidate_base;
const endpoint = `repos/${s.repository}/compare/${base}...${s.head}`;
const diff = '@@ -1 +1 @@\n-old\n+new\n';
const data = Buffer.from('new\n');
const packet = select(Buffer.from(diff), [{ path: 'x', revision: s.head, data }], { max_files: 30, max_bytes: 100000 });
const manifest = { files: packet.files.map(f => ({ name: f.name, bytes: f.data.length, sha256: sha256(f.data) })),
total_bytes: packet.total_bytes, omitted: [], limits: { max_files: 30, max_bytes: 100000 },
first_feedback_at: s.feedback_at, commits: [{ sha: s.head, committed: s.feedback_at }] };
const history = { status: 'ahead', merge_base_commit: { sha: base }, total_commits: 1,
commits: [{ sha: s.head, commit: { committer: { date: s.feedback_at } } }], files: [{ filename: 'x', status: 'modified' }] };
const directory = `${sandbox}/packet`;
const map = path => typeof path === 'string' && path.startsWith(`/tmp/opencode/architecture-review-${id}`) ?
path.replace(`/tmp/opencode/architecture-review-${id}`, directory) : path;
const exit = {};
let failure = null;
let calls = 0;
const wrappedFS = { ...fs };
for (const method of ['readFileSync', 'lstatSync', 'readdirSync', 'mkdirSync', 'mkdtempSync', 'rmSync', 'writeFileSync']) {
wrappedFS[method] = (path, ...args) => {
if (path === `docs/architecture-review/runs/${id}.manifest.json` && method === 'readFileSync') return JSON.stringify(manifest);
if (path === `docs/architecture-review/runs/${id}.rebuild.json` && method === 'writeFileSync') path = `${sandbox}/report.json`;
if (failure === 'staging' && method === 'writeFileSync' && path.includes('.staging-') && path.endsWith('/head/x')) args[0] = 'damaged staging';
return fs[method](map(path), ...args);
};
}
wrappedFS.renameSync = (from, to) => fs.renameSync(map(from), map(to));
const dispatch = mode => {
let status = null;
try {
vm.runInNewContext(source, { Buffer, TextDecoder, console: { log() {} },
process: { argv: ['node', 'recipe', mode, id], exit(code) { status = code; throw exit; } },
require(name) {
if (name === 'node:fs') return wrappedFS;
if (name === 'node:child_process') return { execFileSync(command, args, options) {
calls++;
assert.equal(command, 'gh');
assert.equal(args[0], 'api');
assert.equal(options.timeout, 20000);
assert.equal(options.maxBuffer, 8 * 1024 * 1024);
if (failure === 'retrieval') throw new Error('canned retrieval failure');
if (args[1] === endpoint) return Buffer.from(args.includes('Accept: application/vnd.github.diff') ?
(failure === 'comparison' ? diff + 'bad' : diff) : JSON.stringify(history));
assert.equal(args[1], `repos/${s.repository}/contents/x?ref=${s.head}`);
return Buffer.from(JSON.stringify({ type: 'file', encoding: 'base64', content: data.toString('base64'), size: data.length }));
} };
return require(name);
} });
} catch (error) {
if (error !== exit) return { status: 1, error };
}
return { status };
};
const succeeds = mode => { const result = dispatch(mode); assert.equal(result.status, 0, `${mode}: ${result.error?.stack ?? ''}`); };
const fails = mode => assert.equal(dispatch(mode).status, 1, mode);
try {
succeeds('rebuild');
succeeds('verify');
const before = calls;
for (const [name, kind] of [['head/obsolete.txt', 'file'], ['feedback.json', 'file'],
['feedback', 'directory'], ['head/extra', 'directory'], ['head/link', 'symlink']]) {
const path = `${directory}/${name}`;
if (kind === 'directory') fs.mkdirSync(path);
else if (kind === 'symlink') fs.symlinkSync(`${directory}/diff.patch`, path);
else fs.writeFileSync(path, 'stale feedback');
fails('verify');
succeeds('rebuild');
assert.equal(fs.existsSync(path), false, name);
succeeds('verify');
}
assert.ok(calls > before);
succeeds('rebuild'); // Repeated reconstruction has the same exact packet.
for (const name of ['diff.patch', 'head/x']) {
fs.rmSync(`${directory}/${name}`);
fs.symlinkSync(`${sandbox}/report.json`, `${directory}/${name}`);
fails('verify');
succeeds('rebuild');
}
fs.renameSync(`${directory}/head`, `${sandbox}/old-head`);
fs.symlinkSync(`${sandbox}/old-head`, `${directory}/head`);
fails('verify');
succeeds('rebuild');
fs.renameSync(directory, `${sandbox}/old-packet`);
fs.symlinkSync(`${sandbox}/old-packet`, directory);
fails('verify');
succeeds('rebuild');
fs.writeFileSync(`${directory}/head/x`, 'bad\n');
fails('verify');
for (const reason of ['retrieval', 'comparison', 'staging']) {
failure = reason;
const oldReport = fs.readFileSync(`${sandbox}/report.json`, 'utf8');
fails('rebuild');
assert.equal(fs.readFileSync(`${directory}/head/x`, 'utf8'), 'bad\n', 'failed rebuild does not publish');
assert.equal(fs.readFileSync(`${sandbox}/report.json`, 'utf8'), oldReport);
fails('verify');
fs.rmSync(directory, { recursive: true });
fails('rebuild');
assert.equal(fs.existsSync(directory), false, 'failed rebuild does not create a packet');
failure = null;
succeeds('rebuild');
fs.writeFileSync(`${directory}/head/x`, 'bad\n');
}
fs.rmSync(`${directory}/head/x`);
fails('verify');
failure = null;
succeeds('rebuild');
succeeds('verify');
assert.deepEqual(fs.readdirSync(sandbox).filter(n => n.includes('.staging-')), []);
} finally {
fs.rmSync(sandbox, { recursive: true, force: true });
}
});
test('offline packet annotation and selection fixtures', () => {
const diff = 'diff --git a/x b/x\n--- a/x\n+++ b/x\n@@ -1,2 +1,2 @@\n-old\n+new\n same\n\\ No newline at end of file\n';
assert.equal(annotate(diff), 'diff --git a/x b/x\n--- a/x\n+++ b/x\n@@ -1,2 +1,2 @@\n -old\n 1 +new\n 2 same\n\\ No newline at end of file\n');
assert.equal(annotate('@@ -0,0 +1 @@\n+é\r\n'), '@@ -0,0 +1 @@\n 1 +é\r\n');
assert.equal(annotate('@@ -1 +0,0 @@\n-x'), '@@ -1 +0,0 @@\n -x');
const sources = ['z', 'a', 'large'].map(path => ({ path, data: Buffer.from(path === 'large' ? 'xxx' : 'é') }));
const packet = select(Buffer.from(''), sources, { max_files: 2, max_bytes: 2 });
assert.deepEqual(packet.files.map(f => f.name), ['diff.patch', 'head/a']);
assert.equal(packet.total_bytes, 2);
assert.equal(packet.omitted.length, 2);
assert.equal(select(Buffer.from(''), sources, { max_files: 30, max_bytes: 4 }).files.length, 3);
assert.equal(select(Buffer.from(''), sources, { max_files: 1, max_bytes: 100 }).files.length, 1);
assert.equal(select(Buffer.from(''), sources, { max_files: 30, max_bytes: 1 }).files.length, 1);
assert.throws(() => select(Buffer.from('abc'), [], { max_files: 30, max_bytes: 2 }));
assert.throws(() => select(Buffer.from(''), [{ path: 'a/b', data: Buffer.from('') }, { path: 'a__b', data: Buffer.from('') }], { max_files: 30, max_bytes: 10 }));
assert.throws(() => text(Buffer.from([255])));
const manifest = { files: packet.files.map(f => ({ name: f.name, bytes: f.data.length, sha256: sha256(f.data) })), total_bytes: 2, omitted: packet.omitted };
assert.equal(comparePacket(packet, manifest).all_match, true);
assert.equal(comparePacket({ ...packet, files: packet.files.map(f => ({ ...f, data: Buffer.from('bad') })) }, manifest).all_match, false);
assert.equal(comparePacket(packet, { ...manifest, total_bytes: 3 }).all_match, false);
assert.equal(comparePacket(packet, { ...manifest, omitted: [] }).all_match, false);
assert.equal(comparePacket(packet, { ...manifest, files: manifest.files.map(f => ({ ...f, sha256: '0'.repeat(64) })) }).all_match, false);
assert.equal(comparePacket({ ...packet, files: [...packet.files].reverse() }, manifest).all_match, false);
const s = { candidate_base: 'base', feedback_at: '2026-01-02T00:00:00Z' };
const commits = [{ sha: 'head', commit: { committer: { date: '2026-01-01T00:00:00Z' } } }];
const history = { status: 'ahead', merge_base_commit: { sha: 'base' }, total_commits: 1, commits, files: [] };
const frozen = { first_feedback_at: s.feedback_at, commits: [{ sha: 'head', committed: '2026-01-01T00:00:00Z' }] };
validateHistory(history, s, frozen);
assert.throws(() => validateHistory({ ...history, status: 'diverged' }, s, frozen));
assert.throws(() => validateHistory({ ...history, merge_base_commit: { sha: 'wrong' } }, s, frozen));
assert.throws(() => validateHistory({ ...history, total_commits: 2 }, s, frozen));
assert.throws(() => validateHistory({ ...history, files: Array(300).fill({}) }, s, frozen));
assert.throws(() => validateHistory(history, { ...s, feedback_at: '2025-01-01T00:00:00Z' }, { ...frozen, first_feedback_at: '2025-01-01T00:00:00Z' }));
assert.throws(() => validateHistory(history, s, { ...frozen, commits: [] }));
});
function validateCounts(r) {
const evaluated = sample.samples.filter(s => s.status === 'evaluated');
assert.equal(r.selected_prs, sample.samples.length);
assert.equal(r.evaluated_prs, evaluated.length);
assert.equal(r.evaluated_prs + r.rejected_prs, r.selected_prs);
assert.equal(r.architecture_concerns + r.excluded_nits + r.excluded_acknowledgements + r.excluded_empty_reviews,
r.classification_units);
assert.equal(r.classification_units, r.audited_feedback_records + r.split_extra_units);
assert.equal(r.audited_feedback_records, r.audited_inline_comments + r.audited_review_bodies + r.audited_pr_comments);
assert.equal(r.architecture_concerns, r.concerns.length);
assert.equal(r.executed_reviews, r.runs.length);
assert.deepEqual(r.runs.map(run => run.id), evaluated.map(s => s.id));
assert.ok(r.compared_prs <= r.executed_reviews);
assert.equal(r.status, 'development_pilot');
assert.equal(r.held_out_claim, false);
for (const c of r.concerns) {
assert.ok(['concern', 'question'].includes(c.stratum), c.id);
assert.ok(r.runs.some(run => c.id.startsWith(`${run.id}-`)), c.id);
if (c.matched_by) assert.ok(c.scorable && c.matched_by.startsWith(c.id.split('-')[0]), c.id);
if (!c.scorable) assert.ok(c.unscorable_reason, c.id);
}
for (const f of Object.values(r.findings)) assert.equal(f.matched + f.unmatched, f.total);
for (const [key, stratum] of [['concerns', 'concern'], ['questions', 'question']]) {
const units = r.concerns.filter(c => c.stratum === stratum);
const counts = r.strata[key];
assert.equal(counts.total, units.length);
assert.equal(counts.scorable, units.filter(c => c.scorable).length);
assert.equal(counts.matched, units.filter(c => c.matched_by).length);
assert.equal(counts.missed, counts.scorable - counts.matched);
assert.equal(counts.unscorable, counts.total - counts.scorable);
}
const a = r.unmatched_audit;
assert.equal(a.supported + a.unsupported + a.unresolved, a.audited);
assert.ok(a.audited <= r.findings.defects.unmatched + r.findings.questions.unmatched);
}
test('explicit version and pinned, publicly linked source files', () => {
assert.equal(version, '0.1.0');
assert.equal(results.version, version);
assert.equal(references.references.length, 9);
for (const [file, digest] of Object.entries(results.artifact_sha256)) {
assert.equal(sha256(read(file)), digest, file);
}
const ids = new Set();
for (const r of references.references) {
assert.match(r.revision, /^[a-f0-9]{40}$/);
assert.ok(r.sections.length > 0);
assert.ok(!ids.has(r.id));
ids.add(r.id);
assert.ok(read('README.md').includes(
`https://github.com/${r.repository}/blob/${r.revision}/${r.path}`));
}
});
test('six scoped architecture patterns with explicitly illustrative examples', () => {
const library = read('patterns.md');
const sections = library.split(/^## /m).slice(1);
assert.equal(sections.length, 6);
sections.forEach((s, i) => {
assert.ok(s.startsWith(`AR-0${i + 1}:`));
for (const label of ['Principle', 'Scope/exceptions', 'Source',
'Good (illustrative)', 'Bad (illustrative)']) assert.ok(s.includes(`**${label}:**`));
assert.ok(references.references.some(r => s.includes(r.id)));
});
});
test('prompt covers design dimensions, evidence, uncertainty and read-only boundary', () => {
const prompt = read('prompt.md');
assert.ok(prompt.includes(`prompt ${version}`));
for (const dimension of ['Invariants', 'State ownership', 'Failure and rollback',
'Compatibility and interfaces', 'Dependency ordering', 'Privilege boundaries', 'Testing']) {
assert.ok(prompt.includes(`**${dimension}:**`));
}
for (const phrase of ['untrusted **data**', 'Do not execute', 'Do not edit files',
'concrete PR evidence', 'Uncertainty/questions', 'nit-checklist',
'100,000', 'contaminated', 'Archive this output unchanged', 'references.json'
]) assert.ok(prompt.includes(phrase));
});
test('sample pins are separate from derivation and not leaked into prompt/library', () => {
const ids = new Set(sample.samples.map(s => s.id));
assert.equal(ids.size, sample.samples.length);
for (const s of sample.samples) {
assert.ok(['evaluated', 'rejected'].includes(s.status), s.id);
if (s.status === 'rejected') assert.ok(s.rejection, s.id);
// A later replacement must name what it replaces, never substitute silently.
if (!s.frozen_before_prompt_authoring) {
assert.equal(sample.samples.find(r => r.id === s.replaces)?.status, 'rejected', s.id);
}
for (const sha of [s.head, s.candidate_base, ...(s.reconstructed_base ? [s.reconstructed_base] : [])]) {
assert.match(sha, /^[a-f0-9]{40}$/);
assert.ok(!references.references.some(r => r.revision === sha));
for (const file of ['patterns.md', 'prompt.md']) assert.ok(!read(file).includes(sha));
}
for (const file of ['patterns.md', 'prompt.md']) {
assert.ok(!read(file).includes(`/${s.repository}/pull/${s.pr}`));
}
assert.ok(s.review.startsWith(`https://github.com/${s.repository}/pull/${s.pr}#`));
}
});
test('archived runs match their pinned packets and recorded hashes', () => {
for (const run of results.runs) {
const s = sample.samples.find(x => x.id === run.id);
const manifest = json(run.manifest);
assert.equal(manifest.head, s.head, run.id);
assert.equal(manifest.base, s.reconstructed_base ?? s.candidate_base, run.id);
assert.equal(manifest.ancestry.compare_status, 'ahead', run.id);
assert.ok(manifest.commits.every(c => c.committed <= s.feedback_at), run.id);
assert.ok(manifest.files.length <= manifest.limits.max_files, run.id);
assert.ok(manifest.total_bytes <= manifest.limits.max_bytes, run.id);
assert.equal(sha256(read(run.manifest)), run.manifest_sha256, run.id);
const output = read(run.output);
assert.equal(sha256(output), run.output_sha256, run.id);
assert.ok(output.startsWith('Architecture review'), run.id);
assert.ok(output.includes(s.head), run.id);
}
});
function validateReportIDs(reports) {
const ids = reports.map(r => r.id);
assert.equal(new Set(ids).size, ids.length, 'duplicate rebuild report ID');
assert.deepEqual([...ids].sort(), sample.samples.filter(s => s.status === 'evaluated').map(s => s.id).sort(),
'rebuild report IDs must exactly cover evaluated runs');
}
test('rebuild reports compare every recorded size/digest and selection with frozen manifests', () => {
const reports = results.packet_rebuilds.map(json);
validateReportIDs(reports);
assert.throws(() => validateReportIDs([reports[0], reports[0], reports[0]]), /duplicate/);
assert.throws(() => validateReportIDs(reports.slice(1)), /exactly cover/);
assert.throws(() => validateReportIDs(reports.map((r, i) => i ? r : { ...r, id: 'unknown' })), /exactly cover/);
for (const report of reports) {
const run = results.runs.find(r => r.id === report.id);
const manifest = json(run.manifest);
assert.equal(report.manifest_sha256, sha256(read(run.manifest)));
assert.equal(report.expected_total_bytes, manifest.total_bytes);
assert.equal(report.actual_total_bytes, manifest.total_bytes);
assert.deepEqual(report.expected_file_order, manifest.files.map(f => f.name));
assert.deepEqual(report.actual_file_order, report.expected_file_order);
assert.deepEqual(report.expected_omitted, manifest.omitted);
assert.deepEqual(report.actual_omitted, manifest.omitted);
assert.equal(report.files.length, manifest.files.length);
report.files.forEach((file, i) => {
const expected = manifest.files[i];
assert.equal(file.name, expected.name);
assert.equal(file.expected_bytes, expected.bytes);
assert.equal(file.actual_bytes, expected.bytes);
assert.equal(file.expected_sha256, expected.sha256);
assert.equal(file.actual_sha256, expected.sha256);
assert.equal(file.match, true);
});
for (const flag of ['all_match', 'ordering_match', 'total_match', 'omissions_match']) assert.equal(report[flag], true);
assert.equal(report.ancestry.status, 'ahead');
assert.equal(report.ancestry.merge_base, manifest.base);
assert.equal(report.ancestry.pre_feedback, true);
assert.equal(report.ancestry.commits_match, true);
}
});
test('all paginated feedback records have explicit root-deduplicated dispositions', () => {
const inventory = json(results.feedback_inventory);
const scoring = json(results.independent_scoring);
assert.equal(scoring.provenance.feedback_sha256, sha256(read(results.feedback_inventory)));
assert.equal(inventory.author, 'cgwalters');
assert.deepEqual(inventory.samples.map(s => s.id), results.runs.map(r => r.id));
const records = inventory.samples.flatMap(s => s.units);
assert.equal(records.length, results.audited_feedback_records);
assert.equal(new Set(records.map(r => r.id)).size, records.length);
assert.equal(scoring.units.length, records.length);
for (const [channel, count] of [['inline', results.audited_inline_comments], ['review', results.audited_review_bodies], ['pr', results.audited_pr_comments]]) {
assert.equal(records.filter(r => r.channel === channel).length, count);
}
for (const entry of inventory.samples) {
assert.deepEqual(entry.channels.map(c => c.channel), ['inline', 'review', 'pr']);
for (const channel of entry.channels) {
assert.equal(channel.complete, true);
assert.equal(channel.per_page, 100);
assert.ok(channel.page_sizes.length > 0 && channel.page_sizes.length <= 10);
assert.ok(channel.page_sizes.at(-1) < 100);
assert.ok(channel.page_sizes.slice(0, -1).every(n => n === 100));
}
const frozen = sample.samples.find(s => s.id === entry.id);
assert.equal(entry.units.map(u => u.created_at).sort()[0], frozen.feedback_at);
for (const record of entry.units) {
const units = scoring.units.filter(u => u.record_id === record.id);
assert.equal(units.length, 1);
const unit = units[0];
assert.equal(unit.sample, entry.id);
assert.equal(unit.url, record.url);
assert.ok(unit.reason);
assert.ok(['concern', 'question', 'nit', 'empty'].includes(unit.classification));
if (unit.classification === 'empty') assert.equal(record.body, '');
for (const root of unit.roots) {
const concern = results.concerns.find(c => c.id === root);
assert.ok(concern, root);
assert.equal(concern.stratum, unit.classification);
assert.equal(concern.scorable, unit.scorable);
assert.equal(concern.matched_by, unit.matched_by);
}
}
}
const roots = scoring.units.flatMap(u => u.roots);
assert.equal(new Set(roots).size, roots.length);
assert.deepEqual([...roots].sort(), results.concerns.map(c => c.id).sort());
assert.equal(scoring.units.filter(u => u.classification === 'nit').length, results.excluded_nits);
assert.equal(scoring.units.filter(u => u.classification === 'empty').length, results.excluded_empty_reviews);
assert.equal(results.scoring_passes, 2);
assert.match(results.human_adjudication, /not obtained/);
const totals = scoring.independent_totals;
for (const [key, value] of Object.entries({ feedback_records: records.length, roots: roots.length,
scorable_concerns: results.strata.concerns.scorable, matched_concerns: results.strata.concerns.matched,
scorable_questions: results.strata.questions.scorable, matched_questions: results.strata.questions.matched,
unscorable_concerns: results.strata.concerns.unscorable, excluded_nits: results.excluded_nits,
excluded_empty_reviews: results.excluded_empty_reviews })) assert.equal(totals[key], value);
assert.equal(scoring.finding_dispositions.length, results.findings.defects.total + results.findings.questions.total);
assert.equal(scoring.finding_dispositions.filter(f => f.matched_by_feedback).length, results.findings.questions.matched);
for (const key of ['supported', 'unsupported', 'unresolved']) {
assert.equal(scoring.finding_dispositions.filter(f => f.audit === key).length, results.unmatched_audit[key]);
}
});
test('evaluation counts are consistent and make no held-out claim', () => {
validateCounts(results);
assert.throws(() => validateCounts({ ...results, architecture_concerns: 4 }));
assert.throws(() => validateCounts({ ...results, executed_reviews: 2 }));
assert.throws(() => validateCounts({ ...results, held_out_claim: true }));
assert.throws(() => validateCounts({
...results, concerns: results.concerns.map(c => ({ ...c, scorable: true, unscorable_reason: undefined, matched_by: c.matched_by ?? 'X1' })),
}));
assert.throws(() => validateCounts({
...results, unmatched_audit: { ...results.unmatched_audit, supported: 3 },
}));
});
# poll must pass both of the fix bounds files dispatch selects per target.
test-poll-bounds:
#!/usr/bin/env node
const assert = require('node:assert/strict');
const poll = require('node:fs').readFileSync('.github/workflows/poll.yml', 'utf8');
const command = /"\$AGENT_BOARD" pass .*\\\n(?: {12}.*\\\n)* {12}.*\n/.exec(poll)?.[0];
assert.ok(command, 'the agent-board pass command');
assert.deepEqual([...command.matchAll(/--fix-bounds (\S+)/g)].map(m => m[1]),
['.github/agentic-job/fix.toml', '.github/agentic-job/midstream-fix.toml']);
console.log('ok poll passes both fix bounds files to agent-board pass');
# poll's `next` step script, run with a stub agent-board that prints its
# arguments: the supervisor and release-train wakes are added only under
# their variables' gates. Every flag it passes is checked against the
# pinned CLI's help (AGENT_BOARD), which reads no token. The failure-only
# fallback uses runner tools and makes just one dispatch attempt.
test-poll-next:
#!/usr/bin/env node
const assert = require('node:assert/strict');
const { execFileSync } = require('node:child_process');
const fs = require('node:fs');
const path = require('node:path');
const poll = fs.readFileSync('.github/workflows/poll.yml', 'utf8');
const step = /^ {6}- name: Start the next pass .*\n((?: {8}.*\n)+)/m.exec(poll)?.[1];
assert.ok(step, 'the next pass step');
assert.match(step, /^ {10}RELEASE_TRAIN: \$\{\{ vars\.RELEASE_TRAIN \}\}$/m);
const script = /^ {8}run: \|\n((?: {10}.*\n)+)/m.exec(step)[1].replace(/^ {10}/gm, '');
const temp = fs.mkdtempSync(path.join(require('node:os').tmpdir(), 'poll-next-'));
const stub = path.join(temp, 'agent-board/bin/agent-board');
fs.mkdirSync(path.dirname(stub), { recursive: true });
fs.writeFileSync(stub, '#!/bin/sh\nprintf "%s\\n" "$@"\n', { mode: 0o755 });
const run = (env) => execFileSync('bash', ['--noprofile', '--norc', '-eo', 'pipefail', '-c', script], {
env: { PATH: process.env.PATH, RUNNER_TEMP: temp, DEFAULT_BRANCH: 'main', SUPERVISOR: '', RELEASE_TRAIN: '', ...env },
}).toString().trimEnd().split('\n');
const base = ['pass', 'next', '--workflow', 'poll.yml', '--branch', 'main', '--input', 'apply=true',
'--apply-title', 'pass (apply)', '--wait-minutes', '10'];
const supervisor = ['--supervisor-workflow', 'supervisor.yml', '--supervisor-stale-minutes', '30',
'--supervisor-input', 'prompt=', '--supervisor-input', 'session=false'];
const train = ['--train-workflow', 'release-train.yml', '--train-stale-minutes', '60'];
for (const value of ['', 'enabled', 'Apply', ' apply', 'dry-run ', 'preview']) {
assert.deepEqual(run({ RELEASE_TRAIN: value }), base, `RELEASE_TRAIN=${JSON.stringify(value)}`);
}
for (const value of ['dry-run', 'apply']) {
assert.deepEqual(run({ RELEASE_TRAIN: value }), [...base, ...train], `RELEASE_TRAIN=${value}`);
}
for (const [value, enabled] of [['off', false], ['preview', true], ['apply', true]]) {
assert.deepEqual(run({ SUPERVISOR: value }), enabled ? [...base, ...supervisor] : base, `SUPERVISOR=${value}`);
}
const all = run({ SUPERVISOR: 'preview', RELEASE_TRAIN: 'apply' });
assert.deepEqual(all, [...base, ...supervisor, ...train]);
fs.rmSync(temp, { recursive: true });
console.log('ok poll next wakes the supervisor and release train only under their gates');
if (process.env.AGENT_BOARD) {
const help = execFileSync(process.env.AGENT_BOARD, ['pass', 'next', '--help']).toString();
for (const flag of new Set(all.filter(arg => arg.startsWith('--')))) {
assert.match(help, new RegExp(`${flag}(?![\\w-])`), `${flag} in agent-board pass next --help`);
}
console.log('ok the pinned agent-board pass next takes every flag poll next passes');
} else {
console.log('skipped: set AGENT_BOARD to check the flags against the pinned CLI');
}
const fallback = /^ {2}next-fallback:\n([\s\S]+)$/m.exec(poll)?.[1];
assert.ok(fallback, 'the final fallback job');
assert.match(fallback, /^ {4}needs: next$/m);
const gate = /^ {4}if: \$\{\{ (.*) \}\}$/m.exec(fallback)[1];
for (const result of ['success', 'failure', 'cancelled', 'skipped']) {
for (const cancelled of [false, true]) {
for (const apply of ['', 'enabled', 'disabled']) {
const got = require('node:vm').runInNewContext(gate, {
cancelled: () => cancelled, needs: { next: { result } }, vars: { PASS_APPLY: apply },
});
assert.equal(got, result === 'failure' && !cancelled && apply === 'enabled');
}
}
}
assert.match(fallback, /^ {4}timeout-minutes: 12$/m);
assert.match(fallback, /^ {4}permissions:\n {6}actions: write\n {4}steps:/m);
assert.doesNotMatch(fallback, /uses:|continue-on-error:|^ {2}\S/m);
assert.equal((fallback.match(/^ {6}- /gm) ?? []).length, 1, 'one step without setup');
assert.match(fallback, /^ {10}GH_TOKEN: \$\{\{ github.token \}\}$/m);
assert.match(fallback, /^ {10}DEFAULT_BRANCH: \$\{\{ github.event.repository.default_branch \}\}$/m);
const recovery = /^ {8}run: \|\n((?: {10}.*\n)+)/m.exec(fallback)[1].replace(/^ {10}/gm, '');
const sandbox = fs.mkdtempSync(path.join(require('node:os').tmpdir(), 'poll-fallback-'));
try {
fs.writeFileSync(path.join(sandbox, 'gh'), '#!/bin/bash\nif [[ $1 == api ]]; then printf "%s\\n" "$*" >&2; printf "2026-11-02T17:51:00Z\\n"; exit "${API_EXIT:-0}"; fi\nprintf "%s\\n" "$*"\nexit "${DISPATCH_EXIT:-0}"\n', { mode: 0o755 });
fs.writeFileSync(path.join(sandbox, 'date'), '#!/bin/bash\nif [[ $1 == -d ]]; then printf "1000\\n"; else printf "%s\\n" "$NOW"; fi\n', { mode: 0o755 });
fs.writeFileSync(path.join(sandbox, 'sleep'), '#!/bin/bash\nprintf "sleep %s\\n" "$*"\n', { mode: 0o755 });
const api = 'api repos/owner/repo/actions/runs/42 --jq .run_started_at';
const dispatch = 'workflow run poll.yml --repo owner/repo --ref main -f apply=true';
const recover = (env) => require('node:child_process').spawnSync('bash', ['--noprofile', '--norc', '-eo', 'pipefail', '-c', recovery], {
env: { PATH: `${sandbox}:${process.env.PATH}`, GITHUB_REPOSITORY: 'owner/repo', GITHUB_RUN_ID: '42', DEFAULT_BRANCH: 'main', ...env }, encoding: 'utf8',
});
for (const [now, delay] of [['1120', '480'], ['1600', null], ['1900', null], ['900', '600']]) {
const got = recover({ NOW: now });
assert.equal(got.status, 0, got.stderr);
assert.equal(got.stderr.trimEnd(), api);
assert.deepEqual(got.stdout.trimEnd().split('\n'), [...(delay ? [`sleep ${delay}`] : []), dispatch]);
}
const failed = recover({ NOW: '1900', DISPATCH_EXIT: '1' });
assert.equal(failed.status, 1, 'dispatch failure stays visible');
assert.equal(failed.stderr.trimEnd(), api);
assert.deepEqual(failed.stdout.trimEnd().split('\n'), [dispatch], 'no retry on failure');
const unreadable = recover({ NOW: '1900', API_EXIT: '1' });
assert.equal(unreadable.status, 1);
assert.equal(unreadable.stderr.trimEnd(), api);
assert.equal(unreadable.stdout, '', 'API failure stops recovery');
} finally {
fs.rmSync(sandbox, { recursive: true, force: true });
}
console.log('ok poll fallback is failure-only, waits at most ten minutes and dispatches once');
# supervisor.yml's folded concurrency group and health `if`, evaluated as
# JavaScript on events: a run the gate skips gets a group of its own
# (#803), and a session dispatch, prompt or not, is an agent run.
test-supervisor-workflow:
#!/usr/bin/env node
const assert = require('node:assert/strict');
const fs = require('node:fs');
const vm = require('node:vm');
const workflow = fs.readFileSync('.github/workflows/supervisor.yml', 'utf8');
const folded = (key, indent) => new RegExp(`^${' '.repeat(indent)}${key}: >-\\n((?:${' '.repeat(indent + 2)}.*\\n)+)`, 'm')
.exec(workflow)[1].split('\n').map(line => line.trim()).filter(Boolean).join(' ');
const group = /^\$\{\{ (.*) \}\}$/.exec(folded('group', 2))[1];
const gate = folded('if', 4);
assert.ok(group.startsWith(`(!(${gate}) && format('supervisor-ignored-{0}', github.run_id))`));
const { operator } = JSON.parse(fs.readFileSync('.github/poll/agent-board.json', 'utf8'));
const { issue } = JSON.parse(fs.readFileSync('.github/supervisor/config.json', 'utf8')).agent.requests;
const comment = { user: { login: operator } };
const requests = { number: issue, locked: true };
const dispatch = (inputs, ref) => ({ event_name: 'workflow_dispatch', inputs: { prompt: '', session: false, ...inputs }, ref });
for (const [event, expected] of [
[{ event_name: 'issue_comment', event: { comment, issue: requests } }, 'supervisor-agent'],
[{ event_name: 'issue_comment', event: { comment, issue: { ...requests, locked: false } } }, 'supervisor-ignored-42'],
[{ event_name: 'issue_comment', event: { comment, issue: { ...requests, pull_request: { url: 'x' } } } }, 'supervisor-ignored-42'],
[{ event_name: 'issue_comment', event: { comment: { user: { login: 'outsider' } }, issue: requests } }, 'supervisor-ignored-42'],
[dispatch({ prompt: 'do it' }), 'supervisor-agent'],
[dispatch({ prompt: 'do it' }, 'refs/heads/other'), 'supervisor-ignored-42'],
[dispatch({}), 'supervisor-health'],
[dispatch({}, 'refs/heads/other'), 'supervisor-ignored-42'],
[dispatch({ session: true }), 'supervisor-agent'],
[dispatch({ session: true, prompt: 'look at #1' }), 'supervisor-agent'],
[dispatch({ session: true }, 'refs/heads/other'), 'supervisor-ignored-42'],
[{ event_name: 'schedule', event: { schedule: '*/15 * * * *' } }, 'supervisor-health'],
[{ event_name: 'schedule', event: { schedule: '7 */2 * * *' } }, 'supervisor-agent'],
]) {
const context = {
format: (f, ...a) => f.replace(/\{(\d+)\}/g, (_, i) => a[i]),
fromJSON: JSON.parse,
vars: {},
inputs: event.inputs ?? {},
github: {
event_name: event.event_name, run_id: 42, ref: event.ref ?? 'refs/heads/main',
event: { repository: { default_branch: 'main' }, ...event.event },
},
};
const got = vm.runInNewContext(group, context);
assert.equal(got, expected, JSON.stringify(event));
assert.equal(vm.runInNewContext(gate, context), !got.startsWith('supervisor-ignored-'), JSON.stringify(event));
}
// A session is an explicit request to `due`, and reaches the wrapper
// with session.json's idle timeout.
assert.match(workflow, /^ {10}PROMPT: \$\{\{ inputs\.prompt \|\| \(inputs\.session && 'session' \|\| ''\) \}\}$/m);
assert.match(workflow, /^ {6}session: \$\{\{ needs\.health\.outputs\.session == 'true' \}\}\n {6}idle-minutes: \$\{\{ needs\.health\.outputs\.idle \}\}$/m);
assert.match(workflow, /"idle=\\\(\.idleMinutes\)"' \.github\/agentic-job\/session\.json/);
console.log('ok supervisor concurrency groups and health gate, sessions included');
# The one agent-board commit every workflow pins as ` AGENT_BOARD_REV:`,
# printed; fails when the workflows pin different revisions, or none, so a
# move by hand cannot miss a workflow.
agent-board-pin:
#!/usr/bin/env bash
set -euo pipefail
pins=$(sed -n 's/^ AGENT_BOARD_REV: //p' .github/workflows/*.yml | sort -u)
if ! [[ $pins =~ ^[0-9a-f]{40}$ ]]; then
echo "::error::.github/workflows pin more than one agent-board revision, or none:" >&2
grep -H '^ AGENT_BOARD_REV:' .github/workflows/*.yml >&2 || true
exit 1
fi
echo "$pins"
# Opens or updates the one pull request, from the bot's fork, that moves
# every AGENT_BOARD_REV to the head of agent-board's configured branch
# (.github/poll/agent-board.json), once all that commit's check runs have
# passed and it is ahead of the pin (agent-board-pin.yml). It commits in this
# checkout, and needs GITHUB_REPOSITORY and GH_TOKEN to push and open the PR.
bump-agent-board:
#!/usr/bin/env bash
set -euo pipefail
pin=$({{ just_executable() }} agent-board-pin)
source=$(jq -er .source .github/poll/agent-board.json)
branch=$(jq -er .branch .github/poll/agent-board.json)
bot=$(jq -er .bot .github/poll/agent-board.json)
upstream=${source#https://github.com/}
head=$(gh api "repos/$upstream/commits/$branch" --jq .sha)
[[ $head =~ ^[0-9a-f]{40}$ ]]
if [[ $head == "$pin" ]]; then echo "the pin is $branch's head $head"; exit 0; fi
runs=$(gh api --paginate "repos/$upstream/commits/$head/check-runs" --jq '.check_runs[] | {name, status, conclusion}' | jq -cs .)
if ! jq -e 'length > 0 and all(.[]; .status == "completed" and (.conclusion | IN("success", "neutral", "skipped")))' <<< "$runs" > /dev/null; then
echo "$head: CI is not green yet: $runs"; exit 0
fi
# Publication follows CI: green source alone is not an installable pin.
if ! published=$(gh release view "build-$head" --repo "$upstream" --json isDraft --jq '.isDraft') || [[ $published != false ]]; then
echo "$head: no published binary release yet"; exit 0
fi
(export RUNNER_TEMP=$(mktemp -d); trap 'rm -rf "$RUNNER_TEMP"' EXIT
GITHUB_OUTPUT=/dev/null {{ just_executable() }} build-agent-board "$head")
# Never backwards or onto rewritten history; compare lists at most 250 commits.
compare=$(gh api "repos/$upstream/compare/$pin...$head")
status=$(jq -r .status <<< "$compare")
[[ $status == ahead ]] || { echo "::error::$head is $status of the pin $pin, not ahead"; exit 1; }
title="workflows: Move agent-board to ${head:0:7}"
pr_branch=agent-board-pin
open=$(gh pr list --repo "$GITHUB_REPOSITORY" --author "$bot" --head "$pr_branch" --state open --json number,title --jq '.[0] // empty')
if [[ $(jq -r '.title // ""' <<< "$open") == "$title" ]]; then echo "already proposed: $title"; exit 0; fi
fork=$(gh api --paginate "repos/$GITHUB_REPOSITORY/forks" --jq ".[] | select(.owner.login == \"$bot\") | .full_name" | head -n1)
[[ -n $fork ]] || { echo "::error::$bot has no fork of $GITHUB_REPOSITORY"; exit 1; }
body=$(
echo "Every \`AGENT_BOARD_REV\` in \`.github/workflows/\` moves from $pin to $head, $branch's head, whose check runs all passed. Review [what changes]($source/compare/$pin...$head) in the program given the bot's token. The agent-board commits in between:"
echo
# Subjects in code spans, so their #N and @login do not reference here.
jq -r --arg s "$source" '.commits[] | "- [\(.sha[0:7])](\($s)/commit/\(.sha)) `\(.commit.message | split("\n")[0] | gsub("`"; "'\''"))`"' <<< "$compare"
jq -r 'if .total_commits > (.commits | length) then "\n\(.total_commits - (.commits | length)) more commits are not listed." else empty end' <<< "$compare"
)
sed -i "s/^ AGENT_BOARD_REV: $pin\$/ AGENT_BOARD_REV: $head/" .github/workflows/*.yml
[[ $({{ just_executable() }} agent-board-pin) == "$head" ]]
email=$(gh api "users/$bot" --jq '"\(.id)+\(.login)@users.noreply.github.com"')
git -c user.name="$bot" -c user.email="$email" commit -q -a -m "$title" -m "$body"
gh auth setup-git
git push -q --force "https://github.com/$fork" "HEAD:refs/heads/$pr_branch"
if [[ -n $open ]]; then
gh pr edit "$(jq -r .number <<< "$open")" --repo "$GITHUB_REPOSITORY" --title "$title" --body "$body"
else
gh pr create --repo "$GITHUB_REPOSITORY" --head "${fork%%/*}:$pr_branch" --base "$(gh repo view "$GITHUB_REPOSITORY" --json defaultBranchRef --jq .defaultBranchRef.name)" --title "$title" --body "$body"
fi
# Whether the effective rules of a branch, GitHub's
# `repos/OWNER/REPO/rules/branches/BRANCH` response on stdin, hold what the
# `[unprotected_files]` bounds in dispatch.toml rest on: one pull_request
# rule requiring an approval and dismissing stale approvals on push. Any one
# rule is enough, as rulesets add up (review's own requires none; the org's
# does), but both must be on the same rule. Anything but an array fails.
review-gate:
@jq -e 'type == "array" and any(.[]; .type == "pull_request" and .parameters.required_approving_review_count >= 1 and .parameters.dismiss_stale_reviews_on_push == true)' > /dev/null
# review-gate on each fixture under .github/ruleset-check, which must
# pass if its name starts with pass- and fail otherwise.
test-review-gate:
#!/usr/bin/env bash
set -euo pipefail
for f in .github/ruleset-check/*.json; do
want=fail; [[ $(basename "$f") == pass-* ]] && want=pass
got=fail; {{ just_executable() }} review-gate < "$f" 2>/dev/null && got=pass
[[ $got == "$want" ]] || { echo "$f: want $want, got $got" >&2; exit 1; }
echo "ok $f ($want)"
done
# review-gate on the default branch of every repository in
# unprotected_files.repos (ruleset-check.yml). A repository that fails it,
# or whose rules cannot be read, gets a ::error with the pull_request rules
# found and is listed in the step output `failed`; then the recipe fails.
check-review-gate:
#!/usr/bin/env bash
set -euo pipefail
repos=$(python3 -c 'import sys, tomllib; print(*tomllib.load(open(sys.argv[1], "rb"))["unprotected_files"]["repos"])' .github/agentic-job/dispatch.toml)
failed=()
for repo in $repos; do
rules=
if branch=$(gh api "repos/$repo" --jq .default_branch) &&
rules=$(gh api --paginate "repos/$repo/rules/branches/$branch" | jq -cs add); then
if {{ just_executable() }} review-gate <<< "$rules"; then echo "ok $repo"; continue; fi
found=$(jq -c '[.[]? | select(.type == "pull_request") | {ruleset_id, ruleset_source} + (.parameters | {required_approving_review_count, dismiss_stale_reviews_on_push})]' <<< "$rules")
echo "::error::$repo: no pull_request rule on $branch requires an approval and dismisses stale approvals on push; found $found"
else
echo "::error::$repo: could not read the rules of its default branch"
fi
failed+=("$repo")
done
echo "failed=${failed[*]}" >> "${GITHUB_OUTPUT:-/dev/null}"
[[ ${#failed[@]} -eq 0 ]]
# A repository's drift from the manifest (.github/forge-sync/repos.json,
# #544), one line each, from its fetched files on stdin:
# {"repo": ..., "review": <review.yml or null>, "dependabot": <dependabot.yml
# or null>, "justfile": <bool>, "renovate": <renovate.json or null>}. Fails
# on any drift, a repository the manifest does not list, or an exemption that
# is unknown or gives no reason. The YAML is parsed with yq (its first
# document), so only a job's `uses` counts as a call of the reviewer, and
# every such call must be at the pin. Only a manifest that sets
# renovate_preset requires the Renovate config to list it in `extends`, so
# that no repository's runner label goes stale unnoticed (#1054).
forge-sync-drift manifest:
#!/usr/bin/env bash
set -euo pipefail
input=$(cat)
yaml() { jq -r --arg k "$1" '.[$k] // empty' <<< "$input" | yq -p=yaml -o=json -I=0 'select(document_index == 0)'; }
review=$(yaml review)
dependabot=$(yaml dependabot)
drift=$(jq -r --slurpfile m "$1" --argjson rv "$review" --argjson dp "$dependabot" '
$m[0] as $m | .repo as $r | (first($m.repos[] | select(.repo == $r)) // null) as $e
| if $e == null then "\($r) \($path): expected an entry, found none" else ($e.exempt // {}) as $x
| ($x | to_entries[] | select((.key | IN("review", "dependabot", "justfile", "renovate") | not) or (.value | type != "string" or length == 0))
| "\($r) \($path): expected exempt.\(.key) to be review, dependabot, justfile or renovate with a reason, found \(.value | tojson)"),
("cgwalters-forge/actions/.github/workflows/cgwalters-review.yml@" as $call | "\($call)\($m.review_pin)" as $want
| [$rv | objects | .jobs | objects | .[] | objects | .uses | strings] as $uses
| [$uses[] | select(startswith($call))] as $calls
| select($x.review == null and ($calls == [] or any($calls[]; . != $want)))
| "\($r) .github/workflows/review.yml: expected uses \($want), found \(if .review == null then "no file" else if $calls == [] then $uses else $calls end | join(", ") | if . == "" then "no uses" else . end end)"),
([$dp | objects | .updates | arrays | .[] | objects | select(."package-ecosystem" == "github-actions")
| (.schedule | objects | .interval) // "no interval" | tostring] as $i
| select($x.dependabot == null and ($i == [] or any($i[]; . != "daily")))
| "\($r) .github/dependabot.yml: expected a github-actions update with interval daily, found \(if .dependabot == null then "no file" elif $i == [] then "no github-actions update" else $i | join(", ") end)"),
(select($x.justfile == null and .justfile != true) | "\($r) Justfile: expected a root Justfile, found none"),
($m.renovate_preset as $p | select($p != null and $x.renovate == null)
| if $p | type != "string" or length == 0 then "\($r) \($path): expected renovate_preset to be a preset name, found \($p | tojson)"
else .renovate as $text | (if $text == null then null else try ($text | fromjson) catch null end) as $rc
| select([$rc | objects | .extends | arrays | .[] | select(. == $p)] == [])
| "\($r) renovate.json: expected extends to list \($p), found \(if $text == null then "no file" elif $rc | type != "object" then "no JSON object" else $rc.extends // [] | tojson end)" end)
end' --arg path "$1" <<< "$input")
[[ -z $drift ]] || { echo "$drift"; exit 1; }
# forge-sync-drift on each fixture under .github/forge-sync/testdata, and
# under its renovate/ whose manifest sets renovate_preset, against the
# manifest.json beside it, passing for pass-* and reporting drift for the
# fixture's repository otherwise; then on this
# repository's own files against the real manifest, so moving the pin in
# review.yml fails here until review_pin moves with it, and dropping
# .github/renovate.json or renovate_preset fails too.
test-forge-sync:
#!/usr/bin/env bash
set -euo pipefail
top=.github/forge-sync/testdata
for dir in "$top" "$top/renovate"; do
for f in "$dir"/pass-*.json "$dir"/fail-*.json; do
want=fail; [[ $(basename "$f") == pass-* ]] && want=pass
got=pass; out=$({{ just_executable() }} forge-sync-drift "$dir/manifest.json" < "$f" 2> /dev/null) || got=fail
[[ $got == pass || $out == "$(jq -r .repo "$f") "* ]] || got="an error: $out"
[[ $got == "$want" ]] || { echo "$f: want $want, got $got" >&2; exit 1; }
echo "ok $f ($want)"
done
done
bad=$(mktemp)
# Without renovate_preset no Renovate config is required; with one that
# is not a preset name every repository drifts.
{{ just_executable() }} forge-sync-drift "$top/manifest.json" < "$top/renovate/fail-no-renovate.json"
echo "ok no renovate_preset requires no Renovate config"
jq '.renovate_preset = 1' "$top/renovate/manifest.json" > "$bad"
if {{ just_executable() }} forge-sync-drift "$bad" < "$top/renovate/pass-extends-preset.json" > /dev/null 2>&1; then
echo "forge-sync-drift with renovate_preset 1: want fail, got pass" >&2; exit 1
fi
echo "ok forge-sync-drift refuses renovate_preset 1"
for manifest in '{}' '{"repos": []}' '{"repos": [{}]}' '[' ''; do
if [[ -n $manifest ]]; then echo "$manifest" > "$bad"; else rm -f "$bad"; fi
if {{ just_executable() }} forge-sync "$bad" > /dev/null 2>&1; then
echo "forge-sync with manifest '$manifest': want fail, got pass" >&2; exit 1
fi
echo "ok forge-sync refuses manifest '${manifest:-<missing>}'"
done
self=$(jq -n --rawfile review .github/workflows/review.yml --rawfile dependabot .github/dependabot.yml \
--rawfile renovate .github/renovate.json '{repo: "cgwalters-forge/tracker", $review, $dependabot, justfile: true, $renovate}')
{{ just_executable() }} forge-sync-drift .github/forge-sync/repos.json <<< "$self"
echo "ok this repository against .github/forge-sync/repos.json"
# The real manifest must keep requiring the preset of this repository.
if {{ just_executable() }} forge-sync-drift .github/forge-sync/repos.json <<< "$(jq '.renovate = null' <<< "$self")" > /dev/null 2>&1; then
echo "this repository without .github/renovate.json: want drift, got none" >&2; exit 1
fi
echo "ok this repository without .github/renovate.json drifts"
# forge-sync-drift on the default branch of every repository in the
# manifest, read through the contents API (a 404 is a missing file); fails
# if any repository drifted or could not be read. The step output `failed`
# lists those repositories and `drift`, a JSON string, their drift lines
# (forge-sync.yml).
forge-sync manifest=".github/forge-sync/repos.json":
#!/usr/bin/env bash
set -euo pipefail
err=$(mktemp)
trap 'rm -f "$err"' EXIT
fetch() {
local body
if body=$(gh api "repos/$1/contents/$2" -H 'Accept: application/vnd.github.raw' 2> "$err"); then jq -Rs . <<< "$body"
elif grep -q 'HTTP 404' "$err"; then echo null
else echo "::error::$1: could not read $2: $(cat "$err")" >&2; return 1; fi
}
repos=$(jq -er '.repos | select(length > 0 and all(.[]; .repo | type == "string")) | .[].repo' "$1") || { echo "::error::$1: expected a manifest listing repos" >&2; exit 1; }
# The Renovate config is read only when the manifest asks for a preset,
# from the two JSON places Renovate looks for it that the forge uses.
preset=$(jq -r '.renovate_preset // empty' "$1")
failed=() drift=
for repo in $repos; do
out= renovate=null
if j=$(fetch "$repo" Justfile) && if [[ $j == null ]]; then j=$(fetch "$repo" justfile); fi &&