Skip to content

Commit 332efec

Browse files
bradygasterCopilot
andcommitted
test(ci): enforce structural gh-aw compiler pins
Closes #2103 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
1 parent abb4226 commit 332efec

1 file changed

Lines changed: 139 additions & 10 deletions

File tree

‎test/gh-aw-quality.test.ts‎

Lines changed: 139 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import { tmpdir } from 'node:os';
1515
import { execFileSync, execSync, spawnSync } from 'node:child_process';
1616
import { createHash } from 'node:crypto';
1717
import { minimatch } from 'minimatch';
18+
import { isMap, isSeq, parseDocument, type YAMLMap } from 'yaml';
1819
import { POSIX_SHELL, NO_POSIX_SHELL_MESSAGE, requirePosixShell } from './posix-shell';
1920
import { createFirstInstallFixture } from './helpers/gh-aw-install-fixture.js';
2021
import {
@@ -3515,6 +3516,144 @@ describe('gh-aw: activation roster guard counts only data rows (#1605)', () => {
35153516
});
35163517
});
35173518

3519+
describe('gh-aw: CI compiler pin contract', () => {
3520+
const ci = readText(join(process.cwd(), '.github/workflows/squad-ci.yml'));
3521+
const installs = [
3522+
{
3523+
job: 'test',
3524+
name: 'Install gh-aw extension',
3525+
commands: [`gh extension install --pin ${MIN_GH_AW_VERSION} github/gh-aw`],
3526+
},
3527+
{
3528+
job: 'gh-aw-compile',
3529+
name: `Install gh-aw ${MIN_GH_AW_VERSION} (pinned)`,
3530+
commands: [
3531+
`gh extension install --force --pin ${MIN_GH_AW_VERSION} github/gh-aw`,
3532+
`test "$(gh aw --version 2>&1 | awk '{print $NF}')" = ${MIN_GH_AW_VERSION}`,
3533+
],
3534+
},
3535+
] as const;
3536+
3537+
function assertCompilerPins(source: string): void {
3538+
const workflow = parseDocument(source);
3539+
expect(workflow.errors).toEqual([]);
3540+
const jobs = workflow.get('jobs');
3541+
if (!isMap(jobs)) throw new Error('CI must define jobs');
3542+
const installSteps: Array<{ job: unknown; name: unknown; commands: string[] }> = [];
3543+
for (const { key, value } of jobs.items) {
3544+
if (!isMap(value)) throw new Error('CI job must be a mapping');
3545+
const steps = value.get('steps');
3546+
if (steps === undefined) continue; // Reusable workflow jobs have no steps.
3547+
if (!isSeq(steps)) throw new Error('CI steps must be a sequence');
3548+
for (const step of steps.items) {
3549+
if (!isMap(step)) throw new Error('CI step must be a mapping');
3550+
const run = step.get('run');
3551+
if (run === undefined) continue;
3552+
if (typeof run !== 'string') throw new Error('CI run must be a string');
3553+
const commands = run.split('\n').map(line => line.replace(
3554+
/'[^']*'|"(?:\\.|[^"\\])*"|\\.|(^|\s)#.*$/g,
3555+
(token, comment) => comment === undefined ? token : '',
3556+
).trim()).filter(Boolean);
3557+
const name = step.get('name');
3558+
// Include named slots even if their install was removed or replaced by output.
3559+
// Unexpected install-like scripts fail closed; never accept a shell wrapper
3560+
// merely because it contains the pinned command as a substring.
3561+
if (installs.some(install => install.name === name) || commands.some(line =>
3562+
!/^(?:echo|printf)(?:\s+(?:'[^']*'|"[^"$`]*"))+$/.test(line) &&
3563+
/\bgh\s+(?:extension|ext)\s+install\b.*\bgithub\/gh-aw\b/.test(line))) {
3564+
installSteps.push({ job: String(key), name, commands });
3565+
}
3566+
}
3567+
}
3568+
expect(installSteps).toEqual(installs);
3569+
}
3570+
3571+
function mutateInstall(job: string, change: (step: YAMLMap) => void): string {
3572+
const workflow = parseDocument(ci);
3573+
const steps = workflow.getIn(['jobs', job, 'steps']);
3574+
if (!isSeq(steps)) throw new Error(`Missing steps for ${job}`);
3575+
const name = installs.find(install => install.job === job)?.name;
3576+
const step = steps.items.find(step => isMap(step) && step.get('name') === name);
3577+
if (!isMap(step)) throw new Error(`Missing install step for ${job}`);
3578+
change(step);
3579+
return workflow.toString();
3580+
}
3581+
3582+
it('pins every CI compiler install to the package minimum gh-aw version', () => {
3583+
assertCompilerPins(ci);
3584+
});
3585+
3586+
it('ignores YAML comments, shell comment lines, and non-run strings', () => {
3587+
const workflow = parseDocument(ci);
3588+
workflow.setIn(['env', 'INSTALL_EXAMPLE'], 'gh extension install github/gh-aw');
3589+
workflow.addIn(['jobs', 'changes', 'steps'], {
3590+
name: installs[0].commands[0],
3591+
run: `echo '${installs[0].commands[0]}'\nprintf "%s" "gh extension install github/gh-aw"`,
3592+
});
3593+
const source = mutateInstall('test', step => {
3594+
step.set('run', `# gh extension install github/gh-aw\n\n${installs[0].commands[0]} # pinned\n`);
3595+
step.setIn(['env', 'PIN_EXAMPLE'], installs[1].commands[0]);
3596+
});
3597+
assertCompilerPins(`# gh extension install github/gh-aw\n${source}`);
3598+
assertCompilerPins(workflow.toString());
3599+
});
3600+
3601+
describe.each(installs)('$job install step', ({ job, commands }) => {
3602+
it.each([
3603+
['unpinned', 'gh extension install github/gh-aw'],
3604+
['wrong version', commands[0].replace(MIN_GH_AW_VERSION, 'v0.0.0')],
3605+
['duplicate command', `${commands[0]}\n${commands[0]}`],
3606+
['comment-only pin', `# ${commands[0]}\ngh extension install github/gh-aw`],
3607+
['inline comment pin', `gh extension install github/gh-aw # ${commands[0]}`],
3608+
['echoed pin', `echo "${commands[0]}"`],
3609+
['shell wrapper', `bash -c '${commands[0]}'`],
3610+
['conditional command', `false && ${commands[0]}`],
3611+
['preceding command', `echo preparing\n${commands[0]}`],
3612+
['appended command', `${commands[0]}; gh extension install github/gh-aw`],
3613+
['heredoc text', `cat <<'INSTALL'\n${commands[0]}\nINSTALL`],
3614+
])('rejects %s', (_label, command) => {
3615+
const source = mutateInstall(job, step => {
3616+
step.set('run', [command, ...commands.slice(1)].join('\n'));
3617+
});
3618+
expect(() => assertCompilerPins(source)).toThrow();
3619+
});
3620+
3621+
it('rejects an install moved to an unexpected step', () => {
3622+
const source = mutateInstall(job, step => step.set('name', 'Unexpected install'));
3623+
expect(() => assertCompilerPins(source)).toThrow();
3624+
});
3625+
3626+
it('rejects an install moved to an unexpected job', () => {
3627+
const workflow = parseDocument(ci);
3628+
const original = workflow.getIn(['jobs', job]);
3629+
workflow.deleteIn(['jobs', job]);
3630+
workflow.setIn(['jobs', 'unexpected-job'], original);
3631+
expect(() => assertCompilerPins(workflow.toString())).toThrow();
3632+
});
3633+
3634+
it('rejects a missing install step', () => {
3635+
const source = mutateInstall(job, step => step.delete('run'));
3636+
expect(() => assertCompilerPins(source)).toThrow();
3637+
});
3638+
3639+
it.each([
3640+
['duplicate step', job, commands[0]],
3641+
['extra pinned install', 'changes', commands[0]],
3642+
['extra unpinned install', 'changes', 'gh extension install github/gh-aw'],
3643+
['extra wrapped install', 'changes', `bash -c '${commands[0]}'`],
3644+
['extra install after quoted hash', 'changes', `echo "#"; ${commands[0]}`],
3645+
['extra install in output substitution', 'changes', `echo "$(${commands[0]})"`],
3646+
])('rejects %s', (_label, targetJob, command) => {
3647+
const workflow = parseDocument(ci);
3648+
workflow.addIn(['jobs', targetJob, 'steps'], {
3649+
name: 'Extra install',
3650+
run: command,
3651+
});
3652+
expect(() => assertCompilerPins(workflow.toString())).toThrow();
3653+
});
3654+
});
3655+
});
3656+
35183657
describe('gh-aw: canonical package integrity contract', () => {
35193658
const revisionA = 'a'.repeat(40);
35203659
const revisionB = 'b'.repeat(40);
@@ -3647,16 +3786,6 @@ describe('gh-aw: canonical package integrity contract', () => {
36473786
expect(normalized).not.toContain(revisionA);
36483787
});
36493788

3650-
it('pins every CI compiler install to the package minimum gh-aw version', () => {
3651-
const ci = readText(join(process.cwd(), '.github/workflows/squad-ci.yml'));
3652-
const pins = [...ci.matchAll(
3653-
/gh extension install(?: --force)? --pin (v[0-9.]+) github\/gh-aw/g,
3654-
)].map(match => match[1]);
3655-
3656-
expect(pins).toHaveLength(2);
3657-
expect(new Set(pins)).toEqual(new Set([MIN_GH_AW_VERSION]));
3658-
});
3659-
36603789
it('accepts only the deterministic retained bootstrap trigger sentinel', () => {
36613790
const root = makeConsumer();
36623791
const probePath = join(root, TRIGGER_PROBE_DESTINATION);

0 commit comments

Comments
 (0)