Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
174 changes: 136 additions & 38 deletions .github/extensions/ai-attribution/README.md

Large diffs are not rendered by default.

97 changes: 90 additions & 7 deletions .github/extensions/ai-attribution/extension.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@ import { EXTENSION_DIR, MODEL_SECTIONS, loadModel, loadResearch } from './lib/mo
import { recipeView } from './lib/recipe.mjs';
import { startCanvasServer } from './lib/server.mjs';
import { verifyRecipe } from './lib/verify.mjs';
import { validateIdentityLinks } from './lib/actors.mjs';
import { fetchBilling, billingView } from './lib/billing.mjs';
import { sourceLabels as SOURCE_NAMES } from './ui/format.mjs';

const VIEWS = ['outcomes', 'recipe', 'surfaces', 'intersection', 'model', 'gaps', 'coverage', 'research'];
const DEFAULT_WINDOW_DAYS = 30;
Expand Down Expand Up @@ -62,7 +65,6 @@ function summarize(coverage) {
};
}

const SOURCE_NAMES = { sessionStore: 'CLI/app', traces: 'VS Code', agentHost: 'VS Code Copilot harness' };

function statusLine(coverage) {
const window = coverage.window.days ? `${coverage.window.days}-day window` : 'All history';
Expand All @@ -84,7 +86,30 @@ function instanceFor(ctx) {
}

async function recipeFor(instance) {
return recipeView((await coverageFor(instance.options)).recipe, instance.verification);
const recipe = (await coverageFor(instance.options)).recipe;
return {
...recipeView(recipe, instance.verification, { identityLinks: instance.identityLinks }),
billing: billingView(instance.billing, recipe),
};
}

async function readBilling(instance, scope) {
if (instance.readingBilling) throw new CanvasError('busy', 'A billing request is already running.');
instance.readingBilling = true;
instance.billing = null;
try {
instance.billing = await fetchBilling(scope);
instance.server.broadcast('recipe', {});
return recipeFor(instance);
} finally {
instance.readingBilling = false;
}
Comment on lines +100 to +106
}

async function setIdentityLinks(instance, links) {
instance.identityLinks = validateIdentityLinks(links);
instance.server.broadcast('recipe', {});
return recipeFor(instance);
}

/** Explicit network step: asks GitHub and Azure Boards about the cycles already found locally. */
Expand All @@ -103,8 +128,26 @@ async function verifyInstance(instance, organization) {
return recipeFor(instance);
}

function summarizeRecipe(view) {
function summarizeRecipe(view, filter = {}) {
const developerSources = view.developers.sources
.filter((source) => !filter.sourceId || source.id === filter.sourceId)
.map((source) => {
const owned = filter.perspective === 'ownership';
const selected = filter.actorKey === '__unknown' ? source[owned ? 'unowned' : 'unknown'] :
filter.actorKey ? source.actors.find((actor) => actor.key === filter.actorKey)?.[owned ? 'ownership' : 'usage'] :
owned ? source.ownership : source;
return { id: source.id, summary: selected ? {
total: selected.total, measured: selected.measured, inferred: selected.inferred, unattributed: selected.unattributed,
work: selected.work.map((row) => {
const cycle = view.cycles.find((c) => c.key === row.cycleKey);
return { ...row, repository: cycle?.repository ?? null, branch: cycle?.branch ?? null,
workItems: cycle?.workItems.map((w) => w.reference) ?? [], pullRequest: cycle?.pullRequest?.number ?? null };
}),
} : null };
});
return {
billing: view.billing,
developers: { actors: view.developers.actors, identityLinks: view.developers.identityLinks, sources: developerSources, perspective: filter.perspective ?? 'usage' },
verification: view.verification,
totals: view.totals,
stations: view.stations.map(({ id, any, strict, unknown }) => ({ id, any, strict, unknown })),
Expand All @@ -114,6 +157,7 @@ function summarizeRecipe(view) {
repository: c.repository,
branch: c.branch,
workItems: c.workItems.map((w) => w.reference),
ownership: c.ownership,
pullRequest: c.pullRequest?.number ?? null,
reached: c.reached,
steps: Object.fromEntries(Object.entries(c.steps).map(([id, s]) => [id, s.state])),
Expand All @@ -130,6 +174,9 @@ async function openInstance(ctx) {
options: { windowDays: DEFAULT_WINDOW_DAYS },
verification: null,
verifying: null,
identityLinks: [],
billing: null,
readingBilling: false,
};
instance.server = await startCanvasServer({
uiDir: UI_DIR,
Expand All @@ -145,6 +192,8 @@ async function openInstance(ctx) {
},
recipe: () => recipeFor(instance),
verify: ({ azureDevOpsOrg } = {}) => verifyInstance(instance, azureDevOpsOrg),
identityLinks: (links) => setIdentityLinks(instance, links),
billing: (scope) => readBilling(instance, scope),
getView: () => instance.view,
setView: (view) => {
if (VIEWS.includes(view)) instance.view = view;
Expand All @@ -163,6 +212,7 @@ async function openInstance(ctx) {
if (input.tracesDbPath !== undefined) options.tracesDbPath = input.tracesDbPath || undefined;
if (input.agentHostTracesDbPath !== undefined) options.agentHostTracesDbPath = input.agentHostTracesDbPath || undefined;
if (input.azureDevOpsOrg !== undefined) options.azureDevOpsOrg = input.azureDevOpsOrg || undefined;
if (input.identityLinks !== undefined) instance.identityLinks = validateIdentityLinks(input.identityLinks);
instance.options = options;

// Every open recomputes, so a reopened panel never shows a stale snapshot;
Expand Down Expand Up @@ -200,13 +250,21 @@ await joinSession({
tracesDbPath: { type: 'string', description: "Override for VS Code's agent-traces.db." },
agentHostTracesDbPath: {
type: 'string',
description: "Override for VS Code's Copilot-harness agent-host-traces.db.",
description: "Override for VS Code's Copilot Agent Host harness agent-host-traces.db.",
},
azureDevOpsOrg: {
type: 'string',
description:
'Azure DevOps organization for Recipe verification (name, https://dev.azure.com/<org> or https://<org>.visualstudio.com). Defaults to az devops configure.',
},
identityLinks: {
type: 'array', maxItems: 32,
description: 'Explicit Azure Boards account-key to GitHub-login links. No display-name matching.',
items: {
type: 'object', additionalProperties: false, required: ['boardsKey', 'githubLogin'],
properties: { boardsKey: { type: 'string' }, githubLogin: { type: 'string' } },
},
},
},
},
actions: [
Expand Down Expand Up @@ -250,11 +308,36 @@ await joinSession({
inputSchema: { type: 'object', additionalProperties: false, properties: {} },
handler: async (ctx) => withoutRecipe(await coverageFor(instanceFor(ctx).options)),
},
{
name: 'reconcile_recipe_billing',
description: 'Explicitly read one completed UTC day of GitHub Copilot AI-credit billing for a user and paying account. Compare native captured consumption per source; work amounts are estimated allocations, never PR invoices.',
inputSchema: {
type: 'object', additionalProperties: false, required: ['level', 'account', 'user', 'day'],
properties: {
level: { type: 'string', enum: ['organization', 'enterprise', 'user'] },
account: { type: 'string' }, user: { type: 'string' },
day: { type: 'string', pattern: '^\\d{4}-\\d{2}-\\d{2}$' },
},
},
handler: async (ctx) => summarizeRecipe(await readBilling(instanceFor(ctx), ctx.input)),
},
{
name: 'get_recipe',
description: 'Return the Recipe summary: cycle stations, KPIs, branch-convention adherence and the latest cycles.',
inputSchema: { type: 'object', additionalProperties: false, properties: {} },
handler: async (ctx) => summarizeRecipe(await recipeFor(instanceFor(ctx))),
description: 'Return Recipe and developer credit attribution. Filter by actorKey/sourceId; usage separates measured from inferred, ownership shows all contributors to owned work.',
inputSchema: { type: 'object', additionalProperties: false, properties: {
actorKey: { type: 'string' }, sourceId: { type: 'string', enum: ['sessionStore', 'traces', 'agentHost'] },
perspective: { type: 'string', enum: ['usage', 'ownership'] },
} },
handler: async (ctx) => summarizeRecipe(await recipeFor(instanceFor(ctx)), ctx.input ?? {}),
},
{
name: 'set_recipe_identity_links',
description: 'Set explicit Azure Boards account-key to GitHub-login links for this panel. Never infer them from names or emails.',
inputSchema: { type: 'object', additionalProperties: false, required: ['links'], properties: {
links: { type: 'array', maxItems: 32, items: { type: 'object', additionalProperties: false, required: ['boardsKey', 'githubLogin'],
properties: { boardsKey: { type: 'string' }, githubLogin: { type: 'string' } } } },
} },
handler: async (ctx) => summarizeRecipe(await setIdentityLinks(instanceFor(ctx), ctx.input.links)),
},
{
name: 'verify_recipe',
Expand Down
139 changes: 139 additions & 0 deletions .github/extensions/ai-attribution/lib/actors.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
// Provider-qualified identities. Device names, branch prefixes and display names
// never become account keys; cross-provider links must be supplied explicitly.
export function githubActor(login, kind = 'User') {
if (typeof login !== 'string' || !/^[a-z\d](?:[a-z\d-]{0,37}[a-z\d])?(?:\[bot\])?$/i.test(login)) return null;
const bot = kind === 'Bot' || login.endsWith('[bot]');
return { key: `github:${login.toLowerCase()}`, label: login, provider: 'github', bot };
}

export function telemetryActor(value) {
if (typeof value !== 'string') return null;
const name = value.trim();
if (!name || name.length > 128 || /[\u0000-\u001f\u007f]/.test(name)) return null;
return githubActor(name) ?? { key: `telemetry:${name}`, label: name, provider: 'telemetry', bot: false };
}

export function boardsActor(value, organization) {
if (!value || typeof value.id !== 'string' || !/^[a-f\d-]{8,64}$/i.test(value.id)) return null;
const label = typeof value.displayName === 'string' && value.displayName.length <= 128 && !/[\u0000-\u001f\u007f]/.test(value.displayName)
? value.displayName : value.id;
return { key: `boards:${organization.toLowerCase()}:${value.id.toLowerCase()}`, label, provider: 'azure-boards', bot: false };
}

export function validateIdentityLinks(links) {
if (!Array.isArray(links) || links.length > 32) throw new Error('Provide at most 32 identity links.');
const seen = new Set();
return links.map((link) => {
if (!link || typeof link.boardsKey !== 'string' ||
!/^boards:https:\/\/(?:dev\.azure\.com\/[a-z\d-]+|[a-z\d-]+\.visualstudio\.com):[a-f\d-]{8,64}$/i.test(link.boardsKey) ||
!githubActor(link.githubLogin) || seen.has(link.boardsKey.toLowerCase())) {
throw new Error('Each identity link needs a unique Azure Boards account key and a GitHub login.');
}
const boardsKey = link.boardsKey.toLowerCase();
seen.add(boardsKey);
return { boardsKey, githubLogin: link.githubLogin };
});
}

function resolve(actor, links) {
if (!actor) return null;
const link = links.find((l) => l.boardsKey === actor.key);
return link ? { ...githubActor(link.githubLogin), linkedFrom: actor.key } : actor;
}

/** Current assignment is displayed as a role, never backdated to infer usage. */
export function ownershipFor(workItems, pr, links) {
const roles = [];
if (pr?.author) roles.push({ role: 'PR author', actor: resolve(pr.author, links), identity: pr.author, at: pr.createdAt });
for (const item of workItems) {
for (const [field, role, at] of [
['assignees', 'Assigned now', item.assignmentCheckedAt],
['assigneesAtStart', 'Assigned at pickup', item.assignmentAt],
]) {
for (const actor of item[field] ?? []) roles.push({ role, actor: resolve(actor, links), identity: actor, at, reference: item.reference });
}
}
const historical = roles.filter((r) => r.role !== 'Assigned now' && !r.actor.bot);
const candidates = [...new Map(historical.map((r) => [r.actor.key, r.actor])).values()];
return {
roles,
inferred: candidates.length === 1 ? candidates[0] : null,
inference: candidates.length === 1
? [...new Set(historical.map((r) => r.role))].join(' + ')
: candidates.length > 1 ? 'Conflicting owners' : 'No historical owner',
};
}

function newTally() {
return { calls: 0, creditedCalls: 0, credits: 0 };
}

function add(tally, usage) {
tally.calls += usage.calls;
tally.creditedCalls += usage.creditedCalls;
tally.credits += usage.credits ?? 0;
}

function reported(tally) {
return { ...tally, credits: tally.creditedCalls ? tally.credits : null };
}

/** Partitions every source into disjoint measured, inferred and unattributed usage. */
export function attributeUsage(sources, ownership, links) {
return sources.flatMap((source) => {
const actors = source.actors ?? [{ actor: null, calls: source.calls, creditedCalls: source.creditedCalls, credits: source.credits }];
return actors.map((usage) => ({
source: source.id,
...reported(usage),
actor: resolve(usage.actor, links) ?? ownership?.inferred ?? null,
method: usage.actor ? 'measured' : ownership?.inferred ? 'inferred' : 'unattributed',
evidence: usage.actor ? 'user.name' : ownership?.inference ?? 'No user identity',
}));
});
}

export function developerReport(cycles, unmatched, identityLinks) {
const actors = new Map();
const entries = [
...cycles.flatMap((cycle) => cycle.attribution.map((entry) => ({ ...entry, cycleKey: cycle.key }))),
...attributeUsage(unmatched, null, identityLinks).map((entry) => ({ ...entry, cycleKey: null })),
];
for (const entry of entries) if (entry.actor) actors.set(entry.actor.key, entry.actor);
// Owners without attributed usage remain selectable as work owners, not consumers.
for (const cycle of cycles) for (const role of cycle.ownership.roles) actors.set(role.actor.key, role.actor);
const summarize = (list, withWork = true) => {
const tally = { total: newTally(), measured: newTally(), inferred: newTally(), unattributed: newTally() };
const work = new Map();
for (const entry of list) {
add(tally.total, entry);
add(tally[entry.method], entry);
const group = work.get(entry.cycleKey) ?? [];
group.push(entry);
work.set(entry.cycleKey, group);
}
return {
...Object.fromEntries(Object.entries(tally).map(([key, value]) => [key, reported(value)])),
...(withWork ? { work: [...work.entries()].map(([cycleKey, group]) => ({ cycleKey, ...summarize(group, false) }))
.sort((a, b) => (b.total.credits ?? -1) - (a.total.credits ?? -1)) } : {}),
};
};
const sourceIds = [...new Set(entries.map((entry) => entry.source))];
const ownedCycles = new Set(cycles.filter((cycle) => cycle.ownership.roles.length).map((cycle) => cycle.key));
return {
sources: sourceIds.map((id) => {
const list = entries.filter((entry) => entry.source === id);
return {
id, ...summarize(list),
unknown: summarize(list.filter((entry) => !entry.actor)),
ownership: summarize(list.filter((entry) => ownedCycles.has(entry.cycleKey))),
unowned: summarize(list.filter((entry) => !ownedCycles.has(entry.cycleKey))),
actors: [...actors.keys()].map((key) => {
const owned = new Set(cycles.filter((cycle) => cycle.ownership.roles.some((r) => r.actor.key === key)).map((cycle) => cycle.key));
return { key, usage: summarize(list.filter((entry) => entry.actor?.key === key)), ownership: summarize(list.filter((entry) => owned.has(entry.cycleKey))) };
}),
};
}),
actors: [...actors.values()].sort((a, b) => a.label.localeCompare(b.label)),
identityLinks,
};
}
Loading
Loading