From 92e251b6e34fb9a735e5cb96ca96981e0fa96692 Mon Sep 17 00:00:00 2001 From: Glen Chung Date: Wed, 16 Oct 2024 17:23:16 -0700 Subject: [PATCH 1/2] Support A/B Compiler Arguments Traits - Depends on cpptools' update to provide ProjectContextResult. - Added the following new traits - intellisense: compiler information disclaimer. - intellisenseBegin: to note the begin of IntelliSense information. - compilerArguments: a list of compiler command arguments that could affect Copilot generating completions. - directAsks: direct asking Copilot to do something instead of providing an argument. - intellisenseEnd: to note the end of IntelliSense information. - A/B Experimental flags - copilotcppTraits: boolean flag to enable cpp traits - copilotcppExcludeTraits: string array to exclude individual trait, i.e., compilerArguments. - copilotcppMsvcCompilerArgumentFilter: regex string to match compiler arguments for GCC. - copilotcppClangCompilerArgumentFilter: regex string to match compiler arguments for Clang. - copilotcppGccCompilerArgumentFilter: regex string to match compiler arguments for MSVC. - copilotcppCompilerArgumentDirectAskMap: a stringify map string to map arguments to direct ask statements. --- Extension/src/LanguageServer/client.ts | 22 ++ .../src/LanguageServer/copilotProviders.ts | 49 ++- Extension/src/LanguageServer/lmTool.ts | 128 +++++++- Extension/src/LanguageServer/utils.ts | 6 + Extension/src/telemetry.ts | 2 +- .../tests/copilotProviders.test.ts | 188 +++++++---- .../SingleRootProject/tests/lmTool.test.ts | 308 ++++++++++++++++++ 7 files changed, 613 insertions(+), 90 deletions(-) create mode 100644 Extension/test/scenarios/SingleRootProject/tests/lmTool.test.ts diff --git a/Extension/src/LanguageServer/client.ts b/Extension/src/LanguageServer/client.ts index e46743815e..c05034e643 100644 --- a/Extension/src/LanguageServer/client.ts +++ b/Extension/src/LanguageServer/client.ts @@ -541,6 +541,19 @@ export interface ChatContextResult { targetArchitecture: string; } +export interface FileContextResult { + compilerArguments: string[]; +} + +export interface ProjectContextResult { + language: string; + standardVersion: string; + compiler: string; + targetPlatform: string; + targetArchitecture: string; + fileContext: FileContextResult; +} + // Requests const PreInitializationRequest: RequestType = new RequestType('cpptools/preinitialize'); const InitializationRequest: RequestType = new RequestType('cpptools/initialize'); @@ -561,6 +574,7 @@ const GenerateDoxygenCommentRequest: RequestType = new RequestType('cpptools/didChangeCppProperties'); const IncludesRequest: RequestType = new RequestType('cpptools/getIncludes'); const CppContextRequest: RequestType = new RequestType('cpptools/getChatContext'); +const ProjectContextRequest: RequestType = new RequestType('cpptools/getProjectContext'); // Notifications to the server const DidOpenNotification: NotificationType = new NotificationType('textDocument/didOpen'); @@ -792,6 +806,7 @@ export interface Client { addTrustedCompiler(path: string): Promise; getIncludes(maxDepth: number, token: vscode.CancellationToken): Promise; getChatContext(token: vscode.CancellationToken): Promise; + getProjectContext(token: vscode.CancellationToken): Promise; } export function createClient(workspaceFolder?: vscode.WorkspaceFolder): Client { @@ -2220,6 +2235,12 @@ export class DefaultClient implements Client { () => this.languageClient.sendRequest(CppContextRequest, null, token), token); } + public async getProjectContext(token: vscode.CancellationToken): Promise { + await withCancellation(this.ready, token); + return DefaultClient.withLspCancellationHandling( + () => this.languageClient.sendRequest(ProjectContextRequest, null, token), token); + } + /** * a Promise that can be awaited to know when it's ok to proceed. * @@ -4123,4 +4144,5 @@ class NullClient implements Client { addTrustedCompiler(path: string): Promise { return Promise.resolve(); } getIncludes(maxDepth: number, token: vscode.CancellationToken): Promise { return Promise.resolve({} as GetIncludesResult); } getChatContext(token: vscode.CancellationToken): Promise { return Promise.resolve({} as ChatContextResult); } + getProjectContext(token: vscode.CancellationToken): Promise { return Promise.resolve({} as ProjectContextResult); } } diff --git a/Extension/src/LanguageServer/copilotProviders.ts b/Extension/src/LanguageServer/copilotProviders.ts index f23554f76d..c51f2b88f7 100644 --- a/Extension/src/LanguageServer/copilotProviders.ts +++ b/Extension/src/LanguageServer/copilotProviders.ts @@ -6,8 +6,9 @@ import * as vscode from 'vscode'; import * as util from '../common'; -import { ChatContextResult, GetIncludesResult } from './client'; +import { GetIncludesResult } from './client'; import { getActiveClient } from './extension'; +import { getProjectContext } from './lmTool'; export interface CopilotTrait { name: string; @@ -38,19 +39,51 @@ export async function registerRelatedFilesProvider(): Promise { const getIncludesHandler = async () => (await getIncludesWithCancellation(1, token))?.includedFiles.map(file => vscode.Uri.file(file)) ?? []; const getTraitsHandler = async () => { - const chatContext: ChatContextResult | undefined = await (getActiveClient().getChatContext(token) ?? undefined); + const cppContext = await getProjectContext(context, token); - if (!chatContext) { + if (!cppContext) { return undefined; } let traits: CopilotTrait[] = [ - { name: "language", value: chatContext.language, includeInPrompt: true, promptTextOverride: `The language is ${chatContext.language}.` }, - { name: "compiler", value: chatContext.compiler, includeInPrompt: true, promptTextOverride: `This project compiles using ${chatContext.compiler}.` }, - { name: "standardVersion", value: chatContext.standardVersion, includeInPrompt: true, promptTextOverride: `This project uses the ${chatContext.standardVersion} language standard.` }, - { name: "targetPlatform", value: chatContext.targetPlatform, includeInPrompt: true, promptTextOverride: `This build targets ${chatContext.targetPlatform}.` }, - { name: "targetArchitecture", value: chatContext.targetArchitecture, includeInPrompt: true, promptTextOverride: `This build targets ${chatContext.targetArchitecture}.` } + { name: "intellisense", value: 'intellisense', includeInPrompt: true, promptTextOverride: `IntelliSense is currently configured with the following compiler information. It's best effort to reflect the active configuration, and the project may have more configurations targeting different platforms.` }, + { name: "intellisenseBegin", value: 'Begin', includeInPrompt: true, promptTextOverride: `Begin of IntelliSense information.` } ]; + if (cppContext.language) { + traits.push({ name: "language", value: cppContext.language, includeInPrompt: true, promptTextOverride: `The language is ${cppContext.language}.` }); + } + if (cppContext.compiler) { + traits.push({ name: "compiler", value: cppContext.compiler, includeInPrompt: true, promptTextOverride: `This project compiles using ${cppContext.compiler}.` }); + } + if (cppContext.standardVersion) { + traits.push({ name: "standardVersion", value: cppContext.standardVersion, includeInPrompt: true, promptTextOverride: `This project uses the ${cppContext.standardVersion} language standard.` }); + } + if (cppContext.targetPlatform) { + traits.push({ name: "targetPlatform", value: cppContext.targetPlatform, includeInPrompt: true, promptTextOverride: `This build targets ${cppContext.targetPlatform}.` }); + } + if (cppContext.targetArchitecture) { + traits.push({ name: "targetArchitecture", value: cppContext.targetArchitecture, includeInPrompt: true, promptTextOverride: `This build targets ${cppContext.targetArchitecture}.` }); + } + let directAsks: string = ''; + if (cppContext.compilerArguments.length > 0) { + // Example: JSON.stringify({'-fno-rtti': "Do not generate code using RTTI keywords."}) + const directAskMap: { [key: string]: string } = JSON.parse(context.flags.copilotcppCompilerArgumentDirectAskMap as string ?? '{}'); + const updatedArguments = cppContext.compilerArguments.filter(arg => { + if (directAskMap[arg]) { + directAsks += `${directAskMap[arg]} `; + return false; + } + return true; + }); + + const compilerArgumentsValue = updatedArguments.join(", "); + traits.push({ name: "compilerArguments", value: compilerArgumentsValue, includeInPrompt: true, promptTextOverride: `The compiler arguments include: ${compilerArgumentsValue}.` }); + } + if (directAsks) { + traits.push({ name: "directAsks", value: directAsks, includeInPrompt: true, promptTextOverride: directAsks }); + } + + traits.push({ name: "intellisenseEnd", value: 'End', includeInPrompt: true, promptTextOverride: `End of IntelliSense information.` }); const excludeTraits = context.flags.copilotcppExcludeTraits as string[] ?? []; traits = traits.filter(trait => !excludeTraits.includes(trait.name)); diff --git a/Extension/src/LanguageServer/lmTool.ts b/Extension/src/LanguageServer/lmTool.ts index c3fad8b6eb..82c54d3135 100644 --- a/Extension/src/LanguageServer/lmTool.ts +++ b/Extension/src/LanguageServer/lmTool.ts @@ -9,9 +9,13 @@ import { localize } from 'vscode-nls'; import * as util from '../common'; import * as logger from '../logger'; import * as telemetry from '../telemetry'; -import { ChatContextResult } from './client'; +import { ChatContextResult, ProjectContextResult } from './client'; import { getClients } from './extension'; +import { checkTime } from './utils'; +const MSVC: string = 'MSVC'; +const Clang: string = 'Clang'; +const GCC: string = 'GCC'; const knownValues: { [Property in keyof ChatContextResult]?: { [id: string]: string } } = { language: { 'c': 'C', @@ -19,9 +23,9 @@ const knownValues: { [Property in keyof ChatContextResult]?: { [id: string]: str 'cuda-cpp': 'CUDA C++' }, compiler: { - 'msvc': 'MSVC', - 'clang': 'Clang', - 'gcc': 'GCC' + 'msvc': MSVC, + 'clang': Clang, + 'gcc': GCC }, standardVersion: { 'c++98': 'C++98', @@ -44,6 +48,112 @@ const knownValues: { [Property in keyof ChatContextResult]?: { [id: string]: str } }; +function formatChatContext(context: ChatContextResult | ProjectContextResult): void { + type KnownKeys = 'language' | 'standardVersion' | 'compiler' | 'targetPlatform'; + for (const key in knownValues) { + const knownKey = key as KnownKeys; + if (knownValues[knownKey] && context[knownKey]) { + // Clear the value if it's not in the known values. + context[knownKey] = knownValues[knownKey][context[knownKey]] || ""; + } + } +} + +export interface ProjectContext { + language: string; + standardVersion: string; + compiler: string; + targetPlatform: string; + targetArchitecture: string; + compilerArguments: string[]; +} + +// Set these values for local testing purpose without involving control tower. +const defaultCompilerArgumentFilters: { [id: string]: RegExp | undefined } = { + MSVC: undefined, // Example: /^(\/std:.*|\/EHs-c-|\/GR-|\/await.*)$/, + Clang: undefined, + GCC: undefined // Example: /^(-std=.*|-fno-rtti|-fno-exceptions)$/ +}; + +function filterComplierArguments(compiler: string, compilerArguments: string[], context: { flags: Record }): string[] { + const defaultFilter: RegExp | undefined = defaultCompilerArgumentFilters[compiler] ?? undefined; + let additionalFilter: RegExp | undefined; + switch (compiler) { + case MSVC: + additionalFilter = context.flags.copilotcppMsvcCompilerArgumentFilter ? new RegExp(context.flags.copilotcppMsvcCompilerArgumentFilter as string) : undefined; + break; + case Clang: + additionalFilter = context.flags.copilotcppClangCompilerArgumentFilter ? new RegExp(context.flags.copilotcppClangCompilerArgumentFilter as string) : undefined; + break; + case GCC: + additionalFilter = context.flags.copilotcppGccCompilerArgumentFilter ? new RegExp(context.flags.copilotcppGccCompilerArgumentFilter as string) : undefined; + break; + } + + return compilerArguments.filter(arg => defaultFilter?.test(arg) || additionalFilter?.test(arg)); +} + +export async function getProjectContext(context: { flags: Record }, token: vscode.CancellationToken): Promise { + const telemetryProperties: Record = {}; + try { + const projectContext = await checkTime(async () => await getClients()?.ActiveClient?.getProjectContext(token) ?? undefined); + telemetryProperties["time"] = projectContext.time.toString(); + if (!projectContext.result) { + return undefined; + } + + formatChatContext(projectContext.result); + + const result: ProjectContext = { + language: projectContext.result.language, + standardVersion: projectContext.result.standardVersion, + compiler: projectContext.result.compiler, + targetPlatform: projectContext.result.targetPlatform, + targetArchitecture: projectContext.result.targetArchitecture, + compilerArguments: [] + }; + + if (projectContext.result.language) { + telemetryProperties["language"] = projectContext.result.language; + } + if (projectContext.result.compiler) { + telemetryProperties["compiler"] = projectContext.result.compiler; + } + if (projectContext.result.standardVersion) { + telemetryProperties["standardVersion"] = projectContext.result.standardVersion; + } + if (projectContext.result.targetPlatform) { + telemetryProperties["targetPlatform"] = projectContext.result.targetPlatform; + } + if (projectContext.result.targetArchitecture) { + telemetryProperties["targetArchitecture"] = projectContext.result.targetArchitecture; + } + telemetryProperties["compilerArgumentCount"] = projectContext.result.fileContext.compilerArguments.length.toString(); + // Telemtry to learn about the argument distribution. The filtered arguments are expected to be non-PII. + if (projectContext.result.fileContext.compilerArguments.length) { + const filteredCompilerArguments = filterComplierArguments(projectContext.result.compiler, projectContext.result.fileContext.compilerArguments, context); + if (filteredCompilerArguments.length > 0) { + telemetryProperties["filteredCompilerArguments"] = filteredCompilerArguments.join(', '); + result.compilerArguments = filteredCompilerArguments; + } + } + + return result; + } + catch { + try { + logger.getOutputChannelLogger().appendLine(localize("copilot.cppcontext.error", "Error while retrieving the project context.")); + } + catch { + // Intentionally swallow any exception. + } + telemetryProperties["error"] = "true"; + return undefined; + } finally { + telemetry.logLanguageModelToolEvent('Completions/tool', telemetryProperties); + } +} + export class CppConfigurationLanguageModelTool implements vscode.LanguageModelTool { public async invoke(options: vscode.LanguageModelToolInvocationOptions, token: vscode.CancellationToken): Promise { return new vscode.LanguageModelToolResult([ @@ -63,13 +173,7 @@ export class CppConfigurationLanguageModelTool implements vscode.LanguageModelTo return 'No configuration information is available for the active document.'; } - for (const key in knownValues) { - const knownKey = key as keyof ChatContextResult; - if (knownValues[knownKey] && chatContext[knownKey]) { - // Clear the value if it's not in the known values. - chatContext[knownKey] = knownValues[knownKey][chatContext[knownKey]] || ""; - } - } + formatChatContext(chatContext); let contextString = ""; if (chatContext.language) { @@ -100,7 +204,7 @@ export class CppConfigurationLanguageModelTool implements vscode.LanguageModelTo telemetryProperties["error"] = "true"; return ""; } finally { - telemetry.logLanguageModelToolEvent('cpp', telemetryProperties); + telemetry.logLanguageModelToolEvent('Chat/Tool/cpp', telemetryProperties); } } diff --git a/Extension/src/LanguageServer/utils.ts b/Extension/src/LanguageServer/utils.ts index e8c8073ee1..1ad736dad5 100644 --- a/Extension/src/LanguageServer/utils.ts +++ b/Extension/src/LanguageServer/utils.ts @@ -112,3 +112,9 @@ export async function withCancellation(promise: Promise, token: vscode.Can }); }); } + +export async function checkTime(fn: () => Promise): Promise<{ result: T; time: number }> { + const start = Date.now(); + const result = await fn(); + return { result, time: Date.now() - start }; +} diff --git a/Extension/src/telemetry.ts b/Extension/src/telemetry.ts index 600ffa4c45..434a222854 100644 --- a/Extension/src/telemetry.ts +++ b/Extension/src/telemetry.ts @@ -126,7 +126,7 @@ export function logLanguageServerEvent(eventName: string, properties?: Record, metrics?: Record): void { const sendTelemetry = () => { if (experimentationTelemetry) { - const eventNamePrefix: string = "C_Cpp/Copilot/Chat/Tool/"; + const eventNamePrefix: string = "C_Cpp/Copilot/"; experimentationTelemetry.sendTelemetryEvent(eventNamePrefix + eventName, properties, metrics); } }; diff --git a/Extension/test/scenarios/SingleRootProject/tests/copilotProviders.test.ts b/Extension/test/scenarios/SingleRootProject/tests/copilotProviders.test.ts index 54052e122d..f1c1e8908b 100644 --- a/Extension/test/scenarios/SingleRootProject/tests/copilotProviders.test.ts +++ b/Extension/test/scenarios/SingleRootProject/tests/copilotProviders.test.ts @@ -9,11 +9,13 @@ import * as proxyquire from 'proxyquire'; import * as sinon from 'sinon'; import * as vscode from 'vscode'; import * as util from '../../../../src/common'; -import { ChatContextResult, DefaultClient, GetIncludesResult } from '../../../../src/LanguageServer/client'; +import { DefaultClient, GetIncludesResult } from '../../../../src/LanguageServer/client'; import { CopilotApi, CopilotTrait } from '../../../../src/LanguageServer/copilotProviders'; import * as extension from '../../../../src/LanguageServer/extension'; +import * as lmTool from '../../../../src/LanguageServer/lmTool'; +import { ProjectContext } from '../../../../src/LanguageServer/lmTool'; -describe('registerRelatedFilesProvider', () => { +describe('copilotProviders Tests', () => { let moduleUnderTest: any; let mockCopilotApi: sinon.SinonStubbedInstance; let getActiveClientStub: sinon.SinonStub; @@ -74,12 +76,12 @@ describe('registerRelatedFilesProvider', () => { sinon.restore(); }); - const arrange = ({ vscodeExtension, getIncludeFiles, chatContext, rootUri, flags }: - { vscodeExtension?: vscode.Extension; getIncludeFiles?: GetIncludesResult; chatContext?: ChatContextResult; rootUri?: vscode.Uri; flags?: Record } = - { vscodeExtension: undefined, getIncludeFiles: undefined, chatContext: undefined, rootUri: undefined, flags: {} } + const arrange = ({ vscodeExtension, getIncludeFiles, projectContext, rootUri, flags }: + { vscodeExtension?: vscode.Extension; getIncludeFiles?: GetIncludesResult; projectContext?: ProjectContext; rootUri?: vscode.Uri; flags?: Record } = + { vscodeExtension: undefined, getIncludeFiles: undefined, projectContext: undefined, rootUri: undefined, flags: {} } ) => { activeClientStub.getIncludes.resolves(getIncludeFiles); - activeClientStub.getChatContext.resolves(chatContext); + sinon.stub(lmTool, 'getProjectContext').resolves(projectContext); sinon.stub(activeClientStub, 'RootUri').get(() => rootUri); mockCopilotApi.registerRelatedFilesProvider.callsFake((_providerId: { extensionId: string; languageId: string }, callback: (uri: vscode.Uri, context: { flags: Record }, cancellationToken: vscode.CancellationToken) => Promise<{ entries: vscode.Uri[]; traits?: CopilotTrait[] }>) => { const tokenSource = new vscode.CancellationTokenSource(); @@ -108,11 +110,11 @@ describe('registerRelatedFilesProvider', () => { ok(mockCopilotApi.registerRelatedFilesProvider.calledWithMatch(sinon.match({ extensionId: 'test-extension-id', languageId: sinon.match.in(['c', 'cpp', 'cuda-cpp']) })), 'registerRelatedFilesProvider should be called with the correct providerId and languageId'); }); - it('should not add #cpp traits when ChatContext isn\'t available.', async () => { + it('should not provide project context traits when ChatContext isn\'t available.', async () => { arrange({ vscodeExtension: vscodeExtension, getIncludeFiles: { includedFiles }, - chatContext: undefined, + projectContext: undefined, rootUri, flags: { copilotcppTraits: true } }); @@ -130,17 +132,20 @@ describe('registerRelatedFilesProvider', () => { ok(result.traits === undefined, 'result.traits should be undefined'); }); - it('should not add #cpp traits when copilotcppTraits flag is false.', async () => { + const projectContextNoArgs: ProjectContext = { + language: 'C++', + standardVersion: 'C++20', + compiler: 'MSVC', + targetPlatform: 'Windows', + targetArchitecture: 'x64', + compilerArguments: [] + }; + + it('should not provide project context traits when copilotcppTraits flag is false.', async () => { arrange({ vscodeExtension: vscodeExtension, getIncludeFiles: { includedFiles }, - chatContext: { - language: 'c++', - standardVersion: 'c++20', - compiler: 'msvc', - targetPlatform: 'windows', - targetArchitecture: 'x64' - }, + projectContext: projectContextNoArgs, rootUri, flags: { copilotcppTraits: false } }); @@ -148,27 +153,15 @@ describe('registerRelatedFilesProvider', () => { const result = await callbackPromise; - ok(vscodeGetExtensionsStub.calledOnce, 'vscode.extensions.getExtension should be called once'); - ok(mockCopilotApi.registerRelatedFilesProvider.calledWithMatch(sinon.match({ extensionId: 'test-extension-id', languageId: sinon.match.in(['c', 'cpp', 'cuda-cpp']) })), 'registerRelatedFilesProvider should be called with the correct providerId and languageId'); - ok(getActiveClientStub.callCount !== 0, 'getActiveClient should be called'); - ok(callbackPromise, 'callbackPromise should be defined'); ok(result, 'result should be defined'); - ok(result.entries.length === 1, 'result.entries should have 1 included file'); - ok(result.entries[0].toString() === expectedInclude, `result.entries should have "${expectedInclude}"`); ok(result.traits === undefined, 'result.traits should be undefined'); }); - it('should add #cpp traits when copilotcppTraits flag is true.', async () => { + it('should provide project context traits when copilotcppTraits flag is true.', async () => { arrange({ vscodeExtension: vscodeExtension, getIncludeFiles: { includedFiles }, - chatContext: { - language: 'c++', - standardVersion: 'c++20', - compiler: 'msvc', - targetPlatform: 'windows', - targetArchitecture: 'x64' - }, + projectContext: projectContextNoArgs, rootUri, flags: { copilotcppTraits: true } }); @@ -176,50 +169,49 @@ describe('registerRelatedFilesProvider', () => { const result = await callbackPromise; - ok(vscodeGetExtensionsStub.calledOnce, 'vscode.extensions.getExtension should be called once'); - ok(mockCopilotApi.registerRelatedFilesProvider.calledThrice, 'registerRelatedFilesProvider should be called three times'); - ok(mockCopilotApi.registerRelatedFilesProvider.calledWithMatch(sinon.match({ extensionId: 'test-extension-id', languageId: sinon.match.in(['c', 'cpp', 'cuda-cpp']) })), 'registerRelatedFilesProvider should be called with the correct providerId and languageId'); - ok(getActiveClientStub.callCount !== 0, 'getActiveClient should be called'); - ok(callbackPromise, 'callbackPromise should be defined'); ok(result, 'result should be defined'); - ok(result.entries.length === 1, 'result.entries should have 1 included file'); - ok(result.entries[0].toString() === expectedInclude, `result.entries should have "${expectedInclude}"`); ok(result.traits, 'result.traits should be defined'); - ok(result.traits.length === 5, 'result.traits should have 5 traits'); - ok(result.traits[0].name === 'language', 'result.traits[0].name should be "language"'); - ok(result.traits[0].value === 'c++', 'result.traits[0].value should be "c++"'); - ok(result.traits[0].includeInPrompt, 'result.traits[0].includeInPrompt should be true'); - ok(result.traits[0].promptTextOverride === 'The language is c++.', 'result.traits[0].promptTextOverride should be "The language is c++."'); - ok(result.traits[1].name === 'compiler', 'result.traits[1].name should be "compiler"'); - ok(result.traits[1].value === 'msvc', 'result.traits[1].value should be "msvc"'); - ok(result.traits[1].includeInPrompt, 'result.traits[1].includeInPrompt should be true'); - ok(result.traits[1].promptTextOverride === 'This project compiles using msvc.', 'result.traits[1].promptTextOverride should be "This project compiles using msvc."'); - ok(result.traits[2].name === 'standardVersion', 'result.traits[2].name should be "standardVersion"'); - ok(result.traits[2].value === 'c++20', 'result.traits[2].value should be "c++20"'); - ok(result.traits[2].includeInPrompt, 'result.traits[2].includeInPrompt should be true'); - ok(result.traits[2].promptTextOverride === 'This project uses the c++20 language standard.', 'result.traits[2].promptTextOverride should be "This project uses the c++20 language standard."'); - ok(result.traits[3].name === 'targetPlatform', 'result.traits[3].name should be "targetPlatform"'); - ok(result.traits[3].value === 'windows', 'result.traits[3].value should be "windows"'); - ok(result.traits[3].includeInPrompt, 'result.traits[3].includeInPrompt should be true'); - ok(result.traits[3].promptTextOverride === 'This build targets windows.', 'result.traits[3].promptTextOverride should be "This build targets windows."'); - ok(result.traits[4].name === 'targetArchitecture', 'result.traits[4].name should be "targetArchitecture"'); - ok(result.traits[4].value === 'x64', 'result.traits[4].value should be "x64"'); - ok(result.traits[4].includeInPrompt, 'result.traits[4].includeInPrompt should be true'); - ok(result.traits[4].promptTextOverride === 'This build targets x64.', 'result.traits[4].promptTextOverride should be "This build targets x64."'); + ok(result.traits.length === 8, 'result.traits should have 6 traits if none are excluded'); + ok(result.traits.find((trait) => trait.name === 'intellisense'), 'result.traits should have a intellisense trait'); + ok(result.traits.find((trait) => trait.name === 'intellisense')?.value === 'intellisense', 'result.traits should have a intellisense trait with value "intellisense"'); + ok(result.traits.find((trait) => trait.name === 'intellisense')?.includeInPrompt, 'result.traits should have a intellisense trait with includeInPrompt true'); + ok(result.traits.find((trait) => trait.name === 'intellisense')?.promptTextOverride === 'IntelliSense is currently configured with the following compiler information. It\'s best effort to reflect the active configuration, and the project may have more configurations targeting different platforms.', 'result.traits should have a intellisense trait with promptTextOverride'); + ok(result.traits.find((trait) => trait.name === 'intellisenseBegin'), 'result.traits should have a intellisenseBegin trait'); + ok(result.traits.find((trait) => trait.name === 'intellisenseBegin')?.value === 'Begin', 'result.traits should have a intellisense trait with value "Begin"'); + ok(result.traits.find((trait) => trait.name === 'intellisenseBegin')?.includeInPrompt, 'result.traits should have a intellisenseBegin trait with includeInPrompt true'); + ok(result.traits.find((trait) => trait.name === 'intellisenseBegin')?.promptTextOverride === 'Begin of IntelliSense information.', 'result.traits should have a intellisenseBegin trait with promptTextOverride'); + ok(result.traits.find((trait) => trait.name === 'language'), 'result.traits should have a language trait'); + ok(result.traits.find((trait) => trait.name === 'language')?.value === 'C++', 'result.traits should have a language trait with value "C++"'); + ok(result.traits.find((trait) => trait.name === 'language')?.includeInPrompt, 'result.traits should have a language trait with includeInPrompt true'); + ok(result.traits.find((trait) => trait.name === 'language')?.promptTextOverride === 'The language is C++.', 'result.traits should have a language trait with promptTextOverride'); + ok(result.traits.find((trait) => trait.name === 'compiler'), 'result.traits should have a compiler trait'); + ok(result.traits.find((trait) => trait.name === 'compiler')?.value === 'MSVC', 'result.traits should have a compiler trait with value "MSVC"'); + ok(result.traits.find((trait) => trait.name === 'compiler')?.includeInPrompt, 'result.traits should have a compiler trait with includeInPrompt true'); + ok(result.traits.find((trait) => trait.name === 'compiler')?.promptTextOverride === 'This project compiles using MSVC.', 'result.traits should have a compiler trait with promptTextOverride'); + ok(result.traits.find((trait) => trait.name === 'standardVersion'), 'result.traits should have a standardVersion trait'); + ok(result.traits.find((trait) => trait.name === 'standardVersion')?.value === 'C++20', 'result.traits should have a standardVersion trait with value "C++20"'); + ok(result.traits.find((trait) => trait.name === 'standardVersion')?.includeInPrompt, 'result.traits should have a standardVersion trait with includeInPrompt true'); + ok(result.traits.find((trait) => trait.name === 'standardVersion')?.promptTextOverride === 'This project uses the C++20 language standard.', 'result.traits should have a standardVersion trait with promptTextOverride'); + ok(result.traits.find((trait) => trait.name === 'targetPlatform'), 'result.traits should have a targetPlatform trait'); + ok(result.traits.find((trait) => trait.name === 'targetPlatform')?.value === 'Windows', 'result.traits should have a targetPlatform trait with value "Windows"'); + ok(result.traits.find((trait) => trait.name === 'targetPlatform')?.includeInPrompt, 'result.traits should have a targetPlatform trait with includeInPrompt true'); + ok(result.traits.find((trait) => trait.name === 'targetPlatform')?.promptTextOverride === 'This build targets Windows.', 'result.traits should have a targetPlatform trait with promptTextOverride'); + ok(result.traits.find((trait) => trait.name === 'targetArchitecture'), 'result.traits should have a targetArchitecture trait'); + ok(result.traits.find((trait) => trait.name === 'targetArchitecture')?.value === 'x64', 'result.traits should have a targetArchitecture trait with value "x64"'); + ok(result.traits.find((trait) => trait.name === 'targetArchitecture')?.includeInPrompt, 'result.traits should have a targetArchitecture trait with includeInPrompt true'); + ok(result.traits.find((trait) => trait.name === 'targetArchitecture')?.promptTextOverride === 'This build targets x64.', 'result.traits should have a targetArchitecture trait with promptTextOverride'); + ok(result.traits.find((trait) => trait.name === 'intellisenseEnd'), 'result.traits should have a intellisenseEnd trait'); + ok(result.traits.find((trait) => trait.name === 'intellisenseEnd')?.value === 'End', 'result.traits should have a intellisense trait with value "End"'); + ok(result.traits.find((trait) => trait.name === 'intellisenseEnd')?.includeInPrompt, 'result.traits should have a intellisenseEnd trait with includeInPrompt true'); + ok(result.traits.find((trait) => trait.name === 'intellisenseEnd')?.promptTextOverride === 'End of IntelliSense information.', 'result.traits should have a intellisenseEnd trait with promptTextOverride'); }); - it('should exclude #cpp traits per copilotcppExcludeTraits.', async () => { - const excludeTraits = ['compiler', 'targetPlatform']; + it('should exclude project context traits per copilotcppExcludeTraits.', async () => { + const excludeTraits = ['compiler', 'targetPlatform', 'intellisense', 'intellisenseBegin', 'intellisenseEnd']; arrange({ vscodeExtension: vscodeExtension, getIncludeFiles: { includedFiles }, - chatContext: { - language: 'c++', - standardVersion: 'c++20', - compiler: 'msvc', - targetPlatform: 'windows', - targetArchitecture: 'x64' - }, + projectContext: projectContextNoArgs, rootUri, flags: { copilotcppTraits: true, copilotcppExcludeTraits: excludeTraits } }); @@ -233,8 +225,6 @@ describe('registerRelatedFilesProvider', () => { ok(getActiveClientStub.callCount !== 0, 'getActiveClient should be called'); ok(callbackPromise, 'callbackPromise should be defined'); ok(result, 'result should be defined'); - ok(result.entries.length === 1, 'result.entries should have 1 included file'); - ok(result.entries[0].toString() === expectedInclude, `result.entries should have "${expectedInclude}"`); ok(result.traits, 'result.traits should be defined'); ok(result.traits.length === 3, 'result.traits should have 3 traits'); ok(result.traits.filter(trait => excludeTraits.includes(trait.name)).length === 0, 'result.traits should not include excluded traits'); @@ -248,4 +238,64 @@ describe('registerRelatedFilesProvider', () => { ok(vscodeGetExtensionsStub.calledOnce, 'vscode.extensions.getExtension should be called once'); ok(mockCopilotApi.registerRelatedFilesProvider.notCalled, 'registerRelatedFilesProvider should not be called'); }); + + const projectContext: ProjectContext = { + language: 'C++', + standardVersion: 'C++20', + compiler: 'MSVC', + targetPlatform: 'Windows', + targetArchitecture: 'x64', + compilerArguments: ['/std:c++17', '/GR-', '/EHs-c-', '/await'] + }; + + it('should provide compiler command line traits if available.', async () => { + arrange({ + vscodeExtension: vscodeExtension, + getIncludeFiles: { includedFiles: ['c:\\system\\include\\vector', 'c:\\system\\include\\string', 'C:\\src\\my_project\\foo.h'] }, + projectContext: projectContext, + rootUri: vscode.Uri.file('C:\\src\\my_project'), + flags: { copilotcppTraits: true } + }); + await moduleUnderTest.registerRelatedFilesProvider(); + + const result = await callbackPromise; + + ok(result, 'result should be defined'); + ok(result.traits, 'result.traits should be defined'); + ok(result.traits.find((trait) => trait.name === 'compilerArguments'), 'result.traits should have a compiler arguments trait'); + ok(result.traits.find((trait) => trait.name === 'compilerArguments')?.value === '/std:c++17, /GR-, /EHs-c-, /await', 'result.traits should have a compiler arguments trait with value "/std:c++17, /GR-, /EHs-c-, /await"'); + ok(result.traits.find((trait) => trait.name === 'compilerArguments')?.includeInPrompt, 'result.traits should have a compiler arguments trait with includeInPrompt true'); + ok(result.traits.find((trait) => trait.name === 'compilerArguments')?.promptTextOverride === 'The compiler arguments include: /std:c++17, /GR-, /EHs-c-, /await.', 'result.traits should have a compiler arguments trait with promptTextOverride'); + ok(!result.traits.find((trait) => trait.name === 'directAsks'), 'result.traits should not have a direct asks trait'); + }); + + it('can map compiler arguments to direct asks.', async () => { + arrange({ + vscodeExtension: vscodeExtension, + getIncludeFiles: { includedFiles: ['c:\\system\\include\\vector', 'c:\\system\\include\\string', 'C:\\src\\my_project\\foo.h'] }, + projectContext: projectContext, + rootUri: vscode.Uri.file('C:\\src\\my_project'), + flags: { + copilotcppTraits: true, copilotcppCompilerArgumentDirectAskMap: + JSON.stringify({ + '/GR-': 'Do not generate code using RTTI keywords.', + '/EHs-c-': 'Do not generate code using exception handling keywords.' + }) + } + }); + await moduleUnderTest.registerRelatedFilesProvider(); + + const result = await callbackPromise; + + ok(result, 'result should be defined'); + ok(result.traits, 'result.traits should be defined'); + ok(result.traits.find((trait) => trait.name === 'compilerArguments'), 'result.traits should have a compiler arguments trait'); + ok(result.traits.find((trait) => trait.name === 'compilerArguments')?.value === '/std:c++17, /await', 'result.traits should have a compiler arguments trait with value "/std:c++17, /await"'); + ok(result.traits.find((trait) => trait.name === 'compilerArguments')?.includeInPrompt, 'result.traits should have a compiler arguments trait with includeInPrompt true'); + ok(result.traits.find((trait) => trait.name === 'compilerArguments')?.promptTextOverride === 'The compiler arguments include: /std:c++17, /await.', 'result.traits should have a compiler arguments trait with promptTextOverride'); + ok(result.traits.find((trait) => trait.name === 'directAsks'), 'result.traits should have a direct asks trait'); + ok(result.traits.find((trait) => trait.name === 'directAsks')?.value === 'Do not generate code using RTTI keywords. Do not generate code using exception handling keywords. ', 'result.traits should have a direct asks value'); + ok(result.traits.find((trait) => trait.name === 'directAsks')?.includeInPrompt, 'result.traits should have a direct asks trait with includeInPrompt true'); + ok(result.traits.find((trait) => trait.name === 'directAsks')?.promptTextOverride === 'Do not generate code using RTTI keywords. Do not generate code using exception handling keywords. ', 'result.traits should have a direct ask trait with promptTextOverride'); + }); }); diff --git a/Extension/test/scenarios/SingleRootProject/tests/lmTool.test.ts b/Extension/test/scenarios/SingleRootProject/tests/lmTool.test.ts new file mode 100644 index 0000000000..d0a8bc6de4 --- /dev/null +++ b/Extension/test/scenarios/SingleRootProject/tests/lmTool.test.ts @@ -0,0 +1,308 @@ +/* -------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All Rights Reserved. + * See 'LICENSE' in the project root for license information. + * ------------------------------------------------------------------------------------------ */ + +import { ok } from 'assert'; +import { afterEach, beforeEach, describe, it } from 'mocha'; +import * as sinon from 'sinon'; +import * as vscode from 'vscode'; +import * as util from '../../../../src/common'; +import { ChatContextResult, DefaultClient, ProjectContextResult } from '../../../../src/LanguageServer/client'; +import { ClientCollection } from '../../../../src/LanguageServer/clientCollection'; +import * as extension from '../../../../src/LanguageServer/extension'; +import { CppConfigurationLanguageModelTool, getProjectContext } from '../../../../src/LanguageServer/lmTool'; +import * as telemetry from '../../../../src/telemetry'; + +describe('CppConfigurationLanguageModelTool Tests', () => { + let mockLanguageModelToolInvocationOptions: sinon.SinonStubbedInstance>; + let activeClientStub: sinon.SinonStubbedInstance; + let mockTextEditorStub: MockTextEditor; + let mockTextDocumentStub: sinon.SinonStubbedInstance; + let languageModelToolTelemetryStub: sinon.SinonStub; + + class MockLanguageModelToolInvocationOptions implements vscode.LanguageModelToolInvocationOptions { + tokenizationOptions?: vscode.LanguageModelToolTokenizationOptions | undefined; + toolInvocationToken: undefined; + input: undefined; + } + class MockTextEditor implements vscode.TextEditor { + constructor(selection: vscode.Selection, selections: readonly vscode.Selection[], visibleRanges: readonly vscode.Range[], options: vscode.TextEditorOptions, document: vscode.TextDocument, viewColumn?: vscode.ViewColumn) { + this.selection = selection; + this.selections = selections; + this.visibleRanges = visibleRanges; + this.options = options; + this.viewColumn = viewColumn; + this.document = document; + } + selection: vscode.Selection; + selections: readonly vscode.Selection[]; + visibleRanges: readonly vscode.Range[]; + options: vscode.TextEditorOptions; + viewColumn: vscode.ViewColumn | undefined; + edit(_callback: (editBuilder: vscode.TextEditorEdit) => void, _options?: { readonly undoStopBefore: boolean; readonly undoStopAfter: boolean }): Thenable { + throw new Error('Method not implemented.'); + } + insertSnippet(_snippet: vscode.SnippetString, _location?: vscode.Position | vscode.Range | readonly vscode.Position[] | readonly vscode.Range[], _options?: { readonly undoStopBefore: boolean; readonly undoStopAfter: boolean }): Thenable { + throw new Error('Method not implemented.'); + } + setDecorations(_decorationType: vscode.TextEditorDecorationType, _rangesOrOptions: readonly vscode.Range[] | readonly vscode.DecorationOptions[]): void { + throw new Error('Method not implemented.'); + } + revealRange(_range: vscode.Range, _revealType?: vscode.TextEditorRevealType): void { + throw new Error('Method not implemented.'); + } + show(_column?: vscode.ViewColumn): void { + throw new Error('Method not implemented.'); + } + hide(): void { + throw new Error('Method not implemented.'); + } + document: vscode.TextDocument; + } + class MockTextDocument implements vscode.TextDocument { + uri: vscode.Uri; + constructor(uri: vscode.Uri, fileName: string, isUntitled: boolean, languageId: string, version: number, isDirty: boolean, isClosed: boolean, eol: vscode.EndOfLine, lineCount: number) { + this.uri = uri; + this.fileName = fileName; + this.isUntitled = isUntitled; + this.languageId = languageId; + this.version = version; + this.isDirty = isDirty; + this.isClosed = isClosed; + this.eol = eol; + this.lineCount = lineCount; + } + fileName: string; + isUntitled: boolean; + languageId: string; + version: number; + isDirty: boolean; + isClosed: boolean; + save(): Thenable { + throw new Error('Method not implemented.'); + } + eol: vscode.EndOfLine; + lineCount: number; + + lineAt(line: number): vscode.TextLine; + // eslint-disable-next-line @typescript-eslint/unified-signatures + lineAt(position: vscode.Position): vscode.TextLine; + lineAt(_arg: number | vscode.Position): vscode.TextLine { + throw new Error('Method not implemented.'); + } + offsetAt(_position: vscode.Position): number { + throw new Error('Method not implemented.'); + } + positionAt(_offset: number): vscode.Position { + throw new Error('Method not implemented.'); + } + getText(_range?: vscode.Range): string { + throw new Error('Method not implemented.'); + } + getWordRangeAtPosition(_position: vscode.Position, _regex?: RegExp): vscode.Range | undefined { + throw new Error('Method not implemented.'); + } + validateRange(_range: vscode.Range): vscode.Range { + throw new Error('Method not implemented.'); + } + validatePosition(_position: vscode.Position): vscode.Position { + throw new Error('Method not implemented.'); + } + } + beforeEach(() => { + sinon.stub(util, 'extensionContext').value({ extension: { id: 'test-extension-id' } }); + + mockTextDocumentStub = sinon.createStubInstance(MockTextDocument); + mockTextEditorStub = new MockTextEditor(new vscode.Selection(0, 0, 0, 0), [], [], { tabSize: 4 }, mockTextDocumentStub); + mockLanguageModelToolInvocationOptions = new MockLanguageModelToolInvocationOptions(); + activeClientStub = sinon.createStubInstance(DefaultClient); + const clientsStub = sinon.createStubInstance(ClientCollection); + sinon.stub(extension, 'getClients').returns(clientsStub); + sinon.stub(clientsStub, 'ActiveClient').get(() => activeClientStub); + activeClientStub.getIncludes.resolves({ includedFiles: [] }); + sinon.stub(vscode.window, 'activeTextEditor').get(() => mockTextEditorStub); + languageModelToolTelemetryStub = sinon.stub(telemetry, 'logLanguageModelToolEvent').returns(); + }); + + afterEach(() => { + sinon.restore(); + }); + + const arrangeChatContextFromCppTools = ({ chatContextFromCppTools, isCpp, isHeaderFile }: + { chatContextFromCppTools?: ChatContextResult; isCpp?: boolean; isHeaderFile?: boolean } = + { chatContextFromCppTools: undefined, isCpp: undefined, isHeaderFile: false } + ) => { + activeClientStub.getChatContext.resolves(chatContextFromCppTools); + sinon.stub(util, 'isCpp').returns(isCpp ?? true); + sinon.stub(util, 'isHeaderFile').returns(isHeaderFile ?? false); + }; + + const arrangeProjectContextFromCppTools = ({ projectContextFromCppTools, isCpp, isHeaderFile }: + { projectContextFromCppTools?: ProjectContextResult; isCpp?: boolean; isHeaderFile?: boolean } = + { projectContextFromCppTools: undefined, isCpp: undefined, isHeaderFile: false } + ) => { + activeClientStub.getProjectContext.resolves(projectContextFromCppTools); + sinon.stub(util, 'isCpp').returns(isCpp ?? true); + sinon.stub(util, 'isHeaderFile').returns(isHeaderFile ?? false); + }; + + it('should log telemetry and provide #cpp chat context.', async () => { + arrangeChatContextFromCppTools({ + chatContextFromCppTools: { + language: 'cpp', + standardVersion: 'c++20', + compiler: 'msvc', + targetPlatform: 'windows', + targetArchitecture: 'x64' + } + }); + + const result = await new CppConfigurationLanguageModelTool().invoke(mockLanguageModelToolInvocationOptions, new vscode.CancellationTokenSource().token); + + ok(languageModelToolTelemetryStub.calledOnce, 'logLanguageModelToolEvent should be called once'); + ok(languageModelToolTelemetryStub.calledWithMatch('Chat/Tool/cpp', sinon.match({ + "language": 'C++', + "compiler": 'MSVC', + "standardVersion": 'C++20', + "targetPlatform": 'Windows', + "targetArchitecture": 'x64' + }))); + ok(result, 'result should not be undefined'); + const text = result.content[0] as vscode.LanguageModelTextPart; + ok(text, 'result should contain a text part'); + ok(text.value === 'The user is working on a C++ project. The project uses language version C++20. The project compiles using the MSVC compiler. The project targets the Windows platform. The project targets the x64 architecture. '); + }); + + const testGetProjectContext = async ({ + compiler, + expectedCompiler, + context, + compilerArguments: compilerArguments, + expectedCompilerArguments + }: { + compiler: string; + expectedCompiler: string; + context: { flags: Record }; + compilerArguments: string[]; + expectedCompilerArguments: string[]; + }) => { + arrangeProjectContextFromCppTools({ + projectContextFromCppTools: { + language: 'cpp', + standardVersion: 'c++20', + compiler: compiler, + targetPlatform: 'windows', + targetArchitecture: 'x64', + fileContext: { + compilerArguments: compilerArguments + } + } + }); + + const result = await getProjectContext(context, new vscode.CancellationTokenSource().token); + + ok(languageModelToolTelemetryStub.calledOnce, 'logLanguageModelToolEvent should be called once'); + ok(languageModelToolTelemetryStub.calledWithMatch('Completions/tool', sinon.match({ + "language": 'C++' + }))); + if (expectedCompiler) { + ok(languageModelToolTelemetryStub.calledWithMatch('Completions/tool', sinon.match({ + "compiler": expectedCompiler + }))); + } + ok(languageModelToolTelemetryStub.calledWithMatch('Completions/tool', sinon.match({ + "standardVersion": 'C++20' + }))); + ok(languageModelToolTelemetryStub.calledWithMatch('Completions/tool', sinon.match({ + "targetPlatform": 'Windows' + }))); + ok(languageModelToolTelemetryStub.calledWithMatch('Completions/tool', sinon.match({ + "targetArchitecture": 'x64' + }))); + ok(languageModelToolTelemetryStub.calledWithMatch('Completions/tool', sinon.match({ + "compilerArgumentCount": compilerArguments.length.toString() + }))); + if (expectedCompilerArguments.length > 0) { + ok(languageModelToolTelemetryStub.calledWithMatch('Completions/tool', sinon.match({ + "filteredCompilerArguments": expectedCompilerArguments.join(', ') + }))); + } + ok(languageModelToolTelemetryStub.calledWithMatch('Completions/tool', sinon.match({ + 'time': sinon.match.string + }))); + ok(result, 'result should not be undefined'); + ok(result.language === 'C++'); + ok(result.compiler === expectedCompiler); + ok(result.standardVersion === 'C++20'); + ok(result.targetPlatform === 'Windows'); + ok(result.targetArchitecture === 'x64'); + ok(JSON.stringify(result.compilerArguments) === JSON.stringify(expectedCompilerArguments)); + }; + + it('should log telemetry and provide cpp context properly when experimental flags are not defined.', async () => { + await testGetProjectContext({ + compiler: 'gcc', + expectedCompiler: 'GCC', + context: { flags: {} }, + compilerArguments: ['-Wall', '-Werror', '-std=c++20'], + expectedCompilerArguments: [] + }); + }); + + it('should provide compilerArguments based on copilotcppGccCompilerArgumentFilter.', async () => { + await testGetProjectContext({ + compiler: 'gcc', + expectedCompiler: 'GCC', + context: { flags: { copilotcppGccCompilerArgumentFilter: '^-(fno\-exceptions|fno\-rtti)$' } }, + compilerArguments: ['-Wall', '-Werror', '-std=c++20', '-fno-exceptions', '-fno-rtti', '-pthread', '-O3', '-funroll-loops'], + expectedCompilerArguments: ['-fno-exceptions', '-fno-rtti'] + }); + }); + + it('should filter out all compilerArguments for unkonwn compilers.', async () => { + await testGetProjectContext({ + compiler: 'unknown', + expectedCompiler: '', + context: { + flags: { + copilotcppMsvcCompilerArgumentFilter: '^-(fno\-exceptions|fno\-rtti)$', + copilotcppClangCompilerArgumentFilter: '^-(fno\-exceptions|fno\-rtti)$', + copilotcppGccCompilerArgumentFilter: '^-(fno\-exceptions|fno\-rtti)$' + } + }, + compilerArguments: ['-fno-exceptions', '-fno-rtti'], + expectedCompilerArguments: [] + }); + }); + + it('should not log telemetry for unknown values', async () => { + arrangeProjectContextFromCppTools({ + projectContextFromCppTools: { + language: 'java', + standardVersion: 'foo', + compiler: 'javac', + targetPlatform: 'arduino', + targetArchitecture: 'bar', + fileContext: { + compilerArguments: [] + } + } + }); + + const result = await getProjectContext({ flags: {} }, new vscode.CancellationTokenSource().token); + + ok(languageModelToolTelemetryStub.calledOnce, 'logLanguageModelToolEvent should be called once'); + ok(languageModelToolTelemetryStub.calledWithMatch('Completions/tool', sinon.match({ + "compilerArgumentCount": '0', + "targetArchitecture": 'bar' + }))); + ok(result, 'result should not be undefined'); + ok(result.language === ''); + ok(result.compiler === ''); + ok(result.standardVersion === ''); + ok(result.targetPlatform === ''); + ok(result.targetArchitecture === 'bar'); + ok(result.compilerArguments.length === 0); + }); +}); From 2672a8388ac1dbab9de00991ee5804c5b6eb44c0 Mon Sep 17 00:00:00 2001 From: Glen Chung Date: Tue, 19 Nov 2024 15:43:01 -0800 Subject: [PATCH 2/2] Support User Defines and Macro References AB Experiment - Depends on https://github.com/microsoft/vscode-cpptools/pull/12979 to merge first. - copilotcppMacroReferenceFilter: regex string to filter macro reference for telemetry. - Telemetry related to user defines and macro references. - Added new trait compilerUserDefines to note the relevant user defines to the editing file. --- Extension/src/LanguageServer/client.ts | 2 + .../src/LanguageServer/copilotProviders.ts | 4 + Extension/src/LanguageServer/lmTool.ts | 33 +++++++- .../tests/copilotProviders.test.ts | 10 ++- .../SingleRootProject/tests/lmTool.test.ts | 84 +++++++++++++++++-- 5 files changed, 123 insertions(+), 10 deletions(-) diff --git a/Extension/src/LanguageServer/client.ts b/Extension/src/LanguageServer/client.ts index c05034e643..ee19fdfed6 100644 --- a/Extension/src/LanguageServer/client.ts +++ b/Extension/src/LanguageServer/client.ts @@ -543,6 +543,8 @@ export interface ChatContextResult { export interface FileContextResult { compilerArguments: string[]; + compilerUserDefines: string[]; + macroReferences: string[]; } export interface ProjectContextResult { diff --git a/Extension/src/LanguageServer/copilotProviders.ts b/Extension/src/LanguageServer/copilotProviders.ts index c51f2b88f7..9943e9fb3e 100644 --- a/Extension/src/LanguageServer/copilotProviders.ts +++ b/Extension/src/LanguageServer/copilotProviders.ts @@ -79,6 +79,10 @@ export async function registerRelatedFilesProvider(): Promise { const compilerArgumentsValue = updatedArguments.join(", "); traits.push({ name: "compilerArguments", value: compilerArgumentsValue, includeInPrompt: true, promptTextOverride: `The compiler arguments include: ${compilerArgumentsValue}.` }); } + if (cppContext.compilerUserDefinesRelevant.length > 0) { + const compilerUserDefinesValue = cppContext.compilerUserDefinesRelevant.join(", "); + traits.push({ name: "compilerUserDefines", value: compilerUserDefinesValue, includeInPrompt: true, promptTextOverride: `These compiler command line user defines may be relevent: ${compilerUserDefinesValue}.` }); + } if (directAsks) { traits.push({ name: "directAsks", value: directAsks, includeInPrompt: true, promptTextOverride: directAsks }); } diff --git a/Extension/src/LanguageServer/lmTool.ts b/Extension/src/LanguageServer/lmTool.ts index 82c54d3135..f6acf44033 100644 --- a/Extension/src/LanguageServer/lmTool.ts +++ b/Extension/src/LanguageServer/lmTool.ts @@ -66,6 +66,7 @@ export interface ProjectContext { targetPlatform: string; targetArchitecture: string; compilerArguments: string[]; + compilerUserDefinesRelevant: string[]; } // Set these values for local testing purpose without involving control tower. @@ -93,6 +94,15 @@ function filterComplierArguments(compiler: string, compilerArguments: string[], return compilerArguments.filter(arg => defaultFilter?.test(arg) || additionalFilter?.test(arg)); } +// We can set up copilotcppMacroReferenceFilter feature flag to filter macro references to learn about +// macro usage distribution, i.e., compiler or platform specific macros, or even the presence of certain macros. +const defaultMacroReferenceFilter: RegExp | undefined = undefined; +function filterMacroReferences(macroReferences: string[], context: { flags: Record }): string[] { + const filter: RegExp | undefined = context.flags.copilotcppMacroReferenceFilter ? new RegExp(context.flags.copilotcppMacroReferenceFilter as string) : undefined; + + return macroReferences.filter(macro => defaultMacroReferenceFilter?.test(macro) || filter?.test(macro)); +} + export async function getProjectContext(context: { flags: Record }, token: vscode.CancellationToken): Promise { const telemetryProperties: Record = {}; try { @@ -110,7 +120,8 @@ export async function getProjectContext(context: { flags: Record 0) { @@ -137,6 +149,23 @@ export async function getProjectContext(context: { flags: Record 0) { + const userDefinesWithoutValue = projectContext.result.fileContext.compilerUserDefines.map(value => value.split('=')[0]); + const userDefinesRelatedToThisFile = userDefinesWithoutValue.filter(value => projectContext.result?.fileContext.macroReferences.includes(value)); + if (userDefinesRelatedToThisFile.length > 0) { + // Don't care the actual name of the user define, just the count that's relevant. + telemetryProperties["compilerUserDefinesRelevantCount"] = userDefinesRelatedToThisFile.length.toString(); + result.compilerUserDefinesRelevant = userDefinesRelatedToThisFile; + } + } + telemetryProperties["macroReferenceCount"] = projectContext.result.fileContext.macroReferences.length.toString(); + if (projectContext.result.fileContext.macroReferences.length > 0) { + const filteredMacroReferences = filterMacroReferences(projectContext.result.fileContext.macroReferences, context); + if (filteredMacroReferences.length > 0) { + telemetryProperties["filteredMacroReferences"] = filteredMacroReferences.join(', '); + } + } return result; } diff --git a/Extension/test/scenarios/SingleRootProject/tests/copilotProviders.test.ts b/Extension/test/scenarios/SingleRootProject/tests/copilotProviders.test.ts index f1c1e8908b..a8b48a90e6 100644 --- a/Extension/test/scenarios/SingleRootProject/tests/copilotProviders.test.ts +++ b/Extension/test/scenarios/SingleRootProject/tests/copilotProviders.test.ts @@ -138,7 +138,8 @@ describe('copilotProviders Tests', () => { compiler: 'MSVC', targetPlatform: 'Windows', targetArchitecture: 'x64', - compilerArguments: [] + compilerArguments: [], + compilerUserDefinesRelevant: [] }; it('should not provide project context traits when copilotcppTraits flag is false.', async () => { @@ -245,7 +246,8 @@ describe('copilotProviders Tests', () => { compiler: 'MSVC', targetPlatform: 'Windows', targetArchitecture: 'x64', - compilerArguments: ['/std:c++17', '/GR-', '/EHs-c-', '/await'] + compilerArguments: ['/std:c++17', '/GR-', '/EHs-c-', '/await'], + compilerUserDefinesRelevant: ['MY_FOO', 'MY_BAR'] }; it('should provide compiler command line traits if available.', async () => { @@ -266,6 +268,10 @@ describe('copilotProviders Tests', () => { ok(result.traits.find((trait) => trait.name === 'compilerArguments')?.value === '/std:c++17, /GR-, /EHs-c-, /await', 'result.traits should have a compiler arguments trait with value "/std:c++17, /GR-, /EHs-c-, /await"'); ok(result.traits.find((trait) => trait.name === 'compilerArguments')?.includeInPrompt, 'result.traits should have a compiler arguments trait with includeInPrompt true'); ok(result.traits.find((trait) => trait.name === 'compilerArguments')?.promptTextOverride === 'The compiler arguments include: /std:c++17, /GR-, /EHs-c-, /await.', 'result.traits should have a compiler arguments trait with promptTextOverride'); + ok(result.traits.find((trait) => trait.name === 'compilerUserDefines'), 'result.traits should have a compiler user defines trait'); + ok(result.traits.find((trait) => trait.name === 'compilerUserDefines')?.value === 'MY_FOO, MY_BAR', 'result.traits should have a compiler user defines trait with value "MY_FOO, MY_BAR"'); + ok(result.traits.find((trait) => trait.name === 'compilerUserDefines')?.includeInPrompt, 'result.traits should have a compiler user defines trait with includeInPrompt true'); + ok(result.traits.find((trait) => trait.name === 'compilerUserDefines')?.promptTextOverride === 'These compiler command line user defines may be relevent: MY_FOO, MY_BAR.', 'result.traits should have a compiler args trait with promptTextOverride'); ok(!result.traits.find((trait) => trait.name === 'directAsks'), 'result.traits should not have a direct asks trait'); }); diff --git a/Extension/test/scenarios/SingleRootProject/tests/lmTool.test.ts b/Extension/test/scenarios/SingleRootProject/tests/lmTool.test.ts index d0a8bc6de4..015d21ea96 100644 --- a/Extension/test/scenarios/SingleRootProject/tests/lmTool.test.ts +++ b/Extension/test/scenarios/SingleRootProject/tests/lmTool.test.ts @@ -179,13 +179,21 @@ describe('CppConfigurationLanguageModelTool Tests', () => { expectedCompiler, context, compilerArguments: compilerArguments, - expectedCompilerArguments + expectedCompilerArguments, + compilerUserDefines, + expectedCompilerUserDefines, + macroReferences, + expectedMacroReferences }: { compiler: string; expectedCompiler: string; context: { flags: Record }; compilerArguments: string[]; expectedCompilerArguments: string[]; + compilerUserDefines: string[]; + expectedCompilerUserDefines: string[]; + macroReferences: string[]; + expectedMacroReferences: string[]; }) => { arrangeProjectContextFromCppTools({ projectContextFromCppTools: { @@ -195,7 +203,9 @@ describe('CppConfigurationLanguageModelTool Tests', () => { targetPlatform: 'windows', targetArchitecture: 'x64', fileContext: { - compilerArguments: compilerArguments + compilerArguments: compilerArguments, + compilerUserDefines: compilerUserDefines, + macroReferences: macroReferences } } }); @@ -228,6 +238,22 @@ describe('CppConfigurationLanguageModelTool Tests', () => { "filteredCompilerArguments": expectedCompilerArguments.join(', ') }))); } + ok(languageModelToolTelemetryStub.calledWithMatch('Completions/tool', sinon.match({ + "compilerUserDefinesCount": compilerUserDefines.length.toString() + }))); + if (expectedCompilerUserDefines.length > 0) { + ok(languageModelToolTelemetryStub.calledWithMatch('Completions/tool', sinon.match({ + "compilerUserDefinesRelevantCount": expectedCompilerUserDefines.length.toString() + }))); + } + ok(languageModelToolTelemetryStub.calledWithMatch('Completions/tool', sinon.match({ + 'macroReferenceCount': macroReferences.length.toString() + }))); + if (expectedMacroReferences.length > 0) { + ok(languageModelToolTelemetryStub.calledWithMatch('Completions/tool', sinon.match({ + 'filteredMacroReferences': expectedMacroReferences.join(', ') + }))); + } ok(languageModelToolTelemetryStub.calledWithMatch('Completions/tool', sinon.match({ 'time': sinon.match.string }))); @@ -238,6 +264,7 @@ describe('CppConfigurationLanguageModelTool Tests', () => { ok(result.targetPlatform === 'Windows'); ok(result.targetArchitecture === 'x64'); ok(JSON.stringify(result.compilerArguments) === JSON.stringify(expectedCompilerArguments)); + ok(JSON.stringify(result.compilerUserDefinesRelevant) === JSON.stringify(expectedCompilerUserDefines)); }; it('should log telemetry and provide cpp context properly when experimental flags are not defined.', async () => { @@ -246,7 +273,11 @@ describe('CppConfigurationLanguageModelTool Tests', () => { expectedCompiler: 'GCC', context: { flags: {} }, compilerArguments: ['-Wall', '-Werror', '-std=c++20'], - expectedCompilerArguments: [] + expectedCompilerArguments: [], + compilerUserDefines: [], + expectedCompilerUserDefines: [], + macroReferences: [], + expectedMacroReferences: [] }); }); @@ -256,7 +287,39 @@ describe('CppConfigurationLanguageModelTool Tests', () => { expectedCompiler: 'GCC', context: { flags: { copilotcppGccCompilerArgumentFilter: '^-(fno\-exceptions|fno\-rtti)$' } }, compilerArguments: ['-Wall', '-Werror', '-std=c++20', '-fno-exceptions', '-fno-rtti', '-pthread', '-O3', '-funroll-loops'], - expectedCompilerArguments: ['-fno-exceptions', '-fno-rtti'] + expectedCompilerArguments: ['-fno-exceptions', '-fno-rtti'], + compilerUserDefines: [], + expectedCompilerUserDefines: [], + macroReferences: [], + expectedMacroReferences: [] + }); + }); + + it('should honor content-exclusion and provide user defines based on macro references.', async () => { + await testGetProjectContext({ + compiler: 'gcc', + expectedCompiler: 'GCC', + context: { flags: {} }, + compilerArguments: [], + expectedCompilerArguments: [], + compilerUserDefines: ['FOO', 'BAR', 'XYZ'], + expectedCompilerUserDefines: ['FOO', 'XYZ'], + macroReferences: ['FOO', 'XYZ', 'BAZ'], + expectedMacroReferences: [] + }); + }); + + it('should log macro reference telemetry based on copilotcppMacroReferenceFilter.', async () => { + await testGetProjectContext({ + compiler: 'gcc', + expectedCompiler: 'GCC', + context: { flags: { copilotcppMacroReferenceFilter: '^(FOO|XYZ)$' } }, + compilerArguments: [], + expectedCompilerArguments: [], + compilerUserDefines: [], + expectedCompilerUserDefines: [], + macroReferences: ['FOO', 'XYZ', 'BAZ'], + expectedMacroReferences: ['FOO', 'XYZ'] }); }); @@ -272,7 +335,11 @@ describe('CppConfigurationLanguageModelTool Tests', () => { } }, compilerArguments: ['-fno-exceptions', '-fno-rtti'], - expectedCompilerArguments: [] + expectedCompilerArguments: [], + compilerUserDefines: [], + expectedCompilerUserDefines: [], + macroReferences: [], + expectedMacroReferences: [] }); }); @@ -285,7 +352,9 @@ describe('CppConfigurationLanguageModelTool Tests', () => { targetPlatform: 'arduino', targetArchitecture: 'bar', fileContext: { - compilerArguments: [] + compilerArguments: [], + compilerUserDefines: [], + macroReferences: [] } } }); @@ -295,6 +364,8 @@ describe('CppConfigurationLanguageModelTool Tests', () => { ok(languageModelToolTelemetryStub.calledOnce, 'logLanguageModelToolEvent should be called once'); ok(languageModelToolTelemetryStub.calledWithMatch('Completions/tool', sinon.match({ "compilerArgumentCount": '0', + "compilerUserDefinesCount": '0', + "macroReferenceCount": '0', "targetArchitecture": 'bar' }))); ok(result, 'result should not be undefined'); @@ -304,5 +375,6 @@ describe('CppConfigurationLanguageModelTool Tests', () => { ok(result.targetPlatform === ''); ok(result.targetArchitecture === 'bar'); ok(result.compilerArguments.length === 0); + ok(result.compilerUserDefinesRelevant.length === 0); }); });