From e6721bd8781a8681f589cff323834d55f5c6eb52 Mon Sep 17 00:00:00 2001 From: Glen Chung Date: Wed, 16 Oct 2024 17:23:16 -0700 Subject: [PATCH] Support A/B Compiler Arguments Traits - User defines are not covered in this PR. - Depends on cpptools' update. - A/B Experimental flags - copilotcppTraits: boolean flag to enable cpp traits - copilotcppExcludeTraits: string array to exclude individual trait, i.e., compilerArguments. - copilotcppMsvcCompilerArgumentFilter: regex string to match compiler arguments for GCC. - copilotcppClangCompilerArgumentFilter: regex string to match compiler arguments for Clang. - copilotcppGccCompilerArgumentFilter: regex string to match compiler arguments for MSVC. --- Extension/src/LanguageServer/client.ts | 2 + .../src/LanguageServer/copilotProviders.ts | 20 +- Extension/src/LanguageServer/lmTool.ts | 103 +++++++- Extension/src/telemetry.ts | 2 +- .../tests/copilotProviders.test.ts | 132 +++++----- .../SingleRootProject/tests/lmTool.test.ts | 239 ++++++++++++++++++ 6 files changed, 420 insertions(+), 78 deletions(-) create mode 100644 Extension/test/scenarios/SingleRootProject/tests/lmTool.test.ts diff --git a/Extension/src/LanguageServer/client.ts b/Extension/src/LanguageServer/client.ts index e46743815e..cb1e45f9ee 100644 --- a/Extension/src/LanguageServer/client.ts +++ b/Extension/src/LanguageServer/client.ts @@ -539,6 +539,8 @@ export interface ChatContextResult { compiler: string; targetPlatform: string; targetArchitecture: string; + compilerArgs: string[]; + compilerUserDefines: string[]; } // Requests diff --git a/Extension/src/LanguageServer/copilotProviders.ts b/Extension/src/LanguageServer/copilotProviders.ts index f23554f76d..20eb410f4e 100644 --- a/Extension/src/LanguageServer/copilotProviders.ts +++ b/Extension/src/LanguageServer/copilotProviders.ts @@ -6,8 +6,9 @@ import * as vscode from 'vscode'; import * as util from '../common'; -import { ChatContextResult, GetIncludesResult } from './client'; +import { GetIncludesResult } from './client'; import { getActiveClient } from './extension'; +import { getCppContext } from './lmTool'; export interface CopilotTrait { name: string; @@ -38,19 +39,22 @@ export async function registerRelatedFilesProvider(): Promise { const getIncludesHandler = async () => (await getIncludesWithCancellation(1, token))?.includedFiles.map(file => vscode.Uri.file(file)) ?? []; const getTraitsHandler = async () => { - const chatContext: ChatContextResult | undefined = await (getActiveClient().getChatContext(token) ?? undefined); + const cppContext = await getCppContext(context, token); - if (!chatContext) { + if (!cppContext) { return undefined; } let traits: CopilotTrait[] = [ - { name: "language", value: chatContext.language, includeInPrompt: true, promptTextOverride: `The language is ${chatContext.language}.` }, - { name: "compiler", value: chatContext.compiler, includeInPrompt: true, promptTextOverride: `This project compiles using ${chatContext.compiler}.` }, - { name: "standardVersion", value: chatContext.standardVersion, includeInPrompt: true, promptTextOverride: `This project uses the ${chatContext.standardVersion} language standard.` }, - { name: "targetPlatform", value: chatContext.targetPlatform, includeInPrompt: true, promptTextOverride: `This build targets ${chatContext.targetPlatform}.` }, - { name: "targetArchitecture", value: chatContext.targetArchitecture, includeInPrompt: true, promptTextOverride: `This build targets ${chatContext.targetArchitecture}.` } + { name: "language", value: cppContext.language, includeInPrompt: true, promptTextOverride: `The language is ${cppContext.language}.` }, + { name: "compiler", value: cppContext.compiler, includeInPrompt: true, promptTextOverride: `This project compiles using ${cppContext.compiler}.` }, + { name: "standardVersion", value: cppContext.standardVersion, includeInPrompt: true, promptTextOverride: `This project uses the ${cppContext.standardVersion} language standard.` }, + { name: "targetPlatform", value: cppContext.targetPlatform, includeInPrompt: true, promptTextOverride: `This build targets ${cppContext.targetPlatform}.` }, + { name: "targetArchitecture", value: cppContext.targetArchitecture, includeInPrompt: true, promptTextOverride: `This build targets ${cppContext.targetArchitecture}.` } ]; + if (cppContext.compilerArguments.length > 0) { + traits.push({ name: "compilerArguments", value: cppContext.compilerArguments, includeInPrompt: true, promptTextOverride: `The compiler command line arguments may contain: ${cppContext.compilerArguments}.` }); + } const excludeTraits = context.flags.copilotcppExcludeTraits as string[] ?? []; traits = traits.filter(trait => !excludeTraits.includes(trait.name)); diff --git a/Extension/src/LanguageServer/lmTool.ts b/Extension/src/LanguageServer/lmTool.ts index ed5be61a00..9104e81504 100644 --- a/Extension/src/LanguageServer/lmTool.ts +++ b/Extension/src/LanguageServer/lmTool.ts @@ -12,6 +12,9 @@ import * as telemetry from '../telemetry'; import { ChatContextResult } from './client'; import { getClients } from './extension'; +const MSVC: string = 'MSVC'; +const Clang: string = 'Clang'; +const GCC: string = 'GCC'; const knownValues: { [Property in keyof ChatContextResult]?: { [id: string]: string } } = { language: { 'c': 'C', @@ -19,9 +22,9 @@ const knownValues: { [Property in keyof ChatContextResult]?: { [id: string]: str 'cuda-cpp': 'CUDA C++' }, compiler: { - 'msvc': 'MSVC', - 'clang': 'Clang', - 'gcc': 'GCC' + 'msvc': MSVC, + 'clang': Clang, + 'gcc': GCC }, standardVersion: { 'c++98': 'C++98', @@ -44,6 +47,91 @@ const knownValues: { [Property in keyof ChatContextResult]?: { [id: string]: str } }; +function formatChatContext(chatContext: ChatContextResult): void { + type KnownKeys = 'language' | 'standardVersion' | 'compiler' | 'targetPlatform' | 'targetArchitecture'; + for (const key in knownValues) { + const knownKey = key as KnownKeys; + if (knownKey && knownValues[knownKey] && chatContext[knownKey]) { + chatContext[knownKey] = knownValues[knownKey][chatContext[knownKey] as string] || chatContext[knownKey]; + } + } +} + +export interface CppContext { + language: string; + standardVersion: string; + compiler: string; + targetPlatform: string; + targetArchitecture: string; + compilerArguments: string; +} + +// To be updated after A/B experiments. +const defaultCompilerArgumentFilters: { [id: string]: string } = { + MSVC: '', + Clang: '', + GCC: '' +}; + +function filterComplierArguments(compiler: string, compilerArguments: string[], context: { flags: Record }): string[] { + let defaultFilter: RegExp | undefined; + let additionalFilter: RegExp | undefined; + switch (compiler) { + case MSVC: + defaultFilter = new RegExp(defaultCompilerArgumentFilters[compiler]); + additionalFilter = context.flags.copilotcppMsvcCompilerArgumentFilter ? new RegExp(context.flags.copilotcppMsvcCompilerArgumentFilter as string) : undefined; + break; + case Clang: + defaultFilter = new RegExp(defaultCompilerArgumentFilters[compiler]); + additionalFilter = context.flags.copilotcppClangCompilerArgumentFilter ? new RegExp(context.flags.copilotcppClangCompilerArgumentFilter as string) : undefined; + break; + case GCC: + defaultFilter = new RegExp(defaultCompilerArgumentFilters[compiler]); + additionalFilter = context.flags.copilotcppGccCompilerArgumentFilter ? new RegExp(context.flags.copilotcppGccCompilerArgumentFilter as string) : undefined; + break; + } + + const enableDefaultFilter = context.flags.copilotcppEnableDefaultcompilerArguments as boolean ?? false; + + if (!enableDefaultFilter && additionalFilter === undefined) { + return []; + } + + return compilerArguments.filter(arg => (enableDefaultFilter && defaultFilter?.test(arg)) || additionalFilter?.test(arg)); +} + +export async function getCppContext(context: { flags: Record }, token: vscode.CancellationToken): Promise { + const chatContext: ChatContextResult | undefined = await (getClients()?.ActiveClient?.getChatContext(token) ?? undefined); + if (!chatContext) { + return undefined; + } + + formatChatContext(chatContext); + + const filteredcompilerArguments = filterComplierArguments(chatContext.compiler, chatContext.compilerArgs, context); + + telemetry.logLanguageModelToolEvent( + 'Completions/tool', + { + "language": chatContext.language, + "compiler": chatContext.compiler, + "standardVersion": chatContext.standardVersion, + "targetPlatform": chatContext.targetPlatform, + "targetArchitecture": chatContext.targetArchitecture, + "compilerArgumentCount": chatContext.compilerArgs.length.toString(), + "filteredCompilerArgumentCount": filteredcompilerArguments.length.toString() + }); + + return { + language: chatContext.language, + standardVersion: chatContext.standardVersion, + compiler: chatContext.compiler, + targetPlatform: chatContext.targetPlatform, + targetArchitecture: chatContext.targetArchitecture, + compilerArguments: (filteredcompilerArguments.length > 0) ? filteredcompilerArguments.join(' ') : '' + }; +} + export class CppConfigurationLanguageModelTool implements vscode.LanguageModelTool { public async invoke(options: vscode.LanguageModelToolInvocationOptions, token: vscode.CancellationToken): Promise { return new vscode.LanguageModelToolResult([ @@ -63,7 +151,7 @@ export class CppConfigurationLanguageModelTool implements vscode.LanguageModelTo } telemetry.logLanguageModelToolEvent( - 'cpp', + 'Chat/Tool/cpp', { "language": chatContext.language, "compiler": chatContext.compiler, @@ -72,12 +160,7 @@ export class CppConfigurationLanguageModelTool implements vscode.LanguageModelTo "targetArchitecture": chatContext.targetArchitecture }); - for (const key in knownValues) { - const knownKey = key as keyof ChatContextResult; - if (knownValues[knownKey] && chatContext[knownKey]) { - chatContext[knownKey] = knownValues[knownKey][chatContext[knownKey]] || chatContext[knownKey]; - } - } + formatChatContext(chatContext); return `The user is working on a ${chatContext.language} project. The project uses language version ${chatContext.standardVersion}, compiles using the ${chatContext.compiler} compiler, targets the ${chatContext.targetPlatform} platform, and targets the ${chatContext.targetArchitecture} architecture.`; } diff --git a/Extension/src/telemetry.ts b/Extension/src/telemetry.ts index 600ffa4c45..434a222854 100644 --- a/Extension/src/telemetry.ts +++ b/Extension/src/telemetry.ts @@ -126,7 +126,7 @@ export function logLanguageServerEvent(eventName: string, properties?: Record, metrics?: Record): void { const sendTelemetry = () => { if (experimentationTelemetry) { - const eventNamePrefix: string = "C_Cpp/Copilot/Chat/Tool/"; + const eventNamePrefix: string = "C_Cpp/Copilot/"; experimentationTelemetry.sendTelemetryEvent(eventNamePrefix + eventName, properties, metrics); } }; diff --git a/Extension/test/scenarios/SingleRootProject/tests/copilotProviders.test.ts b/Extension/test/scenarios/SingleRootProject/tests/copilotProviders.test.ts index 54052e122d..c90976a3e7 100644 --- a/Extension/test/scenarios/SingleRootProject/tests/copilotProviders.test.ts +++ b/Extension/test/scenarios/SingleRootProject/tests/copilotProviders.test.ts @@ -9,11 +9,13 @@ import * as proxyquire from 'proxyquire'; import * as sinon from 'sinon'; import * as vscode from 'vscode'; import * as util from '../../../../src/common'; -import { ChatContextResult, DefaultClient, GetIncludesResult } from '../../../../src/LanguageServer/client'; +import { DefaultClient, GetIncludesResult } from '../../../../src/LanguageServer/client'; import { CopilotApi, CopilotTrait } from '../../../../src/LanguageServer/copilotProviders'; import * as extension from '../../../../src/LanguageServer/extension'; +import * as lmTool from '../../../../src/LanguageServer/lmTool'; +import { CppContext } from '../../../../src/LanguageServer/lmTool'; -describe('registerRelatedFilesProvider', () => { +describe('copilotProviders Tests', () => { let moduleUnderTest: any; let mockCopilotApi: sinon.SinonStubbedInstance; let getActiveClientStub: sinon.SinonStub; @@ -74,12 +76,12 @@ describe('registerRelatedFilesProvider', () => { sinon.restore(); }); - const arrange = ({ vscodeExtension, getIncludeFiles, chatContext, rootUri, flags }: - { vscodeExtension?: vscode.Extension; getIncludeFiles?: GetIncludesResult; chatContext?: ChatContextResult; rootUri?: vscode.Uri; flags?: Record } = - { vscodeExtension: undefined, getIncludeFiles: undefined, chatContext: undefined, rootUri: undefined, flags: {} } + const arrange = ({ vscodeExtension, getIncludeFiles, cppContext, rootUri, flags }: + { vscodeExtension?: vscode.Extension; getIncludeFiles?: GetIncludesResult; cppContext?: CppContext; rootUri?: vscode.Uri; flags?: Record } = + { vscodeExtension: undefined, getIncludeFiles: undefined, cppContext: undefined, rootUri: undefined, flags: {} } ) => { activeClientStub.getIncludes.resolves(getIncludeFiles); - activeClientStub.getChatContext.resolves(chatContext); + sinon.stub(lmTool, 'getCppContext').resolves(cppContext); sinon.stub(activeClientStub, 'RootUri').get(() => rootUri); mockCopilotApi.registerRelatedFilesProvider.callsFake((_providerId: { extensionId: string; languageId: string }, callback: (uri: vscode.Uri, context: { flags: Record }, cancellationToken: vscode.CancellationToken) => Promise<{ entries: vscode.Uri[]; traits?: CopilotTrait[] }>) => { const tokenSource = new vscode.CancellationTokenSource(); @@ -108,11 +110,11 @@ describe('registerRelatedFilesProvider', () => { ok(mockCopilotApi.registerRelatedFilesProvider.calledWithMatch(sinon.match({ extensionId: 'test-extension-id', languageId: sinon.match.in(['c', 'cpp', 'cuda-cpp']) })), 'registerRelatedFilesProvider should be called with the correct providerId and languageId'); }); - it('should not add #cpp traits when ChatContext isn\'t available.', async () => { + it('should not provide cpp context traits when ChatContext isn\'t available.', async () => { arrange({ vscodeExtension: vscodeExtension, getIncludeFiles: { includedFiles }, - chatContext: undefined, + cppContext: undefined, rootUri, flags: { copilotcppTraits: true } }); @@ -130,17 +132,20 @@ describe('registerRelatedFilesProvider', () => { ok(result.traits === undefined, 'result.traits should be undefined'); }); - it('should not add #cpp traits when copilotcppTraits flag is false.', async () => { + const cppContextNoArgs: CppContext = { + language: 'C++', + standardVersion: 'C++20', + compiler: 'MSVC', + targetPlatform: 'Windows', + targetArchitecture: 'x64', + compilerArguments: '' + }; + + it('should not provide cpp context traits when copilotcppTraits flag is false.', async () => { arrange({ vscodeExtension: vscodeExtension, getIncludeFiles: { includedFiles }, - chatContext: { - language: 'c++', - standardVersion: 'c++20', - compiler: 'msvc', - targetPlatform: 'windows', - targetArchitecture: 'x64' - }, + cppContext: cppContextNoArgs, rootUri, flags: { copilotcppTraits: false } }); @@ -158,17 +163,11 @@ describe('registerRelatedFilesProvider', () => { ok(result.traits === undefined, 'result.traits should be undefined'); }); - it('should add #cpp traits when copilotcppTraits flag is true.', async () => { + it('should provide cpp context traits when copilotcppTraits flag is true.', async () => { arrange({ vscodeExtension: vscodeExtension, getIncludeFiles: { includedFiles }, - chatContext: { - language: 'c++', - standardVersion: 'c++20', - compiler: 'msvc', - targetPlatform: 'windows', - targetArchitecture: 'x64' - }, + cppContext: cppContextNoArgs, rootUri, flags: { copilotcppTraits: true } }); @@ -176,50 +175,65 @@ describe('registerRelatedFilesProvider', () => { const result = await callbackPromise; - ok(vscodeGetExtensionsStub.calledOnce, 'vscode.extensions.getExtension should be called once'); - ok(mockCopilotApi.registerRelatedFilesProvider.calledThrice, 'registerRelatedFilesProvider should be called three times'); - ok(mockCopilotApi.registerRelatedFilesProvider.calledWithMatch(sinon.match({ extensionId: 'test-extension-id', languageId: sinon.match.in(['c', 'cpp', 'cuda-cpp']) })), 'registerRelatedFilesProvider should be called with the correct providerId and languageId'); - ok(getActiveClientStub.callCount !== 0, 'getActiveClient should be called'); - ok(callbackPromise, 'callbackPromise should be defined'); ok(result, 'result should be defined'); - ok(result.entries.length === 1, 'result.entries should have 1 included file'); - ok(result.entries[0].toString() === expectedInclude, `result.entries should have "${expectedInclude}"`); ok(result.traits, 'result.traits should be defined'); ok(result.traits.length === 5, 'result.traits should have 5 traits'); - ok(result.traits[0].name === 'language', 'result.traits[0].name should be "language"'); - ok(result.traits[0].value === 'c++', 'result.traits[0].value should be "c++"'); - ok(result.traits[0].includeInPrompt, 'result.traits[0].includeInPrompt should be true'); - ok(result.traits[0].promptTextOverride === 'The language is c++.', 'result.traits[0].promptTextOverride should be "The language is c++."'); - ok(result.traits[1].name === 'compiler', 'result.traits[1].name should be "compiler"'); - ok(result.traits[1].value === 'msvc', 'result.traits[1].value should be "msvc"'); - ok(result.traits[1].includeInPrompt, 'result.traits[1].includeInPrompt should be true'); - ok(result.traits[1].promptTextOverride === 'This project compiles using msvc.', 'result.traits[1].promptTextOverride should be "This project compiles using msvc."'); - ok(result.traits[2].name === 'standardVersion', 'result.traits[2].name should be "standardVersion"'); - ok(result.traits[2].value === 'c++20', 'result.traits[2].value should be "c++20"'); - ok(result.traits[2].includeInPrompt, 'result.traits[2].includeInPrompt should be true'); - ok(result.traits[2].promptTextOverride === 'This project uses the c++20 language standard.', 'result.traits[2].promptTextOverride should be "This project uses the c++20 language standard."'); - ok(result.traits[3].name === 'targetPlatform', 'result.traits[3].name should be "targetPlatform"'); - ok(result.traits[3].value === 'windows', 'result.traits[3].value should be "windows"'); - ok(result.traits[3].includeInPrompt, 'result.traits[3].includeInPrompt should be true'); - ok(result.traits[3].promptTextOverride === 'This build targets windows.', 'result.traits[3].promptTextOverride should be "This build targets windows."'); - ok(result.traits[4].name === 'targetArchitecture', 'result.traits[4].name should be "targetArchitecture"'); - ok(result.traits[4].value === 'x64', 'result.traits[4].value should be "x64"'); - ok(result.traits[4].includeInPrompt, 'result.traits[4].includeInPrompt should be true'); - ok(result.traits[4].promptTextOverride === 'This build targets x64.', 'result.traits[4].promptTextOverride should be "This build targets x64."'); + ok(result.traits.find((trait) => trait.name === 'language'), 'result.traits should have a language trait'); + ok(result.traits.find((trait) => trait.name === 'language')?.value === 'C++', 'result.traits should have a language trait with value "C++"'); + ok(result.traits.find((trait) => trait.name === 'language')?.includeInPrompt, 'result.traits should have a language trait with includeInPrompt true'); + ok(result.traits.find((trait) => trait.name === 'language')?.promptTextOverride === 'The language is C++.', 'result.traits should have a language trait with promptTextOverride "The language is C++."'); + ok(result.traits.find((trait) => trait.name === 'compiler'), 'result.traits should have a compiler trait'); + ok(result.traits.find((trait) => trait.name === 'compiler')?.value === 'MSVC', 'result.traits should have a compiler trait with value "MSVC"'); + ok(result.traits.find((trait) => trait.name === 'compiler')?.includeInPrompt, 'result.traits should have a compiler trait with includeInPrompt true'); + ok(result.traits.find((trait) => trait.name === 'compiler')?.promptTextOverride === 'This project compiles using MSVC.', 'result.traits should have a compiler trait with promptTextOverride "This project compiles using MSVC."'); + ok(result.traits.find((trait) => trait.name === 'standardVersion'), 'result.traits should have a standardVersion trait'); + ok(result.traits.find((trait) => trait.name === 'standardVersion')?.value === 'C++20', 'result.traits should have a standardVersion trait with value "C++20"'); + ok(result.traits.find((trait) => trait.name === 'standardVersion')?.includeInPrompt, 'result.traits should have a standardVersion trait with includeInPrompt true'); + ok(result.traits.find((trait) => trait.name === 'standardVersion')?.promptTextOverride === 'This project uses the C++20 language standard.', 'result.traits should have a standardVersion trait with promptTextOverride "This project uses the C++20 language standard."'); + ok(result.traits.find((trait) => trait.name === 'targetPlatform'), 'result.traits should have a targetPlatform trait'); + ok(result.traits.find((trait) => trait.name === 'targetPlatform')?.value === 'Windows', 'result.traits should have a targetPlatform trait with value "Windows"'); + ok(result.traits.find((trait) => trait.name === 'targetPlatform')?.includeInPrompt, 'result.traits should have a targetPlatform trait with includeInPrompt true'); + ok(result.traits.find((trait) => trait.name === 'targetPlatform')?.promptTextOverride === 'This build targets Windows.', 'result.traits should have a targetPlatform trait with promptTextOverride "This build targets Windows."'); + ok(result.traits.find((trait) => trait.name === 'targetArchitecture'), 'result.traits should have a targetArchitecture trait'); + ok(result.traits.find((trait) => trait.name === 'targetArchitecture')?.value === 'x64', 'result.traits should have a targetArchitecture trait with value "x64"'); + ok(result.traits.find((trait) => trait.name === 'targetArchitecture')?.includeInPrompt, 'result.traits should have a targetArchitecture trait with includeInPrompt true'); + ok(result.traits.find((trait) => trait.name === 'targetArchitecture')?.promptTextOverride === 'This build targets x64.', 'result.traits should have a targetArchitecture trait with promptTextOverride "This build targets x64."'); + }); + + const cppContext: CppContext = { + language: 'C++', + standardVersion: 'C++20', + compiler: 'MSVC', + targetPlatform: 'Windows', + targetArchitecture: 'x64', + compilerArguments: '/std:c++17 /permissive-' + }; + it('should provide compiler arguments traits if available.', async () => { + arrange({ + vscodeExtension: vscodeExtension, + getIncludeFiles: { includedFiles: ['c:\\system\\include\\vector', 'c:\\system\\include\\string', 'C:\\src\\my_project\\foo.h'] }, + cppContext, + rootUri: vscode.Uri.file('C:\\src\\my_project'), + flags: { copilotcppTraits: true } + }); + await moduleUnderTest.registerRelatedFilesProvider(); + + const result = await callbackPromise; + + ok(result, 'result should be defined'); + ok(result.traits, 'result.traits should be defined'); + ok(result.traits.find((trait) => trait.name === 'compilerArguments'), 'result.traits should have a compiler args trait'); + ok(result.traits.find((trait) => trait.name === 'compilerArguments')?.value === '/std:c++17 /permissive-', 'result.traits should have a compiler args trait with value "/std:c++17 /permissive-"'); + ok(result.traits.find((trait) => trait.name === 'compilerArguments')?.includeInPrompt, 'result.traits should have a compiler args trait with includeInPrompt true'); + ok(result.traits.find((trait) => trait.name === 'compilerArguments')?.promptTextOverride === 'The compiler command line arguments may contain: /std:c++17 /permissive-.', 'result.traits should have a compiler args trait with promptTextOverride "The compiler command line arguments may contain: /std:c++17 /permissive-"'); }); - it('should exclude #cpp traits per copilotcppExcludeTraits.', async () => { + it('should exclude cpp context traits per copilotcppExcludeTraits.', async () => { const excludeTraits = ['compiler', 'targetPlatform']; arrange({ vscodeExtension: vscodeExtension, getIncludeFiles: { includedFiles }, - chatContext: { - language: 'c++', - standardVersion: 'c++20', - compiler: 'msvc', - targetPlatform: 'windows', - targetArchitecture: 'x64' - }, + cppContext: cppContextNoArgs, rootUri, flags: { copilotcppTraits: true, copilotcppExcludeTraits: excludeTraits } }); diff --git a/Extension/test/scenarios/SingleRootProject/tests/lmTool.test.ts b/Extension/test/scenarios/SingleRootProject/tests/lmTool.test.ts new file mode 100644 index 0000000000..41c5144595 --- /dev/null +++ b/Extension/test/scenarios/SingleRootProject/tests/lmTool.test.ts @@ -0,0 +1,239 @@ +/* -------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All Rights Reserved. + * See 'LICENSE' in the project root for license information. + * ------------------------------------------------------------------------------------------ */ + +import { ok } from 'assert'; +import { afterEach, beforeEach, describe, it } from 'mocha'; +import * as sinon from 'sinon'; +import * as vscode from 'vscode'; +import * as util from '../../../../src/common'; +import { ChatContextResult, DefaultClient } from '../../../../src/LanguageServer/client'; +import { ClientCollection } from '../../../../src/LanguageServer/clientCollection'; +import * as extension from '../../../../src/LanguageServer/extension'; +import { CppConfigurationLanguageModelTool, getCppContext } from '../../../../src/LanguageServer/lmTool'; +import * as telemetry from '../../../../src/telemetry'; + +describe('CppConfigurationLanguageModelTool Tests', () => { + let mockLanguageModelToolInvocationOptions: sinon.SinonStubbedInstance>; + let activeClientStub: sinon.SinonStubbedInstance; + let mockTextEditorStub: MockTextEditor; + let mockTextDocumentStub: sinon.SinonStubbedInstance; + let languageModelToolTelemetryStub: sinon.SinonStub; + + class MockLanguageModelToolInvocationOptions implements vscode.LanguageModelToolInvocationOptions { + tokenizationOptions?: vscode.LanguageModelToolTokenizationOptions | undefined; + toolInvocationToken: undefined; + input: undefined; + } + class MockTextEditor implements vscode.TextEditor { + constructor(selection: vscode.Selection, selections: readonly vscode.Selection[], visibleRanges: readonly vscode.Range[], options: vscode.TextEditorOptions, document: vscode.TextDocument, viewColumn?: vscode.ViewColumn) { + this.selection = selection; + this.selections = selections; + this.visibleRanges = visibleRanges; + this.options = options; + this.viewColumn = viewColumn; + this.document = document; + } + selection: vscode.Selection; + selections: readonly vscode.Selection[]; + visibleRanges: readonly vscode.Range[]; + options: vscode.TextEditorOptions; + viewColumn: vscode.ViewColumn | undefined; + edit(_callback: (editBuilder: vscode.TextEditorEdit) => void, _options?: { readonly undoStopBefore: boolean; readonly undoStopAfter: boolean }): Thenable { + throw new Error('Method not implemented.'); + } + insertSnippet(_snippet: vscode.SnippetString, _location?: vscode.Position | vscode.Range | readonly vscode.Position[] | readonly vscode.Range[], _options?: { readonly undoStopBefore: boolean; readonly undoStopAfter: boolean }): Thenable { + throw new Error('Method not implemented.'); + } + setDecorations(_decorationType: vscode.TextEditorDecorationType, _rangesOrOptions: readonly vscode.Range[] | readonly vscode.DecorationOptions[]): void { + throw new Error('Method not implemented.'); + } + revealRange(_range: vscode.Range, _revealType?: vscode.TextEditorRevealType): void { + throw new Error('Method not implemented.'); + } + show(_column?: vscode.ViewColumn): void { + throw new Error('Method not implemented.'); + } + hide(): void { + throw new Error('Method not implemented.'); + } + document: vscode.TextDocument; + } + class MockTextDocument implements vscode.TextDocument { + uri: vscode.Uri; + constructor(uri: vscode.Uri, fileName: string, isUntitled: boolean, languageId: string, version: number, isDirty: boolean, isClosed: boolean, eol: vscode.EndOfLine, lineCount: number) { + this.uri = uri; + this.fileName = fileName; + this.isUntitled = isUntitled; + this.languageId = languageId; + this.version = version; + this.isDirty = isDirty; + this.isClosed = isClosed; + this.eol = eol; + this.lineCount = lineCount; + } + fileName: string; + isUntitled: boolean; + languageId: string; + version: number; + isDirty: boolean; + isClosed: boolean; + save(): Thenable { + throw new Error('Method not implemented.'); + } + eol: vscode.EndOfLine; + lineCount: number; + + lineAt(line: number): vscode.TextLine; + // eslint-disable-next-line @typescript-eslint/unified-signatures + lineAt(position: vscode.Position): vscode.TextLine; + lineAt(_arg: number | vscode.Position): vscode.TextLine { + throw new Error('Method not implemented.'); + } + offsetAt(_position: vscode.Position): number { + throw new Error('Method not implemented.'); + } + positionAt(_offset: number): vscode.Position { + throw new Error('Method not implemented.'); + } + getText(_range?: vscode.Range): string { + throw new Error('Method not implemented.'); + } + getWordRangeAtPosition(_position: vscode.Position, _regex?: RegExp): vscode.Range | undefined { + throw new Error('Method not implemented.'); + } + validateRange(_range: vscode.Range): vscode.Range { + throw new Error('Method not implemented.'); + } + validatePosition(_position: vscode.Position): vscode.Position { + throw new Error('Method not implemented.'); + } + } + beforeEach(() => { + sinon.stub(util, 'extensionContext').value({ extension: { id: 'test-extension-id' } }); + + mockTextDocumentStub = sinon.createStubInstance(MockTextDocument); + mockTextEditorStub = new MockTextEditor(new vscode.Selection(0, 0, 0, 0), [], [], { tabSize: 4 }, mockTextDocumentStub); + mockLanguageModelToolInvocationOptions = new MockLanguageModelToolInvocationOptions(); + activeClientStub = sinon.createStubInstance(DefaultClient); + const clientsStub = sinon.createStubInstance(ClientCollection); + sinon.stub(extension, 'getClients').returns(clientsStub); + sinon.stub(clientsStub, 'ActiveClient').get(() => activeClientStub); + activeClientStub.getIncludes.resolves({ includedFiles: [] }); + sinon.stub(vscode.window, 'activeTextEditor').get(() => mockTextEditorStub); + languageModelToolTelemetryStub = sinon.stub(telemetry, 'logLanguageModelToolEvent').returns(); + }); + + afterEach(() => { + sinon.restore(); + }); + + const arrange = ({ chatContextFromCppTools, isCpp, isHeaderFile }: + { chatContextFromCppTools?: ChatContextResult; isCpp?: boolean; isHeaderFile?: boolean } = + { chatContextFromCppTools: undefined, isCpp: undefined, isHeaderFile: false } + ) => { + activeClientStub.getChatContext.resolves(chatContextFromCppTools); + sinon.stub(util, 'isCpp').returns(isCpp ?? true); + sinon.stub(util, 'isHeaderFile').returns(isHeaderFile ?? false); + }; + + it('should log telemetry and provide #cpp chat context.', async () => { + arrange({ + chatContextFromCppTools: { + language: 'cpp', + standardVersion: 'c++20', + compiler: 'msvc', + targetPlatform: 'windows', + targetArchitecture: 'x64', + compilerArgs: ['/notused'], + compilerUserDefines: ['NOTUSED'] + } + }); + + const result = await new CppConfigurationLanguageModelTool().invoke(mockLanguageModelToolInvocationOptions, new vscode.CancellationTokenSource().token); + + ok(languageModelToolTelemetryStub.calledOnce, 'logLanguageModelToolEvent should be called once'); + ok(languageModelToolTelemetryStub.calledWithMatch('Chat/Tool/cpp', sinon.match({ + "language": 'cpp', + "compiler": 'msvc', + "standardVersion": 'c++20', + "targetPlatform": 'windows', + "targetArchitecture": 'x64' + }))); + ok(result, 'result should not be undefined'); + const text = result.content[0] as vscode.LanguageModelTextPart; + ok(text, 'result should contain a text part'); + ok(text.value === 'The user is working on a C++ project. The project uses language version C++20, compiles using the MSVC compiler, targets the Windows platform, and targets the x64 architecture.'); + }); + + const testGetCppContext = async ({ + compiler, + expectedCompiler, + context, + compilerArguments: compilerArguments, + exptectedFilteredCompilerArgumentCount, + expectedcompilerArguments }: { + compiler: string; + expectedCompiler: string; + context: { flags: Record }; + compilerArguments: string[]; + exptectedFilteredCompilerArgumentCount: number; + expectedcompilerArguments: string; + }) => { + arrange({ + chatContextFromCppTools: { + language: 'cpp', + standardVersion: 'c++20', + compiler: compiler, + targetPlatform: 'windows', + targetArchitecture: 'x64', + compilerArgs: compilerArguments, + compilerUserDefines: [] + } + }); + + const result = await getCppContext(context, new vscode.CancellationTokenSource().token); + + ok(languageModelToolTelemetryStub.calledOnce, 'logLanguageModelToolEvent should be called once'); + ok(languageModelToolTelemetryStub.calledWithMatch('Completions/tool', sinon.match({ + "language": 'C++', + "compiler": expectedCompiler, + "standardVersion": 'C++20', + "targetPlatform": 'Windows', + "targetArchitecture": 'x64', + "compilerArgumentCount": compilerArguments.length.toString(), + "filteredCompilerArgumentCount": exptectedFilteredCompilerArgumentCount.toString() + }))); + ok(result, 'result should not be undefined'); + ok(result.language === 'C++'); + ok(result.compiler === expectedCompiler); + ok(result.standardVersion === 'C++20'); + ok(result.targetPlatform === 'Windows'); + ok(result.targetArchitecture === 'x64'); + ok(result.compilerArguments === expectedcompilerArguments); + }; + + it('should log telemetry and provide cpp context properly when experimental flags are not defined.', async () => { + await testGetCppContext({ + compiler: 'gcc', + expectedCompiler: 'GCC', + context: { flags: {} }, + compilerArguments: ['-Wall', '-Werror', '-std=c++20'], + exptectedFilteredCompilerArgumentCount: 0, + expectedcompilerArguments: '' + }); + }); + + it('should provide compilerArguments based on copilotcppGccCompilerArgumentFilter.', async () => { + await testGetCppContext({ + compiler: 'gcc', + expectedCompiler: 'GCC', + context: { flags: { copilotcppGccCompilerArgumentFilter: '^-(fno\-exceptions|fno\-rtti)$' } }, + compilerArguments: ['-Wall', '-Werror', '-std=c++20', '-fno-exceptions', '-fno-rtti', '-pthread', '-O3', '-funroll-loops'], + exptectedFilteredCompilerArgumentCount: 2, + expectedcompilerArguments: '-fno-exceptions -fno-rtti' + }); + }); + +});