Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Support Compiler Argument Trait AB Experiment #12979

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
+614 −93
Open

Support Compiler Argument Trait AB Experiment #12979

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions Extension/src/LanguageServer/client.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -541,6 +541,19 @@ export interface ChatContextResult {
targetArchitecture: string;
}

export interface FileContextResult {
compilerArguments: string[];
}

export interface ProjectContextResult {
language: string;
standardVersion: string;
compiler: string;
targetPlatform: string;
targetArchitecture: string;
fileContext: FileContextResult;
}

// Requests
const PreInitializationRequest: RequestType<void, string, void> = new RequestType<void, string, void>('cpptools/preinitialize');
const InitializationRequest: RequestType<CppInitializationParams, void, void> = new RequestType<CppInitializationParams, void, void>('cpptools/initialize');
Expand All @@ -561,6 +574,7 @@ const GenerateDoxygenCommentRequest: RequestType<GenerateDoxygenCommentParams, G
const ChangeCppPropertiesRequest: RequestType<CppPropertiesParams, void, void> = new RequestType<CppPropertiesParams, void, void>('cpptools/didChangeCppProperties');
const IncludesRequest: RequestType<GetIncludesParams, GetIncludesResult, void> = new RequestType<GetIncludesParams, GetIncludesResult, void>('cpptools/getIncludes');
const CppContextRequest: RequestType<void, ChatContextResult, void> = new RequestType<void, ChatContextResult, void>('cpptools/getChatContext');
const ProjectContextRequest: RequestType<void, ProjectContextResult, void> = new RequestType<void, ProjectContextResult, void>('cpptools/getProjectContext');

// Notifications to the server
const DidOpenNotification: NotificationType<DidOpenTextDocumentParams> = new NotificationType<DidOpenTextDocumentParams>('textDocument/didOpen');
Expand Down Expand Up @@ -792,6 +806,7 @@ export interface Client {
addTrustedCompiler(path: string): Promise<void>;
getIncludes(maxDepth: number, token: vscode.CancellationToken): Promise<GetIncludesResult>;
getChatContext(token: vscode.CancellationToken): Promise<ChatContextResult>;
getProjectContext(token: vscode.CancellationToken): Promise<ProjectContextResult>;
}

export function createClient(workspaceFolder?: vscode.WorkspaceFolder): Client {
Expand Down Expand Up @@ -2220,6 +2235,12 @@ export class DefaultClient implements Client {
() => this.languageClient.sendRequest(CppContextRequest, null, token), token);
}

public async getProjectContext(token: vscode.CancellationToken): Promise<ProjectContextResult> {
await withCancellation(this.ready, token);
return DefaultClient.withLspCancellationHandling(
() => this.languageClient.sendRequest(ProjectContextRequest, null, token), token);
}

/**
* a Promise that can be awaited to know when it's ok to proceed.
*
Expand Down Expand Up @@ -4123,4 +4144,5 @@ class NullClient implements Client {
addTrustedCompiler(path: string): Promise<void> { return Promise.resolve(); }
getIncludes(maxDepth: number, token: vscode.CancellationToken): Promise<GetIncludesResult> { return Promise.resolve({} as GetIncludesResult); }
getChatContext(token: vscode.CancellationToken): Promise<ChatContextResult> { return Promise.resolve({} as ChatContextResult); }
getProjectContext(token: vscode.CancellationToken): Promise<ProjectContextResult> { return Promise.resolve({} as ProjectContextResult); }
}
53 changes: 43 additions & 10 deletions Extension/src/LanguageServer/copilotProviders.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,9 @@

import * as vscode from 'vscode';
import * as util from '../common';
import { ChatContextResult, GetIncludesResult } from './client';
import { GetIncludesResult } from './client';
import { getActiveClient } from './extension';
import { getProjectContext } from './lmTool';

export interface CopilotTrait {
name: string;
Expand Down Expand Up @@ -38,22 +39,54 @@ export async function registerRelatedFilesProvider(): Promise<void> {

const getIncludesHandler = async () => (await getIncludesWithCancellation(1, token))?.includedFiles.map(file => vscode.Uri.file(file)) ?? [];
const getTraitsHandler = async () => {
const chatContext: ChatContextResult | undefined = await (getActiveClient().getChatContext(token) ?? undefined);
const cppContext = await getProjectContext(context, token);

if (!chatContext) {
if (!cppContext) {
return undefined;
}

let traits: CopilotTrait[] = [
{ name: "language", value: chatContext.language, includeInPrompt: true, promptTextOverride: `The language is ${chatContext.language}.` },
{ name: "compiler", value: chatContext.compiler, includeInPrompt: true, promptTextOverride: `This project compiles using ${chatContext.compiler}.` },
{ name: "standardVersion", value: chatContext.standardVersion, includeInPrompt: true, promptTextOverride: `This project uses the ${chatContext.standardVersion} language standard.` },
{ name: "targetPlatform", value: chatContext.targetPlatform, includeInPrompt: true, promptTextOverride: `This build targets ${chatContext.targetPlatform}.` },
{ name: "targetArchitecture", value: chatContext.targetArchitecture, includeInPrompt: true, promptTextOverride: `This build targets ${chatContext.targetArchitecture}.` }
{ name: "intelliSenseDisclaimer", value: '', includeInPrompt: true, promptTextOverride: `IntelliSense is currently configured with the following compiler information. It reflects the active configuration, and the project may have more configurations targeting different platforms.` },
{ name: "intelliSenseDisclaimerBeginning", value: '', includeInPrompt: true, promptTextOverride: `Beginning of IntelliSense information.` }
];
if (cppContext.language) {
traits.push({ name: "language", value: cppContext.language, includeInPrompt: true, promptTextOverride: `The language is ${cppContext.language}.` });
}
if (cppContext.compiler) {
traits.push({ name: "compiler", value: cppContext.compiler, includeInPrompt: true, promptTextOverride: `This project compiles using ${cppContext.compiler}.` });
}
if (cppContext.standardVersion) {
traits.push({ name: "standardVersion", value: cppContext.standardVersion, includeInPrompt: true, promptTextOverride: `This project uses the ${cppContext.standardVersion} language standard.` });
}
if (cppContext.targetPlatform) {
traits.push({ name: "targetPlatform", value: cppContext.targetPlatform, includeInPrompt: true, promptTextOverride: `This build targets ${cppContext.targetPlatform}.` });
}
if (cppContext.targetArchitecture) {
traits.push({ name: "targetArchitecture", value: cppContext.targetArchitecture, includeInPrompt: true, promptTextOverride: `This build targets ${cppContext.targetArchitecture}.` });
}
let directAsks: string = '';
if (cppContext.compilerArguments.length > 0) {
// Example: JSON.stringify({'-fno-rtti': "Do not generate code using RTTI keywords."})
const directAskMap: { [key: string]: string } = JSON.parse(context.flags.copilotcppCompilerArgumentDirectAskMap as string ?? '{}');
const updatedArguments = cppContext.compilerArguments.filter(arg => {
if (directAskMap[arg]) {
directAsks += `${directAskMap[arg]} `;
return false;
}
return true;
});

const compilerArgumentsValue = updatedArguments.join(", ");
traits.push({ name: "compilerArguments", value: compilerArgumentsValue, includeInPrompt: true, promptTextOverride: `The compiler arguments include: ${compilerArgumentsValue}.` });
}
if (directAsks) {
traits.push({ name: "directAsks", value: directAsks, includeInPrompt: true, promptTextOverride: directAsks });
}

traits.push({ name: "intelliSenseDisclaimerEnd", value: '', includeInPrompt: true, promptTextOverride: `End of IntelliSense information.` });

const excludeTraits = context.flags.copilotcppExcludeTraits as string[] ?? [];
traits = traits.filter(trait => !excludeTraits.includes(trait.name));
const excludeTraits = new Set(context.flags.copilotcppExcludeTraits as string[] ?? []);
sean-mcmanus marked this conversation as resolved.
Show resolved Hide resolved
traits = traits.filter(trait => !excludeTraits.has(trait.name));

return traits.length > 0 ? traits : undefined;
};
Expand Down
129 changes: 117 additions & 12 deletions Extension/src/LanguageServer/lmTool.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,19 +9,23 @@ import { localize } from 'vscode-nls';
import * as util from '../common';
import * as logger from '../logger';
import * as telemetry from '../telemetry';
import { ChatContextResult } from './client';
import { ChatContextResult, ProjectContextResult } from './client';
import { getClients } from './extension';
import { checkTime } from './utils';

const MSVC: string = 'MSVC';
const Clang: string = 'Clang';
const GCC: string = 'GCC';
const knownValues: { [Property in keyof ChatContextResult]?: { [id: string]: string } } = {
language: {
'c': 'C',
'cpp': 'C++',
'cuda-cpp': 'CUDA C++'
},
compiler: {
'msvc': 'MSVC',
'clang': 'Clang',
'gcc': 'GCC'
'msvc': MSVC,
'clang': Clang,
'gcc': GCC
},
standardVersion: {
'c++98': 'C++98',
Expand All @@ -44,6 +48,113 @@ const knownValues: { [Property in keyof ChatContextResult]?: { [id: string]: str
}
};

function formatChatContext(context: ChatContextResult | ProjectContextResult): void {
type KnownKeys = 'language' | 'standardVersion' | 'compiler' | 'targetPlatform';
for (const key in knownValues) {
const knownKey = key as KnownKeys;
if (knownValues[knownKey] && context[knownKey]) {
// Clear the value if it's not in the known values.
context[knownKey] = knownValues[knownKey][context[knownKey]] || "";
}
}
}

export interface ProjectContext {
language: string;
standardVersion: string;
compiler: string;
targetPlatform: string;
targetArchitecture: string;
compilerArguments: string[];
}

// Set these values for local testing purpose without involving control tower.
const defaultCompilerArgumentFilters: { [id: string]: RegExp | undefined } = {
MSVC: undefined, // Example: /^(\/std:.*|\/EHs-c-|\/GR-|\/await.*)$/,
Clang: undefined,
GCC: undefined // Example: /^(-std=.*|-fno-rtti|-fno-exceptions)$/
};

function filterComplierArguments(compiler: string, compilerArguments: string[], context: { flags: Record<string, unknown> }): string[] {
const defaultFilter: RegExp | undefined = defaultCompilerArgumentFilters[compiler] ?? undefined;
let additionalFilter: RegExp | undefined;
switch (compiler) {
case MSVC:
additionalFilter = context.flags.copilotcppMsvcCompilerArgumentFilter ? new RegExp(context.flags.copilotcppMsvcCompilerArgumentFilter as string) : undefined;
break;
case Clang:
additionalFilter = context.flags.copilotcppClangCompilerArgumentFilter ? new RegExp(context.flags.copilotcppClangCompilerArgumentFilter as string) : undefined;
break;
case GCC:
additionalFilter = context.flags.copilotcppGccCompilerArgumentFilter ? new RegExp(context.flags.copilotcppGccCompilerArgumentFilter as string) : undefined;
break;
}

return compilerArguments.filter(arg => defaultFilter?.test(arg) || additionalFilter?.test(arg));
}

export async function getProjectContext(context: { flags: Record<string, unknown> }, token: vscode.CancellationToken): Promise<ProjectContext | undefined> {
const telemetryProperties: Record<string, string> = {};
try {
const projectContext = await checkTime<ProjectContextResult | undefined>(async () => await getClients()?.ActiveClient?.getProjectContext(token) ?? undefined);
telemetryProperties["time"] = projectContext.time.toFixed(1);
if (!projectContext.result) {
return undefined;
}

formatChatContext(projectContext.result);

const result: ProjectContext = {
language: projectContext.result.language,
standardVersion: projectContext.result.standardVersion,
compiler: projectContext.result.compiler,
targetPlatform: projectContext.result.targetPlatform,
targetArchitecture: projectContext.result.targetArchitecture,
compilerArguments: []
};

if (projectContext.result.language) {
telemetryProperties["language"] = projectContext.result.language;
}
if (projectContext.result.compiler) {
telemetryProperties["compiler"] = projectContext.result.compiler;
}
if (projectContext.result.standardVersion) {
telemetryProperties["standardVersion"] = projectContext.result.standardVersion;
}
if (projectContext.result.targetPlatform) {
telemetryProperties["targetPlatform"] = projectContext.result.targetPlatform;
}
if (projectContext.result.targetArchitecture) {
telemetryProperties["targetArchitecture"] = projectContext.result.targetArchitecture;
}
telemetryProperties["compilerArgumentCount"] = projectContext.result.fileContext.compilerArguments.length.toString();
sean-mcmanus marked this conversation as resolved.
Show resolved Hide resolved
// Telemtry to learn about the argument distribution. The filtered arguments are expected to be non-PII.
if (projectContext.result.fileContext.compilerArguments.length) {
const filteredCompilerArguments = filterComplierArguments(projectContext.result.compiler, projectContext.result.fileContext.compilerArguments, context);
if (filteredCompilerArguments.length > 0) {
telemetryProperties["filteredCompilerArguments"] = filteredCompilerArguments.join(', ');
result.compilerArguments = filteredCompilerArguments;
}
}

return result;
}
catch (exception) {
try {
const err: Error = exception as Error;
logger.getOutputChannelLogger().appendLine(localize("copilot.projectcontext.error", "Error while retrieving the project context. Reason: {0}", err.message));
}
catch {
// Intentionally swallow any exception.
}
telemetryProperties["error"] = "true";
return undefined;
} finally {
telemetry.logLanguageModelEvent('Completions/tool', telemetryProperties);
kuchungmsft marked this conversation as resolved.
Show resolved Hide resolved
}
}

export class CppConfigurationLanguageModelTool implements vscode.LanguageModelTool<void> {
public async invoke(options: vscode.LanguageModelToolInvocationOptions<void>, token: vscode.CancellationToken): Promise<vscode.LanguageModelToolResult> {
return new vscode.LanguageModelToolResult([
Expand All @@ -63,13 +174,7 @@ export class CppConfigurationLanguageModelTool implements vscode.LanguageModelTo
return 'No configuration information is available for the active document.';
}

for (const key in knownValues) {
const knownKey = key as keyof ChatContextResult;
if (knownValues[knownKey] && chatContext[knownKey]) {
// Clear the value if it's not in the known values.
chatContext[knownKey] = knownValues[knownKey][chatContext[knownKey]] || "";
}
}
formatChatContext(chatContext);

let contextString = "";
if (chatContext.language) {
Expand Down Expand Up @@ -100,7 +205,7 @@ export class CppConfigurationLanguageModelTool implements vscode.LanguageModelTo
telemetryProperties["error"] = "true";
return "";
} finally {
telemetry.logLanguageModelToolEvent('cpp', telemetryProperties);
telemetry.logLanguageModelEvent('Chat/Tool/cpp', telemetryProperties);
}
}

Expand Down
6 changes: 6 additions & 0 deletions Extension/src/LanguageServer/utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,3 +112,9 @@ export async function withCancellation<T>(promise: Promise<T>, token: vscode.Can
});
});
}

export async function checkTime<T>(fn: () => Promise<T>): Promise<{ result: T; time: number }> {
const start = performance.now();
const result = await fn();
return { result, time: performance.now() - start };
}
4 changes: 2 additions & 2 deletions Extension/src/telemetry.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,10 +123,10 @@ export function logLanguageServerEvent(eventName: string, properties?: Record<st
sendTelemetry();
}

export function logLanguageModelToolEvent(eventName: string, properties?: Record<string, string>, metrics?: Record<string, number>): void {
export function logLanguageModelEvent(eventName: string, properties?: Record<string, string>, metrics?: Record<string, number>): void {
const sendTelemetry = () => {
if (experimentationTelemetry) {
const eventNamePrefix: string = "C_Cpp/Copilot/Chat/Tool/";
const eventNamePrefix: string = "C_Cpp/Copilot/";
experimentationTelemetry.sendTelemetryEvent(eventNamePrefix + eventName, properties, metrics);
}
};
Expand Down
Loading
Loading