From 9122433268eec9592f25bcef62e9e0bffe472deb Mon Sep 17 00:00:00 2001 From: Tofik Sonono Date: Wed, 4 Sep 2024 08:18:39 +0200 Subject: [PATCH 01/14] Added lcov-parser as dependency Used to parse lcov coverage info files. --- package.json | 1 + yarn.lock | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/package.json b/package.json index f279ccbea..4d2e9aac1 100644 --- a/package.json +++ b/package.json @@ -3760,6 +3760,7 @@ "webpack-cli": "^4.5.0" }, "dependencies": { + "@friedemannsommer/lcov-parser": "^4.0.1", "@types/string.prototype.matchall": "^4.0.4", "ajv": "^7.1.0", "chokidar": "^3.5.1", diff --git a/yarn.lock b/yarn.lock index 8e311934b..c0a147f4f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -165,6 +165,11 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" +"@friedemannsommer/lcov-parser@^4.0.1": + version "4.0.1" + resolved "https://registry.yarnpkg.com/@friedemannsommer/lcov-parser/-/lcov-parser-4.0.1.tgz#192445df2095bb061fd889ea0b0f0f94bf7c58b7" + integrity sha512-NjXrxLeh77FUAL6TzVJyvYvgA2AS+t2hlfRmfOyE5HR3g+2HdLNFTP2l+zTypn5JTnV4jfXgwSO46JLhvJeXlw== + "@gulp-sourcemaps/identity-map@^2.0.1": version "2.0.1" resolved "https://pkgs.dev.azure.com/azure-public/VisualCpp/_packaging/cpp_PublicPackages/npm/registry/@gulp-sourcemaps/identity-map/-/identity-map-2.0.1.tgz" From 1f04a3effa3acd7b66e68c90f93307c9e834cbef Mon Sep 17 00:00:00 2001 From: Tofik Sonono Date: Wed, 4 Sep 2024 08:24:38 +0200 Subject: [PATCH 02/14] Update vscode engine to 1.88.0 Needed for accessing the Test Coverage API in vscode --- package.json | 4 ++-- test/helpers/vscodefake/extensioncontext.ts | 4 ++-- yarn.lock | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index 4d2e9aac1..ed401a6b4 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "syntaxes" ], "engines": { - "vscode": "^1.67.0" + "vscode": "^1.88.0" }, "categories": [ "Other", @@ -3718,7 +3718,7 @@ "@types/rimraf": "^3.0.0", "@types/sinon": "~9.0.10", "@types/tmp": "^0.2.0", - "@types/vscode": "1.63.0", + "@types/vscode": "1.88.0", "@types/which": "~2.0.0", "@types/xml2js": "^0.4.8", "@types/uuid": "~8.3.3", diff --git a/test/helpers/vscodefake/extensioncontext.ts b/test/helpers/vscodefake/extensioncontext.ts index 784f24727..432410617 100644 --- a/test/helpers/vscodefake/extensioncontext.ts +++ b/test/helpers/vscodefake/extensioncontext.ts @@ -17,7 +17,7 @@ export class DefaultExtensionContext implements vscode.ExtensionContext { get extensionPath(): string { throw new Error(notImplementedErr); } - get environmentVariableCollection(): vscode.EnvironmentVariableCollection { + get environmentVariableCollection(): vscode.GlobalEnvironmentVariableCollection { throw new Error(notImplementedErr); } asAbsolutePath(relativePath: string): string { @@ -61,7 +61,7 @@ export class SmokeTestExtensionContext implements vscode.ExtensionContext { get extensionUri(): vscode.Uri { throw new Error(notImplementedErr); } - get environmentVariableCollection(): vscode.EnvironmentVariableCollection { + get environmentVariableCollection(): vscode.GlobalEnvironmentVariableCollection { throw new Error(notImplementedErr); } asAbsolutePath(sub: string): string { diff --git a/yarn.lock b/yarn.lock index c0a147f4f..9e8deb5d3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -647,10 +647,10 @@ resolved "https://pkgs.dev.azure.com/azure-public/VisualCpp/_packaging/cpp_PublicPackages/npm/registry/@types/uuid/-/uuid-8.3.4.tgz" integrity sha512-c/I8ZRb51j+pYGAu5CrFMRxqZ2ke4y2grEBO5AUjgSkSk+qT2Ea+OdWElz/OiMf5MNpn2b17kuVBwZLQJXzihw== -"@types/vscode@1.63.0": - version "1.63.0" - resolved "https://pkgs.dev.azure.com/azure-public/VisualCpp/_packaging/cpp_PublicPackages/npm/registry/@types/vscode/-/vscode-1.63.0.tgz" - integrity sha512-iePu1axOi5WSThV6l2TYcciBIpAlMarjBC8H0y8L8ocsZLxh7MttzwFU3pjoItF5fRVGxHS0Hsvje9jO3yJsfw== +"@types/vscode@1.88.0": + version "1.88.0" + resolved "https://registry.yarnpkg.com/@types/vscode/-/vscode-1.88.0.tgz#2dc690237f7ef049942508c8609b6b9f5216b4d3" + integrity sha512-rWY+Bs6j/f1lvr8jqZTyp5arRMfovdxolcqGi+//+cPDOh8SBvzXH90e7BiSXct5HJ9HGW6jATchbRTpTJpEkw== "@types/which@~2.0.0": version "2.0.1" From 26e0d22c21ce16533c341209ca7de2025bac7d20 Mon Sep 17 00:00:00 2001 From: Tofik Sonono Date: Wed, 4 Sep 2024 15:18:05 +0200 Subject: [PATCH 03/14] Native test coverage implementation Test coverage implementation for the CTest test controller. It relies on lcov coverage info files being specefied by the user in the settings.json of the project. Optionally, the user can specify CMake (utility) targets that should be built before and/or after the tests are/have been executed. These targets could reasonably zero the coverage counters (pre) and filter the coverage info files (post). --- docs/cmake-settings.md | 3 ++ package.json | 21 +++++++++ package.nls.json | 3 ++ src/config.ts | 20 +++++++- src/coverage.ts | 64 ++++++++++++++++++++++++++ src/ctest.ts | 84 +++++++++++++++++++++++++++++++++- test/unit-tests/config.test.ts | 5 +- 7 files changed, 196 insertions(+), 4 deletions(-) create mode 100644 src/coverage.ts diff --git a/docs/cmake-settings.md b/docs/cmake-settings.md index af77856e2..0e23d6c17 100644 --- a/docs/cmake-settings.md +++ b/docs/cmake-settings.md @@ -38,6 +38,9 @@ Options that support substitution, in the table below, allow variable references | `cmake.saveBeforeBuild` | If `true` (the default), saves open text documents when build or configure is invoked before running CMake. | `true` | no | | `cmake.sourceDirectory` | A directory or a list of directories where the root `CMakeLists.txt`s are stored. | `${workspaceFolder}` | yes | | `cmake.testEnvironment` | An object containing `key:value` pairs of environment variables, which will be available when debugging, running and testing with CTest. | `null` (no environment variables) | yes | +| `cmake.preRunCoverageTarget` | Target to build before running tests with coverage using the test explorer | null | no | +| `cmake.postRunCoverageTarget` | Target to build after running tests with coverage using the test explorer | null | no | +| `cmake.coverageInfoFiles` | LCOV coverage info files to be processed after running tests with coverage using the test explorer | [] | yes | ## Variable substitution diff --git a/package.json b/package.json index ed401a6b4..45d23daac 100644 --- a/package.json +++ b/package.json @@ -3592,6 +3592,27 @@ "default": true, "description": "%cmake-tools.configuration.cmake.enableAutomaticKitScan.description%", "scope": "resource" + }, + "cmake.preRunCoverageTarget": { + "type": "string", + "default": null, + "description": "%cmake-tools.configuration.cmake.preRunCoverageTarget.description%", + "scope": "resource" + }, + "cmake.postRunCoverageTarget": { + "type": "string", + "default": null, + "description": "%cmake-tools.configuration.cmake.postRunCoverageTarget.description%", + "scope": "resource" + }, + "cmake.coverageInfoFiles": { + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "description": "%cmake-tools.configuration.cmake.coverageInfoFiles.description%", + "scope": "resource" } } }, diff --git a/package.nls.json b/package.nls.json index c78efd003..e8e22eb4d 100644 --- a/package.nls.json +++ b/package.nls.json @@ -274,6 +274,9 @@ "cmake-tools.configuration.cmake.automaticReconfigure.description": "Automatically configure CMake project directories when the kit or the configuration preset is changed.", "cmake-tools.configuration.cmake.pinnedCommands.description":"List of CMake commands to pin.", "cmake-tools.configuration.cmake.enableAutomaticKitScan.description": "Enable automatic scanning for kits when a kit isn't selected. This will only take affect when CMake Presets aren't being used.", + "cmake-tools.configuration.cmake.preRunCoverageTarget.description": "Target to build before running tests with coverage using the test explorer", + "cmake-tools.configuration.cmake.postRunCoverageTarget.description": "Target to build after running tests with coverage using the test explorer", + "cmake-tools.configuration.cmake.coverageInfoFiles.description": "LCOV coverage info files to be processed after running tests with coverage using the test explorer.", "cmake-tools.debugger.pipeName.description": "Name of the pipe (on Windows) or domain socket (on Unix) to use for debugger communication.", "cmake-tools.debugger.clean.description": "Clean prior to configuring.", "cmake-tools.debugger.configureAll.description": "Configure for all projects.", diff --git a/src/config.ts b/src/config.ts index 6c2345361..ddcc2e41d 100644 --- a/src/config.ts +++ b/src/config.ts @@ -218,6 +218,9 @@ export interface ExtensionConfigurationSettings { automaticReconfigure: boolean; pinnedCommands: string[]; enableAutomaticKitScan: boolean; + preRunCoverageTarget: string | null; + postRunCoverageTarget: string | null; + coverageInfoFiles: string[]; } type EmittersOf = { @@ -566,6 +569,18 @@ export class ConfigurationReader implements vscode.Disposable { return this.configData.enableAutomaticKitScan; } + get preRunCoverageTarget(): string | null { + return this.configData.preRunCoverageTarget; + } + + get postRunCoverageTarget(): string | null { + return this.configData.postRunCoverageTarget; + } + + get coverageInfoFiles(): string[] { + return this.configData.coverageInfoFiles; + } + private readonly emitters: EmittersOf = { autoSelectActiveFolder: new vscode.EventEmitter(), defaultActiveFolder: new vscode.EventEmitter(), @@ -631,7 +646,10 @@ export class ConfigurationReader implements vscode.Disposable { launchBehavior: new vscode.EventEmitter(), automaticReconfigure: new vscode.EventEmitter(), pinnedCommands: new vscode.EventEmitter(), - enableAutomaticKitScan: new vscode.EventEmitter() + enableAutomaticKitScan: new vscode.EventEmitter(), + preRunCoverageTarget: new vscode.EventEmitter(), + postRunCoverageTarget: new vscode.EventEmitter(), + coverageInfoFiles: new vscode.EventEmitter() }; /** diff --git a/src/coverage.ts b/src/coverage.ts new file mode 100644 index 000000000..feb1a10af --- /dev/null +++ b/src/coverage.ts @@ -0,0 +1,64 @@ +import * as vscode from 'vscode'; +import { lcovParser } from "@friedemannsommer/lcov-parser"; +import * as nls from 'vscode-nls'; +import * as logging from '@cmt/logging'; + +nls.config({ messageFormat: nls.MessageFormat.bundle, bundleFormat: nls.BundleFormat.standalone })(); +const localize: nls.LocalizeFunc = nls.loadMessageBundle(); + +const log = logging.createLogger('ctest'); + +export async function handleCoverageInfoFiles(run: vscode.TestRun, coverageInfoFiles: string[], coverageData: WeakMap) { + for (const coverageInfoFile of coverageInfoFiles) { + let contents: Uint8Array; + try { + contents = await vscode.workspace.fs.readFile(vscode.Uri.file(coverageInfoFile)); + } catch (e) { + log.warning(localize('test.openCoverageInfoFile', 'Could not open coverage info file: {0}. Skipping...', coverageInfoFile)); + return; + } + const sections = await lcovParser({ from: contents }); + for (const section of sections) { + const coverage = new vscode.FileCoverage(vscode.Uri.file(section.path), + new vscode.TestCoverageCount( + section.lines.hit, + section.lines.instrumented + ), new vscode.TestCoverageCount( + section.branches.hit, + section.branches.instrumented + ), new vscode.TestCoverageCount( + section.functions.hit, + section.functions.instrumented + )); + + const lineBranches = new Map(); + for (const branch of section.branches.details) { + const branchCoverage = new vscode.BranchCoverage(branch.hit, + new vscode.Position(branch.line - 1, 0), branch.branch); + + const curr = lineBranches.get(branch.line); + if (curr === undefined) { + lineBranches.set(branch.line, [branchCoverage]); + } else { + curr.push(branchCoverage); + lineBranches.set(branch.line, curr); + } + } + + const declarations: vscode.DeclarationCoverage[] = []; + for (const declaration of section.functions.details) { + declarations.push(new vscode.DeclarationCoverage(declaration.name, declaration.hit, + new vscode.Position(declaration.line - 1, 0))); + } + + const statements: vscode.StatementCoverage[] = []; + for (const line of section.lines.details) { + statements.push(new vscode.StatementCoverage(line.hit, + new vscode.Position(line.line - 1, 0), + lineBranches.get(line.line) ?? [])); + } + coverageData.set(coverage, [...statements, ...declarations]); + run.addCoverage(coverage); + } + } +} diff --git a/src/ctest.ts b/src/ctest.ts index 5d1021bdb..7de41fc07 100644 --- a/src/ctest.ts +++ b/src/ctest.ts @@ -15,6 +15,8 @@ import { expandString } from '@cmt/expand'; import * as proc from '@cmt/proc'; import { ProjectController } from '@cmt/projectController'; import { extensionManager } from '@cmt/extension'; +import { CMakeProject } from '@cmt/cmakeProject'; +import { handleCoverageInfoFiles } from '@cmt/coverage'; nls.config({ messageFormat: nls.MessageFormat.bundle, bundleFormat: nls.BundleFormat.standalone })(); const localize: nls.LocalizeFunc = nls.loadMessageBundle(); @@ -116,6 +118,14 @@ interface CTestInfo { version: { major: number; minor: number }; } +interface ProjectCoverageConfig { + project: CMakeProject; + driver: CMakeDriver; + preRunCoverageTarget: string | null; + postRunCoverageTarget: string | null; + coverageInfoFiles: string[]; +} + function parseXmlString(xml: string): Promise { return new Promise((resolve, reject) => { xml2js.parseString(xml, (err, result) => { @@ -227,6 +237,8 @@ export class CTestDriver implements vscode.Disposable { private readonly testingEnabledEmitter = new vscode.EventEmitter(); readonly onTestingEnabledChanged = this.testingEnabledEmitter.event; + private coverageData = new WeakMap(); + dispose() { this.testingEnabledEmitter.dispose(); this.testsChangedEmitter.dispose(); @@ -874,7 +886,64 @@ export class CTestDriver implements vscode.Disposable { return true; } - private async runTestHandler(request: vscode.TestRunRequest, cancellation: vscode.CancellationToken) { + private async handleCoverageOnProjects(run: vscode.TestRun, projectCoverageConfigs: ProjectCoverageConfig[]) { + for (const projectCoverageConfig of projectCoverageConfigs) { + if (projectCoverageConfig.coverageInfoFiles.length === 0) { + log.warning(localize('test.noCoverageInfoFiles', 'No coverage info files for CMake project {0}. No coverage data will be analyzed for this project.', projectCoverageConfig.project.sourceDir)); + continue; + } + const expandedCoverageInfoFiles = await Promise.all(projectCoverageConfig.coverageInfoFiles.map(async (coverageInfoFile: string) => expandString(coverageInfoFile, projectCoverageConfig.driver.expansionOptions))); + await handleCoverageInfoFiles(run, expandedCoverageInfoFiles, this.coverageData); + } + } + + private async coverageCTestHelper(tests: vscode.TestItem[], run: vscode.TestRun, cancellation: vscode.CancellationToken): Promise { + const projectRoots = new Set(); + for (const test of tests) { + projectRoots.add(this.getTestRootFolder(test)); + } + + const projectCoverageConfigs: ProjectCoverageConfig[] = []; + for (const folder of projectRoots) { + const project = await this.projectController?.getProjectForFolder(folder); + if (project) { + const driver = await project.getCMakeDriverInstance(); + if (driver) { + projectCoverageConfigs.push({ + project: project, driver: driver, preRunCoverageTarget: driver.config.preRunCoverageTarget, postRunCoverageTarget: driver.config.postRunCoverageTarget, coverageInfoFiles: driver.config.coverageInfoFiles + }); + } + } + } + + for (const projectCoverageConfig of projectCoverageConfigs) { + if (projectCoverageConfig.preRunCoverageTarget) { + log.info(localize('test.buildingPreRunCoverageTarget', 'Building the preRunCoverageTarget for project {0} before running tests with coverage.', projectCoverageConfig.project.sourceDir)); + const rc = await projectCoverageConfig.project.build([projectCoverageConfig.preRunCoverageTarget]); + if (rc !== 0) { + log.error(localize('test.preRunCoverageTargetFailure', 'Building the preRunCoverageTarget \'{0}\' on project in {1} failed. Skipping running tests.', projectCoverageConfig.preRunCoverageTarget, projectCoverageConfig.project.sourceDir)); + run.end(); + return 0; + } + } + } + const runResult = await this.runCTestHelper(tests, run, cancellation, undefined, undefined, undefined, false, undefined, RunCTestHelperEntryPoint.TestExplorer); + for (const projectCoverageConfig of projectCoverageConfigs) { + if (projectCoverageConfig.postRunCoverageTarget) { + log.info(localize('test.buildingPostRunCoverageTarget', 'Building the postRunCoverageTarget \'{0}\' for project {1} after the tests have run with coverage.', projectCoverageConfig.postRunCoverageTarget, projectCoverageConfig.project.sourceDir)); + const rc = await projectCoverageConfig.project.build([projectCoverageConfig.postRunCoverageTarget]); + if (rc !== 0) { + log.error(localize('test.postRunCoverageTargetFailure', 'Building target postRunCoverageTarget on project in {0} failed. Skipping handling of coverage data.', projectCoverageConfig.project.sourceDir)); + return 0; + } + } + } + + await this.handleCoverageOnProjects(run, projectCoverageConfigs); + return runResult; + } + + private async runTestHandler(request: vscode.TestRunRequest, cancellation: vscode.CancellationToken, isCoverageRun = false) { // NOTE: We expect the testExplorer to be undefined when the cmake.ctest.testExplorerIntegrationEnabled is disabled. if (!testExplorer) { return; @@ -891,7 +960,12 @@ export class CTestDriver implements vscode.Disposable { this.ctestsEnqueued(tests, run); const buildSucceeded = await this.buildTests(tests, run); if (buildSucceeded) { - await this.runCTestHelper(tests, run, cancellation, undefined, undefined, undefined, false, undefined, RunCTestHelperEntryPoint.TestExplorer); + if (isCoverageRun) { + await this.coverageCTestHelper(tests, run, cancellation); + + } else { + await this.runCTestHelper(tests, run, cancellation, undefined, undefined, undefined, false, undefined, RunCTestHelperEntryPoint.TestExplorer); + } } else { log.info(localize('test.skip.run.build.failure', "Not running tests due to build failure.")); } @@ -1200,6 +1274,12 @@ export class CTestDriver implements vscode.Disposable { (request: vscode.TestRunRequest, cancellation: vscode.CancellationToken) => this.runTestHandler(request, cancellation), true ); + testExplorer.createRunProfile( + 'Run Tests with Coverage', + vscode.TestRunProfileKind.Coverage, + (request: vscode.TestRunRequest, cancellation: vscode.CancellationToken) => this.runTestHandler(request, cancellation, true), + true + ).loadDetailedCoverage = async (_, fileCoverage) => this.coverageData.get(fileCoverage) ?? []; testExplorer.createRunProfile( 'Debug Tests', vscode.TestRunProfileKind.Debug, diff --git a/test/unit-tests/config.test.ts b/test/unit-tests/config.test.ts index 2dec3e5c5..909c665c3 100644 --- a/test/unit-tests/config.test.ts +++ b/test/unit-tests/config.test.ts @@ -77,7 +77,10 @@ function createConfig(conf: Partial): Configurat launchBehavior: 'reuseTerminal', ignoreCMakeListsMissing: false, automaticReconfigure: false, - enableAutomaticKitScan: true + enableAutomaticKitScan: true, + preRunCoverageTarget: null, + postRunCoverageTarget: null, + coverageInfoFiles: [] }); ret.updatePartial(conf); return ret; From 6d04a6219ab4f84fc47c58c616ef5761366783db Mon Sep 17 00:00:00 2001 From: Tofik Sonono Date: Thu, 31 Oct 2024 13:40:13 +0100 Subject: [PATCH 04/14] CMake fixes in single-root-UI test --- .../single-root-UI/project-folder/CMakeLists.txt | 2 +- .../single-root-UI/project-folder/CMakePresets.json | 4 ++-- .../single-root-UI/project-folder/CMakeUserPresets.json | 6 +++--- .../project-folder/include/IncludedPresets1.json | 2 +- .../project-folder/include/IncludedPresets2.json | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/end-to-end-tests/single-root-UI/project-folder/CMakeLists.txt b/test/end-to-end-tests/single-root-UI/project-folder/CMakeLists.txt index dff0b8e57..0c51956ce 100644 --- a/test/end-to-end-tests/single-root-UI/project-folder/CMakeLists.txt +++ b/test/end-to-end-tests/single-root-UI/project-folder/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.0) +cmake_minimum_required(VERSION 3.5.0) project(TestBuildProcess VERSION 0.1.0) set(CMT_COOKIE passed-cookie CACHE STRING "Cookie to be written by the main executable") diff --git a/test/end-to-end-tests/single-root-UI/project-folder/CMakePresets.json b/test/end-to-end-tests/single-root-UI/project-folder/CMakePresets.json index e78e95aae..f049729a3 100644 --- a/test/end-to-end-tests/single-root-UI/project-folder/CMakePresets.json +++ b/test/end-to-end-tests/single-root-UI/project-folder/CMakePresets.json @@ -5,12 +5,12 @@ "name": "Linux1", "description": "Sets generator, build and install directory, vcpkg", "generator": "Ninja", - "binaryDir": "${workspaceFolder}/build", + "binaryDir": "${sourceDir}/build", "cacheVariables": { "CMAKE_BUILD_TYPE": "Debug", "CMAKE_TOOLCHAIN_FILE": { "type": "FILEPATH", - "value": "${workspaceFolder}/test-toolchain.cmake" + "value": "${sourceDir}/test-toolchain.cmake" } } } diff --git a/test/end-to-end-tests/single-root-UI/project-folder/CMakeUserPresets.json b/test/end-to-end-tests/single-root-UI/project-folder/CMakeUserPresets.json index e85796119..6fe319e92 100644 --- a/test/end-to-end-tests/single-root-UI/project-folder/CMakeUserPresets.json +++ b/test/end-to-end-tests/single-root-UI/project-folder/CMakeUserPresets.json @@ -8,7 +8,7 @@ "name": "LinuxUser1", "description": "Sets generator, build and install directory, vcpkg", "generator": "Unix Makefiles", - "binaryDir": "${workspaceFolder}/build", + "binaryDir": "${sourceDir}/build", "cacheVariables": { "CMAKE_BUILD_TYPE": "Debug", "CMAKE_C_COMPILER": "gcc", @@ -22,7 +22,7 @@ "name": "WindowsUser1", "description": "Sets generator, build and install directory, vcpkg", "generator": "Visual Studio 16 2019", - "binaryDir": "${workspaceFolder}/build", + "binaryDir": "${sourceDir}/build", "cacheVariables": { "CMAKE_BUILD_TYPE": "Debug", "CMAKE_C_COMPILER": "cl.exe", @@ -35,7 +35,7 @@ { "name": "NoGenerator", "description": "Skips setting a generator", - "binaryDir": "${workspaceFolder}/build", + "binaryDir": "${sourceDir}/build", "cacheVariables": { "CMAKE_BUILD_TYPE": "Debug", "CMAKE_C_COMPILER": "gcc", diff --git a/test/end-to-end-tests/single-root-UI/project-folder/include/IncludedPresets1.json b/test/end-to-end-tests/single-root-UI/project-folder/include/IncludedPresets1.json index e0dfc3f3d..f0b3b74ca 100644 --- a/test/end-to-end-tests/single-root-UI/project-folder/include/IncludedPresets1.json +++ b/test/end-to-end-tests/single-root-UI/project-folder/include/IncludedPresets1.json @@ -8,7 +8,7 @@ "name": "LinuxIncluded1", "description": "Sets generator, build and install directory, vcpkg", "generator": "Unix Makefiles", - "binaryDir": "${workspaceFolder}/build", + "binaryDir": "${sourceDir}/build", "cacheVariables": { "CMAKE_BUILD_TYPE": "Debug", "CMAKE_C_COMPILER": "gcc", diff --git a/test/end-to-end-tests/single-root-UI/project-folder/include/IncludedPresets2.json b/test/end-to-end-tests/single-root-UI/project-folder/include/IncludedPresets2.json index 8faab3d3b..fc3ef1704 100644 --- a/test/end-to-end-tests/single-root-UI/project-folder/include/IncludedPresets2.json +++ b/test/end-to-end-tests/single-root-UI/project-folder/include/IncludedPresets2.json @@ -5,7 +5,7 @@ "name": "LinuxIncluded2", "description": "Sets generator, build and install directory, vcpkg", "generator": "Unix Makefiles", - "binaryDir": "${workspaceFolder}/build", + "binaryDir": "${sourceDir}/build", "cacheVariables": { "CMAKE_BUILD_TYPE": "Debug", "CMAKE_C_COMPILER": "gcc", From 9c109e427b5876d3fca191a6f5d924fd39daeb90 Mon Sep 17 00:00:00 2001 From: Tofik Sonono Date: Thu, 31 Oct 2024 13:41:01 +0100 Subject: [PATCH 05/14] Added end-to-end test for coverage --- src/ctest.ts | 2 +- src/util.ts | 7 +++ .../project-folder/.vscode/settings.json | 7 ++- .../project-folder/CMakeLists.txt | 43 +++++++++++++ .../project-folder/CMakeUserPresets.json | 8 +++ .../single-root-UI/runTest.ts | 1 + .../single-root-UI/test/coverage.test.ts | 63 +++++++++++++++++++ 7 files changed, 129 insertions(+), 2 deletions(-) create mode 100644 test/end-to-end-tests/single-root-UI/test/coverage.test.ts diff --git a/src/ctest.ts b/src/ctest.ts index 7de41fc07..810e9e459 100644 --- a/src/ctest.ts +++ b/src/ctest.ts @@ -691,7 +691,7 @@ export class CTestDriver implements vscode.Disposable { return -1; } - if (util.isTestMode()) { + if (util.isTestMode() && !util.overrideTestModeForTestExplorer()) { // ProjectController can't be initialized in test mode, so we don't have a usable test explorer return 0; } diff --git a/src/util.ts b/src/util.ts index 0145a93b1..76078d887 100644 --- a/src/util.ts +++ b/src/util.ts @@ -835,6 +835,13 @@ export function isTestMode(): boolean { return process.env['CMT_TESTING'] === '1'; } +/** + * Returns true if the test explorer should be enabled even when in test mode. + */ +export function overrideTestModeForTestExplorer(): boolean { + return process.env['CMT_TESTING_OVERRIDE_TEST_EXPLORER'] === '1'; +} + export async function getAllCMakeListsPaths(path: string): Promise { const regex: RegExp = new RegExp(/(\/|\\)CMakeLists\.txt$/); return recGetAllFilePaths(path, regex, await readDir(path), []); diff --git a/test/end-to-end-tests/single-root-UI/project-folder/.vscode/settings.json b/test/end-to-end-tests/single-root-UI/project-folder/.vscode/settings.json index fc6a75163..69dd2297f 100644 --- a/test/end-to-end-tests/single-root-UI/project-folder/.vscode/settings.json +++ b/test/end-to-end-tests/single-root-UI/project-folder/.vscode/settings.json @@ -1,5 +1,10 @@ { "cmake.buildDirectory": "${workspaceFolder}/build", "cmake.useCMakePresets": "never", - "cmake.configureOnOpen": false + "cmake.configureOnOpen": false, + "cmake.preRunCoverageTarget": "init-coverage", + "cmake.postRunCoverageTarget": "capture-coverage", + "cmake.coverageInfoFiles": [ + "${workspaceFolder}/build/lcov.info" + ] } \ No newline at end of file diff --git a/test/end-to-end-tests/single-root-UI/project-folder/CMakeLists.txt b/test/end-to-end-tests/single-root-UI/project-folder/CMakeLists.txt index 0c51956ce..d223aba9e 100644 --- a/test/end-to-end-tests/single-root-UI/project-folder/CMakeLists.txt +++ b/test/end-to-end-tests/single-root-UI/project-folder/CMakeLists.txt @@ -3,9 +3,20 @@ project(TestBuildProcess VERSION 0.1.0) set(CMT_COOKIE passed-cookie CACHE STRING "Cookie to be written by the main executable") +option(USE_COVERAGE "Enable GCOV during the build" OFF) +if(USE_COVERAGE) + add_compile_options(--coverage) + add_link_options(--coverage -Wl,--dynamic-list-data) +endif() + +include(CTest) + add_executable(TestBuildProcess main.cpp) set_property(TARGET TestBuildProcess PROPERTY CXX_STANDARD 98) +add_test(NAME TestBuildProcessAsTest + COMMAND $) + add_custom_command( TARGET TestBuildProcess POST_BUILD @@ -36,3 +47,35 @@ add_custom_target(runTestTarget DEPENDS TestBuildProcess WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMENT "Run test target" ) + +find_program(LCOV lcov) +find_program(GCOV gcov) +set(LCOV_BASE lcov-base.info) +set(LCOV_TEST lcov-test.info) +set(LCOV_TOTAL lcov.info) +set(LCOV_LOG lcov.log) +set(LCOV_ERR lcov.err) +add_custom_target(init-coverage + COMMENT "Collecting initial coverage" + COMMAND ${LCOV} -c -i -d ${CMAKE_CURRENT_BINARY_DIR} --gcov-tool ${GCOV} + -o ${LCOV_BASE} 2>${LCOV_ERR} >${LCOV_LOG}) +add_dependencies(init-coverage reset-coverage) + +add_custom_target(reset-coverage + COMMENT "Reset all coverage counters to zero" + COMMAND ${LCOV} -q -z -d ${CMAKE_CURRENT_BINARY_DIR} --gcov-tool ${GCOV} + -o ${LCOV_BASE} + COMMAND ${LCOV} -q -z -d ${CMAKE_CURRENT_BINARY_DIR} --gcov-tool ${GCOV} + -o ${LCOV_TEST} + COMMAND ${LCOV} -q -z -d ${CMAKE_CURRENT_BINARY_DIR} --gcov-tool ${GCOV} + -o ${LCOV_TOTAL}) + +add_custom_target(capture-coverage + COMMENT "Capture coverage data" + DEPENDS ${LCOV_BASE} + COMMAND ${LCOV} -c -d ${CMAKE_CURRENT_BINARY_DIR} -o ${LCOV_TEST} + --gcov-tool ${GCOV} 2>${LCOV_ERR} >${LCOV_LOG} + COMMAND ${LCOV} -a ${LCOV_BASE} -a ${LCOV_TEST} -o ${LCOV_TOTAL} + >>${LCOV_LOG} + COMMAND ${LCOV} -r ${LCOV_TOTAL} -o ${LCOV_TOTAL} "'/usr/include*'" + >>${LCOV_LOG}) diff --git a/test/end-to-end-tests/single-root-UI/project-folder/CMakeUserPresets.json b/test/end-to-end-tests/single-root-UI/project-folder/CMakeUserPresets.json index 6fe319e92..b9f11c5e7 100644 --- a/test/end-to-end-tests/single-root-UI/project-folder/CMakeUserPresets.json +++ b/test/end-to-end-tests/single-root-UI/project-folder/CMakeUserPresets.json @@ -48,6 +48,14 @@ { "name": "TestInheritFromPreset", "inherits": "Linux1" + }, + { + "name": "LinuxUser2", + "description": "LinuxUser1 with Coverage enabled", + "inherits": "LinuxUser1", + "cacheVariables": { + "USE_COVERAGE": "ON" + } } ] } diff --git a/test/end-to-end-tests/single-root-UI/runTest.ts b/test/end-to-end-tests/single-root-UI/runTest.ts index 40e633c8c..e92b01567 100644 --- a/test/end-to-end-tests/single-root-UI/runTest.ts +++ b/test/end-to-end-tests/single-root-UI/runTest.ts @@ -19,6 +19,7 @@ async function main() { const extensionTestsEnv: { [key: string]: string | undefined } = { "CMT_TESTING": "1", "CMT_QUIET_CONSOLE": "1", + "CMT_TESTING_OVERRIDE_TEST_EXPLORER": "1", "TEST_FILTER": process.env.TEST_FILTER ?? ".*" }; diff --git a/test/end-to-end-tests/single-root-UI/test/coverage.test.ts b/test/end-to-end-tests/single-root-UI/test/coverage.test.ts new file mode 100644 index 000000000..94e280d1d --- /dev/null +++ b/test/end-to-end-tests/single-root-UI/test/coverage.test.ts @@ -0,0 +1,63 @@ +import { DefaultEnvironment, expect } from '@test/util'; +import * as vscode from 'vscode'; + +// From vscode: src/vs/workbench/contrib/testing/common/testTypes.ts +const enum TestResultState { + Unset = 0, + Queued = 1, + Running = 2, + Passed = 3, + Failed = 4, + Skipped = 5, + Errored = 6 +}; + +suite('Coverage integration', () => { + let testEnv: DefaultEnvironment; + + suiteSetup(async function (this: Mocha.Context) { + this.timeout(100000); + + const build_loc = 'build'; + const exe_res = 'output.txt'; + + testEnv = new DefaultEnvironment('test/end-to-end-tests/single-root-UI/project-folder', build_loc, exe_res); + + await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('useCMakePresets', 'always'); + await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); + + testEnv.projectFolder.buildDirectory.clear(); + await vscode.commands.executeCommand('cmake.setConfigurePreset', 'LinuxUser2'); + expect(await vscode.commands.executeCommand('cmake.configure')).to.be.eq(0); + expect(await vscode.commands.executeCommand('cmake.build')).to.be.eq(0); + }); + + suiteTeardown(async function (this: Mocha.Context) { + this.timeout(30000); + testEnv.teardown(); + }); + + test('Bad Run test with coverage', async () => { + await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('preRunCoverageTarget', 'non-existing-target'); + + let testResult: any = await vscode.commands.executeCommand('testing.coverage.uri', vscode.Uri.file(testEnv.projectFolder.location)); + expect(testResult['tasks'][0].hasCoverage).to.be.eq(false); + expect(testResult['items'][2].computedState).to.be.eq(TestResultState.Unset); + + await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('preRunCoverageTarget', 'init-target'); + await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('postRunCoverageTarget', 'non-existing-target'); + + testResult = await vscode.commands.executeCommand('testing.coverage.uri', vscode.Uri.file(testEnv.projectFolder.location)); + expect(testResult['tasks'][0].hasCoverage).to.be.eq(false); + expect(testResult['items'][2].computedState).to.be.eq(TestResultState.Unset); + }).timeout(60000); + + test('Good Run test with coverage', async () => { + await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('preRunCoverageTarget', 'init-coverage'); + await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('postRunCoverageTarget', 'capture-coverage'); + + const testResult: any = await vscode.commands.executeCommand('testing.coverage.uri', vscode.Uri.file(testEnv.projectFolder.location)); + expect(testResult['tasks'][0].hasCoverage).to.be.eq(true); + expect(testResult['items'][2].computedState).to.be.eq(TestResultState.Passed); + }).timeout(60000); +}); From 2d53bdb94650126e8dada9bdfe69ad7fdf34483d Mon Sep 17 00:00:00 2001 From: Tofik Sonono Date: Mon, 4 Nov 2024 14:16:47 +0100 Subject: [PATCH 06/14] Unit test for coverage --- test/unit-tests/coverage.test.ts | 42 ++++++++++++++++++++++++++++++++ test/unit-tests/lcov.info | 16 ++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 test/unit-tests/coverage.test.ts create mode 100644 test/unit-tests/lcov.info diff --git a/test/unit-tests/coverage.test.ts b/test/unit-tests/coverage.test.ts new file mode 100644 index 000000000..0598ada09 --- /dev/null +++ b/test/unit-tests/coverage.test.ts @@ -0,0 +1,42 @@ +import { handleCoverageInfoFiles } from "@cmt/coverage"; +import * as vscode from "vscode"; +import { expect, getTestResourceFilePath } from "@test/util"; + +suite('Coverage Handling', () => { + + test('Coverage Info (LCOV)', async () => { + const filesCoverages: vscode.FileCoverage[] = []; + const testRun: vscode.TestRun = { + name: '', + token: {} as vscode.CancellationToken, + isPersisted: false, + enqueued: (_test: vscode.TestItem) => {}, + started: (_test: vscode.TestItem) => {}, + passed: (_test: vscode.TestItem, _duration: number) => {}, + failed: (_test: vscode.TestItem, _message: vscode.TestMessage | vscode.TestMessage[], _duration: number) => {}, + skipped: (_test: vscode.TestItem) => {}, + errored: (_test: vscode.TestItem, _message: vscode.TestMessage | vscode.TestMessage[], _duration: number) => {}, + appendOutput: (_output: string, _location?: vscode.Location, _test?: vscode.TestItem) => {}, + addCoverage: (fileCoverage: vscode.FileCoverage) => { + filesCoverages.push(fileCoverage); + }, + end: () => {}, + onDidDispose: (_listener: () => void): vscode.Disposable => new vscode.Disposable(() => {}) + }; + + const coverageData = new WeakMap(); + await handleCoverageInfoFiles(testRun, [getTestResourceFilePath('lcov.info')], coverageData); + expect(filesCoverages.length).to.eq(1); + expect(filesCoverages[0].uri.fsPath).to.eq('/tmp/lcov/main.cpp'); + const coverageDetail = coverageData.get(filesCoverages[0]); + expect(coverageDetail).to.not.be.undefined; + + expect(coverageDetail![0].executed).to.eq(1); + expect((coverageDetail![0].location as vscode.Position).line).to.eq(2); + + expect(coverageDetail![1].executed).to.eq(1); + expect((coverageDetail![1].location as vscode.Position).line).to.eq(3); + + console.log(JSON.stringify(coverageDetail, null, 4)); + }); +}); diff --git a/test/unit-tests/lcov.info b/test/unit-tests/lcov.info new file mode 100644 index 000000000..fb7a8349c --- /dev/null +++ b/test/unit-tests/lcov.info @@ -0,0 +1,16 @@ +TN: +SF:/tmp/lcov/main.cpp +FN:3,4,main +FN:5,5,_GLOBAL__sub_I_main +FN:5,5,_Z41__static_initialization_and_destruction_0ii +FNDA:1,main +FNDA:1,_GLOBAL__sub_I_main +FNDA:1,_Z41__static_initialization_and_destruction_0ii +FNF:3 +FNH:3 +DA:3,1 +DA:4,1 +DA:5,4 +LF:3 +LH:3 +end_of_record From f40abf0cde0f7728da41f33590759f2d829cbc65 Mon Sep 17 00:00:00 2001 From: Tofik Sonono Date: Tue, 5 Nov 2024 08:47:39 +0100 Subject: [PATCH 07/14] Removed specifying gcov tool as it is not needed Also removed dynamic `--dynamic-list-data` linker flag --- .../single-root-UI/project-folder/CMakeLists.txt | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/test/end-to-end-tests/single-root-UI/project-folder/CMakeLists.txt b/test/end-to-end-tests/single-root-UI/project-folder/CMakeLists.txt index d223aba9e..207e388e8 100644 --- a/test/end-to-end-tests/single-root-UI/project-folder/CMakeLists.txt +++ b/test/end-to-end-tests/single-root-UI/project-folder/CMakeLists.txt @@ -6,7 +6,7 @@ set(CMT_COOKIE passed-cookie CACHE STRING "Cookie to be written by the main exec option(USE_COVERAGE "Enable GCOV during the build" OFF) if(USE_COVERAGE) add_compile_options(--coverage) - add_link_options(--coverage -Wl,--dynamic-list-data) + add_link_options(--coverage) endif() include(CTest) @@ -49,7 +49,6 @@ add_custom_target(runTestTarget DEPENDS TestBuildProcess ) find_program(LCOV lcov) -find_program(GCOV gcov) set(LCOV_BASE lcov-base.info) set(LCOV_TEST lcov-test.info) set(LCOV_TOTAL lcov.info) @@ -57,24 +56,24 @@ set(LCOV_LOG lcov.log) set(LCOV_ERR lcov.err) add_custom_target(init-coverage COMMENT "Collecting initial coverage" - COMMAND ${LCOV} -c -i -d ${CMAKE_CURRENT_BINARY_DIR} --gcov-tool ${GCOV} + COMMAND ${LCOV} -c -i -d ${CMAKE_CURRENT_BINARY_DIR} -o ${LCOV_BASE} 2>${LCOV_ERR} >${LCOV_LOG}) add_dependencies(init-coverage reset-coverage) add_custom_target(reset-coverage COMMENT "Reset all coverage counters to zero" - COMMAND ${LCOV} -q -z -d ${CMAKE_CURRENT_BINARY_DIR} --gcov-tool ${GCOV} + COMMAND ${LCOV} -q -z -d ${CMAKE_CURRENT_BINARY_DIR} -o ${LCOV_BASE} - COMMAND ${LCOV} -q -z -d ${CMAKE_CURRENT_BINARY_DIR} --gcov-tool ${GCOV} + COMMAND ${LCOV} -q -z -d ${CMAKE_CURRENT_BINARY_DIR} -o ${LCOV_TEST} - COMMAND ${LCOV} -q -z -d ${CMAKE_CURRENT_BINARY_DIR} --gcov-tool ${GCOV} + COMMAND ${LCOV} -q -z -d ${CMAKE_CURRENT_BINARY_DIR} -o ${LCOV_TOTAL}) add_custom_target(capture-coverage COMMENT "Capture coverage data" DEPENDS ${LCOV_BASE} COMMAND ${LCOV} -c -d ${CMAKE_CURRENT_BINARY_DIR} -o ${LCOV_TEST} - --gcov-tool ${GCOV} 2>${LCOV_ERR} >${LCOV_LOG} + 2>${LCOV_ERR} >${LCOV_LOG} COMMAND ${LCOV} -a ${LCOV_BASE} -a ${LCOV_TEST} -o ${LCOV_TOTAL} >>${LCOV_LOG} COMMAND ${LCOV} -r ${LCOV_TOTAL} -o ${LCOV_TOTAL} "'/usr/include*'" From 845d7fa769150fce9405e67c60ee1512081079c9 Mon Sep 17 00:00:00 2001 From: Tofik Sonono Date: Tue, 5 Nov 2024 08:47:52 +0100 Subject: [PATCH 08/14] Use setup-lcov action --- .github/workflows/ci-main-linux.yml | 3 +++ .github/workflows/ci-main-mac.yml | 5 +++-- .github/workflows/ci-main.win.yml | 3 +++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-main-linux.yml b/.github/workflows/ci-main-linux.yml index 20d47015c..b68c1019b 100644 --- a/.github/workflows/ci-main-linux.yml +++ b/.github/workflows/ci-main-linux.yml @@ -13,6 +13,9 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Setup LCOV + uses: hrishikesh-kadam/setup-lcov@v1 + - name: Setup Node environment uses: actions/setup-node@v3 with: diff --git a/.github/workflows/ci-main-mac.yml b/.github/workflows/ci-main-mac.yml index c1de44bb6..a946c4859 100644 --- a/.github/workflows/ci-main-mac.yml +++ b/.github/workflows/ci-main-mac.yml @@ -13,6 +13,9 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Setup LCOV + uses: hrishikesh-kadam/setup-lcov@v1 + - name: Setup Node environment uses: actions/setup-node@v3 with: @@ -43,8 +46,6 @@ jobs: configure-options: -DCMAKE_INSTALL_PREFIX:STRING=${{ github.workspace }}/test/fakebin install-build: true - - - name: Run successful-build test run: yarn endToEndTestsSuccessfulBuild diff --git a/.github/workflows/ci-main.win.yml b/.github/workflows/ci-main.win.yml index 51a03687c..da967df80 100644 --- a/.github/workflows/ci-main.win.yml +++ b/.github/workflows/ci-main.win.yml @@ -14,6 +14,9 @@ jobs: - name: Checkout source code uses: actions/checkout@v3 + - name: Setup LCOV + uses: hrishikesh-kadam/setup-lcov@v1 + - name: Setup Node environment uses: actions/setup-node@v3 with: From 9014ffaed0b2622d4b7fbf2cb9089f553ed8f2a3 Mon Sep 17 00:00:00 2001 From: Tofik Sonono Date: Tue, 5 Nov 2024 09:14:26 +0100 Subject: [PATCH 09/14] Removed log from coverage unit test --- test/unit-tests/coverage.test.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/unit-tests/coverage.test.ts b/test/unit-tests/coverage.test.ts index 0598ada09..00936e6f1 100644 --- a/test/unit-tests/coverage.test.ts +++ b/test/unit-tests/coverage.test.ts @@ -36,7 +36,5 @@ suite('Coverage Handling', () => { expect(coverageDetail![1].executed).to.eq(1); expect((coverageDetail![1].location as vscode.Position).line).to.eq(3); - - console.log(JSON.stringify(coverageDetail, null, 4)); }); }); From a4882234809a3412f8251e290a1e75d105d3f9d6 Mon Sep 17 00:00:00 2001 From: Tofik Sonono Date: Tue, 5 Nov 2024 09:30:54 +0100 Subject: [PATCH 10/14] Test additions and fixes --- .../single-root-UI/project-folder/CMakeLists.txt | 16 ++++++++-------- .../single-root-UI/test/coverage.test.ts | 3 +++ test/unit-tests/coverage.test.ts | 3 ++- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/test/end-to-end-tests/single-root-UI/project-folder/CMakeLists.txt b/test/end-to-end-tests/single-root-UI/project-folder/CMakeLists.txt index 207e388e8..9a2d2ca14 100644 --- a/test/end-to-end-tests/single-root-UI/project-folder/CMakeLists.txt +++ b/test/end-to-end-tests/single-root-UI/project-folder/CMakeLists.txt @@ -48,7 +48,7 @@ add_custom_target(runTestTarget DEPENDS TestBuildProcess COMMENT "Run test target" ) -find_program(LCOV lcov) +find_program(LCOV lcov REQUIRED) set(LCOV_BASE lcov-base.info) set(LCOV_TEST lcov-test.info) set(LCOV_TOTAL lcov.info) @@ -56,25 +56,25 @@ set(LCOV_LOG lcov.log) set(LCOV_ERR lcov.err) add_custom_target(init-coverage COMMENT "Collecting initial coverage" - COMMAND ${LCOV} -c -i -d ${CMAKE_CURRENT_BINARY_DIR} + COMMAND lcov -c -i -d ${CMAKE_CURRENT_BINARY_DIR} -o ${LCOV_BASE} 2>${LCOV_ERR} >${LCOV_LOG}) add_dependencies(init-coverage reset-coverage) add_custom_target(reset-coverage COMMENT "Reset all coverage counters to zero" - COMMAND ${LCOV} -q -z -d ${CMAKE_CURRENT_BINARY_DIR} + COMMAND lcov -q -z -d ${CMAKE_CURRENT_BINARY_DIR} -o ${LCOV_BASE} - COMMAND ${LCOV} -q -z -d ${CMAKE_CURRENT_BINARY_DIR} + COMMAND lcov -q -z -d ${CMAKE_CURRENT_BINARY_DIR} -o ${LCOV_TEST} - COMMAND ${LCOV} -q -z -d ${CMAKE_CURRENT_BINARY_DIR} + COMMAND lcov -q -z -d ${CMAKE_CURRENT_BINARY_DIR} -o ${LCOV_TOTAL}) add_custom_target(capture-coverage COMMENT "Capture coverage data" DEPENDS ${LCOV_BASE} - COMMAND ${LCOV} -c -d ${CMAKE_CURRENT_BINARY_DIR} -o ${LCOV_TEST} + COMMAND lcov -c -d ${CMAKE_CURRENT_BINARY_DIR} -o ${LCOV_TEST} 2>${LCOV_ERR} >${LCOV_LOG} - COMMAND ${LCOV} -a ${LCOV_BASE} -a ${LCOV_TEST} -o ${LCOV_TOTAL} + COMMAND lcov -a ${LCOV_BASE} -a ${LCOV_TEST} -o ${LCOV_TOTAL} >>${LCOV_LOG} - COMMAND ${LCOV} -r ${LCOV_TOTAL} -o ${LCOV_TOTAL} "'/usr/include*'" + COMMAND lcov -r ${LCOV_TOTAL} -o ${LCOV_TOTAL} "'/usr/include*'" >>${LCOV_LOG}) diff --git a/test/end-to-end-tests/single-root-UI/test/coverage.test.ts b/test/end-to-end-tests/single-root-UI/test/coverage.test.ts index 94e280d1d..c836c28c4 100644 --- a/test/end-to-end-tests/single-root-UI/test/coverage.test.ts +++ b/test/end-to-end-tests/single-root-UI/test/coverage.test.ts @@ -1,5 +1,7 @@ import { DefaultEnvironment, expect } from '@test/util'; import * as vscode from 'vscode'; +import * as fs from 'fs'; +import * as path from 'path'; // From vscode: src/vs/workbench/contrib/testing/common/testTypes.ts const enum TestResultState { @@ -59,5 +61,6 @@ suite('Coverage integration', () => { const testResult: any = await vscode.commands.executeCommand('testing.coverage.uri', vscode.Uri.file(testEnv.projectFolder.location)); expect(testResult['tasks'][0].hasCoverage).to.be.eq(true); expect(testResult['items'][2].computedState).to.be.eq(TestResultState.Passed); + expect(fs.existsSync(path.join(testEnv.projectFolder.location, testEnv.buildLocation, 'lcov.info'))).to.be.true; }).timeout(60000); }); diff --git a/test/unit-tests/coverage.test.ts b/test/unit-tests/coverage.test.ts index 00936e6f1..c7f766a7c 100644 --- a/test/unit-tests/coverage.test.ts +++ b/test/unit-tests/coverage.test.ts @@ -1,6 +1,7 @@ import { handleCoverageInfoFiles } from "@cmt/coverage"; import * as vscode from "vscode"; import { expect, getTestResourceFilePath } from "@test/util"; +import * as path from "path"; suite('Coverage Handling', () => { @@ -27,7 +28,7 @@ suite('Coverage Handling', () => { const coverageData = new WeakMap(); await handleCoverageInfoFiles(testRun, [getTestResourceFilePath('lcov.info')], coverageData); expect(filesCoverages.length).to.eq(1); - expect(filesCoverages[0].uri.fsPath).to.eq('/tmp/lcov/main.cpp'); + expect(filesCoverages[0].uri.fsPath).to.eq(path.join(path.sep, 'tmp', 'lcov', 'main.cpp')); const coverageDetail = coverageData.get(filesCoverages[0]); expect(coverageDetail).to.not.be.undefined; From 66fc827accfe8184c48b7d2a79a0244094566df3 Mon Sep 17 00:00:00 2001 From: Tofik Sonono Date: Tue, 5 Nov 2024 15:17:03 +0100 Subject: [PATCH 11/14] Rationale notes in code --- src/ctest.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/ctest.ts b/src/ctest.ts index 810e9e459..d2866f1bf 100644 --- a/src/ctest.ts +++ b/src/ctest.ts @@ -118,6 +118,10 @@ interface CTestInfo { version: { major: number; minor: number }; } +/* The preRunCoverageTarget and postRunCoverageTarget are optional user +configured targets that are run before and after the tests when a "Run with +Coverage" test execution is triggered. These cae be used to zero the +coverage counters, filter coverage results etc. */ interface ProjectCoverageConfig { project: CMakeProject; driver: CMakeDriver; @@ -887,6 +891,7 @@ export class CTestDriver implements vscode.Disposable { } private async handleCoverageOnProjects(run: vscode.TestRun, projectCoverageConfigs: ProjectCoverageConfig[]) { + // Currently only LCOV coverage info files are supported for (const projectCoverageConfig of projectCoverageConfigs) { if (projectCoverageConfig.coverageInfoFiles.length === 0) { log.warning(localize('test.noCoverageInfoFiles', 'No coverage info files for CMake project {0}. No coverage data will be analyzed for this project.', projectCoverageConfig.project.sourceDir)); From 8542165292fbb8b64776dab8787cff5932086fd3 Mon Sep 17 00:00:00 2001 From: Garrett Campbell Date: Wed, 20 Nov 2024 09:52:39 -0500 Subject: [PATCH 12/14] grab from feed --- yarn.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yarn.lock b/yarn.lock index 9e8deb5d3..f3cf9a517 100644 --- a/yarn.lock +++ b/yarn.lock @@ -167,7 +167,7 @@ "@friedemannsommer/lcov-parser@^4.0.1": version "4.0.1" - resolved "https://registry.yarnpkg.com/@friedemannsommer/lcov-parser/-/lcov-parser-4.0.1.tgz#192445df2095bb061fd889ea0b0f0f94bf7c58b7" + resolved "https://pkgs.dev.azure.com/azure-public/VisualCpp/_packaging/cpp_PublicPackages/npm/registry/@friedemannsommer/lcov-parser/-/lcov-parser-4.0.1.tgz#192445df2095bb061fd889ea0b0f0f94bf7c58b7" integrity sha512-NjXrxLeh77FUAL6TzVJyvYvgA2AS+t2hlfRmfOyE5HR3g+2HdLNFTP2l+zTypn5JTnV4jfXgwSO46JLhvJeXlw== "@gulp-sourcemaps/identity-map@^2.0.1": @@ -649,7 +649,7 @@ "@types/vscode@1.88.0": version "1.88.0" - resolved "https://registry.yarnpkg.com/@types/vscode/-/vscode-1.88.0.tgz#2dc690237f7ef049942508c8609b6b9f5216b4d3" + resolved "https://pkgs.dev.azure.com/azure-public/VisualCpp/_packaging/cpp_PublicPackages/npm/registry/@types/vscode/-/vscode-1.88.0.tgz#2dc690237f7ef049942508c8609b6b9f5216b4d3" integrity sha512-rWY+Bs6j/f1lvr8jqZTyp5arRMfovdxolcqGi+//+cPDOh8SBvzXH90e7BiSXct5HJ9HGW6jATchbRTpTJpEkw== "@types/which@~2.0.0": From 08020d95f7fb6301a32e9fc22dbee7f7f4a6e52b Mon Sep 17 00:00:00 2001 From: Tofik Sonono Date: Thu, 21 Nov 2024 11:19:12 +0100 Subject: [PATCH 13/14] Review fixes + disable coverage test on Win MSVC can not produce gcov based coverage data, therefore the coverage end-to-end tests are disabled on Windows. --- src/coverage.ts | 2 +- src/ctest.ts | 4 ++-- .../single-root-UI/test/coverage.test.ts | 12 ++++++++++-- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/coverage.ts b/src/coverage.ts index feb1a10af..b4d4caabe 100644 --- a/src/coverage.ts +++ b/src/coverage.ts @@ -6,7 +6,7 @@ import * as logging from '@cmt/logging'; nls.config({ messageFormat: nls.MessageFormat.bundle, bundleFormat: nls.BundleFormat.standalone })(); const localize: nls.LocalizeFunc = nls.loadMessageBundle(); -const log = logging.createLogger('ctest'); +const log = logging.createLogger('ctest-coverage'); export async function handleCoverageInfoFiles(run: vscode.TestRun, coverageInfoFiles: string[], coverageData: WeakMap) { for (const coverageInfoFile of coverageInfoFiles) { diff --git a/src/ctest.ts b/src/ctest.ts index eadabe6dd..455331782 100644 --- a/src/ctest.ts +++ b/src/ctest.ts @@ -959,7 +959,7 @@ export class CTestDriver implements vscode.Disposable { if (rc !== 0) { log.error(localize('test.preRunCoverageTargetFailure', 'Building the preRunCoverageTarget \'{0}\' on project in {1} failed. Skipping running tests.', projectCoverageConfig.preRunCoverageTarget, projectCoverageConfig.project.sourceDir)); run.end(); - return 0; + return rc; } } } @@ -970,7 +970,7 @@ export class CTestDriver implements vscode.Disposable { const rc = await projectCoverageConfig.project.build([projectCoverageConfig.postRunCoverageTarget]); if (rc !== 0) { log.error(localize('test.postRunCoverageTargetFailure', 'Building target postRunCoverageTarget on project in {0} failed. Skipping handling of coverage data.', projectCoverageConfig.project.sourceDir)); - return 0; + return rc; } } } diff --git a/test/end-to-end-tests/single-root-UI/test/coverage.test.ts b/test/end-to-end-tests/single-root-UI/test/coverage.test.ts index c836c28c4..10bc54682 100644 --- a/test/end-to-end-tests/single-root-UI/test/coverage.test.ts +++ b/test/end-to-end-tests/single-root-UI/test/coverage.test.ts @@ -25,6 +25,11 @@ suite('Coverage integration', () => { testEnv = new DefaultEnvironment('test/end-to-end-tests/single-root-UI/project-folder', build_loc, exe_res); + if (process.platform === 'win32') { + // MSVC compiler does not produce gcov based coverage data + return this.skip(); + } + await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('useCMakePresets', 'always'); await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); @@ -50,8 +55,11 @@ suite('Coverage integration', () => { await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('postRunCoverageTarget', 'non-existing-target'); testResult = await vscode.commands.executeCommand('testing.coverage.uri', vscode.Uri.file(testEnv.projectFolder.location)); - expect(testResult['tasks'][0].hasCoverage).to.be.eq(false); - expect(testResult['items'][2].computedState).to.be.eq(TestResultState.Unset); + if (testResult !== undefined) { + // May or may not be undefined in this case evidently based on platform + expect(testResult['tasks'][0].hasCoverage).to.be.eq(false); + expect(testResult['items'][2].computedState).to.be.eq(TestResultState.Unset); + } }).timeout(60000); test('Good Run test with coverage', async () => { From e3bf947cfdc4fc3813ea0bea1fbb5feab4ca1b52 Mon Sep 17 00:00:00 2001 From: Tofik Sonono Date: Thu, 21 Nov 2024 20:57:31 +0100 Subject: [PATCH 14/14] Added changelog entry for the coverage feature --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5acc840ad..9f38056c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ Features: - Add support for Presets v9, which enables more macro expansion for the `include` field. [#3946](https://github.com/microsoft/vscode-cmake-tools/issues/3946) - Add support to configure default folder in workspace setting. [#1078](https://github.com/microsoft/vscode-cmake-tools/issues/1078) +- Add support for processing LCOV based coverage info files when tests are + executed. This adds test execution type, "Run with coverage", on the `ctest` + section of the Testing tab. + [#4040](https://github.com/microsoft/vscode-cmake-tools/issues/4040) Improvements: