diff --git a/.github/workflows/ci-main-linux.yml b/.github/workflows/ci-main-linux.yml index eb4fc8c26..276e88513 100644 --- a/.github/workflows/ci-main-linux.yml +++ b/.github/workflows/ci-main-linux.yml @@ -13,6 +13,9 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Setup LCOV + uses: hrishikesh-kadam/setup-lcov@v1 + - name: Setup Node environment uses: actions/setup-node@v3 with: diff --git a/.github/workflows/ci-main-mac.yml b/.github/workflows/ci-main-mac.yml index 96cde8f69..87ec28158 100644 --- a/.github/workflows/ci-main-mac.yml +++ b/.github/workflows/ci-main-mac.yml @@ -13,6 +13,9 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Setup LCOV + uses: hrishikesh-kadam/setup-lcov@v1 + - name: Setup Node environment uses: actions/setup-node@v3 with: @@ -43,8 +46,6 @@ jobs: configure-options: -DCMAKE_INSTALL_PREFIX:STRING=${{ github.workspace }}/test/fakebin install-build: true - - - name: Run successful-build test run: yarn endToEndTestsSuccessfulBuild diff --git a/.github/workflows/ci-main.win.yml b/.github/workflows/ci-main.win.yml index 26f42b19e..20b567c52 100644 --- a/.github/workflows/ci-main.win.yml +++ b/.github/workflows/ci-main.win.yml @@ -14,6 +14,9 @@ jobs: - name: Checkout source code uses: actions/checkout@v3 + - name: Setup LCOV + uses: hrishikesh-kadam/setup-lcov@v1 + - name: Setup Node environment uses: actions/setup-node@v3 with: diff --git a/CHANGELOG.md b/CHANGELOG.md index cac98e039..ec36d6660 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ Features: - Add support for Presets v9, which enables more macro expansion for the `include` field. [#3946](https://github.com/microsoft/vscode-cmake-tools/issues/3946) - Add support to configure default folder in workspace setting. [#1078](https://github.com/microsoft/vscode-cmake-tools/issues/1078) +- Add support for processing LCOV based coverage info files when tests are + executed. This adds test execution type, "Run with coverage", on the `ctest` + section of the Testing tab. + [#4040](https://github.com/microsoft/vscode-cmake-tools/issues/4040) Improvements: diff --git a/docs/cmake-settings.md b/docs/cmake-settings.md index af77856e2..0e23d6c17 100644 --- a/docs/cmake-settings.md +++ b/docs/cmake-settings.md @@ -38,6 +38,9 @@ Options that support substitution, in the table below, allow variable references | `cmake.saveBeforeBuild` | If `true` (the default), saves open text documents when build or configure is invoked before running CMake. | `true` | no | | `cmake.sourceDirectory` | A directory or a list of directories where the root `CMakeLists.txt`s are stored. | `${workspaceFolder}` | yes | | `cmake.testEnvironment` | An object containing `key:value` pairs of environment variables, which will be available when debugging, running and testing with CTest. | `null` (no environment variables) | yes | +| `cmake.preRunCoverageTarget` | Target to build before running tests with coverage using the test explorer | null | no | +| `cmake.postRunCoverageTarget` | Target to build after running tests with coverage using the test explorer | null | no | +| `cmake.coverageInfoFiles` | LCOV coverage info files to be processed after running tests with coverage using the test explorer | [] | yes | ## Variable substitution diff --git a/package.json b/package.json index 20bc2c09e..65afed899 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,7 @@ "syntaxes" ], "engines": { - "vscode": "^1.67.0" + "vscode": "^1.88.0" }, "categories": [ "Other", @@ -3589,6 +3589,27 @@ "default": true, "description": "%cmake-tools.configuration.cmake.enableAutomaticKitScan.description%", "scope": "resource" + }, + "cmake.preRunCoverageTarget": { + "type": "string", + "default": null, + "description": "%cmake-tools.configuration.cmake.preRunCoverageTarget.description%", + "scope": "resource" + }, + "cmake.postRunCoverageTarget": { + "type": "string", + "default": null, + "description": "%cmake-tools.configuration.cmake.postRunCoverageTarget.description%", + "scope": "resource" + }, + "cmake.coverageInfoFiles": { + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "description": "%cmake-tools.configuration.cmake.coverageInfoFiles.description%", + "scope": "resource" } } }, @@ -3716,7 +3737,7 @@ "@types/rimraf": "^3.0.0", "@types/sinon": "~9.0.10", "@types/tmp": "^0.2.0", - "@types/vscode": "1.63.0", + "@types/vscode": "1.88.0", "@types/which": "~2.0.0", "@types/xml2js": "^0.4.8", "@types/uuid": "~8.3.3", @@ -3758,6 +3779,7 @@ "webpack-cli": "^4.5.0" }, "dependencies": { + "@friedemannsommer/lcov-parser": "^4.0.1", "@types/string.prototype.matchall": "^4.0.4", "ajv": "^7.1.0", "chokidar": "^3.5.1", diff --git a/package.nls.json b/package.nls.json index 74ba6956c..ec986cdaa 100644 --- a/package.nls.json +++ b/package.nls.json @@ -273,6 +273,9 @@ "cmake-tools.configuration.cmake.automaticReconfigure.description": "Automatically configure CMake project directories when the kit or the configuration preset is changed.", "cmake-tools.configuration.cmake.pinnedCommands.description":"List of CMake commands to always pin by default.", "cmake-tools.configuration.cmake.enableAutomaticKitScan.description": "Enable automatic scanning for kits when a kit isn't selected. This will only take affect when CMake Presets aren't being used.", + "cmake-tools.configuration.cmake.preRunCoverageTarget.description": "Target to build before running tests with coverage using the test explorer", + "cmake-tools.configuration.cmake.postRunCoverageTarget.description": "Target to build after running tests with coverage using the test explorer", + "cmake-tools.configuration.cmake.coverageInfoFiles.description": "LCOV coverage info files to be processed after running tests with coverage using the test explorer.", "cmake-tools.debugger.pipeName.description": "Name of the pipe (on Windows) or domain socket (on Unix) to use for debugger communication.", "cmake-tools.debugger.clean.description": "Clean prior to configuring.", "cmake-tools.debugger.configureAll.description": "Configure for all projects.", diff --git a/src/config.ts b/src/config.ts index b65b19ae7..c2ec3021a 100644 --- a/src/config.ts +++ b/src/config.ts @@ -217,6 +217,9 @@ export interface ExtensionConfigurationSettings { automaticReconfigure: boolean; pinnedCommands: string[]; enableAutomaticKitScan: boolean; + preRunCoverageTarget: string | null; + postRunCoverageTarget: string | null; + coverageInfoFiles: string[]; } type EmittersOf = { @@ -565,6 +568,18 @@ export class ConfigurationReader implements vscode.Disposable { return this.configData.enableAutomaticKitScan; } + get preRunCoverageTarget(): string | null { + return this.configData.preRunCoverageTarget; + } + + get postRunCoverageTarget(): string | null { + return this.configData.postRunCoverageTarget; + } + + get coverageInfoFiles(): string[] { + return this.configData.coverageInfoFiles; + } + private readonly emitters: EmittersOf = { autoSelectActiveFolder: new vscode.EventEmitter(), defaultActiveFolder: new vscode.EventEmitter(), @@ -629,7 +644,10 @@ export class ConfigurationReader implements vscode.Disposable { launchBehavior: new vscode.EventEmitter(), automaticReconfigure: new vscode.EventEmitter(), pinnedCommands: new vscode.EventEmitter(), - enableAutomaticKitScan: new vscode.EventEmitter() + enableAutomaticKitScan: new vscode.EventEmitter(), + preRunCoverageTarget: new vscode.EventEmitter(), + postRunCoverageTarget: new vscode.EventEmitter(), + coverageInfoFiles: new vscode.EventEmitter() }; /** diff --git a/src/coverage.ts b/src/coverage.ts new file mode 100644 index 000000000..b4d4caabe --- /dev/null +++ b/src/coverage.ts @@ -0,0 +1,64 @@ +import * as vscode from 'vscode'; +import { lcovParser } from "@friedemannsommer/lcov-parser"; +import * as nls from 'vscode-nls'; +import * as logging from '@cmt/logging'; + +nls.config({ messageFormat: nls.MessageFormat.bundle, bundleFormat: nls.BundleFormat.standalone })(); +const localize: nls.LocalizeFunc = nls.loadMessageBundle(); + +const log = logging.createLogger('ctest-coverage'); + +export async function handleCoverageInfoFiles(run: vscode.TestRun, coverageInfoFiles: string[], coverageData: WeakMap) { + for (const coverageInfoFile of coverageInfoFiles) { + let contents: Uint8Array; + try { + contents = await vscode.workspace.fs.readFile(vscode.Uri.file(coverageInfoFile)); + } catch (e) { + log.warning(localize('test.openCoverageInfoFile', 'Could not open coverage info file: {0}. Skipping...', coverageInfoFile)); + return; + } + const sections = await lcovParser({ from: contents }); + for (const section of sections) { + const coverage = new vscode.FileCoverage(vscode.Uri.file(section.path), + new vscode.TestCoverageCount( + section.lines.hit, + section.lines.instrumented + ), new vscode.TestCoverageCount( + section.branches.hit, + section.branches.instrumented + ), new vscode.TestCoverageCount( + section.functions.hit, + section.functions.instrumented + )); + + const lineBranches = new Map(); + for (const branch of section.branches.details) { + const branchCoverage = new vscode.BranchCoverage(branch.hit, + new vscode.Position(branch.line - 1, 0), branch.branch); + + const curr = lineBranches.get(branch.line); + if (curr === undefined) { + lineBranches.set(branch.line, [branchCoverage]); + } else { + curr.push(branchCoverage); + lineBranches.set(branch.line, curr); + } + } + + const declarations: vscode.DeclarationCoverage[] = []; + for (const declaration of section.functions.details) { + declarations.push(new vscode.DeclarationCoverage(declaration.name, declaration.hit, + new vscode.Position(declaration.line - 1, 0))); + } + + const statements: vscode.StatementCoverage[] = []; + for (const line of section.lines.details) { + statements.push(new vscode.StatementCoverage(line.hit, + new vscode.Position(line.line - 1, 0), + lineBranches.get(line.line) ?? [])); + } + coverageData.set(coverage, [...statements, ...declarations]); + run.addCoverage(coverage); + } + } +} diff --git a/src/ctest.ts b/src/ctest.ts index c2fd575b4..455331782 100644 --- a/src/ctest.ts +++ b/src/ctest.ts @@ -15,6 +15,8 @@ import { expandString } from '@cmt/expand'; import * as proc from '@cmt/proc'; import { ProjectController } from '@cmt/projectController'; import { extensionManager } from '@cmt/extension'; +import { CMakeProject } from '@cmt/cmakeProject'; +import { handleCoverageInfoFiles } from '@cmt/coverage'; nls.config({ messageFormat: nls.MessageFormat.bundle, bundleFormat: nls.BundleFormat.standalone })(); const localize: nls.LocalizeFunc = nls.loadMessageBundle(); @@ -118,6 +120,18 @@ interface CTestInfo { version: { major: number; minor: number }; } +/* The preRunCoverageTarget and postRunCoverageTarget are optional user +configured targets that are run before and after the tests when a "Run with +Coverage" test execution is triggered. These cae be used to zero the +coverage counters, filter coverage results etc. */ +interface ProjectCoverageConfig { + project: CMakeProject; + driver: CMakeDriver; + preRunCoverageTarget: string | null; + postRunCoverageTarget: string | null; + coverageInfoFiles: string[]; +} + function parseXmlString(xml: string): Promise { return new Promise((resolve, reject) => { xml2js.parseString(xml, (err, result) => { @@ -229,6 +243,8 @@ export class CTestDriver implements vscode.Disposable { private readonly testingEnabledEmitter = new vscode.EventEmitter(); readonly onTestingEnabledChanged = this.testingEnabledEmitter.event; + private coverageData = new WeakMap(); + dispose() { this.testingEnabledEmitter.dispose(); this.testsChangedEmitter.dispose(); @@ -710,7 +726,7 @@ export class CTestDriver implements vscode.Disposable { return -1; } - if (util.isTestMode()) { + if (util.isTestMode() && !util.overrideTestModeForTestExplorer()) { // ProjectController can't be initialized in test mode, so we don't have a usable test explorer return 0; } @@ -905,7 +921,65 @@ export class CTestDriver implements vscode.Disposable { return true; } - private async runTestHandler(request: vscode.TestRunRequest, cancellation: vscode.CancellationToken) { + private async handleCoverageOnProjects(run: vscode.TestRun, projectCoverageConfigs: ProjectCoverageConfig[]) { + // Currently only LCOV coverage info files are supported + for (const projectCoverageConfig of projectCoverageConfigs) { + if (projectCoverageConfig.coverageInfoFiles.length === 0) { + log.warning(localize('test.noCoverageInfoFiles', 'No coverage info files for CMake project {0}. No coverage data will be analyzed for this project.', projectCoverageConfig.project.sourceDir)); + continue; + } + const expandedCoverageInfoFiles = await Promise.all(projectCoverageConfig.coverageInfoFiles.map(async (coverageInfoFile: string) => expandString(coverageInfoFile, projectCoverageConfig.driver.expansionOptions))); + await handleCoverageInfoFiles(run, expandedCoverageInfoFiles, this.coverageData); + } + } + + private async coverageCTestHelper(tests: vscode.TestItem[], run: vscode.TestRun, cancellation: vscode.CancellationToken): Promise { + const projectRoots = new Set(); + for (const test of tests) { + projectRoots.add(this.getTestRootFolder(test)); + } + + const projectCoverageConfigs: ProjectCoverageConfig[] = []; + for (const folder of projectRoots) { + const project = await this.projectController?.getProjectForFolder(folder); + if (project) { + const driver = await project.getCMakeDriverInstance(); + if (driver) { + projectCoverageConfigs.push({ + project: project, driver: driver, preRunCoverageTarget: driver.config.preRunCoverageTarget, postRunCoverageTarget: driver.config.postRunCoverageTarget, coverageInfoFiles: driver.config.coverageInfoFiles + }); + } + } + } + + for (const projectCoverageConfig of projectCoverageConfigs) { + if (projectCoverageConfig.preRunCoverageTarget) { + log.info(localize('test.buildingPreRunCoverageTarget', 'Building the preRunCoverageTarget for project {0} before running tests with coverage.', projectCoverageConfig.project.sourceDir)); + const rc = await projectCoverageConfig.project.build([projectCoverageConfig.preRunCoverageTarget]); + if (rc !== 0) { + log.error(localize('test.preRunCoverageTargetFailure', 'Building the preRunCoverageTarget \'{0}\' on project in {1} failed. Skipping running tests.', projectCoverageConfig.preRunCoverageTarget, projectCoverageConfig.project.sourceDir)); + run.end(); + return rc; + } + } + } + const runResult = await this.runCTestHelper(tests, run, cancellation, undefined, undefined, undefined, false, undefined, RunCTestHelperEntryPoint.TestExplorer); + for (const projectCoverageConfig of projectCoverageConfigs) { + if (projectCoverageConfig.postRunCoverageTarget) { + log.info(localize('test.buildingPostRunCoverageTarget', 'Building the postRunCoverageTarget \'{0}\' for project {1} after the tests have run with coverage.', projectCoverageConfig.postRunCoverageTarget, projectCoverageConfig.project.sourceDir)); + const rc = await projectCoverageConfig.project.build([projectCoverageConfig.postRunCoverageTarget]); + if (rc !== 0) { + log.error(localize('test.postRunCoverageTargetFailure', 'Building target postRunCoverageTarget on project in {0} failed. Skipping handling of coverage data.', projectCoverageConfig.project.sourceDir)); + return rc; + } + } + } + + await this.handleCoverageOnProjects(run, projectCoverageConfigs); + return runResult; + } + + private async runTestHandler(request: vscode.TestRunRequest, cancellation: vscode.CancellationToken, isCoverageRun = false) { // NOTE: We expect the testExplorer to be undefined when the cmake.ctest.testExplorerIntegrationEnabled is disabled. if (!testExplorer) { return; @@ -922,7 +996,12 @@ export class CTestDriver implements vscode.Disposable { this.ctestsEnqueued(tests, run); const buildSucceeded = await this.buildTests(tests, run); if (buildSucceeded) { - await this.runCTestHelper(tests, run, cancellation, undefined, undefined, undefined, false, undefined, RunCTestHelperEntryPoint.TestExplorer); + if (isCoverageRun) { + await this.coverageCTestHelper(tests, run, cancellation); + + } else { + await this.runCTestHelper(tests, run, cancellation, undefined, undefined, undefined, false, undefined, RunCTestHelperEntryPoint.TestExplorer); + } } else { log.info(localize('test.skip.run.build.failure', "Not running tests due to build failure.")); } @@ -1231,6 +1310,12 @@ export class CTestDriver implements vscode.Disposable { (request: vscode.TestRunRequest, cancellation: vscode.CancellationToken) => this.runTestHandler(request, cancellation), true ); + testExplorer.createRunProfile( + 'Run Tests with Coverage', + vscode.TestRunProfileKind.Coverage, + (request: vscode.TestRunRequest, cancellation: vscode.CancellationToken) => this.runTestHandler(request, cancellation, true), + true + ).loadDetailedCoverage = async (_, fileCoverage) => this.coverageData.get(fileCoverage) ?? []; testExplorer.createRunProfile( 'Debug Tests', vscode.TestRunProfileKind.Debug, diff --git a/src/util.ts b/src/util.ts index 0145a93b1..76078d887 100644 --- a/src/util.ts +++ b/src/util.ts @@ -835,6 +835,13 @@ export function isTestMode(): boolean { return process.env['CMT_TESTING'] === '1'; } +/** + * Returns true if the test explorer should be enabled even when in test mode. + */ +export function overrideTestModeForTestExplorer(): boolean { + return process.env['CMT_TESTING_OVERRIDE_TEST_EXPLORER'] === '1'; +} + export async function getAllCMakeListsPaths(path: string): Promise { const regex: RegExp = new RegExp(/(\/|\\)CMakeLists\.txt$/); return recGetAllFilePaths(path, regex, await readDir(path), []); diff --git a/test/end-to-end-tests/single-root-UI/project-folder/.vscode/settings.json b/test/end-to-end-tests/single-root-UI/project-folder/.vscode/settings.json index fc6a75163..69dd2297f 100644 --- a/test/end-to-end-tests/single-root-UI/project-folder/.vscode/settings.json +++ b/test/end-to-end-tests/single-root-UI/project-folder/.vscode/settings.json @@ -1,5 +1,10 @@ { "cmake.buildDirectory": "${workspaceFolder}/build", "cmake.useCMakePresets": "never", - "cmake.configureOnOpen": false + "cmake.configureOnOpen": false, + "cmake.preRunCoverageTarget": "init-coverage", + "cmake.postRunCoverageTarget": "capture-coverage", + "cmake.coverageInfoFiles": [ + "${workspaceFolder}/build/lcov.info" + ] } \ No newline at end of file diff --git a/test/end-to-end-tests/single-root-UI/project-folder/CMakeLists.txt b/test/end-to-end-tests/single-root-UI/project-folder/CMakeLists.txt index dff0b8e57..9a2d2ca14 100644 --- a/test/end-to-end-tests/single-root-UI/project-folder/CMakeLists.txt +++ b/test/end-to-end-tests/single-root-UI/project-folder/CMakeLists.txt @@ -1,11 +1,22 @@ -cmake_minimum_required(VERSION 3.0.0) +cmake_minimum_required(VERSION 3.5.0) project(TestBuildProcess VERSION 0.1.0) set(CMT_COOKIE passed-cookie CACHE STRING "Cookie to be written by the main executable") +option(USE_COVERAGE "Enable GCOV during the build" OFF) +if(USE_COVERAGE) + add_compile_options(--coverage) + add_link_options(--coverage) +endif() + +include(CTest) + add_executable(TestBuildProcess main.cpp) set_property(TARGET TestBuildProcess PROPERTY CXX_STANDARD 98) +add_test(NAME TestBuildProcessAsTest + COMMAND $) + add_custom_command( TARGET TestBuildProcess POST_BUILD @@ -36,3 +47,34 @@ add_custom_target(runTestTarget DEPENDS TestBuildProcess WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} COMMENT "Run test target" ) + +find_program(LCOV lcov REQUIRED) +set(LCOV_BASE lcov-base.info) +set(LCOV_TEST lcov-test.info) +set(LCOV_TOTAL lcov.info) +set(LCOV_LOG lcov.log) +set(LCOV_ERR lcov.err) +add_custom_target(init-coverage + COMMENT "Collecting initial coverage" + COMMAND lcov -c -i -d ${CMAKE_CURRENT_BINARY_DIR} + -o ${LCOV_BASE} 2>${LCOV_ERR} >${LCOV_LOG}) +add_dependencies(init-coverage reset-coverage) + +add_custom_target(reset-coverage + COMMENT "Reset all coverage counters to zero" + COMMAND lcov -q -z -d ${CMAKE_CURRENT_BINARY_DIR} + -o ${LCOV_BASE} + COMMAND lcov -q -z -d ${CMAKE_CURRENT_BINARY_DIR} + -o ${LCOV_TEST} + COMMAND lcov -q -z -d ${CMAKE_CURRENT_BINARY_DIR} + -o ${LCOV_TOTAL}) + +add_custom_target(capture-coverage + COMMENT "Capture coverage data" + DEPENDS ${LCOV_BASE} + COMMAND lcov -c -d ${CMAKE_CURRENT_BINARY_DIR} -o ${LCOV_TEST} + 2>${LCOV_ERR} >${LCOV_LOG} + COMMAND lcov -a ${LCOV_BASE} -a ${LCOV_TEST} -o ${LCOV_TOTAL} + >>${LCOV_LOG} + COMMAND lcov -r ${LCOV_TOTAL} -o ${LCOV_TOTAL} "'/usr/include*'" + >>${LCOV_LOG}) diff --git a/test/end-to-end-tests/single-root-UI/project-folder/CMakePresets.json b/test/end-to-end-tests/single-root-UI/project-folder/CMakePresets.json index e78e95aae..f049729a3 100644 --- a/test/end-to-end-tests/single-root-UI/project-folder/CMakePresets.json +++ b/test/end-to-end-tests/single-root-UI/project-folder/CMakePresets.json @@ -5,12 +5,12 @@ "name": "Linux1", "description": "Sets generator, build and install directory, vcpkg", "generator": "Ninja", - "binaryDir": "${workspaceFolder}/build", + "binaryDir": "${sourceDir}/build", "cacheVariables": { "CMAKE_BUILD_TYPE": "Debug", "CMAKE_TOOLCHAIN_FILE": { "type": "FILEPATH", - "value": "${workspaceFolder}/test-toolchain.cmake" + "value": "${sourceDir}/test-toolchain.cmake" } } } diff --git a/test/end-to-end-tests/single-root-UI/project-folder/CMakeUserPresets.json b/test/end-to-end-tests/single-root-UI/project-folder/CMakeUserPresets.json index e85796119..b9f11c5e7 100644 --- a/test/end-to-end-tests/single-root-UI/project-folder/CMakeUserPresets.json +++ b/test/end-to-end-tests/single-root-UI/project-folder/CMakeUserPresets.json @@ -8,7 +8,7 @@ "name": "LinuxUser1", "description": "Sets generator, build and install directory, vcpkg", "generator": "Unix Makefiles", - "binaryDir": "${workspaceFolder}/build", + "binaryDir": "${sourceDir}/build", "cacheVariables": { "CMAKE_BUILD_TYPE": "Debug", "CMAKE_C_COMPILER": "gcc", @@ -22,7 +22,7 @@ "name": "WindowsUser1", "description": "Sets generator, build and install directory, vcpkg", "generator": "Visual Studio 16 2019", - "binaryDir": "${workspaceFolder}/build", + "binaryDir": "${sourceDir}/build", "cacheVariables": { "CMAKE_BUILD_TYPE": "Debug", "CMAKE_C_COMPILER": "cl.exe", @@ -35,7 +35,7 @@ { "name": "NoGenerator", "description": "Skips setting a generator", - "binaryDir": "${workspaceFolder}/build", + "binaryDir": "${sourceDir}/build", "cacheVariables": { "CMAKE_BUILD_TYPE": "Debug", "CMAKE_C_COMPILER": "gcc", @@ -48,6 +48,14 @@ { "name": "TestInheritFromPreset", "inherits": "Linux1" + }, + { + "name": "LinuxUser2", + "description": "LinuxUser1 with Coverage enabled", + "inherits": "LinuxUser1", + "cacheVariables": { + "USE_COVERAGE": "ON" + } } ] } diff --git a/test/end-to-end-tests/single-root-UI/project-folder/include/IncludedPresets1.json b/test/end-to-end-tests/single-root-UI/project-folder/include/IncludedPresets1.json index e0dfc3f3d..f0b3b74ca 100644 --- a/test/end-to-end-tests/single-root-UI/project-folder/include/IncludedPresets1.json +++ b/test/end-to-end-tests/single-root-UI/project-folder/include/IncludedPresets1.json @@ -8,7 +8,7 @@ "name": "LinuxIncluded1", "description": "Sets generator, build and install directory, vcpkg", "generator": "Unix Makefiles", - "binaryDir": "${workspaceFolder}/build", + "binaryDir": "${sourceDir}/build", "cacheVariables": { "CMAKE_BUILD_TYPE": "Debug", "CMAKE_C_COMPILER": "gcc", diff --git a/test/end-to-end-tests/single-root-UI/project-folder/include/IncludedPresets2.json b/test/end-to-end-tests/single-root-UI/project-folder/include/IncludedPresets2.json index 8faab3d3b..fc3ef1704 100644 --- a/test/end-to-end-tests/single-root-UI/project-folder/include/IncludedPresets2.json +++ b/test/end-to-end-tests/single-root-UI/project-folder/include/IncludedPresets2.json @@ -5,7 +5,7 @@ "name": "LinuxIncluded2", "description": "Sets generator, build and install directory, vcpkg", "generator": "Unix Makefiles", - "binaryDir": "${workspaceFolder}/build", + "binaryDir": "${sourceDir}/build", "cacheVariables": { "CMAKE_BUILD_TYPE": "Debug", "CMAKE_C_COMPILER": "gcc", diff --git a/test/end-to-end-tests/single-root-UI/runTest.ts b/test/end-to-end-tests/single-root-UI/runTest.ts index 40e633c8c..e92b01567 100644 --- a/test/end-to-end-tests/single-root-UI/runTest.ts +++ b/test/end-to-end-tests/single-root-UI/runTest.ts @@ -19,6 +19,7 @@ async function main() { const extensionTestsEnv: { [key: string]: string | undefined } = { "CMT_TESTING": "1", "CMT_QUIET_CONSOLE": "1", + "CMT_TESTING_OVERRIDE_TEST_EXPLORER": "1", "TEST_FILTER": process.env.TEST_FILTER ?? ".*" }; diff --git a/test/end-to-end-tests/single-root-UI/test/coverage.test.ts b/test/end-to-end-tests/single-root-UI/test/coverage.test.ts new file mode 100644 index 000000000..10bc54682 --- /dev/null +++ b/test/end-to-end-tests/single-root-UI/test/coverage.test.ts @@ -0,0 +1,74 @@ +import { DefaultEnvironment, expect } from '@test/util'; +import * as vscode from 'vscode'; +import * as fs from 'fs'; +import * as path from 'path'; + +// From vscode: src/vs/workbench/contrib/testing/common/testTypes.ts +const enum TestResultState { + Unset = 0, + Queued = 1, + Running = 2, + Passed = 3, + Failed = 4, + Skipped = 5, + Errored = 6 +}; + +suite('Coverage integration', () => { + let testEnv: DefaultEnvironment; + + suiteSetup(async function (this: Mocha.Context) { + this.timeout(100000); + + const build_loc = 'build'; + const exe_res = 'output.txt'; + + testEnv = new DefaultEnvironment('test/end-to-end-tests/single-root-UI/project-folder', build_loc, exe_res); + + if (process.platform === 'win32') { + // MSVC compiler does not produce gcov based coverage data + return this.skip(); + } + + await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('useCMakePresets', 'always'); + await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); + + testEnv.projectFolder.buildDirectory.clear(); + await vscode.commands.executeCommand('cmake.setConfigurePreset', 'LinuxUser2'); + expect(await vscode.commands.executeCommand('cmake.configure')).to.be.eq(0); + expect(await vscode.commands.executeCommand('cmake.build')).to.be.eq(0); + }); + + suiteTeardown(async function (this: Mocha.Context) { + this.timeout(30000); + testEnv.teardown(); + }); + + test('Bad Run test with coverage', async () => { + await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('preRunCoverageTarget', 'non-existing-target'); + + let testResult: any = await vscode.commands.executeCommand('testing.coverage.uri', vscode.Uri.file(testEnv.projectFolder.location)); + expect(testResult['tasks'][0].hasCoverage).to.be.eq(false); + expect(testResult['items'][2].computedState).to.be.eq(TestResultState.Unset); + + await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('preRunCoverageTarget', 'init-target'); + await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('postRunCoverageTarget', 'non-existing-target'); + + testResult = await vscode.commands.executeCommand('testing.coverage.uri', vscode.Uri.file(testEnv.projectFolder.location)); + if (testResult !== undefined) { + // May or may not be undefined in this case evidently based on platform + expect(testResult['tasks'][0].hasCoverage).to.be.eq(false); + expect(testResult['items'][2].computedState).to.be.eq(TestResultState.Unset); + } + }).timeout(60000); + + test('Good Run test with coverage', async () => { + await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('preRunCoverageTarget', 'init-coverage'); + await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('postRunCoverageTarget', 'capture-coverage'); + + const testResult: any = await vscode.commands.executeCommand('testing.coverage.uri', vscode.Uri.file(testEnv.projectFolder.location)); + expect(testResult['tasks'][0].hasCoverage).to.be.eq(true); + expect(testResult['items'][2].computedState).to.be.eq(TestResultState.Passed); + expect(fs.existsSync(path.join(testEnv.projectFolder.location, testEnv.buildLocation, 'lcov.info'))).to.be.true; + }).timeout(60000); +}); diff --git a/test/helpers/vscodefake/extensioncontext.ts b/test/helpers/vscodefake/extensioncontext.ts index 784f24727..432410617 100644 --- a/test/helpers/vscodefake/extensioncontext.ts +++ b/test/helpers/vscodefake/extensioncontext.ts @@ -17,7 +17,7 @@ export class DefaultExtensionContext implements vscode.ExtensionContext { get extensionPath(): string { throw new Error(notImplementedErr); } - get environmentVariableCollection(): vscode.EnvironmentVariableCollection { + get environmentVariableCollection(): vscode.GlobalEnvironmentVariableCollection { throw new Error(notImplementedErr); } asAbsolutePath(relativePath: string): string { @@ -61,7 +61,7 @@ export class SmokeTestExtensionContext implements vscode.ExtensionContext { get extensionUri(): vscode.Uri { throw new Error(notImplementedErr); } - get environmentVariableCollection(): vscode.EnvironmentVariableCollection { + get environmentVariableCollection(): vscode.GlobalEnvironmentVariableCollection { throw new Error(notImplementedErr); } asAbsolutePath(sub: string): string { diff --git a/test/unit-tests/config.test.ts b/test/unit-tests/config.test.ts index a2d04c04a..e3c27d97c 100644 --- a/test/unit-tests/config.test.ts +++ b/test/unit-tests/config.test.ts @@ -76,7 +76,10 @@ function createConfig(conf: Partial): Configurat launchBehavior: 'reuseTerminal', ignoreCMakeListsMissing: false, automaticReconfigure: false, - enableAutomaticKitScan: true + enableAutomaticKitScan: true, + preRunCoverageTarget: null, + postRunCoverageTarget: null, + coverageInfoFiles: [] }); ret.updatePartial(conf); return ret; diff --git a/test/unit-tests/coverage.test.ts b/test/unit-tests/coverage.test.ts new file mode 100644 index 000000000..c7f766a7c --- /dev/null +++ b/test/unit-tests/coverage.test.ts @@ -0,0 +1,41 @@ +import { handleCoverageInfoFiles } from "@cmt/coverage"; +import * as vscode from "vscode"; +import { expect, getTestResourceFilePath } from "@test/util"; +import * as path from "path"; + +suite('Coverage Handling', () => { + + test('Coverage Info (LCOV)', async () => { + const filesCoverages: vscode.FileCoverage[] = []; + const testRun: vscode.TestRun = { + name: '', + token: {} as vscode.CancellationToken, + isPersisted: false, + enqueued: (_test: vscode.TestItem) => {}, + started: (_test: vscode.TestItem) => {}, + passed: (_test: vscode.TestItem, _duration: number) => {}, + failed: (_test: vscode.TestItem, _message: vscode.TestMessage | vscode.TestMessage[], _duration: number) => {}, + skipped: (_test: vscode.TestItem) => {}, + errored: (_test: vscode.TestItem, _message: vscode.TestMessage | vscode.TestMessage[], _duration: number) => {}, + appendOutput: (_output: string, _location?: vscode.Location, _test?: vscode.TestItem) => {}, + addCoverage: (fileCoverage: vscode.FileCoverage) => { + filesCoverages.push(fileCoverage); + }, + end: () => {}, + onDidDispose: (_listener: () => void): vscode.Disposable => new vscode.Disposable(() => {}) + }; + + const coverageData = new WeakMap(); + await handleCoverageInfoFiles(testRun, [getTestResourceFilePath('lcov.info')], coverageData); + expect(filesCoverages.length).to.eq(1); + expect(filesCoverages[0].uri.fsPath).to.eq(path.join(path.sep, 'tmp', 'lcov', 'main.cpp')); + const coverageDetail = coverageData.get(filesCoverages[0]); + expect(coverageDetail).to.not.be.undefined; + + expect(coverageDetail![0].executed).to.eq(1); + expect((coverageDetail![0].location as vscode.Position).line).to.eq(2); + + expect(coverageDetail![1].executed).to.eq(1); + expect((coverageDetail![1].location as vscode.Position).line).to.eq(3); + }); +}); diff --git a/test/unit-tests/lcov.info b/test/unit-tests/lcov.info new file mode 100644 index 000000000..fb7a8349c --- /dev/null +++ b/test/unit-tests/lcov.info @@ -0,0 +1,16 @@ +TN: +SF:/tmp/lcov/main.cpp +FN:3,4,main +FN:5,5,_GLOBAL__sub_I_main +FN:5,5,_Z41__static_initialization_and_destruction_0ii +FNDA:1,main +FNDA:1,_GLOBAL__sub_I_main +FNDA:1,_Z41__static_initialization_and_destruction_0ii +FNF:3 +FNH:3 +DA:3,1 +DA:4,1 +DA:5,4 +LF:3 +LH:3 +end_of_record diff --git a/yarn.lock b/yarn.lock index 8e311934b..f3cf9a517 100644 --- a/yarn.lock +++ b/yarn.lock @@ -165,6 +165,11 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" +"@friedemannsommer/lcov-parser@^4.0.1": + version "4.0.1" + resolved "https://pkgs.dev.azure.com/azure-public/VisualCpp/_packaging/cpp_PublicPackages/npm/registry/@friedemannsommer/lcov-parser/-/lcov-parser-4.0.1.tgz#192445df2095bb061fd889ea0b0f0f94bf7c58b7" + integrity sha512-NjXrxLeh77FUAL6TzVJyvYvgA2AS+t2hlfRmfOyE5HR3g+2HdLNFTP2l+zTypn5JTnV4jfXgwSO46JLhvJeXlw== + "@gulp-sourcemaps/identity-map@^2.0.1": version "2.0.1" resolved "https://pkgs.dev.azure.com/azure-public/VisualCpp/_packaging/cpp_PublicPackages/npm/registry/@gulp-sourcemaps/identity-map/-/identity-map-2.0.1.tgz" @@ -642,10 +647,10 @@ resolved "https://pkgs.dev.azure.com/azure-public/VisualCpp/_packaging/cpp_PublicPackages/npm/registry/@types/uuid/-/uuid-8.3.4.tgz" integrity sha512-c/I8ZRb51j+pYGAu5CrFMRxqZ2ke4y2grEBO5AUjgSkSk+qT2Ea+OdWElz/OiMf5MNpn2b17kuVBwZLQJXzihw== -"@types/vscode@1.63.0": - version "1.63.0" - resolved "https://pkgs.dev.azure.com/azure-public/VisualCpp/_packaging/cpp_PublicPackages/npm/registry/@types/vscode/-/vscode-1.63.0.tgz" - integrity sha512-iePu1axOi5WSThV6l2TYcciBIpAlMarjBC8H0y8L8ocsZLxh7MttzwFU3pjoItF5fRVGxHS0Hsvje9jO3yJsfw== +"@types/vscode@1.88.0": + version "1.88.0" + resolved "https://pkgs.dev.azure.com/azure-public/VisualCpp/_packaging/cpp_PublicPackages/npm/registry/@types/vscode/-/vscode-1.88.0.tgz#2dc690237f7ef049942508c8609b6b9f5216b4d3" + integrity sha512-rWY+Bs6j/f1lvr8jqZTyp5arRMfovdxolcqGi+//+cPDOh8SBvzXH90e7BiSXct5HJ9HGW6jATchbRTpTJpEkw== "@types/which@~2.0.0": version "2.0.1"