From a321e5d2f7688a7c9ce92144033d930ae4764937 Mon Sep 17 00:00:00 2001 From: Tofik Sonono Date: Wed, 4 Sep 2024 15:18:05 +0200 Subject: [PATCH] Native test coverage implementation Test coverage implementation for the CTest test controller. It relies on lcov coverage info files being specefied by the user in the settings.json of the project. Optionally, the user can specify CMake (utility) targets that should be built before and/or after the tests are/have been executed. These targets could reasonably zero the coverage counters (pre) and filter the coverage info files (post). --- docs/cmake-settings.md | 3 + package.json | 21 +++++ package.nls.json | 3 + src/config.ts | 20 ++++- src/ctest.ts | 139 ++++++++++++++++++++++++++++++++- test/unit-tests/config.test.ts | 5 +- 6 files changed, 187 insertions(+), 4 deletions(-) diff --git a/docs/cmake-settings.md b/docs/cmake-settings.md index e6b625833..ff82c393a 100644 --- a/docs/cmake-settings.md +++ b/docs/cmake-settings.md @@ -37,6 +37,9 @@ Options that support substitution, in the table below, allow variable references | `cmake.saveBeforeBuild` | If `true` (the default), saves open text documents when build or configure is invoked before running CMake. | `true` | no | | `cmake.sourceDirectory` | A directory or a list of directories where the root `CMakeLists.txt`s are stored. | `${workspaceFolder}` | yes | | `cmake.testEnvironment` | An object containing `key:value` pairs of environment variables, which will be available when debugging, running and testing with CTest. | `null` (no environment variables) | yes | +| `cmake.preRunCoverageTarget` | Target to build before running tests with coverage using the test explorer | null | no | +| `cmake.postRunCoverageTarget` | Target to build after running tests with coverage using the test explorer | null | no | +| `cmake.coverageInfoFiles` | LCOV coverage info files to be processed after running tests with coverage using the test explorer | [] | yes | ## Variable substitution diff --git a/package.json b/package.json index 6a62272fa..3aaab48fb 100644 --- a/package.json +++ b/package.json @@ -3582,6 +3582,27 @@ "default": true, "description": "%cmake-tools.configuration.cmake.enableAutomaticKitScan.description%", "scope": "resource" + }, + "cmake.preRunCoverageTarget": { + "type": "string", + "default": null, + "description": "%cmake-tools.configuration.cmake.preRunCoverageTarget.description%", + "scope": "resource" + }, + "cmake.postRunCoverageTarget": { + "type": "string", + "default": null, + "description": "%cmake-tools.configuration.cmake.postRunCoverageTarget.description%", + "scope": "resource" + }, + "cmake.coverageInfoFiles": { + "type": "array", + "items": { + "type": "string" + }, + "default": [], + "description": "%cmake-tools.configuration.cmake.coverageInfoFiles.description%", + "scope": "resource" } } }, diff --git a/package.nls.json b/package.nls.json index 911a453d1..1e59c67a2 100644 --- a/package.nls.json +++ b/package.nls.json @@ -273,6 +273,9 @@ "cmake-tools.configuration.cmake.automaticReconfigure.description": "Automatically configure CMake project directories when the kit or the configuration preset is changed.", "cmake-tools.configuration.cmake.pinnedCommands.description":"List of CMake commands to pin.", "cmake-tools.configuration.cmake.enableAutomaticKitScan.description": "Enable automatic scanning for kits when a kit isn't selected. This will only take affect when CMake Presets aren't being used.", + "cmake-tools.configuration.cmake.preRunCoverageTarget.description": "Target to build before running tests with coverage using the test explorer", + "cmake-tools.configuration.cmake.postRunCoverageTarget.description": "Target to build after running tests with coverage using the test explorer", + "cmake-tools.configuration.cmake.coverageInfoFiles.description": "LCOV coverage info files to be processed after running tests with coverage using the test explorer.", "cmake-tools.debugger.pipeName.description": "Name of the pipe (on Windows) or domain socket (on Unix) to use for debugger communication.", "cmake-tools.debugger.clean.description": "Clean prior to configuring.", "cmake-tools.debugger.configureAll.description": "Configure for all projects.", diff --git a/src/config.ts b/src/config.ts index ee4aaa785..546671c9b 100644 --- a/src/config.ts +++ b/src/config.ts @@ -217,6 +217,9 @@ export interface ExtensionConfigurationSettings { automaticReconfigure: boolean; pinnedCommands: string[]; enableAutomaticKitScan: boolean; + preRunCoverageTarget: string | null; + postRunCoverageTarget: string | null; + coverageInfoFiles: string[]; } type EmittersOf = { @@ -562,6 +565,18 @@ export class ConfigurationReader implements vscode.Disposable { return this.configData.enableAutomaticKitScan; } + get preRunCoverageTarget(): string | null { + return this.configData.preRunCoverageTarget; + } + + get postRunCoverageTarget(): string | null { + return this.configData.postRunCoverageTarget; + } + + get coverageInfoFiles(): string[] { + return this.configData.coverageInfoFiles; + } + private readonly emitters: EmittersOf = { autoSelectActiveFolder: new vscode.EventEmitter(), cmakePath: new vscode.EventEmitter(), @@ -626,7 +641,10 @@ export class ConfigurationReader implements vscode.Disposable { launchBehavior: new vscode.EventEmitter(), automaticReconfigure: new vscode.EventEmitter(), pinnedCommands: new vscode.EventEmitter(), - enableAutomaticKitScan: new vscode.EventEmitter() + enableAutomaticKitScan: new vscode.EventEmitter(), + preRunCoverageTarget: new vscode.EventEmitter(), + postRunCoverageTarget: new vscode.EventEmitter(), + coverageInfoFiles: new vscode.EventEmitter() }; /** diff --git a/src/ctest.ts b/src/ctest.ts index 3df61efb3..98309a082 100644 --- a/src/ctest.ts +++ b/src/ctest.ts @@ -15,6 +15,8 @@ import { expandString } from './expand'; import * as proc from '@cmt/proc'; import { ProjectController } from './projectController'; import { extensionManager } from './extension'; +import { lcovParser } from "@friedemannsommer/lcov-parser"; +import CMakeProject from './cmakeProject'; nls.config({ messageFormat: nls.MessageFormat.bundle, bundleFormat: nls.BundleFormat.standalone })(); const localize: nls.LocalizeFunc = nls.loadMessageBundle(); @@ -116,6 +118,14 @@ interface CTestInfo { version: { major: number; minor: number }; } +interface ProjectCoverageConfig { + project: CMakeProject; + driver: CMakeDriver; + preRunCoverageTarget: string | null; + postRunCoverageTarget: string | null; + coverageInfoFiles: string[]; +} + function parseXmlString(xml: string): Promise { return new Promise((resolve, reject) => { xml2js.parseString(xml, (err, result) => { @@ -227,6 +237,8 @@ export class CTestDriver implements vscode.Disposable { private readonly testingEnabledEmitter = new vscode.EventEmitter(); readonly onTestingEnabledChanged = this.testingEnabledEmitter.event; + private coverageData = new WeakMap(); + dispose() { this.testingEnabledEmitter.dispose(); this.testsChangedEmitter.dispose(); @@ -839,7 +851,119 @@ export class CTestDriver implements vscode.Disposable { return true; } - private async runTestHandler(request: vscode.TestRunRequest, cancellation: vscode.CancellationToken) { + private async handleCoverageOnProjects(run: vscode.TestRun, projectCoverageConfigs: ProjectCoverageConfig[]) { + for (const projectCoverageConfig of projectCoverageConfigs) { + if (projectCoverageConfig.coverageInfoFiles.length === 0) { + log.warning(localize('test.noCoverageInfoFiles', 'No coverage info files for CMake project {0}. No coverage data will be analyzed for this project.', projectCoverageConfig.project.sourceDir)); + continue; + } + const expandedCoverageInfoFiles = await Promise.all(projectCoverageConfig.coverageInfoFiles.map(async (coverageInfoFile: string) => expandString(coverageInfoFile, projectCoverageConfig.driver.expansionOptions))); + await this.handleCoverageData(run, expandedCoverageInfoFiles); + } + } + + private async handleCoverageData(run: vscode.TestRun, coverageInfoFiles: string[]) { + for (const coverageInfoFile of coverageInfoFiles) { + let contents: Uint8Array; + try { + contents = await vscode.workspace.fs.readFile(vscode.Uri.file(coverageInfoFile)); + } catch (e) { + log.warning(localize('test.openCoverageInfoFile', 'Could not open coverage info file: {0}. Skipping...', coverageInfoFile)); + return; + } + const sections = await lcovParser({ from: contents }); + for (const section of sections) { + const coverage = new vscode.FileCoverage(vscode.Uri.file(section.path), + new vscode.TestCoverageCount( + section.lines.hit, + section.lines.instrumented + ), new vscode.TestCoverageCount( + section.branches.hit, + section.branches.instrumented + ), new vscode.TestCoverageCount( + section.functions.hit, + section.functions.instrumented + )); + + const lineBranches = new Map(); + for (const branch of section.branches.details) { + const branchCoverage = new vscode.BranchCoverage(branch.hit, + new vscode.Position(branch.line - 1, 0), branch.branch); + + const curr = lineBranches.get(branch.line); + if (curr === undefined) { + lineBranches.set(branch.line, [branchCoverage]); + } else { + curr.push(branchCoverage); + lineBranches.set(branch.line, curr); + } + } + + const declarations: vscode.DeclarationCoverage[] = []; + for (const declaration of section.functions.details) { + declarations.push(new vscode.DeclarationCoverage(declaration.name, declaration.hit, + new vscode.Position(declaration.line - 1, 0))); + } + + const statements: vscode.StatementCoverage[] = []; + for (const line of section.lines.details) { + statements.push(new vscode.StatementCoverage(line.hit, + new vscode.Position(line.line - 1, 0), + lineBranches.get(line.line) ?? [])); + } + this.coverageData.set(coverage, [...statements, ...declarations]); + run.addCoverage(coverage); + } + } + } + + private async coverageCTestHelper(tests: vscode.TestItem[], run: vscode.TestRun, cancellation?: vscode.CancellationToken): Promise { + const projectRoots = new Set(); + for (const test of tests) { + projectRoots.add(this.getTestRootFolder(test)); + } + + const projectCoverageConfigs: ProjectCoverageConfig[] = []; + for (const folder of projectRoots) { + const project = await this.projectController?.getProjectForFolder(folder); + if (project) { + const driver = await project.getCMakeDriverInstance(); + if (driver) { + projectCoverageConfigs.push({ + project: project, driver: driver, preRunCoverageTarget: driver.config.preRunCoverageTarget, postRunCoverageTarget: driver.config.postRunCoverageTarget, coverageInfoFiles: driver.config.coverageInfoFiles + }); + } + } + } + + for (const projectCoverageConfig of projectCoverageConfigs) { + if (projectCoverageConfig.preRunCoverageTarget) { + log.info(localize('test.buildingPreRunCoverageTarget', 'Building the preRunCoverageTarget for project {0} before running tests with coverage.', projectCoverageConfig.project.sourceDir)); + const rc = await projectCoverageConfig.project.build([projectCoverageConfig.preRunCoverageTarget]); + if (rc !== 0) { + log.error(localize('test.preRunCoverageTargetFailure', 'Building the preRunCoverageTarget \'{0}\' on project in {1} failed. Skipping running tests.', projectCoverageConfig.preRunCoverageTarget, projectCoverageConfig.project.sourceDir)); + run.end(); + return 0; + } + } + } + const runResult = await this.runCTestHelper(tests, run, undefined, undefined, undefined, cancellation, false, undefined, RunCTestHelperEntryPoint.TestExplorer); + for (const projectCoverageConfig of projectCoverageConfigs) { + if (projectCoverageConfig.postRunCoverageTarget) { + log.info(localize('test.buildingPostRunCoverageTarget', 'Building the postRunCoverageTarget \'{0}\' for project {1} after the tests have run with coverage.', projectCoverageConfig.postRunCoverageTarget, projectCoverageConfig.project.sourceDir)); + const rc = await projectCoverageConfig.project.build([projectCoverageConfig.postRunCoverageTarget]); + if (rc !== 0) { + log.error(localize('test.postRunCoverageTargetFailure', 'Building target postRunCoverageTarget on project in {0} failed. Skipping handling of coverage data.', projectCoverageConfig.project.sourceDir)); + return 0; + } + } + } + + await this.handleCoverageOnProjects(run, projectCoverageConfigs); + return runResult; + } + + private async runTestHandler(request: vscode.TestRunRequest, cancellation: vscode.CancellationToken, isCoverageRun = false) { // NOTE: We expect the testExplorer to be undefined when the cmake.ctest.testExplorerIntegrationEnabled is disabled. if (!testExplorer) { return; @@ -856,7 +980,12 @@ export class CTestDriver implements vscode.Disposable { this.ctestsEnqueued(tests, run); const buildSucceeded = await this.buildTests(tests, run); if (buildSucceeded) { - await this.runCTestHelper(tests, run, undefined, undefined, undefined, cancellation, false, undefined, RunCTestHelperEntryPoint.TestExplorer); + if (isCoverageRun) { + await this.coverageCTestHelper(tests, run, cancellation); + + } else { + await this.runCTestHelper(tests, run, undefined, undefined, undefined, cancellation, false, undefined, RunCTestHelperEntryPoint.TestExplorer); + } } else { log.info(localize('test.skip.run.build.failure', "Not running tests due to build failure.")); } @@ -1165,6 +1294,12 @@ export class CTestDriver implements vscode.Disposable { (request: vscode.TestRunRequest, cancellation: vscode.CancellationToken) => this.runTestHandler(request, cancellation), true ); + testExplorer.createRunProfile( + 'Run Tests with Coverage', + vscode.TestRunProfileKind.Coverage, + (request: vscode.TestRunRequest, cancellation: vscode.CancellationToken) => this.runTestHandler(request, cancellation, true), + true + ).loadDetailedCoverage = async (_, fileCoverage) => this.coverageData.get(fileCoverage) ?? []; testExplorer.createRunProfile( 'Debug Tests', vscode.TestRunProfileKind.Debug, diff --git a/test/unit-tests/config.test.ts b/test/unit-tests/config.test.ts index bc071cc75..25e659115 100644 --- a/test/unit-tests/config.test.ts +++ b/test/unit-tests/config.test.ts @@ -76,7 +76,10 @@ function createConfig(conf: Partial): Configurat launchBehavior: 'reuseTerminal', ignoreCMakeListsMissing: false, automaticReconfigure: false, - enableAutomaticKitScan: true + enableAutomaticKitScan: true, + preRunCoverageTarget: null, + postRunCoverageTarget: null, + coverageInfoFiles: [] }); ret.updatePartial(conf); return ret;