Native coverage integration for the CTest test controller

.github/workflows/ci-main-linux.yml

@@ -13,6 +13,9 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      - name: Setup LCOV
+        uses: hrishikesh-kadam/setup-lcov@v1
+
       - name: Setup Node environment
         uses: actions/setup-node@v3
         with:

.github/workflows/ci-main-mac.yml

@@ -13,6 +13,9 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      - name: Setup LCOV
+        uses: hrishikesh-kadam/setup-lcov@v1
+
       - name: Setup Node environment
         uses: actions/setup-node@v3
         with:
@@ -43,8 +46,6 @@ jobs:
           configure-options: -DCMAKE_INSTALL_PREFIX:STRING=${{ github.workspace }}/test/fakebin
           install-build: true
 
-
-
       - name: Run successful-build test
         run: yarn endToEndTestsSuccessfulBuild
 

.github/workflows/ci-main.win.yml

@@ -14,6 +14,9 @@ jobs:
       - name: Checkout source code
         uses: actions/checkout@v3
 
+      - name: Setup LCOV
+        uses: hrishikesh-kadam/setup-lcov@v1
+
       - name: Setup Node environment
         uses: actions/setup-node@v3
         with:

CHANGELOG.md

@@ -6,6 +6,10 @@ Features:
 
 - Add support for Presets v9, which enables more macro expansion for the `include` field. [#3946](https://github.com/microsoft/vscode-cmake-tools/issues/3946)
 - Add support to configure default folder in workspace setting. [#1078](https://github.com/microsoft/vscode-cmake-tools/issues/1078)
+- Add support for processing LCOV based coverage info files when tests are
+  executed. This adds test execution type, "Run with coverage", on the `ctest`
+  section of the Testing tab.
+  [#4040](https://github.com/microsoft/vscode-cmake-tools/issues/4040)
 
 Improvements:
 

docs/cmake-settings.md

@@ -38,6 +38,9 @@ Options that support substitution, in the table below, allow variable references
 | `cmake.saveBeforeBuild` | If `true` (the default), saves open text documents when build or configure is invoked before running CMake. | `true` | no |
 | `cmake.sourceDirectory` | A directory or a list of directories where the root `CMakeLists.txt`s are stored. | `${workspaceFolder}` | yes |
 | `cmake.testEnvironment` | An object containing `key:value` pairs of environment variables, which will be available when debugging, running and testing with CTest. | `null` (no environment variables) | yes |
+| `cmake.preRunCoverageTarget` | Target to build before running tests with coverage using the test explorer | null | no |
+| `cmake.postRunCoverageTarget` | Target to build after running tests with coverage using the test explorer | null | no |
+| `cmake.coverageInfoFiles` | LCOV coverage info files to be processed after running tests with coverage using the test explorer | [] | yes |
 
 ## Variable substitution
 

package.json

@@ -22,7 +22,7 @@
     "syntaxes"
   ],
   "engines": {
-    "vscode": "^1.67.0"
+    "vscode": "^1.88.0"
   },
   "categories": [
     "Other",
@@ -3589,6 +3589,27 @@
           "default": true,
           "description": "%cmake-tools.configuration.cmake.enableAutomaticKitScan.description%",
           "scope": "resource"
+        },
+        "cmake.preRunCoverageTarget": {
+          "type": "string",
+          "default": null,
+          "description": "%cmake-tools.configuration.cmake.preRunCoverageTarget.description%",
+          "scope": "resource"
+        },
+        "cmake.postRunCoverageTarget": {
+          "type": "string",
+          "default": null,
+          "description": "%cmake-tools.configuration.cmake.postRunCoverageTarget.description%",
+          "scope": "resource"
+        },
+        "cmake.coverageInfoFiles": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "default": [],
+          "description": "%cmake-tools.configuration.cmake.coverageInfoFiles.description%",
+          "scope": "resource"
         }
       }
     },
@@ -3716,7 +3737,7 @@
     "@types/rimraf": "^3.0.0",
     "@types/sinon": "~9.0.10",
     "@types/tmp": "^0.2.0",
-    "@types/vscode": "1.63.0",
+    "@types/vscode": "1.88.0",
     "@types/which": "~2.0.0",
     "@types/xml2js": "^0.4.8",
     "@types/uuid": "~8.3.3",
@@ -3758,6 +3779,7 @@
     "webpack-cli": "^4.5.0"
   },
   "dependencies": {
+    "@friedemannsommer/lcov-parser": "^4.0.1",
     "@types/string.prototype.matchall": "^4.0.4",
     "ajv": "^7.1.0",
     "chokidar": "^3.5.1",

package.nls.json

@@ -273,6 +273,9 @@
     "cmake-tools.configuration.cmake.automaticReconfigure.description": "Automatically configure CMake project directories when the kit or the configuration preset is changed.",
     "cmake-tools.configuration.cmake.pinnedCommands.description":"List of CMake commands to always pin by default.",
     "cmake-tools.configuration.cmake.enableAutomaticKitScan.description": "Enable automatic scanning for kits when a kit isn't selected. This will only take affect when CMake Presets aren't being used.",
+    "cmake-tools.configuration.cmake.preRunCoverageTarget.description": "Target to build before running tests with coverage using the test explorer",
+    "cmake-tools.configuration.cmake.postRunCoverageTarget.description": "Target to build after running tests with coverage using the test explorer",
+    "cmake-tools.configuration.cmake.coverageInfoFiles.description": "LCOV coverage info files to be processed after running tests with coverage using the test explorer.",
     "cmake-tools.debugger.pipeName.description": "Name of the pipe (on Windows) or domain socket (on Unix) to use for debugger communication.",
     "cmake-tools.debugger.clean.description": "Clean prior to configuring.",
     "cmake-tools.debugger.configureAll.description": "Configure for all projects.",

src/config.ts

@@ -217,6 +217,9 @@ export interface ExtensionConfigurationSettings {
     automaticReconfigure: boolean;
     pinnedCommands: string[];
     enableAutomaticKitScan: boolean;
+    preRunCoverageTarget: string | null;
+    postRunCoverageTarget: string | null;
+    coverageInfoFiles: string[];
 }
 
 type EmittersOf<T> = {
@@ -565,6 +568,18 @@ export class ConfigurationReader implements vscode.Disposable {
         return this.configData.enableAutomaticKitScan;
     }
 
+    get preRunCoverageTarget(): string | null {
+        return this.configData.preRunCoverageTarget;
+    }
+
+    get postRunCoverageTarget(): string | null {
+        return this.configData.postRunCoverageTarget;
+    }
+
+    get coverageInfoFiles(): string[] {
+        return this.configData.coverageInfoFiles;
+    }
+
     private readonly emitters: EmittersOf<ExtensionConfigurationSettings> = {
         autoSelectActiveFolder: new vscode.EventEmitter<boolean>(),
         defaultActiveFolder: new vscode.EventEmitter<string | null>(),
@@ -629,7 +644,10 @@ export class ConfigurationReader implements vscode.Disposable {
         launchBehavior: new vscode.EventEmitter<string>(),
         automaticReconfigure: new vscode.EventEmitter<boolean>(),
         pinnedCommands: new vscode.EventEmitter<string[]>(),
-        enableAutomaticKitScan: new vscode.EventEmitter<boolean>()
+        enableAutomaticKitScan: new vscode.EventEmitter<boolean>(),
+        preRunCoverageTarget: new vscode.EventEmitter<string | null>(),
+        postRunCoverageTarget: new vscode.EventEmitter<string | null>(),
+        coverageInfoFiles: new vscode.EventEmitter<string[]>()
     };
 
     /**

src/coverage.ts

@@ -0,0 +1,64 @@
+import * as vscode from 'vscode';
+import { lcovParser } from "@friedemannsommer/lcov-parser";
+import * as nls from 'vscode-nls';
+import * as logging from '@cmt/logging';
+
+nls.config({ messageFormat: nls.MessageFormat.bundle, bundleFormat: nls.BundleFormat.standalone })();
+const localize: nls.LocalizeFunc = nls.loadMessageBundle();
+
+const log = logging.createLogger('ctest-coverage');
+
+export async function handleCoverageInfoFiles(run: vscode.TestRun, coverageInfoFiles: string[], coverageData: WeakMap<vscode.FileCoverage, vscode.FileCoverageDetail[]>) {
+    for (const coverageInfoFile of coverageInfoFiles) {
+        let contents: Uint8Array;
+        try {
+            contents = await vscode.workspace.fs.readFile(vscode.Uri.file(coverageInfoFile));
+        } catch (e) {
+            log.warning(localize('test.openCoverageInfoFile', 'Could not open coverage info file: {0}. Skipping...', coverageInfoFile));
+            return;
+        }
+        const sections = await lcovParser({ from: contents });
+        for (const section of sections) {
+            const coverage = new vscode.FileCoverage(vscode.Uri.file(section.path),
+                new vscode.TestCoverageCount(
+                    section.lines.hit,
+                    section.lines.instrumented
+                ), new vscode.TestCoverageCount(
+                    section.branches.hit,
+                    section.branches.instrumented
+                ), new vscode.TestCoverageCount(
+                    section.functions.hit,
+                    section.functions.instrumented
+                ));
+
+            const lineBranches = new Map<number, vscode.BranchCoverage[]>();
+            for (const branch of section.branches.details) {
+                const branchCoverage = new vscode.BranchCoverage(branch.hit,
+                    new vscode.Position(branch.line - 1, 0), branch.branch);
+
+                const curr = lineBranches.get(branch.line);
+                if (curr === undefined) {
+                    lineBranches.set(branch.line, [branchCoverage]);
+                } else {
+                    curr.push(branchCoverage);
+                    lineBranches.set(branch.line, curr);
+                }
+            }
+
+            const declarations: vscode.DeclarationCoverage[] = [];
+            for (const declaration of section.functions.details) {
+                declarations.push(new vscode.DeclarationCoverage(declaration.name, declaration.hit,
+                    new vscode.Position(declaration.line - 1, 0)));
+            }
+
+            const statements: vscode.StatementCoverage[] = [];
+            for (const line of section.lines.details) {
+                statements.push(new vscode.StatementCoverage(line.hit,
+                    new vscode.Position(line.line - 1, 0),
+                    lineBranches.get(line.line) ?? []));
+            }
+            coverageData.set(coverage, [...statements, ...declarations]);
+            run.addCoverage(coverage);
+        }
+    }
+}

src/ctest.ts

@@ -15,6 +15,8 @@ import { expandString } from '@cmt/expand';
 import * as proc from '@cmt/proc';
 import { ProjectController } from '@cmt/projectController';
 import { extensionManager } from '@cmt/extension';
+import { CMakeProject } from '@cmt/cmakeProject';
+import { handleCoverageInfoFiles } from '@cmt/coverage';
 
 nls.config({ messageFormat: nls.MessageFormat.bundle, bundleFormat: nls.BundleFormat.standalone })();
 const localize: nls.LocalizeFunc = nls.loadMessageBundle();
@@ -118,6 +120,18 @@ interface CTestInfo {
     version: { major: number; minor: number };
 }
 
+/* The preRunCoverageTarget and postRunCoverageTarget are optional user
+configured targets that are run before and after the tests when a "Run with
+Coverage" test execution is triggered. These cae be used to zero the
+coverage counters, filter coverage results etc. */
+interface ProjectCoverageConfig {
+    project: CMakeProject;
+    driver: CMakeDriver;
+    preRunCoverageTarget: string | null;
+    postRunCoverageTarget: string | null;
+    coverageInfoFiles: string[];
+}
+
 function parseXmlString<T>(xml: string): Promise<T> {
     return new Promise((resolve, reject) => {
         xml2js.parseString(xml, (err, result) => {
@@ -229,6 +243,8 @@ export class CTestDriver implements vscode.Disposable {
     private readonly testingEnabledEmitter = new vscode.EventEmitter<boolean>();
     readonly onTestingEnabledChanged = this.testingEnabledEmitter.event;
 
+    private coverageData = new WeakMap<vscode.FileCoverage, vscode.FileCoverageDetail[]>();
+
     dispose() {
         this.testingEnabledEmitter.dispose();
         this.testsChangedEmitter.dispose();
@@ -710,7 +726,7 @@ export class CTestDriver implements vscode.Disposable {
             return -1;
         }
 
-        if (util.isTestMode()) {
+        if (util.isTestMode() && !util.overrideTestModeForTestExplorer()) {
             // ProjectController can't be initialized in test mode, so we don't have a usable test explorer
             return 0;
         }
@@ -905,7 +921,65 @@ export class CTestDriver implements vscode.Disposable {
         return true;
     }
 
-    private async runTestHandler(request: vscode.TestRunRequest, cancellation: vscode.CancellationToken) {
+    private async handleCoverageOnProjects(run: vscode.TestRun, projectCoverageConfigs: ProjectCoverageConfig[]) {
+        // Currently only LCOV coverage info files are supported
+        for (const projectCoverageConfig of projectCoverageConfigs) {
+            if (projectCoverageConfig.coverageInfoFiles.length === 0) {
+                log.warning(localize('test.noCoverageInfoFiles', 'No coverage info files for CMake project {0}. No coverage data will be analyzed for this project.', projectCoverageConfig.project.sourceDir));
+                continue;
+            }
+            const expandedCoverageInfoFiles = await Promise.all(projectCoverageConfig.coverageInfoFiles.map(async (coverageInfoFile: string) => expandString(coverageInfoFile, projectCoverageConfig.driver.expansionOptions)));
+            await handleCoverageInfoFiles(run, expandedCoverageInfoFiles, this.coverageData);
+        }
+    }
+
+    private async coverageCTestHelper(tests: vscode.TestItem[], run: vscode.TestRun, cancellation: vscode.CancellationToken): Promise<number> {
+        const projectRoots = new Set<string>();
+        for (const test of tests) {
+            projectRoots.add(this.getTestRootFolder(test));
+        }
+
+        const projectCoverageConfigs: ProjectCoverageConfig[] = [];
+        for (const folder of projectRoots) {
+            const project = await this.projectController?.getProjectForFolder(folder);
+            if (project) {
+                const driver = await project.getCMakeDriverInstance();
+                if (driver) {
+                    projectCoverageConfigs.push({
+                        project: project, driver: driver, preRunCoverageTarget: driver.config.preRunCoverageTarget, postRunCoverageTarget: driver.config.postRunCoverageTarget, coverageInfoFiles: driver.config.coverageInfoFiles
+                    });
+                }
+            }
+        }
+
+        for (const projectCoverageConfig of projectCoverageConfigs) {
+            if (projectCoverageConfig.preRunCoverageTarget) {
+                log.info(localize('test.buildingPreRunCoverageTarget', 'Building the preRunCoverageTarget for project {0} before running tests with coverage.', projectCoverageConfig.project.sourceDir));
+                const rc = await projectCoverageConfig.project.build([projectCoverageConfig.preRunCoverageTarget]);
+                if (rc !== 0) {
+                    log.error(localize('test.preRunCoverageTargetFailure', 'Building the preRunCoverageTarget \'{0}\' on project in {1} failed. Skipping running tests.', projectCoverageConfig.preRunCoverageTarget, projectCoverageConfig.project.sourceDir));
+                    run.end();
+                    return rc;
+                }
+            }
+        }
+        const runResult = await this.runCTestHelper(tests, run, cancellation, undefined, undefined, undefined, false, undefined, RunCTestHelperEntryPoint.TestExplorer);
+        for (const projectCoverageConfig of projectCoverageConfigs) {
+            if (projectCoverageConfig.postRunCoverageTarget) {
+                log.info(localize('test.buildingPostRunCoverageTarget', 'Building the postRunCoverageTarget \'{0}\' for project {1} after the tests have run with coverage.', projectCoverageConfig.postRunCoverageTarget, projectCoverageConfig.project.sourceDir));
+                const rc = await projectCoverageConfig.project.build([projectCoverageConfig.postRunCoverageTarget]);
+                if (rc !== 0) {
+                    log.error(localize('test.postRunCoverageTargetFailure', 'Building target postRunCoverageTarget on project in {0} failed. Skipping handling of coverage data.', projectCoverageConfig.project.sourceDir));
+                    return rc;
+                }
+            }
+        }
+
+        await this.handleCoverageOnProjects(run, projectCoverageConfigs);
+        return runResult;
+    }
+
+    private async runTestHandler(request: vscode.TestRunRequest, cancellation: vscode.CancellationToken, isCoverageRun = false) {
         // NOTE: We expect the testExplorer to be undefined when the cmake.ctest.testExplorerIntegrationEnabled is disabled.
         if (!testExplorer) {
             return;
@@ -922,7 +996,12 @@ export class CTestDriver implements vscode.Disposable {
         this.ctestsEnqueued(tests, run);
         const buildSucceeded = await this.buildTests(tests, run);
         if (buildSucceeded) {
-            await this.runCTestHelper(tests, run, cancellation, undefined, undefined, undefined, false, undefined, RunCTestHelperEntryPoint.TestExplorer);
+            if (isCoverageRun) {
+                await this.coverageCTestHelper(tests, run, cancellation);
+
+            } else {
+                await this.runCTestHelper(tests, run, cancellation, undefined, undefined, undefined, false, undefined, RunCTestHelperEntryPoint.TestExplorer);
+            }
         } else {
             log.info(localize('test.skip.run.build.failure', "Not running tests due to build failure."));
         }
@@ -1231,6 +1310,12 @@ export class CTestDriver implements vscode.Disposable {
                 (request: vscode.TestRunRequest, cancellation: vscode.CancellationToken) => this.runTestHandler(request, cancellation),
                 true
             );
+            testExplorer.createRunProfile(
+                'Run Tests with Coverage',
+                vscode.TestRunProfileKind.Coverage,
+                (request: vscode.TestRunRequest, cancellation: vscode.CancellationToken) => this.runTestHandler(request, cancellation, true),
+                true
+            ).loadDetailedCoverage = async (_, fileCoverage) => this.coverageData.get(fileCoverage) ?? [];
             testExplorer.createRunProfile(
                 'Debug Tests',
                 vscode.TestRunProfileKind.Debug,

src/util.ts

@@ -835,6 +835,13 @@ export function isTestMode(): boolean {
     return process.env['CMT_TESTING'] === '1';
 }
 
+/**
+ * Returns true if the test explorer should be enabled even when in test mode.
+ */
+export function overrideTestModeForTestExplorer(): boolean {
+    return process.env['CMT_TESTING_OVERRIDE_TEST_EXPLORER'] === '1';
+}
+
 export async function getAllCMakeListsPaths(path: string): Promise<string[] | undefined> {
     const regex: RegExp = new RegExp(/(\/|\\)CMakeLists\.txt$/);
     return recGetAllFilePaths(path, regex, await readDir(path), []);

test/end-to-end-tests/single-root-UI/project-folder/.vscode/settings.json

@@ -1,5 +1,10 @@
 {
     "cmake.buildDirectory": "${workspaceFolder}/build",
     "cmake.useCMakePresets": "never",
-    "cmake.configureOnOpen": false
+    "cmake.configureOnOpen": false,
+    "cmake.preRunCoverageTarget": "init-coverage",
+    "cmake.postRunCoverageTarget": "capture-coverage",
+    "cmake.coverageInfoFiles": [
+        "${workspaceFolder}/build/lcov.info"
+    ]
 }