Added end-to-end test for coverage

microsoft · Nov 4, 2024 · 9c109e4 · 9c109e4
1 parent 6d04a62
commit 9c109e4
Show file tree

Hide file tree

Showing 7 changed files with 129 additions and 2 deletions.
diff --git a/src/ctest.ts b/src/ctest.ts
@@ -691,7 +691,7 @@ export class CTestDriver implements vscode.Disposable {
             return -1;
         }
 
-        if (util.isTestMode()) {
+        if (util.isTestMode() && !util.overrideTestModeForTestExplorer()) {
             // ProjectController can't be initialized in test mode, so we don't have a usable test explorer
             return 0;
         }

diff --git a/src/util.ts b/src/util.ts
@@ -835,6 +835,13 @@ export function isTestMode(): boolean {
     return process.env['CMT_TESTING'] === '1';
 }
 
+/**
+ * Returns true if the test explorer should be enabled even when in test mode.
+ */
+export function overrideTestModeForTestExplorer(): boolean {
+    return process.env['CMT_TESTING_OVERRIDE_TEST_EXPLORER'] === '1';
+}
+
 export async function getAllCMakeListsPaths(path: string): Promise<string[] | undefined> {
     const regex: RegExp = new RegExp(/(\/|\\)CMakeLists\.txt$/);
     return recGetAllFilePaths(path, regex, await readDir(path), []);

diff --git a/test/end-to-end-tests/single-root-UI/project-folder/.vscode/settings.json b/test/end-to-end-tests/single-root-UI/project-folder/.vscode/settings.json
@@ -1,5 +1,10 @@
 {
     "cmake.buildDirectory": "${workspaceFolder}/build",
     "cmake.useCMakePresets": "never",
-    "cmake.configureOnOpen": false
+    "cmake.configureOnOpen": false,
+    "cmake.preRunCoverageTarget": "init-coverage",
+    "cmake.postRunCoverageTarget": "capture-coverage",
+    "cmake.coverageInfoFiles": [
+        "${workspaceFolder}/build/lcov.info"
+    ]
 }
diff --git a/test/end-to-end-tests/single-root-UI/project-folder/CMakeLists.txt b/test/end-to-end-tests/single-root-UI/project-folder/CMakeLists.txt
@@ -3,9 +3,20 @@ project(TestBuildProcess VERSION 0.1.0)
 
 set(CMT_COOKIE passed-cookie CACHE STRING "Cookie to be written by the main executable")
 
+option(USE_COVERAGE "Enable GCOV during the build" OFF)
+if(USE_COVERAGE)
+    add_compile_options(--coverage)
+    add_link_options(--coverage -Wl,--dynamic-list-data)
+endif()
+
+include(CTest)
+
 add_executable(TestBuildProcess main.cpp)
 set_property(TARGET TestBuildProcess PROPERTY CXX_STANDARD 98)
 
+add_test(NAME TestBuildProcessAsTest
+    COMMAND $<TARGET_FILE:TestBuildProcess>)
+
 add_custom_command(
     TARGET TestBuildProcess
     POST_BUILD
@@ -36,3 +47,35 @@ add_custom_target(runTestTarget DEPENDS TestBuildProcess
     WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
     COMMENT "Run test target"
 )
+
+find_program(LCOV lcov)
+find_program(GCOV gcov)
+set(LCOV_BASE lcov-base.info)
+set(LCOV_TEST lcov-test.info)
+set(LCOV_TOTAL lcov.info)
+set(LCOV_LOG lcov.log)
+set(LCOV_ERR lcov.err)
+add_custom_target(init-coverage
+    COMMENT "Collecting initial coverage"
+    COMMAND ${LCOV} -c -i -d ${CMAKE_CURRENT_BINARY_DIR} --gcov-tool ${GCOV}
+        -o ${LCOV_BASE} 2>${LCOV_ERR} >${LCOV_LOG})
+add_dependencies(init-coverage reset-coverage)
+
+add_custom_target(reset-coverage
+    COMMENT "Reset all coverage counters to zero"
+    COMMAND ${LCOV} -q -z -d ${CMAKE_CURRENT_BINARY_DIR} --gcov-tool ${GCOV}
+        -o ${LCOV_BASE}
+    COMMAND ${LCOV} -q -z -d ${CMAKE_CURRENT_BINARY_DIR} --gcov-tool ${GCOV}
+        -o ${LCOV_TEST}
+    COMMAND ${LCOV} -q -z -d ${CMAKE_CURRENT_BINARY_DIR} --gcov-tool ${GCOV}
+        -o ${LCOV_TOTAL})
+
+add_custom_target(capture-coverage
+    COMMENT "Capture coverage data"
+    DEPENDS ${LCOV_BASE}
+    COMMAND ${LCOV} -c -d ${CMAKE_CURRENT_BINARY_DIR} -o ${LCOV_TEST}
+        --gcov-tool ${GCOV} 2>${LCOV_ERR} >${LCOV_LOG}
+    COMMAND ${LCOV} -a ${LCOV_BASE} -a ${LCOV_TEST} -o ${LCOV_TOTAL}
+        >>${LCOV_LOG}
+    COMMAND ${LCOV} -r ${LCOV_TOTAL} -o ${LCOV_TOTAL} "'/usr/include*'"
+        >>${LCOV_LOG})
diff --git a/test/end-to-end-tests/single-root-UI/project-folder/CMakeUserPresets.json b/test/end-to-end-tests/single-root-UI/project-folder/CMakeUserPresets.json
@@ -48,6 +48,14 @@
     {
         "name": "TestInheritFromPreset",
         "inherits": "Linux1"
+    },
+    {
+      "name": "LinuxUser2",
+      "description": "LinuxUser1 with Coverage enabled",
+      "inherits": "LinuxUser1",
+      "cacheVariables": {
+        "USE_COVERAGE": "ON"
+      }
     }
   ]
 }
diff --git a/test/end-to-end-tests/single-root-UI/runTest.ts b/test/end-to-end-tests/single-root-UI/runTest.ts
@@ -19,6 +19,7 @@ async function main() {
         const extensionTestsEnv: { [key: string]: string | undefined } = {
             "CMT_TESTING": "1",
             "CMT_QUIET_CONSOLE": "1",
+            "CMT_TESTING_OVERRIDE_TEST_EXPLORER": "1",
             "TEST_FILTER": process.env.TEST_FILTER ?? ".*"
         };
 

diff --git a/test/end-to-end-tests/single-root-UI/test/coverage.test.ts b/test/end-to-end-tests/single-root-UI/test/coverage.test.ts
@@ -0,0 +1,63 @@
+import { DefaultEnvironment, expect } from '@test/util';
+import * as vscode from 'vscode';
+
+// From vscode: src/vs/workbench/contrib/testing/common/testTypes.ts
+const enum TestResultState {
+    Unset = 0,
+    Queued = 1,
+    Running = 2,
+    Passed = 3,
+    Failed = 4,
+    Skipped = 5,
+    Errored = 6
+};
+
+suite('Coverage integration', () => {
+    let testEnv: DefaultEnvironment;
+
+    suiteSetup(async function (this: Mocha.Context) {
+        this.timeout(100000);
+
+        const build_loc = 'build';
+        const exe_res = 'output.txt';
+
+        testEnv = new DefaultEnvironment('test/end-to-end-tests/single-root-UI/project-folder', build_loc, exe_res);
+
+        await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('useCMakePresets', 'always');
+        await vscode.commands.executeCommand('cmake.getSettingsChangePromise');
+
+        testEnv.projectFolder.buildDirectory.clear();
+        await vscode.commands.executeCommand('cmake.setConfigurePreset', 'LinuxUser2');
+        expect(await vscode.commands.executeCommand('cmake.configure')).to.be.eq(0);
+        expect(await vscode.commands.executeCommand('cmake.build')).to.be.eq(0);
+    });
+
+    suiteTeardown(async function (this: Mocha.Context) {
+        this.timeout(30000);
+        testEnv.teardown();
+    });
+
+    test('Bad Run test with coverage', async () => {
+        await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('preRunCoverageTarget', 'non-existing-target');
+
+        let testResult: any = await vscode.commands.executeCommand('testing.coverage.uri', vscode.Uri.file(testEnv.projectFolder.location));
+        expect(testResult['tasks'][0].hasCoverage).to.be.eq(false);
+        expect(testResult['items'][2].computedState).to.be.eq(TestResultState.Unset);
+
+        await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('preRunCoverageTarget', 'init-target');
+        await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('postRunCoverageTarget', 'non-existing-target');
+
+        testResult = await vscode.commands.executeCommand('testing.coverage.uri', vscode.Uri.file(testEnv.projectFolder.location));
+        expect(testResult['tasks'][0].hasCoverage).to.be.eq(false);
+        expect(testResult['items'][2].computedState).to.be.eq(TestResultState.Unset);
+    }).timeout(60000);
+
+    test('Good Run test with coverage', async () => {
+        await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('preRunCoverageTarget', 'init-coverage');
+        await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('postRunCoverageTarget', 'capture-coverage');
+
+        const testResult: any = await vscode.commands.executeCommand('testing.coverage.uri', vscode.Uri.file(testEnv.projectFolder.location));
+        expect(testResult['tasks'][0].hasCoverage).to.be.eq(true);
+        expect(testResult['items'][2].computedState).to.be.eq(TestResultState.Passed);
+    }).timeout(60000);
+});