From a8f6d9eee62145673db4cbc6cd745972a58f193d Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Thu, 22 Jul 2021 14:24:02 +0200 Subject: [PATCH] Generate Cobertura report (#441) * WIP * WIP * No transformation yet * Update Jenkinsfile * Coverage DTD * Typo * Write attrs for Cobertura report * Enable coverage for CI scripts * Implementation full * Few bugfixes * Fix: hell in builder * Fix: wrong method name * Supports all test types * Fix: package context * Rename Must * Rename --- .ci/Jenkinsfile | 1 + cmd/testrunner.go | 17 ++ internal/builder/packages.go | 36 ++- internal/cobraext/const.go | 3 + internal/docs/readme.go | 2 +- internal/testrunner/coverageoutput.go | 248 ++++++++++++++++++ internal/testrunner/reporters/outputs/file.go | 2 +- scripts/test-check-packages.sh | 2 +- 8 files changed, 299 insertions(+), 12 deletions(-) create mode 100644 internal/testrunner/coverageoutput.go diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile index f856e4d0c..c5bf92df4 100644 --- a/.ci/Jenkinsfile +++ b/.ci/Jenkinsfile @@ -64,6 +64,7 @@ pipeline { junit(allowEmptyResults: false, keepLongStdio: true, testResults: "build/test-results/*.xml") + coverageReport('build/test-coverage') } } } diff --git a/cmd/testrunner.go b/cmd/testrunner.go index f1d04c949..8dc13e523 100644 --- a/cmd/testrunner.go +++ b/cmd/testrunner.go @@ -67,6 +67,7 @@ func setupTestCommand() *cobraext.Command { cmd.PersistentFlags().BoolP(cobraext.GenerateTestResultFlagName, "g", false, cobraext.GenerateTestResultFlagDescription) cmd.PersistentFlags().StringP(cobraext.ReportFormatFlagName, "", string(formats.ReportFormatHuman), cobraext.ReportFormatFlagDescription) cmd.PersistentFlags().StringP(cobraext.ReportOutputFlagName, "", string(outputs.ReportOutputSTDOUT), cobraext.ReportOutputFlagDescription) + cmd.PersistentFlags().BoolP(cobraext.TestCoverageFlagName, "", false, cobraext.TestCoverageFlagDescription) cmd.PersistentFlags().DurationP(cobraext.DeferCleanupFlagName, "", 0, cobraext.DeferCleanupFlagDescription) cmd.PersistentFlags().String(cobraext.VariantFlagName, "", cobraext.VariantFlagDescription) @@ -116,6 +117,11 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command return cobraext.FlagParsingError(err, cobraext.ReportOutputFlagName) } + testCoverage, err := cmd.Flags().GetBool(cobraext.TestCoverageFlagName) + if err != nil { + return cobraext.FlagParsingError(err, cobraext.TestCoverageFlagName) + } + packageRootPath, found, err := packages.FindPackageRoot() if !found { return errors.New("package root not found") @@ -143,6 +149,10 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command if err != nil { return cobraext.FlagParsingError(err, cobraext.DataStreamsFlagName) } + + if len(dataStreams) > 0 { + return cobraext.FlagParsingError(errors.New("test coverage can be calculated only if all data streams are selected"), cobraext.DataStreamsFlagName) + } } if runner.TestFolderRequired() { @@ -217,6 +227,13 @@ func testTypeCommandActionFactory(runner testrunner.TestRunner) cobraext.Command return errors.Wrap(err, "error writing test report") } + if testCoverage { + err := testrunner.WriteCoverage(packageRootPath, m.Name, runner.Type(), results) + if err != nil { + return errors.Wrap(err, "error writing test coverage") + } + } + // Check if there is any error or failure reported for _, r := range results { if r.ErrorMsg != "" || r.FailureMsg != "" { diff --git a/internal/builder/packages.go b/internal/builder/packages.go index e0990f6a0..fdc6ac919 100644 --- a/internal/builder/packages.go +++ b/internal/builder/packages.go @@ -15,8 +15,22 @@ import ( "github.com/pkg/errors" ) -// FindBuildDirectory locates the target build directory. -func FindBuildDirectory() (string, bool, error) { +// BuildDirectory function locates the target build directory. If the directory doesn't exist, it will create it. +func BuildDirectory() (string, error) { + buildDir, found, err := findBuildDirectory() + if err != nil { + return "", errors.Wrap(err, "locating build directory failed") + } + if !found { + buildDir, err = createBuildDirectory() + if err != nil { + return "", errors.Wrap(err, "creating new build directory failed") + } + } + return buildDir, nil +} + +func findBuildDirectory() (string, bool, error) { workDir, err := os.Getwd() if err != nil { return "", false, errors.Wrap(err, "locating working directory failed") @@ -38,15 +52,15 @@ func FindBuildDirectory() (string, bool, error) { return "", false, nil } -// MustFindBuildPackagesDirectory function locates the target build directory for packages. +// BuildPackagesDirectory function locates the target build directory for packages. // If the directories path doesn't exist, it will create it. -func MustFindBuildPackagesDirectory(packageRoot string) (string, error) { +func BuildPackagesDirectory(packageRoot string) (string, error) { buildDir, found, err := FindBuildPackagesDirectory() if err != nil { return "", errors.Wrap(err, "locating build directory failed") } if !found { - buildDir, err = createBuildPackagesDirectory() + buildDir, err = createBuildDirectory("integrations") // TODO add support for other package types if err != nil { return "", errors.Wrap(err, "creating new build directory failed") } @@ -61,7 +75,7 @@ func MustFindBuildPackagesDirectory(packageRoot string) (string, error) { // FindBuildPackagesDirectory function locates the target build directory for packages. func FindBuildPackagesDirectory() (string, bool, error) { - buildDir, found, err := FindBuildDirectory() + buildDir, found, err := findBuildDirectory() if err != nil { return "", false, err } @@ -86,7 +100,7 @@ func FindBuildPackagesDirectory() (string, bool, error) { // BuildPackage function builds the package. func BuildPackage(packageRoot string) (string, error) { - destinationDir, err := MustFindBuildPackagesDirectory(packageRoot) + destinationDir, err := BuildPackagesDirectory(packageRoot) if err != nil { return "", errors.Wrap(err, "locating build directory for package failed") } @@ -118,7 +132,7 @@ func BuildPackage(packageRoot string) (string, error) { return destinationDir, nil } -func createBuildPackagesDirectory() (string, error) { +func createBuildDirectory(dirs ...string) (string, error) { workDir, err := os.Getwd() if err != nil { return "", errors.Wrap(err, "locating working directory failed") @@ -129,7 +143,11 @@ func createBuildPackagesDirectory() (string, error) { path := filepath.Join(dir, ".git") fileInfo, err := os.Stat(path) if err == nil && fileInfo.IsDir() { - buildDir := filepath.Join(dir, "build", "integrations") // TODO add support for other package types + p := []string{dir, "build"} + if len(dirs) > 0 { + p = append(p, dirs...) + } + buildDir := filepath.Join(p...) err = os.MkdirAll(buildDir, 0755) if err != nil { return "", errors.Wrapf(err, "mkdir failed (path: %s)", buildDir) diff --git a/internal/cobraext/const.go b/internal/cobraext/const.go index c016fd1fa..ff8be8844 100644 --- a/internal/cobraext/const.go +++ b/internal/cobraext/const.go @@ -72,6 +72,9 @@ const ( StackDumpOutputFlagName = "output" StackDumpOutputFlagDescription = "output location for the stack dump" + TestCoverageFlagName = "test-coverage" + TestCoverageFlagDescription = "generate Cobertura test coverage reports" + VariantFlagName = "variant" VariantFlagDescription = "service variant" diff --git a/internal/docs/readme.go b/internal/docs/readme.go index 0f2a7b1d6..6b00c1ead 100644 --- a/internal/docs/readme.go +++ b/internal/docs/readme.go @@ -128,7 +128,7 @@ func updateReadme(fileName, packageRoot string) (string, error) { return "", errors.Wrapf(err, "writing %s file failed", fileName) } - packageBuildRoot, err := builder.MustFindBuildPackagesDirectory(packageRoot) + packageBuildRoot, err := builder.BuildPackagesDirectory(packageRoot) if err != nil { return "", errors.Wrap(err, "package build root not found") } diff --git a/internal/testrunner/coverageoutput.go b/internal/testrunner/coverageoutput.go new file mode 100644 index 000000000..f6e4e5545 --- /dev/null +++ b/internal/testrunner/coverageoutput.go @@ -0,0 +1,248 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package testrunner + +import ( + "bytes" + "encoding/xml" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "time" + + "github.com/pkg/errors" + + "github.com/elastic/elastic-package/internal/builder" +) + +const coverageDtd = `` + +type testCoverageDetails struct { + packageName string + testType TestType + dataStreams map[string][]string // : +} + +func newTestCoverageDetails(packageName string, testType TestType) *testCoverageDetails { + return &testCoverageDetails{packageName: packageName, testType: testType, dataStreams: map[string][]string{}} +} + +func (tcd *testCoverageDetails) withUncoveredDataStreams(dataStreams []string) *testCoverageDetails { + for _, wt := range dataStreams { + tcd.dataStreams[wt] = []string{} + } + return tcd +} + +func (tcd *testCoverageDetails) withTestResults(results []TestResult) *testCoverageDetails { + for _, result := range results { + if _, ok := tcd.dataStreams[result.DataStream]; !ok { + tcd.dataStreams[result.DataStream] = []string{} + } + tcd.dataStreams[result.DataStream] = append(tcd.dataStreams[result.DataStream], result.Name) + } + return tcd +} + +type coberturaCoverage struct { + XMLName xml.Name `xml:"coverage"` + LineRate float32 `xml:"line-rate,attr"` + BranchRate float32 `xml:"branch-rate,attr"` + Version string `xml:"version,attr"` + Timestamp int64 `xml:"timestamp,attr"` + LinesCovered int64 `xml:"lines-covered,attr"` + LinesValid int64 `xml:"lines-valid,attr"` + BranchesCovered int64 `xml:"branches-covered,attr"` + BranchesValid int64 `xml:"branches-valid,attr"` + Complexity float32 `xml:"complexity,attr"` + Sources []*coberturaSource `xml:"sources>source"` + Packages []*coberturaPackage `xml:"packages>package"` +} + +type coberturaSource struct { + Path string `xml:",chardata"` +} + +type coberturaPackage struct { + Name string `xml:"name,attr"` + LineRate float32 `xml:"line-rate,attr"` + BranchRate float32 `xml:"branch-rate,attr"` + Complexity float32 `xml:"complexity,attr"` + Classes []*coberturaClass `xml:"classes>class"` +} + +type coberturaClass struct { + Name string `xml:"name,attr"` + Filename string `xml:"filename,attr"` + LineRate float32 `xml:"line-rate,attr"` + BranchRate float32 `xml:"branch-rate,attr"` + Complexity float32 `xml:"complexity,attr"` + Methods []*coberturaMethod `xml:"methods>method"` +} + +type coberturaMethod struct { + Name string `xml:"name,attr"` + Signature string `xml:"signature,attr"` + LineRate float32 `xml:"line-rate,attr"` + BranchRate float32 `xml:"branch-rate,attr"` + Complexity float32 `xml:"complexity,attr"` + Lines coberturaLines `xml:"lines>line"` +} + +type coberturaLine struct { + Number int `xml:"number,attr"` + Hits int64 `xml:"hits,attr"` +} + +type coberturaLines []*coberturaLine + +func (c *coberturaCoverage) bytes() ([]byte, error) { + out, err := xml.MarshalIndent(&c, "", " ") + if err != nil { + return nil, errors.Wrap(err, "unable to format test results as xUnit") + } + + var buffer bytes.Buffer + buffer.WriteString(xml.Header) + buffer.WriteString("\n") + buffer.WriteString(coverageDtd) + buffer.WriteString("\n") + buffer.Write(out) + return buffer.Bytes(), nil +} + +// WriteCoverage function calculates test coverage for the given package. +// It requires to execute tests for all data streams (same test type), so the coverage can be calculated properly. +func WriteCoverage(packageRootPath, packageName string, testType TestType, results []TestResult) error { + details, err := collectTestCoverageDetails(packageRootPath, packageName, testType, results) + if err != nil { + return errors.Wrap(err, "can't collect test coverage details") + } + + report := transformToCoberturaReport(details) + + err = writeCoverageReportFile(report, packageName) + if err != nil { + return errors.Wrap(err, "can't write test coverage report file") + } + return nil +} + +func collectTestCoverageDetails(packageRootPath, packageName string, testType TestType, results []TestResult) (*testCoverageDetails, error) { + withoutTests, err := findDataStreamsWithoutTests(packageRootPath, testType) + if err != nil { + return nil, errors.Wrap(err, "can't find data streams without tests") + } + + details := newTestCoverageDetails(packageName, testType). + withUncoveredDataStreams(withoutTests). + withTestResults(results) + return details, nil +} + +func findDataStreamsWithoutTests(packageRootPath string, testType TestType) ([]string, error) { + dataStreamDir := filepath.Join(packageRootPath, "data_stream") + dataStreams, err := ioutil.ReadDir(dataStreamDir) + if err != nil { + return nil, errors.Wrap(err, "can't list data streams directory") + } + + var noTests []string + for _, dataStream := range dataStreams { + if !dataStream.IsDir() { + continue + } + + dataStreamTestPath := filepath.Join(packageRootPath, "data_stream", dataStream.Name(), "_dev", "test", string(testType)) + _, err := os.Stat(dataStreamTestPath) + if errors.Is(err, os.ErrNotExist) { + noTests = append(noTests, dataStream.Name()) + continue + } + if err != nil { + return nil, errors.Wrapf(err, "can't stat path: %s", dataStreamTestPath) + } + } + return noTests, nil +} + +func transformToCoberturaReport(details *testCoverageDetails) *coberturaCoverage { + var classes []*coberturaClass + for dataStream, testCases := range details.dataStreams { + var methods []*coberturaMethod + + if len(testCases) == 0 { + methods = append(methods, &coberturaMethod{ + Name: "no-test", + Lines: []*coberturaLine{{Number: 1, Hits: 0}}, + }) + } else { + for i, tc := range testCases { + methods = append(methods, &coberturaMethod{ + Name: tc, + Lines: []*coberturaLine{{Number: i + 1, Hits: 1}}, + }) + } + } + + if dataStream == "" { + dataStream = "-" // workaround for Cobertura to properly analyze tests running in the package context (not data stream) + } + + aClass := &coberturaClass{ + Name: string(details.testType), + Filename: details.packageName + "/" + dataStream, + Methods: methods, + } + classes = append(classes, aClass) + } + + return &coberturaCoverage{ + Timestamp: time.Now().UnixNano(), + Packages: []*coberturaPackage{ + { + Name: details.packageName, + Classes: classes, + }, + }, + } +} + +func writeCoverageReportFile(report *coberturaCoverage, packageName string) error { + dest, err := testCoverageReportsDir() + if err != nil { + return errors.Wrap(err, "could not determine test coverage reports folder") + } + + // Create test coverage reports folder if it doesn't exist + _, err = os.Stat(dest) + if err != nil && os.IsNotExist(err) { + if err := os.MkdirAll(dest, 0755); err != nil { + return errors.Wrap(err, "could not create test coverage reports folder") + } + } + + fileName := fmt.Sprintf("coverage-%s-%d-report.xml", packageName, report.Timestamp) + filePath := filepath.Join(dest, fileName) + + b, err := report.bytes() + if err != nil { + return errors.Wrap(err, "can't marshal test coverage report") + } + + if err := ioutil.WriteFile(filePath, b, 0644); err != nil { + return errors.Wrap(err, "could not write test coverage report file") + } + return nil +} + +func testCoverageReportsDir() (string, error) { + buildDir, err := builder.BuildDirectory() + if err != nil { + return "", errors.Wrap(err, "locating build directory failed") + } + return filepath.Join(buildDir, "test-coverage"), nil +} diff --git a/internal/testrunner/reporters/outputs/file.go b/internal/testrunner/reporters/outputs/file.go index cef2b469f..64b5d1dfe 100644 --- a/internal/testrunner/reporters/outputs/file.go +++ b/internal/testrunner/reporters/outputs/file.go @@ -58,7 +58,7 @@ func reportToFile(pkg, report string, format testrunner.TestReportFormat) error // testReportsDir returns the location of the directory to store test reports. func testReportsDir() (string, error) { - buildDir, _, err := builder.FindBuildDirectory() + buildDir, err := builder.BuildDirectory() if err != nil { return "", errors.Wrap(err, "locating build directory failed") } diff --git a/scripts/test-check-packages.sh b/scripts/test-check-packages.sh index 8146622c7..7c86ed8e2 100755 --- a/scripts/test-check-packages.sh +++ b/scripts/test-check-packages.sh @@ -59,7 +59,7 @@ for d in test/packages/*/; do elastic-package install -v # defer-cleanup is set to a short period to verify that the option is available - elastic-package test -v --report-format xUnit --report-output file --defer-cleanup 1s + elastic-package test -v --report-format xUnit --report-output file --defer-cleanup 1s --test-coverage ) cd - done