Merge pull request #39 from cicirello/json-summary-file

Generate a JSON summary of coverage stats

.github/workflows/build.yml

@@ -72,5 +72,39 @@ jobs:
         echo "coverage = ${{ steps.integration3.outputs.coverage }}"
         echo "branch coverage = ${{ steps.integration3.outputs.branches }}"
 
+    - name: Integration test for summary report 
+      id: integration4
+      uses: ./
+      with:
+        jacoco-csv-file: tests/summaryReportTest.csv
+        badges-directory: tests/summary
+        generate-coverage-badge: false
+        generate-branches-badge: false
+        generate-coverage-endpoint: false
+        generate-branches-endpoint: false
+        generate-summary: true
+
+    - name: Log integration summary report test outputs
+      run: |
+        echo "coverage = ${{ steps.integration4.outputs.coverage }}"
+        echo "branch coverage = ${{ steps.integration4.outputs.branches }}"
+
     - name: Verify integration test results
       run: python3 -u -m unittest tests/integration.py
+
+    # This test can be used to test failing the workflow run on decrease.
+    # Uncomment to use. Success is if this fails the workflow.
+    # - name: Integration test for fail on decrease 
+      # id: integrationFailures
+      # uses: ./
+      # with:
+        # jacoco-csv-file: tests/summaryReportTest.csv
+        # badges-directory: tests
+        # generate-coverage-badge: false
+        # generate-branches-badge: false
+        # generate-coverage-endpoint: false
+        # generate-branches-endpoint: false
+        # generate-summary: true
+        # summary-filename: reportTestFail.json
+        # fail-on-coverage-decrease: true
+        # fail-on-branches-decrease: true

CHANGELOG.md

@@ -4,7 +4,7 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-## [Unreleased] - 2021-08-16
+## [Unreleased] - 2021-11-11
 
 ### Added
 
@@ -19,6 +19,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### CI/CD
 
 
+## [2.5.0] - 2021-11-11
+
+### Added
+* Option to generate a simple JSON summary report containing the coverage
+  and branches coverage values as double-precision floating-point values.
+  This may be useful as input to other tools. Additionally, if used in
+  combination with the existing `fail-on-coverage-decrease` and/or 
+  `fail-on-branches-decrease` features, those checks will be more accurate.
+  This new feature is controlled by a pair of new inputs `generate-summary`
+  and `summary-filename`. This feature is disabled by default.
+
+
 ## [2.4.1] - 2021-08-16
 
 ### Fixed

JacocoBadgeGenerator.py

@@ -413,6 +413,33 @@ def coverageDecreasedEndpoint(coverage, jsonFilename, whichBadge) :
         return True
     return False
 
+def coverageDecreasedSummary(checkCoverage, checkBranches, jsonFile, coverage, branches) :
+    """Uses a summary report JSON file for the decreased coverage checks.
+    Returns true if workflow should fail, and also logs appropriate message.
+
+    Keyword arguments:
+    checkCoverage - If true, check if coverage decreased.
+    checkBranches - If true, check if branches coverage decreased.
+    jsonFile - The summary report including full path.
+    coverage - The instructions coverage in interval [0.0, 1.0].
+    branches - The branches coverage in interval [0.0, 1.0].
+    """
+    if not os.path.isfile(jsonFile) :
+        return False
+    try :
+        with open(jsonFile, "r") as f :
+            priorCoverage = json.load(f)
+    except :
+        return False
+    result = False
+    if checkCoverage and "coverage" in priorCoverage and 100*coverage < priorCoverage["coverage"] :
+        print("Coverage decreased from", priorCoverage["coverage"], "to", 100*coverage)
+        result = True
+    if checkBranches and "branches" in priorCoverage and 100*branches < priorCoverage["branches"] :
+        print("Branches coverage decreased from", priorCoverage["branches"], "to", 100*branches)
+        result = True
+    return result
+
 def colorCutoffsStringToNumberList(strCutoffs) :
     """Converts a string of space or comma separated percentages
     to a list of floats.
@@ -422,6 +449,18 @@ def colorCutoffsStringToNumberList(strCutoffs) :
     """
     return list(map(float, strCutoffs.replace(',', ' ').split()))
 
+def coverageDictionary(cov, branches) :
+    """Creates a dictionary with the coverage and branches coverage
+    as double-precision floating-point values, specifically the raw
+    computed values prior to truncation. Enables more accurate implementation
+    of fail on decrease. Coverages are reported in interval [0.0, 100.0].
+
+    Keyword arguments:
+    cov - Instruction coverage in interval [0.0, 1.0]
+    branches - Branches coverage in interval [0.0, 1.0]
+    """
+    return { "coverage" : 100 * cov, "branches" : 100 * branches }
+
 if __name__ == "__main__" :
     jacocoCsvFile = sys.argv[1]
     badgesDirectory = sys.argv[2]
@@ -440,6 +479,8 @@ def colorCutoffsStringToNumberList(strCutoffs) :
     generateBranchesJSON = sys.argv[15].lower() == "true"
     coverageJSON = sys.argv[16]
     branchesJSON = sys.argv[17]
+    generateSummary = sys.argv[18].lower() == "true"
+    summaryFilename = sys.argv[19]
 
     if onMissingReport not in {"fail", "quiet", "badges"} :
         print("ERROR: Invalid value for on-missing-report.")
@@ -469,24 +510,29 @@ def colorCutoffsStringToNumberList(strCutoffs) :
         branchesBadgeWithPath = formFullPathToFile(badgesDirectory, branchesFilename)
         coverageJSONWithPath = formFullPathToFile(badgesDirectory, coverageJSON)
         branchesJSONWithPath = formFullPathToFile(badgesDirectory, branchesJSON)
-
-        if failOnCoverageDecrease and generateCoverageBadge and coverageDecreased(cov, coverageBadgeWithPath, "coverage") :
-            print("Failing the workflow run.")
-            sys.exit(1)
-
-        if failOnBranchesDecrease and generateBranchesBadge and coverageDecreased(branches, branchesBadgeWithPath, "branches") :
-            print("Failing the workflow run.")
-            sys.exit(1)
-
-        if failOnCoverageDecrease and generateCoverageJSON and coverageDecreasedEndpoint(cov, coverageJSONWithPath, "coverage") :
-            print("Failing the workflow run.")
-            sys.exit(1)
-
-        if failOnBranchesDecrease and generateBranchesJSON and coverageDecreasedEndpoint(branches, branchesJSONWithPath, "branches") :
-            print("Failing the workflow run.")
-            sys.exit(1)
+        summaryFilenameWithPath = formFullPathToFile(badgesDirectory, summaryFilename)
+
+        # If using the fail on decrease options, in combination with the summary report, use summary
+        # report for the check since it is more accurate.
+        if (failOnCoverageDecrease or failOnBranchesDecrease) and generateSummary and os.path.isfile(summaryFilenameWithPath) :
+            if coverageDecreasedSummary(failOnCoverageDecrease, failOnBranchesDecrease, summaryFilenameWithPath, cov, branches) :
+                print("Failing the workflow run.")
+                sys.exit(1)
+        else : # Otherwise use the prior coverages as stored in badges / JSON.
+            if failOnCoverageDecrease and generateCoverageBadge and coverageDecreased(cov, coverageBadgeWithPath, "coverage") :
+                print("Failing the workflow run.")
+                sys.exit(1)
+            if failOnBranchesDecrease and generateBranchesBadge and coverageDecreased(branches, branchesBadgeWithPath, "branches") :
+                print("Failing the workflow run.")
+                sys.exit(1)
+            if failOnCoverageDecrease and generateCoverageJSON and coverageDecreasedEndpoint(cov, coverageJSONWithPath, "coverage") :
+                print("Failing the workflow run.")
+                sys.exit(1)
+            if failOnBranchesDecrease and generateBranchesJSON and coverageDecreasedEndpoint(branches, branchesJSONWithPath, "branches") :
+                print("Failing the workflow run.")
+                sys.exit(1)
 
-        if (generateCoverageBadge or generateBranchesBadge or generateCoverageJSON or generateBranchesJSON) and badgesDirectory != "" :
+        if (generateSummary or generateCoverageBadge or generateBranchesBadge or generateCoverageJSON or generateBranchesJSON) and badgesDirectory != "" :
             createOutputDirectories(badgesDirectory)
 
         if generateCoverageBadge or generateCoverageJSON :
@@ -507,6 +553,10 @@ def colorCutoffsStringToNumberList(strCutoffs) :
                 with open(branchesJSONWithPath, "w") as endpoint :
                     json.dump(generateDictionaryForEndpoint(covStr, color, "branches"), endpoint, sort_keys=True)
 
+        if generateSummary :
+            with open(summaryFilenameWithPath, "w") as summaryFile :
+                json.dump(coverageDictionary(cov, branches), summaryFile, sort_keys=True)
+
         print("::set-output name=coverage::" + str(cov))
         print("::set-output name=branches::" + str(branches))
 

README.md

@@ -26,7 +26,10 @@ from all modules, provided that the individual reports are independent.
 
 The jacoco-badge-generator can also optionally be used as part of a pull-request check. Specifically, you can
 configure it to fail the workflow run if coverage decreased relative to prior run, and/or if coverage is below
-a target threshold. See the [Inputs](#inputs) section for details of how to configure it for this purpose.
+a target threshold. See the [Inputs](#inputs) section for details of how to configure it for this purpose. The
+action can also be configured to generate a simple JSON file containing the coverages as double-precision 
+floating-point values, either instead of or in addition to generating the badges, which may be useful as input
+to other tools.
 
 _The developers of the jacoco-badge-generator GitHub Action are not affiliated 
 with the developers of JaCoCo, although we are a fan and user of their excellent 
@@ -328,6 +331,26 @@ created within the `badges-directory`
 directory. __The action doesn't commit the JSON file. You will 
 need to have additional steps in your workflow to do that.__
 
+### `generate-summary`
+
+This input controls whether or not to generate a simple JSON
+summary report of the following form:
+
+```JSON
+{"branches": 77.77777777777779, "coverage": 72.72727272727273}
+```
+
+The default is `generate-summary: false`. To enable, use
+`generate-summary: true`.
+
+### `summary-filename`
+
+This input is the filename for the summary report (see above). The
+default is `summary-filename: coverage-summary.json`, and will be
+created within the `badges-directory`
+directory. __The action doesn't commit the JSON file. You will 
+need to have additional steps in your workflow to do that.__
+
 ### `colors`
 
 This input can be used to change the colors used for the badges.
@@ -439,20 +462,34 @@ Values greater than 1 are assumed percents.
 ### `fail-on-coverage-decrease`
 
 This input enables directing the action to fail the workflow run if
-the computed coverage is less than it was on the previous run as recorded in the
-existing coverage badge, if one exists. The default is `false`. 
-Use `fail-on-coverage-decrease: true` to enable. The `generate-coverage-badge`
-input must also be `true` (the default), as the action will otherwise assume
-that there is no existing badge from which to get the prior coverage.
+the computed coverage is less than it was on the previous run as recorded in either the
+existing coverage badge, the existing coverage Shields endpoint, or the JSON summary 
+report (see the `generate-summary` input), if one of these exists. The default 
+is `false`.  Use `fail-on-coverage-decrease: true` to enable. 
+
+Additionally, at least one of the `generate-summary`, `generate-coverage-badge`, 
+or `generate-coverage-endpoint` inputs must also be `true`, as the action will otherwise assume
+that there is no existing badge or summary report from which to get the prior coverage.
+If more than one of these exist, this feature will use the summary report to determine if coverage
+decreased since it is more precise than the truncated coverage percentage stored in 
+the badge or Shields endpoint. __Therefore, when using this feature, it is recommended that
+you also set `generate-summary: true` and commit the summary report JSON file to the repository.__
 
 ### `fail-on-branches-decrease`
 
 This input enables directing the action to fail the workflow run if
-the computed branches coverage is less than it was on the previous run as recorded in the
-existing branches badge, if one exists. The default is `false`. 
-Use `fail-on-branches-decrease: true` to enable. The `generate-branches-badge`
-input must also be `true`, as the action will otherwise assume
-that there is no existing badge from which to get the prior branches coverage.
+the computed branches coverage is less than it was on the previous run as recorded in either the
+existing branches coverage badge, the existing branches coverage Shields endpoint, or the JSON summary 
+report (see the `generate-summary` input), if one of these exists. The default is `false`. 
+Use `fail-on-branches-decrease: true` to enable. 
+
+Additionally, at least one of the `generate-summary`, `generate-branches-badge`, 
+or `generate-branches-endpoint` inputs must also be `true`, as the action will otherwise assume
+that there is no existing badge or summary report from which to get the prior coverage.
+If more than one of these exist, this feature will use the summary report to determine if branches coverage
+decreased since it is more precise than the truncated coverage percentage stored in 
+the badge or Shields endpoint. __Therefore, when using this feature, it is recommended that
+you also set `generate-summary: true` and commit the summary report JSON file to the repository.__
 
 
 ## Outputs
@@ -583,7 +620,7 @@ You can also use a specific release with:
 
 ```yml
     - name: Generate JaCoCo Badge
-      uses: cicirello/jacoco-badge-generator@v2.3.0
+      uses: cicirello/jacoco-badge-generator@v2.5.0
       with:
         generate-branches-badge: true
 ```
@@ -610,6 +647,8 @@ what these inputs do.
         coverage-endpoint-filename: jacoco.json
         generate-branches-endpoint: false
         branches-endpoint-filename: branches.json
+        generate-summary: false
+        summary-filename: coverage-summary.json
         colors: '#4c1 #97ca00 #a4a61d #dfb317 #fe7d37 #e05d44'
         intervals: 100 90 80 70 60 0
         on-missing-report: fail

action.yml

@@ -100,6 +100,14 @@ inputs:
     description: 'The filename of the branches coverage JSON endpoint.'
     required: false
     default: 'branches.json'
+  generate-summary:
+    description: 'Controls whether or not to generate a JSON file containing the coverage percentages as floating-point values.'
+    required: false
+    default: false
+  summary-filename:
+    description: 'The filename of the summary file.'
+    required: false
+    default: 'coverage-summary.json'
 outputs:
   coverage:
     description: 'The jacoco coverage percentage as computed from the data in the jacoco.csv file.'
@@ -126,3 +134,5 @@ runs:
     - ${{ inputs.generate-branches-endpoint }}
     - ${{ inputs.coverage-endpoint-filename }}
     - ${{ inputs.branches-endpoint-filename }}
+    - ${{ inputs.generate-summary }}
+    - ${{ inputs.summary-filename }}

tests/integration.py

@@ -49,6 +49,12 @@ def testIntegrationMultiJacocoReportsCase(self) :
             with open("tests/badges/branchesMulti.svg","r") as generated :
                 self.assertEqual(expected.read(), generated.read())
 
+    def testIntegrationSummaryReport(self) :
+        with open("tests/summary/coverage-summary.json", "r") as f :
+            d = json.load(f)
+            self.assertAlmostEqual(72.72727272727272, d["coverage"])
+            self.assertAlmostEqual(77.77777777777777, d["branches"])
+
     def testIntegrationInstructionsJSON(self) :
         with open("tests/endpoints/jacoco.json", "r") as f :
             d = json.load(f)

tests/reportTest.json

@@ -0,0 +1 @@
+{"branches": 77.77777777777777, "coverage": 72.72727272727272}

tests/reportTestFail.json

@@ -0,0 +1 @@
+{"branches": 77.777777787777777, "coverage": 72.72727282727272}

tests/summaryReportTest.csv

@@ -0,0 +1,4 @@
+GROUP,PACKAGE,CLASS,INSTRUCTION_MISSED,INSTRUCTION_COVERED,BRANCH_MISSED,BRANCH_COVERED,LINE_MISSED,LINE_COVERED,COMPLEXITY_MISSED,COMPLEXITY_COVERED,METHOD_MISSED,METHOD_COVERED
+Program Name,org.something.package,OneClass,50,100,50,200,0,33,0,13,0,7
+Program Name,org.something.package,AnotherClass,100,300,150,300,0,37,0,15,0,7
+Program Name,org.something.package,YetAnotherClass,150,400,0,200,0,37,0,15,0,7