coverage_report.sh

#!/usr/bin/env bash
##
## Trivial Tickets Ticketsystem
## Copyright (C) 2019 The Contributors
##
## This program is free software: you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation, either version 3 of the License, or
## (at your option) any later version.
##
## This program is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
## GNU General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with this program.  If not, see <http://www.gnu.org/licenses/>.
##
##
## Ticketsystem Trivial Tickets
##
## Matriculation numbers: 3040018, 6694964, 3478222
## Lecture:               Programmieren II, INF16B
## Lecturer:              Herr Prof. Dr. Helmut Neemann
## Institute:             Duale Hochschule Baden-Württemberg Mosbach
##
## ---------------
## Script to generate a coverage report
## from the tests in the packages
##
## Get more information about this script here:
## https://github.com/mortenterhart/trivial-tickets/wiki/Generating-a-Coverage-Report
##

# Script constants
script="${0##*/}"
root_dir="${0%/*}"

# Output Options
output_dir=""
output_file=""
coverage_profile="${root_dir}/coverage.txt"

# Test packages
default_package="${root_dir}/..."

# Test and Coverage Options
checks=""
coverage_mode="atomic"
coverage_format="html"
preserve_profile="false"
race="false"
verbose="false"

# printError prints an error message with an help advice
# to stdout. The message is built through concatenation
# of all arguments.
#
# Parameters:
#   $*: the error message
function printError() {
    printf "%s: %s\n" "${script}" "$*" >&2
    printf "Try '%s --help' for more information.\n" "${script}" >&2
}

# printCoverageModes prints the available coverage modes
# to stdout. This is called in case an invalid mode was
# specified on the command-line.
function printCoverageModes() {
    printf "%s: These coverage modes are available:\n" "${script}"
    printf "  set     test which statements run\n"
    printf "  count   test how often a statement runs\n"
    printf "  atomic  (default) count runs of statement, but\n"
    printf "          correct in multithreaded tests\n"
}

# printHelp outputs a help message with usage instructions
# and documentation for all options to the console.
function printHelp() {
    printf "Usage: %s [option(s)] [--] [package] ...\n" "${script}"
    printf "Generate a coverage profile for the named packages\n"
    printf "and report the coverage results in an output format\n\n"
    printf "The following options are available:\n"
    printf "  -c, --checks=<LIST>   Specify code checks to be run by 'go vet'\n"
    printf "                        before the actual tests. LIST is a comma\n"
    printf "                        separated list of check names. Special values\n"
    printf "                        of LIST are:   all       run all checks\n"
    printf "                                       default   selects all checks that are\n"
    printf "                                                 assumed to be always worth\n"
    printf "                                       off       no checks are done by 'go vet'\n"
    printf "                        The default for LIST is 'default'. Look at\n"
    printf "                        the options documented at 'go tool vet -help'\n"
    printf "                        for a complete list of checks.\n"
    printf "  -m, --coverage-mode=<MODE>\n"
    printf "                        Set the mode for coverage analysis for the\n"
    printf "                        packages being tested. MODE can be one of the\n"
    printf "                        following:     set       test which statements run\n"
    printf "                                       count     test how often a statement runs\n"
    printf "                                       atomic    count runs of statement, but\n"
    printf "                                                 correct in multithreaded tests\n"
    printf "                        The default for MODE is 'atomic'.\n"
    printf "  -p, --coverage-profile=<FILE>\n"
    printf "                        Write the coverage profile generated by 'go test'\n"
    printf "                        to FILE. If --output-directory is supplied the file\n"
    printf "                        is written to the specified output directory.\n"
    printf "  -f, --functions       Generate coverage information for each function\n"
    printf "                        in the applied packages. Overrides --html.\n"
    printf "  -h, --help            Display this help message and exit.\n"
    printf "  -H, --html            Generate a HTML representation of the coverage\n"
    printf "                        profile and open it in the default web browser\n"
    printf "                        unless --output-file is specified in which case\n"
    printf "                        the HTML is written to the specified file.\n"
    printf "                        Overrides --functions.\n"
    printf "  -d, --output-directory=<DIR>\n"
    printf "                        Write the coverage profile and the output file\n"
    printf "                        to the applied directory. DIR must exist and needs\n"
    printf "                        to be a directory.\n"
    printf "  -o, --output-file=<FILE>\n"
    printf "                        Write the outgoing coverage report from 'go tool cover'\n"
    printf "                        to FILE. This can be either a plain text file if\n"
    printf "                        --functions is supplied, or a HTML file if --html\n"
    printf "                        was given. The FILE should not exist yet, otherwise\n"
    printf "                        it is going to be overwritten. The leading directory,\n"
    printf "                        if any, is stripped if --output-directory is given.\n"
    printf "      --preserve-profile\n"
    printf "                        Don't remove the coverage profile after output file\n"
    printf "                        has been generated.\n"
    printf "  -r, --race            Examine the tests and search for race conditions in\n"
    printf "                        parallelized tests or code and report warnings, if any.\n"
    printf "  -v, --verbose         Print information for each executed test such as\n"
    printf "                        test logs and results.\n"
    printf "Mandatory arguments to long options are also mandatory to short options.\n\n"
    printf "By default, the coverage profile is named 'coverage.txt' and is written\n"
    printf "to the directory in which the tests were executed. The --output-directory\n"
    printf "option can change the location, whereas the directory must exist.\n"
    printf "Note that the coverage results are not generated if one of the tests fails.\n\n"
    printf "The default coverage mode is 'atomic' which is a complete analysis\n"
    printf "of the significance of the executed statements. It counts the number\n"
    printf "of executions for each statement and is also correct for tests which run\n"
    printf "on multiple go routines. By default, all checks assumed to be beneficial\n"
    printf "are run before the actual tests.\n\n"
    printf "The option --html is assumed if none of --html or --functions is specified.\n"
    printf "These options override each other. If --output-file is not specified, the\n"
    printf "HTML output will be shown in the default web browser.\n"
}

# compareVersions compares the numbers of two given version
# strings. The exit code indicates the comparison result. The
# versions are split on dots '.' so no other version delimiter
# is supported.
#
# If one version contains more numbers separated by dots
# the missing numbers in the other version are filled with
# zeros.
#
# The exit code is 0 if the versions are identical, 1 if
# the first version is greater than the second version
# and 2 if the first is lower than the second.
#
# Parameters:
#   $1: the first version
#   $2: the second version
#
# Examples:
#   compareVersions "1.10.1" "1.10.0"     # exit code: 1
#   compareVersions "1.11.5.2" "1.11.5"   # exit code: 2
#   compareVersions "1.10.0.0" "1.10"     # exit code: 0
function compareVersions() {
    local v1="$1"
    local v2="$2"

    # If the version strings are equal, return 0 directly
    if [[ "${v1}" == "${v2}" ]]; then
        return 0
    fi

    # Split the version strings on dots
    local IFS="."
    local -i i
    local ver1=(${v1}) ver2=(${v2})

    # Fill empty fields in ver1 with zeros
    for ((i = "${#ver1[@]}"; i < "${#ver2[@]}"; i++)); do
        ver1[i]=0
    done

    for ((i = 0; i < "${#ver1[@]}"; i++)); do
        if [ -z "${ver2[i]}" ]; then
            # Fill empty fields in ver2 with zeros
            ver2[i]=0
        fi

        # Ensure the version numbers are decimal numbers
        # (base 10).
        # If a version number contains a leading zero,
        # such as "08", the Bash 'let' builtin evaluates
        # it as octal number. The '10#xx' syntax prevents
        # this behaviour and reads the number 'xx' in
        # base 10 (decimal numbers).
        local ver1_decimal="$(( 10#${ver1[i]} ))"
        local ver2_decimal="$(( 10#${ver2[i]} ))"

        if ((ver1_decimal > ver2_decimal)); then
            return 1
        fi

        if ((ver1_decimal < ver2_decimal)); then
            return 2
        fi
    done

    return 0
}

# testVersion makes a version comparison between two provided
# version strings. The versions can be compared using different
# comparison operators which can be one of the following:
#
#   <, <=, ==, >, >=
#
# The operator is passed as second argument in between the two
# versions and should be quoted with double quotes to prevent
# interpretation by the shell.
#
# The exit status indicates if the test was successful or not.
# The exit code 0 is returned if the condition matches and 1
# otherwise.
#
# Parameters:
#   $1: first version string
#   $2: comparison operator as mentioned above
#   $3: second version string
#
# Examples:
#   testVersion 1.11.5 "<" 1.12     # true
#   testVersion 1.12.0 ">" 1.11.5   # true
#   testVersion 2.0.0 "==" 2        # true
#   testVersion 1.11.3 ">=" 1.11.4  # false
function testVersion() {
    local version1="$1"
    local compareOperator="$2"
    local version2="$3"

    local matching="false"

    # Check if the applied comparison operator
    # is valid
    case "${compareOperator}" in
        "<" | "<=" | "==" | ">" | ">=") ;;
        *)
            printf "%s: testVersion: invalid compare operator: %s\n" "${script}" "${compareOperator}" >&2
            return 5
            ;;
    esac

    local actualOperator

    # Compare the versions and check exit code for
    # comparison result
    compareVersions "${version1}" "${version2}"
    case "$?" in
        0) actualOperator="==" ;;
        1) actualOperator=">"  ;;
        2) actualOperator="<"  ;;
    esac

    if [ "${compareOperator}" == "<=" ]; then
        if [ "${actualOperator}" == "<" ] || [ "${actualOperator}" == "==" ]; then
            matching="true"
        fi
    elif [ "${compareOperator}" == ">=" ]; then
        if [ "${actualOperator}" == ">" ] || [ "${actualOperator}" == "==" ]; then
            matching="true"
        fi
    elif [ "${actualOperator}" == "${compareOperator}" ]; then
        matching="true"
    fi

    if "${matching}"; then
        return 0
    fi

    return 1
}

# getGoVersion writes the version of the currently active
# Go runtime to the variable specified in the first
# parameter $1. It uses the output of the 'go version'
# command to get the version.
#
# Parameters:
#   $1: the name of the variable where the go version is
#       written to (callback variable)
#
# Example:
#   local go_version
#   getGoVersion go_version
#   echo "${go_version}"     # Example Output: 1.11.5
function getGoVersion() {
    local callback_variable="$1"

    # Get the Go version in the format: "go1.11.5"
    local version="$(go version | grep -Eo "go[0-9]+(\.[0-9]+)+")"

    # Write the raw Go version number into the callback variable
    printf -v "${callback_variable}" "${version#go}"
}

# assignCoverageMode checks the provided coverage mode
# and sets the coverage mode option if the mode is valid.
# If the mode is invalid the script exits with an error.
#
# Parameters:
#   $1: the argument to the --coverage-mode option
function assignCoverageMode() {
    local mode_argument="$1"

    case "${mode_argument}" in
        atomic | count | set)
            coverage_mode="${mode_argument}"
            ;;
        *)
            printf "%s: invalid coverage mode: '%s'\n" "${script}" "${mode_argument}" >&2
            printCoverageModes >&2
            exit 3
    esac
}

# assignCoverageProfile sets the file to which the
# coverage profile is going to be written. The profile
# is generated by the 'go test' step. The function
# checks if the given file argument exists and if so
# it further checks if the file is regular and writable.
# If one of the checks is unsuccessful the script exits
# with an error.
#
# Parameters:
#   $1: the argument to the --coverage-profile option
function assignCoverageProfile() {
    local file_argument="$1"

    if [ -e "${file_argument}" ]; then
        if ! [ -f "${file_argument}" ]; then
            printError "file exists and is not a regular file: ${file_argument}"
            exit 3
        fi

        if ! [ -w "${file_argument}" ]; then
            printError "file is not writable: ${file_argument}"
            exit 3
        fi

        printf "%s: warning: file '%s' already exists and will be overwritten\n" "${script}" "${file_argument}"
    fi

    coverage_profile="${file_argument}"
}

# assignChecks sets the option which causes the test step
# to perform code checks using 'go vet' before executing
# the actual tests. There are three predefined check arguments:
#
#   all      run all checks
#   default  select all checks that are assumed to be
#            always worth
#   off      do not perform any checks
#
# Other arguments are also valid and are checked against
# the available code checks that 'go vet' offers. If an
# unrecognized check is supplied the script exits with
# an error. See the complete list of checks with
# 'go tool vet -help' for Go versions <= 1.12 or
# 'go tool vet help' for Go 1.12 and later.
#
# Parameters:
#   $1: the argument to the --checks option
function assignChecks() {
    local checks_argument="$1"

    case "${checks_argument}" in
        all)
            checks="all"
            ;;
        default)
            checks=""
            ;;
        off)
            checks="off"
            ;;
        *)
            # Split the check list in the argument on commas,
            # so do not put quotes around the variable expansion
            local IFS=","
            local all_checks=(${checks_argument})

            # Declare local variable to insert the help text
            # of 'go tool vet' with all defined options depending
            # on the Go version.
            local go_vet_options

            # Get the version of the current Go runtime
            local go_version
            getGoVersion go_version

            # Test whether the Go version is below 1.12 or not. This
            # is necessary because the command-line interface of
            # 'go tool vet' was changed in the Go 1.12 release.
            if testVersion "${go_version}" ">=" "1.12"; then
                go_vet_options="$(go tool vet help 2>&1 > /dev/null)"
            else
                go_vet_options="$(go tool vet -help 2>&1)"
            fi

            # For each specified check, examine if the check exists
            for c in "${all_checks[@]}"; do
                if ! grep -q -- "-\b${c}\b" <<< "${go_vet_options}"; then
                    printError "undefined check: '${c}'" >&2
                    exit 3
                fi
            done

            checks="${checks_argument}"
            ;;
    esac
}

# assignOutputDirectory sets the output directory option
# only if the directory exists and is a directory. If the
# option argument does not exist the scripts exits with
# with an error.
#
# Parameters:
#   $1: the argument to the --output-directory option
function assignOutputDirectory() {
    local directory_argument="$1"

    if [ -d "${directory_argument}" ]; then
        output_dir="${directory_argument}"
    else
        printError "output directory does not exist: '${directory_argument}'"
        exit 3
    fi
}

# assignOutputFile sets the output file option to the
# specified file. If the file already exists a warning
# is printed to stdout telling that the file will be
# overwritten.
#
# Parameters:
#   $1: the argument to the --output-file option
function assignOutputFile() {
    local file_argument="$1"

    if [ -d "${file_argument}" ]; then
        printError "output file is a directory: ${file_argument}"
        exit 3
    elif [ -e "${file_argument}" ]; then
        if ! [ -w "${file_argument}" ]; then
            printError "file exists, but is not writable: ${file_argument}"
            exit 3
        fi

        printf "%s: warning: file '%s' already exists and will be overwritten\n" "${script}" "${file_argument}"
    fi

    output_file="${file_argument}"
}

# Parse the command-line options
while getopts ":c:d:fhHm:o:p:rv-:" option "$@"; do
    case "${option}" in
        c)
            assignChecks "${OPTARG}"
            ;;
        d)
            assignOutputDirectory "${OPTARG}"
            ;;
        f)
            coverage_format="functions"
            ;;
        h)
            printHelp
            exit 0
            ;;
        H)
            coverage_format="html"
            ;;
        m)
            assignCoverageMode "${OPTARG}"
            ;;
        o)
            assignOutputFile "${OPTARG}"
            ;;
        p)
            assignCoverageProfile "${OPTARG}"
            ;;
        r)
            race="true"
            ;;
        v)
            verbose="true"
            ;;
        -)
            option_name="${OPTARG%%=*}"
            option_argument="${OPTARG#*=}"

            long_options="checks coverage-mode coverage-profile functions help html output-directory
                          output-file preserve-profile race verbose"
            long_options_arguments="checks coverage-mode coverage-profile output-directory output-file"

            option_matches=($(compgen -W "${long_options}" -- "${option_name}"))

            if [ "${#option_matches[@]}" -eq 0 ]; then
                printError "unrecognized option: '--${option_name}'" >&2
                exit 1
            elif [ "${#option_matches[@]}" -eq 1 ]; then
                option_name="${option_matches[0]}"

                if compgen -W "${long_options_arguments}" -- "${option_name}" >/dev/null 2>&1; then
                    if [[ "${OPTARG}" != *=* ]]; then
                        printError "option requires an argument: '--${option_name}'" >&2
                        exit 2
                    elif [ -z "${option_argument}" ]; then
                        printError "option requires a non-empty argument: '--${option_name}'"
                        exit 2
                    fi
                elif ! compgen -W "${long_options_arguments}" -- "${option_name}" >/dev/null 2>&1 && [[ "${OPTARG}" == *=* ]]; then
                    printError "option does not allow an argument: '--${option_name}'" >&2
                    exit 2
                fi

                case "${option_name}" in
                    checks)
                        assignChecks "${option_argument}"
                        ;;
                    coverage-mode)
                        assignCoverageMode "${option_argument}"
                        ;;
                    coverage-profile)
                        assignCoverageProfile "${option_argument}"
                        ;;
                    functions)
                        coverage_format="functions"
                        ;;
                    help)
                        printHelp
                        exit 0
                        ;;
                    html)
                        coverage_format="html"
                        ;;
                    output-file)
                        assignOutputFile "${option_argument}"
                        ;;
                    output-directory)
                        assignOutputDirectory "${option_argument}"
                        ;;
                    preserve-profile)
                        preserve_profile="true"
                        ;;
                    race)
                        race="true"
                        ;;
                    verbose)
                        verbose="true"
                        ;;
                esac
            else
                option_pos="$((OPTIND - 1))"
                printf "%s: ambiguous option: '%s'; could be one of:\n" "${script}" "${!option_pos}" >&2
                printf "  > --%s\n" "${option_matches[@]}" >&2
                printf "Try '%s --help' for more information.\n" "${script}" >&2
                exit 1
            fi
            ;;
        \?)
            printError "invalid option -- ${OPTARG}"
            exit 1
            ;;
        :)
            printError "option requires an argument -- ${OPTARG}"
            exit 2
            ;;
    esac
done

# Shift all the processed options so that only
# non-option parameters remain
shift "$((OPTIND - 1))"

# All non-option parameters are package paths
packages=("$@")
if [ "${#packages[@]}" -eq 0 ]; then
    packages[0]="${default_package}"
else
    for pack in "${packages[@]}"; do
        if [[ "${pack}" == */... ]]; then
            pack="${pack%/...}"
        fi

        # Check if all packages provided exist
        if ! [ -d "${pack}" ]; then
            printError "package does not exist: '${pack}'"
            exit 1
        fi
    done
fi

# Arrays with options for 'go test' and
# 'go tool cover'
go_test_options=()
go_cover_options=()

# Set the test flag for output directory and strip
# the leading directory of the coverage profile and
# output file paths
if [ -n "${output_dir}" ]; then
    go_test_options+=("-outputdir=${output_dir}")

    coverage_profile="${coverage_profile##*/}"

    if [ -n "${output_file}" ]; then
        output_file="${output_dir}/${output_file##*/}"
    fi
fi

# Set the coverage profile, coverage mode and
# vet options
go_test_options+=("-coverprofile=${coverage_profile}")
go_test_options+=("-covermode=${coverage_mode}")
go_test_options+=("-vet=${checks}")

# Set the race option
if "${race}"; then
    go_test_options+=("-race")
fi

# Set the verbose option
if "${verbose}"; then
    go_test_options+=("-v")
fi

# Set the output file option
if [ -n "${output_file}" ]; then
    go_cover_options+=("-o" "${output_file}")
fi

# Set the correct path to the coverage profile
# if an output directory was specified
if [ -n "${output_dir}" ]; then
    coverage_profile="${output_dir}/${coverage_profile}"
fi

# Set the output format option to either functions
# or HTML
case "${coverage_format}" in
    functions)
        go_cover_options+=("-func=${coverage_profile}")
        ;;
    html)
        go_cover_options+=("-html=${coverage_profile}")
        ;;
esac

## ----------------------------------------
## Begin of generating coverage report
## ----------------------------------------

# Check if Go is installed correctly
if ! type -P "go" > /dev/null 2>&1; then
    printError "executable \"go\" is not installed or not inside your \$PATH"
    exit 2
fi

printf "%s: Checking for missing Go test dependencies\n" "${script}"

# Check and install the required test dependencies
go get -t -v "${packages[@]}"

printf "%s: Building coverage profile by executing tests in package(s):\n" "${script}"
printf "  - %s\n" "${packages[@]}"

# Execute the tests with coverage and the
# specified options
go test "${go_test_options[@]}" "${packages[@]}"

# Get the exit status (successful test result)
test_status="$?"
exit_status="${test_status}"

# If the tests were successful generate the coverage report
if [ "${test_status}" -eq 0 ] && [ -f "${coverage_profile}" ]; then
    printf "%s: Tests succeeded, generating coverage report\n" "${script}"
    go tool cover "${go_cover_options[@]}"
    exit_status=$?
elif [ "${test_status}" -eq 1 ]; then
    printf "%s: Some of the tests failed, coverage report could not be generated.\n" "${script}"
elif [ "${test_status}" -eq 2 ]; then
    printf "%s: There were build errors in the tests, coverage report could not be generated.\n" "${script}"
elif [ -e "${coverage_profile}" ]; then
    printf "%s: Coverage profile cannot be read from '%s', report could not be generated.\n" "${script}" "${coverage_profile}"
fi

# Only delete the coverage profile if the --preserve-report
# option is not set and the file is owned by the current user
# so that the file can be deleted
if ! "${preserve_profile}" && [ -O "${coverage_profile}" ]; then
    rm -f "${coverage_profile}"
fi

# Exit the script with the exit status
exit "${exit_status}"