diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f33c274b5..1fded69b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,7 +25,7 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Setup Build Env + - name: Install Build Tools run: sudo ./scripts/install-build-tools.sh - name: Setup Local Dependencies run: ./scripts/setup-dependencies.sh @@ -38,7 +38,7 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Setup Build Env + - name: Install Build Tools run: sudo ./scripts/install-build-tools.sh - name: Setup Local Dependencies run: ./scripts/setup-dependencies.sh @@ -50,7 +50,7 @@ jobs: name: Pylint runs-on: ubuntu-22.04 continue-on-error: true - timeout-minutes: 10 + timeout-minutes: 5 strategy: matrix: python-version: ["3.10"] @@ -62,10 +62,25 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Setup Build Env + - name: Install Build Tools run: sudo ./scripts/install-build-tools.sh - name: Lint with Pylint run: ./scripts/pylint.sh + shellcheck: + name: Shellcheck + runs-on: ubuntu-22.04 + continue-on-error: true + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - name: Install shellcheck + run: | + sudo apt-get update + sudo apt-get install -y shellcheck + - name: Lint with Shellcheck + run: ./scripts/shellcheck.sh -S error --color=auto unit-and-integration-test: name: Unit and Integration Tests runs-on: ubuntu-22.04 @@ -74,7 +89,7 @@ jobs: - uses: actions/checkout@v4 with: submodules: recursive - - name: Setup Build Env + - name: Install Build Tools run: sudo ./scripts/install-build-tools.sh - name: Setup Local Dependencies run: ./scripts/setup-dependencies.sh @@ -84,7 +99,7 @@ jobs: run: ./scripts/test.sh - name: Shorten SHA id: vars - run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)" + run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_ENV - uses: actions/upload-artifact@v4 if: ${{ !env.ACT }} name: Archive Test Results @@ -114,4 +129,3 @@ jobs: name: OpenCBDC Transaction Processor docs for ${{ steps.vars.outputs.sha_short }} path: ./doxygen_generated/html/* retention-days: 7 - diff --git a/scripts/create-e2e-report.sh b/scripts/create-e2e-report.sh index 51214a4d0..771fb698c 100755 --- a/scripts/create-e2e-report.sh +++ b/scripts/create-e2e-report.sh @@ -11,9 +11,9 @@ function readAndFormatLogs() { return fi - for logfile in $(ls $logdir); do + for logfile in "$logdir"/*; do logfile_path="$logdir/$logfile" - logfile_content=$(cat $logfile_path) + logfile_content=$(<"$logfile_path") message+="\n
\n$logfile\n\n\`\`\`\n$logfile_content\n\`\`\`\n
\n" done echo "$message" diff --git a/scripts/install-build-tools.sh b/scripts/install-build-tools.sh index 523789d99..197ab904b 100755 --- a/scripts/install-build-tools.sh +++ b/scripts/install-build-tools.sh @@ -17,7 +17,7 @@ fi # Supporting these versions for buildflow PYTHON_VERSIONS=("3.10" "3.11" "3.12") -echo "Python3 versions supported: ${PYTHON_VERSIONS[@]}" +echo "Python3 versions supported: ${PYTHON_VERSIONS[*]}" # check if supported version of python3 is already installed, and save the version PY_INSTALLED='' diff --git a/scripts/lint.sh b/scripts/lint.sh index c6e4b0936..2197ab050 100755 --- a/scripts/lint.sh +++ b/scripts/lint.sh @@ -25,9 +25,12 @@ if [ -n "$whitespace_files" ] || [ -n "$newline_files" ] ; then exit 1 fi -check_format_files=$(git ls-files | grep -E "tools|tests|src|cmake-tests" \ - | grep -E "\..*pp") -clang-format --style=file --Werror --dry-run ${check_format_files[@]} +check_format_files=$(git ls-files | \ + grep -E "tools|tests|src|cmake-tests" | \ + grep -E "\..*pp") + +echo "${check_format_files}" | \ + xargs -n1 -I{} clang-format --style=file --Werror --dry-run {} if ! command -v clang-tidy &>/dev/null; then echo "clang-tidy does not appear to be installed" diff --git a/scripts/native-system-benchmark.sh b/scripts/native-system-benchmark.sh index 1b7923fa2..f295f2861 100755 --- a/scripts/native-system-benchmark.sh +++ b/scripts/native-system-benchmark.sh @@ -151,7 +151,7 @@ on_int() { printf 'Interrupting all components\n' trap '' SIGINT # avoid interrupting ourself for i in $PIDS; do # intentionally unquoted - if [[ -n "RECORD" ]]; then + if [[ -n "$RECORD" ]]; then kill -SIGINT -- "-$i" else kill -SIGINT -- "$i" @@ -194,7 +194,7 @@ on_int() { printf 'Terminating any remaining processes\n' for i in $PIDS; do # intentionally unquoted - if [[ -n "RECORD" ]]; then + if [[ -n "$RECORD" ]]; then kill -SIGTERM -- "-$i" else kill -SIGTERM -- "$i" @@ -253,7 +253,7 @@ run() { COMP= case "$RECORD" in perf) - $@ &> "$PROC_LOG" & + "$@" &> "$PROC_LOG" & COMP="$!" perf record -F 99 -a -g -o "$PNAME".perf -p "$COMP" &> "$PERF_LOG" & PERFS="$PERFS $!";; @@ -261,7 +261,7 @@ run() { ${DBG} "$@" &> "$PROC_LOG" & COMP="$!";; *) - $@ &> "$PROC_LOG" & + "$@" &> "$PROC_LOG" & COMP="$!";; esac @@ -324,7 +324,7 @@ launch() { "$RT"/scripts/wait-for-it.sh -q -t 5 -h localhost -p "$ep" done printf 'Launched logical %s %d, replica %d [PID: %d]\n' "$1" "$id" "$node" "$PID" - if [[ -n "RECORD" ]]; then + if [[ -n "$RECORD" ]]; then PIDS="$PIDS $(getpgid $PID)" else PIDS="$PIDS $PID" @@ -337,7 +337,7 @@ launch() { "$RT"/scripts/wait-for-it.sh -q -t 5 -h localhost -p "$ep" done printf 'Launched %s %d [PID: %d]\n' "$1" "$id" "$PID" - if [[ -n "RECORD" ]]; then + if [[ -n "$RECORD" ]]; then PIDS="$PIDS $(getpgid $PID)" else PIDS="$PIDS $PID" diff --git a/scripts/shellcheck.sh b/scripts/shellcheck.sh new file mode 100755 index 000000000..00dd59fbb --- /dev/null +++ b/scripts/shellcheck.sh @@ -0,0 +1,352 @@ +#!/usr/bin/env bash + +START_TIME=$(date "+%s") +ROOT="$(cd "$(dirname "$0")"/.. && pwd)" +SHELLCHECK_REPORT="${ROOT}/shellcheck-report.txt" +SHELLCHECK_REPORT_ALL_MSGS="${ROOT}/shellcheck-report-all-msgs.txt" + +SEVERITY="error" +EXCLUDE_CODES= +VIEW= +COLOR="auto" + +RED="\e[31m" +GREEN="\e[32m" +YELLOW="\e[33m" +BLUE="\e[34m" +CYAN="\e[36m" +RST_COLOR="\e[0m" + +exit_on_error() { + if [[ "$COLOR" == "never" ]]; then + echo; echo "[ERROR] $1" + else + echo; echo -e "${RED}[ERROR]${RST_COLOR} $1" + fi + echo; echo "Exiting..."; echo + exit 1 +} + +check_shellcheck_install() { + if ! command -v shellcheck &>/dev/null; then + exit_on_error "shellcheck is not installed.\n\n\ + Run '# ./scripts/install-build-tools.sh' to install shellcheck." + fi +} + +exit_bad_arg() { + if [[ -z "$1" ]]; then + # must pass argument to function + exit_on_error "No argument passed to exit_bad_arg function" + fi + show_usage + exit_on_error "Invalid argument: $1" +} + +show_usage() { + cat << EOF +Usage: $0 [options] + +Options: + -h, --help print this help and exit + -C, --color colorize the output, default is 'auto' + -e, --exclude-code exclude specific error code, can be repeated + -S, --severity=LEVEL set severity level (style, info, warning, error), default is 'error' + -v, --view view shellcheck report, default is False + +Usage: $ ./scripts/shellcheck.sh [-e|--exclude-code=CODE] [-S|--severity=LEVEL] [-v|--view] + +example: $ ./scripts/shellcheck.sh -e SC1091 -e SC1090 -S warning -v -C always + +EOF +} + +parse_cli_args() { + echo + while [[ $# -gt 0 ]]; do + optarg= + shft_cnt=1 + # if -- is passed then stop parsing + if [[ "$1" = '--' ]]; then + break + # --option=value + elif [[ "$1" =~ ^-- && ! "$1" =~ ^--$ ]]; then + optarg="${1#*=}"; shft_cnt=1 + # -o=value + elif [[ "$1" =~ ^-- && $# -gt 1 && ! "$2" =~ ^- ]]; then + optarg="$2"; shft_cnt=2 + # -o value + elif [[ "$1" =~ ^-[^-] && $# -gt 1 && ! "$2" =~ ^- ]]; then + optarg="$2"; shft_cnt=2 + # -o + elif [[ "$1" =~ ^-[^-] ]]; then + optarg="${1/??/}" + fi + + case "$1" in + -S*|--severity*) + case "${optarg}" in + style|info|warning|error) + SEVERITY="${optarg}" ;; + *) + exit_bad_arg "$optarg" ;; + esac + shift "$shft_cnt" ;; + -C*|--color*) + case "${optarg}" in + always|auto|never) + COLOR="${optarg}" ;; + *) + exit_bad_arg "$optarg" ;; + esac + shift "$shft_cnt" ;; + -e*|--exclude-code*) + # valid if matching format SC1000-SC9999 + if [[ "${optarg}" =~ ^SC[0-9]{4}$ ]]; then + # if empty, fill with error code, else append comma and new code + if [[ -z "${EXCLUDE_CODES}" ]]; then + EXCLUDE_CODES+="${optarg}" + else + EXCLUDE_CODES+=",${optarg}" + fi + else + exit_bad_arg "$optarg" + fi + shift "$shft_cnt" ;; + -v|--view) + VIEW="True" + shift "$shft_cnt" ;; + -h|--help) + echo; echo "Command line arguments: $0 $*"; echo + show_usage + exit 0 ;; + *) + exit_bad_arg "$optarg" ;; + esac + done +} + +get_num_cores() { + local CORE_COUNT=1 + if [[ "$OSTYPE" == "linux-gnu"* ]]; then + CORE_COUNT=$(grep -c ^processor /proc/cpuinfo) + elif [[ "$OSTYPE" == "darwin"* ]]; then + CORE_COUNT=$(sysctl -n hw.ncpu) + fi + printf "%d\n" "$CORE_COUNT" +} + +run_shellcheck() { + # check if git is installed + if command -v git &>/dev/null; then + if [[ -z "$EXCLUDE_CODES" ]]; then + git ls-files '*.sh' | xargs -n 1 -P "$NUM_CORES" shellcheck -s bash -C"$COLOR" > "$SHELLCHECK_REPORT_ALL_MSGS" + git ls-files '*.sh' | xargs -n 1 -P "$NUM_CORES" shellcheck -s bash -C"$COLOR" -S "$SEVERITY" > "$SHELLCHECK_REPORT" + else + git ls-files '*.sh' | xargs -n 1 -P "$NUM_CORES" shellcheck -s bash -C"$COLOR" > "$SHELLCHECK_REPORT_ALL_MSGS" + git ls-files '*.sh' | xargs -n 1 -P "$NUM_CORES" shellcheck -s bash -C"$COLOR" -S "$SEVERITY" --exclude="$EXCLUDE_CODES" > "$SHELLCHECK_REPORT" + fi + else + echo "git is not installed. Using find to compile list of shell scripts..."; echo + if [[ -z "$EXCLUDE_CODES" ]]; then + find "$ROOT" -name '*.sh' -print0 | xargs -0 -n 1 -P "$NUM_CORES" shellcheck -s bash -C"$COLOR" > "$SHELLCHECK_REPORT_ALL_MSGS" + find "$ROOT" -name '*.sh' -print0 | xargs -0 -n 1 -P "$NUM_CORES" shellcheck -s bash -C"$COLOR" -S "$SEVERITY" > "$SHELLCHECK_REPORT" + else + find "$ROOT" -name '*.sh' -print0 | xargs -0 -n 1 -P "$NUM_CORES" shellcheck -s bash -C"$COLOR" > "$SHELLCHECK_REPORT_ALL_MSGS" + find "$ROOT" -name '*.sh' -print0 | xargs -0 -n 1 -P "$NUM_CORES" shellcheck -s bash -C"$COLOR" -S "$SEVERITY" --exclude="$EXCLUDE_CODES" > "$SHELLCHECK_REPORT" + fi + fi +} + +view_report() { + if [[ "$#" -ne 1 ]]; then + exit_on_error "view_report function requires 1 argument" + elif [[ ! -f "$1" ]]; then + exit_on_error "view_report function requires a file as an argument" + fi + SHELL_REPORT="$1" + # view non-empty shellcheck report, includes info, warnings, errors + echo + if [[ "$COLOR" == "never" ]]; then + echo "----------------------- Start of Shellcheck report -----------------------" + echo; cat "$SHELL_REPORT"; echo + echo "------------------------ End of Shellcheck report ------------------------" + else + echo -e "----------------------- ${BLUE}Start of Shellcheck report${RST_COLOR} -----------------------" + echo; cat "$SHELL_REPORT"; echo + echo -e "------------------------ ${BLUE}End of Shellcheck report${RST_COLOR} ------------------------" + fi + echo +} + +check_report() { + SHELL_REPORT="$1" + # check if shellcheck file was created + if [[ ! -f "$SHELL_REPORT" ]]; then + exit_on_error "Shellcheck report was not created: ${SHELL_REPORT}" + # if shellcheck report was generated within the last minute then ok + elif [[ -f "$SHELL_REPORT" ]]; then + FILE_MODIFIED=$(date -r "$SHELL_REPORT" "+%s") + TIME_DIFF=$(( FILE_MODIFIED - START_TIME )) + # in testing takes a few seconds at most + if [[ "$TIME_DIFF" -gt 60 ]]; then + exit_on_error "Shellcheck report was not created within the last minute: ${SHELL_REPORT}" + fi + # check if shellcheck report is empty + elif [[ ! -s "$SHELL_REPORT" ]]; then + echo "Shellcheck report is empty: ${SHELL_REPORT}" + echo "Either there are no info/warning/error messages for all shell scripts" + echo "in the codebase or shellcheck failed to run successfully. Exiting..." + exit 0 + fi + + if [[ "$VIEW" == "True" ]]; then + view_report "$SHELL_REPORT" + fi +} + +determine_pass_fail() { + # detect if fatal errors are in shellcheck report + if [[ "$COLOR" == "never" ]]; then + echo -e "Checking for errors in shellcheck report with strictness level '${SEVERITY}'" + if [[ -n "$EXCLUDE_CODES" ]]; then + echo -e "Excluding error codes: '${EXCLUDE_CODES}'" + fi + else + echo -e "Checking for errors in shellcheck report with strictness level '${CYAN}${SEVERITY}${RST_COLOR}'" + if [[ -n "$EXCLUDE_CODES" ]]; then + echo -e "Excluding error codes: '${YELLOW}${EXCLUDE_CODES}${RST_COLOR}'"; echo + fi + fi + + TOTAL_ERRORS=0 + for SEV_LEVEL in "${REGEX_SEVERITY[@]}"; do + # get count of errors of severity level in main shellcheck report not waived version + ERROR_COUNT=$(grep -cE "\s\(${SEV_LEVEL}\):\s" "$SHELLCHECK_REPORT") + FATAL_ERROR_COUNTS["$SEV_LEVEL"]=$(( FATAL_ERROR_COUNTS["$SEV_LEVEL"] + ERROR_COUNT )) + TOTAL_ERRORS=$(( TOTAL_ERRORS + ERROR_COUNT )) + done + + if [[ "$TOTAL_ERRORS" -gt 0 ]]; then + return 1 + fi + return 0 +} + +calc_waived_error_stats() { + # find count of all violations of each severity level ignoring exclude codes, level + for SEV_LEVEL in error warning info style; do + # get count of errors of severity level + ERROR_COUNT=$(grep -cE "\s\(${SEV_LEVEL}\):\s" "$SHELLCHECK_REPORT_ALL_MSGS") + ALL_ERROR_COUNTS["$SEV_LEVEL"]=$(( ALL_ERROR_COUNTS["$SEV_LEVEL"] + ERROR_COUNT )) + done + + # waive stricter levels all at once + for WAIVE_LEV in "${WAIVE_STRICTER[@]}"; do + WAIVED_ERROR_COUNTS["$WAIVE_LEV"]="${ALL_ERROR_COUNTS["$WAIVE_LEV"]}" + done + + if [[ -n "$EXCLUDE_CODES" ]]; then + # Process each severity level in REGEX_SEVERITY + for SEV in "${REGEX_SEVERITY[@]}"; do + for EX_CODE in ${EXCLUDE_CODES//,/ }; do + # Count occurrences of exclude codes at the current severity level + EX_COUNT=$(grep -cE "\s${EX_CODE}\s\(${SEV}\):\s" "$SHELLCHECK_REPORT_ALL_MSGS") + # Add the count to the waived error counts for the current severity + WAIVED_ERROR_COUNTS["$SEV"]=$(( WAIVED_ERROR_COUNTS["$SEV"] + EX_COUNT )) + done + done + fi +} + +print_summary_table() { + echo; echo "-----------------------------------------------------" + if [[ "$COLOR" == "never" ]]; then + echo -e "Severity Level Counts Summary" + else + echo -e "${BLUE}Severity Level Counts Summary${RST_COLOR}" + fi + echo "-----------------------------------------------------" + printf "%-10s %-10s %-10s %-10s %-10s\n" "Type" "Error" "Warning" "Info" "Style" + echo "-----------------------------------------------------" + printf "%-10s %-10d %-10d %-10d %-10d\n" "Total" "${ALL_ERROR_COUNTS["error"]}" "${ALL_ERROR_COUNTS["warning"]}" "${ALL_ERROR_COUNTS["info"]}" "${ALL_ERROR_COUNTS["style"]}" + printf "%-10s %-10d %-10d %-10d %-10d\n" "Waived" "${WAIVED_ERROR_COUNTS["error"]}" "${WAIVED_ERROR_COUNTS["warning"]}" "${WAIVED_ERROR_COUNTS["info"]}" "${WAIVED_ERROR_COUNTS["style"]}" + if [[ "$COLOR" == "never" ]]; then + printf "%-10s %-10d %-10d %-10d %-10d\n" "Fatal" "${FATAL_ERROR_COUNTS["error"]}" "${FATAL_ERROR_COUNTS["warning"]}" "${FATAL_ERROR_COUNTS["info"]}" "${FATAL_ERROR_COUNTS["style"]}" + else + printf "%-10s ${RED}%-10d${RST_COLOR} ${RED}%-10d${RST_COLOR} ${RED}%-10d${RST_COLOR} ${RED}%-10d${RST_COLOR}\n" \ + "Fatal" "${FATAL_ERROR_COUNTS["error"]}" "${FATAL_ERROR_COUNTS["warning"]}" "${FATAL_ERROR_COUNTS["info"]}" "${FATAL_ERROR_COUNTS["style"]}" + fi + echo "-----------------------------------------------------"; echo +} + +print_final_message() { + if [[ "$#" -ne 1 ]] || [[ "$1" -lt 0 && "$0" -gt 1 ]]; then + exit_on_error "This function needs a decimal status code as input" + fi + EXIT_STATUS="$1" + # if no errors found, then shellcheck passed + if [[ "$COLOR" == "never" ]]; then + if [[ "$EXIT_STATUS" -eq 0 ]]; then + echo "[PASS] Shellcheck did not detect violations" + echo "Shellcheck passed. See report: ${SHELLCHECK_REPORT}" + exit "$EXIT_STATUS" + else + echo "[FAIL] Shellcheck found unexcused violations" + echo "Shellcheck failed. See report: ${SHELLCHECK_REPORT}" + fi + else + if [[ "$EXIT_STATUS" -eq 0 ]]; then + echo -e "${GREEN}[PASS]${RST_COLOR} Shellcheck did not detect violations" + echo; echo -e "${GREEN}Shellcheck passed.${RST_COLOR} See report: ${SHELLCHECK_REPORT}"; echo + exit "$EXIT_STATUS" + else + echo -e "${RED}[FAIL]${RST_COLOR} Shellcheck found unexcused violations" + exit_on_error "Shellcheck failed. See report: ${SHELLCHECK_REPORT}" + fi + fi +} + +main() { + + check_shellcheck_install + parse_cli_args "$@" + NUM_CORES=$(get_num_cores) + + run_shellcheck + check_report "$SHELLCHECK_REPORT" + + declare -A FATAL_ERROR_COUNTS; declare -A ALL_ERROR_COUNTS; declare -A WAIVED_ERROR_COUNTS + + FATAL_ERROR_COUNTS=( ["style"]=0 ["info"]=0 ["warning"]=0 ["error"]=0 ) + ALL_ERROR_COUNTS=( ["style"]=0 ["info"]=0 ["warning"]=0 ["error"]=0 ) + WAIVED_ERROR_COUNTS=( ["style"]=0 ["info"]=0 ["warning"]=0 ["error"]=0 ) + + # if any messages of severity level or more strict are present, use for grepping report + case "$SEVERITY" in + "style") REGEX_SEVERITY=("error" "warning" "info" "style") ;; + "info") REGEX_SEVERITY=("error" "warning" "info") ;; + "warning") REGEX_SEVERITY=("error" "warning") ;; + "error") REGEX_SEVERITY=("error") ;; + esac + + determine_pass_fail + STATUS="$?" + + check_report "$SHELLCHECK_REPORT_ALL_MSGS" + + # waive stricter levels if possible + case "$SEVERITY" in + "style") WAIVE_STRICTER=("") ;; + "info") WAIVE_STRICTER=("style") ;; + "warning") WAIVE_STRICTER=("style" "info") ;; + "error") WAIVE_STRICTER=("style" "info" "warning") ;; + esac + + calc_waived_error_stats + + print_summary_table + print_final_message "$STATUS" +} + +main "$@" diff --git a/scripts/test-e2e-minikube.sh b/scripts/test-e2e-minikube.sh index d25900163..b4dc55759 100755 --- a/scripts/test-e2e-minikube.sh +++ b/scripts/test-e2e-minikube.sh @@ -7,7 +7,7 @@ BUILD_DOCKER=${TESTRUN_BUILD_DOCKER:-1} # Make sure we have the necessary tools installed required_executables=(minikube docker go helm kubectl) -for e in ${required_executables[@]}; do +for e in "${required_executables[@]}"; do if ! command -v $e &> /dev/null; then echo "'$e' command not be found! This is required to run. Please install it." exit 1 diff --git a/scripts/wait-for-it.sh b/scripts/wait-for-it.sh index d990e0d36..df6ad2646 100755 --- a/scripts/wait-for-it.sh +++ b/scripts/wait-for-it.sh @@ -150,7 +150,7 @@ if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then WAITFORIT_ISBUSY=1 # Check if busybox timeout uses -t flag # (recent Alpine versions don't support -t anymore) - if timeout &>/dev/stdout | grep -q -e '-t '; then + if timeout |& tee /dev/stdout | grep -q -e '-t '; then WAITFORIT_BUSYTIMEFLAG="-t" fi else