Skip to content

Commit 1d4dc50

Browse files
committed
ci: add shellcheck gh action; fix fatal shellcheck errors
This commit made with the assistance of github copilot Signed-off-by: Morgan Rockett <[email protected]>
1 parent f6e5401 commit 1d4dc50

8 files changed

+236
-21
lines changed

.github/workflows/ci.yml

+21-7
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ jobs:
2525
- uses: actions/checkout@v4
2626
with:
2727
submodules: recursive
28-
- name: Setup Build Env
28+
- name: Install Build Tools
2929
run: sudo ./scripts/install-build-tools.sh
3030
- name: Setup Local Dependencies
3131
run: ./scripts/setup-dependencies.sh
@@ -38,7 +38,7 @@ jobs:
3838
- uses: actions/checkout@v4
3939
with:
4040
submodules: recursive
41-
- name: Setup Build Env
41+
- name: Install Build Tools
4242
run: sudo ./scripts/install-build-tools.sh
4343
- name: Setup Local Dependencies
4444
run: ./scripts/setup-dependencies.sh
@@ -50,7 +50,7 @@ jobs:
5050
name: Pylint
5151
runs-on: ubuntu-22.04
5252
continue-on-error: true
53-
timeout-minutes: 10
53+
timeout-minutes: 5
5454
strategy:
5555
matrix:
5656
python-version: ["3.10"]
@@ -62,10 +62,25 @@ jobs:
6262
uses: actions/setup-python@v5
6363
with:
6464
python-version: ${{ matrix.python-version }}
65-
- name: Setup Build Env
65+
- name: Install Build Tools
6666
run: sudo ./scripts/install-build-tools.sh
6767
- name: Lint with Pylint
6868
run: ./scripts/pylint.sh
69+
shellcheck:
70+
name: Shellcheck
71+
runs-on: ubuntu-22.04
72+
continue-on-error: true
73+
timeout-minutes: 5
74+
steps:
75+
- uses: actions/checkout@v4
76+
with:
77+
submodules: recursive
78+
- name: Install shellcheck
79+
run: |
80+
sudo apt-get update
81+
sudo apt-get install -y shellcheck
82+
- name: Lint with Shellcheck
83+
run: ./scripts/shellcheck.sh -S error
6984
unit-and-integration-test:
7085
name: Unit and Integration Tests
7186
runs-on: ubuntu-22.04
@@ -74,7 +89,7 @@ jobs:
7489
- uses: actions/checkout@v4
7590
with:
7691
submodules: recursive
77-
- name: Setup Build Env
92+
- name: Install Build Tools
7893
run: sudo ./scripts/install-build-tools.sh
7994
- name: Setup Local Dependencies
8095
run: ./scripts/setup-dependencies.sh
@@ -84,7 +99,7 @@ jobs:
8499
run: ./scripts/test.sh
85100
- name: Shorten SHA
86101
id: vars
87-
run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
102+
run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
88103
- uses: actions/upload-artifact@v4
89104
if: ${{ !env.ACT }}
90105
name: Archive Test Results
@@ -114,4 +129,3 @@ jobs:
114129
name: OpenCBDC Transaction Processor docs for ${{ steps.vars.outputs.sha_short }}
115130
path: ./doxygen_generated/html/*
116131
retention-days: 7
117-

scripts/create-e2e-report.sh

+2-2
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,9 @@ function readAndFormatLogs() {
1111
return
1212
fi
1313

14-
for logfile in $(ls $logdir); do
14+
for logfile in "$logdir"/*; do
1515
logfile_path="$logdir/$logfile"
16-
logfile_content=$(cat $logfile_path)
16+
logfile_content=$(<"$logfile_path")
1717
message+="\n<details>\n<summary>$logfile</summary>\n\n\`\`\`\n$logfile_content\n\`\`\`\n</details>\n"
1818
done
1919
echo "$message"

scripts/install-build-tools.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ fi
1717

1818
# Supporting these versions for buildflow
1919
PYTHON_VERSIONS=("3.10" "3.11" "3.12")
20-
echo "Python3 versions supported: ${PYTHON_VERSIONS[@]}"
20+
echo "Python3 versions supported: ${PYTHON_VERSIONS[*]}"
2121

2222
# check if supported version of python3 is already installed, and save the version
2323
PY_INSTALLED=''

scripts/lint.sh

+6-3
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,12 @@ if [ -n "$whitespace_files" ] || [ -n "$newline_files" ] ; then
2525
exit 1
2626
fi
2727

28-
check_format_files=$(git ls-files | grep -E "tools|tests|src|cmake-tests" \
29-
| grep -E "\..*pp")
30-
clang-format --style=file --Werror --dry-run ${check_format_files[@]}
28+
check_format_files=$(git ls-files | \
29+
grep -E "tools|tests|src|cmake-tests" | \
30+
grep -E "\..*pp")
31+
32+
echo "${check_format_files}" | \
33+
xargs -n1 -I{} clang-format --style=file --Werror --dry-run {}
3134

3235
if ! command -v clang-tidy &>/dev/null; then
3336
echo "clang-tidy does not appear to be installed"

scripts/native-system-benchmark.sh

+6-6
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ on_int() {
151151
printf 'Interrupting all components\n'
152152
trap '' SIGINT # avoid interrupting ourself
153153
for i in $PIDS; do # intentionally unquoted
154-
if [[ -n "RECORD" ]]; then
154+
if [[ -n "$RECORD" ]]; then
155155
kill -SIGINT -- "-$i"
156156
else
157157
kill -SIGINT -- "$i"
@@ -194,7 +194,7 @@ on_int() {
194194

195195
printf 'Terminating any remaining processes\n'
196196
for i in $PIDS; do # intentionally unquoted
197-
if [[ -n "RECORD" ]]; then
197+
if [[ -n "$RECORD" ]]; then
198198
kill -SIGTERM -- "-$i"
199199
else
200200
kill -SIGTERM -- "$i"
@@ -253,15 +253,15 @@ run() {
253253
COMP=
254254
case "$RECORD" in
255255
perf)
256-
$@ &> "$PROC_LOG" &
256+
"$@" &> "$PROC_LOG" &
257257
COMP="$!"
258258
perf record -F 99 -a -g -o "$PNAME".perf -p "$COMP" &> "$PERF_LOG" &
259259
PERFS="$PERFS $!";;
260260
debug)
261261
${DBG} "$@" &> "$PROC_LOG" &
262262
COMP="$!";;
263263
*)
264-
$@ &> "$PROC_LOG" &
264+
"$@" &> "$PROC_LOG" &
265265
COMP="$!";;
266266
esac
267267

@@ -324,7 +324,7 @@ launch() {
324324
"$RT"/scripts/wait-for-it.sh -q -t 5 -h localhost -p "$ep"
325325
done
326326
printf 'Launched logical %s %d, replica %d [PID: %d]\n' "$1" "$id" "$node" "$PID"
327-
if [[ -n "RECORD" ]]; then
327+
if [[ -n "$RECORD" ]]; then
328328
PIDS="$PIDS $(getpgid $PID)"
329329
else
330330
PIDS="$PIDS $PID"
@@ -337,7 +337,7 @@ launch() {
337337
"$RT"/scripts/wait-for-it.sh -q -t 5 -h localhost -p "$ep"
338338
done
339339
printf 'Launched %s %d [PID: %d]\n' "$1" "$id" "$PID"
340-
if [[ -n "RECORD" ]]; then
340+
if [[ -n "$RECORD" ]]; then
341341
PIDS="$PIDS $(getpgid $PID)"
342342
else
343343
PIDS="$PIDS $PID"

scripts/shellcheck.sh

+198
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
#!/usr/bin/env bash
2+
3+
RED="\e[31m"
4+
GREEN="\e[32m"
5+
RST_COLOR="\e[0m"
6+
7+
if ! command -v shellcheck &>/dev/null; then
8+
echo -e "${RED}[ERROR]${RST_COLOR} shellcheck is not installed."
9+
echo "Run 'sudo ./scripts/install-build-tools.sh' to install shellcheck."
10+
exit 1
11+
fi
12+
13+
# Usage: ./scripts/shellcheck.sh [-e|--exclude-code=CODE] [-S|--severity=LEVEL] [-v|--view]
14+
IFS='' read -rd '' usage <<'EOF'
15+
Usage: %s [options]
16+
17+
Options:
18+
-h, --help print this help and exit
19+
-e, --exclude-code exclude specific error code, can be repeated
20+
-S, --severity=LEVEL set severity level (info, warning, error)
21+
-v, --view view shellcheck report
22+
23+
example: ./scripts/shellcheck.sh -e SC1091 -e SC1090 -S warning -v
24+
EOF
25+
26+
echo; echo "Command line arguments: $0 $*"; echo
27+
28+
SEVERITY=
29+
EXCLUDE_CODES=
30+
VIEW="False"
31+
32+
_help=
33+
_err=0
34+
while [[ $# -gt 0 ]]; do
35+
optarg=
36+
shft_cnt=1
37+
if [[ "$1" = '--' ]]; then
38+
shift 1
39+
break
40+
elif [[ "$1" =~ [=] ]]; then
41+
optarg="${1#*=}"
42+
elif [[ "$1" =~ ^-- && $# -gt 1 && ! "$2" =~ ^- ]]; then
43+
optarg="$2"
44+
shft_cnt=2
45+
elif [[ "$1" =~ ^-[^-] && $# -gt 1 && ! "$2" =~ ^- ]]; then
46+
optarg="$2"
47+
shft_cnt=2
48+
elif [[ "$1" =~ ^-[^-] ]]; then
49+
optarg="${1/??/}"
50+
fi
51+
52+
case "$1" in
53+
-S*|--severity*)
54+
# don't let the user enter -S LEVEL more than once
55+
# SEV=$(echo "${optarg}" | tr -d '[:space:]')
56+
if [[ -n "$SEVERITY" ]]; then
57+
printf "${RED}[Error]${RST_COLOR} Severity level already set to: %s\n" "${SEVERITY}"
58+
_help=1; _err=1
59+
# valid if -S has any of 'info', 'warning', 'error'
60+
elif [[ "${optarg}" == "info" || "${optarg}" == "warning" || "${optarg}" == "error" ]]; then
61+
SEVERITY="${optarg}"
62+
else
63+
# continue and disregard invalid severity level
64+
printf "${RED}[Error]${RST_COLOR} severity level: %s\n" "${optarg}"
65+
_err=1
66+
fi
67+
shift "$shft_cnt"
68+
;;
69+
-e*|--exclude-code*)
70+
# strip whitespace from optarg
71+
CODE=$(echo "${optarg}" | tr -d '[:space:]')
72+
# valid if matching format SC1000-SC9999
73+
if [[ "${optarg}" =~ ^SC[0-9]{4}$ ]]; then
74+
# if empty then populate with just error code, otherwise add pipe before new code for grep later
75+
if [[ -z "${EXCLUDE_CODES}" ]]; then
76+
EXCLUDE_CODES+="${CODE}"
77+
else
78+
EXCLUDE_CODES+="|${CODE}"
79+
fi
80+
else
81+
# continue just don't save invalid error code
82+
printf "${RED}[Error]${RST_COLOR} Invalid error code entered: %s\n" "${optarg}"
83+
_err=1
84+
fi
85+
shift "$shft_cnt"
86+
;;
87+
-v|--view)
88+
VIEW="True"
89+
shift "$shft_cnt"
90+
;;
91+
-h|--help)
92+
_help=1
93+
;;
94+
*)
95+
printf "${RED}[Error]${RST_COLOR} Unrecognized option: %s\n" "$1"
96+
_err=1
97+
shift "$shft_cnt"
98+
;;
99+
esac
100+
101+
# exit on help message
102+
if [[ $_help -eq 1 ]]; then
103+
printf "%s\n" "$usage"
104+
exit 0
105+
fi
106+
# continue on invalid arg, let user know but don't exit
107+
if [[ "$_err" -eq 1 ]]; then
108+
printf "${RED}[Error]${RST_COLOR} Invalid argument: %s\n" "${optarg}"
109+
printf "%s\n" "$usage"
110+
fi
111+
done
112+
113+
# if severity not set, set it to error as default
114+
if [[ -z "$SEVERITY" ]]; then
115+
SEVERITY="error"
116+
fi
117+
118+
ROOT="$(cd "$(dirname "$0")"/.. && pwd)"
119+
SHELLCHECK_REPORT="${ROOT}/shellcheck-report.txt"
120+
121+
NUM_CORES=1
122+
if [[ "$OSTYPE" == "linux-gnu"* ]]; then
123+
NUM_CORES=$(grep -c ^processor /proc/cpuinfo)
124+
elif [[ "$OSTYPE" == "darwin"* ]]; then
125+
NUM_CORES=$(sysctl -n hw.ncpu)
126+
fi
127+
128+
# run shellcheck in parallel on all tracked shell scripts
129+
#
130+
# checking status of this run will give failure if info/warning/error is found by default
131+
#
132+
# determine status by parsing shellcheck report to see if any messages
133+
# of the severity level or more strict are present to determine failure (true errors)
134+
135+
# check if git is installed
136+
if command -v git &>/dev/null; then
137+
echo "Using git ls-files to find shell scripts..."; echo
138+
git ls-files '*.sh' | xargs -n 1 -P "$NUM_CORES" shellcheck > "$SHELLCHECK_REPORT"
139+
else
140+
echo "git is not installed. Using find to compile list of shell scripts..."; echo
141+
if [[ -z "$EXCLUDE_CODES" ]]; then
142+
find "$ROOT" -name '*.sh' -print0 | xargs -0 -n 1 -P "$NUM_CORES" shellcheck > "$SHELLCHECK_REPORT"
143+
fi
144+
fi
145+
146+
# if shell check report exists to determine if shellcheck run was successful
147+
if [[ -z "$SHELLCHECK_REPORT" ]]; then
148+
echo "${RED}[FAIL]${RST_COLOR}Shellcheck report ${SHELLCHECK_REPORT} not found. Exiting..."
149+
exit 1
150+
else
151+
if [[ ! -s "$SHELLCHECK_REPORT" ]]; then
152+
echo "Shellcheck report is empty: ${SHELLCHECK_REPORT}"
153+
echo "Either there are no info/warning/error messages for all shell scripts"
154+
echo "in the codebase or shellcheck failed to run successfully. Exiting..."
155+
exit 0
156+
fi
157+
fi
158+
159+
# view non-empty shellcheck report, includes info, warnings, errors
160+
if [[ "$VIEW" == "True" ]]; then
161+
echo "Shellcheck report: ${SHELLCHECK_REPORT}"
162+
cat "$SHELLCHECK_REPORT"
163+
fi
164+
165+
# detect if fatal errors are in shellcheck report
166+
echo "Checking for errors in shellcheck report with severity level ${SEVERITY}"
167+
READABLE_EXCLUDE_CODES=("${EXCLUDE_CODES//|/, }")
168+
echo "Excluding error codes: ${READABLE_EXCLUDE_CODES[*]}"; echo
169+
170+
# if any messages of severity level or more strict are present, use for grepping report
171+
case "$SEVERITY" in
172+
"info") REGEX_SEVERITY="info|warning|error" ;;
173+
"warning") REGEX_SEVERITY="warning|error" ;;
174+
*) REGEX_SEVERITY="error" ;;
175+
esac
176+
177+
# just grep report for severity level if no exclude codes, otherwise pipe and exclude codes from matches
178+
SEARCH_CMD() {
179+
if [[ "${#EXCLUDE_CODES[@]}" -eq 0 ]]; then
180+
grep -E "\(${REGEX_SEVERITY}\):" "$SHELLCHECK_REPORT"
181+
else
182+
grep -E "\(${REGEX_SEVERITY}\):" "$SHELLCHECK_REPORT" | grep -v -E "\"${EXCLUDE_CODES}"\"
183+
fi
184+
}
185+
186+
# if grep yielded no output then no violations of severity level or higher found (success)
187+
SEARCH_RESULTS="$(SEARCH_CMD)"
188+
if [[ -z "$SEARCH_RESULTS" ]]; then
189+
echo -e "${GREEN}[PASS]${RST_COLOR} Shellcheck did not detect violations scanning with severity level '${SEVERITY}'"
190+
echo; echo -e "${GREEN}Shellcheck passed.${RST_COLOR} See report: ${SHELLCHECK_REPORT}"; echo
191+
exit 0
192+
else
193+
COUNT=$(echo "$SEARCH_RESULTS" | wc -l | tr -d '[:space:]')
194+
echo -e "${RED}[FAIL]${RST_COLOR} Shellcheck found ${RED}${COUNT}${RST_COLOR}"\
195+
"unexcused violations scanning with severity level '${SEVERITY}'"; echo
196+
echo -e "${RED}Shellcheck failed.${RST_COLOR} See report: ${SHELLCHECK_REPORT}"; echo
197+
exit 1
198+
fi

scripts/test-e2e-minikube.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ BUILD_DOCKER=${TESTRUN_BUILD_DOCKER:-1}
77

88
# Make sure we have the necessary tools installed
99
required_executables=(minikube docker go helm kubectl)
10-
for e in ${required_executables[@]}; do
10+
for e in "${required_executables[@]}"; do
1111
if ! command -v $e &> /dev/null; then
1212
echo "'$e' command not be found! This is required to run. Please install it."
1313
exit 1

scripts/wait-for-it.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,7 @@ if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
150150
WAITFORIT_ISBUSY=1
151151
# Check if busybox timeout uses -t flag
152152
# (recent Alpine versions don't support -t anymore)
153-
if timeout &>/dev/stdout | grep -q -e '-t '; then
153+
if timeout |& tee /dev/stdout | grep -q -e '-t '; then
154154
WAITFORIT_BUSYTIMEFLAG="-t"
155155
fi
156156
else

0 commit comments

Comments
 (0)