Skip to content

Commit

Permalink
ci: add shellcheck gh action; fix fatal shellcheck errors
Browse files Browse the repository at this point in the history
This commit made with the assistance of github copilot

Signed-off-by: Morgan Rockett <[email protected]>
  • Loading branch information
rockett-m committed Aug 15, 2024
1 parent f6e5401 commit 1d4dc50
Show file tree
Hide file tree
Showing 8 changed files with 236 additions and 21 deletions.
28 changes: 21 additions & 7 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ jobs:
- uses: actions/checkout@v4
with:
submodules: recursive
- name: Setup Build Env
- name: Install Build Tools
run: sudo ./scripts/install-build-tools.sh
- name: Setup Local Dependencies
run: ./scripts/setup-dependencies.sh
Expand All @@ -38,7 +38,7 @@ jobs:
- uses: actions/checkout@v4
with:
submodules: recursive
- name: Setup Build Env
- name: Install Build Tools
run: sudo ./scripts/install-build-tools.sh
- name: Setup Local Dependencies
run: ./scripts/setup-dependencies.sh
Expand All @@ -50,7 +50,7 @@ jobs:
name: Pylint
runs-on: ubuntu-22.04
continue-on-error: true
timeout-minutes: 10
timeout-minutes: 5
strategy:
matrix:
python-version: ["3.10"]
Expand All @@ -62,10 +62,25 @@ jobs:
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: Setup Build Env
- name: Install Build Tools
run: sudo ./scripts/install-build-tools.sh
- name: Lint with Pylint
run: ./scripts/pylint.sh
shellcheck:
name: Shellcheck
runs-on: ubuntu-22.04
continue-on-error: true
timeout-minutes: 5
steps:
- uses: actions/checkout@v4
with:
submodules: recursive
- name: Install shellcheck
run: |
sudo apt-get update
sudo apt-get install -y shellcheck
- name: Lint with Shellcheck
run: ./scripts/shellcheck.sh -S error
unit-and-integration-test:
name: Unit and Integration Tests
runs-on: ubuntu-22.04
Expand All @@ -74,7 +89,7 @@ jobs:
- uses: actions/checkout@v4
with:
submodules: recursive
- name: Setup Build Env
- name: Install Build Tools
run: sudo ./scripts/install-build-tools.sh
- name: Setup Local Dependencies
run: ./scripts/setup-dependencies.sh
Expand All @@ -84,7 +99,7 @@ jobs:
run: ./scripts/test.sh
- name: Shorten SHA
id: vars
run: echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
- uses: actions/upload-artifact@v4
if: ${{ !env.ACT }}
name: Archive Test Results
Expand Down Expand Up @@ -114,4 +129,3 @@ jobs:
name: OpenCBDC Transaction Processor docs for ${{ steps.vars.outputs.sha_short }}
path: ./doxygen_generated/html/*
retention-days: 7

4 changes: 2 additions & 2 deletions scripts/create-e2e-report.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,9 @@ function readAndFormatLogs() {
return
fi

for logfile in $(ls $logdir); do
for logfile in "$logdir"/*; do
logfile_path="$logdir/$logfile"
logfile_content=$(cat $logfile_path)
logfile_content=$(<"$logfile_path")
message+="\n<details>\n<summary>$logfile</summary>\n\n\`\`\`\n$logfile_content\n\`\`\`\n</details>\n"
done
echo "$message"
Expand Down
2 changes: 1 addition & 1 deletion scripts/install-build-tools.sh
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ fi

# Supporting these versions for buildflow
PYTHON_VERSIONS=("3.10" "3.11" "3.12")
echo "Python3 versions supported: ${PYTHON_VERSIONS[@]}"
echo "Python3 versions supported: ${PYTHON_VERSIONS[*]}"

# check if supported version of python3 is already installed, and save the version
PY_INSTALLED=''
Expand Down
9 changes: 6 additions & 3 deletions scripts/lint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,12 @@ if [ -n "$whitespace_files" ] || [ -n "$newline_files" ] ; then
exit 1
fi

check_format_files=$(git ls-files | grep -E "tools|tests|src|cmake-tests" \
| grep -E "\..*pp")
clang-format --style=file --Werror --dry-run ${check_format_files[@]}
check_format_files=$(git ls-files | \
grep -E "tools|tests|src|cmake-tests" | \
grep -E "\..*pp")

echo "${check_format_files}" | \
xargs -n1 -I{} clang-format --style=file --Werror --dry-run {}

if ! command -v clang-tidy &>/dev/null; then
echo "clang-tidy does not appear to be installed"
Expand Down
12 changes: 6 additions & 6 deletions scripts/native-system-benchmark.sh
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ on_int() {
printf 'Interrupting all components\n'
trap '' SIGINT # avoid interrupting ourself
for i in $PIDS; do # intentionally unquoted
if [[ -n "RECORD" ]]; then
if [[ -n "$RECORD" ]]; then
kill -SIGINT -- "-$i"
else
kill -SIGINT -- "$i"
Expand Down Expand Up @@ -194,7 +194,7 @@ on_int() {

printf 'Terminating any remaining processes\n'
for i in $PIDS; do # intentionally unquoted
if [[ -n "RECORD" ]]; then
if [[ -n "$RECORD" ]]; then
kill -SIGTERM -- "-$i"
else
kill -SIGTERM -- "$i"
Expand Down Expand Up @@ -253,15 +253,15 @@ run() {
COMP=
case "$RECORD" in
perf)
$@ &> "$PROC_LOG" &
"$@" &> "$PROC_LOG" &
COMP="$!"
perf record -F 99 -a -g -o "$PNAME".perf -p "$COMP" &> "$PERF_LOG" &
PERFS="$PERFS $!";;
debug)
${DBG} "$@" &> "$PROC_LOG" &
COMP="$!";;
*)
$@ &> "$PROC_LOG" &
"$@" &> "$PROC_LOG" &
COMP="$!";;
esac

Expand Down Expand Up @@ -324,7 +324,7 @@ launch() {
"$RT"/scripts/wait-for-it.sh -q -t 5 -h localhost -p "$ep"
done
printf 'Launched logical %s %d, replica %d [PID: %d]\n' "$1" "$id" "$node" "$PID"
if [[ -n "RECORD" ]]; then
if [[ -n "$RECORD" ]]; then
PIDS="$PIDS $(getpgid $PID)"
else
PIDS="$PIDS $PID"
Expand All @@ -337,7 +337,7 @@ launch() {
"$RT"/scripts/wait-for-it.sh -q -t 5 -h localhost -p "$ep"
done
printf 'Launched %s %d [PID: %d]\n' "$1" "$id" "$PID"
if [[ -n "RECORD" ]]; then
if [[ -n "$RECORD" ]]; then
PIDS="$PIDS $(getpgid $PID)"
else
PIDS="$PIDS $PID"
Expand Down
198 changes: 198 additions & 0 deletions scripts/shellcheck.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,198 @@
#!/usr/bin/env bash

RED="\e[31m"
GREEN="\e[32m"
RST_COLOR="\e[0m"

if ! command -v shellcheck &>/dev/null; then
echo -e "${RED}[ERROR]${RST_COLOR} shellcheck is not installed."
echo "Run 'sudo ./scripts/install-build-tools.sh' to install shellcheck."
exit 1
fi

# Usage: ./scripts/shellcheck.sh [-e|--exclude-code=CODE] [-S|--severity=LEVEL] [-v|--view]
IFS='' read -rd '' usage <<'EOF'
Usage: %s [options]
Options:
-h, --help print this help and exit
-e, --exclude-code exclude specific error code, can be repeated
-S, --severity=LEVEL set severity level (info, warning, error)
-v, --view view shellcheck report
example: ./scripts/shellcheck.sh -e SC1091 -e SC1090 -S warning -v
EOF

echo; echo "Command line arguments: $0 $*"; echo

SEVERITY=
EXCLUDE_CODES=
VIEW="False"

_help=
_err=0
while [[ $# -gt 0 ]]; do
optarg=
shft_cnt=1
if [[ "$1" = '--' ]]; then
shift 1
break
elif [[ "$1" =~ [=] ]]; then
optarg="${1#*=}"
elif [[ "$1" =~ ^-- && $# -gt 1 && ! "$2" =~ ^- ]]; then
optarg="$2"
shft_cnt=2
elif [[ "$1" =~ ^-[^-] && $# -gt 1 && ! "$2" =~ ^- ]]; then
optarg="$2"
shft_cnt=2
elif [[ "$1" =~ ^-[^-] ]]; then
optarg="${1/??/}"
fi

case "$1" in
-S*|--severity*)
# don't let the user enter -S LEVEL more than once
# SEV=$(echo "${optarg}" | tr -d '[:space:]')
if [[ -n "$SEVERITY" ]]; then
printf "${RED}[Error]${RST_COLOR} Severity level already set to: %s\n" "${SEVERITY}"
_help=1; _err=1
# valid if -S has any of 'info', 'warning', 'error'
elif [[ "${optarg}" == "info" || "${optarg}" == "warning" || "${optarg}" == "error" ]]; then
SEVERITY="${optarg}"
else
# continue and disregard invalid severity level
printf "${RED}[Error]${RST_COLOR} severity level: %s\n" "${optarg}"
_err=1
fi
shift "$shft_cnt"
;;
-e*|--exclude-code*)
# strip whitespace from optarg
CODE=$(echo "${optarg}" | tr -d '[:space:]')
# valid if matching format SC1000-SC9999
if [[ "${optarg}" =~ ^SC[0-9]{4}$ ]]; then
# if empty then populate with just error code, otherwise add pipe before new code for grep later
if [[ -z "${EXCLUDE_CODES}" ]]; then
EXCLUDE_CODES+="${CODE}"
else
EXCLUDE_CODES+="|${CODE}"
fi
else
# continue just don't save invalid error code
printf "${RED}[Error]${RST_COLOR} Invalid error code entered: %s\n" "${optarg}"
_err=1
fi
shift "$shft_cnt"
;;
-v|--view)
VIEW="True"
shift "$shft_cnt"
;;
-h|--help)
_help=1
;;
*)
printf "${RED}[Error]${RST_COLOR} Unrecognized option: %s\n" "$1"
_err=1
shift "$shft_cnt"
;;
esac

# exit on help message
if [[ $_help -eq 1 ]]; then
printf "%s\n" "$usage"
exit 0
fi
# continue on invalid arg, let user know but don't exit
if [[ "$_err" -eq 1 ]]; then
printf "${RED}[Error]${RST_COLOR} Invalid argument: %s\n" "${optarg}"
printf "%s\n" "$usage"
fi
done

# if severity not set, set it to error as default
if [[ -z "$SEVERITY" ]]; then
SEVERITY="error"
fi

ROOT="$(cd "$(dirname "$0")"/.. && pwd)"
SHELLCHECK_REPORT="${ROOT}/shellcheck-report.txt"

NUM_CORES=1
if [[ "$OSTYPE" == "linux-gnu"* ]]; then
NUM_CORES=$(grep -c ^processor /proc/cpuinfo)
elif [[ "$OSTYPE" == "darwin"* ]]; then
NUM_CORES=$(sysctl -n hw.ncpu)
fi

# run shellcheck in parallel on all tracked shell scripts
#
# checking status of this run will give failure if info/warning/error is found by default
#
# determine status by parsing shellcheck report to see if any messages
# of the severity level or more strict are present to determine failure (true errors)

# check if git is installed
if command -v git &>/dev/null; then
echo "Using git ls-files to find shell scripts..."; echo
git ls-files '*.sh' | xargs -n 1 -P "$NUM_CORES" shellcheck > "$SHELLCHECK_REPORT"
else
echo "git is not installed. Using find to compile list of shell scripts..."; echo
if [[ -z "$EXCLUDE_CODES" ]]; then
find "$ROOT" -name '*.sh' -print0 | xargs -0 -n 1 -P "$NUM_CORES" shellcheck > "$SHELLCHECK_REPORT"
fi
fi

# if shell check report exists to determine if shellcheck run was successful
if [[ -z "$SHELLCHECK_REPORT" ]]; then
echo "${RED}[FAIL]${RST_COLOR}Shellcheck report ${SHELLCHECK_REPORT} not found. Exiting..."
exit 1
else
if [[ ! -s "$SHELLCHECK_REPORT" ]]; then
echo "Shellcheck report is empty: ${SHELLCHECK_REPORT}"
echo "Either there are no info/warning/error messages for all shell scripts"
echo "in the codebase or shellcheck failed to run successfully. Exiting..."
exit 0
fi
fi

# view non-empty shellcheck report, includes info, warnings, errors
if [[ "$VIEW" == "True" ]]; then
echo "Shellcheck report: ${SHELLCHECK_REPORT}"
cat "$SHELLCHECK_REPORT"
fi

# detect if fatal errors are in shellcheck report
echo "Checking for errors in shellcheck report with severity level ${SEVERITY}"
READABLE_EXCLUDE_CODES=("${EXCLUDE_CODES//|/, }")
echo "Excluding error codes: ${READABLE_EXCLUDE_CODES[*]}"; echo

# if any messages of severity level or more strict are present, use for grepping report
case "$SEVERITY" in
"info") REGEX_SEVERITY="info|warning|error" ;;
"warning") REGEX_SEVERITY="warning|error" ;;
*) REGEX_SEVERITY="error" ;;
esac

# just grep report for severity level if no exclude codes, otherwise pipe and exclude codes from matches
SEARCH_CMD() {
if [[ "${#EXCLUDE_CODES[@]}" -eq 0 ]]; then
grep -E "\(${REGEX_SEVERITY}\):" "$SHELLCHECK_REPORT"
else
grep -E "\(${REGEX_SEVERITY}\):" "$SHELLCHECK_REPORT" | grep -v -E "\"${EXCLUDE_CODES}"\"
fi
}

# if grep yielded no output then no violations of severity level or higher found (success)
SEARCH_RESULTS="$(SEARCH_CMD)"
if [[ -z "$SEARCH_RESULTS" ]]; then
echo -e "${GREEN}[PASS]${RST_COLOR} Shellcheck did not detect violations scanning with severity level '${SEVERITY}'"
echo; echo -e "${GREEN}Shellcheck passed.${RST_COLOR} See report: ${SHELLCHECK_REPORT}"; echo
exit 0
else
COUNT=$(echo "$SEARCH_RESULTS" | wc -l | tr -d '[:space:]')
echo -e "${RED}[FAIL]${RST_COLOR} Shellcheck found ${RED}${COUNT}${RST_COLOR}"\
"unexcused violations scanning with severity level '${SEVERITY}'"; echo
echo -e "${RED}Shellcheck failed.${RST_COLOR} See report: ${SHELLCHECK_REPORT}"; echo
exit 1
fi
2 changes: 1 addition & 1 deletion scripts/test-e2e-minikube.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ BUILD_DOCKER=${TESTRUN_BUILD_DOCKER:-1}

# Make sure we have the necessary tools installed
required_executables=(minikube docker go helm kubectl)
for e in ${required_executables[@]}; do
for e in "${required_executables[@]}"; do
if ! command -v $e &> /dev/null; then
echo "'$e' command not be found! This is required to run. Please install it."
exit 1
Expand Down
2 changes: 1 addition & 1 deletion scripts/wait-for-it.sh
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
WAITFORIT_ISBUSY=1
# Check if busybox timeout uses -t flag
# (recent Alpine versions don't support -t anymore)
if timeout &>/dev/stdout | grep -q -e '-t '; then
if timeout |& tee /dev/stdout | grep -q -e '-t '; then
WAITFORIT_BUSYTIMEFLAG="-t"
fi
else
Expand Down

0 comments on commit 1d4dc50

Please sign in to comment.