Fix races in pre-commit checks and improve error handling (#365)

# Summary

Introduction of parallelism in pre-commit hook introduced races or even
caused ignoring errors for some checks executed in background jobs.

## Changes
- in pre_commit function each check is send to background with tracking
its pid to. Previously we dispatched tasks into background and execute
`wait` builtin without arguments. While it's OK for just waiting until
completion it's ignoring any errors. Now, we're using `wait "${pid}"`,
which fails if the background job return non zero code.
- Note, that we cannot just use simple wait with $(jobs -p) because it's
race'y (see the next point)
- in start_shellcheck we're using xargs that errors if any of the checks
in background resulted in error. Previously there was a race condition
because until we reached `jobs -p` some of the checks might be already
failed and finished and it won't show on the job list.
  

## Proof of Work
* [Clean lint_repo in
evg](https://parsley.mongodb.com/evergreen/mongodb_kubernetes_unit_tests_lint_repo_patch_637923f515139843848eb9bd335bc6d66aab3f78_68aa3ac8e2d2f40007605d15_25_08_23_22_03_54/0/task?bookmarks=0%2C925&selectedLineRange=L916&shareLine=916)
```
[2025/08/24 00:11:23.223] pre-commit: All checks passed!
```

* [Deliberate error in
shellcheck](https://parsley.mongodb.com/evergreen/mongodb_kubernetes_unit_tests_lint_repo_patch_637923f515139843848eb9bd335bc6d66aab3f78_68aa4cf0a398bc00070a8e1b_25_08_23_23_21_21/0/task?bookmarks=0%2C974&selectedLineRange=L963&shareLine=963):
```
[2025/08/24 01:26:57.709] Some checks have failed:
[2025/08/24 01:26:57.709]     shellcheck (PID 4211)
```
<!-- Enter your proof that it works here.-->

## Checklist

- [ ] Have you linked a jira ticket and/or is the ticket in the title?
- [ ] Have you checked whether your jira ticket required DOCSP changes?
- [ ] Have you added changelog file?
    - use `skip-changelog` label if not needed
- refer to [Changelog files and Release
Notes](https://github.com/mongodb/mongodb-kubernetes/blob/master/CONTRIBUTING.md#changelog-files-and-release-notes)
section in CONTRIBUTING.md for more details

Author: lsierant

.githooks/pre-commit

@@ -18,14 +18,14 @@ fi
 
 mkdir -p "$(go env GOPATH)/bin"
 
-function update_mco_tests() {
+update_mco_tests() {
   echo "Regenerating MCO evergreen tests configuration"
   python scripts/evergreen/e2e/mco/create_mco_tests.py >.evergreen-mco.yml
   git add .evergreen-mco.yml
 }
 
 # Generates a yaml file to install the operator from the helm sources.
-function generate_standalone_yaml() {
+generate_standalone_yaml() {
   HELM_OPTS=$@
 
   charttmpdir=$(mktemp -d 2>/dev/null || mktemp -d -t 'charttmpdir')
@@ -73,7 +73,7 @@ function generate_standalone_yaml() {
 
 }
 
-function python_formatting() {
+python_formatting() {
   # installing Black
   if ! command -v "black" >/dev/null; then
     pip install -r requirements.txt
@@ -85,15 +85,15 @@ function python_formatting() {
   black .
 }
 
-function generate_manifests() {
+generate_manifests() {
   make manifests
 
   git add config/crd/bases
   git add helm_chart/crds
   git add public/crds.yaml
 }
 
-function update_values_yaml_files() {
+update_values_yaml_files() {
   # ensure that all helm values files are up to date.
   # shellcheck disable=SC2154
   python scripts/evergreen/release/update_helm_values_files.py
@@ -107,7 +107,7 @@ function update_values_yaml_files() {
   git add go.sum
 }
 
-function update_release_json() {
+update_release_json() {
   # ensure that release.json is up 2 date
   # shellcheck disable=SC2154
   python scripts/evergreen/release/update_release.py
@@ -116,7 +116,7 @@ function update_release_json() {
   git add release.json
 }
 
-function regenerate_public_rbac_multi_cluster() {
+regenerate_public_rbac_multi_cluster() {
     if echo "$git_last_changed" | grep -q -e 'cmd/kubectl-mongodb' -e 'pkg/kubectl-mongodb'; then
       echo 'regenerating multicluster RBAC public example'
       pushd pkg/kubectl-mongodb/common/
@@ -126,78 +126,128 @@ function regenerate_public_rbac_multi_cluster() {
     fi
 }
 
-function update_licenses() {
-    echo 'regenerating licenses'
-    time scripts/evergreen/update_licenses.sh 2>&1 | prepend "update_licenses"
-    git add LICENSE-THIRD-PARTY
+update_licenses() {
+    if [[ "${MDB_UPDATE_LICENSES:-""}" == "true" ]]; then
+      echo 'regenerating licenses'
+      time scripts/evergreen/update_licenses.sh 2>&1 | prepend "update_licenses"
+      git add LICENSE-THIRD-PARTY
+    fi
 }
 
-function check_erroneous_kubebuilder_annotations() {
+check_erroneous_kubebuilder_annotations() {
   # Makes sure there are not erroneous kubebuilder annotations that can
   # end up in CRDs as descriptions.
   if grep "// kubebuilder" ./* -r --exclude-dir=vendor --include=\*.go; then
-    echo "Found an erroneous kubebuilder annotation"
+    echo -e "${RED}Found an erroneous kubebuilder annotation${NO_COLOR}"
     exit 1
   fi
 }
 
-function check_incorrect_makefile_variable_brackets() {
+check_incorrect_makefile_variable_brackets() {
   if find . -name "Makefile" | grep -v vendor | xargs grep "\${"; then
-    echo 'ERROR: Makefiles should NEVER contain curly brackets variables'
+    echo -e "${RED}ERROR: Makefiles should NEVER contain curly brackets variables${NO_COLOR}"
     exit 1
   fi
 }
 
-function pre_commit() {
-  if [[ "${MDB_UPDATE_LICENSES:-""}" == "true" ]]; then
-    ( (time update_licenses) 2>&1 | prepend "update_licenses" ) &
-  fi
-  ( (time scripts/evergreen/lint_code.sh) 2>&1 | prepend "lint_code.sh" ) &
-  ( (time start_shellcheck) 2>&1 | prepend "shellcheck" ) &
+update_jobs() {
+     # Update release.json first in case there is a newer version
+    time update_release_json
+    # We need to generate the values files first
+    time update_values_yaml_files
+    # The values files are used for generating the standalone yaml
+    time generate_standalone_yaml
+}
 
-  # Update release.json first in case there is a newer version
-  (time update_release_json) 2>&1 | prepend "update_release_json"
-  # We need to generate the values files first
-  (time update_values_yaml_files) 2>&1 | prepend "update_values_yaml_files"
-  # The values files are used for generating the standalone yaml
-  (time generate_standalone_yaml) 2>&1 | prepend "generate_standalone_yaml"
+lint_code() {
+  scripts/evergreen/lint_code.sh
+}
+
+# bg_job_ vars are global; run_job_in_background function is appending to them on each call
+bg_job_pids=()
+bg_job_pids_with_names=()
+
+get_job_name() {
+  local search_pid="$1"
+  local match
+  match=$(printf '%s\n' "${bg_job_pids_with_names[@]}" | grep "^${search_pid}:")
+  echo "${match#*:}"  # Remove everything up to and including the colon
+}
 
-  ( (time regenerate_public_rbac_multi_cluster) 2>&1 | prepend "regenerate_public_rbac_multi_cluster" ) &
+# Executes function given on the first argument as background job.
+# It's ensuring logs are properly prefixed by the name and
+# the job's pid is captured in bg_jobs array in order to wait for completion.
+run_job_in_background() {
+  job_name=$1
+  time ${job_name} 2>&1 | prepend "${job_name}" &
+
+  local job_pid=$!
+  bg_job_pids+=("${job_pid}")
+  bg_job_pids_with_names+=("${job_pid}:${job_name}")
+  echo "Started ${job_name} with PID: ${job_pid}"
+}
 
-  # Run black and isort on python files that have changed
-  ( (time python_formatting) 2>&1 | prepend "python_formatting") &
+# Waits for all background jobs stored in bg_job_pids and check their exit codes.
+wait_for_all_background_jobs() {
+    failures=()
+    for pid in "${bg_job_pids[@]}"; do
+      wait "${pid}" || {
+        job_name=$(get_job_name "${pid}")
+        failures+=("    ${RED}${job_name} (PID ${pid})${NO_COLOR}")
+      }
+    done
+
+    if [[ ${#failures[@]} -gt 0 ]]; then
+      echo -e "${RED}Some checks have failed:${NO_COLOR}"
+      for failure in "${failures[@]}"; do
+        echo -e "$failure"
+      done
+      echo -e "${RED}To see the details look for the job's logs by it's prefixed name (e.g. \"shellcheck:\").${NO_COLOR}"
+      return 1
+    fi
 
-  ( (time check_erroneous_kubebuilder_annotations) 2>&1 | prepend "check_erroneous_kubebuilder_annotations" ) &
+    return 0
+}
 
-  wait
+pre_commit() {
+  run_job_in_background "update_jobs"
+  run_job_in_background "update_licenses"
+  run_job_in_background "lint_code"
+  run_job_in_background "start_shellcheck"
+  run_job_in_background "regenerate_public_rbac_multi_cluster"
+  run_job_in_background "python_formatting"
+  run_job_in_background "check_erroneous_kubebuilder_annotations"
+
+  if wait_for_all_background_jobs; then
+    echo -e "${GREEN}pre-commit: All checks passed!${NO_COLOR}"
+    return 0
+  else
+    return 1
+  fi
 }
 
 # Function to run shellcheck on a single file
 run_shellcheck() {
   local file="$1"
   echo "Running shellcheck on $file"
   if ! shellcheck --color=always -x "$file" -e SC2154 -e SC1091 -e SC1090 -e SC2148 -o require-variable-braces -P "scripts"; then
-    echo "shellcheck failed on $file"
+    echo -e "${RED}shellcheck failed on $file${NO_COLOR}"
     exit 1
   fi
 }
 
-start_shellcheck() {
-  files_1=$(find scripts -type f -name "*.sh")
-  files_2=$(find scripts/dev/contexts -type f | grep -v private-context)
-  files_3=$(find scripts/funcs -type f)
-  files_4=$(find public/architectures -type f -name "*.sh")
-  files=$(echo -e "$files_1\n$files_2\n$files_3\n$files_4")
-  # Process each file in parallel
-  for file in $files; do
-    run_shellcheck "$file" &
-  done
-
-  # Wait for all background jobs
-  for job in $(jobs -p); do
-    wait "$job" || exit 1
-  done
+# Export function so it's available in subshells (for xargs)
+export -f run_shellcheck
 
+start_shellcheck() {
+  # shellcheck disable=SC2016
+  {
+    find scripts -type f -name "*.sh"
+    find scripts/dev/contexts -type f | grep -v private-context
+    find scripts/funcs -type f
+    find public/architectures -type f -name "*.sh"
+    find docs/ -type f -name "*.sh"
+  } | xargs -I {} -P 20 bash -c 'run_shellcheck "$1"' _ {}
 }
 
 cmd=${1:-"pre-commit"}
@@ -210,5 +260,5 @@ elif [[ "${cmd}" == "pre-commit" ]]; then
 elif [[ "${cmd}" == "shellcheck" ]]; then
   start_shellcheck
 elif [[ "${cmd}" == "lint" ]]; then
-  source scripts/evergreen/lint_code.sh
+  lint_code
 fi

scripts/funcs/printing

@@ -20,4 +20,5 @@ prepend() {
 }
 
 export RED='\033[0;31m'
+export GREEN='\033[0;32m'
 export NO_COLOR='\033[0m'