Merge pull request #17604 from dundargoc/ci/better-error-output

Better CI error output

.github/workflows/ci.yml

@@ -80,11 +80,11 @@ jobs:
         run: ./ci/before_script.sh
 
       - name: Build nvim
-        run: ./ci/run_tests.sh build
+        run: ./ci/run_tests.sh build_nvim
 
       - if: "!cancelled()"
-        name: clint
-        run: ./ci/run_lint.sh clint
+        name: clint-full
+        run: ./ci/run_lint.sh clint-full
 
       - if: "!cancelled()"
         name: lualint
@@ -99,8 +99,8 @@ jobs:
         run: ./ci/run_lint.sh shlint
 
       - if: "!cancelled()"
-        name: single-includes
-        run: ./ci/run_lint.sh single-includes
+        name: check-single-includes
+        run: ./ci/run_lint.sh check-single-includes
 
       - name: Cache dependencies
         run: ./ci/before_cache.sh
@@ -201,7 +201,7 @@ jobs:
         run: ./ci/before_script.sh
 
       - name: Build
-        run: ./ci/run_tests.sh build
+        run: ./ci/run_tests.sh build_nvim
 
       - if: matrix.flavor != 'tsan' && matrix.flavor != 'functionaltest-lua' && !cancelled()
         name: Unittests

ci/common/suite.sh

@@ -1,9 +1,3 @@
-# HACK: get newline for use in strings given that "\n" and $'' do not work.
-NL="$(printf '\nE')"
-NL="${NL%E}"
-
-FAIL_SUMMARY=""
-
 # Test success marker. If END_MARKER file exists, we know that all tests 
 # finished. If FAIL_SUMMARY_FILE exists we know that some tests failed, this 
 # file will contain information about failed tests. Build is considered 
@@ -11,35 +5,15 @@ FAIL_SUMMARY=""
 END_MARKER="$BUILD_DIR/.tests_finished"
 FAIL_SUMMARY_FILE="$BUILD_DIR/.test_errors"
 
-enter_suite() {
-  FAILED=0
-  rm -f "${END_MARKER}"
-  local suite_name="$1"
-  export NVIM_TEST_CURRENT_SUITE="${NVIM_TEST_CURRENT_SUITE}/$suite_name"
-}
-
-exit_suite() {
-  if test $FAILED -ne 0 ; then
-    echo "Suite ${NVIM_TEST_CURRENT_SUITE} failed, summary:"
-    echo "${FAIL_SUMMARY}"
-  fi
-  export NVIM_TEST_CURRENT_SUITE="${NVIM_TEST_CURRENT_SUITE%/*}"
-  FAILED=0
-}
-
 fail() {
   local test_name="$1"
-  local fail_char="$2"
-  local message="$3"
+  local message="$2"
 
-  : ${fail_char:=F}
   : ${message:=Test $test_name failed}
 
-  local full_msg="$fail_char $NVIM_TEST_CURRENT_SUITE|$test_name :: $message"
-  FAIL_SUMMARY="${FAIL_SUMMARY}${NL}${full_msg}"
+  local full_msg="$test_name :: $message"
   echo "${full_msg}" >> "${FAIL_SUMMARY_FILE}"
   echo "Failed: $full_msg"
-  FAILED=1
 }
 
 ended_successfully() {
@@ -64,13 +38,3 @@ end_tests() {
   touch "${END_MARKER}"
   ended_successfully
 }
-
-run_suite() {
-  local command="$1"
-  local suite_name="$2"
-
-  enter_suite "$suite_name"
-  eval "$command" || fail "$suite_name"
-  exit_suite
-}
-

ci/common/test.sh

@@ -51,7 +51,7 @@ check_core_dumps() {
     fi
   done
   if test "$app" != quiet ; then
-    fail 'cores' E 'Core dumps found'
+    fail 'cores' 'Core dumps found'
   fi
 }
 
@@ -72,7 +72,7 @@ check_logs() {
     rm "${log}"
   done
   if test -n "${err}" ; then
-    fail 'logs' E 'Runtime errors detected.'
+    fail 'logs' 'Runtime errors detected.'
   fi
 }
 
@@ -86,19 +86,19 @@ check_sanitizer() {
   fi
 }
 
-run_unittests() {(
+unittests() {(
   ulimit -c unlimited || true
   if ! build_make unittest ; then
-    fail 'unittests' F 'Unit tests failed'
+    fail 'unittests' 'Unit tests failed'
   fi
   submit_coverage unittest
   check_core_dumps "$(command -v luajit)"
 )}
 
-run_functionaltests() {(
+functionaltests() {(
   ulimit -c unlimited || true
   if ! build_make ${FUNCTIONALTEST}; then
-    fail 'functionaltests' F 'Functional tests failed'
+    fail 'functionaltests' 'Functional tests failed'
   fi
   submit_coverage functionaltest
   check_sanitizer "${LOG_DIR}"
@@ -106,11 +106,11 @@ run_functionaltests() {(
   check_core_dumps
 )}
 
-run_oldtests() {(
+oldtests() {(
   ulimit -c unlimited || true
   if ! make oldtest; then
     reset
-    fail 'oldtests' F 'Legacy tests failed'
+    fail 'oldtests' 'Legacy tests failed'
   fi
   submit_coverage oldtest
   check_sanitizer "${LOG_DIR}"
@@ -129,26 +129,25 @@ check_runtime_files() {(
     # Prefer failing the build over using more robust construct because files
     # with IFS are not welcome.
     if ! test -e "$file" ; then
-      fail "$test_name" E \
-        "It appears that $file is only a part of the file name"
+      fail "$test_name" "It appears that $file is only a part of the file name"
     fi
     if ! test "$tst" "$INSTALL_PREFIX/share/nvim/runtime/$file" ; then
-      fail "$test_name" F "$(printf "$message" "$file")"
+      fail "$test_name" "$(printf "$message" "$file")"
     fi
   done
 )}
 
 install_nvim() {(
   if ! build_make install ; then
-    fail 'install' E 'make install failed'
-    exit_suite
+    fail 'install' 'make install failed'
+    exit 1
   fi
 
   "${INSTALL_PREFIX}/bin/nvim" --version
   if ! "${INSTALL_PREFIX}/bin/nvim" -u NONE -e -c ':help' -c ':qall' ; then
     echo "Running ':help' in the installed nvim failed."
     echo "Maybe the helptags have not been generated properly."
-    fail 'help' F 'Failed running :help'
+    fail 'help' 'Failed running :help'
   fi
 
   # Check that all runtime files were installed
@@ -169,6 +168,6 @@ install_nvim() {(
   local genvimsynf=syntax/vim/generated.vim
   local gpat='syn keyword vimFuncName .*eval'
   if ! grep -q "$gpat" "${INSTALL_PREFIX}/share/nvim/runtime/$genvimsynf" ; then
-    fail 'funcnames' F "It appears that $genvimsynf does not contain $gpat."
+    fail 'funcnames' "It appears that $genvimsynf does not contain $gpat."
   fi
 )}

ci/run_lint.sh

@@ -8,34 +8,17 @@ CI_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 source "${CI_DIR}/common/build.sh"
 source "${CI_DIR}/common/suite.sh"
 
-if [[ "$GITHUB_ACTIONS" != "true" ]]; then
-  run_suite 'make clint-full' 'clint'
-  run_suite 'make lualint' 'lualint'
-  run_suite 'make pylint' 'pylint'
-  run_suite 'make shlint' 'shlint'
-  run_suite 'make check-single-includes' 'single-includes'
+rm -f "$END_MARKER"
 
-  end_tests
+# Run all tests if no input argument is given
+if (($# == 0)); then
+  tests=('clint-full' 'lualint' 'pylint' 'shlint' 'check-single-includes')
 else
-  case "$1" in
-    clint)
-      run_suite 'make clint-full' 'clint'
-      ;;
-    lualint)
-      run_suite 'make lualint' 'lualint'
-      ;;
-    pylint)
-      run_suite 'make pylint' 'pylint'
-      ;;
-    shlint)
-      run_suite 'make shlint' 'shlint'
-      ;;
-    single-includes)
-      run_suite 'make check-single-includes' 'single-includes'
-      ;;
-    *)
-      :;;
-  esac
-
-  end_tests
+  tests=("$@")
 fi
+
+for i in "${tests[@]}"; do
+  make "$i" || fail "$i"
+done
+
+end_tests

ci/run_tests.sh

@@ -8,42 +8,28 @@ source "${CI_DIR}/common/build.sh"
 source "${CI_DIR}/common/test.sh"
 source "${CI_DIR}/common/suite.sh"
 
+rm -f "$END_MARKER"
 
-if [[ "$GITHUB_ACTIONS" != "true" ]]; then
-  run_suite 'build_nvim' 'build'
+# Run all tests (with some caveats) if no input argument is given
+if (($# == 0)); then
+  tests=('build_nvim')
 
   if test "$CLANG_SANITIZER" != "TSAN"; then
     # Additional threads are only created when the builtin UI starts, which
     # doesn't happen in the unit/functional tests
     if test "${FUNCTIONALTEST}" != "functionaltest-lua"; then
-      run_suite run_unittests unittests
+      tests+=('unittests')
     fi
-    run_suite run_functionaltests functionaltests
+    tests+=('functionaltests')
   fi
-  run_suite run_oldtests oldtests
-  run_suite install_nvim install_nvim
 
-  end_tests
+  tests+=('oldtests' 'install_nvim')
 else
-  case "$1" in
-    build)
-      run_suite 'build_nvim' 'build'
-      ;;
-    unittests)
-      run_suite 'run_unittests' 'unittests'
-      ;;
-    functionaltests)
-      run_suite 'run_functionaltests' 'functionaltests'
-      ;;
-    oldtests)
-      run_suite 'run_oldtests' 'oldtests'
-      ;;
-    install_nvim)
-      run_suite 'install_nvim' 'install_nvim'
-      ;;
-    *)
-      :;;
-  esac
-
-  end_tests
+  tests=("$@")
 fi
+
+for i in "${tests[@]}"; do
+  eval "$i" || fail "$i"
+done
+
+end_tests