diff --git a/ci/common/suite.sh b/ci/common/suite.sh
index c0c470dce1..0320ac15c3 100644
--- a/ci/common/suite.sh
+++ b/ci/common/suite.sh
@@ -14,6 +14,7 @@ fail() {
   local full_msg="$test_name :: $message"
   echo "${full_msg}" >> "${FAIL_SUMMARY_FILE}"
   echo "Failed: $full_msg"
+  FAILED=1
 }
 
 ended_successfully() {
diff --git a/src/nvim/testdir/runnvim.sh b/src/nvim/testdir/runnvim.sh
index 72d88f9f93..322265737a 100755
--- a/src/nvim/testdir/runnvim.sh
+++ b/src/nvim/testdir/runnvim.sh
@@ -38,7 +38,7 @@ main() {(
     -S runnvim.vim \
     "$tlog" > "out-$tlog" 2> "err-$tlog"
   then
-    fail "$test_name" F "Nvim exited with non-zero code"
+    fail "$test_name" "Nvim exited with non-zero code"
   fi
   {
     echo "Stdout of :terminal runner"
@@ -53,7 +53,7 @@ main() {(
   if test "$oldesttest" = 1 ; then
     if ! diff -q test.out "$test_name.ok" > /dev/null 2>&1 ; then
       if test -f test.out ; then
-        fail "$test_name" F "Oldest test .out file differs from .ok file"
+        fail "$test_name" "Oldest test .out file differs from .ok file"
         {
           echo "Diff between test.out and $test_name.ok"
           echo "$separator"