Received-SPF: softfail (google.com: domain of transitioning linux-kernel-owner@vger.kernel.org does not designate 23.128.96.19 as permitted sender) client-ip=23.128.96.19;
From:   =?UTF-8?q?Micka=C3=ABl=20Sala=C3=BCn?= <mic@digikod.net>
To:     Shuah Khan <shuah@kernel.org>
Cc:     =?UTF-8?q?Micka=C3=ABl=20Sala=C3=BCn?= <mic@digikod.net>,
        Greg Kroah-Hartman <gregkh@linuxfoundation.org>,
        SeongJae Park <sjpark@amazon.de>, linux-kernel@vger.kernel.org,
        linux-kselftest@vger.kernel.org, stable@vger.kernel.org
Subject: [PATCH v1] selftests/kselftest: Make failed tests exit with 1
Date:   Fri, 27 May 2022 18:24:17 +0200
Message-Id: <20220527162417.2646998-1-mic@digikod.net>
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Precedence: bulk

To check that tests are passing we must parse the kselftest runner
output.  It is much more convenient to check the return value of the
runner instead: 0 is OK, and 1 implies one or more errors.

This has an impact on the kselftest's gen_tar bundle and the run_tests
make target.

Backporting this change would be useful to consistently test older
kernels as well.

Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
Cc: SeongJae Park <sjpark@amazon.de>
Cc: Shuah Khan <shuah@kernel.org>
Cc: stable@vger.kernel.org # 303f8e2d0200: selftests/kselftest/runner/run_one(): allow running non-executable files
Signed-off-by: Mickaël Salaün <mic@digikod.net>
Link: https://lore.kernel.org/r/20220527162417.2646998-1-mic@digikod.net
---
 tools/testing/selftests/kselftest/runner.sh | 36 +++++++++++++--------
 tools/testing/selftests/run_kselftest.sh    |  4 ++-
 2 files changed, 26 insertions(+), 14 deletions(-)

diff --git a/tools/testing/selftests/kselftest/runner.sh b/tools/testing/selftests/kselftest/runner.sh
index 294619ade49f..f6488a53a78c 100644
--- a/tools/testing/selftests/kselftest/runner.sh
+++ b/tools/testing/selftests/kselftest/runner.sh
@@ -46,6 +46,8 @@ run_one()
 	DIR="$1"
 	TEST="$2"
 	NUM="$3"
+	local rc=1
+	local ret=1
 
 	BASENAME_TEST=$(basename $TEST)
 
@@ -107,29 +109,36 @@ run_one()
 				cmd="$interpreter ./$BASENAME_TEST"
 			else
 				echo "not ok $test_num $TEST_HDR_MSG"
-				return
+				return 1
 			fi
 		fi
 		cd `dirname $TEST` > /dev/null
-		((((( tap_timeout "$cmd" 2>&1; echo $? >&3) |
+		if (((( tap_timeout "$cmd" 2>&1; echo $? >&3) |
 			tap_prefix >&4) 3>&1) |
-			(read xs; exit $xs)) 4>>"$logfile" &&
-		echo "ok $test_num $TEST_HDR_MSG") ||
-		(rc=$?;	\
-		if [ $rc -eq $skip_rc ]; then	\
-			echo "ok $test_num $TEST_HDR_MSG # SKIP"
-		elif [ $rc -eq $timeout_rc ]; then \
-			echo "#"
-			echo "not ok $test_num $TEST_HDR_MSG # TIMEOUT $kselftest_timeout seconds"
+			(read xs; exit $xs)) 4>>"$logfile"; then
+			echo "ok $test_num $TEST_HDR_MSG"
+			ret=0
 		else
-			echo "not ok $test_num $TEST_HDR_MSG # exit=$rc"
-		fi)
+			rc=$?
+			if [ $rc -eq $skip_rc ]; then
+				echo "ok $test_num $TEST_HDR_MSG # SKIP"
+				ret=0
+			elif [ $rc -eq $timeout_rc ]; then
+				echo "#"
+				echo "not ok $test_num $TEST_HDR_MSG # TIMEOUT $kselftest_timeout seconds"
+			else
+				echo "not ok $test_num $TEST_HDR_MSG # exit=$rc"
+			fi
+		fi
 		cd - >/dev/null
 	fi
+	return $ret
 }
 
 run_many()
 {
+	local ret=0
+
 	echo "TAP version 13"
 	DIR="${PWD#${BASE_DIR}/}"
 	test_num=0
@@ -142,6 +151,7 @@ run_many()
 			logfile="/tmp/$BASENAME_TEST"
 			cat /dev/null > "$logfile"
 		fi
-		run_one "$DIR" "$TEST" "$test_num"
+		run_one "$DIR" "$TEST" "$test_num" || ret=1
 	done
+	return $ret
 }
diff --git a/tools/testing/selftests/run_kselftest.sh b/tools/testing/selftests/run_kselftest.sh
index 97165a83df63..6164314f837e 100755
--- a/tools/testing/selftests/run_kselftest.sh
+++ b/tools/testing/selftests/run_kselftest.sh
@@ -85,9 +85,11 @@ if [ -n "$TESTS" ]; then
 	available="$(echo "$valid" | sed -e 's/ /\n/g')"
 fi
 
+ret=0
 collections=$(echo "$available" | cut -d: -f1 | uniq)
 for collection in $collections ; do
 	[ -w /dev/kmsg ] && echo "kselftest: Running tests in $collection" >> /dev/kmsg
 	tests=$(echo "$available" | grep "^$collection:" | cut -d: -f2)
-	($dryrun cd "$collection" && $dryrun run_many $tests)
+	($dryrun cd "$collection" && $dryrun run_many $tests) || ret=1
 done
+exit $ret

base-commit: 7e284070abe53d448517b80493863595af4ab5f0
-- 
2.36.0