To check that tests are passing we must parse the kselftest runner
output. It is much more convenient to check the return value of the
runner instead: 0 is OK, and 1 implies one or more errors.
This has an impact on the kselftest's gen_tar bundle and the run_tests
make target.
Backporting this change would be useful to consistently test older
kernels as well.
Cc: Greg Kroah-Hartman <[email protected]>
Cc: SeongJae Park <[email protected]>
Cc: Shuah Khan <[email protected]>
Cc: [email protected] # 303f8e2d0200: selftests/kselftest/runner/run_one(): allow running non-executable files
Signed-off-by: Mickaël Salaün <[email protected]>
Link: https://lore.kernel.org/r/[email protected]
---
tools/testing/selftests/kselftest/runner.sh | 36 +++++++++++++--------
tools/testing/selftests/run_kselftest.sh | 4 ++-
2 files changed, 26 insertions(+), 14 deletions(-)
diff --git a/tools/testing/selftests/kselftest/runner.sh b/tools/testing/selftests/kselftest/runner.sh
index 294619ade49f..f6488a53a78c 100644
--- a/tools/testing/selftests/kselftest/runner.sh
+++ b/tools/testing/selftests/kselftest/runner.sh
@@ -46,6 +46,8 @@ run_one()
DIR="$1"
TEST="$2"
NUM="$3"
+ local rc=1
+ local ret=1
BASENAME_TEST=$(basename $TEST)
@@ -107,29 +109,36 @@ run_one()
cmd="$interpreter ./$BASENAME_TEST"
else
echo "not ok $test_num $TEST_HDR_MSG"
- return
+ return 1
fi
fi
cd `dirname $TEST` > /dev/null
- ((((( tap_timeout "$cmd" 2>&1; echo $? >&3) |
+ if (((( tap_timeout "$cmd" 2>&1; echo $? >&3) |
tap_prefix >&4) 3>&1) |
- (read xs; exit $xs)) 4>>"$logfile" &&
- echo "ok $test_num $TEST_HDR_MSG") ||
- (rc=$?; \
- if [ $rc -eq $skip_rc ]; then \
- echo "ok $test_num $TEST_HDR_MSG # SKIP"
- elif [ $rc -eq $timeout_rc ]; then \
- echo "#"
- echo "not ok $test_num $TEST_HDR_MSG # TIMEOUT $kselftest_timeout seconds"
+ (read xs; exit $xs)) 4>>"$logfile"; then
+ echo "ok $test_num $TEST_HDR_MSG"
+ ret=0
else
- echo "not ok $test_num $TEST_HDR_MSG # exit=$rc"
- fi)
+ rc=$?
+ if [ $rc -eq $skip_rc ]; then
+ echo "ok $test_num $TEST_HDR_MSG # SKIP"
+ ret=0
+ elif [ $rc -eq $timeout_rc ]; then
+ echo "#"
+ echo "not ok $test_num $TEST_HDR_MSG # TIMEOUT $kselftest_timeout seconds"
+ else
+ echo "not ok $test_num $TEST_HDR_MSG # exit=$rc"
+ fi
+ fi
cd - >/dev/null
fi
+ return $ret
}
run_many()
{
+ local ret=0
+
echo "TAP version 13"
DIR="${PWD#${BASE_DIR}/}"
test_num=0
@@ -142,6 +151,7 @@ run_many()
logfile="/tmp/$BASENAME_TEST"
cat /dev/null > "$logfile"
fi
- run_one "$DIR" "$TEST" "$test_num"
+ run_one "$DIR" "$TEST" "$test_num" || ret=1
done
+ return $ret
}
diff --git a/tools/testing/selftests/run_kselftest.sh b/tools/testing/selftests/run_kselftest.sh
index 97165a83df63..6164314f837e 100755
--- a/tools/testing/selftests/run_kselftest.sh
+++ b/tools/testing/selftests/run_kselftest.sh
@@ -85,9 +85,11 @@ if [ -n "$TESTS" ]; then
available="$(echo "$valid" | sed -e 's/ /\n/g')"
fi
+ret=0
collections=$(echo "$available" | cut -d: -f1 | uniq)
for collection in $collections ; do
[ -w /dev/kmsg ] && echo "kselftest: Running tests in $collection" >> /dev/kmsg
tests=$(echo "$available" | grep "^$collection:" | cut -d: -f2)
- ($dryrun cd "$collection" && $dryrun run_many $tests)
+ ($dryrun cd "$collection" && $dryrun run_many $tests) || ret=1
done
+exit $ret
base-commit: 7e284070abe53d448517b80493863595af4ab5f0
--
2.36.0
Greeting,
FYI, we noticed the following commit (built with gcc-11):
commit: 3c89247543c0ca796c2bf7c3c15ba82d6f5d2c47 ("[PATCH v1] selftests/kselftest: Make failed tests exit with 1")
url: https://github.com/intel-lab-lkp/linux/commits/Micka-l-Sala-n/selftests-kselftest-Make-failed-tests-exit-with-1/20220528-002910
patch link: https://lore.kernel.org/lkml/[email protected]
in testcase: kernel-selftests
version: kernel-selftests-x86_64-8d3977ef-1_20220523
with following parameters:
sc_nr_hugepages: 2
group: vm
ucode: 0xc2
test-description: The kernel contains a set of "self tests" under the tools/testing/selftests/ directory. These are intended to be small unit tests to exercise individual code paths in the kernel.
test-url: https://www.kernel.org/doc/Documentation/kselftest.txt
on test machine: 20 threads 1 sockets Commet Lake with 16G memory
caused below changes (please refer to attached dmesg/kmsg for entire log/backtrace):
If you fix the issue, kindly add following tag
Reported-by: kernel test robot <[email protected]>
KERNEL SELFTESTS: linux_headers_dir is /usr/src/linux-headers-x86_64-rhel-8.3-kselftests-3c89247543c0ca796c2bf7c3c15ba82d6f5d2c47
2022-05-29 09:56:50 ln -sf /usr/bin/clang
2022-05-29 09:56:50 ln -sf /usr/bin/llc
2022-05-29 09:56:51 sed -i s/default_timeout=45/default_timeout=300/ kselftest/runner.sh
LKP WARN miss config CONFIG_MEM_SOFT_DIRTY= of vm/config
2022-05-29 09:56:51 make -C vm
make: Entering directory '/usr/src/perf_selftests-x86_64-rhel-8.3-kselftests-3c89247543c0ca796c2bf7c3c15ba82d6f5d2c47/tools/testing/selftests/vm'
/bin/sh ./check_config.sh gcc
make --no-builtin-rules ARCH=x86 -C ../../../.. headers_install
make[1]: Entering directory '/usr/src/perf_selftests-x86_64-rhel-8.3-kselftests-3c89247543c0ca796c2bf7c3c15ba82d6f5d2c47'
....
ok 4 selftests: vm: run_vmtests.sh # SKIP
make: *** [../lib.mk:94: run_tests] Error 1
make: Leaving directory '/usr/src/perf_selftests-x86_64-rhel-8.3-kselftests-3c89247543c0ca796c2bf7c3c15ba82d6f5d2c47/tools/testing/selftests/vm'
To reproduce:
git clone https://github.com/intel/lkp-tests.git
cd lkp-tests
sudo bin/lkp install job.yaml # job file is attached in this email
bin/lkp split-job --compatible job.yaml # generate the yaml file for lkp run
sudo bin/lkp run generated-yaml-file
# if come across any failure that blocks the test,
# please remove ~/.lkp and /lkp dir to run from a clean state.
--
0-DAY CI Kernel Test Service
https://01.org/lkp