This series demonstrates how KTAP output can be used by nolibc-test to
make the test results better to read for people and machines.
Especially when running multiple invocations for different architectors
or build configurations we can make use of the kernels TAP parser to
automatically provide aggregated test reports.
The code is very hacky and incomplete and mostly meant to validate if
the output format is useful.
Start with the last patch of the series to actually see the generated
format, or run it for yourself.
Signed-off-by: Thomas Weißschuh <[email protected]>
---
Thomas Weißschuh (7):
selftests/nolibc: statically calculate number of testsuites
selftests/nolibc: use unsigned indices for testcases
selftests/nolibc: replace repetitive test structure with macro
selftests/nolibc: count subtests
kselftest: support KTAP format
kselftest: support skipping tests with testname
selftests/nolibc: proof of concept for TAP output
tools/testing/selftests/kselftest.h | 20 +++
tools/testing/selftests/nolibc/nolibc-test.c | 197 ++++++++++--------------
tools/testing/selftests/nolibc/run-all-tests.sh | 22 +++
3 files changed, 127 insertions(+), 112 deletions(-)
---
base-commit: dfef4fc45d5713eb23d87f0863aff9c33bd4bfaf
change-id: 20230718-nolibc-ktap-tmp-4408f505408d
Best regards,
--
Thomas Weißschuh <[email protected]>
They should never be negative.
Signed-off-by: Thomas Weißschuh <[email protected]>
---
tools/testing/selftests/nolibc/nolibc-test.c | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/tools/testing/selftests/nolibc/nolibc-test.c b/tools/testing/selftests/nolibc/nolibc-test.c
index 698af242d3e4..1bd99e0fab4d 100644
--- a/tools/testing/selftests/nolibc/nolibc-test.c
+++ b/tools/testing/selftests/nolibc/nolibc-test.c
@@ -62,7 +62,7 @@ static const char *argv0;
/* definition of a series of tests */
struct test {
const char *name; /* test name */
- int (*func)(int min, int max); /* handler */
+ int (*func)(unsigned int min, unsigned int max); /* handler */
};
#ifndef _NOLIBC_STDLIB_H
@@ -570,7 +570,7 @@ static int expect_strne(const char *expr, int llen, const char *cmp)
#define CASE_TEST(name) \
case __LINE__: llen += printf("%d %s", test, #name);
-int run_startup(int min, int max)
+int run_startup(unsigned int min, unsigned int max)
{
int test;
int ret = 0;
@@ -773,7 +773,7 @@ int test_mmap_munmap(void)
/* Run syscall tests between IDs <min> and <max>.
* Return 0 on success, non-zero on failure.
*/
-int run_syscall(int min, int max)
+int run_syscall(unsigned int min, unsigned int max)
{
struct timeval tv;
struct timezone tz;
@@ -884,7 +884,7 @@ int run_syscall(int min, int max)
return ret;
}
-int run_stdlib(int min, int max)
+int run_stdlib(unsigned int min, unsigned int max)
{
int test;
int tmp;
@@ -1027,7 +1027,7 @@ static int expect_vfprintf(int llen, size_t c, const char *expected, const char
return ret;
}
-static int run_vfprintf(int min, int max)
+static int run_vfprintf(unsigned int min, unsigned int max)
{
int test;
int tmp;
@@ -1070,7 +1070,7 @@ static int smash_stack(void)
return 1;
}
-static int run_protection(int min, int max)
+static int run_protection(unsigned int min, unsigned int max)
{
pid_t pid;
int llen = 0, status;
--
2.41.0
This makes it slightly nicer to report the number of suites.
Signed-off-by: Thomas Weißschuh <[email protected]>
---
tools/testing/selftests/nolibc/nolibc-test.c | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/tools/testing/selftests/nolibc/nolibc-test.c b/tools/testing/selftests/nolibc/nolibc-test.c
index 03b1d30f5507..698af242d3e4 100644
--- a/tools/testing/selftests/nolibc/nolibc-test.c
+++ b/tools/testing/selftests/nolibc/nolibc-test.c
@@ -45,6 +45,8 @@
#define SINT_MAX_OF_TYPE(type) (((type)1 << (sizeof(type) * 8 - 2)) - (type)1 + ((type)1 << (sizeof(type) * 8 - 2)))
#define SINT_MIN_OF_TYPE(type) (-SINT_MAX_OF_TYPE(type) - 1)
+#define ARRAY_SIZE(x) (sizeof(x) / sizeof(x[0]))
+
/* will be used to test initialization of environ */
static char **test_envp;
@@ -1184,7 +1186,6 @@ static const struct test test_names[] = {
{ .name = "stdlib", .func = run_stdlib },
{ .name = "vfprintf", .func = run_vfprintf },
{ .name = "protection", .func = run_protection },
- { 0 }
};
int is_setting_valid(char *test)
@@ -1259,7 +1260,7 @@ int main(int argc, char **argv, char **envp)
if (colon)
*(colon++) = '\0';
- for (idx = 0; test_names[idx].name; idx++) {
+ for (idx = 0; idx < ARRAY_SIZE(test_names); idx++) {
if (strcmp(test, test_names[idx].name) == 0)
break;
}
@@ -1305,7 +1306,7 @@ int main(int argc, char **argv, char **envp)
} while (test && *test);
} else {
/* no test mentioned, run everything */
- for (idx = 0; test_names[idx].name; idx++) {
+ for (idx = 0; idx < ARRAY_SIZE(test_names); idx++) {
printf("Running test '%s'\n", test_names[idx].name);
err = test_names[idx].func(min, max);
ret += err;
--
2.41.0
This will be needed for KTAP output.
Signed-off-by: Thomas Weißschuh <[email protected]>
---
tools/testing/selftests/nolibc/nolibc-test.c | 30 +++++++++++++++++++++-------
1 file changed, 23 insertions(+), 7 deletions(-)
diff --git a/tools/testing/selftests/nolibc/nolibc-test.c b/tools/testing/selftests/nolibc/nolibc-test.c
index 76bd6a0a8132..03f64ce1dda6 100644
--- a/tools/testing/selftests/nolibc/nolibc-test.c
+++ b/tools/testing/selftests/nolibc/nolibc-test.c
@@ -65,6 +65,13 @@ struct test {
int (*func)(unsigned int min, unsigned int max); /* handler */
};
+#define COUNT_SUBTESTS (-1)
+
+static unsigned int count_subtests(const struct test *test)
+{
+ return test->func(COUNT_SUBTESTS, COUNT_SUBTESTS);
+}
+
#ifndef _NOLIBC_STDLIB_H
char *itoa(int i)
{
@@ -571,10 +578,10 @@ static int expect_strne(const char *expr, int llen, const char *cmp)
case __LINE__: llen += printf("%d %s", test, #name);
#define SWITCH_TEST \
- switch (test + __LINE__ + 1) {
+ int _tests_start = __LINE__; switch (test + __LINE__ + 1) {
#define SWITCH_TEST_END \
- case __LINE__: return ret; }
+ case __LINE__: return ret; default: return __LINE__ - _tests_start - 1; }
int run_startup(unsigned int min, unsigned int max)
@@ -592,7 +599,7 @@ int run_startup(unsigned int min, unsigned int max)
test_auxv = _auxv;
#endif
- for (test = min; test >= 0 && test <= max; test++) {
+ for (test = min; test == COUNT_SUBTESTS || (test >= 0 && test <= max); test++) {
int llen = 0; /* line length */
/* avoid leaving empty lines below, this will insert holes into
@@ -801,7 +808,7 @@ int run_syscall(unsigned int min, unsigned int max)
has_gettid = __GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 30);
#endif
- for (test = min; test >= 0 && test <= max; test++) {
+ for (test = min; test == COUNT_SUBTESTS || (test >= 0 && test <= max); test++) {
int llen = 0; /* line length */
/* avoid leaving empty lines below, this will insert holes into
@@ -892,7 +899,7 @@ int run_stdlib(unsigned int min, unsigned int max)
int ret = 0;
void *p1, *p2;
- for (test = min; test >= 0 && test <= max; test++) {
+ for (test = min; test == COUNT_SUBTESTS || (test >= 0 && test <= max); test++) {
int llen = 0; /* line length */
/* avoid leaving empty lines below, this will insert holes into
@@ -1032,7 +1039,7 @@ static int run_vfprintf(unsigned int min, unsigned int max)
int ret = 0;
void *p1, *p2;
- for (test = min; test >= 0 && test <= max; test++) {
+ for (test = min; test == COUNT_SUBTESTS || (test >= 0 && test <= max); test++) {
int llen = 0; /* line length */
/* avoid leaving empty lines below, this will insert holes into
@@ -1070,6 +1077,9 @@ static int run_protection(unsigned int min, unsigned int max)
pid_t pid;
int llen = 0, status;
+ if (min == COUNT_SUBTESTS)
+ return 1;
+
llen += printf("0 -fstackprotector ");
#if !defined(_NOLIBC_STACKPROTECTOR)
@@ -1219,6 +1229,7 @@ int main(int argc, char **argv, char **envp)
int ret = 0;
int err;
int idx;
+ unsigned int subtests;
char *test;
argv0 = argv[0];
@@ -1265,6 +1276,8 @@ int main(int argc, char **argv, char **envp)
* once. We may have an optional range at <colon>
* here, which defaults to the full range.
*/
+
+ subtests = count_subtests(&test_names[idx]);
do {
min = 0; max = INT_MAX;
value = colon;
@@ -1285,11 +1298,14 @@ int main(int argc, char **argv, char **envp)
else if (*dash)
max = atoi(dash);
+ if (max > subtests)
+ max = subtests;
+
value = colon;
}
/* now's time to call the test */
- printf("Running test '%s'\n", test_names[idx].name);
+ printf("Running test '%s' (%d-%d of %d)\n", test_names[idx].name, min, max, subtests);
err = test_names[idx].func(min, max);
ret += err;
printf("Errors during this test: %d\n\n", err);
--
2.41.0