Hello everyone,
This patch series adds a test attributes framework to KUnit.
There has been interest in filtering out "slow" KUnit tests. Most notably,
a new config, CONFIG_MEMCPY_SLOW_KUNIT_TEST, has been added to exclude a
particularly slow memcpy test
(https://lore.kernel.org/all/[email protected]/).
This attributes framework can be used to save and access test associated
data, including whether a test is slow. These attributes are reportable
(via KTAP and command line output) and are also filterable.
This framework is designed to allow for the addition of other attributes in
the future. These attributes could include whether the test can be run
concurrently, test file path, etc.
To try out the framework I suggest running:
"./tools/testing/kunit/kunit.py run --filter speed!=slow"
This patch series was originally sent out as an RFC. Here is a link to the
RFC v2:
https://lore.kernel.org/all/[email protected]/
Thanks!
Rae
Rae Moar (9):
kunit: Add test attributes API structure
kunit: Add speed attribute
kunit: Add module attribute
kunit: Add ability to filter attributes
kunit: tool: Add command line interface to filter and report
attributes
kunit: memcpy: Mark tests as slow using test attributes
kunit: time: Mark test as slow using test attributes
kunit: add tests for filtering attributes
kunit: Add documentation of KUnit test attributes
.../dev-tools/kunit/running_tips.rst | 166 +++++++
include/kunit/attributes.h | 50 +++
include/kunit/test.h | 70 ++-
kernel/time/time_test.c | 2 +-
lib/Kconfig.debug | 3 +
lib/kunit/Makefile | 3 +-
lib/kunit/attributes.c | 418 ++++++++++++++++++
lib/kunit/executor.c | 115 ++++-
lib/kunit/executor_test.c | 128 +++++-
lib/kunit/kunit-example-test.c | 9 +
lib/kunit/test.c | 27 +-
lib/memcpy_kunit.c | 8 +-
tools/testing/kunit/kunit.py | 70 ++-
tools/testing/kunit/kunit_kernel.py | 8 +-
tools/testing/kunit/kunit_parser.py | 11 +-
tools/testing/kunit/kunit_tool_test.py | 39 +-
16 files changed, 1051 insertions(+), 76 deletions(-)
create mode 100644 include/kunit/attributes.h
create mode 100644 lib/kunit/attributes.c
base-commit: 64bd4641310c41a1ecf07c13c67bc0ed61045dfd
--
2.41.0.487.g6d72f3e995-goog
Add module attribute to the test attribute API. This attribute stores the
module name associated with the test using KBUILD_MODNAME.
The name of a test suite and the module name often do not match. A
reference to the module name associated with the suite could be extremely
helpful in running tests as modules without needing to check the codebase.
This attribute will be printed for each suite.
Reviewed-by: David Gow <[email protected]>
Signed-off-by: Rae Moar <[email protected]>
---
Changes since v1:
- Change kunit_attr_list definition to fix compile error
Changes since RFC v2:
- No changes.
Changes: since RFC v1:
- This is a new patch.
include/kunit/test.h | 13 ++++++++-----
lib/kunit/attributes.c | 25 +++++++++++++++++++++++++
2 files changed, 33 insertions(+), 5 deletions(-)
diff --git a/include/kunit/test.h b/include/kunit/test.h
index ed5f5000a095..011e0d6bb506 100644
--- a/include/kunit/test.h
+++ b/include/kunit/test.h
@@ -131,6 +131,7 @@ struct kunit_case {
/* private: internal use only. */
enum kunit_status status;
+ char *module_name;
char *log;
};
@@ -155,7 +156,9 @@ static inline char *kunit_status_to_ok_not_ok(enum kunit_status status)
* &struct kunit_case object from it. See the documentation for
* &struct kunit_case for an example on how to use it.
*/
-#define KUNIT_CASE(test_name) { .run_case = test_name, .name = #test_name }
+#define KUNIT_CASE(test_name) \
+ { .run_case = test_name, .name = #test_name, \
+ .module_name = KBUILD_MODNAME}
/**
* KUNIT_CASE_ATTR - A helper for creating a &struct kunit_case
@@ -167,7 +170,7 @@ static inline char *kunit_status_to_ok_not_ok(enum kunit_status status)
*/
#define KUNIT_CASE_ATTR(test_name, attributes) \
{ .run_case = test_name, .name = #test_name, \
- .attr = attributes }
+ .attr = attributes, .module_name = KBUILD_MODNAME}
/**
* KUNIT_CASE_SLOW - A helper for creating a &struct kunit_case
@@ -178,7 +181,7 @@ static inline char *kunit_status_to_ok_not_ok(enum kunit_status status)
#define KUNIT_CASE_SLOW(test_name) \
{ .run_case = test_name, .name = #test_name, \
- .attr.speed = KUNIT_SPEED_SLOW }
+ .attr.speed = KUNIT_SPEED_SLOW, .module_name = KBUILD_MODNAME}
/**
* KUNIT_CASE_PARAM - A helper for creation a parameterized &struct kunit_case
@@ -199,7 +202,7 @@ static inline char *kunit_status_to_ok_not_ok(enum kunit_status status)
*/
#define KUNIT_CASE_PARAM(test_name, gen_params) \
{ .run_case = test_name, .name = #test_name, \
- .generate_params = gen_params }
+ .generate_params = gen_params, .module_name = KBUILD_MODNAME}
/**
* KUNIT_CASE_PARAM_ATTR - A helper for creating a parameterized &struct
@@ -213,7 +216,7 @@ static inline char *kunit_status_to_ok_not_ok(enum kunit_status status)
#define KUNIT_CASE_PARAM_ATTR(test_name, gen_params, attributes) \
{ .run_case = test_name, .name = #test_name, \
.generate_params = gen_params, \
- .attr = attributes }
+ .attr = attributes, .module_name = KBUILD_MODNAME}
/**
* struct kunit_suite - describes a related collection of &struct kunit_case
diff --git a/lib/kunit/attributes.c b/lib/kunit/attributes.c
index ffd0d692b334..9dce4f4d726c 100644
--- a/lib/kunit/attributes.c
+++ b/lib/kunit/attributes.c
@@ -61,6 +61,12 @@ static const char *attr_speed_to_string(void *attr, bool *to_free)
return attr_enum_to_string(attr, speed_str_list, to_free);
}
+static const char *attr_string_to_string(void *attr, bool *to_free)
+{
+ *to_free = false;
+ return (char *) attr;
+}
+
/* Get Attribute Methods */
static void *attr_speed_get(void *test_or_suite, bool is_test)
@@ -74,6 +80,18 @@ static void *attr_speed_get(void *test_or_suite, bool is_test)
return ((void *) suite->attr.speed);
}
+static void *attr_module_get(void *test_or_suite, bool is_test)
+{
+ struct kunit_suite *suite = is_test ? NULL : test_or_suite;
+ struct kunit_case *test = is_test ? test_or_suite : NULL;
+
+ // Suites get their module attribute from their first test_case
+ if (test)
+ return ((void *) test->module_name);
+ else
+ return ((void *) suite->test_cases[0].module_name);
+}
+
/* List of all Test Attributes */
static struct kunit_attr kunit_attr_list[] = {
@@ -84,6 +102,13 @@ static struct kunit_attr kunit_attr_list[] = {
.attr_default = (void *)KUNIT_SPEED_NORMAL,
.print = PRINT_ALWAYS,
},
+ {
+ .name = "module",
+ .get_attr = attr_module_get,
+ .to_string = attr_string_to_string,
+ .attr_default = (void *)"",
+ .print = PRINT_SUITE,
+ }
};
/* Helper Functions to Access Attributes */
--
2.41.0.487.g6d72f3e995-goog
Add four tests to executor_test.c to test behavior of filtering attributes.
- parse_filter_attr_test - to test the parsing of inputted filters
- filter_attr_test - to test the filtering procedure on attributes
- filter_attr_empty_test - to test the behavior when all tests are filtered
out
- filter_attr_skip_test - to test the configurable filter_action=skip
option
Signed-off-by: Rae Moar <[email protected]>
---
Changes since v1:
- No changes.
Changes since RFC v2:
- Change fake suite and test case names.
- Add a few ASSERT statements.
Changes since RFC v1:
- This is a new patch.
lib/kunit/executor_test.c | 116 ++++++++++++++++++++++++++++++++++++++
1 file changed, 116 insertions(+)
diff --git a/lib/kunit/executor_test.c b/lib/kunit/executor_test.c
index d7ab069324b5..01280cb8d451 100644
--- a/lib/kunit/executor_test.c
+++ b/lib/kunit/executor_test.c
@@ -7,6 +7,7 @@
*/
#include <kunit/test.h>
+#include <kunit/attributes.h>
static void kfree_at_end(struct kunit *test, const void *to_free);
static struct kunit_suite *alloc_fake_suite(struct kunit *test,
@@ -108,11 +109,126 @@ static void filter_suites_to_empty_test(struct kunit *test)
"should be empty to indicate no match");
}
+static void parse_filter_attr_test(struct kunit *test)
+{
+ int j, filter_count;
+ struct kunit_attr_filter *parsed_filters;
+ char *filters = "speed>slow, module!=example";
+ int err = 0;
+
+ filter_count = kunit_get_filter_count(filters);
+ KUNIT_EXPECT_EQ(test, filter_count, 2);
+
+ parsed_filters = kunit_kcalloc(test, filter_count + 1, sizeof(*parsed_filters),
+ GFP_KERNEL);
+ for (j = 0; j < filter_count; j++) {
+ parsed_filters[j] = kunit_next_attr_filter(&filters, &err);
+ KUNIT_ASSERT_EQ_MSG(test, err, 0, "failed to parse filter '%s'", filters[j]);
+ }
+
+ KUNIT_EXPECT_STREQ(test, kunit_attr_filter_name(parsed_filters[0]), "speed");
+ KUNIT_EXPECT_STREQ(test, parsed_filters[0].input, ">slow");
+
+ KUNIT_EXPECT_STREQ(test, kunit_attr_filter_name(parsed_filters[1]), "module");
+ KUNIT_EXPECT_STREQ(test, parsed_filters[1].input, "!=example");
+}
+
+static struct kunit_case dummy_attr_test_cases[] = {
+ /* .run_case is not important, just needs to be non-NULL */
+ { .name = "slow", .run_case = dummy_test, .module_name = "dummy",
+ .attr.speed = KUNIT_SPEED_SLOW },
+ { .name = "normal", .run_case = dummy_test, .module_name = "dummy" },
+ {},
+};
+
+static void filter_attr_test(struct kunit *test)
+{
+ struct kunit_suite *subsuite[3] = {NULL, NULL};
+ struct suite_set suite_set = {.start = subsuite, .end = &subsuite[2]};
+ struct suite_set got;
+ int err = 0;
+
+ subsuite[0] = alloc_fake_suite(test, "normal_suite", dummy_attr_test_cases);
+ subsuite[1] = alloc_fake_suite(test, "slow_suite", dummy_attr_test_cases);
+ subsuite[1]->attr.speed = KUNIT_SPEED_SLOW; // Set suite attribute
+
+ /*
+ * Want: normal_suite(slow, normal), slow_suite(slow, normal),
+ * NULL -> normal_suite(normal), NULL
+ *
+ * The normal test in slow_suite is filtered out because the speed
+ * attribute is unset and thus, the filtering is based on the parent attribute
+ * of slow.
+ */
+ got = kunit_filter_suites(&suite_set, NULL, "speed>slow", NULL, &err);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, got.start);
+ KUNIT_ASSERT_EQ(test, err, 0);
+ kfree_at_end(test, got.start);
+
+ /* Validate we just have normal_suite */
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, got.start[0]);
+ KUNIT_EXPECT_STREQ(test, got.start[0]->name, "normal_suite");
+ KUNIT_ASSERT_EQ(test, got.end - got.start, 1);
+
+ /* Now validate we just have normal test case */
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, got.start[0]->test_cases);
+ KUNIT_EXPECT_STREQ(test, got.start[0]->test_cases[0].name, "normal");
+ KUNIT_EXPECT_FALSE(test, got.start[0]->test_cases[1].name);
+}
+
+static void filter_attr_empty_test(struct kunit *test)
+{
+ struct kunit_suite *subsuite[3] = {NULL, NULL};
+ struct suite_set suite_set = {.start = subsuite, .end = &subsuite[2]};
+ struct suite_set got;
+ int err = 0;
+
+ subsuite[0] = alloc_fake_suite(test, "suite1", dummy_attr_test_cases);
+ subsuite[1] = alloc_fake_suite(test, "suite2", dummy_attr_test_cases);
+
+ got = kunit_filter_suites(&suite_set, NULL, "module!=dummy", NULL, &err);
+ KUNIT_ASSERT_EQ(test, err, 0);
+ kfree_at_end(test, got.start); /* just in case */
+
+ KUNIT_EXPECT_PTR_EQ_MSG(test, got.start, got.end,
+ "should be empty to indicate no match");
+}
+
+static void filter_attr_skip_test(struct kunit *test)
+{
+ struct kunit_suite *subsuite[2] = {NULL};
+ struct suite_set suite_set = {.start = subsuite, .end = &subsuite[1]};
+ struct suite_set got;
+ int err = 0;
+
+ subsuite[0] = alloc_fake_suite(test, "suite", dummy_attr_test_cases);
+
+ /* Want: suite(slow, normal), NULL -> suite(slow with SKIP, normal), NULL */
+ got = kunit_filter_suites(&suite_set, NULL, "speed>slow", "skip", &err);
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, got.start);
+ KUNIT_ASSERT_EQ(test, err, 0);
+ kfree_at_end(test, got.start);
+
+ /* Validate we have both the slow and normal test */
+ KUNIT_ASSERT_NOT_ERR_OR_NULL(test, got.start[0]->test_cases);
+ KUNIT_ASSERT_EQ(test, kunit_suite_num_test_cases(got.start[0]), 2);
+ KUNIT_EXPECT_STREQ(test, got.start[0]->test_cases[0].name, "slow");
+ KUNIT_EXPECT_STREQ(test, got.start[0]->test_cases[1].name, "normal");
+
+ /* Now ensure slow is skipped and normal is not */
+ KUNIT_EXPECT_EQ(test, got.start[0]->test_cases[0].status, KUNIT_SKIPPED);
+ KUNIT_EXPECT_FALSE(test, got.start[0]->test_cases[1].status);
+}
+
static struct kunit_case executor_test_cases[] = {
KUNIT_CASE(parse_filter_test),
KUNIT_CASE(filter_suites_test),
KUNIT_CASE(filter_suites_test_glob_test),
KUNIT_CASE(filter_suites_to_empty_test),
+ KUNIT_CASE(parse_filter_attr_test),
+ KUNIT_CASE(filter_attr_test),
+ KUNIT_CASE(filter_attr_empty_test),
+ KUNIT_CASE(filter_attr_skip_test),
{}
};
--
2.41.0.487.g6d72f3e995-goog
Add documentation on the use of test attributes under the section "Tips for
Running KUnit Tests" in the KUnit docs.
Documentation includes three sections on how to mark tests with attributes,
how attributes are reported, and how the user can filter tests using test
attributes.
Reviewed-by: David Gow <[email protected]>
Signed-off-by: Rae Moar <[email protected]>
---
Changes since v1:
- No changes.
Changes since RFC v2:
- Add comment on KUNIT_CASE_SLOW() to documentation.
- Add comment on how to expose raw kernel output.
- Remove an extra line at the end of file.
Changes since RFC v1:
- This is a new patch.
.../dev-tools/kunit/running_tips.rst | 166 ++++++++++++++++++
1 file changed, 166 insertions(+)
diff --git a/Documentation/dev-tools/kunit/running_tips.rst b/Documentation/dev-tools/kunit/running_tips.rst
index 8e8c493f17d1..766f9cdea0fa 100644
--- a/Documentation/dev-tools/kunit/running_tips.rst
+++ b/Documentation/dev-tools/kunit/running_tips.rst
@@ -262,3 +262,169 @@ other code executed during boot, e.g.
# Reset coverage counters before running the test.
$ echo 0 > /sys/kernel/debug/gcov/reset
$ modprobe kunit-example-test
+
+
+Test Attributes and Filtering
+=============================
+
+Test suites and cases can be marked with test attributes, such as speed of
+test. These attributes will later be printed in test output and can be used to
+filter test execution.
+
+Marking Test Attributes
+-----------------------
+
+Tests are marked with an attribute by including a ``kunit_attributes`` object
+in the test definition.
+
+Test cases can be marked using the ``KUNIT_CASE_ATTR(test_name, attributes)``
+macro to define the test case instead of ``KUNIT_CASE(test_name)``.
+
+.. code-block:: c
+
+ static const struct kunit_attributes example_attr = {
+ .speed = KUNIT_VERY_SLOW,
+ };
+
+ static struct kunit_case example_test_cases[] = {
+ KUNIT_CASE_ATTR(example_test, example_attr),
+ };
+
+.. note::
+ To mark a test case as slow, you can also use ``KUNIT_CASE_SLOW(test_name)``.
+ This is a helpful macro as the slow attribute is the most commonly used.
+
+Test suites can be marked with an attribute by setting the "attr" field in the
+suite definition.
+
+.. code-block:: c
+
+ static const struct kunit_attributes example_attr = {
+ .speed = KUNIT_VERY_SLOW,
+ };
+
+ static struct kunit_suite example_test_suite = {
+ ...,
+ .attr = example_attr,
+ };
+
+.. note::
+ Not all attributes need to be set in a ``kunit_attributes`` object. Unset
+ attributes will remain uninitialized and act as though the attribute is set
+ to 0 or NULL. Thus, if an attribute is set to 0, it is treated as unset.
+ These unset attributes will not be reported and may act as a default value
+ for filtering purposes.
+
+Reporting Attributes
+--------------------
+
+When a user runs tests, attributes will be present in the raw kernel output (in
+KTAP format). Note that attributes will be hidden by default in kunit.py output
+for all passing tests but the raw kernel output can be accessed using the
+``--raw_output`` flag. This is an example of how test attributes for test cases
+will be formatted in kernel output:
+
+.. code-block:: none
+
+ # example_test.speed: slow
+ ok 1 example_test
+
+This is an example of how test attributes for test suites will be formatted in
+kernel output:
+
+.. code-block:: none
+
+ KTAP version 2
+ # Subtest: example_suite
+ # module: kunit_example_test
+ 1..3
+ ...
+ ok 1 example_suite
+
+Additionally, users can output a full attribute report of tests with their
+attributes, using the command line flag ``--list_tests_attr``:
+
+.. code-block:: bash
+
+ kunit.py run "example" --list_tests_attr
+
+.. note::
+ This report can be accessed when running KUnit manually by passing in the
+ module_param ``kunit.action=list_attr``.
+
+Filtering
+---------
+
+Users can filter tests using the ``--filter`` command line flag when running
+tests. As an example:
+
+.. code-block:: bash
+
+ kunit.py run --filter speed=slow
+
+
+You can also use the following operations on filters: "<", ">", "<=", ">=",
+"!=", and "=". Example:
+
+.. code-block:: bash
+
+ kunit.py run --filter "speed>slow"
+
+This example will run all tests with speeds faster than slow. Note that the
+characters < and > are often interpreted by the shell, so they may need to be
+quoted or escaped, as above.
+
+Additionally, you can use multiple filters at once. Simply separate filters
+using commas. Example:
+
+.. code-block:: bash
+
+ kunit.py run --filter "speed>slow, module=kunit_example_test"
+
+.. note::
+ You can use this filtering feature when running KUnit manually by passing
+ the filter as a module param: ``kunit.filter="speed>slow, speed<=normal"``.
+
+Filtered tests will not run or show up in the test output. You can use the
+``--filter_action=skip`` flag to skip filtered tests instead. These tests will be
+shown in the test output in the test but will not run. To use this feature when
+running KUnit manually, use the module param ``kunit.filter_action=skip``.
+
+Rules of Filtering Procedure
+----------------------------
+
+Since both suites and test cases can have attributes, there may be conflicts
+between attributes during filtering. The process of filtering follows these
+rules:
+
+- Filtering always operates at a per-test level.
+
+- If a test has an attribute set, then the test's value is filtered on.
+
+- Otherwise, the value falls back to the suite's value.
+
+- If neither are set, the attribute has a global "default" value, which is used.
+
+List of Current Attributes
+--------------------------
+
+``speed``
+
+This attribute indicates the speed of a test's execution (how slow or fast the
+test is).
+
+This attribute is saved as an enum with the following categories: "normal",
+"slow", or "very_slow". The assumed default speed for tests is "normal". This
+indicates that the test takes a relatively trivial amount of time (less than
+1 second), regardless of the machine it is running on. Any test slower than
+this could be marked as "slow" or "very_slow".
+
+The macro ``KUNIT_CASE_SLOW(test_name)`` can be easily used to set the speed
+of a test case to "slow".
+
+``module``
+
+This attribute indicates the name of the module associated with the test.
+
+This attribute is automatically saved as a string and is printed for each suite.
+Tests can also be filtered using this attribute.
--
2.41.0.487.g6d72f3e995-goog
On Tue, 25 Jul 2023 at 00:31, Rae Moar <[email protected]> wrote:
>
> Add four tests to executor_test.c to test behavior of filtering attributes.
>
> - parse_filter_attr_test - to test the parsing of inputted filters
>
> - filter_attr_test - to test the filtering procedure on attributes
>
> - filter_attr_empty_test - to test the behavior when all tests are filtered
> out
>
> - filter_attr_skip_test - to test the configurable filter_action=skip
> option
>
> Signed-off-by: Rae Moar <[email protected]>
> ---
Glad to see some tests for the parser-y bits.
Reviewed-by: David Gow <[email protected]>
Cheers,
-- David
>
> Changes since v1:
> - No changes.
> Changes since RFC v2:
> - Change fake suite and test case names.
> - Add a few ASSERT statements.
> Changes since RFC v1:
> - This is a new patch.
>
> lib/kunit/executor_test.c | 116 ++++++++++++++++++++++++++++++++++++++
> 1 file changed, 116 insertions(+)
>
> diff --git a/lib/kunit/executor_test.c b/lib/kunit/executor_test.c
> index d7ab069324b5..01280cb8d451 100644
> --- a/lib/kunit/executor_test.c
> +++ b/lib/kunit/executor_test.c
> @@ -7,6 +7,7 @@
> */
>
> #include <kunit/test.h>
> +#include <kunit/attributes.h>
>
> static void kfree_at_end(struct kunit *test, const void *to_free);
> static struct kunit_suite *alloc_fake_suite(struct kunit *test,
> @@ -108,11 +109,126 @@ static void filter_suites_to_empty_test(struct kunit *test)
> "should be empty to indicate no match");
> }
>
> +static void parse_filter_attr_test(struct kunit *test)
> +{
> + int j, filter_count;
> + struct kunit_attr_filter *parsed_filters;
> + char *filters = "speed>slow, module!=example";
> + int err = 0;
> +
> + filter_count = kunit_get_filter_count(filters);
> + KUNIT_EXPECT_EQ(test, filter_count, 2);
> +
> + parsed_filters = kunit_kcalloc(test, filter_count + 1, sizeof(*parsed_filters),
> + GFP_KERNEL);
> + for (j = 0; j < filter_count; j++) {
> + parsed_filters[j] = kunit_next_attr_filter(&filters, &err);
> + KUNIT_ASSERT_EQ_MSG(test, err, 0, "failed to parse filter '%s'", filters[j]);
> + }
> +
> + KUNIT_EXPECT_STREQ(test, kunit_attr_filter_name(parsed_filters[0]), "speed");
> + KUNIT_EXPECT_STREQ(test, parsed_filters[0].input, ">slow");
> +
> + KUNIT_EXPECT_STREQ(test, kunit_attr_filter_name(parsed_filters[1]), "module");
> + KUNIT_EXPECT_STREQ(test, parsed_filters[1].input, "!=example");
> +}
> +
> +static struct kunit_case dummy_attr_test_cases[] = {
> + /* .run_case is not important, just needs to be non-NULL */
> + { .name = "slow", .run_case = dummy_test, .module_name = "dummy",
> + .attr.speed = KUNIT_SPEED_SLOW },
> + { .name = "normal", .run_case = dummy_test, .module_name = "dummy" },
> + {},
> +};
> +
> +static void filter_attr_test(struct kunit *test)
> +{
> + struct kunit_suite *subsuite[3] = {NULL, NULL};
> + struct suite_set suite_set = {.start = subsuite, .end = &subsuite[2]};
> + struct suite_set got;
> + int err = 0;
> +
> + subsuite[0] = alloc_fake_suite(test, "normal_suite", dummy_attr_test_cases);
> + subsuite[1] = alloc_fake_suite(test, "slow_suite", dummy_attr_test_cases);
> + subsuite[1]->attr.speed = KUNIT_SPEED_SLOW; // Set suite attribute
> +
> + /*
> + * Want: normal_suite(slow, normal), slow_suite(slow, normal),
> + * NULL -> normal_suite(normal), NULL
> + *
> + * The normal test in slow_suite is filtered out because the speed
> + * attribute is unset and thus, the filtering is based on the parent attribute
> + * of slow.
> + */
> + got = kunit_filter_suites(&suite_set, NULL, "speed>slow", NULL, &err);
> + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, got.start);
> + KUNIT_ASSERT_EQ(test, err, 0);
> + kfree_at_end(test, got.start);
> +
> + /* Validate we just have normal_suite */
> + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, got.start[0]);
> + KUNIT_EXPECT_STREQ(test, got.start[0]->name, "normal_suite");
> + KUNIT_ASSERT_EQ(test, got.end - got.start, 1);
> +
> + /* Now validate we just have normal test case */
> + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, got.start[0]->test_cases);
> + KUNIT_EXPECT_STREQ(test, got.start[0]->test_cases[0].name, "normal");
> + KUNIT_EXPECT_FALSE(test, got.start[0]->test_cases[1].name);
> +}
> +
> +static void filter_attr_empty_test(struct kunit *test)
> +{
> + struct kunit_suite *subsuite[3] = {NULL, NULL};
> + struct suite_set suite_set = {.start = subsuite, .end = &subsuite[2]};
> + struct suite_set got;
> + int err = 0;
> +
> + subsuite[0] = alloc_fake_suite(test, "suite1", dummy_attr_test_cases);
> + subsuite[1] = alloc_fake_suite(test, "suite2", dummy_attr_test_cases);
> +
> + got = kunit_filter_suites(&suite_set, NULL, "module!=dummy", NULL, &err);
> + KUNIT_ASSERT_EQ(test, err, 0);
> + kfree_at_end(test, got.start); /* just in case */
> +
> + KUNIT_EXPECT_PTR_EQ_MSG(test, got.start, got.end,
> + "should be empty to indicate no match");
> +}
> +
> +static void filter_attr_skip_test(struct kunit *test)
> +{
> + struct kunit_suite *subsuite[2] = {NULL};
> + struct suite_set suite_set = {.start = subsuite, .end = &subsuite[1]};
> + struct suite_set got;
> + int err = 0;
> +
> + subsuite[0] = alloc_fake_suite(test, "suite", dummy_attr_test_cases);
> +
> + /* Want: suite(slow, normal), NULL -> suite(slow with SKIP, normal), NULL */
> + got = kunit_filter_suites(&suite_set, NULL, "speed>slow", "skip", &err);
> + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, got.start);
> + KUNIT_ASSERT_EQ(test, err, 0);
> + kfree_at_end(test, got.start);
> +
> + /* Validate we have both the slow and normal test */
> + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, got.start[0]->test_cases);
> + KUNIT_ASSERT_EQ(test, kunit_suite_num_test_cases(got.start[0]), 2);
> + KUNIT_EXPECT_STREQ(test, got.start[0]->test_cases[0].name, "slow");
> + KUNIT_EXPECT_STREQ(test, got.start[0]->test_cases[1].name, "normal");
> +
> + /* Now ensure slow is skipped and normal is not */
> + KUNIT_EXPECT_EQ(test, got.start[0]->test_cases[0].status, KUNIT_SKIPPED);
> + KUNIT_EXPECT_FALSE(test, got.start[0]->test_cases[1].status);
> +}
> +
> static struct kunit_case executor_test_cases[] = {
> KUNIT_CASE(parse_filter_test),
> KUNIT_CASE(filter_suites_test),
> KUNIT_CASE(filter_suites_test_glob_test),
> KUNIT_CASE(filter_suites_to_empty_test),
> + KUNIT_CASE(parse_filter_attr_test),
> + KUNIT_CASE(filter_attr_test),
> + KUNIT_CASE(filter_attr_empty_test),
> + KUNIT_CASE(filter_attr_skip_test),
> {}
> };
>
> --
> 2.41.0.487.g6d72f3e995-goog
>
On Tue, 25 Jul 2023 at 00:31, Rae Moar <[email protected]> wrote:
>
> Add documentation on the use of test attributes under the section "Tips for
> Running KUnit Tests" in the KUnit docs.
>
> Documentation includes three sections on how to mark tests with attributes,
> how attributes are reported, and how the user can filter tests using test
> attributes.
>
> Reviewed-by: David Gow <[email protected]>
> Signed-off-by: Rae Moar <[email protected]>
> ---
This still looks good, but an idea for an improvement (either in this
patch or a future series) would be to add some docs for --filter and
--filter_action to run_wrapper.rst, which has a list of kunit.py
options.
> Changes since v1:
> - No changes.
>
> Changes since RFC v2:
> - Add comment on KUNIT_CASE_SLOW() to documentation.
> - Add comment on how to expose raw kernel output.
> - Remove an extra line at the end of file.
>
> Changes since RFC v1:
> - This is a new patch.
>
> .../dev-tools/kunit/running_tips.rst | 166 ++++++++++++++++++
> 1 file changed, 166 insertions(+)
>
> diff --git a/Documentation/dev-tools/kunit/running_tips.rst b/Documentation/dev-tools/kunit/running_tips.rst
> index 8e8c493f17d1..766f9cdea0fa 100644
> --- a/Documentation/dev-tools/kunit/running_tips.rst
> +++ b/Documentation/dev-tools/kunit/running_tips.rst
> @@ -262,3 +262,169 @@ other code executed during boot, e.g.
> # Reset coverage counters before running the test.
> $ echo 0 > /sys/kernel/debug/gcov/reset
> $ modprobe kunit-example-test
> +
> +
> +Test Attributes and Filtering
> +=============================
> +
> +Test suites and cases can be marked with test attributes, such as speed of
> +test. These attributes will later be printed in test output and can be used to
> +filter test execution.
> +
> +Marking Test Attributes
> +-----------------------
> +
> +Tests are marked with an attribute by including a ``kunit_attributes`` object
> +in the test definition.
> +
> +Test cases can be marked using the ``KUNIT_CASE_ATTR(test_name, attributes)``
> +macro to define the test case instead of ``KUNIT_CASE(test_name)``.
> +
> +.. code-block:: c
> +
> + static const struct kunit_attributes example_attr = {
> + .speed = KUNIT_VERY_SLOW,
> + };
> +
> + static struct kunit_case example_test_cases[] = {
> + KUNIT_CASE_ATTR(example_test, example_attr),
> + };
> +
> +.. note::
> + To mark a test case as slow, you can also use ``KUNIT_CASE_SLOW(test_name)``.
> + This is a helpful macro as the slow attribute is the most commonly used.
> +
> +Test suites can be marked with an attribute by setting the "attr" field in the
> +suite definition.
> +
> +.. code-block:: c
> +
> + static const struct kunit_attributes example_attr = {
> + .speed = KUNIT_VERY_SLOW,
> + };
> +
> + static struct kunit_suite example_test_suite = {
> + ...,
> + .attr = example_attr,
> + };
> +
> +.. note::
> + Not all attributes need to be set in a ``kunit_attributes`` object. Unset
> + attributes will remain uninitialized and act as though the attribute is set
> + to 0 or NULL. Thus, if an attribute is set to 0, it is treated as unset.
> + These unset attributes will not be reported and may act as a default value
> + for filtering purposes.
> +
> +Reporting Attributes
> +--------------------
> +
> +When a user runs tests, attributes will be present in the raw kernel output (in
> +KTAP format). Note that attributes will be hidden by default in kunit.py output
> +for all passing tests but the raw kernel output can be accessed using the
> +``--raw_output`` flag. This is an example of how test attributes for test cases
> +will be formatted in kernel output:
> +
> +.. code-block:: none
> +
> + # example_test.speed: slow
> + ok 1 example_test
> +
> +This is an example of how test attributes for test suites will be formatted in
> +kernel output:
> +
> +.. code-block:: none
> +
> + KTAP version 2
> + # Subtest: example_suite
> + # module: kunit_example_test
> + 1..3
> + ...
> + ok 1 example_suite
> +
> +Additionally, users can output a full attribute report of tests with their
> +attributes, using the command line flag ``--list_tests_attr``:
> +
> +.. code-block:: bash
> +
> + kunit.py run "example" --list_tests_attr
> +
> +.. note::
> + This report can be accessed when running KUnit manually by passing in the
> + module_param ``kunit.action=list_attr``.
> +
> +Filtering
> +---------
> +
> +Users can filter tests using the ``--filter`` command line flag when running
> +tests. As an example:
> +
> +.. code-block:: bash
> +
> + kunit.py run --filter speed=slow
> +
> +
> +You can also use the following operations on filters: "<", ">", "<=", ">=",
> +"!=", and "=". Example:
> +
> +.. code-block:: bash
> +
> + kunit.py run --filter "speed>slow"
> +
> +This example will run all tests with speeds faster than slow. Note that the
> +characters < and > are often interpreted by the shell, so they may need to be
> +quoted or escaped, as above.
> +
> +Additionally, you can use multiple filters at once. Simply separate filters
> +using commas. Example:
> +
> +.. code-block:: bash
> +
> + kunit.py run --filter "speed>slow, module=kunit_example_test"
> +
> +.. note::
> + You can use this filtering feature when running KUnit manually by passing
> + the filter as a module param: ``kunit.filter="speed>slow, speed<=normal"``.
> +
> +Filtered tests will not run or show up in the test output. You can use the
> +``--filter_action=skip`` flag to skip filtered tests instead. These tests will be
> +shown in the test output in the test but will not run. To use this feature when
> +running KUnit manually, use the module param ``kunit.filter_action=skip``.
> +
> +Rules of Filtering Procedure
> +----------------------------
> +
> +Since both suites and test cases can have attributes, there may be conflicts
> +between attributes during filtering. The process of filtering follows these
> +rules:
> +
> +- Filtering always operates at a per-test level.
> +
> +- If a test has an attribute set, then the test's value is filtered on.
> +
> +- Otherwise, the value falls back to the suite's value.
> +
> +- If neither are set, the attribute has a global "default" value, which is used.
> +
> +List of Current Attributes
> +--------------------------
> +
> +``speed``
> +
> +This attribute indicates the speed of a test's execution (how slow or fast the
> +test is).
> +
> +This attribute is saved as an enum with the following categories: "normal",
> +"slow", or "very_slow". The assumed default speed for tests is "normal". This
> +indicates that the test takes a relatively trivial amount of time (less than
> +1 second), regardless of the machine it is running on. Any test slower than
> +this could be marked as "slow" or "very_slow".
> +
> +The macro ``KUNIT_CASE_SLOW(test_name)`` can be easily used to set the speed
> +of a test case to "slow".
> +
> +``module``
> +
> +This attribute indicates the name of the module associated with the test.
> +
> +This attribute is automatically saved as a string and is printed for each suite.
> +Tests can also be filtered using this attribute.
> --
> 2.41.0.487.g6d72f3e995-goog
>