Makes Dept able to track PG_locked waits and events. It's going to be
useful in practice. See the following link that shows dept worked with
PG_locked and can detect real issues:
https://lore.kernel.org/lkml/[email protected]/
Signed-off-by: Byungchul Park <[email protected]>
---
include/linux/mm_types.h | 2 +
include/linux/page-flags.h | 105 ++++++++++++++++++++++++++++++++-----
include/linux/pagemap.h | 7 ++-
mm/filemap.c | 26 +++++++++
mm/mm_init.c | 2 +
5 files changed, 129 insertions(+), 13 deletions(-)
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 957ce38768b2..5c1112bc7a46 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -19,6 +19,7 @@
#include <linux/workqueue.h>
#include <linux/seqlock.h>
#include <linux/percpu_counter.h>
+#include <linux/dept.h>
#include <asm/mmu.h>
@@ -203,6 +204,7 @@ struct page {
struct page *kmsan_shadow;
struct page *kmsan_origin;
#endif
+ struct dept_ext_wgen PG_locked_wgen;
} _struct_page_alignment;
/*
diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index a88e64acebfe..0a498f2c4543 100644
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -198,6 +198,43 @@ enum pageflags {
#ifndef __GENERATING_BOUNDS_H
+#ifdef CONFIG_DEPT
+#include <linux/kernel.h>
+#include <linux/dept.h>
+
+extern struct dept_map PG_locked_map;
+
+/*
+ * Place the following annotations in its suitable point in code:
+ *
+ * Annotate dept_page_set_bit() around firstly set_bit*()
+ * Annotate dept_page_clear_bit() around clear_bit*()
+ * Annotate dept_page_wait_on_bit() around wait_on_bit*()
+ */
+
+static inline void dept_page_set_bit(struct page *p, int bit_nr)
+{
+ if (bit_nr == PG_locked)
+ dept_request_event(&PG_locked_map, &p->PG_locked_wgen);
+}
+
+static inline void dept_page_clear_bit(struct page *p, int bit_nr)
+{
+ if (bit_nr == PG_locked)
+ dept_event(&PG_locked_map, 1UL, _RET_IP_, __func__, &p->PG_locked_wgen);
+}
+
+static inline void dept_page_wait_on_bit(struct page *p, int bit_nr)
+{
+ if (bit_nr == PG_locked)
+ dept_wait(&PG_locked_map, 1UL, _RET_IP_, __func__, 0, -1L);
+}
+#else
+#define dept_page_set_bit(p, bit_nr) do { } while (0)
+#define dept_page_clear_bit(p, bit_nr) do { } while (0)
+#define dept_page_wait_on_bit(p, bit_nr) do { } while (0)
+#endif
+
#ifdef CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP
DECLARE_STATIC_KEY_FALSE(hugetlb_optimize_vmemmap_key);
@@ -379,44 +416,88 @@ static __always_inline int Page##uname(struct page *page) \
#define SETPAGEFLAG(uname, lname, policy) \
static __always_inline \
void folio_set_##lname(struct folio *folio) \
-{ set_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); } \
+{ \
+ set_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); \
+ dept_page_set_bit(&folio->page, PG_##lname); \
+} \
static __always_inline void SetPage##uname(struct page *page) \
-{ set_bit(PG_##lname, &policy(page, 1)->flags); }
+{ \
+ set_bit(PG_##lname, &policy(page, 1)->flags); \
+ dept_page_set_bit(page, PG_##lname); \
+}
#define CLEARPAGEFLAG(uname, lname, policy) \
static __always_inline \
void folio_clear_##lname(struct folio *folio) \
-{ clear_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); } \
+{ \
+ clear_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); \
+ dept_page_clear_bit(&folio->page, PG_##lname); \
+} \
static __always_inline void ClearPage##uname(struct page *page) \
-{ clear_bit(PG_##lname, &policy(page, 1)->flags); }
+{ \
+ clear_bit(PG_##lname, &policy(page, 1)->flags); \
+ dept_page_clear_bit(page, PG_##lname); \
+}
#define __SETPAGEFLAG(uname, lname, policy) \
static __always_inline \
void __folio_set_##lname(struct folio *folio) \
-{ __set_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); } \
+{ \
+ __set_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); \
+ dept_page_set_bit(&folio->page, PG_##lname); \
+} \
static __always_inline void __SetPage##uname(struct page *page) \
-{ __set_bit(PG_##lname, &policy(page, 1)->flags); }
+{ \
+ __set_bit(PG_##lname, &policy(page, 1)->flags); \
+ dept_page_set_bit(page, PG_##lname); \
+}
#define __CLEARPAGEFLAG(uname, lname, policy) \
static __always_inline \
void __folio_clear_##lname(struct folio *folio) \
-{ __clear_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); } \
+{ \
+ __clear_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); \
+ dept_page_clear_bit(&folio->page, PG_##lname); \
+} \
static __always_inline void __ClearPage##uname(struct page *page) \
-{ __clear_bit(PG_##lname, &policy(page, 1)->flags); }
+{ \
+ __clear_bit(PG_##lname, &policy(page, 1)->flags); \
+ dept_page_clear_bit(page, PG_##lname); \
+}
#define TESTSETFLAG(uname, lname, policy) \
static __always_inline \
bool folio_test_set_##lname(struct folio *folio) \
-{ return test_and_set_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); } \
+{ \
+ bool ret = test_and_set_bit(PG_##lname, folio_flags(folio, FOLIO_##policy));\
+ if (!ret) \
+ dept_page_set_bit(&folio->page, PG_##lname); \
+ return ret; \
+} \
static __always_inline int TestSetPage##uname(struct page *page) \
-{ return test_and_set_bit(PG_##lname, &policy(page, 1)->flags); }
+{ \
+ bool ret = test_and_set_bit(PG_##lname, &policy(page, 1)->flags);\
+ if (!ret) \
+ dept_page_set_bit(page, PG_##lname); \
+ return ret; \
+}
#define TESTCLEARFLAG(uname, lname, policy) \
static __always_inline \
bool folio_test_clear_##lname(struct folio *folio) \
-{ return test_and_clear_bit(PG_##lname, folio_flags(folio, FOLIO_##policy)); } \
+{ \
+ bool ret = test_and_clear_bit(PG_##lname, folio_flags(folio, FOLIO_##policy));\
+ if (ret) \
+ dept_page_clear_bit(&folio->page, PG_##lname); \
+ return ret; \
+} \
static __always_inline int TestClearPage##uname(struct page *page) \
-{ return test_and_clear_bit(PG_##lname, &policy(page, 1)->flags); }
+{ \
+ bool ret = test_and_clear_bit(PG_##lname, &policy(page, 1)->flags);\
+ if (ret) \
+ dept_page_clear_bit(page, PG_##lname); \
+ return ret; \
+}
#define PAGEFLAG(uname, lname, policy) \
TESTPAGEFLAG(uname, lname, policy) \
diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h
index 06142ff7f9ce..c6683b228b20 100644
--- a/include/linux/pagemap.h
+++ b/include/linux/pagemap.h
@@ -991,7 +991,12 @@ void folio_unlock(struct folio *folio);
*/
static inline bool folio_trylock(struct folio *folio)
{
- return likely(!test_and_set_bit_lock(PG_locked, folio_flags(folio, 0)));
+ bool ret = !test_and_set_bit_lock(PG_locked, folio_flags(folio, 0));
+
+ if (ret)
+ dept_page_set_bit(&folio->page, PG_locked);
+
+ return likely(ret);
}
/*
diff --git a/mm/filemap.c b/mm/filemap.c
index ad5b4aa049a3..241a67a363b0 100644
--- a/mm/filemap.c
+++ b/mm/filemap.c
@@ -45,6 +45,7 @@
#include <linux/migrate.h>
#include <linux/pipe_fs_i.h>
#include <linux/splice.h>
+#include <linux/dept.h>
#include <asm/pgalloc.h>
#include <asm/tlbflush.h>
#include "internal.h"
@@ -1098,6 +1099,7 @@ static int wake_page_function(wait_queue_entry_t *wait, unsigned mode, int sync,
if (flags & WQ_FLAG_CUSTOM) {
if (test_and_set_bit(key->bit_nr, &key->folio->flags))
return -1;
+ dept_page_set_bit(&key->folio->page, key->bit_nr);
flags |= WQ_FLAG_DONE;
}
}
@@ -1181,6 +1183,7 @@ static inline bool folio_trylock_flag(struct folio *folio, int bit_nr,
if (wait->flags & WQ_FLAG_EXCLUSIVE) {
if (test_and_set_bit(bit_nr, &folio->flags))
return false;
+ dept_page_set_bit(&folio->page, bit_nr);
} else if (test_bit(bit_nr, &folio->flags))
return false;
@@ -1191,6 +1194,9 @@ static inline bool folio_trylock_flag(struct folio *folio, int bit_nr,
/* How many times do we accept lock stealing from under a waiter? */
int sysctl_page_lock_unfairness = 5;
+struct dept_map __maybe_unused PG_locked_map = DEPT_MAP_INITIALIZER(PG_locked_map, NULL);
+EXPORT_SYMBOL(PG_locked_map);
+
static inline int folio_wait_bit_common(struct folio *folio, int bit_nr,
int state, enum behavior behavior)
{
@@ -1202,6 +1208,8 @@ static inline int folio_wait_bit_common(struct folio *folio, int bit_nr,
unsigned long pflags;
bool in_thrashing;
+ dept_page_wait_on_bit(&folio->page, bit_nr);
+
if (bit_nr == PG_locked &&
!folio_test_uptodate(folio) && folio_test_workingset(folio)) {
delayacct_thrashing_start(&in_thrashing);
@@ -1295,6 +1303,23 @@ static inline int folio_wait_bit_common(struct folio *folio, int bit_nr,
break;
}
+ /*
+ * dept_page_set_bit() might have been called already in
+ * folio_trylock_flag(), wake_page_function() or somewhere.
+ * However, call it again to reset the wgen of dept to ensure
+ * dept_page_wait_on_bit() is called prior to
+ * dept_page_set_bit().
+ *
+ * Remind dept considers all the waits between
+ * dept_page_set_bit() and dept_page_clear_bit() as potential
+ * event disturbers. Ensure the correct sequence so that dept
+ * can make correct decisions:
+ *
+ * wait -> acquire(set bit) -> release(clear bit)
+ */
+ if (wait->flags & WQ_FLAG_DONE)
+ dept_page_set_bit(&folio->page, bit_nr);
+
/*
* If a signal happened, this 'finish_wait()' may remove the last
* waiter from the wait-queues, but the folio waiters bit will remain
@@ -1471,6 +1496,7 @@ void folio_unlock(struct folio *folio)
BUILD_BUG_ON(PG_waiters != 7);
BUILD_BUG_ON(PG_locked > 7);
VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
+ dept_page_clear_bit(&folio->page, PG_locked);
if (folio_xor_flags_has_waiters(folio, 1 << PG_locked))
folio_wake_bit(folio, PG_locked);
}
diff --git a/mm/mm_init.c b/mm/mm_init.c
index 077bfe393b5e..fc150d7a3686 100644
--- a/mm/mm_init.c
+++ b/mm/mm_init.c
@@ -26,6 +26,7 @@
#include <linux/pgtable.h>
#include <linux/swap.h>
#include <linux/cma.h>
+#include <linux/dept.h>
#include "internal.h"
#include "slab.h"
#include "shuffle.h"
@@ -564,6 +565,7 @@ void __meminit __init_single_page(struct page *page, unsigned long pfn,
page_mapcount_reset(page);
page_cpupid_reset_last(page);
page_kasan_tag_reset(page);
+ dept_ext_wgen_init(&page->PG_locked_wgen);
INIT_LIST_HEAD(&page->lru);
#ifdef WANT_PAGE_VIRTUAL
--
2.17.1