Add object_phys_tree_root to store the objects allocated with
physical address. Distinguish it from object_tree_root by
OBJECT_PHYS flag or function argument.
Suggested-by: Catalin Marinas <[email protected]>
Signed-off-by: Patrick Wang <[email protected]>
---
mm/kmemleak.c | 99 +++++++++++++++++++++++++++++----------------------
1 file changed, 57 insertions(+), 42 deletions(-)
diff --git a/mm/kmemleak.c b/mm/kmemleak.c
index 1e9e0aa93ae5..218144392446 100644
--- a/mm/kmemleak.c
+++ b/mm/kmemleak.c
@@ -14,14 +14,16 @@
* The following locks and mutexes are used by kmemleak:
*
* - kmemleak_lock (raw_spinlock_t): protects the object_list modifications and
- * accesses to the object_tree_root. The object_list is the main list
- * holding the metadata (struct kmemleak_object) for the allocated memory
- * blocks. The object_tree_root is a red black tree used to look-up
- * metadata based on a pointer to the corresponding memory block. The
- * kmemleak_object structures are added to the object_list and
- * object_tree_root in the create_object() function called from the
- * kmemleak_alloc() callback and removed in delete_object() called from the
- * kmemleak_free() callback
+ * accesses to the object_tree_root (or object_phys_tree_root). The
+ * object_list is the main list holding the metadata (struct kmemleak_object)
+ * for the allocated memory blocks. The object_tree_root and object_phys_tree_root
+ * are red black trees used to look-up metadata based on a pointer to the
+ * corresponding memory block. The object_phys_tree_root is for objects
+ * allocated with physical address. The kmemleak_object structures are
+ * added to the object_list and object_tree_root (or object_phys_tree_root)
+ * in the create_object() function called from the kmemleak_alloc() (or
+ * kmemleak_alloc_phys()) callback and removed in delete_object() called from
+ * the kmemleak_free() callback
* - kmemleak_object.lock (raw_spinlock_t): protects a kmemleak_object.
* Accesses to the metadata (e.g. count) are protected by this lock. Note
* that some members of this structure may be protected by other means
@@ -195,7 +197,9 @@ static int mem_pool_free_count = ARRAY_SIZE(mem_pool);
static LIST_HEAD(mem_pool_free_list);
/* search tree for object boundaries */
static struct rb_root object_tree_root = RB_ROOT;
-/* protecting the access to object_list and object_tree_root */
+/* search tree for object (with OBJECT_PHYS flag) boundaries */
+static struct rb_root object_phys_tree_root = RB_ROOT;
+/* protecting the access to object_list, object_tree_root (or object_phys_tree_root) */
static DEFINE_RAW_SPINLOCK(kmemleak_lock);
/* allocation caches for kmemleak internal data */
@@ -380,9 +384,11 @@ static void dump_object_info(struct kmemleak_object *object)
* beginning of the memory block are allowed. The kmemleak_lock must be held
* when calling this function.
*/
-static struct kmemleak_object *lookup_object(unsigned long ptr, int alias)
+static struct kmemleak_object *lookup_object(unsigned long ptr, int alias,
+ bool is_phys)
{
- struct rb_node *rb = object_tree_root.rb_node;
+ struct rb_node *rb = is_phys ? object_phys_tree_root.rb_node :
+ object_tree_root.rb_node;
unsigned long untagged_ptr = (unsigned long)kasan_reset_tag((void *)ptr);
while (rb) {
@@ -517,14 +523,15 @@ static void put_object(struct kmemleak_object *object)
/*
* Look up an object in the object search tree and increase its use_count.
*/
-static struct kmemleak_object *find_and_get_object(unsigned long ptr, int alias)
+static struct kmemleak_object *find_and_get_object(unsigned long ptr, int alias,
+ bool is_phys)
{
unsigned long flags;
struct kmemleak_object *object;
rcu_read_lock();
raw_spin_lock_irqsave(&kmemleak_lock, flags);
- object = lookup_object(ptr, alias);
+ object = lookup_object(ptr, alias, is_phys);
raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
/* check whether the object is still available */
@@ -536,27 +543,32 @@ static struct kmemleak_object *find_and_get_object(unsigned long ptr, int alias)
}
/*
- * Remove an object from the object_tree_root and object_list. Must be called
- * with the kmemleak_lock held _if_ kmemleak is still enabled.
+ * Remove an object from the object_tree_root (or object_phys_tree_root)
+ * and object_list. Must be called with the kmemleak_lock held _if_ kmemleak
+ * is still enabled.
*/
static void __remove_object(struct kmemleak_object *object)
{
- rb_erase(&object->rb_node, &object_tree_root);
+ rb_erase(&object->rb_node, object->flags & OBJECT_PHYS ?
+ &object_phys_tree_root :
+ &object_tree_root);
list_del_rcu(&object->object_list);
}
/*
* Look up an object in the object search tree and remove it from both
- * object_tree_root and object_list. The returned object's use_count should be
- * at least 1, as initially set by create_object().
+ * object_tree_root (or object_phys_tree_root) and object_list. The
+ * returned object's use_count should be at least 1, as initially set
+ * by create_object().
*/
-static struct kmemleak_object *find_and_remove_object(unsigned long ptr, int alias)
+static struct kmemleak_object *find_and_remove_object(unsigned long ptr, int alias,
+ bool is_phys)
{
unsigned long flags;
struct kmemleak_object *object;
raw_spin_lock_irqsave(&kmemleak_lock, flags);
- object = lookup_object(ptr, alias);
+ object = lookup_object(ptr, alias, is_phys);
if (object)
__remove_object(object);
raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
@@ -574,7 +586,8 @@ static int __save_stack_trace(unsigned long *trace)
/*
* Create the metadata (struct kmemleak_object) corresponding to an allocated
- * memory block and add it to the object_list and object_tree_root.
+ * memory block and add it to the object_list and object_tree_root (or
+ * object_phys_tree_root).
*/
static struct kmemleak_object *create_object(unsigned long ptr, size_t size,
int min_count, gfp_t gfp,
@@ -633,7 +646,8 @@ static struct kmemleak_object *create_object(unsigned long ptr, size_t size,
untagged_ptr = (unsigned long)kasan_reset_tag((void *)ptr);
min_addr = min(min_addr, untagged_ptr);
max_addr = max(max_addr, untagged_ptr + size);
- link = &object_tree_root.rb_node;
+ link = is_phys ? &object_phys_tree_root.rb_node :
+ &object_tree_root.rb_node;
rb_parent = NULL;
while (*link) {
rb_parent = *link;
@@ -657,7 +671,8 @@ static struct kmemleak_object *create_object(unsigned long ptr, size_t size,
}
}
rb_link_node(&object->rb_node, rb_parent, link);
- rb_insert_color(&object->rb_node, &object_tree_root);
+ rb_insert_color(&object->rb_node, is_phys ? &object_phys_tree_root :
+ &object_tree_root);
list_add_tail_rcu(&object->object_list, &object_list);
out:
@@ -693,7 +708,7 @@ static void delete_object_full(unsigned long ptr)
{
struct kmemleak_object *object;
- object = find_and_remove_object(ptr, 0);
+ object = find_and_remove_object(ptr, 0, false);
if (!object) {
#ifdef DEBUG
kmemleak_warn("Freeing unknown object at 0x%08lx\n",
@@ -709,12 +724,12 @@ static void delete_object_full(unsigned long ptr)
* delete it. If the memory block is partially freed, the function may create
* additional metadata for the remaining parts of the block.
*/
-static void delete_object_part(unsigned long ptr, size_t size)
+static void delete_object_part(unsigned long ptr, size_t size, bool is_phys)
{
struct kmemleak_object *object;
unsigned long start, end;
- object = find_and_remove_object(ptr, 1);
+ object = find_and_remove_object(ptr, 1, is_phys);
if (!object) {
#ifdef DEBUG
kmemleak_warn("Partially freeing unknown object at 0x%08lx (size %zu)\n",
@@ -756,11 +771,11 @@ static void paint_it(struct kmemleak_object *object, int color)
raw_spin_unlock_irqrestore(&object->lock, flags);
}
-static void paint_ptr(unsigned long ptr, int color)
+static void paint_ptr(unsigned long ptr, int color, bool is_phys)
{
struct kmemleak_object *object;
- object = find_and_get_object(ptr, 0);
+ object = find_and_get_object(ptr, 0, is_phys);
if (!object) {
kmemleak_warn("Trying to color unknown object at 0x%08lx as %s\n",
ptr,
@@ -776,18 +791,18 @@ static void paint_ptr(unsigned long ptr, int color)
* Mark an object permanently as gray-colored so that it can no longer be
* reported as a leak. This is used in general to mark a false positive.
*/
-static void make_gray_object(unsigned long ptr)
+static void make_gray_object(unsigned long ptr, bool is_phys)
{
- paint_ptr(ptr, KMEMLEAK_GREY);
+ paint_ptr(ptr, KMEMLEAK_GREY, is_phys);
}
/*
* Mark the object as black-colored so that it is ignored from scans and
* reporting.
*/
-static void make_black_object(unsigned long ptr)
+static void make_black_object(unsigned long ptr, bool is_phys)
{
- paint_ptr(ptr, KMEMLEAK_BLACK);
+ paint_ptr(ptr, KMEMLEAK_BLACK, is_phys);
}
/*
@@ -802,7 +817,7 @@ static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp)
unsigned long untagged_ptr;
unsigned long untagged_objp;
- object = find_and_get_object(ptr, 1);
+ object = find_and_get_object(ptr, 1, false);
if (!object) {
kmemleak_warn("Adding scan area to unknown object at 0x%08lx\n",
ptr);
@@ -852,7 +867,7 @@ static void object_set_excess_ref(unsigned long ptr, unsigned long excess_ref)
unsigned long flags;
struct kmemleak_object *object;
- object = find_and_get_object(ptr, 0);
+ object = find_and_get_object(ptr, 0, false);
if (!object) {
kmemleak_warn("Setting excess_ref on unknown object at 0x%08lx\n",
ptr);
@@ -875,7 +890,7 @@ static void object_no_scan(unsigned long ptr)
unsigned long flags;
struct kmemleak_object *object;
- object = find_and_get_object(ptr, 0);
+ object = find_and_get_object(ptr, 0, false);
if (!object) {
kmemleak_warn("Not scanning unknown object at 0x%08lx\n", ptr);
return;
@@ -993,7 +1008,7 @@ void __ref kmemleak_free_part(const void *ptr, size_t size)
pr_debug("%s(0x%p)\n", __func__, ptr);
if (kmemleak_enabled && ptr && !IS_ERR(ptr))
- delete_object_part((unsigned long)ptr, size);
+ delete_object_part((unsigned long)ptr, size, false);
}
EXPORT_SYMBOL_GPL(kmemleak_free_part);
@@ -1034,7 +1049,7 @@ void __ref kmemleak_update_trace(const void *ptr)
if (!kmemleak_enabled || IS_ERR_OR_NULL(ptr))
return;
- object = find_and_get_object((unsigned long)ptr, 1);
+ object = find_and_get_object((unsigned long)ptr, 1, false);
if (!object) {
#ifdef DEBUG
kmemleak_warn("Updating stack trace for unknown object at %p\n",
@@ -1063,7 +1078,7 @@ void __ref kmemleak_not_leak(const void *ptr)
pr_debug("%s(0x%p)\n", __func__, ptr);
if (kmemleak_enabled && ptr && !IS_ERR(ptr))
- make_gray_object((unsigned long)ptr);
+ make_gray_object((unsigned long)ptr, false);
}
EXPORT_SYMBOL(kmemleak_not_leak);
@@ -1081,7 +1096,7 @@ void __ref kmemleak_ignore(const void *ptr)
pr_debug("%s(0x%p)\n", __func__, ptr);
if (kmemleak_enabled && ptr && !IS_ERR(ptr))
- make_black_object((unsigned long)ptr);
+ make_black_object((unsigned long)ptr, false);
}
EXPORT_SYMBOL(kmemleak_ignore);
@@ -1275,7 +1290,7 @@ static void scan_block(void *_start, void *_end,
* is still present in object_tree_root and object_list
* (with updates protected by kmemleak_lock).
*/
- object = lookup_object(pointer, 1);
+ object = lookup_object(pointer, 1, false);
if (!object)
continue;
if (object == scanned)
@@ -1299,7 +1314,7 @@ static void scan_block(void *_start, void *_end,
raw_spin_unlock(&object->lock);
if (excess_ref) {
- object = lookup_object(excess_ref, 0);
+ object = lookup_object(excess_ref, 0, false);
if (!object)
continue;
if (object == scanned)
@@ -1728,7 +1743,7 @@ static int dump_str_object_info(const char *str)
if (kstrtoul(str, 0, &addr))
return -EINVAL;
- object = find_and_get_object(addr, 0);
+ object = find_and_get_object(addr, 0, false);
if (!object) {
pr_info("Unknown object at 0x%08lx\n", addr);
return -EINVAL;
--
2.25.1
On Fri, Jun 03, 2022 at 11:54:13AM +0800, Patrick Wang wrote:
> @@ -536,27 +543,32 @@ static struct kmemleak_object *find_and_get_object(unsigned long ptr, int alias)
> }
>
> /*
> - * Remove an object from the object_tree_root and object_list. Must be called
> - * with the kmemleak_lock held _if_ kmemleak is still enabled.
> + * Remove an object from the object_tree_root (or object_phys_tree_root)
> + * and object_list. Must be called with the kmemleak_lock held _if_ kmemleak
> + * is still enabled.
> */
> static void __remove_object(struct kmemleak_object *object)
> {
> - rb_erase(&object->rb_node, &object_tree_root);
> + rb_erase(&object->rb_node, object->flags & OBJECT_PHYS ?
> + &object_phys_tree_root :
> + &object_tree_root);
This pattern appears in a few place, I guess it's better with a macro,
say get_object_tree_root(object). But see how many are left, I have some
comments below on reducing the diff.
> @@ -709,12 +724,12 @@ static void delete_object_full(unsigned long ptr)
> * delete it. If the memory block is partially freed, the function may create
> * additional metadata for the remaining parts of the block.
> */
> -static void delete_object_part(unsigned long ptr, size_t size)
> +static void delete_object_part(unsigned long ptr, size_t size, bool is_phys)
> {
> struct kmemleak_object *object;
> unsigned long start, end;
>
> - object = find_and_remove_object(ptr, 1);
> + object = find_and_remove_object(ptr, 1, is_phys);
> if (!object) {
> #ifdef DEBUG
> kmemleak_warn("Partially freeing unknown object at 0x%08lx (size %zu)\n",
The previous patch introduced a check on object->flags for
delete_object_part(). I think you can just use is_phys directly now when
calling create_object().
> @@ -756,11 +771,11 @@ static void paint_it(struct kmemleak_object *object, int color)
> raw_spin_unlock_irqrestore(&object->lock, flags);
> }
>
> -static void paint_ptr(unsigned long ptr, int color)
> +static void paint_ptr(unsigned long ptr, int color, bool is_phys)
> {
> struct kmemleak_object *object;
>
> - object = find_and_get_object(ptr, 0);
> + object = find_and_get_object(ptr, 0, is_phys);
> if (!object) {
> kmemleak_warn("Trying to color unknown object at 0x%08lx as %s\n",
> ptr,
> @@ -776,18 +791,18 @@ static void paint_ptr(unsigned long ptr, int color)
> * Mark an object permanently as gray-colored so that it can no longer be
> * reported as a leak. This is used in general to mark a false positive.
> */
> -static void make_gray_object(unsigned long ptr)
> +static void make_gray_object(unsigned long ptr, bool is_phys)
> {
> - paint_ptr(ptr, KMEMLEAK_GREY);
> + paint_ptr(ptr, KMEMLEAK_GREY, is_phys);
> }
>
> /*
> * Mark the object as black-colored so that it is ignored from scans and
> * reporting.
> */
> -static void make_black_object(unsigned long ptr)
> +static void make_black_object(unsigned long ptr, bool is_phys)
> {
> - paint_ptr(ptr, KMEMLEAK_BLACK);
> + paint_ptr(ptr, KMEMLEAK_BLACK, is_phys);
> }
We won't need any of these functions to get an 'is_phys' argument if we
make kmemleak_alloc_phys() always create gray objects (do this as one of
the first patches in the series).
>
> /*
> @@ -802,7 +817,7 @@ static void add_scan_area(unsigned long ptr, size_t size, gfp_t gfp)
> unsigned long untagged_ptr;
> unsigned long untagged_objp;
>
> - object = find_and_get_object(ptr, 1);
> + object = find_and_get_object(ptr, 1, false);
> if (!object) {
> kmemleak_warn("Adding scan area to unknown object at 0x%08lx\n",
> ptr);
> @@ -852,7 +867,7 @@ static void object_set_excess_ref(unsigned long ptr, unsigned long excess_ref)
> unsigned long flags;
> struct kmemleak_object *object;
>
> - object = find_and_get_object(ptr, 0);
> + object = find_and_get_object(ptr, 0, false);
> if (!object) {
> kmemleak_warn("Setting excess_ref on unknown object at 0x%08lx\n",
> ptr);
> @@ -875,7 +890,7 @@ static void object_no_scan(unsigned long ptr)
> unsigned long flags;
> struct kmemleak_object *object;
>
> - object = find_and_get_object(ptr, 0);
> + object = find_and_get_object(ptr, 0, false);
> if (!object) {
> kmemleak_warn("Not scanning unknown object at 0x%08lx\n", ptr);
> return;
Same for these.
> @@ -993,7 +1008,7 @@ void __ref kmemleak_free_part(const void *ptr, size_t size)
> pr_debug("%s(0x%p)\n", __func__, ptr);
>
> if (kmemleak_enabled && ptr && !IS_ERR(ptr))
> - delete_object_part((unsigned long)ptr, size);
> + delete_object_part((unsigned long)ptr, size, false);
> }
> EXPORT_SYMBOL_GPL(kmemleak_free_part);
>
> @@ -1034,7 +1049,7 @@ void __ref kmemleak_update_trace(const void *ptr)
> if (!kmemleak_enabled || IS_ERR_OR_NULL(ptr))
> return;
>
> - object = find_and_get_object((unsigned long)ptr, 1);
> + object = find_and_get_object((unsigned long)ptr, 1, false);
> if (!object) {
> #ifdef DEBUG
> kmemleak_warn("Updating stack trace for unknown object at %p\n",
> @@ -1063,7 +1078,7 @@ void __ref kmemleak_not_leak(const void *ptr)
> pr_debug("%s(0x%p)\n", __func__, ptr);
>
> if (kmemleak_enabled && ptr && !IS_ERR(ptr))
> - make_gray_object((unsigned long)ptr);
> + make_gray_object((unsigned long)ptr, false);
> }
> EXPORT_SYMBOL(kmemleak_not_leak);
>
> @@ -1081,7 +1096,7 @@ void __ref kmemleak_ignore(const void *ptr)
> pr_debug("%s(0x%p)\n", __func__, ptr);
>
> if (kmemleak_enabled && ptr && !IS_ERR(ptr))
> - make_black_object((unsigned long)ptr);
> + make_black_object((unsigned long)ptr, false);
> }
> EXPORT_SYMBOL(kmemleak_ignore);
If we avoid changing make_*_object(), we won't need these anymore.
> @@ -1275,7 +1290,7 @@ static void scan_block(void *_start, void *_end,
> * is still present in object_tree_root and object_list
> * (with updates protected by kmemleak_lock).
> */
> - object = lookup_object(pointer, 1);
> + object = lookup_object(pointer, 1, false);
> if (!object)
> continue;
> if (object == scanned)
> @@ -1299,7 +1314,7 @@ static void scan_block(void *_start, void *_end,
> raw_spin_unlock(&object->lock);
>
> if (excess_ref) {
> - object = lookup_object(excess_ref, 0);
> + object = lookup_object(excess_ref, 0, false);
> if (!object)
> continue;
> if (object == scanned)
> @@ -1728,7 +1743,7 @@ static int dump_str_object_info(const char *str)
>
> if (kstrtoul(str, 0, &addr))
> return -EINVAL;
> - object = find_and_get_object(addr, 0);
> + object = find_and_get_object(addr, 0, false);
> if (!object) {
> pr_info("Unknown object at 0x%08lx\n", addr);
> return -EINVAL;
I think find_and_get_object() is never called on a phys object, so you
can probably simplify these a bit. Just add an is_phys argument where
strictly necessary and maybe even add a separate function like
lookup_object_phys() to reduce the other changes.
--
Catalin
On 2022/6/6 22:38, Catalin Marinas wrote:
> On Fri, Jun 03, 2022 at 11:54:13AM +0800, Patrick Wang wrote:
>> @@ -536,27 +543,32 @@ static struct kmemleak_object *find_and_get_object(unsigned long ptr, int alias)
>> }
>>
>> /*
>> - * Remove an object from the object_tree_root and object_list. Must be called
>> - * with the kmemleak_lock held _if_ kmemleak is still enabled.
>> + * Remove an object from the object_tree_root (or object_phys_tree_root)
>> + * and object_list. Must be called with the kmemleak_lock held _if_ kmemleak
>> + * is still enabled.
>> */
>> static void __remove_object(struct kmemleak_object *object)
>> {
>> - rb_erase(&object->rb_node, &object_tree_root);
>> + rb_erase(&object->rb_node, object->flags & OBJECT_PHYS ?
>> + &object_phys_tree_root :
>> + &object_tree_root);
>
> This pattern appears in a few place, I guess it's better with a macro,
> say get_object_tree_root(object). But see how many are left, I have some
> comments below on reducing the diff.
Will do.
>
>> @@ -709,12 +724,12 @@ static void delete_object_full(unsigned long ptr)
>> * delete it. If the memory block is partially freed, the function may create
>> * additional metadata for the remaining parts of the block.
>> */
>> -static void delete_object_part(unsigned long ptr, size_t size)
>> +static void delete_object_part(unsigned long ptr, size_t size, bool is_phys)
>> {
>> struct kmemleak_object *object;
>> unsigned long start, end;
>>
>> - object = find_and_remove_object(ptr, 1);
>> + object = find_and_remove_object(ptr, 1, is_phys);
>> if (!object) {
>> #ifdef DEBUG
>> kmemleak_warn("Partially freeing unknown object at 0x%08lx (size %zu)\n",
>
> The previous patch introduced a check on object->flags for
> delete_object_part(). I think you can just use is_phys directly now when
> calling create_object().
Will do.
>
>> @@ -1275,7 +1290,7 @@ static void scan_block(void *_start, void *_end,
>> * is still present in object_tree_root and object_list
>> * (with updates protected by kmemleak_lock).
>> */
>> - object = lookup_object(pointer, 1);
>> + object = lookup_object(pointer, 1, false);
>> if (!object)
>> continue;
>> if (object == scanned)
>> @@ -1299,7 +1314,7 @@ static void scan_block(void *_start, void *_end,
>> raw_spin_unlock(&object->lock);
>>
>> if (excess_ref) {
>> - object = lookup_object(excess_ref, 0);
>> + object = lookup_object(excess_ref, 0, false);
>> if (!object)
>> continue;
>> if (object == scanned)
>> @@ -1728,7 +1743,7 @@ static int dump_str_object_info(const char *str)
>>
>> if (kstrtoul(str, 0, &addr))
>> return -EINVAL;
>> - object = find_and_get_object(addr, 0);
>> + object = find_and_get_object(addr, 0, false);
>> if (!object) {
>> pr_info("Unknown object at 0x%08lx\n", addr);
>> return -EINVAL;
>
> I think find_and_get_object() is never called on a phys object, so you
> can probably simplify these a bit. Just add an is_phys argument where
> strictly necessary and maybe even add a separate function like
> lookup_object_phys() to reduce the other changes.
Will add lookup_object_phys() function and find_and_get_object_phys()
function. The find_and_get_object() function is also called in many
places.
Thanks,
Patrick