To generalize kfree in later patch, implement __kmem_cache_free() that
takes caller address and make kmem_cache_free() wrapper of it.
Now that kmem_cache_free() is inline function, we should
use _THIS_IP_ instead of _RET_IP_ for consistency.
Signed-off-by: Hyeonggon Yoo <[email protected]>
---
include/linux/slab.h | 16 +++++++++++++++-
mm/slab.c | 17 +++++------------
mm/slob.c | 13 +++++++------
mm/slub.c | 9 +++++----
4 files changed, 32 insertions(+), 23 deletions(-)
diff --git a/include/linux/slab.h b/include/linux/slab.h
index c8c82087c3f9..0630c37ee630 100644
--- a/include/linux/slab.h
+++ b/include/linux/slab.h
@@ -462,7 +462,21 @@ void *kmem_cache_alloc_lru(struct kmem_cache *s, struct list_lru *lru, gfp_t gfp
return __kmem_cache_alloc_node(s, lru, gfpflags, NUMA_NO_NODE, _THIS_IP_);
}
-void kmem_cache_free(struct kmem_cache *s, void *objp);
+void __kmem_cache_free(struct kmem_cache *s, void *objp, unsigned long caller __maybe_unused);
+
+/**
+ * kmem_cache_free - Deallocate an object
+ * @s: The cache the allocation was from.
+ * @objp: The previously allocated object.
+ *
+ * Free an object which was previously allocated from this
+ * cache.
+ */
+static __always_inline void kmem_cache_free(struct kmem_cache *s, void *objp)
+{
+ __kmem_cache_free(s, objp, _THIS_IP_);
+}
+
/*
* Bulk allocation and freeing operations. These are accelerated in an
diff --git a/mm/slab.c b/mm/slab.c
index 424168b96790..d35873da5572 100644
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -3579,30 +3579,23 @@ void kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab)
}
#endif
-/**
- * kmem_cache_free - Deallocate an object
- * @cachep: The cache the allocation was from.
- * @objp: The previously allocated object.
- *
- * Free an object which was previously allocated from this
- * cache.
- */
-void kmem_cache_free(struct kmem_cache *cachep, void *objp)
+void __kmem_cache_free(struct kmem_cache *cachep, void *objp,
+ unsigned long caller __maybe_unused)
{
unsigned long flags;
cachep = cache_from_obj(cachep, objp);
if (!cachep)
return;
- trace_kmem_cache_free(cachep->name, _RET_IP_, objp);
+ trace_kmem_cache_free(cachep->name, caller, objp);
local_irq_save(flags);
debug_check_no_locks_freed(objp, cachep->object_size);
if (!(cachep->flags & SLAB_DEBUG_OBJECTS))
debug_check_no_obj_freed(objp, cachep->object_size);
- __cache_free(cachep, objp, _RET_IP_);
+ __cache_free(cachep, objp, caller);
local_irq_restore(flags);
}
-EXPORT_SYMBOL(kmem_cache_free);
+EXPORT_SYMBOL(__kmem_cache_free);
void kmem_cache_free_bulk(struct kmem_cache *orig_s, size_t size, void **p)
{
diff --git a/mm/slob.c b/mm/slob.c
index 1bb4c577b908..e893d182d471 100644
--- a/mm/slob.c
+++ b/mm/slob.c
@@ -631,7 +631,7 @@ void *__kmem_cache_alloc_node(struct kmem_cache *cachep, struct list_lru *lru __
}
EXPORT_SYMBOL(__kmem_cache_alloc_node);
-static void __kmem_cache_free(void *b, int size)
+static void ____kmem_cache_free(void *b, int size)
{
if (size < PAGE_SIZE)
slob_free(b, size);
@@ -644,23 +644,24 @@ static void kmem_rcu_free(struct rcu_head *head)
struct slob_rcu *slob_rcu = (struct slob_rcu *)head;
void *b = (void *)slob_rcu - (slob_rcu->size - sizeof(struct slob_rcu));
- __kmem_cache_free(b, slob_rcu->size);
+ ____kmem_cache_free(b, slob_rcu->size);
}
-void kmem_cache_free(struct kmem_cache *c, void *b)
+void __kmem_cache_free(struct kmem_cache *c, void *b,
+ unsigned long caller __maybe_unused)
{
kmemleak_free_recursive(b, c->flags);
- trace_kmem_cache_free(c->name, _RET_IP_, b);
+ trace_kmem_cache_free(c->name, caller, b);
if (unlikely(c->flags & SLAB_TYPESAFE_BY_RCU)) {
struct slob_rcu *slob_rcu;
slob_rcu = b + (c->size - sizeof(struct slob_rcu));
slob_rcu->size = c->size;
call_rcu(&slob_rcu->head, kmem_rcu_free);
} else {
- __kmem_cache_free(b, c->size);
+ ____kmem_cache_free(b, c->size);
}
}
-EXPORT_SYMBOL(kmem_cache_free);
+EXPORT_SYMBOL(__kmem_cache_free);
void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
{
diff --git a/mm/slub.c b/mm/slub.c
index a088d4fa1062..a72a2d08e793 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -3466,15 +3466,16 @@ void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr)
}
#endif
-void kmem_cache_free(struct kmem_cache *s, void *x)
+void __kmem_cache_free(struct kmem_cache *s, void *x,
+ unsigned long caller __maybe_unused)
{
s = cache_from_obj(s, x);
if (!s)
return;
- trace_kmem_cache_free(s->name, _RET_IP_, x);
- slab_free(s, virt_to_slab(x), x, NULL, 1, _RET_IP_);
+ trace_kmem_cache_free(s->name, caller, x);
+ slab_free(s, virt_to_slab(x), x, NULL, 1, caller);
}
-EXPORT_SYMBOL(kmem_cache_free);
+EXPORT_SYMBOL(__kmem_cache_free);
struct detached_freelist {
struct slab *slab;
--
2.32.0