The patchset contains few changes to the vmalloc code, which are
leading to some performance gains and code simplification.
Also, it exports a number of pages, used by vmalloc(),
in /proc/meminfo.
Patch (1) removes some redundancy on __vunmap().
Patch (2) separates memory allocation and data initialization
in alloc_vmap_area()
Patch (3) adds vmalloc counter to /proc/meminfo.
RFC->v1:
- removed bogus empty lines (suggested by Matthew Wilcox)
- made nr_vmalloc_pages static (suggested by Matthew Wilcox)
- dropped patch 3 from RFC patchset, will post later with
some other changes
- dropped RFC
Roman Gushchin (3):
mm: refactor __vunmap() to avoid duplicated call to find_vm_area()
mm: separate memory allocation and actual work in alloc_vmap_area()
mm: show number of vmalloc pages in /proc/meminfo
fs/proc/meminfo.c | 2 +-
include/linux/vmalloc.h | 2 +
mm/vmalloc.c | 107 ++++++++++++++++++++++++++--------------
3 files changed, 73 insertions(+), 38 deletions(-)
--
2.19.2
__vunmap() calls find_vm_area() twice without an obvious reason:
first directly to get the area pointer, second indirectly by calling
remove_vm_area(), which is again searching for the area.
To remove this redundancy, let's split remove_vm_area() into
__remove_vm_area(struct vmap_area *), which performs the actual area
removal, and remove_vm_area(const void *addr) wrapper, which can
be used everywhere, where it has been used before.
On my test setup, I've got up to 12% speed up on vfree()'ing 1000000
of 4-pages vmalloc blocks.
Signed-off-by: Roman Gushchin <[email protected]>
Acked-by: Johannes Weiner <[email protected]>
Reviewed-by: Matthew Wilcox <[email protected]>
---
mm/vmalloc.c | 47 +++++++++++++++++++++++++++--------------------
1 file changed, 27 insertions(+), 20 deletions(-)
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index 871e41c55e23..7660e3ef4133 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -1462,6 +1462,24 @@ struct vm_struct *find_vm_area(const void *addr)
return NULL;
}
+static struct vm_struct *__remove_vm_area(struct vmap_area *va)
+{
+ struct vm_struct *vm = va->vm;
+
+ might_sleep();
+
+ spin_lock(&vmap_area_lock);
+ va->vm = NULL;
+ va->flags &= ~VM_VM_AREA;
+ va->flags |= VM_LAZY_FREE;
+ spin_unlock(&vmap_area_lock);
+
+ kasan_free_shadow(vm);
+ free_unmap_vmap_area(va);
+
+ return vm;
+}
+
/**
* remove_vm_area - find and remove a continuous kernel virtual area
* @addr: base address
@@ -1472,31 +1490,20 @@ struct vm_struct *find_vm_area(const void *addr)
*/
struct vm_struct *remove_vm_area(const void *addr)
{
+ struct vm_struct *vm = NULL;
struct vmap_area *va;
- might_sleep();
-
va = find_vmap_area((unsigned long)addr);
- if (va && va->flags & VM_VM_AREA) {
- struct vm_struct *vm = va->vm;
-
- spin_lock(&vmap_area_lock);
- va->vm = NULL;
- va->flags &= ~VM_VM_AREA;
- va->flags |= VM_LAZY_FREE;
- spin_unlock(&vmap_area_lock);
-
- kasan_free_shadow(vm);
- free_unmap_vmap_area(va);
+ if (va && va->flags & VM_VM_AREA)
+ vm = __remove_vm_area(va);
- return vm;
- }
- return NULL;
+ return vm;
}
static void __vunmap(const void *addr, int deallocate_pages)
{
struct vm_struct *area;
+ struct vmap_area *va;
if (!addr)
return;
@@ -1505,17 +1512,18 @@ static void __vunmap(const void *addr, int deallocate_pages)
addr))
return;
- area = find_vmap_area((unsigned long)addr)->vm;
- if (unlikely(!area)) {
+ va = find_vmap_area((unsigned long)addr);
+ if (unlikely(!va || !va->vm)) {
WARN(1, KERN_ERR "Trying to vfree() nonexistent vm area (%p)\n",
addr);
return;
}
+ area = va->vm;
debug_check_no_locks_freed(area->addr, get_vm_area_size(area));
debug_check_no_obj_freed(area->addr, get_vm_area_size(area));
- remove_vm_area(addr);
+ __remove_vm_area(va);
if (deallocate_pages) {
int i;
@@ -1530,7 +1538,6 @@ static void __vunmap(const void *addr, int deallocate_pages)
}
kfree(area);
- return;
}
static inline void __vfree_deferred(const void *addr)
--
2.19.2
alloc_vmap_area() is allocating memory for the vmap_area, and
performing the actual lookup of the vm area and vmap_area
initialization.
This prevents us from using a pre-allocated memory for the map_area
structure, which can be used in some cases to minimize the number
of required memory allocations.
Let's keep the memory allocation part in alloc_vmap_area() and
separate everything else into init_vmap_area().
Signed-off-by: Roman Gushchin <[email protected]>
Acked-by: Johannes Weiner <[email protected]>
Reviewed-by: Matthew Wilcox <[email protected]>
---
mm/vmalloc.c | 50 +++++++++++++++++++++++++++++++++-----------------
1 file changed, 33 insertions(+), 17 deletions(-)
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index 7660e3ef4133..042175d7d95f 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -395,16 +395,10 @@ static void purge_vmap_area_lazy(void);
static BLOCKING_NOTIFIER_HEAD(vmap_notify_list);
-/*
- * Allocate a region of KVA of the specified size and alignment, within the
- * vstart and vend.
- */
-static struct vmap_area *alloc_vmap_area(unsigned long size,
- unsigned long align,
- unsigned long vstart, unsigned long vend,
- int node, gfp_t gfp_mask)
+static int init_vmap_area(struct vmap_area *va, unsigned long size,
+ unsigned long align, unsigned long vstart,
+ unsigned long vend, int node, gfp_t gfp_mask)
{
- struct vmap_area *va;
struct rb_node *n;
unsigned long addr;
int purged = 0;
@@ -416,11 +410,6 @@ static struct vmap_area *alloc_vmap_area(unsigned long size,
might_sleep();
- va = kmalloc_node(sizeof(struct vmap_area),
- gfp_mask & GFP_RECLAIM_MASK, node);
- if (unlikely(!va))
- return ERR_PTR(-ENOMEM);
-
/*
* Only scan the relevant parts containing pointers to other objects
* to avoid false negatives.
@@ -512,7 +501,7 @@ static struct vmap_area *alloc_vmap_area(unsigned long size,
BUG_ON(va->va_start < vstart);
BUG_ON(va->va_end > vend);
- return va;
+ return 0;
overflow:
spin_unlock(&vmap_area_lock);
@@ -534,8 +523,35 @@ static struct vmap_area *alloc_vmap_area(unsigned long size,
if (!(gfp_mask & __GFP_NOWARN) && printk_ratelimit())
pr_warn("vmap allocation for size %lu failed: use vmalloc=<size> to increase size\n",
size);
- kfree(va);
- return ERR_PTR(-EBUSY);
+
+ return -EBUSY;
+}
+
+/*
+ * Allocate a region of KVA of the specified size and alignment, within the
+ * vstart and vend.
+ */
+static struct vmap_area *alloc_vmap_area(unsigned long size,
+ unsigned long align,
+ unsigned long vstart,
+ unsigned long vend,
+ int node, gfp_t gfp_mask)
+{
+ struct vmap_area *va;
+ int ret;
+
+ va = kmalloc_node(sizeof(struct vmap_area),
+ gfp_mask & GFP_RECLAIM_MASK, node);
+ if (unlikely(!va))
+ return ERR_PTR(-ENOMEM);
+
+ ret = init_vmap_area(va, size, align, vstart, vend, node, gfp_mask);
+ if (ret) {
+ kfree(va);
+ return ERR_PTR(ret);
+ }
+
+ return va;
}
int register_vmap_purge_notifier(struct notifier_block *nb)
--
2.19.2
Vmalloc() is getting more and more used these days (kernel stacks,
bpf and percpu allocator are new top users), and the total %
of memory consumed by vmalloc() can be pretty significant
and changes dynamically.
/proc/meminfo is the best place to display this information:
its top goal is to show top consumers of the memory.
Since the VmallocUsed field in /proc/meminfo is not in use
for quite a long time (it has been defined to 0 by the
commit a5ad88ce8c7f ("mm: get rid of 'vmalloc_info' from
/proc/meminfo")), let's reuse it for showing the actual
physical memory consumption of vmalloc().
Signed-off-by: Roman Gushchin <[email protected]>
Acked-by: Johannes Weiner <[email protected]>
---
fs/proc/meminfo.c | 2 +-
include/linux/vmalloc.h | 2 ++
mm/vmalloc.c | 10 ++++++++++
3 files changed, 13 insertions(+), 1 deletion(-)
diff --git a/fs/proc/meminfo.c b/fs/proc/meminfo.c
index 568d90e17c17..465ea0153b2a 100644
--- a/fs/proc/meminfo.c
+++ b/fs/proc/meminfo.c
@@ -120,7 +120,7 @@ static int meminfo_proc_show(struct seq_file *m, void *v)
show_val_kb(m, "Committed_AS: ", committed);
seq_printf(m, "VmallocTotal: %8lu kB\n",
(unsigned long)VMALLOC_TOTAL >> 10);
- show_val_kb(m, "VmallocUsed: ", 0ul);
+ show_val_kb(m, "VmallocUsed: ", vmalloc_nr_pages());
show_val_kb(m, "VmallocChunk: ", 0ul);
show_val_kb(m, "Percpu: ", pcpu_nr_pages());
diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h
index 398e9c95cd61..0b497408272b 100644
--- a/include/linux/vmalloc.h
+++ b/include/linux/vmalloc.h
@@ -63,10 +63,12 @@ extern void vm_unmap_aliases(void);
#ifdef CONFIG_MMU
extern void __init vmalloc_init(void);
+extern unsigned long vmalloc_nr_pages(void);
#else
static inline void vmalloc_init(void)
{
}
+static inline unsigned long vmalloc_nr_pages(void) { return 0; }
#endif
extern void *vmalloc(unsigned long size);
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index 042175d7d95f..efca916940e9 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -340,6 +340,13 @@ static unsigned long cached_align;
static unsigned long vmap_area_pcpu_hole;
+static atomic_long_t nr_vmalloc_pages;
+
+unsigned long vmalloc_nr_pages(void)
+{
+ return atomic_long_read(&nr_vmalloc_pages);
+}
+
static struct vmap_area *__find_vmap_area(unsigned long addr)
{
struct rb_node *n = vmap_area_root.rb_node;
@@ -1549,6 +1556,7 @@ static void __vunmap(const void *addr, int deallocate_pages)
BUG_ON(!page);
__free_pages(page, 0);
}
+ atomic_long_sub(area->nr_pages, &nr_vmalloc_pages);
kvfree(area->pages);
}
@@ -1717,12 +1725,14 @@ static void *__vmalloc_area_node(struct vm_struct *area, gfp_t gfp_mask,
if (unlikely(!page)) {
/* Successfully allocated i pages, free them in __vunmap() */
area->nr_pages = i;
+ atomic_long_add(area->nr_pages, &nr_vmalloc_pages);
goto fail;
}
area->pages[i] = page;
if (gfpflags_allow_blocking(gfp_mask|highmem_mask))
cond_resched();
}
+ atomic_long_add(area->nr_pages, &nr_vmalloc_pages);
if (map_vm_area(area, prot, pages))
goto fail;
--
2.19.2
These slipped through the cracks. +CC Andrew directly.
Andrew, if it's not too late yet, could you consider them for 5.1?
On Wed, Dec 19, 2018 at 09:37:48AM -0800, Roman Gushchin wrote:
> The patchset contains few changes to the vmalloc code, which are
> leading to some performance gains and code simplification.
>
> Also, it exports a number of pages, used by vmalloc(),
> in /proc/meminfo.
>
> Patch (1) removes some redundancy on __vunmap().
> Patch (2) separates memory allocation and data initialization
> in alloc_vmap_area()
> Patch (3) adds vmalloc counter to /proc/meminfo.
>
> RFC->v1:
> - removed bogus empty lines (suggested by Matthew Wilcox)
> - made nr_vmalloc_pages static (suggested by Matthew Wilcox)
> - dropped patch 3 from RFC patchset, will post later with
> some other changes
> - dropped RFC
>
> Roman Gushchin (3):
> mm: refactor __vunmap() to avoid duplicated call to find_vm_area()
> mm: separate memory allocation and actual work in alloc_vmap_area()
> mm: show number of vmalloc pages in /proc/meminfo
>
> fs/proc/meminfo.c | 2 +-
> include/linux/vmalloc.h | 2 +
> mm/vmalloc.c | 107 ++++++++++++++++++++++++++--------------
> 3 files changed, 73 insertions(+), 38 deletions(-)
>
> --
> 2.19.2
>
On Mon, 11 Feb 2019 14:08:22 -0500 Johannes Weiner <[email protected]> wrote:
> On Wed, Dec 19, 2018 at 09:37:48AM -0800, Roman Gushchin wrote:
> > The patchset contains few changes to the vmalloc code, which are
> > leading to some performance gains and code simplification.
> >
> > Also, it exports a number of pages, used by vmalloc(),
> > in /proc/meminfo.
>
> These slipped through the cracks. +CC Andrew directly.
> Andrew, if it's not too late yet, could you consider them for 5.1?
>
There's been some activity in vmalloc.c lately and these have
bitrotted. They'll need a redo, please.
On Mon, Feb 11, 2019 at 11:18:45AM -0800, Andrew Morton wrote:
> On Mon, 11 Feb 2019 14:08:22 -0500 Johannes Weiner <[email protected]> wrote:
>
> > On Wed, Dec 19, 2018 at 09:37:48AM -0800, Roman Gushchin wrote:
> > > The patchset contains few changes to the vmalloc code, which are
> > > leading to some performance gains and code simplification.
> > >
> > > Also, it exports a number of pages, used by vmalloc(),
> > > in /proc/meminfo.
> >
> > These slipped through the cracks. +CC Andrew directly.
> > Andrew, if it's not too late yet, could you consider them for 5.1?
> >
>
> There's been some activity in vmalloc.c lately and these have
> bitrotted. They'll need a redo, please.
>
Will do. Thanks!