vmaps are temporary kernel mappings that may be of long duration.
Reusing a vmap on an object is preferrable for a driver as the cost of
setting up the vmap can otherwise dominate the operation on the object.
However, the vmap address space is rather limited on 32bit systems and
so we add a notification for vmap pressure in order for the driver to
release any cached vmappings.
The interface is styled after the oom-notifier where the callees are
passed a pointer to an unsigned long counter for them to indicate if they
have freed any space.
Signed-off-by: Chris Wilson <[email protected]>
Cc: Andrew Morton <[email protected]>
Cc: David Rientjes <[email protected]>
Cc: Roman Pen <[email protected]>
Cc: Mel Gorman <[email protected]>
Cc: [email protected]
Cc: [email protected]
---
include/linux/vmalloc.h | 4 ++++
mm/vmalloc.c | 22 ++++++++++++++++++++++
2 files changed, 26 insertions(+)
diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h
index d1f1d338af20..edd676b8e112 100644
--- a/include/linux/vmalloc.h
+++ b/include/linux/vmalloc.h
@@ -187,4 +187,8 @@ pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms)
#define VMALLOC_TOTAL 0UL
#endif
+struct notitifer_block;
+int register_vmap_purge_notifier(struct notifier_block *nb);
+int unregister_vmap_purge_notifier(struct notifier_block *nb);
+
#endif /* _LINUX_VMALLOC_H */
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index fb42a5bffe47..fd2ca94c2732 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -21,6 +21,7 @@
#include <linux/debugobjects.h>
#include <linux/kallsyms.h>
#include <linux/list.h>
+#include <linux/notifier.h>
#include <linux/rbtree.h>
#include <linux/radix-tree.h>
#include <linux/rcupdate.h>
@@ -344,6 +345,8 @@ static void __insert_vmap_area(struct vmap_area *va)
static void purge_vmap_area_lazy(void);
+static BLOCKING_NOTIFIER_HEAD(vmap_notify_list);
+
/*
* Allocate a region of KVA of the specified size and alignment, within the
* vstart and vend.
@@ -356,6 +359,7 @@ static struct vmap_area *alloc_vmap_area(unsigned long size,
struct vmap_area *va;
struct rb_node *n;
unsigned long addr;
+ unsigned long freed;
int purged = 0;
struct vmap_area *first;
@@ -468,6 +472,12 @@ overflow:
purged = 1;
goto retry;
}
+ freed = 0;
+ blocking_notifier_call_chain(&vmap_notify_list, 0, &freed);
+ if (freed > 0) {
+ purged = 0;
+ goto retry;
+ }
if (printk_ratelimit())
pr_warn("vmap allocation for size %lu failed: "
"use vmalloc=<size> to increase size.\n", size);
@@ -475,6 +485,18 @@ overflow:
return ERR_PTR(-EBUSY);
}
+int register_vmap_purge_notifier(struct notifier_block *nb)
+{
+ return blocking_notifier_chain_register(&vmap_notify_list, nb);
+}
+EXPORT_SYMBOL_GPL(register_vmap_purge_notifier);
+
+int unregister_vmap_purge_notifier(struct notifier_block *nb)
+{
+ return blocking_notifier_chain_unregister(&vmap_notify_list, nb);
+}
+EXPORT_SYMBOL_GPL(unregister_vmap_purge_notifier);
+
static void __free_vmap_area(struct vmap_area *va)
{
BUG_ON(RB_EMPTY_NODE(&va->rb_node));
--
2.8.0.rc3
If the core runs out of vmap address space, it will call a notifier in
case any driver can reap some of its vmaps. As i915.ko is possibily
holding onto vmap address space that could be recovered, hook into the
notifier chain and try and reap objects holding onto vmaps.
Signed-off-by: Chris Wilson <[email protected]>
Cc: Andrew Morton <[email protected]>
Cc: David Rientjes <[email protected]>
Cc: Roman Pen <[email protected]>
Cc: Mel Gorman <[email protected]>
Cc: [email protected]
Cc: [email protected]
---
drivers/gpu/drm/i915/i915_drv.h | 1 +
drivers/gpu/drm/i915/i915_gem_shrinker.c | 39 ++++++++++++++++++++++++++++++++
2 files changed, 40 insertions(+)
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index b9989d05f82a..4646b8504b84 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -1257,6 +1257,7 @@ struct i915_gem_mm {
struct i915_hw_ppgtt *aliasing_ppgtt;
struct notifier_block oom_notifier;
+ struct notifier_block vmap_notifier;
struct shrinker shrinker;
bool shrinker_no_lock_stealing;
diff --git a/drivers/gpu/drm/i915/i915_gem_shrinker.c b/drivers/gpu/drm/i915/i915_gem_shrinker.c
index d3c473ffb90a..54943f983dc4 100644
--- a/drivers/gpu/drm/i915/i915_gem_shrinker.c
+++ b/drivers/gpu/drm/i915/i915_gem_shrinker.c
@@ -28,6 +28,7 @@
#include <linux/swap.h>
#include <linux/pci.h>
#include <linux/dma-buf.h>
+#include <linux/vmalloc.h>
#include <drm/drmP.h>
#include <drm/i915_drm.h>
@@ -356,6 +357,40 @@ i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr)
return NOTIFY_DONE;
}
+static int
+i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr)
+{
+ struct drm_i915_private *dev_priv =
+ container_of(nb, struct drm_i915_private, mm.vmap_notifier);
+ struct drm_device *dev = dev_priv->dev;
+ unsigned long timeout = msecs_to_jiffies(5000) + 1;
+ unsigned long freed_pages;
+ bool was_interruptible;
+ bool unlock;
+
+ while (!i915_gem_shrinker_lock(dev, &unlock) && --timeout) {
+ schedule_timeout_killable(1);
+ if (fatal_signal_pending(current))
+ return NOTIFY_DONE;
+ }
+ if (timeout == 0) {
+ pr_err("Unable to purge GPU vmaps due to lock contention.\n");
+ return NOTIFY_DONE;
+ }
+
+ was_interruptible = dev_priv->mm.interruptible;
+ dev_priv->mm.interruptible = false;
+
+ freed_pages = i915_gem_shrink_all(dev_priv);
+
+ dev_priv->mm.interruptible = was_interruptible;
+ if (unlock)
+ mutex_unlock(&dev->struct_mutex);
+
+ *(unsigned long *)ptr += freed_pages;
+ return NOTIFY_DONE;
+}
+
/**
* i915_gem_shrinker_init - Initialize i915 shrinker
* @dev_priv: i915 device
@@ -371,6 +406,9 @@ void i915_gem_shrinker_init(struct drm_i915_private *dev_priv)
dev_priv->mm.oom_notifier.notifier_call = i915_gem_shrinker_oom;
WARN_ON(register_oom_notifier(&dev_priv->mm.oom_notifier));
+
+ dev_priv->mm.vmap_notifier.notifier_call = i915_gem_shrinker_vmap;
+ WARN_ON(register_vmap_purge_notifier(&dev_priv->mm.vmap_notifier));
}
/**
@@ -381,6 +419,7 @@ void i915_gem_shrinker_init(struct drm_i915_private *dev_priv)
*/
void i915_gem_shrinker_cleanup(struct drm_i915_private *dev_priv)
{
+ WARN_ON(unregister_vmap_purge_notifier(&dev_priv->mm.vmap_notifier));
WARN_ON(unregister_oom_notifier(&dev_priv->mm.oom_notifier));
unregister_shrinker(&dev_priv->mm.shrinker);
}
--
2.8.0.rc3
Hi, Chris.
Comment is below.
On Thu, Mar 17, 2016 at 12:59 PM, Chris Wilson <[email protected]> wrote:
> vmaps are temporary kernel mappings that may be of long duration.
> Reusing a vmap on an object is preferrable for a driver as the cost of
> setting up the vmap can otherwise dominate the operation on the object.
> However, the vmap address space is rather limited on 32bit systems and
> so we add a notification for vmap pressure in order for the driver to
> release any cached vmappings.
>
> The interface is styled after the oom-notifier where the callees are
> passed a pointer to an unsigned long counter for them to indicate if they
> have freed any space.
>
> Signed-off-by: Chris Wilson <[email protected]>
> Cc: Andrew Morton <[email protected]>
> Cc: David Rientjes <[email protected]>
> Cc: Roman Pen <[email protected]>
> Cc: Mel Gorman <[email protected]>
> Cc: [email protected]
> Cc: [email protected]
> ---
> include/linux/vmalloc.h | 4 ++++
> mm/vmalloc.c | 22 ++++++++++++++++++++++
> 2 files changed, 26 insertions(+)
>
> diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h
> index d1f1d338af20..edd676b8e112 100644
> --- a/include/linux/vmalloc.h
> +++ b/include/linux/vmalloc.h
> @@ -187,4 +187,8 @@ pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms)
> #define VMALLOC_TOTAL 0UL
> #endif
>
> +struct notitifer_block;
> +int register_vmap_purge_notifier(struct notifier_block *nb);
> +int unregister_vmap_purge_notifier(struct notifier_block *nb);
> +
> #endif /* _LINUX_VMALLOC_H */
> diff --git a/mm/vmalloc.c b/mm/vmalloc.c
> index fb42a5bffe47..fd2ca94c2732 100644
> --- a/mm/vmalloc.c
> +++ b/mm/vmalloc.c
> @@ -21,6 +21,7 @@
> #include <linux/debugobjects.h>
> #include <linux/kallsyms.h>
> #include <linux/list.h>
> +#include <linux/notifier.h>
> #include <linux/rbtree.h>
> #include <linux/radix-tree.h>
> #include <linux/rcupdate.h>
> @@ -344,6 +345,8 @@ static void __insert_vmap_area(struct vmap_area *va)
>
> static void purge_vmap_area_lazy(void);
>
> +static BLOCKING_NOTIFIER_HEAD(vmap_notify_list);
> +
> /*
> * Allocate a region of KVA of the specified size and alignment, within the
> * vstart and vend.
> @@ -356,6 +359,7 @@ static struct vmap_area *alloc_vmap_area(unsigned long size,
> struct vmap_area *va;
> struct rb_node *n;
> unsigned long addr;
> + unsigned long freed;
> int purged = 0;
> struct vmap_area *first;
>
> @@ -468,6 +472,12 @@ overflow:
> purged = 1;
> goto retry;
> }
> + freed = 0;
> + blocking_notifier_call_chain(&vmap_notify_list, 0, &freed);
It seems to me that alloc_vmap_area() was designed not to sleep,
at least on GFP_NOWAIT path (__GFP_DIRECT_RECLAIM is not set).
But blocking_notifier_call_chain() might sleep.
Roman.
On Thu, Mar 17, 2016 at 1:57 PM, Chris Wilson <[email protected]> wrote:
> On Thu, Mar 17, 2016 at 01:37:06PM +0100, Roman Peniaev wrote:
>> > + freed = 0;
>> > + blocking_notifier_call_chain(&vmap_notify_list, 0, &freed);
>>
>> It seems to me that alloc_vmap_area() was designed not to sleep,
>> at least on GFP_NOWAIT path (__GFP_DIRECT_RECLAIM is not set).
>>
>> But blocking_notifier_call_chain() might sleep.
>
> Indeed, I had not anticipated anybody using GFP_ATOMIC or equivalently
> restrictive gfp_t for vmap and yes there are such callers.
>
> Would guarding the notifier with gfp & __GFP_DIRECT_RECLAIM and
> !(gfp & __GFP_NORETRY) == be sufficient? Is that enough for GFP_NOFS?
I would use gfpflags_allow_blocking() for that purpose.
Roman
On Thu, Mar 17, 2016 at 01:37:06PM +0100, Roman Peniaev wrote:
> > + freed = 0;
> > + blocking_notifier_call_chain(&vmap_notify_list, 0, &freed);
>
> It seems to me that alloc_vmap_area() was designed not to sleep,
> at least on GFP_NOWAIT path (__GFP_DIRECT_RECLAIM is not set).
>
> But blocking_notifier_call_chain() might sleep.
Indeed, I had not anticipated anybody using GFP_ATOMIC or equivalently
restrictive gfp_t for vmap and yes there are such callers.
Would guarding the notifier with gfp & __GFP_DIRECT_RECLAIM and
!(gfp & __GFP_NORETRY) == be sufficient? Is that enough for GFP_NOFS?
-Chris
--
Chris Wilson, Intel Open Source Technology Centre
On Thu, Mar 17, 2016 at 02:21:40PM +0100, Roman Peniaev wrote:
> On Thu, Mar 17, 2016 at 1:57 PM, Chris Wilson <[email protected]> wrote:
> > On Thu, Mar 17, 2016 at 01:37:06PM +0100, Roman Peniaev wrote:
> >> > + freed = 0;
> >> > + blocking_notifier_call_chain(&vmap_notify_list, 0, &freed);
> >>
> >> It seems to me that alloc_vmap_area() was designed not to sleep,
> >> at least on GFP_NOWAIT path (__GFP_DIRECT_RECLAIM is not set).
> >>
> >> But blocking_notifier_call_chain() might sleep.
> >
> > Indeed, I had not anticipated anybody using GFP_ATOMIC or equivalently
> > restrictive gfp_t for vmap and yes there are such callers.
> >
> > Would guarding the notifier with gfp & __GFP_DIRECT_RECLAIM and
> > !(gfp & __GFP_NORETRY) == be sufficient? Is that enough for GFP_NOFS?
>
> I would use gfpflags_allow_blocking() for that purpose.
Thanks,
-Chris
--
Chris Wilson, Intel Open Source Technology Centre
vmaps are temporary kernel mappings that may be of long duration.
Reusing a vmap on an object is preferrable for a driver as the cost of
setting up the vmap can otherwise dominate the operation on the object.
However, the vmap address space is rather limited on 32bit systems and
so we add a notification for vmap pressure in order for the driver to
release any cached vmappings.
The interface is styled after the oom-notifier where the callees are
passed a pointer to an unsigned long counter for them to indicate if they
have freed any space.
v2: Guard the blocking notifier call with gfpflags_allow_blocking()
Signed-off-by: Chris Wilson <[email protected]>
Cc: Andrew Morton <[email protected]>
Cc: David Rientjes <[email protected]>
Cc: Roman Peniaev <[email protected]>
Cc: Mel Gorman <[email protected]>
Cc: [email protected]
Cc: [email protected]
---
include/linux/vmalloc.h | 4 ++++
mm/vmalloc.c | 27 +++++++++++++++++++++++++++
2 files changed, 31 insertions(+)
diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h
index d1f1d338af20..edd676b8e112 100644
--- a/include/linux/vmalloc.h
+++ b/include/linux/vmalloc.h
@@ -187,4 +187,8 @@ pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms)
#define VMALLOC_TOTAL 0UL
#endif
+struct notitifer_block;
+int register_vmap_purge_notifier(struct notifier_block *nb);
+int unregister_vmap_purge_notifier(struct notifier_block *nb);
+
#endif /* _LINUX_VMALLOC_H */
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index fb42a5bffe47..12d27ac303ae 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -21,6 +21,7 @@
#include <linux/debugobjects.h>
#include <linux/kallsyms.h>
#include <linux/list.h>
+#include <linux/notifier.h>
#include <linux/rbtree.h>
#include <linux/radix-tree.h>
#include <linux/rcupdate.h>
@@ -344,6 +345,8 @@ static void __insert_vmap_area(struct vmap_area *va)
static void purge_vmap_area_lazy(void);
+static BLOCKING_NOTIFIER_HEAD(vmap_notify_list);
+
/*
* Allocate a region of KVA of the specified size and alignment, within the
* vstart and vend.
@@ -363,6 +366,8 @@ static struct vmap_area *alloc_vmap_area(unsigned long size,
BUG_ON(offset_in_page(size));
BUG_ON(!is_power_of_2(align));
+ might_sleep_if(gfpflags_allow_blocking(gfp_mask));
+
va = kmalloc_node(sizeof(struct vmap_area),
gfp_mask & GFP_RECLAIM_MASK, node);
if (unlikely(!va))
@@ -468,6 +473,16 @@ overflow:
purged = 1;
goto retry;
}
+
+ if (gfpflags_allow_blocking(gfp_mask)) {
+ unsigned long freed = 0;
+ blocking_notifier_call_chain(&vmap_notify_list, 0, &freed);
+ if (freed > 0) {
+ purged = 0;
+ goto retry;
+ }
+ }
+
if (printk_ratelimit())
pr_warn("vmap allocation for size %lu failed: "
"use vmalloc=<size> to increase size.\n", size);
@@ -475,6 +490,18 @@ overflow:
return ERR_PTR(-EBUSY);
}
+int register_vmap_purge_notifier(struct notifier_block *nb)
+{
+ return blocking_notifier_chain_register(&vmap_notify_list, nb);
+}
+EXPORT_SYMBOL_GPL(register_vmap_purge_notifier);
+
+int unregister_vmap_purge_notifier(struct notifier_block *nb)
+{
+ return blocking_notifier_chain_unregister(&vmap_notify_list, nb);
+}
+EXPORT_SYMBOL_GPL(unregister_vmap_purge_notifier);
+
static void __free_vmap_area(struct vmap_area *va)
{
BUG_ON(RB_EMPTY_NODE(&va->rb_node));
--
2.8.0.rc3
On Thu, Mar 17, 2016 at 01:34:59PM +0000, Chris Wilson wrote:
> vmaps are temporary kernel mappings that may be of long duration.
> Reusing a vmap on an object is preferrable for a driver as the cost of
> setting up the vmap can otherwise dominate the operation on the object.
> However, the vmap address space is rather limited on 32bit systems and
> so we add a notification for vmap pressure in order for the driver to
> release any cached vmappings.
>
> The interface is styled after the oom-notifier where the callees are
> passed a pointer to an unsigned long counter for them to indicate if they
> have freed any space.
>
> v2: Guard the blocking notifier call with gfpflags_allow_blocking()
>
> Signed-off-by: Chris Wilson <[email protected]>
> Cc: Andrew Morton <[email protected]>
> Cc: David Rientjes <[email protected]>
> Cc: Roman Peniaev <[email protected]>
> Cc: Mel Gorman <[email protected]>
> Cc: [email protected]
> Cc: [email protected]
> ---
> include/linux/vmalloc.h | 4 ++++
> mm/vmalloc.c | 27 +++++++++++++++++++++++++++
> 2 files changed, 31 insertions(+)
>
> diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h
> index d1f1d338af20..edd676b8e112 100644
> --- a/include/linux/vmalloc.h
> +++ b/include/linux/vmalloc.h
> @@ -187,4 +187,8 @@ pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms)
> #define VMALLOC_TOTAL 0UL
> #endif
>
> +struct notitifer_block;
Omg. /o\
-Chris
--
Chris Wilson, Intel Open Source Technology Centre
On Thu, 17 Mar 2016 13:41:56 +0000 Chris Wilson <[email protected]> wrote:
> On Thu, Mar 17, 2016 at 01:34:59PM +0000, Chris Wilson wrote:
> > vmaps are temporary kernel mappings that may be of long duration.
> > Reusing a vmap on an object is preferrable for a driver as the cost of
> > setting up the vmap can otherwise dominate the operation on the object.
> > However, the vmap address space is rather limited on 32bit systems and
> > so we add a notification for vmap pressure in order for the driver to
> > release any cached vmappings.
> >
> > The interface is styled after the oom-notifier where the callees are
> > passed a pointer to an unsigned long counter for them to indicate if they
> > have freed any space.
> >
> > v2: Guard the blocking notifier call with gfpflags_allow_blocking()
> >
> > Signed-off-by: Chris Wilson <[email protected]>
> > Cc: Andrew Morton <[email protected]>
> > Cc: David Rientjes <[email protected]>
> > Cc: Roman Peniaev <[email protected]>
> > Cc: Mel Gorman <[email protected]>
> > Cc: [email protected]
> > Cc: [email protected]
> > ---
> > include/linux/vmalloc.h | 4 ++++
> > mm/vmalloc.c | 27 +++++++++++++++++++++++++++
> > 2 files changed, 31 insertions(+)
> >
> > diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h
> > index d1f1d338af20..edd676b8e112 100644
> > --- a/include/linux/vmalloc.h
> > +++ b/include/linux/vmalloc.h
> > @@ -187,4 +187,8 @@ pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms)
> > #define VMALLOC_TOTAL 0UL
> > #endif
> >
> > +struct notitifer_block;
> Omg. /o\
Hah.
Please move the forward declaration to top-of-file. This prevents
people from later adding the same thing at line 100 - this has happened
before.
Apart from that, all looks OK to me - please merge it via the DRM tree
if that is more convenient.
vmaps are temporary kernel mappings that may be of long duration.
Reusing a vmap on an object is preferrable for a driver as the cost of
setting up the vmap can otherwise dominate the operation on the object.
However, the vmap address space is rather limited on 32bit systems and
so we add a notification for vmap pressure in order for the driver to
release any cached vmappings.
The interface is styled after the oom-notifier where the callees are
passed a pointer to an unsigned long counter for them to indicate if they
have freed any space.
v2: Guard the blocking notifier call with gfpflags_allow_blocking()
v3: Correct typo in forward declaration and move to head of file
Signed-off-by: Chris Wilson <[email protected]>
Cc: Andrew Morton <[email protected]>
Cc: David Rientjes <[email protected]>
Cc: Roman Peniaev <[email protected]>
Cc: Mel Gorman <[email protected]>
Cc: [email protected]
Cc: [email protected]
Acked-by: Andrew Morton <[email protected]> # for inclusion via DRM
---
Thanks Andrew, may I trouble someone for a Reviewed-by?
---
include/linux/vmalloc.h | 4 ++++
mm/vmalloc.c | 27 +++++++++++++++++++++++++++
2 files changed, 31 insertions(+)
diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h
index d1f1d338af20..8b51df3ab334 100644
--- a/include/linux/vmalloc.h
+++ b/include/linux/vmalloc.h
@@ -8,6 +8,7 @@
#include <linux/rbtree.h>
struct vm_area_struct; /* vma defining user mapping in mm_types.h */
+struct notifier_block; /* in notifier.h */
/* bits in flags of vmalloc's vm_struct below */
#define VM_IOREMAP 0x00000001 /* ioremap() and friends */
@@ -187,4 +188,7 @@ pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms)
#define VMALLOC_TOTAL 0UL
#endif
+int register_vmap_purge_notifier(struct notifier_block *nb);
+int unregister_vmap_purge_notifier(struct notifier_block *nb);
+
#endif /* _LINUX_VMALLOC_H */
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index fb42a5bffe47..12d27ac303ae 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -21,6 +21,7 @@
#include <linux/debugobjects.h>
#include <linux/kallsyms.h>
#include <linux/list.h>
+#include <linux/notifier.h>
#include <linux/rbtree.h>
#include <linux/radix-tree.h>
#include <linux/rcupdate.h>
@@ -344,6 +345,8 @@ static void __insert_vmap_area(struct vmap_area *va)
static void purge_vmap_area_lazy(void);
+static BLOCKING_NOTIFIER_HEAD(vmap_notify_list);
+
/*
* Allocate a region of KVA of the specified size and alignment, within the
* vstart and vend.
@@ -363,6 +366,8 @@ static struct vmap_area *alloc_vmap_area(unsigned long size,
BUG_ON(offset_in_page(size));
BUG_ON(!is_power_of_2(align));
+ might_sleep_if(gfpflags_allow_blocking(gfp_mask));
+
va = kmalloc_node(sizeof(struct vmap_area),
gfp_mask & GFP_RECLAIM_MASK, node);
if (unlikely(!va))
@@ -468,6 +473,16 @@ overflow:
purged = 1;
goto retry;
}
+
+ if (gfpflags_allow_blocking(gfp_mask)) {
+ unsigned long freed = 0;
+ blocking_notifier_call_chain(&vmap_notify_list, 0, &freed);
+ if (freed > 0) {
+ purged = 0;
+ goto retry;
+ }
+ }
+
if (printk_ratelimit())
pr_warn("vmap allocation for size %lu failed: "
"use vmalloc=<size> to increase size.\n", size);
@@ -475,6 +490,18 @@ overflow:
return ERR_PTR(-EBUSY);
}
+int register_vmap_purge_notifier(struct notifier_block *nb)
+{
+ return blocking_notifier_chain_register(&vmap_notify_list, nb);
+}
+EXPORT_SYMBOL_GPL(register_vmap_purge_notifier);
+
+int unregister_vmap_purge_notifier(struct notifier_block *nb)
+{
+ return blocking_notifier_chain_unregister(&vmap_notify_list, nb);
+}
+EXPORT_SYMBOL_GPL(unregister_vmap_purge_notifier);
+
static void __free_vmap_area(struct vmap_area *va)
{
BUG_ON(RB_EMPTY_NODE(&va->rb_node));
--
2.8.0.rc3