mmu_interval_notifier_insert() can't be called safely from inside the
invalidate() callback because it can acquire the mmap_sem lock which
might already be held. Insertion might be needed when the invalidate()
callback creates a "hole" in the interval being tracked (i.e., the event
type MMU_NOTIFY_UNMAP) and the interval needs to be split in order to
continue receiving callbacks for the remaining left and right intervals.
Add a new function mmu_interval_notifier_insert_safe() which can be called
from the invalidate() callback.
Signed-off-by: Ralph Campbell <[email protected]>
---
include/linux/mmu_notifier.h | 4 ++++
mm/mmu_notifier.c | 45 ++++++++++++++++++++++++++++++++----
2 files changed, 45 insertions(+), 4 deletions(-)
diff --git a/include/linux/mmu_notifier.h b/include/linux/mmu_notifier.h
index 9e6caa8ecd19..027c9c8f3a69 100644
--- a/include/linux/mmu_notifier.h
+++ b/include/linux/mmu_notifier.h
@@ -299,6 +299,10 @@ int mmu_interval_notifier_insert_locked(
struct mmu_interval_notifier *mni, struct mm_struct *mm,
unsigned long start, unsigned long length,
const struct mmu_interval_notifier_ops *ops);
+int mmu_interval_notifier_insert_safe(
+ struct mmu_interval_notifier *mni, struct mm_struct *mm,
+ unsigned long start, unsigned long length,
+ const struct mmu_interval_notifier_ops *ops);
void mmu_interval_notifier_remove(struct mmu_interval_notifier *mni);
/**
diff --git a/mm/mmu_notifier.c b/mm/mmu_notifier.c
index f76ea05b1cb0..a5ff19cd1bc5 100644
--- a/mm/mmu_notifier.c
+++ b/mm/mmu_notifier.c
@@ -913,16 +913,17 @@ static int __mmu_interval_notifier_insert(
/**
* mmu_interval_notifier_insert - Insert an interval notifier
* @mni: Interval notifier to register
+ * @mm: mm_struct to attach to
* @start: Starting virtual address to monitor
* @length: Length of the range to monitor
- * @mm : mm_struct to attach to
+ * @ops: Interval notifier callback operations
*
* This function subscribes the interval notifier for notifications from the
- * mm. Upon return the ops related to mmu_interval_notifier will be called
+ * mm. Upon return, the ops related to mmu_interval_notifier will be called
* whenever an event that intersects with the given range occurs.
*
- * Upon return the range_notifier may not be present in the interval tree yet.
- * The caller must use the normal interval notifier read flow via
+ * Upon return, the mmu_interval_notifier may not be present in the interval
+ * tree yet. The caller must use the normal interval notifier read flow via
* mmu_interval_read_begin() to establish SPTEs for this range.
*/
int mmu_interval_notifier_insert(struct mmu_interval_notifier *mni,
@@ -969,6 +970,42 @@ int mmu_interval_notifier_insert_locked(
}
EXPORT_SYMBOL_GPL(mmu_interval_notifier_insert_locked);
+/**
+ * mmu_interval_notifier_insert_safe - Insert an interval notifier
+ * @mni: Interval notifier to register
+ * @mm: mm_struct to attach to
+ * @start: Starting virtual address to monitor
+ * @length: Length of the range to monitor
+ * @ops: Interval notifier callback operations
+ *
+ * Return: -EINVAL if @mm hasn't been initialized for interval notifiers
+ * by calling mmu_notifier_register(NULL, mm) or
+ * __mmu_notifier_register(NULL, mm).
+ *
+ * This function subscribes the interval notifier for notifications from the
+ * mm. Upon return, the ops related to mmu_interval_notifier will be called
+ * whenever an event that intersects with the given range occurs.
+ *
+ * This function is safe to call from the ops->invalidate() function.
+ * Upon return, the mmu_interval_notifier may not be present in the interval
+ * tree yet. The caller must use the normal interval notifier read flow via
+ * mmu_interval_read_begin() to establish SPTEs for this range.
+ */
+int mmu_interval_notifier_insert_safe(
+ struct mmu_interval_notifier *mni, struct mm_struct *mm,
+ unsigned long start, unsigned long length,
+ const struct mmu_interval_notifier_ops *ops)
+{
+ struct mmu_notifier_mm *mmn_mm;
+
+ mmn_mm = mm->mmu_notifier_mm;
+ if (!mmn_mm || !mmn_mm->has_itree)
+ return -EINVAL;
+ return __mmu_interval_notifier_insert(mni, mm, mmn_mm, start, length,
+ ops);
+}
+EXPORT_SYMBOL_GPL(mmu_interval_notifier_insert_safe);
+
/**
* mmu_interval_notifier_remove - Remove a interval notifier
* @mni: Interval notifier to unregister
--
2.20.1
The name is weird, normally we call functions like this _locked.
Additionally it will need a lockdep_assert_held to ensure the caller
actually holds the lock.