Changes from v1[0]:
- Got rid of more code in patch 1 by using the end - 1 for closed
intervals, instead of keeping the overlap-check.
- added an additional cleanup patch.
Hi,
I'm sending this series again in this format as the interval tree
node conversion will, at a minimum, take longer than hoped for
(ie: Jason still removing interval tree users for the mmu_notifier
rework[1]). There is also a chance this will never see be done.
As such, I'm resending this series (where patch 1 is the only
interesting one and which Ingo acked previously, with the exception
that the nodes remain fully closed). In the future, it would be
trivial to port pat tree to semi open nodes, but for now think that
it makes sense to just get the pat changes in.
Please consider for v5.5. Thanks!
[0] https://lore.kernel.org/lkml/[email protected]/
[1] https://marc.info/?l=linux-mm&m=157116340411211
Davidlohr Bueso (4):
x86/mm, pat: Convert pat tree to generic interval tree
x86,mm/pat: Cleanup some of the local memtype_rb_* calls
x86,mm/pat: Drop rbt suffix from external memtype calls
x86/mm, pat: Rename pat_rbtree.c to pat_interval.c
arch/x86/mm/pat.c | 8 +-
arch/x86/mm/pat_internal.h | 20 ++--
arch/x86/mm/pat_interval.c | 186 +++++++++++++++++++++++++++++++
arch/x86/mm/pat_rbtree.c | 268 ---------------------------------------------
4 files changed, 200 insertions(+), 282 deletions(-)
create mode 100644 arch/x86/mm/pat_interval.c
delete mode 100644 arch/x86/mm/pat_rbtree.c
--
2.16.4
Drop the rbt_memtype_* calls (those used by pat.c) as we
no longer use an rbtree directly.
Signed-off-by: Davidlohr Bueso <[email protected]>
---
arch/x86/mm/pat.c | 8 ++++----
arch/x86/mm/pat_internal.h | 20 ++++++++++----------
arch/x86/mm/pat_rbtree.c | 12 ++++++------
3 files changed, 20 insertions(+), 20 deletions(-)
diff --git a/arch/x86/mm/pat.c b/arch/x86/mm/pat.c
index d9fbd4f69920..2d758e19ef22 100644
--- a/arch/x86/mm/pat.c
+++ b/arch/x86/mm/pat.c
@@ -603,7 +603,7 @@ int reserve_memtype(u64 start, u64 end, enum page_cache_mode req_type,
spin_lock(&memtype_lock);
- err = rbt_memtype_check_insert(new, new_type);
+ err = memtype_check_insert(new, new_type);
if (err) {
pr_info("x86/PAT: reserve_memtype failed [mem %#010Lx-%#010Lx], track %s, req %s\n",
start, end - 1,
@@ -650,7 +650,7 @@ int free_memtype(u64 start, u64 end)
}
spin_lock(&memtype_lock);
- entry = rbt_memtype_erase(start, end);
+ entry = memtype_erase(start, end);
spin_unlock(&memtype_lock);
if (IS_ERR(entry)) {
@@ -693,7 +693,7 @@ static enum page_cache_mode lookup_memtype(u64 paddr)
spin_lock(&memtype_lock);
- entry = rbt_memtype_lookup(paddr);
+ entry = memtype_lookup(paddr);
if (entry != NULL)
rettype = entry->type;
else
@@ -1109,7 +1109,7 @@ static struct memtype *memtype_get_idx(loff_t pos)
return NULL;
spin_lock(&memtype_lock);
- ret = rbt_memtype_copy_nth_element(print_entry, pos);
+ ret = memtype_copy_nth_element(print_entry, pos);
spin_unlock(&memtype_lock);
if (!ret) {
diff --git a/arch/x86/mm/pat_internal.h b/arch/x86/mm/pat_internal.h
index eeb5caeb089b..79a06684349e 100644
--- a/arch/x86/mm/pat_internal.h
+++ b/arch/x86/mm/pat_internal.h
@@ -29,20 +29,20 @@ static inline char *cattr_name(enum page_cache_mode pcm)
}
#ifdef CONFIG_X86_PAT
-extern int rbt_memtype_check_insert(struct memtype *new,
- enum page_cache_mode *new_type);
-extern struct memtype *rbt_memtype_erase(u64 start, u64 end);
-extern struct memtype *rbt_memtype_lookup(u64 addr);
-extern int rbt_memtype_copy_nth_element(struct memtype *out, loff_t pos);
+extern int memtype_check_insert(struct memtype *new,
+ enum page_cache_mode *new_type);
+extern struct memtype *memtype_erase(u64 start, u64 end);
+extern struct memtype *memtype_lookup(u64 addr);
+extern int memtype_copy_nth_element(struct memtype *out, loff_t pos);
#else
-static inline int rbt_memtype_check_insert(struct memtype *new,
- enum page_cache_mode *new_type)
+static inline int memtype_check_insert(struct memtype *new,
+ enum page_cache_mode *new_type)
{ return 0; }
-static inline struct memtype *rbt_memtype_erase(u64 start, u64 end)
+static inline struct memtype *memtype_erase(u64 start, u64 end)
{ return NULL; }
-static inline struct memtype *rbt_memtype_lookup(u64 addr)
+static inline struct memtype *memtype_lookup(u64 addr)
{ return NULL; }
-static inline int rbt_memtype_copy_nth_element(struct memtype *out, loff_t pos)
+static inline int memtype_copy_nth_element(struct memtype *out, loff_t pos)
{ return 0; }
#endif
diff --git a/arch/x86/mm/pat_rbtree.c b/arch/x86/mm/pat_rbtree.c
index 7974136ea1f6..ef59e0ab00ed 100644
--- a/arch/x86/mm/pat_rbtree.c
+++ b/arch/x86/mm/pat_rbtree.c
@@ -109,8 +109,8 @@ static int memtype_check_conflict(u64 start, u64 end,
return -EBUSY;
}
-int rbt_memtype_check_insert(struct memtype *new,
- enum page_cache_mode *ret_type)
+int memtype_check_insert(struct memtype *new,
+ enum page_cache_mode *ret_type)
{
int err = 0;
@@ -126,13 +126,13 @@ int rbt_memtype_check_insert(struct memtype *new,
return err;
}
-struct memtype *rbt_memtype_erase(u64 start, u64 end)
+struct memtype *memtype_erase(u64 start, u64 end)
{
struct memtype *data;
/*
* Since the memtype_rbroot tree allows overlapping ranges,
- * rbt_memtype_erase() checks with EXACT_MATCH first, i.e. free
+ * memtype_erase() checks with EXACT_MATCH first, i.e. free
* a whole node for the munmap case. If no such entry is found,
* it then checks with END_MATCH, i.e. shrink the size of a node
* from the end for the mremap case.
@@ -158,14 +158,14 @@ struct memtype *rbt_memtype_erase(u64 start, u64 end)
return data;
}
-struct memtype *rbt_memtype_lookup(u64 addr)
+struct memtype *memtype_lookup(u64 addr)
{
return memtype_interval_iter_first(&memtype_rbroot, addr,
addr + PAGE_SIZE);
}
#if defined(CONFIG_DEBUG_FS)
-int rbt_memtype_copy_nth_element(struct memtype *out, loff_t pos)
+int memtype_copy_nth_element(struct memtype *out, loff_t pos)
{
struct memtype *match;
int i = 1;
--
2.16.4
With some considerations, the custom pat_rbtree implementation can be
simplified to use most of the generic interval_tree machinery.
o The tree inorder traversal can slightly differ when there are key
('start') collisions in the tree due to one going left and another right.
This, however, only affects the output of debugfs' pat_memtype_list file.
o Generic interval trees are now fully closed [a, b], for which we need
to adjust the last endpoint (ie: end - 1).
o Erasing logic must remain untouched as well.
In order for the types to remain u64, the 'memtype_interval' calls are
introduced, as opposed to simply using struct interval_tree.
In addition, pat tree might potentially also benefit by the fast overlap
detection for the insertion case when looking up the first overlapping node
in the tree.
Finally, I've tested this on various servers, via sanity warnings, running
side by side with the current version and so far see no differences in the
returned pointer node when doing memtype_rb_lowest_match() lookups.
Signed-off-by: Davidlohr Bueso <[email protected]>
---
arch/x86/mm/pat_rbtree.c | 164 +++++++++++++----------------------------------
1 file changed, 43 insertions(+), 121 deletions(-)
diff --git a/arch/x86/mm/pat_rbtree.c b/arch/x86/mm/pat_rbtree.c
index 65ebe4b88f7c..4998d690d927 100644
--- a/arch/x86/mm/pat_rbtree.c
+++ b/arch/x86/mm/pat_rbtree.c
@@ -5,14 +5,13 @@
* Authors: Venkatesh Pallipadi <[email protected]>
* Suresh B Siddha <[email protected]>
*
- * Interval tree (augmented rbtree) used to store the PAT memory type
- * reservations.
+ * Interval tree used to store the PAT memory type reservations.
*/
#include <linux/seq_file.h>
#include <linux/debugfs.h>
#include <linux/kernel.h>
-#include <linux/rbtree_augmented.h>
+#include <linux/interval_tree_generic.h>
#include <linux/sched.h>
#include <linux/gfp.h>
@@ -33,72 +32,33 @@
*
* memtype_lock protects the rbtree.
*/
-
-static struct rb_root memtype_rbroot = RB_ROOT;
-
-static int is_node_overlap(struct memtype *node, u64 start, u64 end)
+static inline u64 memtype_interval_start(struct memtype *memtype)
{
- if (node->start >= end || node->end <= start)
- return 0;
-
- return 1;
+ return memtype->start;
}
-static u64 get_subtree_max_end(struct rb_node *node)
+static inline u64 memtype_interval_end(struct memtype *memtype)
{
- u64 ret = 0;
- if (node) {
- struct memtype *data = rb_entry(node, struct memtype, rb);
- ret = data->subtree_max_end;
- }
- return ret;
+ return memtype->end - 1;
}
+INTERVAL_TREE_DEFINE(struct memtype, rb, u64, subtree_max_end,
+ memtype_interval_start, memtype_interval_end,
+ static, memtype_interval)
-#define NODE_END(node) ((node)->end)
-
-RB_DECLARE_CALLBACKS_MAX(static, memtype_rb_augment_cb,
- struct memtype, rb, u64, subtree_max_end, NODE_END)
-
-/* Find the first (lowest start addr) overlapping range from rb tree */
-static struct memtype *memtype_rb_lowest_match(struct rb_root *root,
- u64 start, u64 end)
-{
- struct rb_node *node = root->rb_node;
- struct memtype *last_lower = NULL;
-
- while (node) {
- struct memtype *data = rb_entry(node, struct memtype, rb);
-
- if (get_subtree_max_end(node->rb_left) > start) {
- /* Lowest overlap if any must be on left side */
- node = node->rb_left;
- } else if (is_node_overlap(data, start, end)) {
- last_lower = data;
- break;
- } else if (start >= data->start) {
- /* Lowest overlap if any must be on right side */
- node = node->rb_right;
- } else {
- break;
- }
- }
- return last_lower; /* Returns NULL if there is no overlap */
-}
+static struct rb_root_cached memtype_rbroot = RB_ROOT_CACHED;
enum {
MEMTYPE_EXACT_MATCH = 0,
MEMTYPE_END_MATCH = 1
};
-static struct memtype *memtype_rb_match(struct rb_root *root,
- u64 start, u64 end, int match_type)
+static struct memtype *memtype_match(struct rb_root_cached *root,
+ u64 start, u64 end, int match_type)
{
struct memtype *match;
- match = memtype_rb_lowest_match(root, start, end);
+ match = memtype_interval_iter_first(root, start, end);
while (match != NULL && match->start < end) {
- struct rb_node *node;
-
if ((match_type == MEMTYPE_EXACT_MATCH) &&
(match->start == start) && (match->end == end))
return match;
@@ -107,26 +67,21 @@ static struct memtype *memtype_rb_match(struct rb_root *root,
(match->start < start) && (match->end == end))
return match;
- node = rb_next(&match->rb);
- if (node)
- match = rb_entry(node, struct memtype, rb);
- else
- match = NULL;
+ match = memtype_interval_iter_next(match, start, end);
}
return NULL; /* Returns NULL if there is no match */
}
-static int memtype_rb_check_conflict(struct rb_root *root,
+static int memtype_rb_check_conflict(struct rb_root_cached *root,
u64 start, u64 end,
enum page_cache_mode reqtype,
enum page_cache_mode *newtype)
{
- struct rb_node *node;
struct memtype *match;
enum page_cache_mode found_type = reqtype;
- match = memtype_rb_lowest_match(&memtype_rbroot, start, end);
+ match = memtype_interval_iter_first(&memtype_rbroot, start, end);
if (match == NULL)
goto success;
@@ -136,19 +91,12 @@ static int memtype_rb_check_conflict(struct rb_root *root,
dprintk("Overlap at 0x%Lx-0x%Lx\n", match->start, match->end);
found_type = match->type;
- node = rb_next(&match->rb);
- while (node) {
- match = rb_entry(node, struct memtype, rb);
-
- if (match->start >= end) /* Checked all possible matches */
- goto success;
-
- if (is_node_overlap(match, start, end) &&
- match->type != found_type) {
+ match = memtype_interval_iter_next(match, start, end);
+ while (match) {
+ if (match->type != found_type)
goto failure;
- }
- node = rb_next(&match->rb);
+ match = memtype_interval_iter_next(match, start, end);
}
success:
if (newtype)
@@ -163,43 +111,20 @@ static int memtype_rb_check_conflict(struct rb_root *root,
return -EBUSY;
}
-static void memtype_rb_insert(struct rb_root *root, struct memtype *newdata)
-{
- struct rb_node **node = &(root->rb_node);
- struct rb_node *parent = NULL;
-
- while (*node) {
- struct memtype *data = rb_entry(*node, struct memtype, rb);
-
- parent = *node;
- if (data->subtree_max_end < newdata->end)
- data->subtree_max_end = newdata->end;
- if (newdata->start <= data->start)
- node = &((*node)->rb_left);
- else if (newdata->start > data->start)
- node = &((*node)->rb_right);
- }
-
- newdata->subtree_max_end = newdata->end;
- rb_link_node(&newdata->rb, parent, node);
- rb_insert_augmented(&newdata->rb, root, &memtype_rb_augment_cb);
-}
-
int rbt_memtype_check_insert(struct memtype *new,
enum page_cache_mode *ret_type)
{
int err = 0;
err = memtype_rb_check_conflict(&memtype_rbroot, new->start, new->end,
- new->type, ret_type);
-
- if (!err) {
- if (ret_type)
- new->type = *ret_type;
-
- new->subtree_max_end = new->end;
- memtype_rb_insert(&memtype_rbroot, new);
- }
+ new->type, ret_type);
+ if (err)
+ goto done;
+
+ if (ret_type)
+ new->type = *ret_type;
+ memtype_interval_insert(new, &memtype_rbroot);
+done:
return err;
}
@@ -214,26 +139,23 @@ struct memtype *rbt_memtype_erase(u64 start, u64 end)
* it then checks with END_MATCH, i.e. shrink the size of a node
* from the end for the mremap case.
*/
- data = memtype_rb_match(&memtype_rbroot, start, end,
- MEMTYPE_EXACT_MATCH);
+ data = memtype_match(&memtype_rbroot, start, end,
+ MEMTYPE_EXACT_MATCH);
if (!data) {
- data = memtype_rb_match(&memtype_rbroot, start, end,
- MEMTYPE_END_MATCH);
+ data = memtype_match(&memtype_rbroot, start, end,
+ MEMTYPE_END_MATCH);
if (!data)
return ERR_PTR(-EINVAL);
}
if (data->start == start) {
/* munmap: erase this node */
- rb_erase_augmented(&data->rb, &memtype_rbroot,
- &memtype_rb_augment_cb);
+ memtype_interval_remove(data, &memtype_rbroot);
} else {
/* mremap: update the end value of this node */
- rb_erase_augmented(&data->rb, &memtype_rbroot,
- &memtype_rb_augment_cb);
+ memtype_interval_remove(data, &memtype_rbroot);
data->end = start;
- data->subtree_max_end = data->end;
- memtype_rb_insert(&memtype_rbroot, data);
+ memtype_interval_insert(data, &memtype_rbroot);
return NULL;
}
@@ -242,24 +164,24 @@ struct memtype *rbt_memtype_erase(u64 start, u64 end)
struct memtype *rbt_memtype_lookup(u64 addr)
{
- return memtype_rb_lowest_match(&memtype_rbroot, addr, addr + PAGE_SIZE);
+ return memtype_interval_iter_first(&memtype_rbroot, addr,
+ addr + PAGE_SIZE);
}
#if defined(CONFIG_DEBUG_FS)
int rbt_memtype_copy_nth_element(struct memtype *out, loff_t pos)
{
- struct rb_node *node;
+ struct memtype *match;
int i = 1;
- node = rb_first(&memtype_rbroot);
- while (node && pos != i) {
- node = rb_next(node);
+ match = memtype_interval_iter_first(&memtype_rbroot, 0, ULONG_MAX);
+ while (match && pos != i) {
+ match = memtype_interval_iter_next(match, 0, ULONG_MAX);
i++;
}
- if (node) { /* pos == i */
- struct memtype *this = rb_entry(node, struct memtype, rb);
- *out = *this;
+ if (match) { /* pos == i */
+ *out = *match;
return 0;
} else {
return 1;
--
2.16.4
Cleanup by both getting rid of passing the rb_root down the helper
calls; there is only one. Secondly rename some of the calls from
memtype_rb_*.
Signed-off-by: Davidlohr Bueso <[email protected]>
---
arch/x86/mm/pat_rbtree.c | 20 ++++++++------------
1 file changed, 8 insertions(+), 12 deletions(-)
diff --git a/arch/x86/mm/pat_rbtree.c b/arch/x86/mm/pat_rbtree.c
index 4998d690d927..7974136ea1f6 100644
--- a/arch/x86/mm/pat_rbtree.c
+++ b/arch/x86/mm/pat_rbtree.c
@@ -52,12 +52,11 @@ enum {
MEMTYPE_END_MATCH = 1
};
-static struct memtype *memtype_match(struct rb_root_cached *root,
- u64 start, u64 end, int match_type)
+static struct memtype *memtype_match(u64 start, u64 end, int match_type)
{
struct memtype *match;
- match = memtype_interval_iter_first(root, start, end);
+ match = memtype_interval_iter_first(&memtype_rbroot, start, end);
while (match != NULL && match->start < end) {
if ((match_type == MEMTYPE_EXACT_MATCH) &&
(match->start == start) && (match->end == end))
@@ -73,10 +72,9 @@ static struct memtype *memtype_match(struct rb_root_cached *root,
return NULL; /* Returns NULL if there is no match */
}
-static int memtype_rb_check_conflict(struct rb_root_cached *root,
- u64 start, u64 end,
- enum page_cache_mode reqtype,
- enum page_cache_mode *newtype)
+static int memtype_check_conflict(u64 start, u64 end,
+ enum page_cache_mode reqtype,
+ enum page_cache_mode *newtype)
{
struct memtype *match;
enum page_cache_mode found_type = reqtype;
@@ -116,7 +114,7 @@ int rbt_memtype_check_insert(struct memtype *new,
{
int err = 0;
- err = memtype_rb_check_conflict(&memtype_rbroot, new->start, new->end,
+ err = memtype_check_conflict(new->start, new->end,
new->type, ret_type);
if (err)
goto done;
@@ -139,11 +137,9 @@ struct memtype *rbt_memtype_erase(u64 start, u64 end)
* it then checks with END_MATCH, i.e. shrink the size of a node
* from the end for the mremap case.
*/
- data = memtype_match(&memtype_rbroot, start, end,
- MEMTYPE_EXACT_MATCH);
+ data = memtype_match(start, end, MEMTYPE_EXACT_MATCH);
if (!data) {
- data = memtype_match(&memtype_rbroot, start, end,
- MEMTYPE_END_MATCH);
+ data = memtype_match(start, end, MEMTYPE_END_MATCH);
if (!data)
return ERR_PTR(-EINVAL);
}
--
2.16.4
Considering the previous changes, this is a more proper name.
Signed-off-by: Davidlohr Bueso <[email protected]>
---
arch/x86/mm/{pat_rbtree.c => pat_interval.c} | 0
1 file changed, 0 insertions(+), 0 deletions(-)
rename arch/x86/mm/{pat_rbtree.c => pat_interval.c} (100%)
diff --git a/arch/x86/mm/pat_rbtree.c b/arch/x86/mm/pat_interval.c
similarity index 100%
rename from arch/x86/mm/pat_rbtree.c
rename to arch/x86/mm/pat_interval.c
--
2.16.4
ping?
With another week of rc, can this still make it for v5.5?
Thanks,
Davidlohr
On Mon, 21 Oct 2019, Davidlohr Bueso wrote:
>Changes from v1[0]:
> - Got rid of more code in patch 1 by using the end - 1 for closed
> intervals, instead of keeping the overlap-check.
>
> - added an additional cleanup patch.
>
>Hi,
>
>I'm sending this series again in this format as the interval tree
>node conversion will, at a minimum, take longer than hoped for
>(ie: Jason still removing interval tree users for the mmu_notifier
>rework[1]). There is also a chance this will never see be done.
>
>As such, I'm resending this series (where patch 1 is the only
>interesting one and which Ingo acked previously, with the exception
>that the nodes remain fully closed). In the future, it would be
>trivial to port pat tree to semi open nodes, but for now think that
>it makes sense to just get the pat changes in.
>
>Please consider for v5.5. Thanks!
>
>[0] https://lore.kernel.org/lkml/[email protected]/
>[1] https://marc.info/?l=linux-mm&m=157116340411211
>
>Davidlohr Bueso (4):
> x86/mm, pat: Convert pat tree to generic interval tree
> x86,mm/pat: Cleanup some of the local memtype_rb_* calls
> x86,mm/pat: Drop rbt suffix from external memtype calls
> x86/mm, pat: Rename pat_rbtree.c to pat_interval.c
>
> arch/x86/mm/pat.c | 8 +-
> arch/x86/mm/pat_internal.h | 20 ++--
> arch/x86/mm/pat_interval.c | 186 +++++++++++++++++++++++++++++++
> arch/x86/mm/pat_rbtree.c | 268 ---------------------------------------------
> 4 files changed, 200 insertions(+), 282 deletions(-)
> create mode 100644 arch/x86/mm/pat_interval.c
> delete mode 100644 arch/x86/mm/pat_rbtree.c
>
>--
>2.16.4
>
* Davidlohr Bueso <[email protected]> wrote:
> Considering the previous changes, this is a more proper name.
>
> Signed-off-by: Davidlohr Bueso <[email protected]>
> ---
> arch/x86/mm/{pat_rbtree.c => pat_interval.c} | 0
> 1 file changed, 0 insertions(+), 0 deletions(-)
> rename arch/x86/mm/{pat_rbtree.c => pat_interval.c} (100%)
>
> diff --git a/arch/x86/mm/pat_rbtree.c b/arch/x86/mm/pat_interval.c
> similarity index 100%
> rename from arch/x86/mm/pat_rbtree.c
> rename to arch/x86/mm/pat_interval.c
For some reason this patch was missing the kbuild adjustment for the
rename of the .c file:
--- a/arch/x86/mm/Makefile
+++ b/arch/x86/mm/Makefile
@@ -23,7 +23,7 @@ CFLAGS_mem_encrypt_identity.o := $(nostackp)
CFLAGS_fault.o := -I $(srctree)/$(src)/../include/asm/trace
-obj-$(CONFIG_X86_PAT) += pat_rbtree.o
+obj-$(CONFIG_X86_PAT) += pat_interval.o
obj-$(CONFIG_X86_32) += pgtable_32.o iomap_32.o
Thanks,
Ingo
On Tue, 19 Nov 2019, Ingo Molnar wrote:
>
>* Davidlohr Bueso <[email protected]> wrote:
>
>> Considering the previous changes, this is a more proper name.
>>
>> Signed-off-by: Davidlohr Bueso <[email protected]>
>> ---
>> arch/x86/mm/{pat_rbtree.c => pat_interval.c} | 0
>> 1 file changed, 0 insertions(+), 0 deletions(-)
>> rename arch/x86/mm/{pat_rbtree.c => pat_interval.c} (100%)
>>
>> diff --git a/arch/x86/mm/pat_rbtree.c b/arch/x86/mm/pat_interval.c
>> similarity index 100%
>> rename from arch/x86/mm/pat_rbtree.c
>> rename to arch/x86/mm/pat_interval.c
>
>For some reason this patch was missing the kbuild adjustment for the
>rename of the .c file:
Sorry about that, I don't know how I missed this - the series was
certainly tested. Per the below I assume you don't need a v2 (and
avoid more spam).
Thanks,
Davidlohr
On Mon, 21 Oct 2019, Davidlohr Bueso wrote:
> int rbt_memtype_check_insert(struct memtype *new,
> enum page_cache_mode *ret_type)
> {
> int err = 0;
>
> err = memtype_rb_check_conflict(&memtype_rbroot, new->start, new->end,
> - new->type, ret_type);
> -
> - if (!err) {
> - if (ret_type)
> - new->type = *ret_type;
> -
> - new->subtree_max_end = new->end;
> - memtype_rb_insert(&memtype_rbroot, new);
> - }
> + new->type, ret_type);
> + if (err)
> + goto done;
Please return err here. That goto is pointless.
> +
> + if (ret_type)
> + new->type = *ret_type;
> + memtype_interval_insert(new, &memtype_rbroot);
> +done:
> return err;
> }
Other than that.
Reviewed-by: Thomas Gleixner <[email protected]>
On Mon, 21 Oct 2019, Davidlohr Bueso wrote:
> Cleanup by both getting rid of passing the rb_root down the helper
> calls; there is only one. Secondly rename some of the calls from
> memtype_rb_*.
>
> Signed-off-by: Davidlohr Bueso <[email protected]>
Reviewed-by: Thomas Gleixner <[email protected]>
On Mon, 21 Oct 2019, Davidlohr Bueso wrote:
> Drop the rbt_memtype_* calls (those used by pat.c) as we
> no longer use an rbtree directly.
Reviewed-by: Thomas Gleixner <[email protected]>
The following commit has been merged into the x86/mm branch of tip:
Commit-ID: 3309be371c20275c346fe482767e2d11e29d732e
Gitweb: https://git.kernel.org/tip/3309be371c20275c346fe482767e2d11e29d732e
Author: Davidlohr Bueso <[email protected]>
AuthorDate: Mon, 21 Oct 2019 16:19:22 -07:00
Committer: Ingo Molnar <[email protected]>
CommitterDate: Tue, 19 Nov 2019 09:08:42 +01:00
x86/mm/pat: Clean up some of the local memtype_rb_*() calls
Clean up by both getting rid of passing the rb_root down the helper
calls; there is only one. Secondly rename some of the calls still
using the now inaccurate memtype_rb_*() namespace.
Signed-off-by: Davidlohr Bueso <[email protected]>
Cc: Andy Lutomirski <[email protected]>
Cc: Borislav Petkov <[email protected]>
Cc: Dave Hansen <[email protected]>
Cc: H. Peter Anvin <[email protected]>
Cc: Linus Torvalds <[email protected]>
Cc: Peter Zijlstra <[email protected]>
Cc: Rik van Riel <[email protected]>
Cc: Thomas Gleixner <[email protected]>
Cc: [email protected]
Link: https://lkml.kernel.org/r/[email protected]
Signed-off-by: Ingo Molnar <[email protected]>
---
arch/x86/mm/pat_rbtree.c | 20 ++++++++------------
1 file changed, 8 insertions(+), 12 deletions(-)
diff --git a/arch/x86/mm/pat_rbtree.c b/arch/x86/mm/pat_rbtree.c
index 4998d69..7974136 100644
--- a/arch/x86/mm/pat_rbtree.c
+++ b/arch/x86/mm/pat_rbtree.c
@@ -52,12 +52,11 @@ enum {
MEMTYPE_END_MATCH = 1
};
-static struct memtype *memtype_match(struct rb_root_cached *root,
- u64 start, u64 end, int match_type)
+static struct memtype *memtype_match(u64 start, u64 end, int match_type)
{
struct memtype *match;
- match = memtype_interval_iter_first(root, start, end);
+ match = memtype_interval_iter_first(&memtype_rbroot, start, end);
while (match != NULL && match->start < end) {
if ((match_type == MEMTYPE_EXACT_MATCH) &&
(match->start == start) && (match->end == end))
@@ -73,10 +72,9 @@ static struct memtype *memtype_match(struct rb_root_cached *root,
return NULL; /* Returns NULL if there is no match */
}
-static int memtype_rb_check_conflict(struct rb_root_cached *root,
- u64 start, u64 end,
- enum page_cache_mode reqtype,
- enum page_cache_mode *newtype)
+static int memtype_check_conflict(u64 start, u64 end,
+ enum page_cache_mode reqtype,
+ enum page_cache_mode *newtype)
{
struct memtype *match;
enum page_cache_mode found_type = reqtype;
@@ -116,7 +114,7 @@ int rbt_memtype_check_insert(struct memtype *new,
{
int err = 0;
- err = memtype_rb_check_conflict(&memtype_rbroot, new->start, new->end,
+ err = memtype_check_conflict(new->start, new->end,
new->type, ret_type);
if (err)
goto done;
@@ -139,11 +137,9 @@ struct memtype *rbt_memtype_erase(u64 start, u64 end)
* it then checks with END_MATCH, i.e. shrink the size of a node
* from the end for the mremap case.
*/
- data = memtype_match(&memtype_rbroot, start, end,
- MEMTYPE_EXACT_MATCH);
+ data = memtype_match(start, end, MEMTYPE_EXACT_MATCH);
if (!data) {
- data = memtype_match(&memtype_rbroot, start, end,
- MEMTYPE_END_MATCH);
+ data = memtype_match(start, end, MEMTYPE_END_MATCH);
if (!data)
return ERR_PTR(-EINVAL);
}
The following commit has been merged into the x86/mm branch of tip:
Commit-ID: b40805c214c5b00151e168796dca5a4ea3b2882a
Gitweb: https://git.kernel.org/tip/b40805c214c5b00151e168796dca5a4ea3b2882a
Author: Davidlohr Bueso <[email protected]>
AuthorDate: Mon, 21 Oct 2019 16:19:23 -07:00
Committer: Ingo Molnar <[email protected]>
CommitterDate: Tue, 19 Nov 2019 09:08:43 +01:00
x86/mm/pat: Drop the rbt_ prefix from external memtype function names
Rename:
rbt_memtype_* => memtype_*()
... as we no longer use an rbtree directly.
Signed-off-by: Davidlohr Bueso <[email protected]>
Cc: Andy Lutomirski <[email protected]>
Cc: Borislav Petkov <[email protected]>
Cc: Dave Hansen <[email protected]>
Cc: H. Peter Anvin <[email protected]>
Cc: Linus Torvalds <[email protected]>
Cc: Peter Zijlstra <[email protected]>
Cc: Rik van Riel <[email protected]>
Cc: Thomas Gleixner <[email protected]>
Cc: [email protected]
Link: https://lkml.kernel.org/r/[email protected]
Signed-off-by: Ingo Molnar <[email protected]>
---
arch/x86/mm/pat.c | 8 ++++----
arch/x86/mm/pat_internal.h | 20 ++++++++++----------
arch/x86/mm/pat_rbtree.c | 12 ++++++------
3 files changed, 20 insertions(+), 20 deletions(-)
diff --git a/arch/x86/mm/pat.c b/arch/x86/mm/pat.c
index d9fbd4f..2d758e1 100644
--- a/arch/x86/mm/pat.c
+++ b/arch/x86/mm/pat.c
@@ -603,7 +603,7 @@ int reserve_memtype(u64 start, u64 end, enum page_cache_mode req_type,
spin_lock(&memtype_lock);
- err = rbt_memtype_check_insert(new, new_type);
+ err = memtype_check_insert(new, new_type);
if (err) {
pr_info("x86/PAT: reserve_memtype failed [mem %#010Lx-%#010Lx], track %s, req %s\n",
start, end - 1,
@@ -650,7 +650,7 @@ int free_memtype(u64 start, u64 end)
}
spin_lock(&memtype_lock);
- entry = rbt_memtype_erase(start, end);
+ entry = memtype_erase(start, end);
spin_unlock(&memtype_lock);
if (IS_ERR(entry)) {
@@ -693,7 +693,7 @@ static enum page_cache_mode lookup_memtype(u64 paddr)
spin_lock(&memtype_lock);
- entry = rbt_memtype_lookup(paddr);
+ entry = memtype_lookup(paddr);
if (entry != NULL)
rettype = entry->type;
else
@@ -1109,7 +1109,7 @@ static struct memtype *memtype_get_idx(loff_t pos)
return NULL;
spin_lock(&memtype_lock);
- ret = rbt_memtype_copy_nth_element(print_entry, pos);
+ ret = memtype_copy_nth_element(print_entry, pos);
spin_unlock(&memtype_lock);
if (!ret) {
diff --git a/arch/x86/mm/pat_internal.h b/arch/x86/mm/pat_internal.h
index eeb5cae..79a0668 100644
--- a/arch/x86/mm/pat_internal.h
+++ b/arch/x86/mm/pat_internal.h
@@ -29,20 +29,20 @@ static inline char *cattr_name(enum page_cache_mode pcm)
}
#ifdef CONFIG_X86_PAT
-extern int rbt_memtype_check_insert(struct memtype *new,
- enum page_cache_mode *new_type);
-extern struct memtype *rbt_memtype_erase(u64 start, u64 end);
-extern struct memtype *rbt_memtype_lookup(u64 addr);
-extern int rbt_memtype_copy_nth_element(struct memtype *out, loff_t pos);
+extern int memtype_check_insert(struct memtype *new,
+ enum page_cache_mode *new_type);
+extern struct memtype *memtype_erase(u64 start, u64 end);
+extern struct memtype *memtype_lookup(u64 addr);
+extern int memtype_copy_nth_element(struct memtype *out, loff_t pos);
#else
-static inline int rbt_memtype_check_insert(struct memtype *new,
- enum page_cache_mode *new_type)
+static inline int memtype_check_insert(struct memtype *new,
+ enum page_cache_mode *new_type)
{ return 0; }
-static inline struct memtype *rbt_memtype_erase(u64 start, u64 end)
+static inline struct memtype *memtype_erase(u64 start, u64 end)
{ return NULL; }
-static inline struct memtype *rbt_memtype_lookup(u64 addr)
+static inline struct memtype *memtype_lookup(u64 addr)
{ return NULL; }
-static inline int rbt_memtype_copy_nth_element(struct memtype *out, loff_t pos)
+static inline int memtype_copy_nth_element(struct memtype *out, loff_t pos)
{ return 0; }
#endif
diff --git a/arch/x86/mm/pat_rbtree.c b/arch/x86/mm/pat_rbtree.c
index 7974136..ef59e0a 100644
--- a/arch/x86/mm/pat_rbtree.c
+++ b/arch/x86/mm/pat_rbtree.c
@@ -109,8 +109,8 @@ failure:
return -EBUSY;
}
-int rbt_memtype_check_insert(struct memtype *new,
- enum page_cache_mode *ret_type)
+int memtype_check_insert(struct memtype *new,
+ enum page_cache_mode *ret_type)
{
int err = 0;
@@ -126,13 +126,13 @@ done:
return err;
}
-struct memtype *rbt_memtype_erase(u64 start, u64 end)
+struct memtype *memtype_erase(u64 start, u64 end)
{
struct memtype *data;
/*
* Since the memtype_rbroot tree allows overlapping ranges,
- * rbt_memtype_erase() checks with EXACT_MATCH first, i.e. free
+ * memtype_erase() checks with EXACT_MATCH first, i.e. free
* a whole node for the munmap case. If no such entry is found,
* it then checks with END_MATCH, i.e. shrink the size of a node
* from the end for the mremap case.
@@ -158,14 +158,14 @@ struct memtype *rbt_memtype_erase(u64 start, u64 end)
return data;
}
-struct memtype *rbt_memtype_lookup(u64 addr)
+struct memtype *memtype_lookup(u64 addr)
{
return memtype_interval_iter_first(&memtype_rbroot, addr,
addr + PAGE_SIZE);
}
#if defined(CONFIG_DEBUG_FS)
-int rbt_memtype_copy_nth_element(struct memtype *out, loff_t pos)
+int memtype_copy_nth_element(struct memtype *out, loff_t pos)
{
struct memtype *match;
int i = 1;
The following commit has been merged into the x86/mm branch of tip:
Commit-ID: 218bf1a8c73b9dcc2f85df9919578050359aa6ef
Gitweb: https://git.kernel.org/tip/218bf1a8c73b9dcc2f85df9919578050359aa6ef
Author: Davidlohr Bueso <[email protected]>
AuthorDate: Mon, 21 Oct 2019 16:19:21 -07:00
Committer: Ingo Molnar <[email protected]>
CommitterDate: Tue, 19 Nov 2019 09:08:42 +01:00
x86/mm/pat: Convert the PAT tree to a generic interval tree
With some considerations, the custom pat_rbtree implementation can be
simplified to use most of the generic interval_tree machinery:
- The tree inorder traversal can slightly differ when there are key
('start') collisions in the tree due to one going left and another right.
This, however, only affects the output of debugfs' pat_memtype_list file.
- Generic interval trees are now fully closed [a, b], for which we need
to adjust the last endpoint (ie: end - 1).
- Erasing logic must remain untouched as well.
In order for the types to remain u64, 'memtype_interval' calls are
introduced, as opposed to simply using struct interval_tree.
In addition, the PAT tree might potentially also benefit by the fast overlap
detection for the insertion case when looking up the first overlapping node
in the tree.
Finally, I've tested this on various servers, via sanity warnings, running
side by side with the current version and so far see no differences in the
returned pointer node when doing memtype_rb_lowest_match() lookups.
No change in functionality intended.
Signed-off-by: Davidlohr Bueso <[email protected]>
Cc: Andy Lutomirski <[email protected]>
Cc: Borislav Petkov <[email protected]>
Cc: Dave Hansen <[email protected]>
Cc: H. Peter Anvin <[email protected]>
Cc: Linus Torvalds <[email protected]>
Cc: Peter Zijlstra <[email protected]>
Cc: Rik van Riel <[email protected]>
Cc: Thomas Gleixner <[email protected]>
Cc: [email protected]
Link: https://lkml.kernel.org/r/[email protected]
Signed-off-by: Ingo Molnar <[email protected]>
---
arch/x86/mm/pat_rbtree.c | 164 +++++++++-----------------------------
1 file changed, 43 insertions(+), 121 deletions(-)
diff --git a/arch/x86/mm/pat_rbtree.c b/arch/x86/mm/pat_rbtree.c
index 65ebe4b..4998d69 100644
--- a/arch/x86/mm/pat_rbtree.c
+++ b/arch/x86/mm/pat_rbtree.c
@@ -5,14 +5,13 @@
* Authors: Venkatesh Pallipadi <[email protected]>
* Suresh B Siddha <[email protected]>
*
- * Interval tree (augmented rbtree) used to store the PAT memory type
- * reservations.
+ * Interval tree used to store the PAT memory type reservations.
*/
#include <linux/seq_file.h>
#include <linux/debugfs.h>
#include <linux/kernel.h>
-#include <linux/rbtree_augmented.h>
+#include <linux/interval_tree_generic.h>
#include <linux/sched.h>
#include <linux/gfp.h>
@@ -33,72 +32,33 @@
*
* memtype_lock protects the rbtree.
*/
-
-static struct rb_root memtype_rbroot = RB_ROOT;
-
-static int is_node_overlap(struct memtype *node, u64 start, u64 end)
+static inline u64 memtype_interval_start(struct memtype *memtype)
{
- if (node->start >= end || node->end <= start)
- return 0;
-
- return 1;
+ return memtype->start;
}
-static u64 get_subtree_max_end(struct rb_node *node)
+static inline u64 memtype_interval_end(struct memtype *memtype)
{
- u64 ret = 0;
- if (node) {
- struct memtype *data = rb_entry(node, struct memtype, rb);
- ret = data->subtree_max_end;
- }
- return ret;
+ return memtype->end - 1;
}
+INTERVAL_TREE_DEFINE(struct memtype, rb, u64, subtree_max_end,
+ memtype_interval_start, memtype_interval_end,
+ static, memtype_interval)
-#define NODE_END(node) ((node)->end)
-
-RB_DECLARE_CALLBACKS_MAX(static, memtype_rb_augment_cb,
- struct memtype, rb, u64, subtree_max_end, NODE_END)
-
-/* Find the first (lowest start addr) overlapping range from rb tree */
-static struct memtype *memtype_rb_lowest_match(struct rb_root *root,
- u64 start, u64 end)
-{
- struct rb_node *node = root->rb_node;
- struct memtype *last_lower = NULL;
-
- while (node) {
- struct memtype *data = rb_entry(node, struct memtype, rb);
-
- if (get_subtree_max_end(node->rb_left) > start) {
- /* Lowest overlap if any must be on left side */
- node = node->rb_left;
- } else if (is_node_overlap(data, start, end)) {
- last_lower = data;
- break;
- } else if (start >= data->start) {
- /* Lowest overlap if any must be on right side */
- node = node->rb_right;
- } else {
- break;
- }
- }
- return last_lower; /* Returns NULL if there is no overlap */
-}
+static struct rb_root_cached memtype_rbroot = RB_ROOT_CACHED;
enum {
MEMTYPE_EXACT_MATCH = 0,
MEMTYPE_END_MATCH = 1
};
-static struct memtype *memtype_rb_match(struct rb_root *root,
- u64 start, u64 end, int match_type)
+static struct memtype *memtype_match(struct rb_root_cached *root,
+ u64 start, u64 end, int match_type)
{
struct memtype *match;
- match = memtype_rb_lowest_match(root, start, end);
+ match = memtype_interval_iter_first(root, start, end);
while (match != NULL && match->start < end) {
- struct rb_node *node;
-
if ((match_type == MEMTYPE_EXACT_MATCH) &&
(match->start == start) && (match->end == end))
return match;
@@ -107,26 +67,21 @@ static struct memtype *memtype_rb_match(struct rb_root *root,
(match->start < start) && (match->end == end))
return match;
- node = rb_next(&match->rb);
- if (node)
- match = rb_entry(node, struct memtype, rb);
- else
- match = NULL;
+ match = memtype_interval_iter_next(match, start, end);
}
return NULL; /* Returns NULL if there is no match */
}
-static int memtype_rb_check_conflict(struct rb_root *root,
+static int memtype_rb_check_conflict(struct rb_root_cached *root,
u64 start, u64 end,
enum page_cache_mode reqtype,
enum page_cache_mode *newtype)
{
- struct rb_node *node;
struct memtype *match;
enum page_cache_mode found_type = reqtype;
- match = memtype_rb_lowest_match(&memtype_rbroot, start, end);
+ match = memtype_interval_iter_first(&memtype_rbroot, start, end);
if (match == NULL)
goto success;
@@ -136,19 +91,12 @@ static int memtype_rb_check_conflict(struct rb_root *root,
dprintk("Overlap at 0x%Lx-0x%Lx\n", match->start, match->end);
found_type = match->type;
- node = rb_next(&match->rb);
- while (node) {
- match = rb_entry(node, struct memtype, rb);
-
- if (match->start >= end) /* Checked all possible matches */
- goto success;
-
- if (is_node_overlap(match, start, end) &&
- match->type != found_type) {
+ match = memtype_interval_iter_next(match, start, end);
+ while (match) {
+ if (match->type != found_type)
goto failure;
- }
- node = rb_next(&match->rb);
+ match = memtype_interval_iter_next(match, start, end);
}
success:
if (newtype)
@@ -163,43 +111,20 @@ failure:
return -EBUSY;
}
-static void memtype_rb_insert(struct rb_root *root, struct memtype *newdata)
-{
- struct rb_node **node = &(root->rb_node);
- struct rb_node *parent = NULL;
-
- while (*node) {
- struct memtype *data = rb_entry(*node, struct memtype, rb);
-
- parent = *node;
- if (data->subtree_max_end < newdata->end)
- data->subtree_max_end = newdata->end;
- if (newdata->start <= data->start)
- node = &((*node)->rb_left);
- else if (newdata->start > data->start)
- node = &((*node)->rb_right);
- }
-
- newdata->subtree_max_end = newdata->end;
- rb_link_node(&newdata->rb, parent, node);
- rb_insert_augmented(&newdata->rb, root, &memtype_rb_augment_cb);
-}
-
int rbt_memtype_check_insert(struct memtype *new,
enum page_cache_mode *ret_type)
{
int err = 0;
err = memtype_rb_check_conflict(&memtype_rbroot, new->start, new->end,
- new->type, ret_type);
-
- if (!err) {
- if (ret_type)
- new->type = *ret_type;
-
- new->subtree_max_end = new->end;
- memtype_rb_insert(&memtype_rbroot, new);
- }
+ new->type, ret_type);
+ if (err)
+ goto done;
+
+ if (ret_type)
+ new->type = *ret_type;
+ memtype_interval_insert(new, &memtype_rbroot);
+done:
return err;
}
@@ -214,26 +139,23 @@ struct memtype *rbt_memtype_erase(u64 start, u64 end)
* it then checks with END_MATCH, i.e. shrink the size of a node
* from the end for the mremap case.
*/
- data = memtype_rb_match(&memtype_rbroot, start, end,
- MEMTYPE_EXACT_MATCH);
+ data = memtype_match(&memtype_rbroot, start, end,
+ MEMTYPE_EXACT_MATCH);
if (!data) {
- data = memtype_rb_match(&memtype_rbroot, start, end,
- MEMTYPE_END_MATCH);
+ data = memtype_match(&memtype_rbroot, start, end,
+ MEMTYPE_END_MATCH);
if (!data)
return ERR_PTR(-EINVAL);
}
if (data->start == start) {
/* munmap: erase this node */
- rb_erase_augmented(&data->rb, &memtype_rbroot,
- &memtype_rb_augment_cb);
+ memtype_interval_remove(data, &memtype_rbroot);
} else {
/* mremap: update the end value of this node */
- rb_erase_augmented(&data->rb, &memtype_rbroot,
- &memtype_rb_augment_cb);
+ memtype_interval_remove(data, &memtype_rbroot);
data->end = start;
- data->subtree_max_end = data->end;
- memtype_rb_insert(&memtype_rbroot, data);
+ memtype_interval_insert(data, &memtype_rbroot);
return NULL;
}
@@ -242,24 +164,24 @@ struct memtype *rbt_memtype_erase(u64 start, u64 end)
struct memtype *rbt_memtype_lookup(u64 addr)
{
- return memtype_rb_lowest_match(&memtype_rbroot, addr, addr + PAGE_SIZE);
+ return memtype_interval_iter_first(&memtype_rbroot, addr,
+ addr + PAGE_SIZE);
}
#if defined(CONFIG_DEBUG_FS)
int rbt_memtype_copy_nth_element(struct memtype *out, loff_t pos)
{
- struct rb_node *node;
+ struct memtype *match;
int i = 1;
- node = rb_first(&memtype_rbroot);
- while (node && pos != i) {
- node = rb_next(node);
+ match = memtype_interval_iter_first(&memtype_rbroot, 0, ULONG_MAX);
+ while (match && pos != i) {
+ match = memtype_interval_iter_next(match, 0, ULONG_MAX);
i++;
}
- if (node) { /* pos == i */
- struct memtype *this = rb_entry(node, struct memtype, rb);
- *out = *this;
+ if (match) { /* pos == i */
+ *out = *match;
return 0;
} else {
return 1;
The following commit has been merged into the x86/mm branch of tip:
Commit-ID: ee4e7b04b718cc8a674810e3e0a339a493833640
Gitweb: https://git.kernel.org/tip/ee4e7b04b718cc8a674810e3e0a339a493833640
Author: Davidlohr Bueso <[email protected]>
AuthorDate: Mon, 21 Oct 2019 16:19:24 -07:00
Committer: Ingo Molnar <[email protected]>
CommitterDate: Tue, 19 Nov 2019 09:16:00 +01:00
x86/mm/pat: Rename pat_rbtree.c to pat_interval.c
Considering that we don't use an rbtree but an interval tree,
rename the main file accordingly.
Signed-off-by: Davidlohr Bueso <[email protected]>
Cc: Andy Lutomirski <[email protected]>
Cc: Borislav Petkov <[email protected]>
Cc: Dave Hansen <[email protected]>
Cc: H. Peter Anvin <[email protected]>
Cc: Linus Torvalds <[email protected]>
Cc: Peter Zijlstra <[email protected]>
Cc: Rik van Riel <[email protected]>
Cc: Thomas Gleixner <[email protected]>
Cc: [email protected]
Link: https://lkml.kernel.org/r/[email protected]
Signed-off-by: Ingo Molnar <[email protected]>
---
arch/x86/mm/Makefile | 2 +-
arch/x86/mm/pat_interval.c | 186 ++++++++++++++++++++++++++++++++++++-
arch/x86/mm/pat_rbtree.c | 186 +------------------------------------
3 files changed, 187 insertions(+), 187 deletions(-)
create mode 100644 arch/x86/mm/pat_interval.c
delete mode 100644 arch/x86/mm/pat_rbtree.c
diff --git a/arch/x86/mm/Makefile b/arch/x86/mm/Makefile
index 84373dc..de403df 100644
--- a/arch/x86/mm/Makefile
+++ b/arch/x86/mm/Makefile
@@ -23,7 +23,7 @@ CFLAGS_mem_encrypt_identity.o := $(nostackp)
CFLAGS_fault.o := -I $(srctree)/$(src)/../include/asm/trace
-obj-$(CONFIG_X86_PAT) += pat_rbtree.o
+obj-$(CONFIG_X86_PAT) += pat_interval.o
obj-$(CONFIG_X86_32) += pgtable_32.o iomap_32.o
diff --git a/arch/x86/mm/pat_interval.c b/arch/x86/mm/pat_interval.c
new file mode 100644
index 0000000..ef59e0a
--- /dev/null
+++ b/arch/x86/mm/pat_interval.c
@@ -0,0 +1,186 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Handle caching attributes in page tables (PAT)
+ *
+ * Authors: Venkatesh Pallipadi <[email protected]>
+ * Suresh B Siddha <[email protected]>
+ *
+ * Interval tree used to store the PAT memory type reservations.
+ */
+
+#include <linux/seq_file.h>
+#include <linux/debugfs.h>
+#include <linux/kernel.h>
+#include <linux/interval_tree_generic.h>
+#include <linux/sched.h>
+#include <linux/gfp.h>
+
+#include <asm/pgtable.h>
+#include <asm/pat.h>
+
+#include "pat_internal.h"
+
+/*
+ * The memtype tree keeps track of memory type for specific
+ * physical memory areas. Without proper tracking, conflicting memory
+ * types in different mappings can cause CPU cache corruption.
+ *
+ * The tree is an interval tree (augmented rbtree) with tree ordered
+ * on starting address. Tree can contain multiple entries for
+ * different regions which overlap. All the aliases have the same
+ * cache attributes of course.
+ *
+ * memtype_lock protects the rbtree.
+ */
+static inline u64 memtype_interval_start(struct memtype *memtype)
+{
+ return memtype->start;
+}
+
+static inline u64 memtype_interval_end(struct memtype *memtype)
+{
+ return memtype->end - 1;
+}
+INTERVAL_TREE_DEFINE(struct memtype, rb, u64, subtree_max_end,
+ memtype_interval_start, memtype_interval_end,
+ static, memtype_interval)
+
+static struct rb_root_cached memtype_rbroot = RB_ROOT_CACHED;
+
+enum {
+ MEMTYPE_EXACT_MATCH = 0,
+ MEMTYPE_END_MATCH = 1
+};
+
+static struct memtype *memtype_match(u64 start, u64 end, int match_type)
+{
+ struct memtype *match;
+
+ match = memtype_interval_iter_first(&memtype_rbroot, start, end);
+ while (match != NULL && match->start < end) {
+ if ((match_type == MEMTYPE_EXACT_MATCH) &&
+ (match->start == start) && (match->end == end))
+ return match;
+
+ if ((match_type == MEMTYPE_END_MATCH) &&
+ (match->start < start) && (match->end == end))
+ return match;
+
+ match = memtype_interval_iter_next(match, start, end);
+ }
+
+ return NULL; /* Returns NULL if there is no match */
+}
+
+static int memtype_check_conflict(u64 start, u64 end,
+ enum page_cache_mode reqtype,
+ enum page_cache_mode *newtype)
+{
+ struct memtype *match;
+ enum page_cache_mode found_type = reqtype;
+
+ match = memtype_interval_iter_first(&memtype_rbroot, start, end);
+ if (match == NULL)
+ goto success;
+
+ if (match->type != found_type && newtype == NULL)
+ goto failure;
+
+ dprintk("Overlap at 0x%Lx-0x%Lx\n", match->start, match->end);
+ found_type = match->type;
+
+ match = memtype_interval_iter_next(match, start, end);
+ while (match) {
+ if (match->type != found_type)
+ goto failure;
+
+ match = memtype_interval_iter_next(match, start, end);
+ }
+success:
+ if (newtype)
+ *newtype = found_type;
+
+ return 0;
+
+failure:
+ pr_info("x86/PAT: %s:%d conflicting memory types %Lx-%Lx %s<->%s\n",
+ current->comm, current->pid, start, end,
+ cattr_name(found_type), cattr_name(match->type));
+ return -EBUSY;
+}
+
+int memtype_check_insert(struct memtype *new,
+ enum page_cache_mode *ret_type)
+{
+ int err = 0;
+
+ err = memtype_check_conflict(new->start, new->end,
+ new->type, ret_type);
+ if (err)
+ goto done;
+
+ if (ret_type)
+ new->type = *ret_type;
+ memtype_interval_insert(new, &memtype_rbroot);
+done:
+ return err;
+}
+
+struct memtype *memtype_erase(u64 start, u64 end)
+{
+ struct memtype *data;
+
+ /*
+ * Since the memtype_rbroot tree allows overlapping ranges,
+ * memtype_erase() checks with EXACT_MATCH first, i.e. free
+ * a whole node for the munmap case. If no such entry is found,
+ * it then checks with END_MATCH, i.e. shrink the size of a node
+ * from the end for the mremap case.
+ */
+ data = memtype_match(start, end, MEMTYPE_EXACT_MATCH);
+ if (!data) {
+ data = memtype_match(start, end, MEMTYPE_END_MATCH);
+ if (!data)
+ return ERR_PTR(-EINVAL);
+ }
+
+ if (data->start == start) {
+ /* munmap: erase this node */
+ memtype_interval_remove(data, &memtype_rbroot);
+ } else {
+ /* mremap: update the end value of this node */
+ memtype_interval_remove(data, &memtype_rbroot);
+ data->end = start;
+ memtype_interval_insert(data, &memtype_rbroot);
+ return NULL;
+ }
+
+ return data;
+}
+
+struct memtype *memtype_lookup(u64 addr)
+{
+ return memtype_interval_iter_first(&memtype_rbroot, addr,
+ addr + PAGE_SIZE);
+}
+
+#if defined(CONFIG_DEBUG_FS)
+int memtype_copy_nth_element(struct memtype *out, loff_t pos)
+{
+ struct memtype *match;
+ int i = 1;
+
+ match = memtype_interval_iter_first(&memtype_rbroot, 0, ULONG_MAX);
+ while (match && pos != i) {
+ match = memtype_interval_iter_next(match, 0, ULONG_MAX);
+ i++;
+ }
+
+ if (match) { /* pos == i */
+ *out = *match;
+ return 0;
+ } else {
+ return 1;
+ }
+}
+#endif
diff --git a/arch/x86/mm/pat_rbtree.c b/arch/x86/mm/pat_rbtree.c
deleted file mode 100644
index ef59e0a..0000000
--- a/arch/x86/mm/pat_rbtree.c
+++ /dev/null
@@ -1,186 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/*
- * Handle caching attributes in page tables (PAT)
- *
- * Authors: Venkatesh Pallipadi <[email protected]>
- * Suresh B Siddha <[email protected]>
- *
- * Interval tree used to store the PAT memory type reservations.
- */
-
-#include <linux/seq_file.h>
-#include <linux/debugfs.h>
-#include <linux/kernel.h>
-#include <linux/interval_tree_generic.h>
-#include <linux/sched.h>
-#include <linux/gfp.h>
-
-#include <asm/pgtable.h>
-#include <asm/pat.h>
-
-#include "pat_internal.h"
-
-/*
- * The memtype tree keeps track of memory type for specific
- * physical memory areas. Without proper tracking, conflicting memory
- * types in different mappings can cause CPU cache corruption.
- *
- * The tree is an interval tree (augmented rbtree) with tree ordered
- * on starting address. Tree can contain multiple entries for
- * different regions which overlap. All the aliases have the same
- * cache attributes of course.
- *
- * memtype_lock protects the rbtree.
- */
-static inline u64 memtype_interval_start(struct memtype *memtype)
-{
- return memtype->start;
-}
-
-static inline u64 memtype_interval_end(struct memtype *memtype)
-{
- return memtype->end - 1;
-}
-INTERVAL_TREE_DEFINE(struct memtype, rb, u64, subtree_max_end,
- memtype_interval_start, memtype_interval_end,
- static, memtype_interval)
-
-static struct rb_root_cached memtype_rbroot = RB_ROOT_CACHED;
-
-enum {
- MEMTYPE_EXACT_MATCH = 0,
- MEMTYPE_END_MATCH = 1
-};
-
-static struct memtype *memtype_match(u64 start, u64 end, int match_type)
-{
- struct memtype *match;
-
- match = memtype_interval_iter_first(&memtype_rbroot, start, end);
- while (match != NULL && match->start < end) {
- if ((match_type == MEMTYPE_EXACT_MATCH) &&
- (match->start == start) && (match->end == end))
- return match;
-
- if ((match_type == MEMTYPE_END_MATCH) &&
- (match->start < start) && (match->end == end))
- return match;
-
- match = memtype_interval_iter_next(match, start, end);
- }
-
- return NULL; /* Returns NULL if there is no match */
-}
-
-static int memtype_check_conflict(u64 start, u64 end,
- enum page_cache_mode reqtype,
- enum page_cache_mode *newtype)
-{
- struct memtype *match;
- enum page_cache_mode found_type = reqtype;
-
- match = memtype_interval_iter_first(&memtype_rbroot, start, end);
- if (match == NULL)
- goto success;
-
- if (match->type != found_type && newtype == NULL)
- goto failure;
-
- dprintk("Overlap at 0x%Lx-0x%Lx\n", match->start, match->end);
- found_type = match->type;
-
- match = memtype_interval_iter_next(match, start, end);
- while (match) {
- if (match->type != found_type)
- goto failure;
-
- match = memtype_interval_iter_next(match, start, end);
- }
-success:
- if (newtype)
- *newtype = found_type;
-
- return 0;
-
-failure:
- pr_info("x86/PAT: %s:%d conflicting memory types %Lx-%Lx %s<->%s\n",
- current->comm, current->pid, start, end,
- cattr_name(found_type), cattr_name(match->type));
- return -EBUSY;
-}
-
-int memtype_check_insert(struct memtype *new,
- enum page_cache_mode *ret_type)
-{
- int err = 0;
-
- err = memtype_check_conflict(new->start, new->end,
- new->type, ret_type);
- if (err)
- goto done;
-
- if (ret_type)
- new->type = *ret_type;
- memtype_interval_insert(new, &memtype_rbroot);
-done:
- return err;
-}
-
-struct memtype *memtype_erase(u64 start, u64 end)
-{
- struct memtype *data;
-
- /*
- * Since the memtype_rbroot tree allows overlapping ranges,
- * memtype_erase() checks with EXACT_MATCH first, i.e. free
- * a whole node for the munmap case. If no such entry is found,
- * it then checks with END_MATCH, i.e. shrink the size of a node
- * from the end for the mremap case.
- */
- data = memtype_match(start, end, MEMTYPE_EXACT_MATCH);
- if (!data) {
- data = memtype_match(start, end, MEMTYPE_END_MATCH);
- if (!data)
- return ERR_PTR(-EINVAL);
- }
-
- if (data->start == start) {
- /* munmap: erase this node */
- memtype_interval_remove(data, &memtype_rbroot);
- } else {
- /* mremap: update the end value of this node */
- memtype_interval_remove(data, &memtype_rbroot);
- data->end = start;
- memtype_interval_insert(data, &memtype_rbroot);
- return NULL;
- }
-
- return data;
-}
-
-struct memtype *memtype_lookup(u64 addr)
-{
- return memtype_interval_iter_first(&memtype_rbroot, addr,
- addr + PAGE_SIZE);
-}
-
-#if defined(CONFIG_DEBUG_FS)
-int memtype_copy_nth_element(struct memtype *out, loff_t pos)
-{
- struct memtype *match;
- int i = 1;
-
- match = memtype_interval_iter_first(&memtype_rbroot, 0, ULONG_MAX);
- while (match && pos != i) {
- match = memtype_interval_iter_next(match, 0, ULONG_MAX);
- i++;
- }
-
- if (match) { /* pos == i */
- *out = *match;
- return 0;
- } else {
- return 1;
- }
-}
-#endif
* Thomas Gleixner <[email protected]> wrote:
> On Mon, 21 Oct 2019, Davidlohr Bueso wrote:
> > int rbt_memtype_check_insert(struct memtype *new,
> > enum page_cache_mode *ret_type)
> > {
> > int err = 0;
> >
> > err = memtype_rb_check_conflict(&memtype_rbroot, new->start, new->end,
> > - new->type, ret_type);
> > -
> > - if (!err) {
> > - if (ret_type)
> > - new->type = *ret_type;
> > -
> > - new->subtree_max_end = new->end;
> > - memtype_rb_insert(&memtype_rbroot, new);
> > - }
> > + new->type, ret_type);
> > + if (err)
> > + goto done;
>
> Please return err here. That goto is pointless.
>
> > +
> > + if (ret_type)
> > + new->type = *ret_type;
> > + memtype_interval_insert(new, &memtype_rbroot);
> > +done:
> > return err;
> > }
>
> Other than that.
>
> Reviewed-by: Thomas Gleixner <[email protected]>
Thanks - I rebased the v2 version in x86/mm with this cleanup included.
Two days ago I noticed a similarly quirky control flow in free_memtype()
as well, and fixed it via the patch below.
Ingo
==============>
From: Ingo Molnar <[email protected]>
Date: Tue, 19 Nov 2019 10:18:56 +0100
Subject: [PATCH] x86/mm/pat: Simplify the free_memtype() control flow
Simplify/streamline the quirky handling of the pat_pagerange_is_ram() logic,
and get rid of the 'err' local variable.
Signed-off-by: Ingo Molnar <[email protected]>
---
arch/x86/mm/pat.c | 11 +++--------
1 file changed, 3 insertions(+), 8 deletions(-)
diff --git a/arch/x86/mm/pat.c b/arch/x86/mm/pat.c
index ea7da7e62e17..af049920e59a 100644
--- a/arch/x86/mm/pat.c
+++ b/arch/x86/mm/pat.c
@@ -656,7 +656,6 @@ int reserve_memtype(u64 start, u64 end, enum page_cache_mode req_type,
int free_memtype(u64 start, u64 end)
{
- int err = -EINVAL;
int is_range_ram;
struct memtype *entry;
@@ -671,14 +670,10 @@ int free_memtype(u64 start, u64 end)
return 0;
is_range_ram = pat_pagerange_is_ram(start, end);
- if (is_range_ram == 1) {
-
- err = free_ram_pages_type(start, end);
-
- return err;
- } else if (is_range_ram < 0) {
+ if (is_range_ram == 1)
+ return free_ram_pages_type(start, end);
+ if (is_range_ram < 0)
return -EINVAL;
- }
spin_lock(&memtype_lock);
entry = memtype_erase(start, end);