Pagewalk ignores hugepd entries and walk down the tables
as if it was traditionnal entries, leading to crazy result.
Add walk_hugepd_range() and use it to walk hugepage tables.
Signed-off-by: Christophe Leroy <[email protected]>
---
mm/pagewalk.c | 54 +++++++++++++++++++++++++++++++++++++++++++++------
1 file changed, 48 insertions(+), 6 deletions(-)
diff --git a/mm/pagewalk.c b/mm/pagewalk.c
index e81640d9f177..410a9d8f7572 100644
--- a/mm/pagewalk.c
+++ b/mm/pagewalk.c
@@ -58,6 +58,32 @@ static int walk_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
return err;
}
+static int walk_hugepd_range(hugepd_t *phpd, unsigned long addr,
+ unsigned long end, struct mm_walk *walk, int pdshift)
+{
+ int err = 0;
+#ifdef CONFIG_ARCH_HAS_HUGEPD
+ const struct mm_walk_ops *ops = walk->ops;
+ int shift = hugepd_shift(*phpd);
+ int page_size = 1 << shift;
+
+ if (addr & (page_size - 1))
+ return 0;
+
+ for (;;) {
+ pte_t *pte = hugepte_offset(*phpd, addr, pdshift);
+
+ err = ops->pte_entry(pte, addr, addr + page_size, walk);
+ if (err)
+ break;
+ if (addr >= end - page_size)
+ break;
+ addr += page_size;
+ }
+#endif
+ return err;
+}
+
static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
struct mm_walk *walk)
{
@@ -108,7 +134,10 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
goto again;
}
- err = walk_pte_range(pmd, addr, next, walk);
+ if (is_hugepd(__hugepd(pmd_val(*pmd))))
+ err = walk_hugepd_range((hugepd_t *)pmd, addr, next, walk, PMD_SHIFT);
+ else
+ err = walk_pte_range(pmd, addr, next, walk);
if (err)
break;
} while (pmd++, addr = next, addr != end);
@@ -157,7 +186,10 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,
if (pud_none(*pud))
goto again;
- err = walk_pmd_range(pud, addr, next, walk);
+ if (is_hugepd(__hugepd(pud_val(*pud))))
+ err = walk_hugepd_range((hugepd_t *)pud, addr, next, walk, PUD_SHIFT);
+ else
+ err = walk_pmd_range(pud, addr, next, walk);
if (err)
break;
} while (pud++, addr = next, addr != end);
@@ -189,8 +221,13 @@ static int walk_p4d_range(pgd_t *pgd, unsigned long addr, unsigned long end,
if (err)
break;
}
- if (ops->pud_entry || ops->pmd_entry || ops->pte_entry)
- err = walk_pud_range(p4d, addr, next, walk);
+ if (ops->pud_entry || ops->pmd_entry || ops->pte_entry) {
+ if (is_hugepd(__hugepd(p4d_val(*p4d))))
+ err = walk_hugepd_range((hugepd_t *)p4d, addr, next, walk,
+ P4D_SHIFT);
+ else
+ err = walk_pud_range(p4d, addr, next, walk);
+ }
if (err)
break;
} while (p4d++, addr = next, addr != end);
@@ -225,8 +262,13 @@ static int walk_pgd_range(unsigned long addr, unsigned long end,
break;
}
if (ops->p4d_entry || ops->pud_entry || ops->pmd_entry ||
- ops->pte_entry)
- err = walk_p4d_range(pgd, addr, next, walk);
+ ops->pte_entry) {
+ if (is_hugepd(__hugepd(pgd_val(*pgd))))
+ err = walk_hugepd_range((hugepd_t *)pgd, addr, next, walk,
+ PGDIR_SHIFT);
+ else
+ err = walk_p4d_range(pgd, addr, next, walk);
+ }
if (err)
break;
} while (pgd++, addr = next, addr != end);
--
2.25.0
Hi Christophe,
> Pagewalk ignores hugepd entries and walk down the tables
> as if it was traditionnal entries, leading to crazy result.
>
> Add walk_hugepd_range() and use it to walk hugepage tables.
>
> Signed-off-by: Christophe Leroy <[email protected]>
> ---
> mm/pagewalk.c | 54 +++++++++++++++++++++++++++++++++++++++++++++------
> 1 file changed, 48 insertions(+), 6 deletions(-)
>
> diff --git a/mm/pagewalk.c b/mm/pagewalk.c
> index e81640d9f177..410a9d8f7572 100644
> --- a/mm/pagewalk.c
> +++ b/mm/pagewalk.c
> @@ -58,6 +58,32 @@ static int walk_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
> return err;
> }
>
> +static int walk_hugepd_range(hugepd_t *phpd, unsigned long addr,
> + unsigned long end, struct mm_walk *walk, int pdshift)
> +{
> + int err = 0;
> +#ifdef CONFIG_ARCH_HAS_HUGEPD
> + const struct mm_walk_ops *ops = walk->ops;
> + int shift = hugepd_shift(*phpd);
> + int page_size = 1 << shift;
> +
> + if (addr & (page_size - 1))
> + return 0;
> +
> + for (;;) {
> + pte_t *pte = hugepte_offset(*phpd, addr, pdshift);
> +
> + err = ops->pte_entry(pte, addr, addr + page_size, walk);
> + if (err)
> + break;
> + if (addr >= end - page_size)
> + break;
> + addr += page_size;
> + }
Initially I thought this was a somewhat unintuitive way to structure
this loop, but I see it parallels the structure of walk_pte_range_inner,
so I think the consistency is worth it.
I notice the pte walking code potentially takes some locks: does this
code need to do that?
arch/powerpc/mm/hugetlbpage.c says that hugepds are protected by the
mm->page_table_lock, but I don't think we're taking it in this code.
> +#endif
> + return err;
> +}
> +
> static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
> struct mm_walk *walk)
> {
> @@ -108,7 +134,10 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
> goto again;
> }
>
> - err = walk_pte_range(pmd, addr, next, walk);
> + if (is_hugepd(__hugepd(pmd_val(*pmd))))
> + err = walk_hugepd_range((hugepd_t *)pmd, addr, next, walk, PMD_SHIFT);
> + else
> + err = walk_pte_range(pmd, addr, next, walk);
> if (err)
> break;
> } while (pmd++, addr = next, addr != end);
> @@ -157,7 +186,10 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,
> if (pud_none(*pud))
> goto again;
>
> - err = walk_pmd_range(pud, addr, next, walk);
> + if (is_hugepd(__hugepd(pud_val(*pud))))
> + err = walk_hugepd_range((hugepd_t *)pud, addr, next, walk, PUD_SHIFT);
> + else
> + err = walk_pmd_range(pud, addr, next, walk);
I'm a bit worried you might end up calling into walk_hugepd_range with
ops->pte_entry == NULL, and then jumping to 0.
static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,
struct mm_walk *walk)
{
...
pud = pud_offset(p4d, addr);
do {
...
if ((!walk->vma && (pud_leaf(*pud) || !pud_present(*pud))) ||
walk->action == ACTION_CONTINUE ||
!(ops->pmd_entry || ops->pte_entry)) <<< THIS CHECK
continue;
...
if (is_hugepd(__hugepd(pud_val(*pud))))
err = walk_hugepd_range((hugepd_t *)pud, addr, next, walk, PUD_SHIFT);
else
err = walk_pmd_range(pud, addr, next, walk);
if (err)
break;
} while (pud++, addr = next, addr != end);
walk_pud_range will proceed if there is _either_ an ops->pmd_entry _or_
an ops->pte_entry, but walk_hugepd_range will call ops->pte_entry
unconditionally.
The same issue applies to walk_{p4d,pgd}_range...
Kind regards,
Daniel
Le 16/04/2021 à 00:43, Daniel Axtens a écrit :
> Hi Christophe,
>
>> Pagewalk ignores hugepd entries and walk down the tables
>> as if it was traditionnal entries, leading to crazy result.
>>
>> Add walk_hugepd_range() and use it to walk hugepage tables.
>>
>> Signed-off-by: Christophe Leroy <[email protected]>
>> ---
>> mm/pagewalk.c | 54 +++++++++++++++++++++++++++++++++++++++++++++------
>> 1 file changed, 48 insertions(+), 6 deletions(-)
>>
>> diff --git a/mm/pagewalk.c b/mm/pagewalk.c
>> index e81640d9f177..410a9d8f7572 100644
>> --- a/mm/pagewalk.c
>> +++ b/mm/pagewalk.c
>> @@ -58,6 +58,32 @@ static int walk_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
>> return err;
>> }
>>
>> +static int walk_hugepd_range(hugepd_t *phpd, unsigned long addr,
>> + unsigned long end, struct mm_walk *walk, int pdshift)
>> +{
>> + int err = 0;
>> +#ifdef CONFIG_ARCH_HAS_HUGEPD
>> + const struct mm_walk_ops *ops = walk->ops;
>> + int shift = hugepd_shift(*phpd);
>> + int page_size = 1 << shift;
>> +
>> + if (addr & (page_size - 1))
>> + return 0;
>> +
>> + for (;;) {
>> + pte_t *pte = hugepte_offset(*phpd, addr, pdshift);
>> +
>> + err = ops->pte_entry(pte, addr, addr + page_size, walk);
>> + if (err)
>> + break;
>> + if (addr >= end - page_size)
>> + break;
>> + addr += page_size;
>> + }
>
> Initially I thought this was a somewhat unintuitive way to structure
> this loop, but I see it parallels the structure of walk_pte_range_inner,
> so I think the consistency is worth it.
>
> I notice the pte walking code potentially takes some locks: does this
> code need to do that?
>
> arch/powerpc/mm/hugetlbpage.c says that hugepds are protected by the
> mm->page_table_lock, but I don't think we're taking it in this code.
I'll add it, thanks.
>
>> +#endif
>> + return err;
>> +}
>> +
>> static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
>> struct mm_walk *walk)
>> {
>> @@ -108,7 +134,10 @@ static int walk_pmd_range(pud_t *pud, unsigned long addr, unsigned long end,
>> goto again;
>> }
>>
>> - err = walk_pte_range(pmd, addr, next, walk);
>> + if (is_hugepd(__hugepd(pmd_val(*pmd))))
>> + err = walk_hugepd_range((hugepd_t *)pmd, addr, next, walk, PMD_SHIFT);
>> + else
>> + err = walk_pte_range(pmd, addr, next, walk);
>> if (err)
>> break;
>> } while (pmd++, addr = next, addr != end);
>> @@ -157,7 +186,10 @@ static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,
>> if (pud_none(*pud))
>> goto again;
>>
>> - err = walk_pmd_range(pud, addr, next, walk);
>> + if (is_hugepd(__hugepd(pud_val(*pud))))
>> + err = walk_hugepd_range((hugepd_t *)pud, addr, next, walk, PUD_SHIFT);
>> + else
>> + err = walk_pmd_range(pud, addr, next, walk);
>
> I'm a bit worried you might end up calling into walk_hugepd_range with
> ops->pte_entry == NULL, and then jumping to 0.
You are right, I missed it.
I'll bail out of walk_hugepd_range() when ops->pte_entry is NULL.
>
> static int walk_pud_range(p4d_t *p4d, unsigned long addr, unsigned long end,
> struct mm_walk *walk)
> {
> ...
> pud = pud_offset(p4d, addr);
> do {
> ...
> if ((!walk->vma && (pud_leaf(*pud) || !pud_present(*pud))) ||
> walk->action == ACTION_CONTINUE ||
> !(ops->pmd_entry || ops->pte_entry)) <<< THIS CHECK
> continue;
> ...
> if (is_hugepd(__hugepd(pud_val(*pud))))
> err = walk_hugepd_range((hugepd_t *)pud, addr, next, walk, PUD_SHIFT);
> else
> err = walk_pmd_range(pud, addr, next, walk);
> if (err)
> break;
> } while (pud++, addr = next, addr != end);
>
> walk_pud_range will proceed if there is _either_ an ops->pmd_entry _or_
> an ops->pte_entry, but walk_hugepd_range will call ops->pte_entry
> unconditionally.
>
> The same issue applies to walk_{p4d,pgd}_range...
>
> Kind regards,
> Daniel
>
Thanks
Christophe