This series of commits mainly adds the update_mmu_tlb_range() to
batch update tlb in an address range.
After the commit 19eaf44954df ("mm: thp: support allocation of anonymous
multi-size THP"), We may need to batch update tlb of a certain address
range by calling update_mmu_tlb() in a loop. Using the
update_mmu_tlb_range(), we can simplify the code and possibly reduce the
execution of some unnecessary code in some architectures.
Bang Li (5):
LoongArch: Add update_mmu_tlb_range()
mips: Add update_mmu_tlb_range()
riscv: Add update_mmu_tlb_range()
xtensa: Add update_mmu_tlb_range()
mm: Add update_mmu_tlb_range()
arch/loongarch/include/asm/pgtable.h | 2 ++
arch/mips/include/asm/pgtable.h | 2 ++
arch/riscv/include/asm/pgtable.h | 2 ++
arch/xtensa/include/asm/pgtable.h | 2 ++
arch/xtensa/mm/tlb.c | 6 ++++++
include/linux/pgtable.h | 5 +++++
mm/memory.c | 4 +---
7 files changed, 20 insertions(+), 3 deletions(-)
--
2.19.1.6.gb485710b
Added update_mmu_tlb_range function, we can batch update tlb of an
address range.
Signed-off-by: Bang Li <[email protected]>
---
arch/mips/include/asm/pgtable.h | 2 ++
1 file changed, 2 insertions(+)
diff --git a/arch/mips/include/asm/pgtable.h b/arch/mips/include/asm/pgtable.h
index e27a4c83c548..0891ad7d43b6 100644
--- a/arch/mips/include/asm/pgtable.h
+++ b/arch/mips/include/asm/pgtable.h
@@ -596,6 +596,8 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
#define __HAVE_ARCH_UPDATE_MMU_TLB
#define update_mmu_tlb update_mmu_cache
+#define update_mmu_tlb_range(vma, address, ptep, nr) \
+ update_mmu_cache_range(NULL, vma, address, ptep, nr)
static inline void update_mmu_cache_pmd(struct vm_area_struct *vma,
unsigned long address, pmd_t *pmdp)
--
2.19.1.6.gb485710b
After the commit 19eaf44954df ("mm: thp: support allocation of anonymous
multi-size THP"), it may need to batch update tlb of an address range
through the update_mmu_tlb function. We can simplify this operation by
adding the update_mmu_tlb_range function, which may also reduce the
execution of some unnecessary code in some architectures.
Signed-off-by: Bang Li <[email protected]>
---
include/linux/pgtable.h | 5 +++++
mm/memory.c | 4 +---
2 files changed, 6 insertions(+), 3 deletions(-)
diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h
index 18019f037bae..73411dfebf7a 100644
--- a/include/linux/pgtable.h
+++ b/include/linux/pgtable.h
@@ -734,6 +734,11 @@ static inline void update_mmu_tlb(struct vm_area_struct *vma,
unsigned long address, pte_t *ptep)
{
}
+
+static inline void update_mmu_tlb_range(struct vm_area_struct *vma,
+ unsigned long address, pte_t *ptep, unsigned int nr)
+{
+}
#define __HAVE_ARCH_UPDATE_MMU_TLB
#endif
diff --git a/mm/memory.c b/mm/memory.c
index 6647685fd3c4..1f0ca362b82a 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -4396,7 +4396,6 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
vm_fault_t ret = 0;
int nr_pages = 1;
pte_t entry;
- int i;
/* File mapping without ->vm_ops ? */
if (vma->vm_flags & VM_SHARED)
@@ -4465,8 +4464,7 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
update_mmu_tlb(vma, addr, vmf->pte);
goto release;
} else if (nr_pages > 1 && !pte_range_none(vmf->pte, nr_pages)) {
- for (i = 0; i < nr_pages; i++)
- update_mmu_tlb(vma, addr + PAGE_SIZE * i, vmf->pte + i);
+ update_mmu_tlb_range(vma, addr, vmf->pte, nr_pages);
goto release;
}
--
2.19.1.6.gb485710b
Added update_mmu_tlb_range function, we can batch update tlb of an
address range.
Signed-off-by: Bang Li <[email protected]>
---
arch/xtensa/include/asm/pgtable.h | 2 ++
arch/xtensa/mm/tlb.c | 6 ++++++
2 files changed, 8 insertions(+)
diff --git a/arch/xtensa/include/asm/pgtable.h b/arch/xtensa/include/asm/pgtable.h
index 9a7e5e57ee9a..50ccfc988256 100644
--- a/arch/xtensa/include/asm/pgtable.h
+++ b/arch/xtensa/include/asm/pgtable.h
@@ -412,6 +412,8 @@ typedef pte_t *pte_addr_t;
void update_mmu_tlb(struct vm_area_struct *vma,
unsigned long address, pte_t *ptep);
+void update_mmu_tlb_range(struct vm_area_struct *vma,
+ unsigned long address, pte_t *ptep, unsigned int nr);
#define __HAVE_ARCH_UPDATE_MMU_TLB
#endif /* !defined (__ASSEMBLY__) */
diff --git a/arch/xtensa/mm/tlb.c b/arch/xtensa/mm/tlb.c
index d8b60d6e50a8..05efba86b870 100644
--- a/arch/xtensa/mm/tlb.c
+++ b/arch/xtensa/mm/tlb.c
@@ -169,6 +169,12 @@ void update_mmu_tlb(struct vm_area_struct *vma,
local_flush_tlb_page(vma, address);
}
+void update_mmu_tlb_range(struct vm_area_struct *vma,
+ unsigned long address, pte_t *ptep, unsigned int nr)
+{
+ local_flush_tlb_range(vma, address, address + PAGE_SIZE * nr);
+}
+
#ifdef CONFIG_DEBUG_TLB_SANITY
static unsigned get_pte_for_vaddr(unsigned vaddr)
--
2.19.1.6.gb485710b
Added update_mmu_tlb_range function, we can batch update tlb of an
address range.
Signed-off-by: Bang Li <[email protected]>
---
arch/loongarch/include/asm/pgtable.h | 2 ++
1 file changed, 2 insertions(+)
diff --git a/arch/loongarch/include/asm/pgtable.h b/arch/loongarch/include/asm/pgtable.h
index af3acdf3481a..5ccc2a3a6f7a 100644
--- a/arch/loongarch/include/asm/pgtable.h
+++ b/arch/loongarch/include/asm/pgtable.h
@@ -469,6 +469,8 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
#define __HAVE_ARCH_UPDATE_MMU_TLB
#define update_mmu_tlb update_mmu_cache
+#define update_mmu_tlb_range(vma, addr, ptep, nr) \
+ update_mmu_cache_range(NULL, vma, addr, ptep, nr)
static inline void update_mmu_cache_pmd(struct vm_area_struct *vma,
unsigned long address, pmd_t *pmdp)
--
2.19.1.6.gb485710b
Added update_mmu_tlb_range function, we can batch update tlb of an
address range.
Signed-off-by: Bang Li <[email protected]>
---
arch/riscv/include/asm/pgtable.h | 2 ++
1 file changed, 2 insertions(+)
diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h
index f2d5973a011b..d515a11a52cd 100644
--- a/arch/riscv/include/asm/pgtable.h
+++ b/arch/riscv/include/asm/pgtable.h
@@ -488,6 +488,8 @@ static inline void update_mmu_cache_range(struct vm_fault *vmf,
#define __HAVE_ARCH_UPDATE_MMU_TLB
#define update_mmu_tlb update_mmu_cache
+#define update_mmu_tlb_range(vma, addr, ptep, nr) \
+ update_mmu_cache_range(NULL, vma, addr, ptep, nr)
static inline void update_mmu_cache_pmd(struct vm_area_struct *vma,
unsigned long address, pmd_t *pmdp)
--
2.19.1.6.gb485710b
Hey Bang,
On Mon, Apr 29, 2024 at 6:36 PM Bang Li <[email protected]> wrote:
>
> After the commit 19eaf44954df ("mm: thp: support allocation of anonymous
> multi-size THP"), it may need to batch update tlb of an address range
> through the update_mmu_tlb function. We can simplify this operation by
> adding the update_mmu_tlb_range function, which may also reduce the
> execution of some unnecessary code in some architectures.
>
> Signed-off-by: Bang Li <[email protected]>
> ---
> include/linux/pgtable.h | 5 +++++
> mm/memory.c | 4 +---
> 2 files changed, 6 insertions(+), 3 deletions(-)
>
> diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h
> index 18019f037bae..73411dfebf7a 100644
> --- a/include/linux/pgtable.h
> +++ b/include/linux/pgtable.h
> @@ -734,6 +734,11 @@ static inline void update_mmu_tlb(struct vm_area_struct *vma,
> unsigned long address, pte_t *ptep)
> {
> }
> +
> +static inline void update_mmu_tlb_range(struct vm_area_struct *vma,
> + unsigned long address, pte_t *ptep, unsigned int nr)
> +{
> +}
> #define __HAVE_ARCH_UPDATE_MMU_TLB
> #endif
IMO, it might be better to use a separate definition to determine whether
update_mmu_tlb_range() is overridden by a specific architecture.
Thanks,
Lance
>
> diff --git a/mm/memory.c b/mm/memory.c
> index 6647685fd3c4..1f0ca362b82a 100644
> --- a/mm/memory.c
> +++ b/mm/memory.c
> @@ -4396,7 +4396,6 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
> vm_fault_t ret = 0;
> int nr_pages = 1;
> pte_t entry;
> - int i;
>
> /* File mapping without ->vm_ops ? */
> if (vma->vm_flags & VM_SHARED)
> @@ -4465,8 +4464,7 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
> update_mmu_tlb(vma, addr, vmf->pte);
> goto release;
> } else if (nr_pages > 1 && !pte_range_none(vmf->pte, nr_pages)) {
> - for (i = 0; i < nr_pages; i++)
> - update_mmu_tlb(vma, addr + PAGE_SIZE * i, vmf->pte + i);
> + update_mmu_tlb_range(vma, addr, vmf->pte, nr_pages);
> goto release;
> }
>
> --
> 2.19.1.6.gb485710b
>
Hey Lance,
Thanks for taking time to review!
On 2024/4/29 19:23, Lance Yang wrote:
> Hey Bang,
>
> On Mon, Apr 29, 2024 at 6:36 PM Bang Li <[email protected]> wrote:
>>
>> After the commit 19eaf44954df ("mm: thp: support allocation of anonymous
>> multi-size THP"), it may need to batch update tlb of an address range
>> through the update_mmu_tlb function. We can simplify this operation by
>> adding the update_mmu_tlb_range function, which may also reduce the
>> execution of some unnecessary code in some architectures.
>>
>> Signed-off-by: Bang Li <[email protected]>
>> ---
>> include/linux/pgtable.h | 5 +++++
>> mm/memory.c | 4 +---
>> 2 files changed, 6 insertions(+), 3 deletions(-)
>>
>> diff --git a/include/linux/pgtable.h b/include/linux/pgtable.h
>> index 18019f037bae..73411dfebf7a 100644
>> --- a/include/linux/pgtable.h
>> +++ b/include/linux/pgtable.h
>> @@ -734,6 +734,11 @@ static inline void update_mmu_tlb(struct vm_area_struct *vma,
>> unsigned long address, pte_t *ptep)
>> {
>> }
>> +
>> +static inline void update_mmu_tlb_range(struct vm_area_struct *vma,
>> + unsigned long address, pte_t *ptep, unsigned int nr)
>> +{
>> +}
>> #define __HAVE_ARCH_UPDATE_MMU_TLB
>> #endif
>
> IMO, it might be better to use a separate definition to determine whether
> update_mmu_tlb_range() is overridden by a specific architecture.
I have also considered this, and I will modify it in the next version.
thank you again for your review!
Thanks,
Bang
>
> Thanks,
> Lance
>
>>
>> diff --git a/mm/memory.c b/mm/memory.c
>> index 6647685fd3c4..1f0ca362b82a 100644
>> --- a/mm/memory.c
>> +++ b/mm/memory.c
>> @@ -4396,7 +4396,6 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
>> vm_fault_t ret = 0;
>> int nr_pages = 1;
>> pte_t entry;
>> - int i;
>>
>> /* File mapping without ->vm_ops ? */
>> if (vma->vm_flags & VM_SHARED)
>> @@ -4465,8 +4464,7 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
>> update_mmu_tlb(vma, addr, vmf->pte);
>> goto release;
>> } else if (nr_pages > 1 && !pte_range_none(vmf->pte, nr_pages)) {
>> - for (i = 0; i < nr_pages; i++)
>> - update_mmu_tlb(vma, addr + PAGE_SIZE * i, vmf->pte + i);
>> + update_mmu_tlb_range(vma, addr, vmf->pte, nr_pages);
>> goto release;
>> }
>>
>> --
>> 2.19.1.6.gb485710b
>>