--
ChangeList:
v3:
use vma->vm_flags to replace mm->context.flags.
v2:
build the patch on Marc's NV series[1].
v1:
add support for TTL field in arm64.
--
ARMv8.4-TTL provides the TTL field in tlbi instruction to indicate
the level of translation table walk holding the leaf entry for the
address that is being invalidated. Hardware can use this information
to determine if there was a risk of splintering.
Marc has provided basic support for ARM64-TTL features on his
NV series[1] patches. NV is a large feature, however, only
patches 62[2] and 67[3] are need by this patch set.
** You only need read those two patches before review this patch. **
Some of this patch depends on a feature powered by @Will Deacon
two years ago, which tracking the level of page tables in mm_gather.
See more in commit a6d60245.
[1] git://git.kernel.org/pub/scm/linux/kernel/git/maz/arm-platforms.git kvm-arm64/nv-5.6-rc1
[2] https://lore.kernel.org/linux-arm-kernel/[email protected]/
[3] https://lore.kernel.org/linux-arm-kernel/[email protected]/
Zhenyu Ye (4):
arm64: Add level-hinted TLB invalidation helper to tlbi_user
mm: Add page table level flags to vm_flags
arm64: tlb: Use translation level hint in vm_flags
mm: Set VM_LEVEL flags in some tlb_flush functions
arch/arm64/include/asm/mmu.h | 2 ++
arch/arm64/include/asm/tlb.h | 12 +++++++++
arch/arm64/include/asm/tlbflush.h | 44 ++++++++++++++++++++++++++-----
arch/arm64/mm/hugetlbpage.c | 4 +--
arch/arm64/mm/mmu.c | 14 ++++++++++
include/asm-generic/pgtable.h | 16 +++++++++--
include/linux/mm.h | 10 +++++++
include/trace/events/mmflags.h | 15 ++++++++++-
mm/huge_memory.c | 8 +++++-
9 files changed, 113 insertions(+), 12 deletions(-)
--
2.19.1
The relevant functions are:
tlb_flush in asm/tlb.h
get_clear_flush and clear_flush in mm/hugetlbpage.c
flush_pmd|pud_tlb_range in asm-generic/patable.h
do_huge_pmd_numa_page and move_huge_pmd in mm/huge_memory.c
Signed-off-by: Zhenyu Ye <[email protected]>
---
arch/arm64/include/asm/tlb.h | 12 ++++++++++++
arch/arm64/mm/hugetlbpage.c | 4 ++--
include/asm-generic/pgtable.h | 16 ++++++++++++++--
mm/huge_memory.c | 8 +++++++-
4 files changed, 35 insertions(+), 5 deletions(-)
diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h
index b76df828e6b7..77fe942b30b6 100644
--- a/arch/arm64/include/asm/tlb.h
+++ b/arch/arm64/include/asm/tlb.h
@@ -27,6 +27,18 @@ static inline void tlb_flush(struct mmu_gather *tlb)
bool last_level = !tlb->freed_tables;
unsigned long stride = tlb_get_unmap_size(tlb);
+ /*
+ * mm_gather tracked which levels of the page tables
+ * have been cleared, we can use this info to set
+ * vm->vm_flags.
+ */
+ if (tlb->cleared_ptes)
+ vma.vm_flags |= VM_LEVEL_PTE;
+ else if (tlb->cleared_pmds)
+ vma.vm_flags |= VM_LEVEL_PMD;
+ else if (tlb->cleared_puds)
+ vma.vm_flags |= VM_LEVEL_PUD;
+
/*
* If we're tearing down the address space then we only care about
* invalidating the walk-cache, since the ASID allocator won't
diff --git a/arch/arm64/mm/hugetlbpage.c b/arch/arm64/mm/hugetlbpage.c
index bbeb6a5a6ba6..c35a1bd06bd0 100644
--- a/arch/arm64/mm/hugetlbpage.c
+++ b/arch/arm64/mm/hugetlbpage.c
@@ -140,7 +140,7 @@ static pte_t get_clear_flush(struct mm_struct *mm,
}
if (valid) {
- struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0);
+ struct vm_area_struct vma = TLB_FLUSH_VMA(mm, VM_LEVEL_PTE);
flush_tlb_range(&vma, saddr, addr);
}
return orig_pte;
@@ -161,7 +161,7 @@ static void clear_flush(struct mm_struct *mm,
unsigned long pgsize,
unsigned long ncontig)
{
- struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0);
+ struct vm_area_struct vma = TLB_FLUSH_VMA(mm, VM_LEVEL_PTE);
unsigned long i, saddr = addr;
for (i = 0; i < ncontig; i++, addr += pgsize, ptep++)
diff --git a/include/asm-generic/pgtable.h b/include/asm-generic/pgtable.h
index e2e2bef07dd2..391e704faf7a 100644
--- a/include/asm-generic/pgtable.h
+++ b/include/asm-generic/pgtable.h
@@ -1160,8 +1160,20 @@ static inline int pmd_free_pte_page(pmd_t *pmd, unsigned long addr)
* invalidate the entire TLB which is not desitable.
* e.g. see arch/arc: flush_pmd_tlb_range
*/
-#define flush_pmd_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end)
-#define flush_pud_tlb_range(vma, addr, end) flush_tlb_range(vma, addr, end)
+#define flush_pmd_tlb_range(vma, addr, end) \
+ do { \
+ vma->vm_flags &= ~(VM_LEVEL_PUD | VM_LEVEL_PTE); \
+ vma->vm_flags |= VM_LEVEL_PMD; \
+ flush_tlb_range(vma, addr, end); \
+ } while (0)
+
+#define flush_pud_tlb_range(vma, addr, end) \
+ do { \
+ vma->vm_flags &= ~(VM_LEVEL_PMD | VM_LEVEL_PTE); \
+ vma->vm_flags |= VM_LEVEL_PUD; \
+ flush_tlb_range(vma, addr, end); \
+ } while (0)
+
#else
#define flush_pmd_tlb_range(vma, addr, end) BUILD_BUG()
#define flush_pud_tlb_range(vma, addr, end) BUILD_BUG()
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index b08b199f9a11..f28ced8d298e 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -1646,6 +1646,8 @@ vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf, pmd_t pmd)
* mapping or not. Hence use the tlb range variant
*/
if (mm_tlb_flush_pending(vma->vm_mm)) {
+ vma->vm_flags &= ~(VM_LEVEL_PUD | VM_LEVEL_PTE);
+ vma->vm_flags |= VM_LEVEL_PMD;
flush_tlb_range(vma, haddr, haddr + HPAGE_PMD_SIZE);
/*
* change_huge_pmd() released the pmd lock before
@@ -1917,8 +1919,12 @@ bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr,
}
pmd = move_soft_dirty_pmd(pmd);
set_pmd_at(mm, new_addr, new_pmd, pmd);
- if (force_flush)
+ if (force_flush) {
+ vma->vm_flags &= ~(VM_LEVEL_PUD | VM_LEVEL_PTE);
+ vma->vm_flags |= VM_LEVEL_PMD;
flush_tlb_range(vma, old_addr, old_addr + PMD_SIZE);
+ }
+
if (new_ptl != old_ptl)
spin_unlock(new_ptl);
spin_unlock(old_ptl);
--
2.19.1
Add VM_LEVEL_[PUD|PMD|PTE] to vm_flags to indicate which level of
the page tables the vma is in. Those flags can be used to reduce
the cost of TLB invalidation.
These should be common flags for all architectures, however, those
flags are only available in 64-bits system currently, because the
lower-order flags are fully used.
These flags are only used by ARM64 architecture now. See in next
patch.
Signed-off-by: Zhenyu Ye <[email protected]>
---
include/linux/mm.h | 10 ++++++++++
include/trace/events/mmflags.h | 15 ++++++++++++++-
2 files changed, 24 insertions(+), 1 deletion(-)
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 52269e56c514..63a04e02a221 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -313,6 +313,16 @@ extern unsigned int kobjsize(const void *objp);
#endif
#endif /* CONFIG_ARCH_HAS_PKEYS */
+#ifdef CONFIG_64BIT
+# define VM_LEVEL_PUD BIT(37) /* vma is in pud-level of page table */
+# define VM_LEVEL_PMD BIT(38) /* vma is in pmd-level of page table */
+# define VM_LEVEL_PTE BIT(39) /* vma is in pte-level of page table */
+#else
+# define VM_LEVEL_PUD 0
+# define VM_LEVEL_PMD 0
+# define VM_LEVEL_PTE 0
+#endif /* CONFIG_64BIT */
+
#if defined(CONFIG_X86)
# define VM_PAT VM_ARCH_1 /* PAT reserves whole VMA at once (x86) */
#elif defined(CONFIG_PPC)
diff --git a/include/trace/events/mmflags.h b/include/trace/events/mmflags.h
index a1675d43777e..9f13cfa96f9f 100644
--- a/include/trace/events/mmflags.h
+++ b/include/trace/events/mmflags.h
@@ -130,6 +130,16 @@ IF_HAVE_PG_IDLE(PG_idle, "idle" )
#define IF_HAVE_VM_SOFTDIRTY(flag,name)
#endif
+#ifdef CONFIG_64BIT
+#define IF_HAVE_VM_LEVEL_PUD(flag,name) {flag, name}
+#define IF_HAVE_VM_LEVEL_PMD(flag,name) {flag, name}
+#define IF_HAVE_VM_LEVEL_PTE(flag,name) {flag, name}
+#else
+#define IF_HAVE_VM_LEVEL_PUD(flag,name)
+#define IF_HAVE_VM_LEVEL_PMD(flag,name)
+#define IF_HAVE_VM_LEVEL_PTE(flag,name)
+#endif
+
#define __def_vmaflag_names \
{VM_READ, "read" }, \
{VM_WRITE, "write" }, \
@@ -161,7 +171,10 @@ IF_HAVE_VM_SOFTDIRTY(VM_SOFTDIRTY, "softdirty" ) \
{VM_MIXEDMAP, "mixedmap" }, \
{VM_HUGEPAGE, "hugepage" }, \
{VM_NOHUGEPAGE, "nohugepage" }, \
- {VM_MERGEABLE, "mergeable" } \
+ {VM_MERGEABLE, "mergeable" }, \
+IF_HAVE_VM_LEVEL_PUD(VM_LEVEL_PUD, "pud-level" ), \
+IF_HAVE_VM_LEVEL_PMD(VM_LEVEL_PMD, "pmd-level" ), \
+IF_HAVE_VM_LEVEL_PTE(VM_LEVEL_PTE, "pte-level" ) \
#define show_vma_flags(flags) \
(flags) ? __print_flags(flags, "|", \
--
2.19.1
Hi Zhenyu,
On 2020/3/21 20:16, Zhenyu Ye wrote:
> --
> ChangeList:
> v3:
> use vma->vm_flags to replace mm->context.flags.
>
> v2:
> build the patch on Marc's NV series[1].
>
> v1:
> add support for TTL field in arm64.
>
> --
> ARMv8.4-TTL provides the TTL field in tlbi instruction to indicate
> the level of translation table walk holding the leaf entry for the
> address that is being invalidated. Hardware can use this information
> to determine if there was a risk of splintering.
>
> Marc has provided basic support for ARM64-TTL features on his
> NV series[1] patches. NV is a large feature, however, only
> patches 62[2] and 67[3] are need by this patch set.
> ** You only need read those two patches before review this patch. **
It'd be good if you can put the whole thing into a series, otherwise
people will have difficulty when reviewing and testing it...
I haven't tracked the previous versions. If Marc is OK to share the
two patches below [2][3], I'd suggest you to pick them up, add them
in your series, rebase on top of mainline and resend it.
Thanks,
Zenghui
>
> Some of this patch depends on a feature powered by @Will Deacon
> two years ago, which tracking the level of page tables in mm_gather.
> See more in commit a6d60245.
>
> [1] git://git.kernel.org/pub/scm/linux/kernel/git/maz/arm-platforms.git kvm-arm64/nv-5.6-rc1
> [2] https://lore.kernel.org/linux-arm-kernel/[email protected]/
> [3] https://lore.kernel.org/linux-arm-kernel/[email protected]/
>
> Zhenyu Ye (4):
> arm64: Add level-hinted TLB invalidation helper to tlbi_user
> mm: Add page table level flags to vm_flags
> arm64: tlb: Use translation level hint in vm_flags
> mm: Set VM_LEVEL flags in some tlb_flush functions
>
> arch/arm64/include/asm/mmu.h | 2 ++
> arch/arm64/include/asm/tlb.h | 12 +++++++++
> arch/arm64/include/asm/tlbflush.h | 44 ++++++++++++++++++++++++++-----
> arch/arm64/mm/hugetlbpage.c | 4 +--
> arch/arm64/mm/mmu.c | 14 ++++++++++
> include/asm-generic/pgtable.h | 16 +++++++++--
> include/linux/mm.h | 10 +++++++
> include/trace/events/mmflags.h | 15 ++++++++++-
> mm/huge_memory.c | 8 +++++-
> 9 files changed, 113 insertions(+), 12 deletions(-)
>
Hi Zenghui,
On 2020/3/24 19:31, Zenghui Yu wrote:
>Hi Zhenyu,
>
>On 2020/3/21 20:16, Zhenyu Ye wrote:
>> --
>> ChangeList:
>> v3:
>> use vma->vm_flags to replace mm->context.flags.
>>
>> v2:
>> build the patch on Marc's NV series[1].
>>
>> v1:
>> add support for TTL field in arm64.
>>
>> --
>> ARMv8.4-TTL provides the TTL field in tlbi instruction to indicate
>> the level of translation table walk holding the leaf entry for the
>> address that is being invalidated. Hardware can use this information
>> to determine if there was a risk of splintering.
>>
>> Marc has provided basic support for ARM64-TTL features on his
>> NV series[1] patches. NV is a large feature, however, only
>> patches 62[2] and 67[3] are need by this patch set.
>> ** You only need read those two patches before review this patch. **
>
>It'd be good if you can put the whole thing into a series, otherwise
>people will have difficulty when reviewing and testing it...
>
>I haven't tracked the previous versions. If Marc is OK to share the
>two patches below [2][3], I'd suggest you to pick them up, add them
>in your series, rebase on top of mainline and resend it.
>
>
>Thanks,
>Zenghui
>
Thanks for your review. I'd take your suggestion and resend a new set
right now.
Thanks,
Zhenyu
>>
>> Some of this patch depends on a feature powered by @Will Deacon
>> two years ago, which tracking the level of page tables in mm_gather.
>> See more in commit a6d60245.
>>
>> [1] git://git.kernel.org/pub/scm/linux/kernel/git/maz/arm-platforms.git kvm-arm64/nv-5.6-rc1
>> [2] https://lore.kernel.org/linux-arm-kernel/[email protected]/
>> [3] https://lore.kernel.org/linux-arm-kernel/[email protected]/
>>
>> Zhenyu Ye (4):
>> arm64: Add level-hinted TLB invalidation helper to tlbi_user
>> mm: Add page table level flags to vm_flags
>> arm64: tlb: Use translation level hint in vm_flags
>> mm: Set VM_LEVEL flags in some tlb_flush functions
>>
>> arch/arm64/include/asm/mmu.h | 2 ++
>> arch/arm64/include/asm/tlb.h | 12 +++++++++
>> arch/arm64/include/asm/tlbflush.h | 44 ++++++++++++++++++++++++++-----
>> arch/arm64/mm/hugetlbpage.c | 4 +--
>> arch/arm64/mm/mmu.c | 14 ++++++++++
>> include/asm-generic/pgtable.h | 16 +++++++++--
>> include/linux/mm.h | 10 +++++++
>> include/trace/events/mmflags.h | 15 ++++++++++-
>> mm/huge_memory.c | 8 +++++-
>> 9 files changed, 113 insertions(+), 12 deletions(-)
>>
>