This series optimizes the tlb flushes on riscv which used to simply
flush the whole tlb whatever the size of the range to flush or the size
of the stride.
Patch 3 introduces a threshold that is microarchitecture specific and
will very likely be modified by vendors, not sure though which mechanism
we'll use to do that (dt? alternatives? vendor initialization code?).
Next steps would be to implement:
- svinval extension as Mayuresh did here [1]
- BATCHED_UNMAP_TLB_FLUSH (I'll wait for arm64 patchset to land)
- MMU_GATHER_RCU_TABLE_FREE
- MMU_GATHER_MERGE_VMAS
Any other idea welcome.
[1] https://lore.kernel.org/linux-riscv/[email protected]/
Changes in v3:
- Add RB from Andrew, thanks!
- Unwrap a few lines, as suggested by Andrew
- Introduce defines for -1 constants used in tlbflush.c, as suggested by Andrew and Conor
- Use huge_page_size() directly instead of using the shift, as suggested by Andrew
- Remove misleading comments as suggested by Conor
Changes in v2:
- Make static tlb_flush_all_threshold, we'll figure out later how to
override this value on a vendor basis, as suggested by Conor and Palmer
- Fix nommu build, as reported by Conor
Alexandre Ghiti (4):
riscv: Improve flush_tlb()
riscv: Improve flush_tlb_range() for hugetlb pages
riscv: Make __flush_tlb_range() loop over pte instead of flushing the
whole tlb
riscv: Improve flush_tlb_kernel_range()
arch/riscv/include/asm/tlb.h | 8 ++-
arch/riscv/include/asm/tlbflush.h | 12 ++--
arch/riscv/mm/tlbflush.c | 98 ++++++++++++++++++++++++++-----
3 files changed, 99 insertions(+), 19 deletions(-)
--
2.39.2
Currently, when the range to flush covers more than one page (a 4K page or
a hugepage), __flush_tlb_range() flushes the whole tlb. Flushing the whole
tlb comes with a greater cost than flushing a single entry so we should
flush single entries up to a certain threshold so that:
threshold * cost of flushing a single entry < cost of flushing the whole
tlb.
Co-developed-by: Mayuresh Chitale <[email protected]>
Signed-off-by: Mayuresh Chitale <[email protected]>
Signed-off-by: Alexandre Ghiti <[email protected]>
Reviewed-by: Andrew Jones <[email protected]>
---
arch/riscv/mm/tlbflush.c | 48 ++++++++++++++++++++++++++++++++++++----
1 file changed, 44 insertions(+), 4 deletions(-)
diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c
index d883df0dee4a..0c955c474f3a 100644
--- a/arch/riscv/mm/tlbflush.c
+++ b/arch/riscv/mm/tlbflush.c
@@ -7,6 +7,9 @@
#include <asm/sbi.h>
#include <asm/mmu_context.h>
+#define FLUSH_TLB_MAX_SIZE ((unsigned long)-1)
+#define FLUSH_TLB_NO_ASID ((unsigned long)-1)
+
static inline void local_flush_tlb_all_asid(unsigned long asid)
{
__asm__ __volatile__ ("sfence.vma x0, %0"
@@ -24,13 +27,48 @@ static inline void local_flush_tlb_page_asid(unsigned long addr,
: "memory");
}
+/*
+ * Flush entire TLB if number of entries to be flushed is greater
+ * than the threshold below.
+ */
+static unsigned long tlb_flush_all_threshold __read_mostly = 64;
+
+static void local_flush_tlb_range_threshold_asid(unsigned long start,
+ unsigned long size,
+ unsigned long stride,
+ unsigned long asid)
+{
+ u16 nr_ptes_in_range = DIV_ROUND_UP(size, stride);
+ int i;
+
+ if (nr_ptes_in_range > tlb_flush_all_threshold) {
+ if (asid != FLUSH_TLB_NO_ASID)
+ local_flush_tlb_all_asid(asid);
+ else
+ local_flush_tlb_all();
+ return;
+ }
+
+ for (i = 0; i < nr_ptes_in_range; ++i) {
+ if (asid != FLUSH_TLB_NO_ASID)
+ local_flush_tlb_page_asid(start, asid);
+ else
+ local_flush_tlb_page(start);
+ start += stride;
+ }
+}
+
static inline void local_flush_tlb_range(unsigned long start,
unsigned long size, unsigned long stride)
{
if (size <= stride)
local_flush_tlb_page(start);
- else
+ else if (size == FLUSH_TLB_MAX_SIZE)
local_flush_tlb_all();
+ else
+ local_flush_tlb_range_threshold_asid(start, size, stride,
+ FLUSH_TLB_NO_ASID);
+
}
static inline void local_flush_tlb_range_asid(unsigned long start,
@@ -38,8 +76,10 @@ static inline void local_flush_tlb_range_asid(unsigned long start,
{
if (size <= stride)
local_flush_tlb_page_asid(start, asid);
- else
+ else if (size == FLUSH_TLB_MAX_SIZE)
local_flush_tlb_all_asid(asid);
+ else
+ local_flush_tlb_range_threshold_asid(start, size, stride, asid);
}
static void __ipi_flush_tlb_all(void *info)
@@ -52,7 +92,7 @@ void flush_tlb_all(void)
if (riscv_use_ipi_for_rfence())
on_each_cpu(__ipi_flush_tlb_all, NULL, 1);
else
- sbi_remote_sfence_vma(NULL, 0, -1);
+ sbi_remote_sfence_vma(NULL, 0, FLUSH_TLB_MAX_SIZE);
}
struct flush_tlb_range_data {
@@ -130,7 +170,7 @@ static void __flush_tlb_range(struct mm_struct *mm, unsigned long start,
void flush_tlb_mm(struct mm_struct *mm)
{
- __flush_tlb_range(mm, 0, -1, PAGE_SIZE);
+ __flush_tlb_range(mm, 0, FLUSH_TLB_MAX_SIZE, PAGE_SIZE);
}
void flush_tlb_mm_range(struct mm_struct *mm,
--
2.39.2