We used to emit a flush_icache_all() whenever a dirty executable
mapping is set in the page table but we can instead call
flush_icache_mm() which will only send IPIs to cores that currently run
this mm and add a deferred icache flush to the others.
The number of calls to sbi_remote_fence_i() (tested without IPI
support):
With a simple buildroot rootfs:
* Before: ~5k
* After : 4 (!)
Tested on HW, the boot to login is ~4.5% faster.
With an ubuntu rootfs:
* Before: ~24k
* After : ~13k
Signed-off-by: Alexandre Ghiti <[email protected]>
---
arch/riscv/include/asm/pgtable.h | 14 +++++++-------
arch/riscv/mm/cacheflush.c | 4 ++--
arch/riscv/mm/pgtable.c | 2 +-
3 files changed, 10 insertions(+), 10 deletions(-)
diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h
index 0c94260b5d0c..058cfa89dd29 100644
--- a/arch/riscv/include/asm/pgtable.h
+++ b/arch/riscv/include/asm/pgtable.h
@@ -513,12 +513,12 @@ static inline void set_pte(pte_t *ptep, pte_t pteval)
WRITE_ONCE(*ptep, pteval);
}
-void flush_icache_pte(pte_t pte);
+void flush_icache_pte(struct mm_struct *mm, pte_t pte);
-static inline void __set_pte_at(pte_t *ptep, pte_t pteval)
+static inline void __set_pte_at(struct mm_struct *mm, pte_t *ptep, pte_t pteval)
{
if (pte_present(pteval) && pte_exec(pteval))
- flush_icache_pte(pteval);
+ flush_icache_pte(mm, pteval);
set_pte(ptep, pteval);
}
@@ -529,7 +529,7 @@ static inline void set_ptes(struct mm_struct *mm, unsigned long addr,
page_table_check_ptes_set(mm, ptep, pteval, nr);
for (;;) {
- __set_pte_at(ptep, pteval);
+ __set_pte_at(mm, ptep, pteval);
if (--nr == 0)
break;
ptep++;
@@ -541,7 +541,7 @@ static inline void set_ptes(struct mm_struct *mm, unsigned long addr,
static inline void pte_clear(struct mm_struct *mm,
unsigned long addr, pte_t *ptep)
{
- __set_pte_at(ptep, __pte(0));
+ __set_pte_at(mm, ptep, __pte(0));
}
#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS /* defined in mm/pgtable.c */
@@ -707,14 +707,14 @@ static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
pmd_t *pmdp, pmd_t pmd)
{
page_table_check_pmd_set(mm, pmdp, pmd);
- return __set_pte_at((pte_t *)pmdp, pmd_pte(pmd));
+ return __set_pte_at(mm, (pte_t *)pmdp, pmd_pte(pmd));
}
static inline void set_pud_at(struct mm_struct *mm, unsigned long addr,
pud_t *pudp, pud_t pud)
{
page_table_check_pud_set(mm, pudp, pud);
- return __set_pte_at((pte_t *)pudp, pud_pte(pud));
+ return __set_pte_at(mm, (pte_t *)pudp, pud_pte(pud));
}
#ifdef CONFIG_PAGE_TABLE_CHECK
diff --git a/arch/riscv/mm/cacheflush.c b/arch/riscv/mm/cacheflush.c
index b0774993c0f3..3e234e95d952 100644
--- a/arch/riscv/mm/cacheflush.c
+++ b/arch/riscv/mm/cacheflush.c
@@ -85,12 +85,12 @@ void flush_icache_mm(struct mm_struct *mm, bool local)
#endif /* CONFIG_SMP */
#ifdef CONFIG_MMU
-void flush_icache_pte(pte_t pte)
+void flush_icache_pte(struct mm_struct *mm, pte_t pte)
{
struct folio *folio = page_folio(pte_page(pte));
if (!test_bit(PG_dcache_clean, &folio->flags)) {
- flush_icache_all();
+ flush_icache_mm(mm, false);
set_bit(PG_dcache_clean, &folio->flags);
}
}
diff --git a/arch/riscv/mm/pgtable.c b/arch/riscv/mm/pgtable.c
index ef887efcb679..533ec9055fa0 100644
--- a/arch/riscv/mm/pgtable.c
+++ b/arch/riscv/mm/pgtable.c
@@ -10,7 +10,7 @@ int ptep_set_access_flags(struct vm_area_struct *vma,
pte_t entry, int dirty)
{
if (!pte_same(ptep_get(ptep), entry))
- __set_pte_at(ptep, entry);
+ __set_pte_at(vma->vm_mm, ptep, entry);
/*
* update_mmu_cache will unconditionally execute, handling both
* the case that the PTE changed and the spurious fault case.
--
2.39.2
On Fri, Feb 02, 2024 at 01:47:11PM +0100, Alexandre Ghiti wrote:
> We used to emit a flush_icache_all() whenever a dirty executable
> mapping is set in the page table but we can instead call
> flush_icache_mm() which will only send IPIs to cores that currently run
> this mm and add a deferred icache flush to the others.
>
> The number of calls to sbi_remote_fence_i() (tested without IPI
> support):
>
> With a simple buildroot rootfs:
> * Before: ~5k
> * After : 4 (!)
>
> Tested on HW, the boot to login is ~4.5% faster.
>
> With an ubuntu rootfs:
> * Before: ~24k
> * After : ~13k
>
> Signed-off-by: Alexandre Ghiti <[email protected]>
> ---
> arch/riscv/include/asm/pgtable.h | 14 +++++++-------
> arch/riscv/mm/cacheflush.c | 4 ++--
> arch/riscv/mm/pgtable.c | 2 +-
> 3 files changed, 10 insertions(+), 10 deletions(-)
>
> diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h
> index 0c94260b5d0c..058cfa89dd29 100644
> --- a/arch/riscv/include/asm/pgtable.h
> +++ b/arch/riscv/include/asm/pgtable.h
> @@ -513,12 +513,12 @@ static inline void set_pte(pte_t *ptep, pte_t pteval)
> WRITE_ONCE(*ptep, pteval);
> }
>
> -void flush_icache_pte(pte_t pte);
> +void flush_icache_pte(struct mm_struct *mm, pte_t pte);
>
> -static inline void __set_pte_at(pte_t *ptep, pte_t pteval)
> +static inline void __set_pte_at(struct mm_struct *mm, pte_t *ptep, pte_t pteval)
> {
> if (pte_present(pteval) && pte_exec(pteval))
> - flush_icache_pte(pteval);
> + flush_icache_pte(mm, pteval);
>
> set_pte(ptep, pteval);
> }
> @@ -529,7 +529,7 @@ static inline void set_ptes(struct mm_struct *mm, unsigned long addr,
> page_table_check_ptes_set(mm, ptep, pteval, nr);
>
> for (;;) {
> - __set_pte_at(ptep, pteval);
> + __set_pte_at(mm, ptep, pteval);
> if (--nr == 0)
> break;
> ptep++;
> @@ -541,7 +541,7 @@ static inline void set_ptes(struct mm_struct *mm, unsigned long addr,
> static inline void pte_clear(struct mm_struct *mm,
> unsigned long addr, pte_t *ptep)
> {
> - __set_pte_at(ptep, __pte(0));
> + __set_pte_at(mm, ptep, __pte(0));
> }
>
> #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS /* defined in mm/pgtable.c */
> @@ -707,14 +707,14 @@ static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr,
> pmd_t *pmdp, pmd_t pmd)
> {
> page_table_check_pmd_set(mm, pmdp, pmd);
> - return __set_pte_at((pte_t *)pmdp, pmd_pte(pmd));
> + return __set_pte_at(mm, (pte_t *)pmdp, pmd_pte(pmd));
> }
>
> static inline void set_pud_at(struct mm_struct *mm, unsigned long addr,
> pud_t *pudp, pud_t pud)
> {
> page_table_check_pud_set(mm, pudp, pud);
> - return __set_pte_at((pte_t *)pudp, pud_pte(pud));
> + return __set_pte_at(mm, (pte_t *)pudp, pud_pte(pud));
> }
>
> #ifdef CONFIG_PAGE_TABLE_CHECK
> diff --git a/arch/riscv/mm/cacheflush.c b/arch/riscv/mm/cacheflush.c
> index b0774993c0f3..3e234e95d952 100644
> --- a/arch/riscv/mm/cacheflush.c
> +++ b/arch/riscv/mm/cacheflush.c
> @@ -85,12 +85,12 @@ void flush_icache_mm(struct mm_struct *mm, bool local)
> #endif /* CONFIG_SMP */
>
> #ifdef CONFIG_MMU
> -void flush_icache_pte(pte_t pte)
> +void flush_icache_pte(struct mm_struct *mm, pte_t pte)
> {
> struct folio *folio = page_folio(pte_page(pte));
>
> if (!test_bit(PG_dcache_clean, &folio->flags)) {
> - flush_icache_all();
> + flush_icache_mm(mm, false);
> set_bit(PG_dcache_clean, &folio->flags);
> }
> }
> diff --git a/arch/riscv/mm/pgtable.c b/arch/riscv/mm/pgtable.c
> index ef887efcb679..533ec9055fa0 100644
> --- a/arch/riscv/mm/pgtable.c
> +++ b/arch/riscv/mm/pgtable.c
> @@ -10,7 +10,7 @@ int ptep_set_access_flags(struct vm_area_struct *vma,
> pte_t entry, int dirty)
> {
> if (!pte_same(ptep_get(ptep), entry))
> - __set_pte_at(ptep, entry);
> + __set_pte_at(vma->vm_mm, ptep, entry);
> /*
> * update_mmu_cache will unconditionally execute, handling both
> * the case that the PTE changed and the spurious fault case.
> --
> 2.39.2
>
>
> _______________________________________________
> linux-riscv mailing list
> [email protected]
> http://lists.infradead.org/mailman/listinfo/linux-riscv
Reviewed-by: Charlie Jenkins <[email protected]>
Hello:
This patch was applied to riscv/linux.git (for-next)
by Palmer Dabbelt <[email protected]>:
On Fri, 2 Feb 2024 13:47:11 +0100 you wrote:
> We used to emit a flush_icache_all() whenever a dirty executable
> mapping is set in the page table but we can instead call
> flush_icache_mm() which will only send IPIs to cores that currently run
> this mm and add a deferred icache flush to the others.
>
> The number of calls to sbi_remote_fence_i() (tested without IPI
> support):
>
> [...]
Here is the summary with links:
- riscv: Only flush the mm icache when setting an exec pte
https://git.kernel.org/riscv/c/01261e24cfab
You are awesome, thank you!
--
Deet-doot-dot, I am a bot.
https://korg.docs.kernel.org/patchwork/pwbot.html