LinuxLists.cc - [PATCH v2] arm64: mm: fix linear mapping mem access performance degradation

2022-06-30 08:59:52

Subject: [PATCH v2] arm64: mm: fix linear mapping mem access performance degradation

The arm64 can build 2M/1G block/sectiion mapping. When using DMA/DMA32 zone
(enable crashkernel, disable rodata full, disable kfence), the mem_map will
use non block/section mapping(for crashkernel requires to shrink the region
in page granularity). But it will degrade performance when doing larging
continuous mem access in kernel(memcpy/memmove, etc).

There are many changes and discussions:
commit 031495635b46 ("arm64: Do not defer reserve_crashkernel() for
platforms with no DMA memory zones")
commit 0a30c53573b0 ("arm64: mm: Move reserve_crashkernel() into
mem_init()")
commit 2687275a5843 ("arm64: Force NO_BLOCK_MAPPINGS if crashkernel
reservation is required")

This patch changes mem_map to use block/section mapping with crashkernel.
Firstly, do block/section mapping(normally 2M or 1G) for all avail mem at
mem_map, reserve crashkernel memory. And then walking pagetable to split
block/section mapping to non block/section mapping(normally 4K) [[[only]]]
for crashkernel mem. So the linear mem mapping use block/section mapping
as more as possible. We will reduce the cpu dTLB miss conspicuously, and
accelerate mem access about 10-20% performance improvement.

I have tested it with pft(Page Fault Test) and fio, obtained great
performace improvement.

For fio test:
1.prepare ramdisk
modprobe -r brd
modprobe brd rd_nr=1 rd_size=67108864
dmsetup remove_all
wipefs -a --force /dev/ram0
mkfs -t ext4 -E lazy_itable_init=0,lazy_journal_init=0 -q -F /dev/ram0
mkdir -p /fs/ram0
mount -t ext4 /dev/ram0 /fs/ram0

2.prepare fio paremeter in x.fio file:
[global]
bs=4k
ioengine=psync
iodepth=128
size=32G
direct=1
invalidate=1
group_reporting
thread=1
rw=read
directory=/fs/ram0
numjobs=1

[task_0]
cpus_allowed=16
stonewall=1

3.run testcase:
perf stat -e dTLB-load-misses fio x.fio

4.contrast
------------------------
without patch with patch
fio READ aggrb=1493.2MB/s aggrb=1775.3MB/s
dTLB-load-misses 1,818,320,693 438,729,774
time elapsed(s) 70.500326434 62.877316408
user(s) 15.926332000 15.684721000
sys(s) 54.211939000 47.046165000

5.conclusion
Using this patch will reduce dTLB misses and improve performace greatly.

Signed-off-by: Guanghui Feng <[email protected]>
---
arch/arm64/include/asm/mmu.h | 1 +
arch/arm64/mm/init.c | 8 +--
arch/arm64/mm/mmu.c | 163 ++++++++++++++++++++++++++++++-------------
3 files changed, 116 insertions(+), 56 deletions(-)

diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h
index 48f8466..1a46b81 100644
--- a/arch/arm64/include/asm/mmu.h
+++ b/arch/arm64/include/asm/mmu.h
@@ -63,6 +63,7 @@ static inline bool arm64_kernel_unmapped_at_el0(void)
extern void arm64_memblock_init(void);
extern void paging_init(void);
extern void bootmem_init(void);
+extern void map_crashkernel(void);
extern void __iomem *early_io_map(phys_addr_t phys, unsigned long virt);
extern void init_mem_pgprot(void);
extern void create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index 339ee84..241d27e 100644
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -190,6 +190,7 @@ static void __init reserve_crashkernel(void)
crashk_res.start = crash_base;
crashk_res.end = crash_base + crash_size - 1;
insert_resource(&iomem_resource, &crashk_res);
+ map_crashkernel();
}

/*
@@ -388,10 +389,6 @@ void __init arm64_memblock_init(void)
}

early_init_fdt_scan_reserved_mem();
-
- if (!IS_ENABLED(CONFIG_ZONE_DMA) && !IS_ENABLED(CONFIG_ZONE_DMA32))
- reserve_crashkernel();
-
high_memory = __va(memblock_end_of_DRAM() - 1) + 1;
}

@@ -438,8 +435,7 @@ void __init bootmem_init(void)
* request_standard_resources() depends on crashkernel's memory being
* reserved, so do it here.
*/
- if (IS_ENABLED(CONFIG_ZONE_DMA) || IS_ENABLED(CONFIG_ZONE_DMA32))
- reserve_crashkernel();
+ reserve_crashkernel();

memblock_dump_all();
}
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 626ec32..c832848 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -42,6 +42,7 @@
#define NO_BLOCK_MAPPINGS BIT(0)
#define NO_CONT_MAPPINGS BIT(1)
#define NO_EXEC_MAPPINGS BIT(2) /* assumes FEAT_HPDS is not used */
+#define NO_SEC_REMAPPINGS BIT(3) /* rebuild with non block/sec mapping*/

u64 idmap_t0sz = TCR_T0SZ(VA_BITS_MIN);
u64 idmap_ptrs_per_pgd = PTRS_PER_PGD;
@@ -156,11 +157,12 @@ static bool pgattr_change_is_safe(u64 old, u64 new)
}

static void init_pte(pmd_t *pmdp, unsigned long addr, unsigned long end,
- phys_addr_t phys, pgprot_t prot)
+ phys_addr_t phys, pgprot_t prot, int flags)
{
pte_t *ptep;

- ptep = pte_set_fixmap_offset(pmdp, addr);
+ ptep = (flags & NO_SEC_REMAPPINGS) ? pte_offset_kernel(pmdp, addr) :
+ pte_set_fixmap_offset(pmdp, addr);
do {
pte_t old_pte = READ_ONCE(*ptep);

@@ -176,7 +178,8 @@ static void init_pte(pmd_t *pmdp, unsigned long addr, unsigned long end,
phys += PAGE_SIZE;
} while (ptep++, addr += PAGE_SIZE, addr != end);

- pte_clear_fixmap();
+ if (!(flags & NO_SEC_REMAPPINGS))
+ pte_clear_fixmap();
}

static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
@@ -208,11 +211,12 @@ static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
next = pte_cont_addr_end(addr, end);

/* use a contiguous mapping if the range is suitably aligned */
- if ((((addr | next | phys) & ~CONT_PTE_MASK) == 0) &&
+ if (!(flags & NO_SEC_REMAPPINGS) &&
+ (((addr | next | phys) & ~CONT_PTE_MASK) == 0) &&
(flags & NO_CONT_MAPPINGS) == 0)
__prot = __pgprot(pgprot_val(prot) | PTE_CONT);

- init_pte(pmdp, addr, next, phys, __prot);
+ init_pte(pmdp, addr, next, phys, __prot, flags);

phys += next - addr;
} while (addr = next, addr != end);
@@ -224,15 +228,44 @@ static void init_pmd(pud_t *pudp, unsigned long addr, unsigned long end,
{
unsigned long next;
pmd_t *pmdp;
+ phys_addr_t map_offset;
+ pmdval_t pmdval;

- pmdp = pmd_set_fixmap_offset(pudp, addr);
+ pmdp = (flags & NO_SEC_REMAPPINGS) ? pmd_offset(pudp, addr) :
+ pmd_set_fixmap_offset(pudp, addr);
do {
pmd_t old_pmd = READ_ONCE(*pmdp);

next = pmd_addr_end(addr, end);

- /* try section mapping first */
- if (((addr | next | phys) & ~PMD_MASK) == 0 &&
+ if (flags & NO_SEC_REMAPPINGS) {
+ if (!pmd_none(*pmdp) && pmd_sect(*pmdp)) {
+ phys_addr_t pte_phys = pgtable_alloc(PAGE_SHIFT);
+ pmd_clear(pmdp);
+ pmdval = PMD_TYPE_TABLE | PMD_TABLE_UXN;
+ if (flags & NO_EXEC_MAPPINGS)
+ pmdval |= PMD_TABLE_PXN;
+ __pmd_populate(pmdp, pte_phys, pmdval);
+ flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
+
+ map_offset = addr - (addr & PMD_MASK);
+ if (map_offset)
+ alloc_init_cont_pte(pmdp, addr & PMD_MASK, addr,
+ phys - map_offset, prot,
+ pgtable_alloc,
+ flags & (~NO_SEC_REMAPPINGS));
+
+ if (next < (addr & PMD_MASK) + PMD_SIZE)
+ alloc_init_cont_pte(pmdp, next,
+ (addr & PUD_MASK) + PUD_SIZE,
+ next - addr + phys,
+ prot, pgtable_alloc,
+ flags & (~NO_SEC_REMAPPINGS));
+ }
+ alloc_init_cont_pte(pmdp, addr, next, phys, prot,
+ pgtable_alloc, flags);
+ }/* try section mapping first */
+ else if (((addr | next | phys) & ~PMD_MASK) == 0 &&
(flags & NO_BLOCK_MAPPINGS) == 0) {
pmd_set_huge(pmdp, phys, prot);

@@ -252,7 +285,8 @@ static void init_pmd(pud_t *pudp, unsigned long addr, unsigned long end,
phys += next - addr;
} while (pmdp++, addr = next, addr != end);

- pmd_clear_fixmap();
+ if (!(flags & NO_SEC_REMAPPINGS))
+ pmd_clear_fixmap();
}

static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
@@ -286,7 +320,8 @@ static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
next = pmd_cont_addr_end(addr, end);

/* use a contiguous mapping if the range is suitably aligned */
- if ((((addr | next | phys) & ~CONT_PMD_MASK) == 0) &&
+ if (!(flags & NO_SEC_REMAPPINGS) &&
+ (((addr | next | phys) & ~CONT_PMD_MASK) == 0) &&
(flags & NO_CONT_MAPPINGS) == 0)
__prot = __pgprot(pgprot_val(prot) | PTE_CONT);

@@ -301,7 +336,9 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
phys_addr_t (*pgtable_alloc)(int),
int flags)
{
+ phys_addr_t map_offset;
unsigned long next;
+ pudval_t pudval;
pud_t *pudp;
p4d_t *p4dp = p4d_offset(pgdp, addr);
p4d_t p4d = READ_ONCE(*p4dp);
@@ -325,18 +362,48 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
*/
if (system_state != SYSTEM_BOOTING)
mutex_lock(&fixmap_lock);
- pudp = pud_set_fixmap_offset(p4dp, addr);
+
+ pudp = (flags & NO_SEC_REMAPPINGS) ? pud_offset(p4dp, addr) :
+ pud_set_fixmap_offset(p4dp, addr);
do {
pud_t old_pud = READ_ONCE(*pudp);
-
next = pud_addr_end(addr, end);

+ if (flags & NO_SEC_REMAPPINGS) {
+ if (!pud_none(*pudp) && pud_sect(*pudp)) {
+ phys_addr_t pmd_phys = pgtable_alloc(PMD_SHIFT);
+ pud_clear(pudp);
+ pudval = PUD_TYPE_TABLE | PUD_TABLE_UXN;
+ if (flags & NO_EXEC_MAPPINGS)
+ pudval |= PUD_TABLE_PXN;
+
+ __pud_populate(pudp, pmd_phys, pudval);
+ flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
+
+ map_offset = addr - (addr & PUD_MASK);
+ if (map_offset)
+ alloc_init_cont_pmd(pudp, addr & PUD_MASK,
+ addr, phys - map_offset,
+ prot, pgtable_alloc,
+ flags & (~NO_SEC_REMAPPINGS));
+
+ if (next < (addr & PUD_MASK) + PUD_SIZE)
+ alloc_init_cont_pmd(pudp, next,
+ (addr & PUD_MASK) +
+ PUD_SIZE,
+ next - addr + phys,
+ prot, pgtable_alloc,
+ flags & (~NO_SEC_REMAPPINGS));
+ }
+ alloc_init_cont_pmd(pudp, addr, next, phys, prot,
+ pgtable_alloc, flags);
+ }
/*
* For 4K granule only, attempt to put down a 1GB block
*/
- if (pud_sect_supported() &&
- ((addr | next | phys) & ~PUD_MASK) == 0 &&
- (flags & NO_BLOCK_MAPPINGS) == 0) {
+ else if (pud_sect_supported() &&
+ ((addr | next | phys) & ~PUD_MASK) == 0 &&
+ (flags & NO_BLOCK_MAPPINGS) == 0) {
pud_set_huge(pudp, phys, prot);

/*
@@ -355,7 +422,8 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
phys += next - addr;
} while (pudp++, addr = next, addr != end);

- pud_clear_fixmap();
+ if (!(flags & NO_SEC_REMAPPINGS))
+ pud_clear_fixmap();
if (system_state != SYSTEM_BOOTING)
mutex_unlock(&fixmap_lock);
}
@@ -483,20 +551,39 @@ void __init mark_linear_text_alias_ro(void)
PAGE_KERNEL_RO);
}

-static bool crash_mem_map __initdata;
+#ifdef CONFIG_KEXEC_CORE
+static phys_addr_t __init early_crashkernel_pgtable_alloc(int shift)
+{
+ phys_addr_t phys;
+ void *ptr;

-static int __init enable_crash_mem_map(char *arg)
+ phys = memblock_phys_alloc_range(PAGE_SIZE, PAGE_SIZE, 0,
+ MEMBLOCK_ALLOC_NOLEAKTRACE);
+ if (!phys)
+ panic("Failed to allocate page table page\n");
+
+ ptr = (void *)__phys_to_virt(phys);
+ memset(ptr, 0, PAGE_SIZE);
+ return phys;
+}
+
+void __init map_crashkernel(void)
{
- /*
- * Proper parameter parsing is done by reserve_crashkernel(). We only
- * need to know if the linear map has to avoid block mappings so that
- * the crashkernel reservations can be unmapped later.
- */
- crash_mem_map = true;
+ if (can_set_direct_map() || IS_ENABLED(CONFIG_KFENCE))
+ return;

- return 0;
+ if (!crashk_res.end)
+ return;
+
+ __create_pgd_mapping(swapper_pg_dir, crashk_res.start,
+ __phys_to_virt(crashk_res.start),
+ crashk_res.end + 1 - crashk_res.start, PAGE_KERNEL,
+ early_crashkernel_pgtable_alloc,
+ NO_EXEC_MAPPINGS | NO_SEC_REMAPPINGS);
}
-early_param("crashkernel", enable_crash_mem_map);
+#else
+void __init map_crashkernel(void) {}
+#endif

static void __init map_mem(pgd_t *pgdp)
{
@@ -527,17 +614,6 @@ static void __init map_mem(pgd_t *pgdp)
*/
memblock_mark_nomap(kernel_start, kernel_end - kernel_start);

-#ifdef CONFIG_KEXEC_CORE
- if (crash_mem_map) {
- if (IS_ENABLED(CONFIG_ZONE_DMA) ||
- IS_ENABLED(CONFIG_ZONE_DMA32))
- flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
- else if (crashk_res.end)
- memblock_mark_nomap(crashk_res.start,
- resource_size(&crashk_res));
- }
-#endif
-
/* map all the memory banks */
for_each_mem_range(i, &start, &end) {
if (start >= end)
@@ -570,19 +646,6 @@ static void __init map_mem(pgd_t *pgdp)
* in page granularity and put back unused memory to buddy system
* through /sys/kernel/kexec_crash_size interface.
*/
-#ifdef CONFIG_KEXEC_CORE
- if (crash_mem_map &&
- !IS_ENABLED(CONFIG_ZONE_DMA) && !IS_ENABLED(CONFIG_ZONE_DMA32)) {
- if (crashk_res.end) {
- __map_memblock(pgdp, crashk_res.start,
- crashk_res.end + 1,
- PAGE_KERNEL,
- NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS);
- memblock_clear_nomap(crashk_res.start,
- resource_size(&crashk_res));
- }
- }
-#endif
}

void mark_rodata_ro(void)
--
1.8.3.1

2022-06-30 12:12:07

by Guanghui Feng

[permalink] [raw]

Subject: Re: [PATCH v2] arm64: mm: fix linear mapping mem access performance degradation

I'm very sorry!
Please DO'T review this patch, I have rewrited it.

在 2022/6/30 16:38, Guanghui Feng 写道:
> The arm64 can build 2M/1G block/sectiion mapping. When using DMA/DMA32 zone
> (enable crashkernel, disable rodata full, disable kfence), the mem_map will
> use non block/section mapping(for crashkernel requires to shrink the region
> in page granularity). But it will degrade performance when doing larging
> continuous mem access in kernel(memcpy/memmove, etc).
>
> There are many changes and discussions:
> commit 031495635b46 ("arm64: Do not defer reserve_crashkernel() for
> platforms with no DMA memory zones")
> commit 0a30c53573b0 ("arm64: mm: Move reserve_crashkernel() into
> mem_init()")
> commit 2687275a5843 ("arm64: Force NO_BLOCK_MAPPINGS if crashkernel
> reservation is required")
>
> This patch changes mem_map to use block/section mapping with crashkernel.
> Firstly, do block/section mapping(normally 2M or 1G) for all avail mem at
> mem_map, reserve crashkernel memory. And then walking pagetable to split
> block/section mapping to non block/section mapping(normally 4K) [[[only]]]
> for crashkernel mem. So the linear mem mapping use block/section mapping
> as more as possible. We will reduce the cpu dTLB miss conspicuously, and
> accelerate mem access about 10-20% performance improvement.
>
> I have tested it with pft(Page Fault Test) and fio, obtained great
> performace improvement.
>
> For fio test:
> 1.prepare ramdisk
> modprobe -r brd
> modprobe brd rd_nr=1 rd_size=67108864
> dmsetup remove_all
> wipefs -a --force /dev/ram0
> mkfs -t ext4 -E lazy_itable_init=0,lazy_journal_init=0 -q -F /dev/ram0
> mkdir -p /fs/ram0
> mount -t ext4 /dev/ram0 /fs/ram0
>
> 2.prepare fio paremeter in x.fio file:
> [global]
> bs=4k
> ioengine=psync
> iodepth=128
> size=32G
> direct=1
> invalidate=1
> group_reporting
> thread=1
> rw=read
> directory=/fs/ram0
> numjobs=1
>
> [task_0]
> cpus_allowed=16
> stonewall=1
>
> 3.run testcase:
> perf stat -e dTLB-load-misses fio x.fio
>
> 4.contrast
> ------------------------
> without patch with patch
> fio READ aggrb=1493.2MB/s aggrb=1775.3MB/s
> dTLB-load-misses 1,818,320,693 438,729,774
> time elapsed(s) 70.500326434 62.877316408
> user(s) 15.926332000 15.684721000
> sys(s) 54.211939000 47.046165000
>
> 5.conclusion
> Using this patch will reduce dTLB misses and improve performace greatly.
>
> Signed-off-by: Guanghui Feng <[email protected]>
> ---
> arch/arm64/include/asm/mmu.h | 1 +
> arch/arm64/mm/init.c | 8 +--
> arch/arm64/mm/mmu.c | 163 ++++++++++++++++++++++++++++++-------------
> 3 files changed, 116 insertions(+), 56 deletions(-)
>
> diff --git a/arch/arm64/include/asm/mmu.h b/arch/arm64/include/asm/mmu.h
> index 48f8466..1a46b81 100644
> --- a/arch/arm64/include/asm/mmu.h
> +++ b/arch/arm64/include/asm/mmu.h
> @@ -63,6 +63,7 @@ static inline bool arm64_kernel_unmapped_at_el0(void)
> extern void arm64_memblock_init(void);
> extern void paging_init(void);
> extern void bootmem_init(void);
> +extern void map_crashkernel(void);
> extern void __iomem *early_io_map(phys_addr_t phys, unsigned long virt);
> extern void init_mem_pgprot(void);
> extern void create_pgd_mapping(struct mm_struct *mm, phys_addr_t phys,
> diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
> index 339ee84..241d27e 100644
> --- a/arch/arm64/mm/init.c
> +++ b/arch/arm64/mm/init.c
> @@ -190,6 +190,7 @@ static void __init reserve_crashkernel(void)
> crashk_res.start = crash_base;
> crashk_res.end = crash_base + crash_size - 1;
> insert_resource(&iomem_resource, &crashk_res);
> + map_crashkernel();
> }
>
> /*
> @@ -388,10 +389,6 @@ void __init arm64_memblock_init(void)
> }
>
> early_init_fdt_scan_reserved_mem();
> -
> - if (!IS_ENABLED(CONFIG_ZONE_DMA) && !IS_ENABLED(CONFIG_ZONE_DMA32))
> - reserve_crashkernel();
> -
> high_memory = __va(memblock_end_of_DRAM() - 1) + 1;
> }
>
> @@ -438,8 +435,7 @@ void __init bootmem_init(void)
> * request_standard_resources() depends on crashkernel's memory being
> * reserved, so do it here.
> */
> - if (IS_ENABLED(CONFIG_ZONE_DMA) || IS_ENABLED(CONFIG_ZONE_DMA32))
> - reserve_crashkernel();
> + reserve_crashkernel();
>
> memblock_dump_all();
> }
> diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
> index 626ec32..c832848 100644
> --- a/arch/arm64/mm/mmu.c
> +++ b/arch/arm64/mm/mmu.c
> @@ -42,6 +42,7 @@
> #define NO_BLOCK_MAPPINGS BIT(0)
> #define NO_CONT_MAPPINGS BIT(1)
> #define NO_EXEC_MAPPINGS BIT(2) /* assumes FEAT_HPDS is not used */
> +#define NO_SEC_REMAPPINGS BIT(3) /* rebuild with non block/sec mapping*/
>
> u64 idmap_t0sz = TCR_T0SZ(VA_BITS_MIN);
> u64 idmap_ptrs_per_pgd = PTRS_PER_PGD;
> @@ -156,11 +157,12 @@ static bool pgattr_change_is_safe(u64 old, u64 new)
> }
>
> static void init_pte(pmd_t *pmdp, unsigned long addr, unsigned long end,
> - phys_addr_t phys, pgprot_t prot)
> + phys_addr_t phys, pgprot_t prot, int flags)
> {
> pte_t *ptep;
>
> - ptep = pte_set_fixmap_offset(pmdp, addr);
> + ptep = (flags & NO_SEC_REMAPPINGS) ? pte_offset_kernel(pmdp, addr) :
> + pte_set_fixmap_offset(pmdp, addr);
> do {
> pte_t old_pte = READ_ONCE(*ptep);
>
> @@ -176,7 +178,8 @@ static void init_pte(pmd_t *pmdp, unsigned long addr, unsigned long end,
> phys += PAGE_SIZE;
> } while (ptep++, addr += PAGE_SIZE, addr != end);
>
> - pte_clear_fixmap();
> + if (!(flags & NO_SEC_REMAPPINGS))
> + pte_clear_fixmap();
> }
>
> static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
> @@ -208,11 +211,12 @@ static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
> next = pte_cont_addr_end(addr, end);
>
> /* use a contiguous mapping if the range is suitably aligned */
> - if ((((addr | next | phys) & ~CONT_PTE_MASK) == 0) &&
> + if (!(flags & NO_SEC_REMAPPINGS) &&
> + (((addr | next | phys) & ~CONT_PTE_MASK) == 0) &&
> (flags & NO_CONT_MAPPINGS) == 0)
> __prot = __pgprot(pgprot_val(prot) | PTE_CONT);
>
> - init_pte(pmdp, addr, next, phys, __prot);
> + init_pte(pmdp, addr, next, phys, __prot, flags);
>
> phys += next - addr;
> } while (addr = next, addr != end);
> @@ -224,15 +228,44 @@ static void init_pmd(pud_t *pudp, unsigned long addr, unsigned long end,
> {
> unsigned long next;
> pmd_t *pmdp;
> + phys_addr_t map_offset;
> + pmdval_t pmdval;
>
> - pmdp = pmd_set_fixmap_offset(pudp, addr);
> + pmdp = (flags & NO_SEC_REMAPPINGS) ? pmd_offset(pudp, addr) :
> + pmd_set_fixmap_offset(pudp, addr);
> do {
> pmd_t old_pmd = READ_ONCE(*pmdp);
>
> next = pmd_addr_end(addr, end);
>
> - /* try section mapping first */
> - if (((addr | next | phys) & ~PMD_MASK) == 0 &&
> + if (flags & NO_SEC_REMAPPINGS) {
> + if (!pmd_none(*pmdp) && pmd_sect(*pmdp)) {
> + phys_addr_t pte_phys = pgtable_alloc(PAGE_SHIFT);
> + pmd_clear(pmdp);
> + pmdval = PMD_TYPE_TABLE | PMD_TABLE_UXN;
> + if (flags & NO_EXEC_MAPPINGS)
> + pmdval |= PMD_TABLE_PXN;
> + __pmd_populate(pmdp, pte_phys, pmdval);
> + flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
> +
> + map_offset = addr - (addr & PMD_MASK);
> + if (map_offset)
> + alloc_init_cont_pte(pmdp, addr & PMD_MASK, addr,
> + phys - map_offset, prot,
> + pgtable_alloc,
> + flags & (~NO_SEC_REMAPPINGS));
> +
> + if (next < (addr & PMD_MASK) + PMD_SIZE)
> + alloc_init_cont_pte(pmdp, next,
> + (addr & PUD_MASK) + PUD_SIZE,
> + next - addr + phys,
> + prot, pgtable_alloc,
> + flags & (~NO_SEC_REMAPPINGS));
> + }
> + alloc_init_cont_pte(pmdp, addr, next, phys, prot,
> + pgtable_alloc, flags);
> + }/* try section mapping first */
> + else if (((addr | next | phys) & ~PMD_MASK) == 0 &&
> (flags & NO_BLOCK_MAPPINGS) == 0) {
> pmd_set_huge(pmdp, phys, prot);
>
> @@ -252,7 +285,8 @@ static void init_pmd(pud_t *pudp, unsigned long addr, unsigned long end,
> phys += next - addr;
> } while (pmdp++, addr = next, addr != end);
>
> - pmd_clear_fixmap();
> + if (!(flags & NO_SEC_REMAPPINGS))
> + pmd_clear_fixmap();
> }
>
> static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
> @@ -286,7 +320,8 @@ static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
> next = pmd_cont_addr_end(addr, end);
>
> /* use a contiguous mapping if the range is suitably aligned */
> - if ((((addr | next | phys) & ~CONT_PMD_MASK) == 0) &&
> + if (!(flags & NO_SEC_REMAPPINGS) &&
> + (((addr | next | phys) & ~CONT_PMD_MASK) == 0) &&
> (flags & NO_CONT_MAPPINGS) == 0)
> __prot = __pgprot(pgprot_val(prot) | PTE_CONT);
>
> @@ -301,7 +336,9 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
> phys_addr_t (*pgtable_alloc)(int),
> int flags)
> {
> + phys_addr_t map_offset;
> unsigned long next;
> + pudval_t pudval;
> pud_t *pudp;
> p4d_t *p4dp = p4d_offset(pgdp, addr);
> p4d_t p4d = READ_ONCE(*p4dp);
> @@ -325,18 +362,48 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
> */
> if (system_state != SYSTEM_BOOTING)
> mutex_lock(&fixmap_lock);
> - pudp = pud_set_fixmap_offset(p4dp, addr);
> +
> + pudp = (flags & NO_SEC_REMAPPINGS) ? pud_offset(p4dp, addr) :
> + pud_set_fixmap_offset(p4dp, addr);
> do {
> pud_t old_pud = READ_ONCE(*pudp);
> -
> next = pud_addr_end(addr, end);
>
> + if (flags & NO_SEC_REMAPPINGS) {
> + if (!pud_none(*pudp) && pud_sect(*pudp)) {
> + phys_addr_t pmd_phys = pgtable_alloc(PMD_SHIFT);
> + pud_clear(pudp);
> + pudval = PUD_TYPE_TABLE | PUD_TABLE_UXN;
> + if (flags & NO_EXEC_MAPPINGS)
> + pudval |= PUD_TABLE_PXN;
> +
> + __pud_populate(pudp, pmd_phys, pudval);
> + flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
> +
> + map_offset = addr - (addr & PUD_MASK);
> + if (map_offset)
> + alloc_init_cont_pmd(pudp, addr & PUD_MASK,
> + addr, phys - map_offset,
> + prot, pgtable_alloc,
> + flags & (~NO_SEC_REMAPPINGS));
> +
> + if (next < (addr & PUD_MASK) + PUD_SIZE)
> + alloc_init_cont_pmd(pudp, next,
> + (addr & PUD_MASK) +
> + PUD_SIZE,
> + next - addr + phys,
> + prot, pgtable_alloc,
> + flags & (~NO_SEC_REMAPPINGS));
> + }
> + alloc_init_cont_pmd(pudp, addr, next, phys, prot,
> + pgtable_alloc, flags);
> + }
> /*
> * For 4K granule only, attempt to put down a 1GB block
> */
> - if (pud_sect_supported() &&
> - ((addr | next | phys) & ~PUD_MASK) == 0 &&
> - (flags & NO_BLOCK_MAPPINGS) == 0) {
> + else if (pud_sect_supported() &&
> + ((addr | next | phys) & ~PUD_MASK) == 0 &&
> + (flags & NO_BLOCK_MAPPINGS) == 0) {
> pud_set_huge(pudp, phys, prot);
>
> /*
> @@ -355,7 +422,8 @@ static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
> phys += next - addr;
> } while (pudp++, addr = next, addr != end);
>
> - pud_clear_fixmap();
> + if (!(flags & NO_SEC_REMAPPINGS))
> + pud_clear_fixmap();
> if (system_state != SYSTEM_BOOTING)
> mutex_unlock(&fixmap_lock);
> }
> @@ -483,20 +551,39 @@ void __init mark_linear_text_alias_ro(void)
> PAGE_KERNEL_RO);
> }
>
> -static bool crash_mem_map __initdata;
> +#ifdef CONFIG_KEXEC_CORE
> +static phys_addr_t __init early_crashkernel_pgtable_alloc(int shift)
> +{
> + phys_addr_t phys;
> + void *ptr;
>
> -static int __init enable_crash_mem_map(char *arg)
> + phys = memblock_phys_alloc_range(PAGE_SIZE, PAGE_SIZE, 0,
> + MEMBLOCK_ALLOC_NOLEAKTRACE);
> + if (!phys)
> + panic("Failed to allocate page table page\n");
> +
> + ptr = (void *)__phys_to_virt(phys);
> + memset(ptr, 0, PAGE_SIZE);
> + return phys;
> +}
> +
> +void __init map_crashkernel(void)
> {
> - /*
> - * Proper parameter parsing is done by reserve_crashkernel(). We only
> - * need to know if the linear map has to avoid block mappings so that
> - * the crashkernel reservations can be unmapped later.
> - */
> - crash_mem_map = true;
> + if (can_set_direct_map() || IS_ENABLED(CONFIG_KFENCE))
> + return;
>
> - return 0;
> + if (!crashk_res.end)
> + return;
> +
> + __create_pgd_mapping(swapper_pg_dir, crashk_res.start,
> + __phys_to_virt(crashk_res.start),
> + crashk_res.end + 1 - crashk_res.start, PAGE_KERNEL,
> + early_crashkernel_pgtable_alloc,
> + NO_EXEC_MAPPINGS | NO_SEC_REMAPPINGS);
> }
> -early_param("crashkernel", enable_crash_mem_map);
> +#else
> +void __init map_crashkernel(void) {}
> +#endif
>
> static void __init map_mem(pgd_t *pgdp)
> {
> @@ -527,17 +614,6 @@ static void __init map_mem(pgd_t *pgdp)
> */
> memblock_mark_nomap(kernel_start, kernel_end - kernel_start);
>
> -#ifdef CONFIG_KEXEC_CORE
> - if (crash_mem_map) {
> - if (IS_ENABLED(CONFIG_ZONE_DMA) ||
> - IS_ENABLED(CONFIG_ZONE_DMA32))
> - flags |= NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS;
> - else if (crashk_res.end)
> - memblock_mark_nomap(crashk_res.start,
> - resource_size(&crashk_res));
> - }
> -#endif
> -
> /* map all the memory banks */
> for_each_mem_range(i, &start, &end) {
> if (start >= end)
> @@ -570,19 +646,6 @@ static void __init map_mem(pgd_t *pgdp)
> * in page granularity and put back unused memory to buddy system
> * through /sys/kernel/kexec_crash_size interface.
> */
> -#ifdef CONFIG_KEXEC_CORE
> - if (crash_mem_map &&
> - !IS_ENABLED(CONFIG_ZONE_DMA) && !IS_ENABLED(CONFIG_ZONE_DMA32)) {
> - if (crashk_res.end) {
> - __map_memblock(pgdp, crashk_res.start,
> - crashk_res.end + 1,
> - PAGE_KERNEL,
> - NO_BLOCK_MAPPINGS | NO_CONT_MAPPINGS);
> - memblock_clear_nomap(crashk_res.start,
> - resource_size(&crashk_res));
> - }
> - }
> -#endif
> }
>
> void mark_rodata_ro(void)

2022-06-30 21:01:11

by kernel test robot

[permalink] [raw]

Subject: Re: [PATCH v2] arm64: mm: fix linear mapping mem access performance degradation

Hi Guanghui,

Thank you for the patch!
[auto build test WARNING [also build test WARNING [cannot apply to arm64/for-next/core]
wget mkdir build_dir COMPILER_INSTALL_PATH=$HOME/0day
If you fix the issue, Reported-by: kernel
All warnings (new ones
arch/arm64/mm/mmu.c: >> arch/arm64/mm/mmu.c:426:35: 426 | |

vim +/if +426 arch/arm64/mm/mmu.c

d27cfa1fc823d3 Ard Biesheuvel 20a004e7b017cc Will Deacon da141706aea52c Laura Abbott 90292aca9854a2 Yu Zhao c0951366d4b7e0 Ard Biesheuvel c1cc1552616d0f Catalin Marinas 168f0d56f0a26c Guanghui Feng c1cc1552616d0f Catalin Marinas 168f0d56f0a26c Guanghui Feng 20a004e7b017cc Will Deacon e9f6376858b979 Mike Rapoport e9f6376858b979 Mike Rapoport c1cc1552616d0f Catalin Marinas e9f6376858b979 Mike Rapoport 87143f404f338d Ard Biesheuvel 132233a759580f Laura Abbott 87143f404f338d Ard Biesheuvel 87143f404f338d Ard Biesheuvel 87143f404f338d Ard Biesheuvel 132233a759580f Laura Abbott 90292aca9854a2 Yu Zhao 87143f404f338d Ard Biesheuvel e9f6376858b979 Mike Rapoport c79b954bf6c006 Jungseok Lee e9f6376858b979 Mike Rapoport c79b954bf6c006 Jungseok Lee ee017ee353506f Jianyong Wu ee017ee353506f Jianyong Wu ee017ee353506f Jianyong Wu ee017ee353506f Jianyong Wu ee017ee353506f Jianyong Wu ee017ee353506f Jianyong Wu 168f0d56f0a26c Guanghui Feng 168f0d56f0a26c Guanghui Feng 168f0d56f0a26c Guanghui Feng c1cc1552616d0f Catalin Marinas 20a004e7b017cc Will Deacon c1cc1552616d0f Catalin Marinas 206a2a73a62d37 Steve Capper 168f0d56f0a26c Guanghui Feng 168f0d56f0a26c Guanghui Feng 168f0d56f0a26c Guanghui Feng 168f0d56f0a26c Guanghui Feng 168f0d56f0a26c Guanghui Feng 168f0d56f0a26c Guanghui Feng 168f0d56f0a26c Guanghui Feng 168f0d56f0a26c Guanghui Feng 168f0d56f0a26c Guanghui Feng 168f0d56f0a26c Guanghui Feng 168f0d56f0a26c Guanghui Feng 168f0d56f0a26c Guanghui Feng 168f0d56f0a26c Guanghui Feng 168f0d56f0a26c Guanghui Feng 168f0d56f0a26c Guanghui Feng 168f0d56f0a26c Guanghui Feng 168f0d56f0a26c Guanghui Feng 168f0d56f0a26c Guanghui Feng 168f0d56f0a26c Guanghui Feng 168f0d56f0a26c Guanghui Feng 168f0d56f0a26c Guanghui Feng 168f0d56f0a26c Guanghui Feng 168f0d56f0a26c Guanghui Feng 168f0d56f0a26c Guanghui Feng 168f0d56f0a26c Guanghui Feng 168f0d56f0a26c Guanghui Feng 168f0d56f0a26c Guanghui Feng 168f0d56f0a26c Guanghui Feng 168f0d56f0a26c Guanghui Feng 206a2a73a62d37 Steve Capper 206a2a73a62d37 Steve Capper 206a2a73a62d37 Steve Capper 168f0d56f0a26c Guanghui Feng 1310222c276b79 Anshuman c0951366d4b7e0 Ard Biesheuvel 20a004e7b017cc Will Deacon 206a2a73a62d37 Steve Capper 206a2a73a62d37 Steve Capper e98216b52176ba Ard Biesheuvel e98216b52176ba Ard Biesheuvel 206a2a73a62d37 Steve Capper e98216b52176ba Ard Biesheuvel 20a004e7b017cc Will Deacon 206a2a73a62d37 Steve Capper 20a004e7b017cc Will Deacon c0951366d4b7e0 Ard Biesheuvel e98216b52176ba Ard Biesheuvel e98216b52176ba Ard Biesheuvel 20a004e7b017cc Will Deacon 206a2a73a62d37 Steve Capper c1cc1552616d0f Catalin Marinas 20a004e7b017cc Will Deacon f4710445458c0a Mark Rutland 168f0d56f0a26c Guanghui Feng f4710445458c0a Mark Rutland ee017ee353506f Jianyong Wu ee017ee353506f Jianyong Wu c1cc1552616d0f Catalin Marinas c1cc1552616d0f Catalin Marinas
--
0-DAY CI Kernel Test Service
https://git-scm.com/docs/git-format-patch]
thub.com/intel-lab-lkp/linux/commits/Guanghui-Feng/arm64-mm-fix-linear-mapping-mem-access-performance-degradation/20220630-163924">https://github.com/intel-lab-lkp/linux/commits/Guanghui-Feng/arm64-mm-fix-linear-mapping-mem-access-performance-degradation/20220630-163924
76b086c76f7a612856a9a
href="https://download.01.org/0day-ci/archive/20220701/202207010409.ajCVDXYO-lkp@intel.com/config)">https://download.01.org/0day-ci/archive/20220701/[email protected]/config)
(GCC) 11.3.0
://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross">https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
github.com/intel-lab-lkp/linux/commit/168f0d56f0a26c4f7a9470d6f1c398b4e1c1b5b9">https://github.com/intel-lab-lkp/linux/commit/168f0d56f0a26c4f7a9470d6f1c398b4e1c1b5b9
href="https://github.com/intel-lab-lkp/linux">https://github.com/intel-lab-lkp/linux
linux-review Guanghui-Feng/arm64-mm-fix-linear-mapping-mem-access-performance-degradation/20220630-163924
168f0d56f0a26c4f7a9470d6f1c398b4e1c1b5b9
&& cp config build_dir/.config
COMPILER=gcc-11.3.0 make.cross W=1 O=build_dir ARCH=arm64 SHELL=/bin/bash arch/arm64/mm/
kindly add following tag where applicable
test robot <[email protected]>
prefixed by >>):
In function 'alloc_init_pud':
warning: suggest braces around empty body in an 'if' statement [-Wempty-body]
pud_clear_fixmap();
^
2017-03-09 333
2018-02-15 334 static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
2015-01-21 335 phys_addr_t phys, pgprot_t prot,
2019-03-11 336 phys_addr_t (*pgtable_alloc)(int),
2017-03-09 337 int flags)
2012-03-05 338 {
2022-06-30 339 phys_addr_t map_offset;
2012-03-05 340 unsigned long next;
2022-06-30 341 pudval_t pudval;
2018-02-15 342 pud_t *pudp;
2020-06-04 343 p4d_t *p4dp = p4d_offset(pgdp, addr);
2020-06-04 344 p4d_t p4d = READ_ONCE(*p4dp);
2012-03-05 345
2020-06-04 346 if (p4d_none(p4d)) {
2021-03-10 347 p4dval_t p4dval = P4D_TYPE_TABLE | P4D_TABLE_UXN;
2016-02-05 348 phys_addr_t pud_phys;
2021-03-10 349
2021-03-10 350 if (flags & NO_EXEC_MAPPINGS)
2021-03-10 351 p4dval |= P4D_TABLE_PXN;
2016-02-05 352 BUG_ON(!pgtable_alloc);
2019-03-11 353 pud_phys = pgtable_alloc(PUD_SHIFT);
2021-03-10 354 __p4d_populate(p4dp, pud_phys, p4dval);
2020-06-04 355 p4d = READ_ONCE(*p4dp);
2014-05-12 356 }
2020-06-04 357 BUG_ON(p4d_bad(p4d));
2014-05-12 358
2022-02-01 359 /*
2022-02-01 360 * No need for locking during early boot. And it doesn't work as
2022-02-01 361 * expected with KASLR enabled.
2022-02-01 362 */
2022-02-01 363 if (system_state != SYSTEM_BOOTING)
2022-02-01 364 mutex_lock(&fixmap_lock);
2022-06-30 365
2022-06-30 366 pudp = (flags & NO_SEC_REMAPPINGS) ? pud_offset(p4dp, addr) :
2022-06-30 367 pud_set_fixmap_offset(p4dp, addr);
2012-03-05 368 do {
2018-02-15 369 pud_t old_pud = READ_ONCE(*pudp);
2012-03-05 370 next = pud_addr_end(addr, end);
2014-05-06 371
2022-06-30 372 if (flags & NO_SEC_REMAPPINGS) {
2022-06-30 373 if (!pud_none(*pudp) && pud_sect(*pudp)) {
2022-06-30 374 phys_addr_t pmd_phys = pgtable_alloc(PMD_SHIFT);
2022-06-30 375 pud_clear(pudp);
2022-06-30 376 pudval = PUD_TYPE_TABLE | PUD_TABLE_UXN;
2022-06-30 377 if (flags & NO_EXEC_MAPPINGS)
2022-06-30 378 pudval |= PUD_TABLE_PXN;
2022-06-30 379
2022-06-30 380 __pud_populate(pudp, pmd_phys, pudval);
2022-06-30 381 flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
2022-06-30 382
2022-06-30 383 map_offset = addr - (addr & PUD_MASK);
2022-06-30 384 if (map_offset)
2022-06-30 385 alloc_init_cont_pmd(pudp, addr & PUD_MASK,
2022-06-30 386 addr, phys - map_offset,
2022-06-30 387 prot, pgtable_alloc,
2022-06-30 388 flags & (~NO_SEC_REMAPPINGS));
2022-06-30 389
2022-06-30 390 if (next < (addr & PUD_MASK) + PUD_SIZE)
2022-06-30 391 alloc_init_cont_pmd(pudp, next,
2022-06-30 392 (addr & PUD_MASK) +
2022-06-30 393 PUD_SIZE,
2022-06-30 394 next - addr + phys,
2022-06-30 395 prot, pgtable_alloc,
2022-06-30 396 flags & (~NO_SEC_REMAPPINGS));
2022-06-30 397 }
2022-06-30 398 alloc_init_cont_pmd(pudp, addr, next, phys, prot,
2022-06-30 399 pgtable_alloc, flags);
2022-06-30 400 }
2014-05-06 401 /*
2014-05-06 402 * For 4K granule only, attempt to put down a 1GB block
2014-05-06 403 */
2022-06-30 404 else if (pud_sect_supported() &&
Khandual 2022-02-16 405 ((addr | next | phys) & ~PUD_MASK) == 0 &&
2017-03-09 406 (flags & NO_BLOCK_MAPPINGS) == 0) {
2018-02-15 407 pud_set_huge(pudp, phys, prot);
2014-05-06 408
2014-05-06 409 /*
2016-10-21 410 * After the PUD entry has been populated once, we
2016-10-21 411 * only allow updates to the permission attributes.
2014-05-06 412 */
2016-10-21 413 BUG_ON(!pgattr_change_is_safe(pud_val(old_pud),
2018-02-15 414 READ_ONCE(pud_val(*pudp))));
2014-05-06 415 } else {
2018-02-15 416 alloc_init_cont_pmd(pudp, addr, next, phys, prot,
2017-03-09 417 pgtable_alloc, flags);
2016-10-21 418
2016-10-21 419 BUG_ON(pud_val(old_pud) != 0 &&
2018-02-15 420 pud_val(old_pud) != READ_ONCE(pud_val(*pudp)));
2014-05-06 421 }
2012-03-05 422 phys += next - addr;
2018-02-15 423 } while (pudp++, addr = next, addr != end);
2016-01-25 424
2022-06-30 425 if (!(flags & NO_SEC_REMAPPINGS))
2016-01-25 @426 pud_clear_fixmap();
2022-02-01 427 if (system_state != SYSTEM_BOOTING)
2022-02-01 428 mutex_unlock(&fixmap_lock);
2012-03-05 429 }
2012-03-05 430
">https://01.org/lkp