Hi,
this series implements highmem support on xtensa cores with aliasing cache.
It does so by making sure that high memory pages are always mapped at
virtual addresses with color that match color of their physical address.
This involves changing the generic kmap code to make it aware of cache
coloring. This part with corresponding arch changes is cc'd linux-mm,
linux-arch and linux-mips.
The whole series can also be found at:
git://github.com/jcmvbkbc/linux-xtensa.git xtensa-highmem-ca
Leonid Yegoshin (1):
mm/highmem: make kmap cache coloring aware
Max Filippov (7):
xtensa: make fixmap region addressing grow with index
xtensa: allow fixmap and kmap span more than one page table
xtensa: fix TLBTEMP_BASE_2 region handling in fast_second_level_miss
xtensa: implement clear_user_highpage and copy_user_highpage
xtensa: support aliasing cache in k[un]map_atomic
xtensa: support aliasing cache in kmap
xtensa: support highmem in aliasing cache flushing code
arch/xtensa/include/asm/cacheflush.h | 2 +
arch/xtensa/include/asm/fixmap.h | 30 +++++++--
arch/xtensa/include/asm/highmem.h | 18 +++++-
arch/xtensa/include/asm/page.h | 14 ++++-
arch/xtensa/include/asm/pgtable.h | 7 ++-
arch/xtensa/kernel/entry.S | 2 +-
arch/xtensa/mm/cache.c | 77 ++++++++++++++++++++---
arch/xtensa/mm/highmem.c | 24 +++++---
arch/xtensa/mm/misc.S | 116 ++++++++++++++++-------------------
arch/xtensa/mm/mmu.c | 38 +++++++-----
mm/highmem.c | 19 +++++-
11 files changed, 235 insertions(+), 112 deletions(-)
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Cc: David Rientjes <[email protected]>
--
1.8.1.4
To support aliasing cache both kmap region sizes are multiplied by the
number of data cache colors. After that expansion page tables that cover
kmap regions may become larger than one page. Correctly allocate and
initialize page tables in this case.
Signed-off-by: Max Filippov <[email protected]>
---
arch/xtensa/mm/mmu.c | 38 ++++++++++++++++++++++----------------
1 file changed, 22 insertions(+), 16 deletions(-)
diff --git a/arch/xtensa/mm/mmu.c b/arch/xtensa/mm/mmu.c
index 3429b48..abe4513 100644
--- a/arch/xtensa/mm/mmu.c
+++ b/arch/xtensa/mm/mmu.c
@@ -18,32 +18,38 @@
#include <asm/io.h>
#if defined(CONFIG_HIGHMEM)
-static void * __init init_pmd(unsigned long vaddr)
+static void * __init init_pmd(unsigned long vaddr, unsigned long n_pages)
{
pgd_t *pgd = pgd_offset_k(vaddr);
pmd_t *pmd = pmd_offset(pgd, vaddr);
+ pte_t *pte;
+ unsigned long i;
- if (pmd_none(*pmd)) {
- unsigned i;
- pte_t *pte = alloc_bootmem_low_pages(PAGE_SIZE);
+ n_pages = ALIGN(n_pages, PTRS_PER_PTE);
- for (i = 0; i < 1024; i++)
- pte_clear(NULL, 0, pte + i);
+ pr_debug("%s: vaddr: 0x%08lx, n_pages: %ld\n",
+ __func__, vaddr, n_pages);
- set_pmd(pmd, __pmd(((unsigned long)pte) & PAGE_MASK));
- BUG_ON(pte != pte_offset_kernel(pmd, 0));
- pr_debug("%s: vaddr: 0x%08lx, pmd: 0x%p, pte: 0x%p\n",
- __func__, vaddr, pmd, pte);
- return pte;
- } else {
- return pte_offset_kernel(pmd, 0);
+ pte = alloc_bootmem_low_pages(n_pages * sizeof(pte_t));
+
+ for (i = 0; i < n_pages; ++i)
+ pte_clear(NULL, 0, pte + i);
+
+ for (i = 0; i < n_pages; i += PTRS_PER_PTE, ++pmd) {
+ pte_t *cur_pte = pte + i;
+
+ BUG_ON(!pmd_none(*pmd));
+ set_pmd(pmd, __pmd(((unsigned long)cur_pte) & PAGE_MASK));
+ BUG_ON(cur_pte != pte_offset_kernel(pmd, 0));
+ pr_debug("%s: pmd: 0x%p, pte: 0x%p\n",
+ __func__, pmd, cur_pte);
}
+ return pte;
}
static void __init fixedrange_init(void)
{
- BUILD_BUG_ON(FIXADDR_SIZE > PMD_SIZE);
- init_pmd(__fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK);
+ init_pmd(__fix_to_virt(0), __end_of_fixed_addresses);
}
#endif
@@ -52,7 +58,7 @@ void __init paging_init(void)
memset(swapper_pg_dir, 0, PAGE_SIZE);
#ifdef CONFIG_HIGHMEM
fixedrange_init();
- pkmap_page_table = init_pmd(PKMAP_BASE);
+ pkmap_page_table = init_pmd(PKMAP_BASE, LAST_PKMAP);
kmap_init();
#endif
}
--
1.8.1.4
Existing clear_user_page and copy_user_page cannot be used with highmem
because they calculate physical page address from its virtual address
and do it incorrectly in case of high memory page mapped with
kmap_atomic. Also kmap is not needed, as most likely userspace mapping
color would be different from the kmapped color.
Provide clear_user_highpage and copy_user_highpage functions that
determine if temporary mapping is needed for the pages. Move most of the
logic of the former clear_user_page and copy_user_page to
xtensa/mm/cache.c only leaving temporary mapping setup, invalidation and
clearing/copying in the xtensa/mm/misc.S. Rename these functions to
clear_page_alias and copy_page_alias.
Signed-off-by: Max Filippov <[email protected]>
---
arch/xtensa/include/asm/cacheflush.h | 2 +
arch/xtensa/include/asm/page.h | 12 +++-
arch/xtensa/mm/cache.c | 63 +++++++++++++++++++
arch/xtensa/mm/misc.S | 116 ++++++++++++++++-------------------
4 files changed, 127 insertions(+), 66 deletions(-)
diff --git a/arch/xtensa/include/asm/cacheflush.h b/arch/xtensa/include/asm/cacheflush.h
index 555a98a..e72aaca 100644
--- a/arch/xtensa/include/asm/cacheflush.h
+++ b/arch/xtensa/include/asm/cacheflush.h
@@ -37,6 +37,7 @@
* specials for cache aliasing:
*
* __flush_invalidate_dcache_page_alias(vaddr,paddr)
+ * __invalidate_dcache_page_alias(vaddr,paddr)
* __invalidate_icache_page_alias(vaddr,paddr)
*/
@@ -62,6 +63,7 @@ extern void __flush_invalidate_dcache_range(unsigned long, unsigned long);
#if defined(CONFIG_MMU) && (DCACHE_WAY_SIZE > PAGE_SIZE)
extern void __flush_invalidate_dcache_page_alias(unsigned long, unsigned long);
+extern void __invalidate_dcache_page_alias(unsigned long, unsigned long);
#else
static inline void __flush_invalidate_dcache_page_alias(unsigned long virt,
unsigned long phys) { }
diff --git a/arch/xtensa/include/asm/page.h b/arch/xtensa/include/asm/page.h
index 47f5823..11721cc 100644
--- a/arch/xtensa/include/asm/page.h
+++ b/arch/xtensa/include/asm/page.h
@@ -134,6 +134,7 @@ static inline __attribute_const__ int get_order(unsigned long size)
#endif
struct page;
+struct vm_area_struct;
extern void clear_page(void *page);
extern void copy_page(void *to, void *from);
@@ -143,8 +144,15 @@ extern void copy_page(void *to, void *from);
*/
#if DCACHE_WAY_SIZE > PAGE_SIZE
-extern void clear_user_page(void*, unsigned long, struct page*);
-extern void copy_user_page(void*, void*, unsigned long, struct page*);
+extern void clear_page_alias(void *vaddr, unsigned long paddr);
+extern void copy_page_alias(void *to, void *from,
+ unsigned long to_paddr, unsigned long from_paddr);
+
+#define clear_user_highpage clear_user_highpage
+void clear_user_highpage(struct page *page, unsigned long vaddr);
+#define __HAVE_ARCH_COPY_USER_HIGHPAGE
+void copy_user_highpage(struct page *to, struct page *from,
+ unsigned long vaddr, struct vm_area_struct *vma);
#else
# define clear_user_page(page, vaddr, pg) clear_page(page)
# define copy_user_page(to, from, vaddr, pg) copy_page(to, from)
diff --git a/arch/xtensa/mm/cache.c b/arch/xtensa/mm/cache.c
index 63cbb86..96aea66 100644
--- a/arch/xtensa/mm/cache.c
+++ b/arch/xtensa/mm/cache.c
@@ -63,6 +63,69 @@
#error "HIGHMEM is not supported on cores with aliasing cache."
#endif
+#if (DCACHE_WAY_SIZE > PAGE_SIZE)
+static inline void kmap_invalidate_coherent(struct page *page,
+ unsigned long vaddr)
+{
+ if (!DCACHE_ALIAS_EQ(page_to_phys(page), vaddr)) {
+ unsigned long kvaddr;
+
+ if (!PageHighMem(page)) {
+ kvaddr = (unsigned long)page_to_virt(page);
+
+ __invalidate_dcache_page(kvaddr);
+ } else {
+ kvaddr = TLBTEMP_BASE_1 +
+ (page_to_phys(page) & DCACHE_ALIAS_MASK);
+
+ __invalidate_dcache_page_alias(kvaddr,
+ page_to_phys(page));
+ }
+ }
+}
+
+static inline void *coherent_kvaddr(struct page *page, unsigned long base,
+ unsigned long vaddr, unsigned long *paddr)
+{
+ if (PageHighMem(page) || !DCACHE_ALIAS_EQ(page_to_phys(page), vaddr)) {
+ *paddr = page_to_phys(page);
+ return (void *)(base + (vaddr & DCACHE_ALIAS_MASK));
+ } else {
+ *paddr = 0;
+ return page_to_virt(page);
+ }
+}
+
+void clear_user_highpage(struct page *page, unsigned long vaddr)
+{
+ unsigned long paddr;
+ void *kvaddr = coherent_kvaddr(page, TLBTEMP_BASE_1, vaddr, &paddr);
+
+ pagefault_disable();
+ kmap_invalidate_coherent(page, vaddr);
+ set_bit(PG_arch_1, &page->flags);
+ clear_page_alias(kvaddr, paddr);
+ pagefault_enable();
+}
+
+void copy_user_highpage(struct page *dst, struct page *src,
+ unsigned long vaddr, struct vm_area_struct *vma)
+{
+ unsigned long dst_paddr, src_paddr;
+ void *dst_vaddr = coherent_kvaddr(dst, TLBTEMP_BASE_1, vaddr,
+ &dst_paddr);
+ void *src_vaddr = coherent_kvaddr(src, TLBTEMP_BASE_2, vaddr,
+ &src_paddr);
+
+ pagefault_disable();
+ kmap_invalidate_coherent(dst, vaddr);
+ set_bit(PG_arch_1, &dst->flags);
+ copy_page_alias(dst_vaddr, src_vaddr, dst_paddr, src_paddr);
+ pagefault_enable();
+}
+
+#endif /* DCACHE_WAY_SIZE > PAGE_SIZE */
+
#if (DCACHE_WAY_SIZE > PAGE_SIZE) && XCHAL_DCACHE_IS_WRITEBACK
/*
diff --git a/arch/xtensa/mm/misc.S b/arch/xtensa/mm/misc.S
index 1f68558..11a01c3 100644
--- a/arch/xtensa/mm/misc.S
+++ b/arch/xtensa/mm/misc.S
@@ -110,41 +110,24 @@ ENTRY(__tlbtemp_mapping_start)
#if (DCACHE_WAY_SIZE > PAGE_SIZE)
/*
- * clear_user_page (void *addr, unsigned long vaddr, struct page *page)
- * a2 a3 a4
+ * clear_page_alias(void *addr, unsigned long paddr)
+ * a2 a3
*/
-ENTRY(clear_user_page)
+ENTRY(clear_page_alias)
entry a1, 32
- /* Mark page dirty and determine alias. */
+ /* Skip setting up a temporary DTLB if not aliased low page. */
- movi a7, (1 << PG_ARCH_1)
- l32i a5, a4, PAGE_FLAGS
- xor a6, a2, a3
- extui a3, a3, PAGE_SHIFT, DCACHE_ALIAS_ORDER
- extui a6, a6, PAGE_SHIFT, DCACHE_ALIAS_ORDER
- or a5, a5, a7
- slli a3, a3, PAGE_SHIFT
- s32i a5, a4, PAGE_FLAGS
+ movi a5, PAGE_OFFSET
+ movi a6, 0
+ beqz a3, 1f
- /* Skip setting up a temporary DTLB if not aliased. */
-
- beqz a6, 1f
-
- /* Invalidate kernel page. */
-
- mov a10, a2
- call8 __invalidate_dcache_page
-
- /* Setup a temporary DTLB with the color of the VPN */
-
- movi a4, ((PAGE_KERNEL | _PAGE_HW_WRITE) - PAGE_OFFSET) & 0xffffffff
- movi a5, TLBTEMP_BASE_1 # virt
- add a6, a2, a4 # ppn
- add a2, a5, a3 # add 'color'
+ /* Setup a temporary DTLB for the addr. */
+ addi a6, a3, (PAGE_KERNEL | _PAGE_HW_WRITE)
+ mov a4, a2
wdtlb a6, a2
dsync
@@ -165,62 +148,43 @@ ENTRY(clear_user_page)
/* We need to invalidate the temporary idtlb entry, if any. */
-1: addi a2, a2, -PAGE_SIZE
- idtlb a2
+1: idtlb a4
dsync
retw
-ENDPROC(clear_user_page)
+ENDPROC(clear_page_alias)
/*
- * copy_page_user (void *to, void *from, unsigned long vaddr, struct page *page)
- * a2 a3 a4 a5
+ * copy_page_alias(void *to, void *from,
+ * a2 a3
+ * unsigned long to_paddr, unsigned long from_paddr)
+ * a4 a5
*/
-ENTRY(copy_user_page)
+ENTRY(copy_page_alias)
entry a1, 32
- /* Mark page dirty and determine alias for destination. */
-
- movi a8, (1 << PG_ARCH_1)
- l32i a9, a5, PAGE_FLAGS
- xor a6, a2, a4
- xor a7, a3, a4
- extui a4, a4, PAGE_SHIFT, DCACHE_ALIAS_ORDER
- extui a6, a6, PAGE_SHIFT, DCACHE_ALIAS_ORDER
- extui a7, a7, PAGE_SHIFT, DCACHE_ALIAS_ORDER
- or a9, a9, a8
- slli a4, a4, PAGE_SHIFT
- s32i a9, a5, PAGE_FLAGS
- movi a5, ((PAGE_KERNEL | _PAGE_HW_WRITE) - PAGE_OFFSET) & 0xffffffff
-
- beqz a6, 1f
-
- /* Invalidate dcache */
-
- mov a10, a2
- call8 __invalidate_dcache_page
+ /* Skip setting up a temporary DTLB for destination if not aliased. */
- /* Setup a temporary DTLB with a matching color. */
+ movi a6, 0
+ movi a7, 0
+ beqz a4, 1f
- movi a8, TLBTEMP_BASE_1 # base
- add a6, a2, a5 # ppn
- add a2, a8, a4 # add 'color'
+ /* Setup a temporary DTLB for destination. */
+ addi a6, a4, (PAGE_KERNEL | _PAGE_HW_WRITE)
wdtlb a6, a2
dsync
- /* Skip setting up a temporary DTLB for destination if not aliased. */
+ /* Skip setting up a temporary DTLB for source if not aliased. */
-1: beqz a7, 1f
+1: beqz a5, 1f
- /* Setup a temporary DTLB with a matching color. */
+ /* Setup a temporary DTLB for source. */
- movi a8, TLBTEMP_BASE_2 # base
- add a7, a3, a5 # ppn
- add a3, a8, a4
+ addi a7, a5, PAGE_KERNEL
addi a8, a3, 1 # way1
wdtlb a7, a8
@@ -271,7 +235,7 @@ ENTRY(copy_user_page)
retw
-ENDPROC(copy_user_page)
+ENDPROC(copy_page_alias)
#endif
@@ -300,6 +264,30 @@ ENTRY(__flush_invalidate_dcache_page_alias)
retw
ENDPROC(__flush_invalidate_dcache_page_alias)
+
+/*
+ * void __invalidate_dcache_page_alias (addr, phys)
+ * a2 a3
+ */
+
+ENTRY(__invalidate_dcache_page_alias)
+
+ entry sp, 16
+
+ movi a7, 0 # required for exception handler
+ addi a6, a3, (PAGE_KERNEL | _PAGE_HW_WRITE)
+ mov a4, a2
+ wdtlb a6, a2
+ dsync
+
+ ___invalidate_dcache_page a2 a3
+
+ idtlb a4
+ dsync
+
+ retw
+
+ENDPROC(__invalidate_dcache_page_alias)
#endif
ENTRY(__tlbtemp_mapping_itlb)
--
1.8.1.4
From: Leonid Yegoshin <[email protected]>
Provide hooks that allow architectures with aliasing cache to align
mapping address of high pages according to their color. Such architectures
may enforce similar coloring of low- and high-memory page mappings and
reuse existing cache management functions to support highmem.
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Cc: David Rientjes <[email protected]>
Signed-off-by: Leonid Yegoshin <[email protected]>
[ Max: extract architecture-independent part of the original patch, clean
up checkpatch and build warnings. ]
Signed-off-by: Max Filippov <[email protected]>
---
Changes since the initial version:
- define set_pkmap_color(pg, cl) as do { } while (0) instead of /* */;
- rename is_no_more_pkmaps to no_more_pkmaps;
- change 'if (count > 0)' to 'if (count)' to better match the original
code behavior;
mm/highmem.c | 19 ++++++++++++++++---
1 file changed, 16 insertions(+), 3 deletions(-)
diff --git a/mm/highmem.c b/mm/highmem.c
index b32b70c..88fb62e 100644
--- a/mm/highmem.c
+++ b/mm/highmem.c
@@ -44,6 +44,14 @@ DEFINE_PER_CPU(int, __kmap_atomic_idx);
*/
#ifdef CONFIG_HIGHMEM
+#ifndef ARCH_PKMAP_COLORING
+#define set_pkmap_color(pg, cl) do { } while (0)
+#define get_last_pkmap_nr(p, cl) (p)
+#define get_next_pkmap_nr(p, cl) (((p) + 1) & LAST_PKMAP_MASK)
+#define no_more_pkmaps(p, cl) (!(p))
+#define get_next_pkmap_counter(c, cl) ((c) - 1)
+#endif
+
unsigned long totalhigh_pages __read_mostly;
EXPORT_SYMBOL(totalhigh_pages);
@@ -161,19 +169,24 @@ static inline unsigned long map_new_virtual(struct page *page)
{
unsigned long vaddr;
int count;
+ int color __maybe_unused;
+
+ set_pkmap_color(page, color);
+ last_pkmap_nr = get_last_pkmap_nr(last_pkmap_nr, color);
start:
count = LAST_PKMAP;
/* Find an empty entry */
for (;;) {
- last_pkmap_nr = (last_pkmap_nr + 1) & LAST_PKMAP_MASK;
- if (!last_pkmap_nr) {
+ last_pkmap_nr = get_next_pkmap_nr(last_pkmap_nr, color);
+ if (no_more_pkmaps(last_pkmap_nr, color)) {
flush_all_zero_pkmaps();
count = LAST_PKMAP;
}
if (!pkmap_count[last_pkmap_nr])
break; /* Found a usable entry */
- if (--count)
+ count = get_next_pkmap_counter(count, color);
+ if (count)
continue;
/*
--
1.8.1.4
Define ARCH_PKMAP_COLORING and provide corresponding macro definitions
on cores with aliasing data cache.
Instead of single last_pkmap_nr maintain an array last_pkmap_nr_arr of
pkmap counters for each page color. Make sure that kmap maps physical
page at virtual address with color matching its physical address.
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Cc: David Rientjes <[email protected]>
Signed-off-by: Max Filippov <[email protected]>
---
arch/xtensa/include/asm/highmem.h | 18 ++++++++++++++++--
arch/xtensa/mm/highmem.c | 1 +
2 files changed, 17 insertions(+), 2 deletions(-)
diff --git a/arch/xtensa/include/asm/highmem.h b/arch/xtensa/include/asm/highmem.h
index 2653ef5..a5c3380 100644
--- a/arch/xtensa/include/asm/highmem.h
+++ b/arch/xtensa/include/asm/highmem.h
@@ -17,14 +17,28 @@
#include <asm/kmap_types.h>
#include <asm/pgtable.h>
-#define PKMAP_BASE (FIXADDR_START - PMD_SIZE)
-#define LAST_PKMAP PTRS_PER_PTE
+#define PKMAP_BASE ((FIXADDR_START - \
+ (LAST_PKMAP + 1) * PAGE_SIZE) & PMD_MASK)
+#define LAST_PKMAP (PTRS_PER_PTE * DCACHE_N_COLORS)
#define LAST_PKMAP_MASK (LAST_PKMAP - 1)
#define PKMAP_NR(virt) (((virt) - PKMAP_BASE) >> PAGE_SHIFT)
#define PKMAP_ADDR(nr) (PKMAP_BASE + ((nr) << PAGE_SHIFT))
#define kmap_prot PAGE_KERNEL
+#if DCACHE_WAY_SIZE > PAGE_SIZE
+#define ARCH_PKMAP_COLORING
+#define set_pkmap_color(pg, cl) ((cl) = DCACHE_ALIAS(page_to_phys(pg)))
+#define get_last_pkmap_nr(p, cl) (last_pkmap_nr_arr[cl] + (cl))
+#define get_next_pkmap_nr(p, cl) \
+ ((last_pkmap_nr_arr[cl] = ((last_pkmap_nr_arr[cl] + DCACHE_N_COLORS) & \
+ LAST_PKMAP_MASK)) + (cl))
+#define no_more_pkmaps(p, cl) ((p) < DCACHE_N_COLORS)
+#define get_next_pkmap_counter(c, cl) ((c) - DCACHE_N_COLORS)
+
+extern unsigned int last_pkmap_nr_arr[];
+#endif
+
extern pte_t *pkmap_page_table;
void *kmap_high(struct page *page);
diff --git a/arch/xtensa/mm/highmem.c b/arch/xtensa/mm/highmem.c
index 466abae..3742a37 100644
--- a/arch/xtensa/mm/highmem.c
+++ b/arch/xtensa/mm/highmem.c
@@ -12,6 +12,7 @@
#include <linux/highmem.h>
#include <asm/tlbflush.h>
+unsigned int last_pkmap_nr_arr[DCACHE_N_COLORS];
static pte_t *kmap_pte;
static inline enum fixed_addresses kmap_idx(int type, unsigned long color)
--
1.8.1.4
Use __flush_invalidate_dcache_page_alias with alias set to color of the
page physical address instead of __flush_invalidate_dcache_page: this
works for high memory pages and mapping/unmapping to the TLBTEMP area is
virtually free.
Allow building configurations with aliasing cache and highmem enabled.
Signed-off-by: Max Filippov <[email protected]>
---
arch/xtensa/mm/cache.c | 16 ++++++----------
1 file changed, 6 insertions(+), 10 deletions(-)
diff --git a/arch/xtensa/mm/cache.c b/arch/xtensa/mm/cache.c
index 96aea66..d75aa14 100644
--- a/arch/xtensa/mm/cache.c
+++ b/arch/xtensa/mm/cache.c
@@ -59,10 +59,6 @@
*
*/
-#if (DCACHE_WAY_SIZE > PAGE_SIZE) && defined(CONFIG_HIGHMEM)
-#error "HIGHMEM is not supported on cores with aliasing cache."
-#endif
-
#if (DCACHE_WAY_SIZE > PAGE_SIZE)
static inline void kmap_invalidate_coherent(struct page *page,
unsigned long vaddr)
@@ -166,7 +162,8 @@ void flush_dcache_page(struct page *page)
if (!alias && !mapping)
return;
- __flush_invalidate_dcache_page((long)page_address(page));
+ virt = TLBTEMP_BASE_1 + (phys & DCACHE_ALIAS_MASK);
+ __flush_invalidate_dcache_page_alias(virt, phys);
virt = TLBTEMP_BASE_1 + (temp & DCACHE_ALIAS_MASK);
@@ -231,13 +228,12 @@ update_mmu_cache(struct vm_area_struct * vma, unsigned long addr, pte_t *ptep)
#if (DCACHE_WAY_SIZE > PAGE_SIZE) && XCHAL_DCACHE_IS_WRITEBACK
if (!PageReserved(page) && test_bit(PG_arch_1, &page->flags)) {
-
- unsigned long paddr = (unsigned long) page_address(page);
unsigned long phys = page_to_phys(page);
- unsigned long tmp = TLBTEMP_BASE_1 + (addr & DCACHE_ALIAS_MASK);
-
- __flush_invalidate_dcache_page(paddr);
+ unsigned long tmp;
+ tmp = TLBTEMP_BASE_1 + (phys & DCACHE_ALIAS_MASK);
+ __flush_invalidate_dcache_page_alias(tmp, phys);
+ tmp = TLBTEMP_BASE_1 + (addr & DCACHE_ALIAS_MASK);
__flush_invalidate_dcache_page_alias(tmp, phys);
__invalidate_icache_page_alias(tmp, phys);
--
1.8.1.4
Map high memory pages at virtual addresses with color that match color
of their physical address. Existing cache alias management mechanisms
may be used with such pages.
Signed-off-by: Max Filippov <[email protected]>
---
arch/xtensa/include/asm/fixmap.h | 3 ++-
arch/xtensa/include/asm/page.h | 2 ++
arch/xtensa/mm/highmem.c | 17 ++++++++++-------
3 files changed, 14 insertions(+), 8 deletions(-)
diff --git a/arch/xtensa/include/asm/fixmap.h b/arch/xtensa/include/asm/fixmap.h
index a43cd52..62b507d 100644
--- a/arch/xtensa/include/asm/fixmap.h
+++ b/arch/xtensa/include/asm/fixmap.h
@@ -38,7 +38,8 @@ enum fixed_addresses {
#ifdef CONFIG_HIGHMEM
/* reserved pte's for temporary kernel mappings */
FIX_KMAP_BEGIN,
- FIX_KMAP_END = FIX_KMAP_BEGIN + (KM_TYPE_NR * NR_CPUS) - 1,
+ FIX_KMAP_END = FIX_KMAP_BEGIN +
+ (KM_TYPE_NR * NR_CPUS * DCACHE_N_COLORS) - 1,
#endif
__end_of_fixed_addresses
};
diff --git a/arch/xtensa/include/asm/page.h b/arch/xtensa/include/asm/page.h
index 11721cc..abe24c6 100644
--- a/arch/xtensa/include/asm/page.h
+++ b/arch/xtensa/include/asm/page.h
@@ -78,7 +78,9 @@
# define DCACHE_ALIAS_EQ(a,b) ((((a) ^ (b)) & DCACHE_ALIAS_MASK) == 0)
#else
# define DCACHE_ALIAS_ORDER 0
+# define DCACHE_ALIAS(a) ((void)(a), 0)
#endif
+#define DCACHE_N_COLORS (1 << DCACHE_ALIAS_ORDER)
#if ICACHE_WAY_SIZE > PAGE_SIZE
# define ICACHE_ALIAS_ORDER (ICACHE_WAY_SHIFT - PAGE_SHIFT)
diff --git a/arch/xtensa/mm/highmem.c b/arch/xtensa/mm/highmem.c
index 2e95a76..466abae 100644
--- a/arch/xtensa/mm/highmem.c
+++ b/arch/xtensa/mm/highmem.c
@@ -14,18 +14,23 @@
static pte_t *kmap_pte;
+static inline enum fixed_addresses kmap_idx(int type, unsigned long color)
+{
+ return (type + KM_TYPE_NR * smp_processor_id()) * DCACHE_N_COLORS +
+ color;
+}
+
void *kmap_atomic(struct page *page)
{
enum fixed_addresses idx;
unsigned long vaddr;
- int type;
pagefault_disable();
if (!PageHighMem(page))
return page_address(page);
- type = kmap_atomic_idx_push();
- idx = type + KM_TYPE_NR * smp_processor_id();
+ idx = kmap_idx(kmap_atomic_idx_push(),
+ DCACHE_ALIAS(page_to_phys(page)));
vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
#ifdef CONFIG_DEBUG_HIGHMEM
BUG_ON(!pte_none(*(kmap_pte + idx)));
@@ -38,12 +43,10 @@ EXPORT_SYMBOL(kmap_atomic);
void __kunmap_atomic(void *kvaddr)
{
- int idx, type;
-
if (kvaddr >= (void *)FIXADDR_START &&
kvaddr < (void *)FIXADDR_TOP) {
- type = kmap_atomic_idx();
- idx = type + KM_TYPE_NR * smp_processor_id();
+ int idx = kmap_idx(kmap_atomic_idx(),
+ DCACHE_ALIAS((unsigned long)kvaddr));
/*
* Force other mappings to Oops if they'll try to access this
--
1.8.1.4
Current definition of TLBTEMP_BASE_2 is always 32K above the
TLBTEMP_BASE_1, whereas fast_second_level_miss handler for the TLBTEMP
region analyzes virtual address bit (PAGE_SHIFT + DCACHE_ALIAS_ORDER)
to determine TLBTEMP region where the fault happened. The size of the
TLBTEMP region is also checked incorrectly: not 64K, but twice data
cache way size (whicht may as well be less than the instruction cache
way size).
Fix TLBTEMP_BASE_2 to be TLBTEMP_BASE_1 + data cache way size.
Provide TLBTEMP_SIZE that is a greater of doubled data cache way size or
the instruction cache way size, and use it to determine if the second
level TLB miss occured in the TLBTEMP region.
Practical occurence of page faults in the TLBTEMP area is extremely
rare, this code can be tested by deletion of all w[di]tlb instructions
in the tlbtemp_mapping region.
Cc: [email protected]
Signed-off-by: Max Filippov <[email protected]>
---
arch/xtensa/include/asm/pgtable.h | 7 ++++++-
arch/xtensa/kernel/entry.S | 2 +-
2 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/arch/xtensa/include/asm/pgtable.h b/arch/xtensa/include/asm/pgtable.h
index 4b0ca35..b2173e5 100644
--- a/arch/xtensa/include/asm/pgtable.h
+++ b/arch/xtensa/include/asm/pgtable.h
@@ -67,7 +67,12 @@
#define VMALLOC_START 0xC0000000
#define VMALLOC_END 0xC7FEFFFF
#define TLBTEMP_BASE_1 0xC7FF0000
-#define TLBTEMP_BASE_2 0xC7FF8000
+#define TLBTEMP_BASE_2 (TLBTEMP_BASE_1 + DCACHE_WAY_SIZE)
+#if 2 * DCACHE_WAY_SIZE > ICACHE_WAY_SIZE
+#define TLBTEMP_SIZE (2 * DCACHE_WAY_SIZE)
+#else
+#define TLBTEMP_SIZE ICACHE_WAY_SIZE
+#endif
/*
* For the Xtensa architecture, the PTE layout is as follows:
diff --git a/arch/xtensa/kernel/entry.S b/arch/xtensa/kernel/entry.S
index ef7f499..a5c586d 100644
--- a/arch/xtensa/kernel/entry.S
+++ b/arch/xtensa/kernel/entry.S
@@ -1565,7 +1565,7 @@ ENTRY(fast_second_level_miss)
rsr a0, excvaddr
bltu a0, a3, 2f
- addi a1, a0, -(2 << (DCACHE_ALIAS_ORDER + PAGE_SHIFT))
+ addi a1, a0, -TLBTEMP_SIZE
bgeu a1, a3, 2f
/* Check if we have to restore an ITLB mapping. */
--
1.8.1.4
It's much easier to reason about alignment and coloring of regions
located in the fixmap when fixmap index is just a PFN within the fixmap
region. Change fixmap addressing so that index 0 corresponds to
FIXADDR_START instead of the FIXADDR_TOP.
Signed-off-by: Max Filippov <[email protected]>
---
arch/xtensa/include/asm/fixmap.h | 27 ++++++++++++++++++++++++---
arch/xtensa/mm/highmem.c | 6 +++---
2 files changed, 27 insertions(+), 6 deletions(-)
diff --git a/arch/xtensa/include/asm/fixmap.h b/arch/xtensa/include/asm/fixmap.h
index 9f6c33d0..a43cd52 100644
--- a/arch/xtensa/include/asm/fixmap.h
+++ b/arch/xtensa/include/asm/fixmap.h
@@ -23,8 +23,8 @@
* Here we define all the compile-time 'special' virtual
* addresses. The point is to have a constant address at
* compile time, but to set the physical address only
- * in the boot process. We allocate these special addresses
- * from the end of the consistent memory region backwards.
+ * in the boot process. We allocate these special addresses
+ * from the start of the consistent memory region upwards.
* Also this lets us do fail-safe vmalloc(), we
* can guarantee that these special addresses and
* vmalloc()-ed addresses never overlap.
@@ -47,7 +47,28 @@ enum fixed_addresses {
#define FIXADDR_SIZE (__end_of_fixed_addresses << PAGE_SHIFT)
#define FIXADDR_START ((FIXADDR_TOP - FIXADDR_SIZE) & PMD_MASK)
-#include <asm-generic/fixmap.h>
+#define __fix_to_virt(x) (FIXADDR_START + ((x) << PAGE_SHIFT))
+#define __virt_to_fix(x) (((x) - FIXADDR_START) >> PAGE_SHIFT)
+
+#ifndef __ASSEMBLY__
+/*
+ * 'index to address' translation. If anyone tries to use the idx
+ * directly without translation, we catch the bug with a NULL-deference
+ * kernel oops. Illegal ranges of incoming indices are caught too.
+ */
+static __always_inline unsigned long fix_to_virt(const unsigned int idx)
+{
+ BUILD_BUG_ON(idx >= __end_of_fixed_addresses);
+ return __fix_to_virt(idx);
+}
+
+static inline unsigned long virt_to_fix(const unsigned long vaddr)
+{
+ BUG_ON(vaddr >= FIXADDR_TOP || vaddr < FIXADDR_START);
+ return __virt_to_fix(vaddr);
+}
+
+#endif
#define kmap_get_fixmap_pte(vaddr) \
pte_offset_kernel( \
diff --git a/arch/xtensa/mm/highmem.c b/arch/xtensa/mm/highmem.c
index 17a8c0d..2e95a76 100644
--- a/arch/xtensa/mm/highmem.c
+++ b/arch/xtensa/mm/highmem.c
@@ -28,9 +28,9 @@ void *kmap_atomic(struct page *page)
idx = type + KM_TYPE_NR * smp_processor_id();
vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx);
#ifdef CONFIG_DEBUG_HIGHMEM
- BUG_ON(!pte_none(*(kmap_pte - idx)));
+ BUG_ON(!pte_none(*(kmap_pte + idx)));
#endif
- set_pte(kmap_pte - idx, mk_pte(page, PAGE_KERNEL_EXEC));
+ set_pte(kmap_pte + idx, mk_pte(page, PAGE_KERNEL_EXEC));
return (void *)vaddr;
}
@@ -51,7 +51,7 @@ void __kunmap_atomic(void *kvaddr)
* is a bad idea also, in case the page changes cacheability
* attributes or becomes a protected page in a hypervisor.
*/
- pte_clear(&init_mm, kvaddr, kmap_pte - idx);
+ pte_clear(&init_mm, kvaddr, kmap_pte + idx);
local_flush_tlb_kernel_range((unsigned long)kvaddr,
(unsigned long)kvaddr + PAGE_SIZE);
--
1.8.1.4
On 07/22/2014 12:01 PM, Max Filippov wrote:
> From: Leonid Yegoshin <[email protected]>
>
> Provide hooks that allow architectures with aliasing cache to align
> mapping address of high pages according to their color. Such architectures
> may enforce similar coloring of low- and high-memory page mappings and
> reuse existing cache management functions to support highmem.
>
> Cc: [email protected]
> Cc: [email protected]
> Cc: [email protected]
> Cc: David Rientjes <[email protected]>
> Signed-off-by: Leonid Yegoshin <[email protected]>
> [ Max: extract architecture-independent part of the original patch, clean
> up checkpatch and build warnings. ]
> Signed-off-by: Max Filippov <[email protected]>
> ---
> Changes since the initial version:
> - define set_pkmap_color(pg, cl) as do { } while (0) instead of /* */;
> - rename is_no_more_pkmaps to no_more_pkmaps;
> - change 'if (count > 0)' to 'if (count)' to better match the original
> code behavior;
>
> mm/highmem.c | 19 ++++++++++++++++---
> 1 file changed, 16 insertions(+), 3 deletions(-)
>
> diff --git a/mm/highmem.c b/mm/highmem.c
> index b32b70c..88fb62e 100644
> --- a/mm/highmem.c
> +++ b/mm/highmem.c
> @@ -44,6 +44,14 @@ DEFINE_PER_CPU(int, __kmap_atomic_idx);
> */
> #ifdef CONFIG_HIGHMEM
>
> +#ifndef ARCH_PKMAP_COLORING
> +#define set_pkmap_color(pg, cl) do { } while (0)
> +#define get_last_pkmap_nr(p, cl) (p)
> +#define get_next_pkmap_nr(p, cl) (((p) + 1) & LAST_PKMAP_MASK)
> +#define no_more_pkmaps(p, cl) (!(p))
> +#define get_next_pkmap_counter(c, cl) ((c) - 1)
> +#endif
> +
> unsigned long totalhigh_pages __read_mostly;
> EXPORT_SYMBOL(totalhigh_pages);
>
> @@ -161,19 +169,24 @@ static inline unsigned long map_new_virtual(struct page *page)
> {
> unsigned long vaddr;
> int count;
> + int color __maybe_unused;
> +
> + set_pkmap_color(page, color);
> + last_pkmap_nr = get_last_pkmap_nr(last_pkmap_nr, color);
>
> start:
> count = LAST_PKMAP;
> /* Find an empty entry */
> for (;;) {
> - last_pkmap_nr = (last_pkmap_nr + 1) & LAST_PKMAP_MASK;
> - if (!last_pkmap_nr) {
> + last_pkmap_nr = get_next_pkmap_nr(last_pkmap_nr, color);
> + if (no_more_pkmaps(last_pkmap_nr, color)) {
> flush_all_zero_pkmaps();
> count = LAST_PKMAP;
> }
> if (!pkmap_count[last_pkmap_nr])
> break; /* Found a usable entry */
> - if (--count)
> + count = get_next_pkmap_counter(count, color);
> + if (count)
> continue;
>
> /*
I would like to return back to "if (count >0)".
The reason is in easy way to jump through the same coloured pages - next
element is calculated via decrementing by non "1" value of colours and
it can easy become negative on last page available:
#define get_next_pkmap_counter(c,cl) (c - FIX_N_COLOURS)
where FIX_N_COLOURS is a max number of page colours.
Besides that it is a good practice in stopping cycle.
- Leonid.
On Tue, Jul 22, 2014 at 11:35 PM, Leonid Yegoshin
<[email protected]> wrote:
> On 07/22/2014 12:01 PM, Max Filippov wrote:
>>
>> From: Leonid Yegoshin <[email protected]>
>>
>> Provide hooks that allow architectures with aliasing cache to align
>> mapping address of high pages according to their color. Such architectures
>> may enforce similar coloring of low- and high-memory page mappings and
>> reuse existing cache management functions to support highmem.
>>
>> Cc: [email protected]
>> Cc: [email protected]
>> Cc: [email protected]
>> Cc: David Rientjes <[email protected]>
>> Signed-off-by: Leonid Yegoshin <[email protected]>
>> [ Max: extract architecture-independent part of the original patch, clean
>> up checkpatch and build warnings. ]
>> Signed-off-by: Max Filippov <[email protected]>
>> ---
>> Changes since the initial version:
>> - define set_pkmap_color(pg, cl) as do { } while (0) instead of /* */;
>> - rename is_no_more_pkmaps to no_more_pkmaps;
>> - change 'if (count > 0)' to 'if (count)' to better match the original
>> code behavior;
>>
>> mm/highmem.c | 19 ++++++++++++++++---
>> 1 file changed, 16 insertions(+), 3 deletions(-)
>>
>> diff --git a/mm/highmem.c b/mm/highmem.c
>> index b32b70c..88fb62e 100644
>> --- a/mm/highmem.c
>> +++ b/mm/highmem.c
>> @@ -44,6 +44,14 @@ DEFINE_PER_CPU(int, __kmap_atomic_idx);
>> */
>> #ifdef CONFIG_HIGHMEM
>> +#ifndef ARCH_PKMAP_COLORING
>> +#define set_pkmap_color(pg, cl) do { } while (0)
>> +#define get_last_pkmap_nr(p, cl) (p)
>> +#define get_next_pkmap_nr(p, cl) (((p) + 1) & LAST_PKMAP_MASK)
>> +#define no_more_pkmaps(p, cl) (!(p))
>> +#define get_next_pkmap_counter(c, cl) ((c) - 1)
>> +#endif
>> +
>> unsigned long totalhigh_pages __read_mostly;
>> EXPORT_SYMBOL(totalhigh_pages);
>> @@ -161,19 +169,24 @@ static inline unsigned long map_new_virtual(struct
>> page *page)
>> {
>> unsigned long vaddr;
>> int count;
>> + int color __maybe_unused;
>> +
>> + set_pkmap_color(page, color);
>> + last_pkmap_nr = get_last_pkmap_nr(last_pkmap_nr, color);
>> start:
>> count = LAST_PKMAP;
>> /* Find an empty entry */
>> for (;;) {
>> - last_pkmap_nr = (last_pkmap_nr + 1) & LAST_PKMAP_MASK;
>> - if (!last_pkmap_nr) {
>> + last_pkmap_nr = get_next_pkmap_nr(last_pkmap_nr, color);
>> + if (no_more_pkmaps(last_pkmap_nr, color)) {
>> flush_all_zero_pkmaps();
>> count = LAST_PKMAP;
>> }
>> if (!pkmap_count[last_pkmap_nr])
>> break; /* Found a usable entry */
>> - if (--count)
>> + count = get_next_pkmap_counter(count, color);
>> + if (count)
>> continue;
>> /*
>
> I would like to return back to "if (count >0)".
>
> The reason is in easy way to jump through the same coloured pages - next
> element is calculated via decrementing by non "1" value of colours and it
> can easy become negative on last page available:
>
> #define get_next_pkmap_counter(c,cl) (c - FIX_N_COLOURS)
>
> where FIX_N_COLOURS is a max number of page colours.
Initial value of c (i.e. LAST_PKMAP) should be a multiple of FIX_N_COLOURS,
so that should not be a problem.
> Besides that it is a good practice in stopping cycle.
But I agree with that.
--
Thanks.
-- Max