This and the following patches aim to optimize the code dealing with
page tables and TLB operations. Each patch reduces the time it takes
to gzip a 16 MB file slightly, but I expect things like fork() and
mmap() will improve somewhat more.
This patch deals with the low-level TLB operations:
* Remove unused _TLBEHI_I define
* Use gcc builtins instead of inline assembly
* Remove a few unnecessary pipeline flushes and nops
* Introduce NR_TLB_ENTRIES define and use it instead of hardcoding it
to 32 a few places throughout the code.
* Use sysreg bitops instead of hardcoded shifts and masks
* Make a few needlessly global functions static
Signed-off-by: Haavard Skinnemoen <[email protected]>
---
arch/avr32/mm/tlb.c | 175 +++++++++++++++++++++---------------------
include/asm-avr32/tlbflush.h | 1 -
2 files changed, 87 insertions(+), 89 deletions(-)
diff --git a/arch/avr32/mm/tlb.c b/arch/avr32/mm/tlb.c
index cd12edb..06677be 100644
--- a/arch/avr32/mm/tlb.c
+++ b/arch/avr32/mm/tlb.c
@@ -11,21 +11,21 @@
#include <asm/mmu_context.h>
-#define _TLBEHI_I 0x100
+/* TODO: Get the correct number from the CONFIG1 system register */
+#define NR_TLB_ENTRIES 32
-void show_dtlb_entry(unsigned int index)
+static void show_dtlb_entry(unsigned int index)
{
- unsigned int tlbehi, tlbehi_save, tlbelo, mmucr, mmucr_save;
+ u32 tlbehi, tlbehi_save, tlbelo, mmucr, mmucr_save;
unsigned long flags;
local_irq_save(flags);
mmucr_save = sysreg_read(MMUCR);
tlbehi_save = sysreg_read(TLBEHI);
- mmucr = mmucr_save & 0x13;
- mmucr |= index << 14;
+ mmucr = SYSREG_BFINS(DRP, index, mmucr_save);
sysreg_write(MMUCR, mmucr);
- asm volatile("tlbr" : : : "memory");
+ __builtin_tlbr();
cpu_sync_pipeline();
tlbehi = sysreg_read(TLBEHI);
@@ -33,15 +33,17 @@ void show_dtlb_entry(unsigned int index)
printk("%2u: %c %c %02x %05x %05x %o %o %c %c %c %c\n",
index,
- (tlbehi & 0x200)?'1':'0',
- (tlbelo & 0x100)?'1':'0',
- (tlbehi & 0xff),
- (tlbehi >> 12), (tlbelo >> 12),
- (tlbelo >> 4) & 7, (tlbelo >> 2) & 3,
- (tlbelo & 0x200)?'1':'0',
- (tlbelo & 0x080)?'1':'0',
- (tlbelo & 0x001)?'1':'0',
- (tlbelo & 0x002)?'1':'0');
+ SYSREG_BFEXT(TLBEHI_V, tlbehi) ? '1' : '0',
+ SYSREG_BFEXT(G, tlbelo) ? '1' : '0',
+ SYSREG_BFEXT(ASID, tlbehi),
+ SYSREG_BFEXT(VPN, tlbehi) >> 2,
+ SYSREG_BFEXT(PFN, tlbelo) >> 2,
+ SYSREG_BFEXT(AP, tlbelo),
+ SYSREG_BFEXT(SZ, tlbelo),
+ SYSREG_BFEXT(TLBELO_C, tlbelo) ? 'C' : ' ',
+ SYSREG_BFEXT(B, tlbelo) ? 'B' : ' ',
+ SYSREG_BFEXT(W, tlbelo) ? 'W' : ' ',
+ SYSREG_BFEXT(TLBELO_D, tlbelo) ? 'D' : ' ');
sysreg_write(MMUCR, mmucr_save);
sysreg_write(TLBEHI, tlbehi_save);
@@ -54,29 +56,33 @@ void dump_dtlb(void)
unsigned int i;
printk("ID V G ASID VPN PFN AP SZ C B W D\n");
- for (i = 0; i < 32; i++)
+ for (i = 0; i < NR_TLB_ENTRIES; i++)
show_dtlb_entry(i);
}
-static unsigned long last_mmucr;
-
-static inline void set_replacement_pointer(unsigned shift)
+static void update_dtlb(unsigned long address, pte_t pte)
{
- unsigned long mmucr, mmucr_save;
+ u32 tlbehi;
+ u32 mmucr;
- mmucr = mmucr_save = sysreg_read(MMUCR);
+ /*
+ * We're not changing the ASID here, so no need to flush the
+ * pipeline.
+ */
+ tlbehi = sysreg_read(TLBEHI);
+ tlbehi = SYSREG_BF(ASID, SYSREG_BFEXT(ASID, tlbehi));
+ tlbehi |= address & MMU_VPN_MASK;
+ tlbehi |= SYSREG_BIT(TLBEHI_V);
+ sysreg_write(TLBEHI, tlbehi);
/* Does this mapping already exist? */
- __asm__ __volatile__(
- " tlbs\n"
- " mfsr %0, %1"
- : "=r"(mmucr)
- : "i"(SYSREG_MMUCR));
+ __builtin_tlbs();
+ mmucr = sysreg_read(MMUCR);
if (mmucr & SYSREG_BIT(MMUCR_N)) {
/* Not found -- pick a not-recently-accessed entry */
- unsigned long rp;
- unsigned long tlbar = sysreg_read(TLBARLO);
+ unsigned int rp;
+ u32 tlbar = sysreg_read(TLBARLO);
rp = 32 - fls(tlbar);
if (rp == 32) {
@@ -84,30 +90,14 @@ static inline void set_replacement_pointer(unsigned shift)
sysreg_write(TLBARLO, -1L);
}
- mmucr &= 0x13;
- mmucr |= (rp << shift);
-
+ mmucr = SYSREG_BFINS(DRP, rp, mmucr);
sysreg_write(MMUCR, mmucr);
}
- last_mmucr = mmucr;
-}
-
-static void update_dtlb(unsigned long address, pte_t pte, unsigned long asid)
-{
- unsigned long vpn;
-
- vpn = (address & MMU_VPN_MASK) | _TLBEHI_VALID | asid;
- sysreg_write(TLBEHI, vpn);
- cpu_sync_pipeline();
-
- set_replacement_pointer(14);
-
sysreg_write(TLBELO, pte_val(pte) & _PAGE_FLAGS_HARDWARE_MASK);
/* Let's go */
- asm volatile("nop\n\ttlbw" : : : "memory");
- cpu_sync_pipeline();
+ __builtin_tlbw();
}
void update_mmu_cache(struct vm_area_struct *vma,
@@ -120,39 +110,40 @@ void update_mmu_cache(struct vm_area_struct *vma,
return;
local_irq_save(flags);
- update_dtlb(address, pte, get_asid());
+ update_dtlb(address, pte);
local_irq_restore(flags);
}
-void __flush_tlb_page(unsigned long asid, unsigned long page)
+static void __flush_tlb_page(unsigned long asid, unsigned long page)
{
- unsigned long mmucr, tlbehi;
+ u32 mmucr, tlbehi;
- page |= asid;
- sysreg_write(TLBEHI, page);
- cpu_sync_pipeline();
- asm volatile("tlbs");
+ /*
+ * Caller is responsible for masking out non-PFN bits in page
+ * and changing the current ASID if necessary. This means that
+ * we don't need to flush the pipeline after writing TLBEHI.
+ */
+ tlbehi = page | asid;
+ sysreg_write(TLBEHI, tlbehi);
+
+ __builtin_tlbs();
mmucr = sysreg_read(MMUCR);
if (!(mmucr & SYSREG_BIT(MMUCR_N))) {
- unsigned long tlbarlo;
- unsigned long entry;
+ unsigned int entry;
+ u32 tlbarlo;
/* Clear the "valid" bit */
- tlbehi = sysreg_read(TLBEHI);
- tlbehi &= ~_TLBEHI_VALID;
sysreg_write(TLBEHI, tlbehi);
- cpu_sync_pipeline();
/* mark the entry as "not accessed" */
- entry = (mmucr >> 14) & 0x3f;
+ entry = SYSREG_BFEXT(DRP, mmucr);
tlbarlo = sysreg_read(TLBARLO);
- tlbarlo |= (0x80000000 >> entry);
+ tlbarlo |= (0x80000000UL >> entry);
sysreg_write(TLBARLO, tlbarlo);
/* update the entry with valid bit clear */
- asm volatile("tlbw");
- cpu_sync_pipeline();
+ __builtin_tlbw();
}
}
@@ -190,17 +181,22 @@ void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
local_irq_save(flags);
size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
+
if (size > (MMU_DTLB_ENTRIES / 4)) { /* Too many entries to flush */
mm->context = NO_CONTEXT;
if (mm == current->mm)
activate_context(mm);
} else {
- unsigned long asid = mm->context & MMU_CONTEXT_ASID_MASK;
- unsigned long saved_asid = MMU_NO_ASID;
+ unsigned long asid;
+ unsigned long saved_asid;
+
+ asid = mm->context & MMU_CONTEXT_ASID_MASK;
+ saved_asid = MMU_NO_ASID;
start &= PAGE_MASK;
end += (PAGE_SIZE - 1);
end &= PAGE_MASK;
+
if (mm != current->mm) {
saved_asid = get_asid();
set_asid(asid);
@@ -218,33 +214,34 @@ void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
}
/*
- * TODO: If this is only called for addresses > TASK_SIZE, we can probably
- * skip the ASID stuff and just use the Global bit...
+ * This function depends on the pages to be flushed having the G
+ * (global) bit set in their pte. This is true for all
+ * PAGE_KERNEL(_RO) pages.
*/
void flush_tlb_kernel_range(unsigned long start, unsigned long end)
{
unsigned long flags;
int size;
- local_irq_save(flags);
size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
if (size > (MMU_DTLB_ENTRIES / 4)) { /* Too many entries to flush */
flush_tlb_all();
} else {
- unsigned long asid = init_mm.context & MMU_CONTEXT_ASID_MASK;
- unsigned long saved_asid = get_asid();
+ unsigned long asid;
+
+ local_irq_save(flags);
+ asid = get_asid();
start &= PAGE_MASK;
end += (PAGE_SIZE - 1);
end &= PAGE_MASK;
- set_asid(asid);
+
while (start < end) {
__flush_tlb_page(asid, start);
start += PAGE_SIZE;
}
- set_asid(saved_asid);
+ local_irq_restore(flags);
}
- local_irq_restore(flags);
}
void flush_tlb_mm(struct mm_struct *mm)
@@ -280,7 +277,7 @@ static void *tlb_start(struct seq_file *tlb, loff_t *pos)
{
static unsigned long tlb_index;
- if (*pos >= 32)
+ if (*pos >= NR_TLB_ENTRIES)
return NULL;
tlb_index = 0;
@@ -291,7 +288,7 @@ static void *tlb_next(struct seq_file *tlb, void *v, loff_t *pos)
{
unsigned long *index = v;
- if (*index >= 31)
+ if (*index >= NR_TLB_ENTRIES - 1)
return NULL;
++*pos;
@@ -313,16 +310,16 @@ static int tlb_show(struct seq_file *tlb, void *v)
if (*index == 0)
seq_puts(tlb, "ID V G ASID VPN PFN AP SZ C B W D\n");
- BUG_ON(*index >= 32);
+ BUG_ON(*index >= NR_TLB_ENTRIES);
local_irq_save(flags);
mmucr_save = sysreg_read(MMUCR);
tlbehi_save = sysreg_read(TLBEHI);
- mmucr = mmucr_save & 0x13;
- mmucr |= *index << 14;
+ mmucr = SYSREG_BFINS(DRP, *index, mmucr_save);
sysreg_write(MMUCR, mmucr);
- asm volatile("tlbr" : : : "memory");
+ /* TLBR might change the ASID */
+ __builtin_tlbr();
cpu_sync_pipeline();
tlbehi = sysreg_read(TLBEHI);
@@ -334,16 +331,18 @@ static int tlb_show(struct seq_file *tlb, void *v)
local_irq_restore(flags);
seq_printf(tlb, "%2lu: %c %c %02x %05x %05x %o %o %c %c %c %c\n",
- *index,
- (tlbehi & 0x200)?'1':'0',
- (tlbelo & 0x100)?'1':'0',
- (tlbehi & 0xff),
- (tlbehi >> 12), (tlbelo >> 12),
- (tlbelo >> 4) & 7, (tlbelo >> 2) & 3,
- (tlbelo & 0x200)?'1':'0',
- (tlbelo & 0x080)?'1':'0',
- (tlbelo & 0x001)?'1':'0',
- (tlbelo & 0x002)?'1':'0');
+ *index,
+ SYSREG_BFEXT(TLBEHI_V, tlbehi) ? '1' : '0',
+ SYSREG_BFEXT(G, tlbelo) ? '1' : '0',
+ SYSREG_BFEXT(ASID, tlbehi),
+ SYSREG_BFEXT(VPN, tlbehi) >> 2,
+ SYSREG_BFEXT(PFN, tlbelo) >> 2,
+ SYSREG_BFEXT(AP, tlbelo),
+ SYSREG_BFEXT(SZ, tlbelo),
+ SYSREG_BFEXT(TLBELO_C, tlbelo) ? '1' : '0',
+ SYSREG_BFEXT(B, tlbelo) ? '1' : '0',
+ SYSREG_BFEXT(W, tlbelo) ? '1' : '0',
+ SYSREG_BFEXT(TLBELO_D, tlbelo) ? '1' : '0');
return 0;
}
diff --git a/include/asm-avr32/tlbflush.h b/include/asm-avr32/tlbflush.h
index 5bc7c88..bf90a78 100644
--- a/include/asm-avr32/tlbflush.h
+++ b/include/asm-avr32/tlbflush.h
@@ -26,7 +26,6 @@ extern void flush_tlb_mm(struct mm_struct *mm);
extern void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
unsigned long end);
extern void flush_tlb_page(struct vm_area_struct *vma, unsigned long page);
-extern void __flush_tlb_page(unsigned long asid, unsigned long page);
extern void flush_tlb_kernel_range(unsigned long start, unsigned long end);
--
1.5.5.4
Using a quicklist to allocate PTEs might be slightly faster than using
the page allocator directly since we might avoid zeroing the page
after each allocation.
Signed-off-by: Haavard Skinnemoen <[email protected]>
---
include/asm-avr32/pgalloc.h | 28 +++++++++++++++-------------
mm/Kconfig | 2 +-
2 files changed, 16 insertions(+), 14 deletions(-)
diff --git a/include/asm-avr32/pgalloc.h b/include/asm-avr32/pgalloc.h
index a291f59..6408213 100644
--- a/include/asm-avr32/pgalloc.h
+++ b/include/asm-avr32/pgalloc.h
@@ -13,6 +13,7 @@
#include <asm/pgtable.h>
#define QUICK_PGD 0 /* Preserve kernel mappings over free */
+#define QUICK_PT 1 /* Zero on free */
static inline void pmd_populate_kernel(struct mm_struct *mm,
pmd_t *pmd, pte_t *pte)
@@ -52,34 +53,34 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
unsigned long address)
{
- pte_t *pte;
-
- pte = (pte_t *)get_zeroed_page(GFP_KERNEL | __GFP_REPEAT);
-
- return pte;
+ return quicklist_alloc(QUICK_PT, GFP_KERNEL | __GFP_REPEAT, NULL);
}
-static inline struct page *pte_alloc_one(struct mm_struct *mm,
+static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
unsigned long address)
{
- struct page *pte;
+ struct page *page;
+ void *pg;
- pte = alloc_page(GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO);
- if (!pte)
+ pg = quicklist_alloc(QUICK_PT, GFP_KERNEL | __GFP_REPEAT, NULL);
+ if (!pg)
return NULL;
- pgtable_page_ctor(pte);
- return pte;
+
+ page = virt_to_page(pg);
+ pgtable_page_ctor(page);
+
+ return page;
}
static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
{
- free_page((unsigned long)pte);
+ quicklist_free(QUICK_PT, NULL, pte);
}
static inline void pte_free(struct mm_struct *mm, pgtable_t pte)
{
pgtable_page_dtor(pte);
- __free_page(pte);
+ quicklist_free_page(QUICK_PT, NULL, pte);
}
#define __pte_free_tlb(tlb,pte) \
@@ -91,6 +92,7 @@ do { \
static inline void check_pgt_cache(void)
{
quicklist_trim(QUICK_PGD, NULL, 25, 16);
+ quicklist_trim(QUICK_PT, NULL, 25, 16);
}
#endif /* __ASM_AVR32_PGALLOC_H */
diff --git a/mm/Kconfig b/mm/Kconfig
index 3aa819d..60b2619 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -199,7 +199,7 @@ config BOUNCE
config NR_QUICK
int
depends on QUICKLIST
- default "2" if SUPERH
+ default "2" if SUPERH || AVR32
default "1"
config VIRT_TO_BUS
--
1.5.5.4
swapper_pg_dir is stored in .bss, so it must already be zeroed out
when we get there.
Signed-off-by: Haavard Skinnemoen <[email protected]>
---
arch/avr32/mm/init.c | 16 +++-------------
1 files changed, 3 insertions(+), 13 deletions(-)
diff --git a/arch/avr32/mm/init.c b/arch/avr32/mm/init.c
index 5ee1e40..0e77578 100644
--- a/arch/avr32/mm/init.c
+++ b/arch/avr32/mm/init.c
@@ -108,19 +108,9 @@ void __init paging_init(void)
zero_page = alloc_bootmem_low_pages_node(NODE_DATA(0),
PAGE_SIZE);
- {
- pgd_t *pg_dir;
- int i;
-
- pg_dir = swapper_pg_dir;
- sysreg_write(PTBR, (unsigned long)pg_dir);
-
- for (i = 0; i < PTRS_PER_PGD; i++)
- pgd_val(pg_dir[i]) = 0;
-
- enable_mmu();
- printk ("CPU: Paging enabled\n");
- }
+ sysreg_write(PTBR, (unsigned long)swapper_pg_dir);
+ enable_mmu();
+ printk ("CPU: Paging enabled\n");
for_each_online_node(nid) {
pg_data_t *pgdat = NODE_DATA(nid);
--
1.5.5.4
Instead of storing physical addresses along with page flags in the
PGD, store virtual addresses and use NULL to indicate a not present
second-level page table. A non-page-aligned page table indicates a bad
PMD.
This simplifies the TLB miss handler since it no longer has to check
the Present bit and no longer has to convert the PGD entry from
physical to virtual address. Instead, it has to check for a NULL
entry, which is slightly cheaper than either.
Signed-off-by: Haavard Skinnemoen <[email protected]>
---
arch/avr32/kernel/entry-avr32b.S | 45 ++++++++++++++++---------------------
arch/avr32/kernel/vmlinux.lds.S | 4 +++
arch/avr32/mm/init.c | 4 ++-
include/asm-avr32/pgalloc.h | 18 ++++++++------
include/asm-avr32/pgtable.h | 34 +++++++++++----------------
5 files changed, 51 insertions(+), 54 deletions(-)
diff --git a/arch/avr32/kernel/entry-avr32b.S b/arch/avr32/kernel/entry-avr32b.S
index 050c006..3cdd707 100644
--- a/arch/avr32/kernel/entry-avr32b.S
+++ b/arch/avr32/kernel/entry-avr32b.S
@@ -74,12 +74,6 @@ exception_vectors:
.align 2
bral do_dtlb_modified
- /*
- * r0 : PGD/PT/PTE
- * r1 : Offending address
- * r2 : Scratch register
- * r3 : Cause (5, 12 or 13)
- */
#define tlbmiss_save pushm r0-r3
#define tlbmiss_restore popm r0-r3
@@ -108,17 +102,17 @@ tlb_miss_common:
bld r0, 31
brcs handle_vmalloc_miss
- /* First level lookup */
+ /*
+ * First level lookup: The PGD contains virtual pointers to
+ * the second-level page tables, but they may be NULL if not
+ * present.
+ */
pgtbl_lookup:
lsr r2, r0, PGDIR_SHIFT
ld.w r3, r1[r2 << 2]
bfextu r1, r0, PAGE_SHIFT, PGDIR_SHIFT - PAGE_SHIFT
- bld r3, _PAGE_BIT_PRESENT
- brcc page_table_not_present
-
- /* Translate to virtual address in P1. */
- andl r3, 0xf000
- sbr r3, 31
+ cp.w r3, 0
+ breq page_table_not_present
/* Second level lookup */
ld.w r2, r3[r1 << 2]
@@ -155,6 +149,19 @@ handle_vmalloc_miss:
orh r1, hi(swapper_pg_dir)
rjmp pgtbl_lookup
+ /* The slow path of the TLB miss handler */
+ .align 2
+page_table_not_present:
+page_not_present:
+ tlbmiss_restore
+ sub sp, 4
+ stmts --sp, r0-lr
+ rcall save_full_context_ex
+ mfsr r12, SYSREG_ECR
+ mov r11, sp
+ rcall do_page_fault
+ rjmp ret_from_exception
+
/* --- System Call --- */
@@ -267,18 +274,6 @@ syscall_exit_work:
brcc syscall_exit_cont
rjmp enter_monitor_mode
- /* The slow path of the TLB miss handler */
-page_table_not_present:
-page_not_present:
- tlbmiss_restore
- sub sp, 4
- stmts --sp, r0-lr
- rcall save_full_context_ex
- mfsr r12, SYSREG_ECR
- mov r11, sp
- rcall do_page_fault
- rjmp ret_from_exception
-
/* This function expects to find offending PC in SYSREG_RAR_EX */
.type save_full_context_ex, @function
.align 2
diff --git a/arch/avr32/kernel/vmlinux.lds.S b/arch/avr32/kernel/vmlinux.lds.S
index 033dd46..5d25d8e 100644
--- a/arch/avr32/kernel/vmlinux.lds.S
+++ b/arch/avr32/kernel/vmlinux.lds.S
@@ -99,6 +99,10 @@ SECTIONS
*/
*(.data.init_task)
+ /* Then, the page-aligned data */
+ . = ALIGN(PAGE_SIZE);
+ *(.data.page_aligned)
+
/* Then, the cacheline aligned data */
. = ALIGN(L1_CACHE_BYTES);
*(.data.cacheline_aligned)
diff --git a/arch/avr32/mm/init.c b/arch/avr32/mm/init.c
index 0e77578..3f90a87 100644
--- a/arch/avr32/mm/init.c
+++ b/arch/avr32/mm/init.c
@@ -24,9 +24,11 @@
#include <asm/setup.h>
#include <asm/sections.h>
+#define __page_aligned __attribute__((section(".data.page_aligned")))
+
DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
-pgd_t swapper_pg_dir[PTRS_PER_PGD];
+pgd_t swapper_pg_dir[PTRS_PER_PGD] __page_aligned;
struct page *empty_zero_page;
EXPORT_SYMBOL(empty_zero_page);
diff --git a/include/asm-avr32/pgalloc.h b/include/asm-avr32/pgalloc.h
index 51fc1f6..5b768fc 100644
--- a/include/asm-avr32/pgalloc.h
+++ b/include/asm-avr32/pgalloc.h
@@ -8,25 +8,27 @@
#ifndef __ASM_AVR32_PGALLOC_H
#define __ASM_AVR32_PGALLOC_H
-#include <asm/processor.h>
-#include <linux/threads.h>
-#include <linux/slab.h>
#include <linux/mm.h>
+#include <linux/sched.h>
+#include <linux/slab.h>
-#define pmd_populate_kernel(mm, pmd, pte) \
- set_pmd(pmd, __pmd(_PAGE_TABLE + __pa(pte)))
+static inline void pmd_populate_kernel(struct mm_struct *mm,
+ pmd_t *pmd, pte_t *pte)
+{
+ set_pmd(pmd, __pmd((unsigned long)pte));
+}
-static __inline__ void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
+static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
pgtable_t pte)
{
- set_pmd(pmd, __pmd(_PAGE_TABLE + page_to_phys(pte)));
+ set_pmd(pmd, __pmd((unsigned long)page_address(pte)));
}
#define pmd_pgtable(pmd) pmd_page(pmd)
/*
* Allocate and free page tables
*/
-static __inline__ pgd_t *pgd_alloc(struct mm_struct *mm)
+static inline pgd_t *pgd_alloc(struct mm_struct *mm)
{
return kcalloc(USER_PTRS_PER_PGD, sizeof(pgd_t), GFP_KERNEL);
}
diff --git a/include/asm-avr32/pgtable.h b/include/asm-avr32/pgtable.h
index c0e5e29..fecdda1 100644
--- a/include/asm-avr32/pgtable.h
+++ b/include/asm-avr32/pgtable.h
@@ -129,13 +129,6 @@ extern struct page *empty_zero_page;
#define _PAGE_FLAGS_CACHE_MASK (_PAGE_CACHABLE | _PAGE_BUFFER | _PAGE_WT)
-/* TODO: Check for saneness */
-/* User-mode page table flags (to be set in a pgd or pmd entry) */
-#define _PAGE_TABLE (_PAGE_PRESENT | _PAGE_TYPE_SMALL | _PAGE_RW \
- | _PAGE_USER | _PAGE_ACCESSED | _PAGE_DIRTY)
-/* Kernel-mode page table flags */
-#define _KERNPG_TABLE (_PAGE_PRESENT | _PAGE_TYPE_SMALL | _PAGE_RW \
- | _PAGE_ACCESSED | _PAGE_DIRTY)
/* Flags that may be modified by software */
#define _PAGE_CHG_MASK (PTE_MASK | _PAGE_ACCESSED | _PAGE_DIRTY \
| _PAGE_FLAGS_CACHE_MASK)
@@ -262,10 +255,14 @@ static inline pte_t pte_mkspecial(pte_t pte)
}
#define pmd_none(x) (!pmd_val(x))
-#define pmd_present(x) (pmd_val(x) & _PAGE_PRESENT)
-#define pmd_clear(xp) do { set_pmd(xp, __pmd(0)); } while (0)
-#define pmd_bad(x) ((pmd_val(x) & (~PAGE_MASK & ~_PAGE_USER)) \
- != _KERNPG_TABLE)
+#define pmd_present(x) (pmd_val(x))
+
+static inline void pmd_clear(pmd_t *pmdp)
+{
+ set_pmd(pmdp, __pmd(0));
+}
+
+#define pmd_bad(x) (pmd_val(x) & ~PAGE_MASK)
/*
* Permanent address of a page. We don't support highmem, so this is
@@ -303,19 +300,16 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
#define page_pte(page) page_pte_prot(page, __pgprot(0))
-#define pmd_page_vaddr(pmd) \
- ((unsigned long) __va(pmd_val(pmd) & PAGE_MASK))
-
-#define pmd_page(pmd) (phys_to_page(pmd_val(pmd)))
+#define pmd_page_vaddr(pmd) pmd_val(pmd)
+#define pmd_page(pmd) (virt_to_page(pmd_val(pmd)))
/* to find an entry in a page-table-directory. */
-#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1))
-#define pgd_offset(mm, address) ((mm)->pgd+pgd_index(address))
-#define pgd_offset_current(address) \
- ((pgd_t *)__mfsr(SYSREG_PTBR) + pgd_index(address))
+#define pgd_index(address) (((address) >> PGDIR_SHIFT) \
+ & (PTRS_PER_PGD - 1))
+#define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address))
/* to find an entry in a kernel page-table-directory */
-#define pgd_offset_k(address) pgd_offset(&init_mm, address)
+#define pgd_offset_k(address) pgd_offset(&init_mm, address)
/* Find an entry in the third-level page table.. */
#define pte_index(address) \
--
1.5.5.4
Expand the per-process PGDs so that they cover the kernel virtual
memory area as well. This simplifies the TLB miss handler fastpath
since it doesn't have to check for kernel addresses anymore.
If a TLB miss happens on a kernel address and a second-level page
table can't be found, we check swapper_pg_dir and copy the PGD entry
into the user PGD if it can be found there.
Signed-off-by: Haavard Skinnemoen <[email protected]>
---
arch/avr32/kernel/entry-avr32b.S | 42 ++++++++++++++++++++++++++++---------
include/asm-avr32/pgalloc.h | 14 +++++++++---
2 files changed, 42 insertions(+), 14 deletions(-)
diff --git a/arch/avr32/kernel/entry-avr32b.S b/arch/avr32/kernel/entry-avr32b.S
index 3cdd707..2b398ca 100644
--- a/arch/avr32/kernel/entry-avr32b.S
+++ b/arch/avr32/kernel/entry-avr32b.S
@@ -98,10 +98,6 @@ tlb_miss_common:
mfsr r0, SYSREG_TLBEAR
mfsr r1, SYSREG_PTBR
- /* Is it the vmalloc space? */
- bld r0, 31
- brcs handle_vmalloc_miss
-
/*
* First level lookup: The PGD contains virtual pointers to
* the second-level page tables, but they may be NULL if not
@@ -143,15 +139,13 @@ pgtbl_lookup:
tlbmiss_restore
rete
-handle_vmalloc_miss:
- /* Simply do the lookup in init's page table */
- mov r1, lo(swapper_pg_dir)
- orh r1, hi(swapper_pg_dir)
- rjmp pgtbl_lookup
-
/* The slow path of the TLB miss handler */
.align 2
page_table_not_present:
+ /* Do we need to synchronize with swapper_pg_dir? */
+ bld r0, 31
+ brcs sync_with_swapper_pg_dir
+
page_not_present:
tlbmiss_restore
sub sp, 4
@@ -162,6 +156,34 @@ page_not_present:
rcall do_page_fault
rjmp ret_from_exception
+ .align 2
+sync_with_swapper_pg_dir:
+ /*
+ * If swapper_pg_dir contains a non-NULL second-level page
+ * table pointer, copy it into the current PGD. If not, we
+ * must handle it as a full-blown page fault.
+ *
+ * Jumping back to pgtbl_lookup causes an unnecessary lookup,
+ * but it is guaranteed to be a cache hit, it won't happen
+ * very often, and we absolutely do not want to sacrifice any
+ * performance in the fast path in order to improve this.
+ */
+ mov r1, lo(swapper_pg_dir)
+ orh r1, hi(swapper_pg_dir)
+ ld.w r3, r1[r2 << 2]
+ cp.w r3, 0
+ breq page_not_present
+ mfsr r1, SYSREG_PTBR
+ st.w r1[r2 << 2], r3
+ rjmp pgtbl_lookup
+
+ /*
+ * We currently have two bytes left at this point until we
+ * crash into the system call handler...
+ *
+ * Don't worry, the assembler will let us know.
+ */
+
/* --- System Call --- */
diff --git a/include/asm-avr32/pgalloc.h b/include/asm-avr32/pgalloc.h
index 5b768fc..e9636d1 100644
--- a/include/asm-avr32/pgalloc.h
+++ b/include/asm-avr32/pgalloc.h
@@ -9,8 +9,6 @@
#define __ASM_AVR32_PGALLOC_H
#include <linux/mm.h>
-#include <linux/sched.h>
-#include <linux/slab.h>
static inline void pmd_populate_kernel(struct mm_struct *mm,
pmd_t *pmd, pte_t *pte)
@@ -30,12 +28,20 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
*/
static inline pgd_t *pgd_alloc(struct mm_struct *mm)
{
- return kcalloc(USER_PTRS_PER_PGD, sizeof(pgd_t), GFP_KERNEL);
+ pgd_t *pgd;
+
+ pgd = (pgd_t *)get_zeroed_page(GFP_KERNEL | __GFP_REPEAT);
+ if (likely(pgd))
+ memcpy(pgd + USER_PTRS_PER_PGD,
+ swapper_pg_dir + USER_PTRS_PER_PGD,
+ (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
+
+ return pgd;
}
static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
{
- kfree(pgd);
+ free_page((unsigned long)pgd);
}
static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
--
1.5.5.4
Use a quicklist to allocate process PGDs. This is expected to be
slightly faster since we need to copy entries from swapper_pg_dir,
which can stay around for pages on the PGD quick list.
Signed-off-by: Haavard Skinnemoen <[email protected]>
---
arch/avr32/Kconfig | 3 +++
include/asm-avr32/pgalloc.h | 32 ++++++++++++++++++++------------
2 files changed, 23 insertions(+), 12 deletions(-)
diff --git a/arch/avr32/Kconfig b/arch/avr32/Kconfig
index 09ad799..a5793c1 100644
--- a/arch/avr32/Kconfig
+++ b/arch/avr32/Kconfig
@@ -147,6 +147,9 @@ config PHYS_OFFSET
source "kernel/Kconfig.preempt"
+config QUICKLIST
+ def_bool y
+
config HAVE_ARCH_BOOTMEM_NODE
def_bool n
diff --git a/include/asm-avr32/pgalloc.h b/include/asm-avr32/pgalloc.h
index e9636d1..a291f59 100644
--- a/include/asm-avr32/pgalloc.h
+++ b/include/asm-avr32/pgalloc.h
@@ -8,7 +8,11 @@
#ifndef __ASM_AVR32_PGALLOC_H
#define __ASM_AVR32_PGALLOC_H
-#include <linux/mm.h>
+#include <linux/quicklist.h>
+#include <asm/page.h>
+#include <asm/pgtable.h>
+
+#define QUICK_PGD 0 /* Preserve kernel mappings over free */
static inline void pmd_populate_kernel(struct mm_struct *mm,
pmd_t *pmd, pte_t *pte)
@@ -23,25 +27,26 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
}
#define pmd_pgtable(pmd) pmd_page(pmd)
+static inline void pgd_ctor(void *x)
+{
+ pgd_t *pgd = x;
+
+ memcpy(pgd + USER_PTRS_PER_PGD,
+ swapper_pg_dir + USER_PTRS_PER_PGD,
+ (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
+}
+
/*
* Allocate and free page tables
*/
static inline pgd_t *pgd_alloc(struct mm_struct *mm)
{
- pgd_t *pgd;
-
- pgd = (pgd_t *)get_zeroed_page(GFP_KERNEL | __GFP_REPEAT);
- if (likely(pgd))
- memcpy(pgd + USER_PTRS_PER_PGD,
- swapper_pg_dir + USER_PTRS_PER_PGD,
- (PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
-
- return pgd;
+ return quicklist_alloc(QUICK_PGD, GFP_KERNEL | __GFP_REPEAT, pgd_ctor);
}
static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
{
- free_page((unsigned long)pgd);
+ quicklist_free(QUICK_PGD, NULL, pgd);
}
static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
@@ -83,6 +88,9 @@ do { \
tlb_remove_page((tlb), pte); \
} while (0)
-#define check_pgt_cache() do { } while(0)
+static inline void check_pgt_cache(void)
+{
+ quicklist_trim(QUICK_PGD, NULL, 25, 16);
+}
#endif /* __ASM_AVR32_PGALLOC_H */
--
1.5.5.4