This the 5th version patchset. New changes are:
1, tlb_flushall_shift knob move to debugfs.
2, include model 58 IVB mobile CPU tlb_flushall_shift setting
3, many small code changes response for most all of comments for v4.
Any more comments are appreciated!
Alex Shi
32bits kernel is using ILP32 model. 64 bits kernel is using LP64 model.
So, here TLB_FLUSH_ALL can be defined as '-1UL', that fit in both
x86_32/x86_64 mode.
Thanks for Rob Landley and Peter Avin's infos
http://www.unix.org/whitepapers/64bit.html
http://www.unix.org/version2/whatsnew/lp64_wp.html
Signed-off-by: Alex Shi <[email protected]>
---
arch/x86/include/asm/tlbflush.h | 6 +-----
1 files changed, 1 insertions(+), 5 deletions(-)
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index c0e108e..7e8a096 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -62,11 +62,7 @@ static inline void __flush_tlb_one(unsigned long addr)
__flush_tlb();
}
-#ifdef CONFIG_X86_32
-# define TLB_FLUSH_ALL 0xffffffff
-#else
-# define TLB_FLUSH_ALL -1ULL
-#endif
+#define TLB_FLUSH_ALL -1UL
/*
* TLB flushing:
--
1.7.5.4
For 4KB pages, x86 CPU has 2 or 1 level TLB, first level is data TLB and
instruction TLB, second level is shared TLB for both data and instructions.
For hupe page TLB, usually there is just one level and seperated by 2MB/4MB
and 1GB.
Although each levels TLB size is important for performance tuning, but for
genernal and rude optimizing, last level TLB entry number is suitable. And
in fact, last level TLB always has the biggest entry number.
This patch will get the biggest TLB entry number and use it in furture TLB
optimizing.
Accroding Borislav's suggestion, except tlb_ll[i/d]_* array, other
function and data will be released after system boot up.
For all kinds of x86 vendor friendly, vendor specific code was moved to its
specific files.
Signed-off-by: Alex Shi <[email protected]>
---
arch/x86/include/asm/processor.h | 11 +++
arch/x86/kernel/cpu/common.c | 21 ++++++
arch/x86/kernel/cpu/cpu.h | 9 +++
arch/x86/kernel/cpu/intel.c | 141 ++++++++++++++++++++++++++++++++++++++
4 files changed, 182 insertions(+), 0 deletions(-)
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 4fa7dcc..797faca 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -61,6 +61,17 @@ static inline void *current_text_addr(void)
# define ARCH_MIN_MMSTRUCT_ALIGN 0
#endif
+enum tlb_infos {
+ ENTRIES,
+ NR_INFO
+};
+
+extern u16 __read_mostly tlb_lli_4k[NR_INFO];
+extern u16 __read_mostly tlb_lli_2m[NR_INFO];
+extern u16 __read_mostly tlb_lli_4m[NR_INFO];
+extern u16 __read_mostly tlb_lld_4k[NR_INFO];
+extern u16 __read_mostly tlb_lld_2m[NR_INFO];
+extern u16 __read_mostly tlb_lld_4m[NR_INFO];
/*
* CPU type and hardware bug flags. Kept separately for each CPU.
* Members of this structure are referenced in head.S, so think twice
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index cf79302..0152082 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -452,6 +452,25 @@ void __cpuinit cpu_detect_cache_sizes(struct cpuinfo_x86 *c)
c->x86_cache_size = l2size;
}
+u16 __read_mostly tlb_lli_4k[NR_INFO];
+u16 __read_mostly tlb_lli_2m[NR_INFO];
+u16 __read_mostly tlb_lli_4m[NR_INFO];
+u16 __read_mostly tlb_lld_4k[NR_INFO];
+u16 __read_mostly tlb_lld_2m[NR_INFO];
+u16 __read_mostly tlb_lld_4m[NR_INFO];
+
+void __cpuinit cpu_detect_tlb(struct cpuinfo_x86 *c)
+{
+ if (c->x86_vendor == X86_VENDOR_INTEL)
+ intel_cpu_detect_tlb(c);
+
+ printk(KERN_INFO "Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n" \
+ "Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d\n",
+ tlb_lli_4k[ENTRIES], tlb_lli_2m[ENTRIES],
+ tlb_lli_4m[ENTRIES], tlb_lld_4k[ENTRIES],
+ tlb_lld_2m[ENTRIES], tlb_lld_4m[ENTRIES]);
+}
+
void __cpuinit detect_ht(struct cpuinfo_x86 *c)
{
#ifdef CONFIG_X86_HT
@@ -911,6 +930,8 @@ void __init identify_boot_cpu(void)
#else
vgetcpu_set_mode();
#endif
+ if (boot_cpu_data.cpuid_level >= 2)
+ cpu_detect_tlb(&boot_cpu_data);
}
void __cpuinit identify_secondary_cpu(struct cpuinfo_x86 *c)
diff --git a/arch/x86/kernel/cpu/cpu.h b/arch/x86/kernel/cpu/cpu.h
index 8bacc78..78db1d9 100644
--- a/arch/x86/kernel/cpu/cpu.h
+++ b/arch/x86/kernel/cpu/cpu.h
@@ -24,6 +24,14 @@ struct cpu_dev {
int c_x86_vendor;
};
+struct _tlb_table {
+ unsigned char descriptor;
+ char tlb_type;
+ unsigned int entries;
+ /* unsigned int ways; */
+ char info[128];
+};
+
#define cpu_dev_register(cpu_devX) \
static const struct cpu_dev *const __cpu_dev_##cpu_devX __used \
__attribute__((__section__(".x86_cpu_dev.init"))) = \
@@ -34,4 +42,5 @@ extern const struct cpu_dev *const __x86_cpu_dev_start[],
extern void get_cpu_cap(struct cpuinfo_x86 *c);
extern void cpu_detect_cache_sizes(struct cpuinfo_x86 *c);
+extern void intel_cpu_detect_tlb(struct cpuinfo_x86 *c) __cpuinit;
#endif /* ARCH_X86_CPU_H */
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index 3e6ff6c..28ecd1b 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -491,6 +491,147 @@ static unsigned int __cpuinit intel_size_cache(struct cpuinfo_x86 *c, unsigned i
}
#endif
+#define TLB_INST_4K 0x01
+#define TLB_INST_4M 0x02
+#define TLB_INST_2M_4M 0x03
+
+#define TLB_INST_ALL 0x05
+#define TLB_INST_1G 0x06
+
+#define TLB_DATA_4K 0x11
+#define TLB_DATA_4M 0x12
+#define TLB_DATA_2M_4M 0x13
+#define TLB_DATA_4K_4M 0x14
+
+#define TLB_DATA_1G 0x16
+
+#define TLB_DATA0_4K 0x21
+#define TLB_DATA0_4M 0x22
+#define TLB_DATA0_2M_4M 0x23
+
+#define STLB_4K 0x41
+
+static const struct _tlb_table intel_tlb_table[] __cpuinitconst = {
+ { 0x01, TLB_INST_4K, 32, " TLB_INST 4 KByte pages, 4-way set associative" },
+ { 0x02, TLB_INST_4M, 2, " TLB_INST 4 MByte pages, full associative" },
+ { 0x03, TLB_DATA_4K, 64, " TLB_DATA 4 KByte pages, 4-way set associative" },
+ { 0x04, TLB_DATA_4M, 8, " TLB_DATA 4 MByte pages, 4-way set associative" },
+ { 0x05, TLB_DATA_4M, 32, " TLB_DATA 4 MByte pages, 4-way set associative" },
+ { 0x0b, TLB_INST_4M, 4, " TLB_INST 4 MByte pages, 4-way set associative" },
+ { 0x4f, TLB_INST_4K, 32, " TLB_INST 4 KByte pages */" },
+ { 0x50, TLB_INST_ALL, 64, " TLB_INST 4 KByte and 2-MByte or 4-MByte pages" },
+ { 0x51, TLB_INST_ALL, 128, " TLB_INST 4 KByte and 2-MByte or 4-MByte pages" },
+ { 0x52, TLB_INST_ALL, 256, " TLB_INST 4 KByte and 2-MByte or 4-MByte pages" },
+ { 0x55, TLB_INST_2M_4M, 7, " TLB_INST 2-MByte or 4-MByte pages, fully associative" },
+ { 0x56, TLB_DATA0_4M, 16, " TLB_DATA0 4 MByte pages, 4-way set associative" },
+ { 0x57, TLB_DATA0_4K, 16, " TLB_DATA0 4 KByte pages, 4-way associative" },
+ { 0x59, TLB_DATA0_4K, 16, " TLB_DATA0 4 KByte pages, fully associative" },
+ { 0x5a, TLB_DATA0_2M_4M, 32, " TLB_DATA0 2-MByte or 4 MByte pages, 4-way set associative" },
+ { 0x5b, TLB_DATA_4K_4M, 64, " TLB_DATA 4 KByte and 4 MByte pages" },
+ { 0x5c, TLB_DATA_4K_4M, 128, " TLB_DATA 4 KByte and 4 MByte pages" },
+ { 0x5d, TLB_DATA_4K_4M, 256, " TLB_DATA 4 KByte and 4 MByte pages" },
+ { 0xb0, TLB_INST_4K, 128, " TLB_INST 4 KByte pages, 4-way set associative" },
+ { 0xb1, TLB_INST_2M_4M, 4, " TLB_INST 2M pages, 4-way, 8 entries or 4M pages, 4-way entries" },
+ { 0xb2, TLB_INST_4K, 64, " TLB_INST 4KByte pages, 4-way set associative" },
+ { 0xb3, TLB_DATA_4K, 128, " TLB_DATA 4 KByte pages, 4-way set associative" },
+ { 0xb4, TLB_DATA_4K, 256, " TLB_DATA 4 KByte pages, 4-way associative" },
+ { 0xba, TLB_DATA_4K, 64, " TLB_DATA 4 KByte pages, 4-way associative" },
+ { 0xc0, TLB_DATA_4K_4M, 8, " TLB_DATA 4 KByte and 4 MByte pages, 4-way associative" },
+ { 0xca, STLB_4K, 512, " STLB 4 KByte pages, 4-way associative" },
+ { 0x00, 0, 0 }
+};
+
+static void __cpuinit intel_tlb_lookup(const unsigned char desc)
+{
+ unsigned char k;
+ if (desc == 0)
+ return;
+
+ /* look up this descriptor in the table */
+ for (k = 0; intel_tlb_table[k].descriptor != desc && \
+ intel_tlb_table[k].descriptor != 0; k++)
+ ;
+
+ if (intel_tlb_table[k].tlb_type == 0)
+ return;
+
+ switch (intel_tlb_table[k].tlb_type) {
+ case STLB_4K:
+ if (tlb_lli_4k[ENTRIES] < intel_tlb_table[k].entries)
+ tlb_lli_4k[ENTRIES] = intel_tlb_table[k].entries;
+ if (tlb_lld_4k[ENTRIES] < intel_tlb_table[k].entries)
+ tlb_lld_4k[ENTRIES] = intel_tlb_table[k].entries;
+ break;
+ case TLB_INST_ALL:
+ if (tlb_lli_4k[ENTRIES] < intel_tlb_table[k].entries)
+ tlb_lli_4k[ENTRIES] = intel_tlb_table[k].entries;
+ if (tlb_lli_2m[ENTRIES] < intel_tlb_table[k].entries)
+ tlb_lli_2m[ENTRIES] = intel_tlb_table[k].entries;
+ if (tlb_lli_4m[ENTRIES] < intel_tlb_table[k].entries)
+ tlb_lli_4m[ENTRIES] = intel_tlb_table[k].entries;
+ break;
+ case TLB_INST_4K:
+ if (tlb_lli_4k[ENTRIES] < intel_tlb_table[k].entries)
+ tlb_lli_4k[ENTRIES] = intel_tlb_table[k].entries;
+ break;
+ case TLB_INST_4M:
+ if (tlb_lli_4m[ENTRIES] < intel_tlb_table[k].entries)
+ tlb_lli_4m[ENTRIES] = intel_tlb_table[k].entries;
+ break;
+ case TLB_INST_2M_4M:
+ if (tlb_lli_2m[ENTRIES] < intel_tlb_table[k].entries)
+ tlb_lli_2m[ENTRIES] = intel_tlb_table[k].entries;
+ if (tlb_lli_4m[ENTRIES] < intel_tlb_table[k].entries)
+ tlb_lli_4m[ENTRIES] = intel_tlb_table[k].entries;
+ break;
+ case TLB_DATA_4K:
+ case TLB_DATA0_4K:
+ if (tlb_lld_4k[ENTRIES] < intel_tlb_table[k].entries)
+ tlb_lld_4k[ENTRIES] = intel_tlb_table[k].entries;
+ break;
+ case TLB_DATA_4M:
+ case TLB_DATA0_4M:
+ if (tlb_lld_4m[ENTRIES] < intel_tlb_table[k].entries)
+ tlb_lld_4m[ENTRIES] = intel_tlb_table[k].entries;
+ break;
+ case TLB_DATA_2M_4M:
+ case TLB_DATA0_2M_4M:
+ if (tlb_lld_2m[ENTRIES] < intel_tlb_table[k].entries)
+ tlb_lld_2m[ENTRIES] = intel_tlb_table[k].entries;
+ if (tlb_lld_4m[ENTRIES] < intel_tlb_table[k].entries)
+ tlb_lld_4m[ENTRIES] = intel_tlb_table[k].entries;
+ break;
+ case TLB_DATA_4K_4M:
+ if (tlb_lld_4k[ENTRIES] < intel_tlb_table[k].entries)
+ tlb_lld_4k[ENTRIES] = intel_tlb_table[k].entries;
+ if (tlb_lld_4m[ENTRIES] < intel_tlb_table[k].entries)
+ tlb_lld_4m[ENTRIES] = intel_tlb_table[k].entries;
+ break;
+ }
+}
+
+void __cpuinit intel_cpu_detect_tlb(struct cpuinfo_x86 *c)
+{
+ int i, j, n;
+ unsigned int regs[4];
+ unsigned char *desc = (unsigned char *)regs;
+ /* Number of times to iterate */
+ n = cpuid_eax(2) & 0xFF;
+
+ for (i = 0 ; i < n ; i++) {
+ cpuid(2, ®s[0], ®s[1], ®s[2], ®s[3]);
+
+ /* If bit 31 is set, this is an unknown format */
+ for (j = 0 ; j < 3 ; j++)
+ if (regs[j] & (1 << 31))
+ regs[j] = 0;
+
+ /* Byte 0 is level count, not a descriptor */
+ for (j = 1 ; j < 16 ; j++)
+ intel_tlb_lookup(desc[j]);
+ }
+}
+
static const struct cpu_dev __cpuinitconst intel_cpu_dev = {
.c_vendor = "Intel",
.c_ident = { "GenuineIntel" },
--
1.7.5.4
x86 has no flush_tlb_range support in instruction level. Currently the
flush_tlb_range just implemented by flushing all page table. That is not
the best solution for all scenarios. In fact, if we just use 'invlpg' to
flush few lines from TLB, we can get the performance gain from later
remain TLB lines accessing.
But the 'invlpg' instruction costs much of time. Its execution time can
compete with cr3 rewriting, and even a bit more on SNB CPU.
So, on a 512 4KB TLB entries CPU, the balance points is at:
(512 - X) * 100ns(assumed TLB refill cost) =
X(TLB flush entries) * 100ns(assumed invlpg cost)
Here, X is 256, that is 1/2 of 512 entries.
But with the mysterious CPU pre-fetcher and page miss handler Unit, the
assumed TLB refill cost is far lower then 100ns in sequential access. And
2 HT siblings in one core makes the memory access more faster if they are
accessing the same memory. So, in the patch, I just do the change when
the target entries is less than 1/16 of whole active tlb entries.
Actually, I have no data support for the percentage '1/16', so any
suggestions are welcomed.
As to hugetlb, guess due to smaller page table, and smaller active TLB
entries, I didn't see benefit via my benchmark, so no optimizing now.
My macro benchmark show in ideal scenarios, the performance improves 70
percent in reading. And in worst scenario, the reading/writing
performance is similar with unpatched 3.4-rc4 kernel.
Here is the reading data on my 2P * 4cores *HT NHM EP machine, with THP
'always':
multi thread testing, '-t' paramter is thread number:
with patch unpatched 3.4-rc4
./mprotect -t 1 14ns 24ns
./mprotect -t 2 13ns 22ns
./mprotect -t 4 12ns 19ns
./mprotect -t 8 14ns 16ns
./mprotect -t 16 28ns 26ns
./mprotect -t 32 54ns 51ns
./mprotect -t 128 200ns 199ns
Single process with sequencial flushing and memory accessing:
with patch unpatched 3.4-rc4
./mprotect 7ns 11ns
./mprotect -p 4096 -l 8 -n 10240
21ns 21ns
I also tried other benchmarks on Intel core2/NHM/SNB EP and NHM EX machine.
No clear performance change on specjbb2005 with openjdk, and oltp reading.
Signed-off-by: Alex Shi <[email protected]>
---
arch/x86/include/asm/paravirt.h | 5 +-
arch/x86/include/asm/paravirt_types.h | 3 +-
arch/x86/include/asm/tlbflush.h | 23 +++-----
arch/x86/include/asm/uv/uv.h | 5 +-
arch/x86/mm/tlb.c | 97 +++++++++++++++++++++++++++------
arch/x86/platform/uv/tlb_uv.c | 6 +-
arch/x86/xen/mmu.c | 9 ++--
include/trace/events/xen.h | 12 +++--
8 files changed, 113 insertions(+), 47 deletions(-)
diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
index aa0f913..03da4ab 100644
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -397,9 +397,10 @@ static inline void __flush_tlb_single(unsigned long addr)
static inline void flush_tlb_others(const struct cpumask *cpumask,
struct mm_struct *mm,
- unsigned long va)
+ unsigned long start,
+ unsigned long end)
{
- PVOP_VCALL3(pv_mmu_ops.flush_tlb_others, cpumask, mm, va);
+ PVOP_VCALL4(pv_mmu_ops.flush_tlb_others, cpumask, mm, start, end);
}
static inline int paravirt_pgd_alloc(struct mm_struct *mm)
diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h
index 8e8b9a4..600a5fcac9 100644
--- a/arch/x86/include/asm/paravirt_types.h
+++ b/arch/x86/include/asm/paravirt_types.h
@@ -250,7 +250,8 @@ struct pv_mmu_ops {
void (*flush_tlb_single)(unsigned long addr);
void (*flush_tlb_others)(const struct cpumask *cpus,
struct mm_struct *mm,
- unsigned long va);
+ unsigned long start,
+ unsigned long end);
/* Hooks for allocating and freeing a pagetable top-level */
int (*pgd_alloc)(struct mm_struct *mm);
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index 7e8a096..c39c94e 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -73,14 +73,10 @@ static inline void __flush_tlb_one(unsigned long addr)
* - flush_tlb_page(vma, vmaddr) flushes one page
* - flush_tlb_range(vma, start, end) flushes a range of pages
* - flush_tlb_kernel_range(start, end) flushes a range of kernel pages
- * - flush_tlb_others(cpumask, mm, va) flushes TLBs on other cpus
+ * - flush_tlb_others(cpumask, mm, start, end) flushes TLBs on other cpus
*
* ..but the i386 has somewhat limited tlb flushing capabilities,
* and page-granular flushes are available only on i486 and up.
- *
- * x86-64 can only flush individual pages or full VMs. For a range flush
- * we always do the full VM. Might be worth trying if for a small
- * range a few INVLPGs in a row are a win.
*/
#ifndef CONFIG_SMP
@@ -111,7 +107,8 @@ static inline void flush_tlb_range(struct vm_area_struct *vma,
static inline void native_flush_tlb_others(const struct cpumask *cpumask,
struct mm_struct *mm,
- unsigned long va)
+ unsigned long start,
+ unsigned long end)
{
}
@@ -129,17 +126,14 @@ extern void flush_tlb_all(void);
extern void flush_tlb_current_task(void);
extern void flush_tlb_mm(struct mm_struct *);
extern void flush_tlb_page(struct vm_area_struct *, unsigned long);
+extern void flush_tlb_range(struct vm_area_struct *vma,
+ unsigned long start, unsigned long end);
#define flush_tlb() flush_tlb_current_task()
-static inline void flush_tlb_range(struct vm_area_struct *vma,
- unsigned long start, unsigned long end)
-{
- flush_tlb_mm(vma->vm_mm);
-}
-
void native_flush_tlb_others(const struct cpumask *cpumask,
- struct mm_struct *mm, unsigned long va);
+ struct mm_struct *mm,
+ unsigned long start, unsigned long end);
#define TLBSTATE_OK 1
#define TLBSTATE_LAZY 2
@@ -159,7 +153,8 @@ static inline void reset_lazy_tlbstate(void)
#endif /* SMP */
#ifndef CONFIG_PARAVIRT
-#define flush_tlb_others(mask, mm, va) native_flush_tlb_others(mask, mm, va)
+#define flush_tlb_others(mask, mm, start, end) \
+ native_flush_tlb_others(mask, mm, start, end)
#endif
static inline void flush_tlb_kernel_range(unsigned long start,
diff --git a/arch/x86/include/asm/uv/uv.h b/arch/x86/include/asm/uv/uv.h
index 3bb9491..b47c2a8 100644
--- a/arch/x86/include/asm/uv/uv.h
+++ b/arch/x86/include/asm/uv/uv.h
@@ -15,7 +15,8 @@ extern void uv_nmi_init(void);
extern void uv_system_init(void);
extern const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask,
struct mm_struct *mm,
- unsigned long va,
+ unsigned long start,
+ unsigned end,
unsigned int cpu);
#else /* X86_UV */
@@ -26,7 +27,7 @@ static inline void uv_cpu_init(void) { }
static inline void uv_system_init(void) { }
static inline const struct cpumask *
uv_flush_tlb_others(const struct cpumask *cpumask, struct mm_struct *mm,
- unsigned long va, unsigned int cpu)
+ unsigned long start, unsigned long end, unsigned int cpu)
{ return cpumask; }
#endif /* X86_UV */
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index d6c0418..7d92079 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -41,7 +41,8 @@ DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate)
union smp_flush_state {
struct {
struct mm_struct *flush_mm;
- unsigned long flush_va;
+ unsigned long flush_start;
+ unsigned long flush_end;
raw_spinlock_t tlbstate_lock;
DECLARE_BITMAP(flush_cpumask, NR_CPUS);
};
@@ -154,10 +155,19 @@ void smp_invalidate_interrupt(struct pt_regs *regs)
if (f->flush_mm == percpu_read(cpu_tlbstate.active_mm)) {
if (percpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
- if (f->flush_va == TLB_FLUSH_ALL)
+ if (f->flush_end == TLB_FLUSH_ALL
+ || !cpu_has_invlpg)
local_flush_tlb();
- else
- __flush_tlb_one(f->flush_va);
+ else if (!f->flush_end)
+ __flush_tlb_single(f->flush_start);
+ else {
+ unsigned long addr;
+ addr = f->flush_start;
+ while (addr <= f->flush_end) {
+ __flush_tlb_single(addr);
+ addr += PAGE_SIZE;
+ }
+ }
} else
leave_mm(cpu);
}
@@ -170,7 +180,8 @@ out:
}
static void flush_tlb_others_ipi(const struct cpumask *cpumask,
- struct mm_struct *mm, unsigned long va)
+ struct mm_struct *mm, unsigned long start,
+ unsigned long end)
{
unsigned int sender;
union smp_flush_state *f;
@@ -183,7 +194,8 @@ static void flush_tlb_others_ipi(const struct cpumask *cpumask,
raw_spin_lock(&f->tlbstate_lock);
f->flush_mm = mm;
- f->flush_va = va;
+ f->flush_start = start;
+ f->flush_end = end;
if (cpumask_andnot(to_cpumask(f->flush_cpumask), cpumask, cpumask_of(smp_processor_id()))) {
/*
* We have to send the IPI only to
@@ -197,24 +209,26 @@ static void flush_tlb_others_ipi(const struct cpumask *cpumask,
}
f->flush_mm = NULL;
- f->flush_va = 0;
+ f->flush_start = 0;
+ f->flush_end = 0;
if (nr_cpu_ids > NUM_INVALIDATE_TLB_VECTORS)
raw_spin_unlock(&f->tlbstate_lock);
}
void native_flush_tlb_others(const struct cpumask *cpumask,
- struct mm_struct *mm, unsigned long va)
+ struct mm_struct *mm, unsigned long start,
+ unsigned long end)
{
if (is_uv_system()) {
unsigned int cpu;
cpu = smp_processor_id();
- cpumask = uv_flush_tlb_others(cpumask, mm, va, cpu);
+ cpumask = uv_flush_tlb_others(cpumask, mm, start, end, cpu);
if (cpumask)
- flush_tlb_others_ipi(cpumask, mm, va);
+ flush_tlb_others_ipi(cpumask, mm, start, end);
return;
}
- flush_tlb_others_ipi(cpumask, mm, va);
+ flush_tlb_others_ipi(cpumask, mm, start, end);
}
static void __cpuinit calculate_tlb_offset(void)
@@ -280,7 +294,7 @@ void flush_tlb_current_task(void)
local_flush_tlb();
if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
- flush_tlb_others(mm_cpumask(mm), mm, TLB_FLUSH_ALL);
+ flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
preempt_enable();
}
@@ -295,12 +309,63 @@ void flush_tlb_mm(struct mm_struct *mm)
leave_mm(smp_processor_id());
}
if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
- flush_tlb_others(mm_cpumask(mm), mm, TLB_FLUSH_ALL);
+ flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
+
+ preempt_enable();
+}
+
+#define FLUSHALL_BAR 16
+
+void flush_tlb_range(struct vm_area_struct *vma,
+ unsigned long start, unsigned long end)
+{
+ struct mm_struct *mm;
+
+ if (!cpu_has_invlpg || vma->vm_flags & VM_HUGETLB) {
+ flush_tlb_mm(vma->vm_mm);
+ return;
+ }
+
+ preempt_disable();
+ mm = vma->vm_mm;
+ if (current->active_mm == mm) {
+ if (current->mm) {
+ unsigned long addr, vmflag = vma->vm_flags;
+ unsigned act_entries, tlb_entries = 0;
+
+ if (vmflag & VM_EXEC)
+ tlb_entries = tlb_lli_4k[ENTRIES];
+ else
+ tlb_entries = tlb_lld_4k[ENTRIES];
+
+ act_entries = tlb_entries > mm->total_vm ?
+ mm->total_vm : tlb_entries;
+ if ((end - start)/PAGE_SIZE > act_entries/FLUSHALL_BAR)
+ local_flush_tlb();
+ else {
+ for (addr = start; addr <= end;
+ addr += PAGE_SIZE)
+ __flush_tlb_single(addr);
+
+ if (cpumask_any_but(mm_cpumask(mm),
+ smp_processor_id()) < nr_cpu_ids)
+ flush_tlb_others(mm_cpumask(mm), mm,
+ start, end);
+ preempt_enable();
+ return;
+ }
+ } else {
+ leave_mm(smp_processor_id());
+ }
+ }
+ if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
+ flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
preempt_enable();
}
-void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
+
+void flush_tlb_page(struct vm_area_struct *vma, unsigned long start)
{
struct mm_struct *mm = vma->vm_mm;
@@ -308,13 +373,13 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
if (current->active_mm == mm) {
if (current->mm)
- __flush_tlb_one(va);
+ __flush_tlb_one(start);
else
leave_mm(smp_processor_id());
}
if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
- flush_tlb_others(mm_cpumask(mm), mm, va);
+ flush_tlb_others(mm_cpumask(mm), mm, start, 0UL);
preempt_enable();
}
diff --git a/arch/x86/platform/uv/tlb_uv.c b/arch/x86/platform/uv/tlb_uv.c
index 3ae0e61..0df5ad2 100644
--- a/arch/x86/platform/uv/tlb_uv.c
+++ b/arch/x86/platform/uv/tlb_uv.c
@@ -1068,8 +1068,8 @@ static int set_distrib_bits(struct cpumask *flush_mask, struct bau_control *bcp,
* done. The returned pointer is valid till preemption is re-enabled.
*/
const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask,
- struct mm_struct *mm, unsigned long va,
- unsigned int cpu)
+ struct mm_struct *mm, unsigned long start,
+ unsigned end, unsigned int cpu)
{
int locals = 0;
int remotes = 0;
@@ -1112,7 +1112,7 @@ const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask,
record_send_statistics(stat, locals, hubs, remotes, bau_desc);
- bau_desc->payload.address = va;
+ bau_desc->payload.address = start;
bau_desc->payload.sending_cpu = cpu;
/*
* uv_flush_send_and_wait returns 0 if all cpu's were messaged,
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index b8e2794..75bab52 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -1239,7 +1239,8 @@ static void xen_flush_tlb_single(unsigned long addr)
}
static void xen_flush_tlb_others(const struct cpumask *cpus,
- struct mm_struct *mm, unsigned long va)
+ struct mm_struct *mm, unsigned long start,
+ unsigned long end)
{
struct {
struct mmuext_op op;
@@ -1251,7 +1252,7 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,
} *args;
struct multicall_space mcs;
- trace_xen_mmu_flush_tlb_others(cpus, mm, va);
+ trace_xen_mmu_flush_tlb_others(cpus, mm, start, end);
if (cpumask_empty(cpus))
return; /* nothing to do */
@@ -1264,11 +1265,11 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,
cpumask_and(to_cpumask(args->mask), cpus, cpu_online_mask);
cpumask_clear_cpu(smp_processor_id(), to_cpumask(args->mask));
- if (va == TLB_FLUSH_ALL) {
+ if (start == TLB_FLUSH_ALL) {
args->op.cmd = MMUEXT_TLB_FLUSH_MULTI;
} else {
args->op.cmd = MMUEXT_INVLPG_MULTI;
- args->op.arg1.linear_addr = va;
+ args->op.arg1.linear_addr = start;
}
MULTI_mmuext_op(mcs.mc, &args->op, 1, NULL, DOMID_SELF);
diff --git a/include/trace/events/xen.h b/include/trace/events/xen.h
index 92f1a79..15ba03b 100644
--- a/include/trace/events/xen.h
+++ b/include/trace/events/xen.h
@@ -397,18 +397,20 @@ TRACE_EVENT(xen_mmu_flush_tlb_single,
TRACE_EVENT(xen_mmu_flush_tlb_others,
TP_PROTO(const struct cpumask *cpus, struct mm_struct *mm,
- unsigned long addr),
- TP_ARGS(cpus, mm, addr),
+ unsigned long addr, unsigned long end),
+ TP_ARGS(cpus, mm, addr, end),
TP_STRUCT__entry(
__field(unsigned, ncpus)
__field(struct mm_struct *, mm)
__field(unsigned long, addr)
+ __field(unsigned long, end)
),
TP_fast_assign(__entry->ncpus = cpumask_weight(cpus);
__entry->mm = mm;
- __entry->addr = addr),
- TP_printk("ncpus %d mm %p addr %lx",
- __entry->ncpus, __entry->mm, __entry->addr)
+ __entry->addr = addr,
+ __entry->end = end),
+ TP_printk("ncpus %d mm %p addr %lx, end %lx",
+ __entry->ncpus, __entry->mm, __entry->addr, __entry->end)
);
TRACE_EVENT(xen_mmu_write_cr3,
--
1.7.5.4
We don't need to flush large pages by PAGE_SIZE step, that just waste
time. and actually, large page don't need 'invlpg' optimizing according
to our macro benchmark. So, just flush whole TLB is enough for them.
The following result is tested on a 2CPU * 4cores * 2HT NHM EP machine,
with THP 'always' setting.
Multi-thread testing, '-t' paramter is thread number:
without this patch with this patch
./mprotect -t 1 14ns 13ns
./mprotect -t 2 13ns 13ns
./mprotect -t 4 12ns 11ns
./mprotect -t 8 14ns 10ns
./mprotect -t 16 28ns 28ns
./mprotect -t 32 54ns 52ns
./mprotect -t 128 200ns 200ns
Signed-off-by: Alex Shi <[email protected]>
---
arch/x86/mm/tlb.c | 34 ++++++++++++++++++++++++++++++++++
1 files changed, 34 insertions(+), 0 deletions(-)
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 7d92079..22e5bb1 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -316,12 +316,42 @@ void flush_tlb_mm(struct mm_struct *mm)
#define FLUSHALL_BAR 16
+#ifdef CONFIG_TRANSPARENT_HUGEPAGE
+static inline int has_large_page(struct mm_struct *mm,
+ unsigned long start, unsigned long end)
+{
+ pgd_t *pgd;
+ pud_t *pud;
+ pmd_t *pmd;
+ unsigned long addr = ALIGN(start, HPAGE_SIZE);
+ for (; addr < end; addr += HPAGE_SIZE) {
+ pgd = pgd_offset(mm, addr);
+ if (likely(!pgd_none(*pgd))) {
+ pud = pud_offset(pgd, addr);
+ if (likely(!pud_none(*pud))) {
+ pmd = pmd_offset(pud, addr);
+ if (likely(!pmd_none(*pmd)))
+ if (pmd_large(*pmd))
+ return 1;
+ }
+ }
+ }
+ return 0;
+}
+#else
+static inline int has_large_page(struct mm_struct *mm,
+ unsigned long start, unsigned long end)
+{
+ return 0;
+}
+#endif
void flush_tlb_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end)
{
struct mm_struct *mm;
if (!cpu_has_invlpg || vma->vm_flags & VM_HUGETLB) {
+flush_all:
flush_tlb_mm(vma->vm_mm);
return;
}
@@ -344,6 +374,10 @@ void flush_tlb_range(struct vm_area_struct *vma,
if ((end - start)/PAGE_SIZE > act_entries/FLUSHALL_BAR)
local_flush_tlb();
else {
+ if (has_large_page(mm, start, end)) {
+ preempt_enable();
+ goto flush_all;
+ }
for (addr = start; addr <= end;
addr += PAGE_SIZE)
__flush_tlb_single(addr);
--
1.7.5.4
Not every flush_tlb_mm execution moment is really need to evacuate all
TLB entries, like in munmap, just few 'invlpg' is better for whole
process performance, since it leaves most of TLB entries for later
accessing.
This patch is changing flush_tlb_mm(mm) to flush_tlb_mm(mm, start, end)
in cases.
The performance balance points checking is left in __flush_tlb_range()
Signed-off-by: Alex Shi <[email protected]>
---
arch/x86/include/asm/tlb.h | 2 +-
arch/x86/include/asm/tlbflush.h | 5 +-
arch/x86/mm/pgtable.c | 2 +-
arch/x86/mm/tlb.c | 114 +++++++++++++++++++-------------------
fs/proc/task_mmu.c | 2 +-
include/asm-generic/tlb.h | 5 +-
include/asm-generic/tlbflush.h | 3 +-
kernel/fork.c | 2 +-
mm/memory.c | 9 ++--
9 files changed, 74 insertions(+), 70 deletions(-)
diff --git a/arch/x86/include/asm/tlb.h b/arch/x86/include/asm/tlb.h
index 829215f..505fdfe 100644
--- a/arch/x86/include/asm/tlb.h
+++ b/arch/x86/include/asm/tlb.h
@@ -4,7 +4,7 @@
#define tlb_start_vma(tlb, vma) do { } while (0)
#define tlb_end_vma(tlb, vma) do { } while (0)
#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
-#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
+#define tlb_flush(tlb, start, end) flush_tlb_mm((tlb)->mm, start, end)
#include <asm-generic/tlb.h>
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index c39c94e..1d07cf1 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -85,7 +85,8 @@ static inline void __flush_tlb_one(unsigned long addr)
#define flush_tlb_all() __flush_tlb_all()
#define local_flush_tlb() __flush_tlb()
-static inline void flush_tlb_mm(struct mm_struct *mm)
+static inline void flush_tlb_mm(struct mm_struct *mm,
+ unsigned long start, unsigned long end)
{
if (mm == current->active_mm)
__flush_tlb();
@@ -124,7 +125,7 @@ static inline void reset_lazy_tlbstate(void)
extern void flush_tlb_all(void);
extern void flush_tlb_current_task(void);
-extern void flush_tlb_mm(struct mm_struct *);
+extern void flush_tlb_mm(struct mm_struct *, unsigned long, unsigned long);
extern void flush_tlb_page(struct vm_area_struct *, unsigned long);
extern void flush_tlb_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end);
diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c
index 8573b83..5aea5b0 100644
--- a/arch/x86/mm/pgtable.c
+++ b/arch/x86/mm/pgtable.c
@@ -168,7 +168,7 @@ void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd)
* section 8.1: in PAE mode we explicitly have to flush the
* TLB via cr3 if the top-level pgd is changed...
*/
- flush_tlb_mm(mm);
+ flush_tlb_mm(mm, 0UL, -1UL);
}
#else /* !CONFIG_X86_PAE */
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 5bf4e85..b2e25e8 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -298,22 +298,6 @@ void flush_tlb_current_task(void)
preempt_enable();
}
-void flush_tlb_mm(struct mm_struct *mm)
-{
- preempt_disable();
-
- if (current->active_mm == mm) {
- if (current->mm)
- local_flush_tlb();
- else
- leave_mm(smp_processor_id());
- }
- if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
- flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
-
- preempt_enable();
-}
-
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
static inline int has_large_page(struct mm_struct *mm,
unsigned long start, unsigned long end)
@@ -343,61 +327,77 @@ static inline int has_large_page(struct mm_struct *mm,
return 0;
}
#endif
-void flush_tlb_range(struct vm_area_struct *vma,
- unsigned long start, unsigned long end)
-{
- struct mm_struct *mm;
- if (!cpu_has_invlpg || vma->vm_flags & VM_HUGETLB
- || tlb_flushall_shift == (u16)TLB_FLUSH_ALL) {
-flush_all:
- flush_tlb_mm(vma->vm_mm);
- return;
- }
+void __flush_tlb_range(struct mm_struct *mm, unsigned long start,
+ unsigned long end, unsigned long vmflag)
+{
+ unsigned long addr;
+ unsigned act_entries, tlb_entries = 0;
preempt_disable();
- mm = vma->vm_mm;
- if (current->active_mm == mm) {
- if (current->mm) {
- unsigned long addr, vmflag = vma->vm_flags;
- unsigned act_entries, tlb_entries = 0;
+ if (current->active_mm != mm)
+ goto flush_all;
- if (vmflag & VM_EXEC)
- tlb_entries = tlb_lli_4k[ENTRIES];
- else
- tlb_entries = tlb_lld_4k[ENTRIES];
+ if (!current->mm) {
+ leave_mm(smp_processor_id());
+ goto flush_all;
+ }
- act_entries = tlb_entries > mm->total_vm ?
- mm->total_vm : tlb_entries;
+ if (end == TLB_FLUSH_ALL ||
+ tlb_flushall_shift == (u16)TLB_FLUSH_ALL) {
+ local_flush_tlb();
+ goto flush_all;
+ }
- if ((end - start) >> PAGE_SHIFT >
- act_entries >> tlb_flushall_shift)
- local_flush_tlb();
- else {
- if (has_large_page(mm, start, end)) {
- preempt_enable();
- goto flush_all;
- }
- for (addr = start; addr <= end;
- addr += PAGE_SIZE)
- __flush_tlb_single(addr);
+ if (vmflag & VM_EXEC)
+ tlb_entries = tlb_lli_4k[ENTRIES];
+ else
+ tlb_entries = tlb_lld_4k[ENTRIES];
+ act_entries = mm->total_vm > tlb_entries ? tlb_entries : mm->total_vm;
- if (cpumask_any_but(mm_cpumask(mm),
- smp_processor_id()) < nr_cpu_ids)
- flush_tlb_others(mm_cpumask(mm), mm,
- start, end);
- preempt_enable();
- return;
- }
- } else {
- leave_mm(smp_processor_id());
+ if ((end - start) >> PAGE_SHIFT > act_entries >> tlb_flushall_shift)
+ local_flush_tlb();
+ else {
+ if (has_large_page(mm, start, end)) {
+ local_flush_tlb();
+ goto flush_all;
}
+ for (addr = start; addr <= end; addr += PAGE_SIZE)
+ __flush_tlb_single(addr);
+
+ if (cpumask_any_but(mm_cpumask(mm),
+ smp_processor_id()) < nr_cpu_ids)
+ flush_tlb_others(mm_cpumask(mm), mm, start, end);
+ preempt_enable();
+ return;
}
+
+flush_all:
if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
preempt_enable();
}
+void flush_tlb_mm(struct mm_struct *mm, unsigned long start, unsigned long end)
+{
+ if (!cpu_has_invlpg)
+ __flush_tlb_range(mm, 0UL, TLB_FLUSH_ALL, 0UL);
+ else
+ __flush_tlb_range(mm, start, end, 0UL);
+}
+
+void flush_tlb_range(struct vm_area_struct *vma,
+ unsigned long start, unsigned long end)
+{
+ struct mm_struct *mm = vma->vm_mm;
+ unsigned long vmflag = vma->vm_flags;
+
+ if (!cpu_has_invlpg || vma->vm_flags & VM_HUGETLB)
+ __flush_tlb_range(mm, 0UL, TLB_FLUSH_ALL, 0UL);
+ else
+ __flush_tlb_range(mm, start, end, vmflag);
+}
+
void flush_tlb_page(struct vm_area_struct *vma, unsigned long start)
{
diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
index 2d60492..5728c8f 100644
--- a/fs/proc/task_mmu.c
+++ b/fs/proc/task_mmu.c
@@ -660,7 +660,7 @@ static ssize_t clear_refs_write(struct file *file, const char __user *buf,
walk_page_range(vma->vm_start, vma->vm_end,
&clear_refs_walk);
}
- flush_tlb_mm(mm);
+ flush_tlb_mm(mm, 0UL, -1UL);
up_read(&mm->mmap_sem);
mmput(mm);
}
diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
index 75e888b..3a24c97 100644
--- a/include/asm-generic/tlb.h
+++ b/include/asm-generic/tlb.h
@@ -112,7 +112,8 @@ static inline int tlb_fast_mode(struct mmu_gather *tlb)
}
void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, bool fullmm);
-void tlb_flush_mmu(struct mmu_gather *tlb);
+void tlb_flush_mmu(struct mmu_gather *tlb, unsigned long start,
+ unsigned long end);
void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start,
unsigned long end);
int __tlb_remove_page(struct mmu_gather *tlb, struct page *page);
@@ -124,7 +125,7 @@ int __tlb_remove_page(struct mmu_gather *tlb, struct page *page);
static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
{
if (!__tlb_remove_page(tlb, page))
- tlb_flush_mmu(tlb);
+ tlb_flush_mmu(tlb, 0UL, -1UL);
}
/**
diff --git a/include/asm-generic/tlbflush.h b/include/asm-generic/tlbflush.h
index d6d0a88..db1d4bb 100644
--- a/include/asm-generic/tlbflush.h
+++ b/include/asm-generic/tlbflush.h
@@ -11,7 +11,8 @@
#include <linux/bug.h>
-static inline void flush_tlb_mm(struct mm_struct *mm)
+static inline void flush_tlb_mm(struct mm_struct *mm,
+ unsigned long start, unsigned long end)
{
BUG();
}
diff --git a/kernel/fork.c b/kernel/fork.c
index b9372a0..a4f0c64 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -427,7 +427,7 @@ static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm)
retval = 0;
out:
up_write(&mm->mmap_sem);
- flush_tlb_mm(oldmm);
+ flush_tlb_mm(oldmm, 0UL, -1UL);
up_write(&oldmm->mmap_sem);
return retval;
fail_nomem_anon_vma_fork:
diff --git a/mm/memory.c b/mm/memory.c
index 6105f47..05e2c2e 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -218,14 +218,15 @@ void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, bool fullmm)
#endif
}
-void tlb_flush_mmu(struct mmu_gather *tlb)
+void tlb_flush_mmu(struct mmu_gather *tlb,
+ unsigned long start, unsigned long end)
{
struct mmu_gather_batch *batch;
if (!tlb->need_flush)
return;
tlb->need_flush = 0;
- tlb_flush(tlb);
+ tlb_flush(tlb, start, end);
#ifdef CONFIG_HAVE_RCU_TABLE_FREE
tlb_table_flush(tlb);
#endif
@@ -248,7 +249,7 @@ void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long e
{
struct mmu_gather_batch *batch, *next;
- tlb_flush_mmu(tlb);
+ tlb_flush_mmu(tlb, start, end);
/* keep the page table cache within bounds */
check_pgt_cache();
@@ -1204,7 +1205,7 @@ again:
*/
if (force_flush) {
force_flush = 0;
- tlb_flush_mmu(tlb);
+ tlb_flush_mmu(tlb, addr, end);
if (addr != end)
goto again;
}
--
1.7.5.4
Testing show different CPU type(micro architectures and NUMA mode) has
different balance points between the TLB flush all and multiple invlpg.
And there also has cases the tlb flush change has no any help.
This patch give a interface to let x86 vendor developers have a chance
to set different shift for different CPU type.
like some machine in my hands, balance points is 16 entries on
Romely-EP; while it is at 8 entries on Bloomfield NHM-EP; but on model
15 core2 Xeon using invlpg has nothing help.
For untested machine, do a conservative optimization, same as NHM CPU.
Signed-off-by: Alex Shi <[email protected]>
---
arch/x86/include/asm/processor.h | 2 ++
arch/x86/kernel/cpu/common.c | 14 ++++++++++++--
arch/x86/kernel/cpu/intel.c | 34 ++++++++++++++++++++++++++++++++++
arch/x86/mm/tlb.c | 8 ++++----
include/asm-generic/tlb.h | 3 ++-
5 files changed, 54 insertions(+), 7 deletions(-)
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 797faca..3c0cc3d 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -72,6 +72,8 @@ extern u16 __read_mostly tlb_lli_4m[NR_INFO];
extern u16 __read_mostly tlb_lld_4k[NR_INFO];
extern u16 __read_mostly tlb_lld_2m[NR_INFO];
extern u16 __read_mostly tlb_lld_4m[NR_INFO];
+extern u16 __read_mostly tlb_flushall_shift;
+
/*
* CPU type and hardware bug flags. Kept separately for each CPU.
* Members of this structure are referenced in head.S, so think twice
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 0152082..24255ca 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -459,16 +459,26 @@ u16 __read_mostly tlb_lld_4k[NR_INFO];
u16 __read_mostly tlb_lld_2m[NR_INFO];
u16 __read_mostly tlb_lld_4m[NR_INFO];
+/*
+ * tlb_flushall_shift shows the balance point in replacing cr3 write
+ * with multiple 'invlpg'. It will do this replacement when
+ * flush_tlb_lines <= active_lines/2^tlb_flushall_shift.
+ * If tlb_flushall_shift is -1, means the replacement will be disabled.
+ */
+u16 __read_mostly tlb_flushall_shift;
+
void __cpuinit cpu_detect_tlb(struct cpuinfo_x86 *c)
{
if (c->x86_vendor == X86_VENDOR_INTEL)
intel_cpu_detect_tlb(c);
printk(KERN_INFO "Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n" \
- "Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d\n",
+ "Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d\n" \
+ "tlb_flushall_shift is 0x%x\n",
tlb_lli_4k[ENTRIES], tlb_lli_2m[ENTRIES],
tlb_lli_4m[ENTRIES], tlb_lld_4k[ENTRIES],
- tlb_lld_2m[ENTRIES], tlb_lld_4m[ENTRIES]);
+ tlb_lld_2m[ENTRIES], tlb_lld_4m[ENTRIES],
+ tlb_flushall_shift);
}
void __cpuinit detect_ht(struct cpuinfo_x86 *c)
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index 28ecd1b..a51ab82 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -610,6 +610,39 @@ static void __cpuinit intel_tlb_lookup(const unsigned char desc)
}
}
+static void __cpuinit intel_tlb_flushall_shift_set(struct cpuinfo_x86 *c)
+{
+ if (!cpu_has_invlpg) {
+ tlb_flushall_shift = -1;
+ return;
+ }
+ switch ((c->x86 << 8) + c->x86_model) {
+ case 0x60f: /* original 65 nm celeron/pentium/core2/xeon, "Merom"/"Conroe" */
+ case 0x616: /* single-core 65 nm celeron/core2solo "Merom-L"/"Conroe-L" */
+ case 0x617: /* current 45 nm celeron/core2/xeon "Penryn"/"Wolfdale" */
+ case 0x61d: /* six-core 45 nm xeon "Dunnington" */
+ tlb_flushall_shift = -1;
+ break;
+ case 0x61a: /* 45 nm nehalem, "Bloomfield" */
+ case 0x61e: /* 45 nm nehalem, "Lynnfield" */
+ case 0x625: /* 32 nm nehalem, "Clarkdale" */
+ case 0x62c: /* 32 nm nehalem, "Gulftown" */
+ case 0x62e: /* 45 nm nehalem-ex, "Beckton" */
+ case 0x62f: /* 32 nm Xeon E7 */
+ tlb_flushall_shift = 6;
+ break;
+ case 0x62a: /* SandyBridge */
+ case 0x62d: /* SandyBridge, "Romely-EP" */
+ tlb_flushall_shift = 5;
+ break;
+ case 0x63a: /* Ivybridge */
+ tlb_flushall_shift = 0;
+ break;
+ default:
+ tlb_flushall_shift = 6;
+ }
+}
+
void __cpuinit intel_cpu_detect_tlb(struct cpuinfo_x86 *c)
{
int i, j, n;
@@ -630,6 +663,7 @@ void __cpuinit intel_cpu_detect_tlb(struct cpuinfo_x86 *c)
for (j = 1 ; j < 16 ; j++)
intel_tlb_lookup(desc[j]);
}
+ intel_tlb_flushall_shift_set(c);
}
static const struct cpu_dev __cpuinitconst intel_cpu_dev = {
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 22e5bb1..5bf4e85 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -314,8 +314,6 @@ void flush_tlb_mm(struct mm_struct *mm)
preempt_enable();
}
-#define FLUSHALL_BAR 16
-
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
static inline int has_large_page(struct mm_struct *mm,
unsigned long start, unsigned long end)
@@ -350,7 +348,8 @@ void flush_tlb_range(struct vm_area_struct *vma,
{
struct mm_struct *mm;
- if (!cpu_has_invlpg || vma->vm_flags & VM_HUGETLB) {
+ if (!cpu_has_invlpg || vma->vm_flags & VM_HUGETLB
+ || tlb_flushall_shift == (u16)TLB_FLUSH_ALL) {
flush_all:
flush_tlb_mm(vma->vm_mm);
return;
@@ -371,7 +370,8 @@ flush_all:
act_entries = tlb_entries > mm->total_vm ?
mm->total_vm : tlb_entries;
- if ((end - start)/PAGE_SIZE > act_entries/FLUSHALL_BAR)
+ if ((end - start) >> PAGE_SHIFT >
+ act_entries >> tlb_flushall_shift)
local_flush_tlb();
else {
if (has_large_page(mm, start, end)) {
diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
index f96a5b5..75e888b 100644
--- a/include/asm-generic/tlb.h
+++ b/include/asm-generic/tlb.h
@@ -113,7 +113,8 @@ static inline int tlb_fast_mode(struct mmu_gather *tlb)
void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, bool fullmm);
void tlb_flush_mmu(struct mmu_gather *tlb);
-void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end);
+void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start,
+ unsigned long end);
int __tlb_remove_page(struct mmu_gather *tlb, struct page *page);
/* tlb_remove_page
--
1.7.5.4
kernel will replace cr3 rewrite with invlpg when
tlb_flush_entries <= active_tlb_entries / 2^tlb_flushall_factor
if tlb_flushall_factor is -1, kernel won't do this replacement.
User can modify its value according to specific CPU/applications.
Thanks for Borislav providing the help message of
CONFIG_DEBUG_TLBFLUSH.
Signed-off-by: Alex Shi <[email protected]>
---
arch/x86/Kconfig.debug | 19 +++++++++++++++++++
arch/x86/mm/tlb.c | 48 ++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 67 insertions(+), 0 deletions(-)
diff --git a/arch/x86/Kconfig.debug b/arch/x86/Kconfig.debug
index e46c214..b322f12 100644
--- a/arch/x86/Kconfig.debug
+++ b/arch/x86/Kconfig.debug
@@ -129,6 +129,25 @@ config DOUBLEFAULT
option saves about 4k and might cause you much additional grey
hair.
+config DEBUG_TLBFLUSH
+ bool "Set upper limit of TLB entries to flush one-by-one"
+ depends on DEBUG_KERNEL && (X86_64 || X86_INVLPG)
+ ---help---
+
+ X86-only for now.
+
+ This option allows the user to tune the amount of TLB entries the
+ kernel flushes one-by-one instead of doing a full TLB flush. In
+ certain situations, the former is cheaper. This is controlled by the
+ tlb_flushall_shift knob under /sys/kernel/debug/x86. If you set it
+ to -1, the code flushes the whole TLB unconditionally. Otherwise,
+ for positive values of it, the kernel will use single TLB entry
+ invalidating instructions according to the following formula:
+
+ flush_entries <= active_tlb_entries / 2^tlb_flushall_shift
+
+ If in doubt, say "N".
+
config IOMMU_DEBUG
bool "Enable IOMMU debugging"
depends on GART_IOMMU && DEBUG_KERNEL
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index b2e25e8..b77c764 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -12,6 +12,7 @@
#include <asm/cache.h>
#include <asm/apic.h>
#include <asm/uv/uv.h>
+#include <linux/debugfs.h>
DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate)
= { &init_mm, 0, };
@@ -429,3 +430,50 @@ void flush_tlb_all(void)
{
on_each_cpu(do_flush_tlb_all, NULL, 1);
}
+
+#ifdef CONFIG_DEBUG_TLBFLUSH
+static ssize_t tlbflush_read_file(struct file *file, char __user *user_buf,
+ size_t count, loff_t *ppos)
+{
+ char buf[32];
+ unsigned int len;
+
+ len = sprintf(buf, "%hd\n", tlb_flushall_shift);
+ return simple_read_from_buffer(user_buf, count, ppos, buf, len);
+}
+
+static ssize_t tlbflush_write_file(struct file *file,
+ const char __user *user_buf, size_t count, loff_t *ppos)
+{
+ char buf[32];
+ ssize_t len;
+ u16 shift;
+
+ len = min(count, sizeof(buf) - 1);
+ if (copy_from_user(buf, user_buf, len))
+ return -EFAULT;
+
+ buf[len] = '\0';
+ if (kstrtos16(buf, 0, &shift))
+ return -EINVAL;
+
+ tlb_flushall_shift = shift;
+ return count;
+}
+
+static const struct file_operations fops_tlbflush = {
+ .read = tlbflush_read_file,
+ .write = tlbflush_write_file,
+ .llseek = default_llseek,
+};
+
+static int __cpuinit create_tlb_flushall_shift(void)
+{
+ if (cpu_has_invlpg) {
+ debugfs_create_file("tlb_flushall_shift", S_IRUSR | S_IWUSR,
+ arch_debugfs_dir, NULL, &fops_tlbflush);
+ }
+ return 0;
+}
+late_initcall(create_tlb_flushall_shift);
+#endif
--
1.7.5.4
On Tue, 2012-05-15 at 16:55 +0800, Alex Shi wrote:
> Not every flush_tlb_mm execution moment is really need to evacuate all
> TLB entries, like in munmap, just few 'invlpg' is better for whole
> process performance, since it leaves most of TLB entries for later
> accessing.
>
> This patch is changing flush_tlb_mm(mm) to flush_tlb_mm(mm, start,
> end)
> in cases.
Ok, so you're not getting it.
NACKED-BY: Peter Zijlstra <[email protected]>
So this should go to linux-arch...
On 15 May 2012 18:55, Alex Shi <[email protected]> wrote:
> Not every flush_tlb_mm execution moment is really need to evacuate all
> TLB entries, like in munmap, just few 'invlpg' is better for whole
> process performance, since it leaves most of TLB entries for later
> accessing.
>
> This patch is changing flush_tlb_mm(mm) to flush_tlb_mm(mm, start, end)
> in cases.
What happened with Peter's comment about using flush_tlb_range for this?
flush_tlb_mm() API should just stay unchanged AFAIKS.
Then you need to work out the best way to give range info to the tlb/mmu gather
API. Possibly passing in the rage for that guy is OK, which x86 can
then implement
as flush range.
On 15 May 2012 19:15, Nick Piggin <[email protected]> wrote:
> So this should go to linux-arch...
>
> On 15 May 2012 18:55, Alex Shi <[email protected]> wrote:
>> Not every flush_tlb_mm execution moment is really need to evacuate all
>> TLB entries, like in munmap, just few 'invlpg' is better for whole
>> process performance, since it leaves most of TLB entries for later
>> accessing.
Did you have microbenchmarks for this like your mprotect numbers,
by the way? Test munmap numbers and see how that looks. Also,
does it show up on any macro-benchmarks like specjbb?
On Tue, 2012-05-15 at 19:15 +1000, Nick Piggin wrote:
> So this should go to linux-arch...
>
> On 15 May 2012 18:55, Alex Shi <[email protected]> wrote:
> > Not every flush_tlb_mm execution moment is really need to evacuate all
> > TLB entries, like in munmap, just few 'invlpg' is better for whole
> > process performance, since it leaves most of TLB entries for later
> > accessing.
> >
> > This patch is changing flush_tlb_mm(mm) to flush_tlb_mm(mm, start, end)
> > in cases.
>
> What happened with Peter's comment about using flush_tlb_range for this?
>
> flush_tlb_mm() API should just stay unchanged AFAIKS.
>
> Then you need to work out the best way to give range info to the tlb/mmu gather
> API. Possibly passing in the rage for that guy is OK, which x86 can
> then implement
> as flush range.
Right, most archs that have tlb_flush_range() do range tracking in
mmu_gather. Our TLB ops fully support that, there's absolutely no need
to go change the interface for thos.
On 15 May 2012 19:18, Peter Zijlstra <[email protected]> wrote:
> On Tue, 2012-05-15 at 19:15 +1000, Nick Piggin wrote:
>> So this should go to linux-arch...
>>
>> On 15 May 2012 18:55, Alex Shi <[email protected]> wrote:
>> > Not every flush_tlb_mm execution moment is really need to evacuate all
>> > TLB entries, like in munmap, just few 'invlpg' is better for whole
>> > process performance, since it leaves most of TLB entries for later
>> > accessing.
>> >
>> > This patch is changing flush_tlb_mm(mm) to flush_tlb_mm(mm, start, end)
>> > in cases.
>>
>> What happened with Peter's comment about using flush_tlb_range for this?
>>
>> flush_tlb_mm() API should just stay unchanged AFAIKS.
>>
>> Then you need to work out the best way to give range info to the tlb/mmu gather
>> API. Possibly passing in the rage for that guy is OK, which x86 can
>> then implement
>> as flush range.
>
> Right, most archs that have tlb_flush_range() do range tracking in
> mmu_gather. Our TLB ops fully support that, there's absolutely no need
> to go change the interface for thos.
It could be warranted to change tlb_flush_mmu to a range API to
avoid doing the per-entry tracking which those architectures do?
The callers have range available easily, so ignoring those could be
noop for generic helpers.
On Tue, 2012-05-15 at 19:52 +1000, Nick Piggin wrote:
>
> It could be warranted to change tlb_flush_mmu to a range API to
> avoid doing the per-entry tracking which those architectures do?
The per-entry could result in a much smaller range, there's no point in
flushing tlbs for unpopulated pages.
Anyway, I don't think even think we'd need to change the API for that,
you could track the entire range through tlb_start_vma() if you wanted
(although nobody does that IIRC).
On 15 May 2012 20:00, Peter Zijlstra <[email protected]> wrote:
> On Tue, 2012-05-15 at 19:52 +1000, Nick Piggin wrote:
>>
>> It could be warranted to change tlb_flush_mmu to a range API to
>> avoid doing the per-entry tracking which those architectures do?
>
> The per-entry could result in a much smaller range, there's no point in
> flushing tlbs for unpopulated pages.
Well common case for small ranges hopefully would be quite dense
I think. It could be not worth the extra work (although maybe it would
be).
>
> Anyway, I don't think even think we'd need to change the API for that,
> you could track the entire range through tlb_start_vma() if you wanted
> (although nobody does that IIRC).
I'm not sure if you can do that very well, because the tlb might have to
be flushed part way through a vma when we fill up the gather, so you
don't want to flush the full range each time.
On Tue, 2012-05-15 at 20:06 +1000, Nick Piggin wrote:
> On 15 May 2012 20:00, Peter Zijlstra <[email protected]> wrote:
> > On Tue, 2012-05-15 at 19:52 +1000, Nick Piggin wrote:
> >>
> >> It could be warranted to change tlb_flush_mmu to a range API to
> >> avoid doing the per-entry tracking which those architectures do?
> >
> > The per-entry could result in a much smaller range, there's no point in
> > flushing tlbs for unpopulated pages.
>
> Well common case for small ranges hopefully would be quite dense
> I think. It could be not worth the extra work (although maybe it would
> be).
>
> >
> > Anyway, I don't think even think we'd need to change the API for that,
> > you could track the entire range through tlb_start_vma() if you wanted
> > (although nobody does that IIRC).
>
> I'm not sure if you can do that very well, because the tlb might have to
> be flushed part way through a vma when we fill up the gather, so you
> don't want to flush the full range each time.
Fair enough. But that's still an entirely unrelated optimization and
should go with proper benchmarking and preferably across all archs that
have flush_tlb_range() :-)
On Tue, May 15, 2012 at 5:17 PM, Nick Piggin <[email protected]> wrote:
> On 15 May 2012 19:15, Nick Piggin <[email protected]> wrote:
>> So this should go to linux-arch...
>>
>> On 15 May 2012 18:55, Alex Shi <[email protected]> wrote:
>>> Not every flush_tlb_mm execution moment is really need to evacuate all
>>> TLB entries, like in munmap, just few 'invlpg' is better for whole
>>> process performance, since it leaves most of TLB entries for later
>>> accessing.
>
> Did you have microbenchmarks for this like your mprotect numbers,
> by the way? Test munmap numbers and see how that looks. Also,
Might be off topic, but I just spent few minutes to test out the difference
between write CR3 vs. invlpg on a pretty old but still reliable P4 desktop
with my simple hardware latency and bandwidth test tool I posted for
RFC several weeks ago on LKML.
Both __native_flush_tlb() and __native_flush_tlb_single(...)
introduced roughly 1 ns latency to tsc sampling executed in
stop_machine_context in two logical CPUs
Just to fuel the discussion. :-)
Cheers,
/l
On Tue, 2012-05-15 at 20:58 +0800, Luming Yu wrote:
>
>
> Both __native_flush_tlb() and __native_flush_tlb_single(...)
> introduced roughly 1 ns latency to tsc sampling executed in
> stop_machine_context in two logical CPUs
But you have to weight that against the cost of re-population, and
that's the difficult bit, since we have no clue how many tlb entries are
in use by the current cr3.
It might be possible for intel to give us this information, I've asked
for something similar for cachelines.
On Tue, May 15, 2012 at 8:58 PM, Luming Yu <[email protected]> wrote:
> On Tue, May 15, 2012 at 5:17 PM, Nick Piggin <[email protected]> wrote:
>> On 15 May 2012 19:15, Nick Piggin <[email protected]> wrote:
>>> So this should go to linux-arch...
>>>
>>> On 15 May 2012 18:55, Alex Shi <[email protected]> wrote:
>>>> Not every flush_tlb_mm execution moment is really need to evacuate all
>>>> TLB entries, like in munmap, just few 'invlpg' is better for whole
>>>> process performance, since it leaves most of TLB entries for later
>>>> accessing.
>>
>> Did you have microbenchmarks for this like your mprotect numbers,
>> by the way? Test munmap numbers and see how that looks. Also,
>
> Might be off topic, but I just spent few minutes to test out the difference
> between write CR3 vs. invlpg on a pretty old but still reliable P4 desktop
> with my simple hardware latency and bandwidth test tool I posted for
> RFC several weeks ago on LKML.
>
> Both __native_flush_tlb() and __native_flush_tlb_single(...)
> introduced roughly 1 ns  latency to tsc sampling executed in
sorry, typo, 1us.. but I should capture nanosecond data. :-(
> stop_machine_context in two logical CPUs
>
> Just to fuel the discussion. :-)
>
> Cheers,
> /l
On 05/15/2012 05:15 PM, Nick Piggin wrote:
> So this should go to linux-arch...
>
> On 15 May 2012 18:55, Alex Shi <[email protected]> wrote:
>> Not every flush_tlb_mm execution moment is really need to evacuate all
>> TLB entries, like in munmap, just few 'invlpg' is better for whole
>> process performance, since it leaves most of TLB entries for later
>> accessing.
>>
>> This patch is changing flush_tlb_mm(mm) to flush_tlb_mm(mm, start, end)
>> in cases.
>
> What happened with Peter's comment about using flush_tlb_range for this?
>
> flush_tlb_mm() API should just stay unchanged AFAIKS.
>
> Then you need to work out the best way to give range info to the tlb/mmu gather
> API. Possibly passing in the rage for that guy is OK, which x86 can
> then implement
> as flush range.
Sorry. I don't understand what's the comments Peter's made days ago. I should ask for more details originally.
So, Peter, the correct change should like following, am I right?
-#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
+#define tlb_flush(tlb, start, end) __flush_tlb_range((tlb)->mm, start, end)
On Tue, May 15, 2012 at 9:06 PM, Peter Zijlstra <[email protected]> wrote:
> On Tue, 2012-05-15 at 20:58 +0800, Luming Yu wrote:
>>
>>
>> Both __native_flush_tlb() and __native_flush_tlb_single(...)
>> introduced roughly 1 ns  latency to tsc sampling executed in
Fix typo, I just observed 1us with current tool, I would check if I
can push the accuracy to nanoseconds level.
>> stop_machine_context in two logical CPUs
>
> But you have to weight that against the cost of re-population, and
Right, it's hard to detect, but I will try if I can get measurement
done in a simple test tool to help people measure
this kind of stuff in few minutes.
> that's the difficult bit, since we have no clue how many tlb entries are
> in use by the current cr3.
>
> It might be possible for intel to give us this information, I've asked
> for something similar for cachelines.
This is the official document
http://www.intel.com/content/dam/doc/manual/64-ia-32-architectures-optimization-manual.pdf
Let me know if it can answer your question.
>
On 05/15/2012 09:27 PM, Luming Yu wrote:
> On Tue, May 15, 2012 at 9:06 PM, Peter Zijlstra <[email protected]> wrote:
>> On Tue, 2012-05-15 at 20:58 +0800, Luming Yu wrote:
>>>
>>>
>>> Both __native_flush_tlb() and __native_flush_tlb_single(...)
>>> introduced roughly 1 ns latency to tsc sampling executed in
>
> Fix typo, I just observed 1us with current tool, I would check if I
> can push the accuracy to nanoseconds level.
>
>>> stop_machine_context in two logical CPUs
>>
>> But you have to weight that against the cost of re-population, and
>
> Right, it's hard to detect, but I will try if I can get measurement
> done in a simple test tool to help people measure
> this kind of stuff in few minutes.
>
>> that's the difficult bit, since we have no clue how many tlb entries are
>> in use by the current cr3.
>>
>> It might be possible for intel to give us this information, I've asked
>> for something similar for cachelines.
>
> This is the official document
> http://www.intel.com/content/dam/doc/manual/64-ia-32-architectures-optimization-manual.pdf
>
Please, such huge documents! and it also has no such info.
> Let me know if it can answer your question.
>
>>
On 05/15/2012 09:06 PM, Peter Zijlstra wrote:
> On Tue, 2012-05-15 at 20:58 +0800, Luming Yu wrote:
>>
>>
>> Both __native_flush_tlb() and __native_flush_tlb_single(...)
>> introduced roughly 1 ns latency to tsc sampling executed in
>> stop_machine_context in two logical CPUs
>
> But you have to weight that against the cost of re-population, and
> that's the difficult bit, since we have no clue how many tlb entries are
> in use by the current cr3.
>
> It might be possible for intel to give us this information, I've asked
> for something similar for cachelines.
>
I don't know if such info exist in cpu. Maybe US engineer know more.
On Tue, 2012-05-15 at 15:06 +0200, Peter Zijlstra wrote:
> On Tue, 2012-05-15 at 20:58 +0800, Luming Yu wrote:
> >
> >
> > Both __native_flush_tlb() and __native_flush_tlb_single(...)
> > introduced roughly 1 ns latency to tsc sampling executed in
> > stop_machine_context in two logical CPUs
>
> But you have to weight that against the cost of re-population, and
> that's the difficult bit, since we have no clue how many tlb entries are
> in use by the current cr3.
>
> It might be possible for intel to give us this information, I've asked
> for something similar for cachelines.
What information? The # of tlb entries in use?
-- Steve
On 05/15/2012 05:18 PM, Peter Zijlstra wrote:
> On Tue, 2012-05-15 at 19:15 +1000, Nick Piggin wrote:
>> So this should go to linux-arch...
>>
>> On 15 May 2012 18:55, Alex Shi <[email protected]> wrote:
>>> Not every flush_tlb_mm execution moment is really need to evacuate all
>>> TLB entries, like in munmap, just few 'invlpg' is better for whole
>>> process performance, since it leaves most of TLB entries for later
>>> accessing.
>>>
>>> This patch is changing flush_tlb_mm(mm) to flush_tlb_mm(mm, start, end)
>>> in cases.
>>
>> What happened with Peter's comment about using flush_tlb_range for this?
>>
>> flush_tlb_mm() API should just stay unchanged AFAIKS.
>>
>> Then you need to work out the best way to give range info to the tlb/mmu gather
>> API. Possibly passing in the rage for that guy is OK, which x86 can
>> then implement
>> as flush range.
>
> Right, most archs that have tlb_flush_range() do range tracking in
> mmu_gather. Our TLB ops fully support that, there's absolutely no need
> to go change the interface for thos.
Ok. this should be your wanted,
-#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
+#define tlb_flush(tlb, start, end) __flush_tlb_range((tlb)->mm, start, end)
If no objection, I will modify the patch accordingly.
On Tue, May 15, 2012 at 09:39:05AM -0400, Steven Rostedt wrote:
> > But you have to weight that against the cost of re-population, and
> > that's the difficult bit, since we have no clue how many tlb entries are
> > in use by the current cr3.
> >
> > It might be possible for intel to give us this information, I've asked
> > for something similar for cachelines.
>
> What information? The # of tlb entries in use?
... by the current %cr3, yes.
And also, before we delve into details, we still don't have a
representative benchmark where this shows any improvement.
--
Regards/Gruss,
Boris.
Advanced Micro Devices GmbH
Einsteinring 24, 85609 Dornach
GM: Alberto Bozzo
Reg: Dornach, Landkreis Muenchen
HRB Nr. 43632 WEEE Registernr: 129 19551
On 05/15/2012 05:17 PM, Nick Piggin wrote:
> On 15 May 2012 19:15, Nick Piggin <[email protected]> wrote:
>> So this should go to linux-arch...
>>
>> On 15 May 2012 18:55, Alex Shi <[email protected]> wrote:
>>> Not every flush_tlb_mm execution moment is really need to evacuate all
>>> TLB entries, like in munmap, just few 'invlpg' is better for whole
>>> process performance, since it leaves most of TLB entries for later
>>> accessing.
>
> Did you have microbenchmarks for this like your mprotect numbers,
> by the way? Test munmap numbers and see how that looks. Also,
> does it show up on any macro-benchmarks like specjbb?
Yongjie has tested the patchset and get some positive data on Virtual
machine.
Yongjie, could you like to share your data?
On Tue, 2012-05-15 at 21:24 +0800, Alex Shi wrote:
> Sorry. I don't understand what's the comments Peter's made days ago. I should ask for more details originally.
>
> So, Peter, the correct change should like following, am I right?
>
> -#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
> +#define tlb_flush(tlb, start, end) __flush_tlb_range((tlb)->mm, start, end)
No.. the correct change is to do range tracking like the other archs
that support flush_tlb_range() do.
You do not modify the tlb interface.
Again, see: http://marc.info/?l=linux-arch&m=129952026504268&w=2
On Tue, 2012-05-15 at 16:36 +0200, Peter Zijlstra wrote:
> On Tue, 2012-05-15 at 21:24 +0800, Alex Shi wrote:
>
> > Sorry. I don't understand what's the comments Peter's made days ago. I should ask for more details originally.
> >
> > So, Peter, the correct change should like following, am I right?
> >
> > -#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
> > +#define tlb_flush(tlb, start, end) __flush_tlb_range((tlb)->mm, start, end)
>
> No.. the correct change is to do range tracking like the other archs
> that support flush_tlb_range() do.
>
> You do not modify the tlb interface.
>
> Again, see: http://marc.info/?l=linux-arch&m=129952026504268&w=2
Just to be _very_ clear, you do not modify:
mm/memory.c | 9 ++--
kernel/fork.c | 2 +-
fs/proc/task_mmu.c | 2 +-
As it stands your patch breaks compilation on a whole bunch of
architectures.
If you touch the TLB interface, you get to touch _ALL_ architectures.
On 05/15/2012 10:57 PM, Peter Zijlstra wrote:
> On Tue, 2012-05-15 at 16:36 +0200, Peter Zijlstra wrote:
>> On Tue, 2012-05-15 at 21:24 +0800, Alex Shi wrote:
>>
>>> Sorry. I don't understand what's the comments Peter's made days ago. I should ask for more details originally.
>>>
>>> So, Peter, the correct change should like following, am I right?
>>>
>>> -#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
>>> +#define tlb_flush(tlb, start, end) __flush_tlb_range((tlb)->mm, start, end)
>>
>> No.. the correct change is to do range tracking like the other archs
>> that support flush_tlb_range() do.
>>
>> You do not modify the tlb interface.
>>
>> Again, see: http://marc.info/?l=linux-arch&m=129952026504268&w=2
this code is for multiple architecture, but x86 still need implement
'flush tlb range' with 'invlpg'.
>
> Just to be _very_ clear, you do not modify:
>
> mm/memory.c | 9 ++--
> kernel/fork.c | 2 +-
> fs/proc/task_mmu.c | 2 +-
>
Thanks a lot. I see.
> As it stands your patch breaks compilation on a whole bunch of
> architectures.
>
> If you touch the TLB interface, you get to touch _ALL_ architectures.
On 05/15/2012 10:57 PM, Peter Zijlstra wrote:
> On Tue, 2012-05-15 at 16:36 +0200, Peter Zijlstra wrote:
>> On Tue, 2012-05-15 at 21:24 +0800, Alex Shi wrote:
>>
>>> Sorry. I don't understand what's the comments Peter's made days ago. I should ask for more details originally.
>>>
>>> So, Peter, the correct change should like following, am I right?
>>>
>>> -#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
>>> +#define tlb_flush(tlb, start, end) __flush_tlb_range((tlb)->mm, start, end)
>>
>> No.. the correct change is to do range tracking like the other archs
>> that support flush_tlb_range() do.
>>
>> You do not modify the tlb interface.
>>
>> Again, see: http://marc.info/?l=linux-arch&m=129952026504268&w=2
>
> Just to be _very_ clear, you do not modify:
>
> mm/memory.c | 9 ++--
> kernel/fork.c | 2 +-
> fs/proc/task_mmu.c | 2 +-
>
> As it stands your patch breaks compilation on a whole bunch of
> architectures.
>
> If you touch the TLB interface, you get to touch _ALL_ architectures.
Thanks for Nick and Peter's comments. I rewrite the patch according to
your opinions. Is this met your expectation?
----
>From a01864af75d8c86668f4fa73d6ca18ebe5835b18 Mon Sep 17 00:00:00 2001
From: Alex Shi <[email protected]>
Date: Mon, 14 May 2012 09:17:03 +0800
Subject: [PATCH 6/7] x86/tlb: optimizing tlb_finish_mmu on x86
Not every tlb_flush execution moment is really need to evacuate all
TLB entries, like in munmap, just few 'invlpg' is better for whole
process performance, since it leaves most of TLB entries for later
accessing.
Thanks for Peter Zijlstra reminder, tlb interfaces in mm/memory.c
are for all architectures. So, I keep current interfaces, just
reimplement x86 specific 'tlb_flush' only. Some of ideas also are
picked up from Peter's old patch, thanks!
This patch also rewrite flush_tlb_range for 2 purposes:
1, split it out to get flush_blt_mm_range function.
2, clean up to reduce line breaking, thanks for Borislav's input.
Signed-off-by: Alex Shi <[email protected]>
---
arch/x86/include/asm/tlb.h | 9 +++-
arch/x86/include/asm/tlbflush.h | 2 +
arch/x86/mm/tlb.c | 120 +++++++++++++++++++++------------------
include/asm-generic/tlb.h | 2 +
mm/memory.c | 6 ++
5 files changed, 82 insertions(+), 57 deletions(-)
diff --git a/arch/x86/include/asm/tlb.h b/arch/x86/include/asm/tlb.h
index 829215f..4fef207 100644
--- a/arch/x86/include/asm/tlb.h
+++ b/arch/x86/include/asm/tlb.h
@@ -4,7 +4,14 @@
#define tlb_start_vma(tlb, vma) do { } while (0)
#define tlb_end_vma(tlb, vma) do { } while (0)
#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
-#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
+
+#define tlb_flush(tlb) \
+{ \
+ if (tlb->fullmm == 0) \
+ flush_tlb_mm_range(tlb->mm, tlb->start, tlb->end, 0UL); \
+ else \
+ flush_tlb_mm_range(tlb->mm, 0UL, TLB_FLUSH_ALL, 0UL); \
+}
#include <asm-generic/tlb.h>
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index c39c94e..0107f3c 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -128,6 +128,8 @@ extern void flush_tlb_mm(struct mm_struct *);
extern void flush_tlb_page(struct vm_area_struct *, unsigned long);
extern void flush_tlb_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end);
+extern void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
+ unsigned long end, unsigned long vmflag);
#define flush_tlb() flush_tlb_current_task()
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 5bf4e85..52f6a5a 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -298,22 +298,6 @@ void flush_tlb_current_task(void)
preempt_enable();
}
-void flush_tlb_mm(struct mm_struct *mm)
-{
- preempt_disable();
-
- if (current->active_mm == mm) {
- if (current->mm)
- local_flush_tlb();
- else
- leave_mm(smp_processor_id());
- }
- if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
- flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
-
- preempt_enable();
-}
-
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
static inline int has_large_page(struct mm_struct *mm,
unsigned long start, unsigned long end)
@@ -343,61 +327,85 @@ static inline int has_large_page(struct mm_struct *mm,
return 0;
}
#endif
-void flush_tlb_range(struct vm_area_struct *vma,
- unsigned long start, unsigned long end)
+
+void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
+ unsigned long end, unsigned long vmflag)
{
- struct mm_struct *mm;
+ unsigned long addr;
+ unsigned act_entries, tlb_entries = 0;
- if (!cpu_has_invlpg || vma->vm_flags & VM_HUGETLB
- || tlb_flushall_shift == (u16)TLB_FLUSH_ALL) {
-flush_all:
- flush_tlb_mm(vma->vm_mm);
- return;
+ preempt_disable();
+ if (current->active_mm != mm)
+ goto flush_all;
+
+ if (!current->mm) {
+ leave_mm(smp_processor_id());
+ goto flush_all;
}
- preempt_disable();
- mm = vma->vm_mm;
- if (current->active_mm == mm) {
- if (current->mm) {
- unsigned long addr, vmflag = vma->vm_flags;
- unsigned act_entries, tlb_entries = 0;
+ if (end == TLB_FLUSH_ALL ||
+ tlb_flushall_shift == (u16)TLB_FLUSH_ALL) {
+ local_flush_tlb();
+ goto flush_all;
+ }
- if (vmflag & VM_EXEC)
- tlb_entries = tlb_lli_4k[ENTRIES];
- else
- tlb_entries = tlb_lld_4k[ENTRIES];
+ if (vmflag & VM_EXEC)
+ tlb_entries = tlb_lli_4k[ENTRIES];
+ else
+ tlb_entries = tlb_lld_4k[ENTRIES];
+ act_entries = mm->total_vm > tlb_entries ? tlb_entries : mm->total_vm;
- act_entries = tlb_entries > mm->total_vm ?
- mm->total_vm : tlb_entries;
+ if ((end - start) >> PAGE_SHIFT > act_entries >> tlb_flushall_shift)
+ local_flush_tlb();
+ else {
+ if (has_large_page(mm, start, end)) {
+ local_flush_tlb();
+ goto flush_all;
+ }
+ for (addr = start; addr <= end; addr += PAGE_SIZE)
+ __flush_tlb_single(addr);
- if ((end - start) >> PAGE_SHIFT >
- act_entries >> tlb_flushall_shift)
- local_flush_tlb();
- else {
- if (has_large_page(mm, start, end)) {
- preempt_enable();
- goto flush_all;
- }
- for (addr = start; addr <= end;
- addr += PAGE_SIZE)
- __flush_tlb_single(addr);
+ if (cpumask_any_but(mm_cpumask(mm),
+ smp_processor_id()) < nr_cpu_ids)
+ flush_tlb_others(mm_cpumask(mm), mm, start, end);
+ preempt_enable();
+ return;
+ }
- if (cpumask_any_but(mm_cpumask(mm),
- smp_processor_id()) < nr_cpu_ids)
- flush_tlb_others(mm_cpumask(mm), mm,
- start, end);
- preempt_enable();
- return;
- }
- } else {
+flush_all:
+ if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
+ flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
+ preempt_enable();
+}
+
+void flush_tlb_mm(struct mm_struct *mm)
+{
+ preempt_disable();
+
+ if (current->active_mm == mm) {
+ if (current->mm)
+ local_flush_tlb();
+ else
leave_mm(smp_processor_id());
- }
}
if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
+
preempt_enable();
}
+void flush_tlb_range(struct vm_area_struct *vma,
+ unsigned long start, unsigned long end)
+{
+ struct mm_struct *mm = vma->vm_mm;
+ unsigned long vmflag = vma->vm_flags;
+
+ if (!cpu_has_invlpg || vma->vm_flags & VM_HUGETLB)
+ flush_tlb_mm_range(mm, 0UL, TLB_FLUSH_ALL, 0UL);
+ else
+ flush_tlb_mm_range(mm, start, end, vmflag);
+}
+
void flush_tlb_page(struct vm_area_struct *vma, unsigned long start)
{
diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
index 75e888b..ed6642a 100644
--- a/include/asm-generic/tlb.h
+++ b/include/asm-generic/tlb.h
@@ -86,6 +86,8 @@ struct mmu_gather {
#ifdef CONFIG_HAVE_RCU_TABLE_FREE
struct mmu_table_batch *batch;
#endif
+ unsigned long start;
+ unsigned long end;
unsigned int need_flush : 1, /* Did free PTEs */
fast_mode : 1; /* No batching */
diff --git a/mm/memory.c b/mm/memory.c
index 6105f47..b176172 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -206,6 +206,8 @@ void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, bool fullmm)
tlb->mm = mm;
tlb->fullmm = fullmm;
+ tlb->start = -1UL;
+ tlb->end = 0;
tlb->need_flush = 0;
tlb->fast_mode = (num_possible_cpus() == 1);
tlb->local.next = NULL;
@@ -248,6 +250,8 @@ void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long e
{
struct mmu_gather_batch *batch, *next;
+ tlb->start = start;
+ tlb->end = end;
tlb_flush_mmu(tlb);
/* keep the page table cache within bounds */
@@ -1204,6 +1208,8 @@ again:
*/
if (force_flush) {
force_flush = 0;
+ tlb->start = addr;
+ tlb->end = end;
tlb_flush_mmu(tlb);
if (addr != end)
goto again;
--
1.7.5.4
On 05/15/2012 04:55 PM, Alex Shi wrote:
update this patch for a little change commit log and conservative
balance point on IVB mobile cpu.
----
>From 0a3d618e5a8b29efc6296e44a56bc5c87b506659 Mon Sep 17 00:00:00 2001
From: Alex Shi <[email protected]>
Date: Mon, 14 May 2012 09:13:28 +0800
Subject: [PATCH 5/7] x86/tlb: add tlb_flushall_shift for specific CPU
Testing show different CPU type(micro architectures and NUMA mode) has
different balance points between the TLB flush all and multiple invlpg.
And there also has cases the tlb flush change has no any help.
This patch give a interface to let x86 vendor developers have a chance
to set different shift for different CPU type.
like some machine in my hands, balance points is 16 entries on
Romely-EP; while it is at 8 entries on Bloomfield NHM-EP; and is 256 on
IVB mobile CPU. but on model 15 core2 Xeon using invlpg has nothing
help.
For untested machine, do a conservative optimization, same as NHM CPU.
Signed-off-by: Alex Shi <[email protected]>
---
arch/x86/include/asm/processor.h | 2 ++
arch/x86/kernel/cpu/common.c | 14 ++++++++++++--
arch/x86/kernel/cpu/intel.c | 34 ++++++++++++++++++++++++++++++++++
arch/x86/mm/tlb.c | 8 ++++----
include/asm-generic/tlb.h | 3 ++-
5 files changed, 54 insertions(+), 7 deletions(-)
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 797faca..3c0cc3d 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -72,6 +72,8 @@ extern u16 __read_mostly tlb_lli_4m[NR_INFO];
extern u16 __read_mostly tlb_lld_4k[NR_INFO];
extern u16 __read_mostly tlb_lld_2m[NR_INFO];
extern u16 __read_mostly tlb_lld_4m[NR_INFO];
+extern u16 __read_mostly tlb_flushall_shift;
+
/*
* CPU type and hardware bug flags. Kept separately for each CPU.
* Members of this structure are referenced in head.S, so think twice
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 0152082..24255ca 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -459,16 +459,26 @@ u16 __read_mostly tlb_lld_4k[NR_INFO];
u16 __read_mostly tlb_lld_2m[NR_INFO];
u16 __read_mostly tlb_lld_4m[NR_INFO];
+/*
+ * tlb_flushall_shift shows the balance point in replacing cr3 write
+ * with multiple 'invlpg'. It will do this replacement when
+ * flush_tlb_lines <= active_lines/2^tlb_flushall_shift.
+ * If tlb_flushall_shift is -1, means the replacement will be disabled.
+ */
+u16 __read_mostly tlb_flushall_shift;
+
void __cpuinit cpu_detect_tlb(struct cpuinfo_x86 *c)
{
if (c->x86_vendor == X86_VENDOR_INTEL)
intel_cpu_detect_tlb(c);
printk(KERN_INFO "Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n" \
- "Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d\n",
+ "Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d\n" \
+ "tlb_flushall_shift is 0x%x\n",
tlb_lli_4k[ENTRIES], tlb_lli_2m[ENTRIES],
tlb_lli_4m[ENTRIES], tlb_lld_4k[ENTRIES],
- tlb_lld_2m[ENTRIES], tlb_lld_4m[ENTRIES]);
+ tlb_lld_2m[ENTRIES], tlb_lld_4m[ENTRIES],
+ tlb_flushall_shift);
}
void __cpuinit detect_ht(struct cpuinfo_x86 *c)
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index 28ecd1b..bb90754 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -610,6 +610,39 @@ static void __cpuinit intel_tlb_lookup(const unsigned char desc)
}
}
+static void __cpuinit intel_tlb_flushall_shift_set(struct cpuinfo_x86 *c)
+{
+ if (!cpu_has_invlpg) {
+ tlb_flushall_shift = -1;
+ return;
+ }
+ switch ((c->x86 << 8) + c->x86_model) {
+ case 0x60f: /* original 65 nm celeron/pentium/core2/xeon, "Merom"/"Conroe" */
+ case 0x616: /* single-core 65 nm celeron/core2solo "Merom-L"/"Conroe-L" */
+ case 0x617: /* current 45 nm celeron/core2/xeon "Penryn"/"Wolfdale" */
+ case 0x61d: /* six-core 45 nm xeon "Dunnington" */
+ tlb_flushall_shift = -1;
+ break;
+ case 0x61a: /* 45 nm nehalem, "Bloomfield" */
+ case 0x61e: /* 45 nm nehalem, "Lynnfield" */
+ case 0x625: /* 32 nm nehalem, "Clarkdale" */
+ case 0x62c: /* 32 nm nehalem, "Gulftown" */
+ case 0x62e: /* 45 nm nehalem-ex, "Beckton" */
+ case 0x62f: /* 32 nm Xeon E7 */
+ tlb_flushall_shift = 6;
+ break;
+ case 0x62a: /* SandyBridge */
+ case 0x62d: /* SandyBridge, "Romely-EP" */
+ tlb_flushall_shift = 5;
+ break;
+ case 0x63a: /* Ivybridge */
+ tlb_flushall_shift = 1;
+ break;
+ default:
+ tlb_flushall_shift = 6;
+ }
+}
+
void __cpuinit intel_cpu_detect_tlb(struct cpuinfo_x86 *c)
{
int i, j, n;
@@ -630,6 +663,7 @@ void __cpuinit intel_cpu_detect_tlb(struct cpuinfo_x86 *c)
for (j = 1 ; j < 16 ; j++)
intel_tlb_lookup(desc[j]);
}
+ intel_tlb_flushall_shift_set(c);
}
static const struct cpu_dev __cpuinitconst intel_cpu_dev = {
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 22e5bb1..5bf4e85 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -314,8 +314,6 @@ void flush_tlb_mm(struct mm_struct *mm)
preempt_enable();
}
-#define FLUSHALL_BAR 16
-
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
static inline int has_large_page(struct mm_struct *mm,
unsigned long start, unsigned long end)
@@ -350,7 +348,8 @@ void flush_tlb_range(struct vm_area_struct *vma,
{
struct mm_struct *mm;
- if (!cpu_has_invlpg || vma->vm_flags & VM_HUGETLB) {
+ if (!cpu_has_invlpg || vma->vm_flags & VM_HUGETLB
+ || tlb_flushall_shift == (u16)TLB_FLUSH_ALL) {
flush_all:
flush_tlb_mm(vma->vm_mm);
return;
@@ -371,7 +370,8 @@ flush_all:
act_entries = tlb_entries > mm->total_vm ?
mm->total_vm : tlb_entries;
- if ((end - start)/PAGE_SIZE > act_entries/FLUSHALL_BAR)
+ if ((end - start) >> PAGE_SHIFT >
+ act_entries >> tlb_flushall_shift)
local_flush_tlb();
else {
if (has_large_page(mm, start, end)) {
diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
index f96a5b5..75e888b 100644
--- a/include/asm-generic/tlb.h
+++ b/include/asm-generic/tlb.h
@@ -113,7 +113,8 @@ static inline int tlb_fast_mode(struct mmu_gather *tlb)
void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, bool fullmm);
void tlb_flush_mmu(struct mmu_gather *tlb);
-void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end);
+void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start,
+ unsigned long end);
int __tlb_remove_page(struct mmu_gather *tlb, struct page *page);
/* tlb_remove_page
--
1.7.5.4
On Wed, 2012-05-16 at 14:46 +0800, Alex Shi wrote:
> diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
> index 75e888b..ed6642a 100644
> --- a/include/asm-generic/tlb.h
> +++ b/include/asm-generic/tlb.h
> @@ -86,6 +86,8 @@ struct mmu_gather {
> #ifdef CONFIG_HAVE_RCU_TABLE_FREE
> struct mmu_table_batch *batch;
> #endif
> + unsigned long start;
> + unsigned long end;
> unsigned int need_flush : 1, /* Did free PTEs */
> fast_mode : 1; /* No batching */
>
> diff --git a/mm/memory.c b/mm/memory.c
> index 6105f47..b176172 100644
> --- a/mm/memory.c
> +++ b/mm/memory.c
> @@ -206,6 +206,8 @@ void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, bool fullmm)
> tlb->mm = mm;
>
> tlb->fullmm = fullmm;
> + tlb->start = -1UL;
> + tlb->end = 0;
> tlb->need_flush = 0;
> tlb->fast_mode = (num_possible_cpus() == 1);
> tlb->local.next = NULL;
> @@ -248,6 +250,8 @@ void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long e
> {
> struct mmu_gather_batch *batch, *next;
>
> + tlb->start = start;
> + tlb->end = end;
> tlb_flush_mmu(tlb);
>
> /* keep the page table cache within bounds */
> @@ -1204,6 +1208,8 @@ again:
> */
> if (force_flush) {
> force_flush = 0;
> + tlb->start = addr;
> + tlb->end = end;
> tlb_flush_mmu(tlb);
> if (addr != end)
> goto again;
ARGH.. no. What bit about you don't need to modify the generic code
don't you get?
Both ARM and IA64 (and possible others) already do range tracking, you
don't need to modify mm/memory.c _AT_ALL_.
Also, if you modify include/asm-generic/tlb.h to include the ranges it
would be very nice to make that optional, most archs using it won't use
this.
Now IF you're going to change the tlb interface like this, you're going
to get to do it for all architectures, along with a sane benchmark to
show its beneficial to track ranges like this.
But as it stands, people are still questioning the validity of your
mprotect micro-bench, so no, you don't get to change the tlb interface.
On Wed, 2012-05-16 at 10:00 +0200, Peter Zijlstra wrote:
> On Wed, 2012-05-16 at 14:46 +0800, Alex Shi wrote:
> > diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
> > index 75e888b..ed6642a 100644
> > --- a/include/asm-generic/tlb.h
> > +++ b/include/asm-generic/tlb.h
> > @@ -86,6 +86,8 @@ struct mmu_gather {
> > #ifdef CONFIG_HAVE_RCU_TABLE_FREE
> > struct mmu_table_batch *batch;
> > #endif
> > + unsigned long start;
> > + unsigned long end;
> > unsigned int need_flush : 1, /* Did free PTEs */
> > fast_mode : 1; /* No batching */
> >
> > diff --git a/mm/memory.c b/mm/memory.c
> > index 6105f47..b176172 100644
> > --- a/mm/memory.c
> > +++ b/mm/memory.c
> > @@ -206,6 +206,8 @@ void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, bool fullmm)
> > tlb->mm = mm;
> >
> > tlb->fullmm = fullmm;
> > + tlb->start = -1UL;
> > + tlb->end = 0;
> > tlb->need_flush = 0;
> > tlb->fast_mode = (num_possible_cpus() == 1);
> > tlb->local.next = NULL;
Also, you just broke compilation on a bunch of archs.. again.
On 05/16/2012 04:04 PM, Peter Zijlstra wrote:
> On Wed, 2012-05-16 at 10:00 +0200, Peter Zijlstra wrote:
>> On Wed, 2012-05-16 at 14:46 +0800, Alex Shi wrote:
>>> diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
>>> index 75e888b..ed6642a 100644
>>> --- a/include/asm-generic/tlb.h
>>> +++ b/include/asm-generic/tlb.h
>>> @@ -86,6 +86,8 @@ struct mmu_gather {
>>> #ifdef CONFIG_HAVE_RCU_TABLE_FREE
>>> struct mmu_table_batch *batch;
>>> #endif
>>> + unsigned long start;
>>> + unsigned long end;
>>> unsigned int need_flush : 1, /* Did free PTEs */
>>> fast_mode : 1; /* No batching */
>>>
>>> diff --git a/mm/memory.c b/mm/memory.c
>>> index 6105f47..b176172 100644
>>> --- a/mm/memory.c
>>> +++ b/mm/memory.c
>>> @@ -206,6 +206,8 @@ void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, bool fullmm)
>>> tlb->mm = mm;
>>>
>>> tlb->fullmm = fullmm;
>>> + tlb->start = -1UL;
>>> + tlb->end = 0;
>>> tlb->need_flush = 0;
>>> tlb->fast_mode = (num_possible_cpus() == 1);
>>> tlb->local.next = NULL;
>
> Also, you just broke compilation on a bunch of archs.. again.
Sorry. Do you mean not every archs use 'include/asm-generic/tlb.h', so
the assignment of tlb->start in tlb_gather_mmu make trouble?
On Wed, 2012-05-16 at 16:53 +0800, Alex Shi wrote:
>
> Sorry. Do you mean not every archs use 'include/asm-generic/tlb.h', so
> the assignment of tlb->start in tlb_gather_mmu make trouble?
>
Yes exactly. I know you work for Intel, but surely its not forbidden by
contract to look outside of arch/x86/ ? I know!, look at arch/ia64/
that's still Intel.
On Wed, May 16, 2012 at 4:58 PM, Peter Zijlstra <[email protected]> wrote:
> On Wed, 2012-05-16 at 16:53 +0800, Alex Shi wrote:
>>
>> Sorry. Do you mean not every archs use 'include/asm-generic/tlb.h', so
>> the assignment of tlb->start in tlb_gather_mmu make trouble?
>>
> Yes exactly. I know you work for Intel, but surely its not forbidden by
> contract to look outside of arch/x86/ ? I know!, look at arch/ia64/
> that's still Intel.
:) It is my fault. 'make cscope' just help me focus insight into x86.
But frankly speaking, it looks mess that every arch implement similar fields of
mmu_gather. If someone can unify the common fields of mmu_gather, and
left private field for specific arch. it will be great and much helpful.
> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to [email protected]
> More majordomo info at ?http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at ?http://www.tux.org/lkml/
On Wed, 2012-05-16 at 18:58 +0800, Alex Shi wrote:
> If someone can unify the common fields of mmu_gather, and
> left private field for specific arch. it will be great and much
> helpful.
I've send you a link to a patch-set that does exactly that twice now.
http://marc.info/?l=linux-mm&m=129952019004146&w=2
There, 3rd time, now go read all 15 patches ;-)
Getting that merged is still on the todo list, I just got preempted by
other stuff :/
On 05/16/2012 07:04 PM, Peter Zijlstra wrote:
> On Wed, 2012-05-16 at 18:58 +0800, Alex Shi wrote:
>> If someone can unify the common fields of mmu_gather, and
>> left private field for specific arch. it will be great and much
>> helpful.
>
> I've send you a link to a patch-set that does exactly that twice now.
>
> http://marc.info/?l=linux-mm&m=129952019004146&w=2
Thanks for resend. Actually, I had read many times above patch, but it's
still hard to catch some lines, maybe due to it can not apply on current
mm/memory.c that is due to your bit newer code 9547d01b on 2011-05-24.
or maybe I am too stupid. :)
On 05/16/2012 04:00 PM, Peter Zijlstra wrote:
> On Wed, 2012-05-16 at 14:46 +0800, Alex Shi wrote:
>> diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
>> index 75e888b..ed6642a 100644
>> --- a/include/asm-generic/tlb.h
>> +++ b/include/asm-generic/tlb.h
>> @@ -86,6 +86,8 @@ struct mmu_gather {
>> #ifdef CONFIG_HAVE_RCU_TABLE_FREE
>> struct mmu_table_batch *batch;
>> #endif
>> + unsigned long start;
>> + unsigned long end;
>> unsigned int need_flush : 1, /* Did free PTEs */
>> fast_mode : 1; /* No batching */
>>
>> diff --git a/mm/memory.c b/mm/memory.c
>> index 6105f47..b176172 100644
>> --- a/mm/memory.c
>> +++ b/mm/memory.c
>> @@ -206,6 +206,8 @@ void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, bool fullmm)
>> tlb->mm = mm;
>>
>> tlb->fullmm = fullmm;
>> + tlb->start = -1UL;
>> + tlb->end = 0;
>> tlb->need_flush = 0;
>> tlb->fast_mode = (num_possible_cpus() == 1);
>> tlb->local.next = NULL;
>> @@ -248,6 +250,8 @@ void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long e
>> {
>> struct mmu_gather_batch *batch, *next;
>>
>> + tlb->start = start;
>> + tlb->end = end;
>> tlb_flush_mmu(tlb);
>>
>> /* keep the page table cache within bounds */
>> @@ -1204,6 +1208,8 @@ again:
>> */
>> if (force_flush) {
>> force_flush = 0;
>> + tlb->start = addr;
>> + tlb->end = end;
>> tlb_flush_mmu(tlb);
>> if (addr != end)
>> goto again;
>
>
> ARGH.. no. What bit about you don't need to modify the generic code
> don't you get?
>
> Both ARM and IA64 (and possible others) already do range tracking, you
> don't need to modify mm/memory.c _AT_ALL_.
Thanks for time and time remdiner. (shame for me)
In my code checking, the other archs can use self mmu_gather struct
since they code are excluded by HAVE_GENERIC_MMU_GATHER. In another word
if the code protected by HAVE_GENERIC_MMU_GATHER, it is safe for others
That is why tlb_flush_mmu/tlb_finish_mmu enabled both in mm/memory.c and
other archs.
So, if the minimum change of tlb->start/end can be protected by
HAVE_GENERIC_MMU_GATHER, it is safe and harmless, am I right?
If so, the following patch should work on any condition.
---
>From ca29d791c3524887c1776136e9274d10d2114624 Mon Sep 17 00:00:00 2001
From: Alex Shi <[email protected]>
Date: Mon, 14 May 2012 09:17:03 +0800
Subject: [PATCH 6/7] x86/tlb: optimizing tlb_finish_mmu on x86
Not every tlb_flush execution moment is really need to evacuate all
TLB entries, like in munmap, just few 'invlpg' is better for whole
process performance, since it leaves most of TLB entries for later
accessing.
Since all of tlb interfaces in mm/memory.c is reused by all
architecture CPU, except few of them which protected under
HAVE_GENERIC_MMU_GATHER, I keeps global interfaces, just
re-implement x86 specific 'tlb_flush' only. and put the minimum
change under HAVE_GENERIC_MMU_GATHER too.
This patch also rewrite flush_tlb_range for 2 purposes:
1, split it out to get flush_blt_mm_range function.
2, clean up to reduce line breaking, thanks for Borislav's input.
Thanks for Peter Zijlstra time and time reminder for multiple
architecture code safe!
Signed-off-by: Alex Shi <[email protected]>
---
arch/x86/include/asm/tlb.h | 9 +++-
arch/x86/include/asm/tlbflush.h | 2 +
arch/x86/mm/tlb.c | 120 +++++++++++++++++++++------------------
include/asm-generic/tlb.h | 2 +
mm/memory.c | 9 +++
5 files changed, 85 insertions(+), 57 deletions(-)
diff --git a/arch/x86/include/asm/tlb.h b/arch/x86/include/asm/tlb.h
index 829215f..4fef207 100644
--- a/arch/x86/include/asm/tlb.h
+++ b/arch/x86/include/asm/tlb.h
@@ -4,7 +4,14 @@
#define tlb_start_vma(tlb, vma) do { } while (0)
#define tlb_end_vma(tlb, vma) do { } while (0)
#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
-#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
+
+#define tlb_flush(tlb) \
+{ \
+ if (tlb->fullmm == 0) \
+ flush_tlb_mm_range(tlb->mm, tlb->start, tlb->end, 0UL); \
+ else \
+ flush_tlb_mm_range(tlb->mm, 0UL, TLB_FLUSH_ALL, 0UL); \
+}
#include <asm-generic/tlb.h>
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index c39c94e..0107f3c 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -128,6 +128,8 @@ extern void flush_tlb_mm(struct mm_struct *);
extern void flush_tlb_page(struct vm_area_struct *, unsigned long);
extern void flush_tlb_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end);
+extern void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
+ unsigned long end, unsigned long vmflag);
#define flush_tlb() flush_tlb_current_task()
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 5bf4e85..52f6a5a 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -298,22 +298,6 @@ void flush_tlb_current_task(void)
preempt_enable();
}
-void flush_tlb_mm(struct mm_struct *mm)
-{
- preempt_disable();
-
- if (current->active_mm == mm) {
- if (current->mm)
- local_flush_tlb();
- else
- leave_mm(smp_processor_id());
- }
- if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
- flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
-
- preempt_enable();
-}
-
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
static inline int has_large_page(struct mm_struct *mm,
unsigned long start, unsigned long end)
@@ -343,61 +327,85 @@ static inline int has_large_page(struct mm_struct *mm,
return 0;
}
#endif
-void flush_tlb_range(struct vm_area_struct *vma,
- unsigned long start, unsigned long end)
+
+void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
+ unsigned long end, unsigned long vmflag)
{
- struct mm_struct *mm;
+ unsigned long addr;
+ unsigned act_entries, tlb_entries = 0;
- if (!cpu_has_invlpg || vma->vm_flags & VM_HUGETLB
- || tlb_flushall_shift == (u16)TLB_FLUSH_ALL) {
-flush_all:
- flush_tlb_mm(vma->vm_mm);
- return;
+ preempt_disable();
+ if (current->active_mm != mm)
+ goto flush_all;
+
+ if (!current->mm) {
+ leave_mm(smp_processor_id());
+ goto flush_all;
}
- preempt_disable();
- mm = vma->vm_mm;
- if (current->active_mm == mm) {
- if (current->mm) {
- unsigned long addr, vmflag = vma->vm_flags;
- unsigned act_entries, tlb_entries = 0;
+ if (end == TLB_FLUSH_ALL ||
+ tlb_flushall_shift == (u16)TLB_FLUSH_ALL) {
+ local_flush_tlb();
+ goto flush_all;
+ }
- if (vmflag & VM_EXEC)
- tlb_entries = tlb_lli_4k[ENTRIES];
- else
- tlb_entries = tlb_lld_4k[ENTRIES];
+ if (vmflag & VM_EXEC)
+ tlb_entries = tlb_lli_4k[ENTRIES];
+ else
+ tlb_entries = tlb_lld_4k[ENTRIES];
+ act_entries = mm->total_vm > tlb_entries ? tlb_entries : mm->total_vm;
- act_entries = tlb_entries > mm->total_vm ?
- mm->total_vm : tlb_entries;
+ if ((end - start) >> PAGE_SHIFT > act_entries >> tlb_flushall_shift)
+ local_flush_tlb();
+ else {
+ if (has_large_page(mm, start, end)) {
+ local_flush_tlb();
+ goto flush_all;
+ }
+ for (addr = start; addr <= end; addr += PAGE_SIZE)
+ __flush_tlb_single(addr);
- if ((end - start) >> PAGE_SHIFT >
- act_entries >> tlb_flushall_shift)
- local_flush_tlb();
- else {
- if (has_large_page(mm, start, end)) {
- preempt_enable();
- goto flush_all;
- }
- for (addr = start; addr <= end;
- addr += PAGE_SIZE)
- __flush_tlb_single(addr);
+ if (cpumask_any_but(mm_cpumask(mm),
+ smp_processor_id()) < nr_cpu_ids)
+ flush_tlb_others(mm_cpumask(mm), mm, start, end);
+ preempt_enable();
+ return;
+ }
- if (cpumask_any_but(mm_cpumask(mm),
- smp_processor_id()) < nr_cpu_ids)
- flush_tlb_others(mm_cpumask(mm), mm,
- start, end);
- preempt_enable();
- return;
- }
- } else {
+flush_all:
+ if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
+ flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
+ preempt_enable();
+}
+
+void flush_tlb_mm(struct mm_struct *mm)
+{
+ preempt_disable();
+
+ if (current->active_mm == mm) {
+ if (current->mm)
+ local_flush_tlb();
+ else
leave_mm(smp_processor_id());
- }
}
if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
+
preempt_enable();
}
+void flush_tlb_range(struct vm_area_struct *vma,
+ unsigned long start, unsigned long end)
+{
+ struct mm_struct *mm = vma->vm_mm;
+ unsigned long vmflag = vma->vm_flags;
+
+ if (!cpu_has_invlpg || vma->vm_flags & VM_HUGETLB)
+ flush_tlb_mm_range(mm, 0UL, TLB_FLUSH_ALL, 0UL);
+ else
+ flush_tlb_mm_range(mm, start, end, vmflag);
+}
+
void flush_tlb_page(struct vm_area_struct *vma, unsigned long start)
{
diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
index 75e888b..ed6642a 100644
--- a/include/asm-generic/tlb.h
+++ b/include/asm-generic/tlb.h
@@ -86,6 +86,8 @@ struct mmu_gather {
#ifdef CONFIG_HAVE_RCU_TABLE_FREE
struct mmu_table_batch *batch;
#endif
+ unsigned long start;
+ unsigned long end;
unsigned int need_flush : 1, /* Did free PTEs */
fast_mode : 1; /* No batching */
diff --git a/mm/memory.c b/mm/memory.c
index 6105f47..a1078af 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -206,6 +206,8 @@ void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, bool fullmm)
tlb->mm = mm;
tlb->fullmm = fullmm;
+ tlb->start = -1UL;
+ tlb->end = 0;
tlb->need_flush = 0;
tlb->fast_mode = (num_possible_cpus() == 1);
tlb->local.next = NULL;
@@ -248,6 +250,8 @@ void tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long e
{
struct mmu_gather_batch *batch, *next;
+ tlb->start = start;
+ tlb->end = end;
tlb_flush_mmu(tlb);
/* keep the page table cache within bounds */
@@ -1204,6 +1208,11 @@ again:
*/
if (force_flush) {
force_flush = 0;
+
+#ifdef HAVE_GENERIC_MMU_GATHER
+ tlb->start = addr;
+ tlb->end = end;
+#endif
tlb_flush_mmu(tlb);
if (addr != end)
goto again;
--
1.7.5.4
> Now IF you're going to change the tlb interface like this, you're going
> to get to do it for all architectures, along with a sane benchmark to
> show its beneficial to track ranges like this.
>
> But as it stands, people are still questioning the validity of your
> mprotect micro-bench, so no, you don't get to change the tlb interface.
Yes, sure. You are definitely right!
On 05/15/2012 11:49 PM, Alex Shi wrote:
> + if (!cpu_has_invlpg || vma->vm_flags & VM_HUGETLB
> + || tlb_flushall_shift == (u16)TLB_FLUSH_ALL) {
I asked for you to fold the cpu_has_invlpg test into tlb_flushall_shift.
Also, (u16)TLB_FLUSH_ALL really is hit up with the ugly stick... why on
earth is this u16? It's a bit shift, so it can't be bigger than 6 bits
anyway... and if you want to be able to use -1 as a sentinel you might
as well make it a signed value and use < 0 or == -1 as the test.
Am I missing something really fundamental here?
-hpa
--
H. Peter Anvin, Intel Open Source Technology Center
I work for Intel. I don't speak on their behalf.
On Wed, 2012-05-16 at 21:34 +0800, Alex Shi wrote:
>
> So, if the minimum change of tlb->start/end can be protected by
> HAVE_GENERIC_MMU_GATHER, it is safe and harmless, am I right?
>
safe yes, but not entirely harmless. A quick look seems to suggest you
fail for VM_HUGETLB. If your mmu_gather spans a vma with VM_HUGETLB
you'll do a regular range flush not a full mm flush like the other paths
do.
Anyway, I did a quick refresh of my series on a recent -tip tree:
git://git.kernel.org/pub/scm/linux/kernel/git/peterz/mmu.git tlb-unify
With that all you need is to "select HAVE_MMU_GATHER_RANGE" for x86 and
implement a useful flush_tlb_range().
In particular, see:
http://git.kernel.org/?p=linux/kernel/git/peterz/mmu.git;a=commitdiff;h=05e53144177e6242fda404045f50f48114bcf185;hp=2cd7dc710652127522392f4b7ecb5fa6e954941e
I've slightly changed the code to address an open issue with the
vm_flags tracking. We now force flush the mmu_gather whenever VM_HUGETLB
flips because most (all?) archs that look at that flag expect pure huge
pages and not a mixture.
I've seem to have misplaced my cross-compiler set, so I've only compiled
x86-64 for now.
On 05/17/2012 05:09 AM, Peter Zijlstra wrote:
> On Wed, 2012-05-16 at 21:34 +0800, Alex Shi wrote:
>>
>> So, if the minimum change of tlb->start/end can be protected by
>> HAVE_GENERIC_MMU_GATHER, it is safe and harmless, am I right?
>>
> safe yes, but not entirely harmless. A quick look seems to suggest you
> fail for VM_HUGETLB. If your mmu_gather spans a vma with VM_HUGETLB
> you'll do a regular range flush not a full mm flush like the other paths
> do.
Thanks!
Uh, HUGETLB will be found by has_large_page() if THP enabled now. And I
will remove THP cost, then HUGETLB will be deal well. Since the max
number of TLB entries is 512, has_large_page just need execute once just
when the start address is align at HPAGE_SIZE.
IMHO, this patch enabled generic mmu range flush support with just teen
lines. and it is supported well by low level x86 architecture. I like it. :)
>
> Anyway, I did a quick refresh of my series on a recent -tip tree:
>
> git://git.kernel.org/pub/scm/linux/kernel/git/peterz/mmu.git tlb-unify
>
> With that all you need is to "select HAVE_MMU_GATHER_RANGE" for x86 and
> implement a useful flush_tlb_range().
>
> In particular, see:
> http://git.kernel.org/?p=linux/kernel/git/peterz/mmu.git;a=commitdiff;h=05e53144177e6242fda404045f50f48114bcf185;hp=2cd7dc710652127522392f4b7ecb5fa6e954941e
>
> I've slightly changed the code to address an open issue with the
> vm_flags tracking. We now force flush the mmu_gather whenever VM_HUGETLB
> flips because most (all?) archs that look at that flag expect pure huge
> pages and not a mixture.
>
> I've seem to have misplaced my cross-compiler set, so I've only compiled
> x86-64 for now.
Oh, I also need a cross-compiler for other archs. Thanks reminder!
On 05/17/2012 01:55 AM, H. Peter Anvin wrote:
> On 05/15/2012 11:49 PM, Alex Shi wrote:
>> + if (!cpu_has_invlpg || vma->vm_flags & VM_HUGETLB
>> + || tlb_flushall_shift == (u16)TLB_FLUSH_ALL) {
>
> I asked for you to fold the cpu_has_invlpg test into tlb_flushall_shift.
> Also, (u16)TLB_FLUSH_ALL really is hit up with the ugly stick... why on
> earth is this u16? It's a bit shift, so it can't be bigger than 6 bits
> anyway... and if you want to be able to use -1 as a sentinel you might
> as well make it a signed value and use < 0 or == -1 as the test.
!cpu_has_invlpg be moved in next patch, but surely, it is better to be
removed here too.
Sure, u16 make code looks ugly. So, s8 is better and worth to add value
check on later knob patch. Since this change cause later patch
modification. I am going to re-update whole patches.
>
> Am I missing something really fundamental here?
No.
>
> -hpa
>
On Thu, 2012-05-17 at 08:43 +0800, Alex Shi wrote:
> > I've seem to have misplaced my cross-compiler set, so I've only compiled
> > x86-64 for now.
>
>
> Oh, I also need a cross-compiler for other archs. Thanks reminder!
Here:
http://kernel.org/pub/tools/crosstool/
Oh, and if you want to automate this. I attached a ktest.pl config that
does it for you. I'll be pushing this config and others into a examples
directory come the next merge window.
Ktest is located in the Linux tree under tools/testing/ktest/
You can run a bunch of cross compiles by doing:
ktest.pl crosstests.conf
-- Steve
On Wed, May 16, 2012 at 11:09:29PM +0200, Peter Zijlstra wrote:
> On Wed, 2012-05-16 at 21:34 +0800, Alex Shi wrote:
> >
> > So, if the minimum change of tlb->start/end can be protected by
> > HAVE_GENERIC_MMU_GATHER, it is safe and harmless, am I right?
> >
> safe yes, but not entirely harmless. A quick look seems to suggest you
> fail for VM_HUGETLB. If your mmu_gather spans a vma with VM_HUGETLB
> you'll do a regular range flush not a full mm flush like the other paths
> do.
>
> Anyway, I did a quick refresh of my series on a recent -tip tree:
>
> git://git.kernel.org/pub/scm/linux/kernel/git/peterz/mmu.git tlb-unify
>
> With that all you need is to "select HAVE_MMU_GATHER_RANGE" for x86 and
> implement a useful flush_tlb_range().
>
> In particular, see:
> http://git.kernel.org/?p=linux/kernel/git/peterz/mmu.git;a=commitdiff;h=05e53144177e6242fda404045f50f48114bcf185;hp=2cd7dc710652127522392f4b7ecb5fa6e954941e
>
> I've slightly changed the code to address an open issue with the
> vm_flags tracking. We now force flush the mmu_gather whenever VM_HUGETLB
> flips because most (all?) archs that look at that flag expect pure huge
> pages and not a mixture.
>
> I've seem to have misplaced my cross-compiler set, so I've only compiled
> x86-64 for now.
It was on my list to test when you sent out the series initially, but
seems to have slipped my mind until I saw this thread. Here's a patch on
top of your tlb-unify branch that gets sh working (tested on all of
2-level, 3-level, and nommu).
I opted to shove the asm/cacheflush.h include in to tlb.h directly since
it is calling flush_cache_range() openly now, and the rest of the
architectures are just getting at it through various whimsical means. sh
was getting it through pagemap.h -> highmem.h, while ARM presently can't
seem to make up its mind and includes pagemap.h for nommu only as well as
cacheflush.h explicitly.
With the reworked interface we don't seem to actually need to stub out
the interface for the nommu case anymore anyways, all of the users are
insular to mm/memory.c which we don't build for nommu.
Signed-off-by: Paul Mundt <[email protected]>
---
diff --git a/arch/sh/include/asm/pgalloc.h b/arch/sh/include/asm/pgalloc.h
index 8c00785..bedc2ed 100644
--- a/arch/sh/include/asm/pgalloc.h
+++ b/arch/sh/include/asm/pgalloc.h
@@ -13,6 +13,8 @@ extern void pgd_free(struct mm_struct *mm, pgd_t *pgd);
extern void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd);
extern pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address);
extern void pmd_free(struct mm_struct *mm, pmd_t *pmd);
+
+#define __pmd_free_tlb(tlb, pmdp, addr) pmd_free((tlb)->mm, pmdp)
#endif
static inline void pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd,
diff --git a/arch/sh/include/asm/tlb.h b/arch/sh/include/asm/tlb.h
index 45e5925..71af915 100644
--- a/arch/sh/include/asm/tlb.h
+++ b/arch/sh/include/asm/tlb.h
@@ -6,18 +6,7 @@
#endif
#ifndef __ASSEMBLY__
-#include <linux/pagemap.h>
-
#ifdef CONFIG_MMU
-#include <linux/swap.h>
-
-#define __tlb_remove_tlb_entry(tlb, ptep, addr) do { } while (0)
-
-#define __pte_free_tlb(tlb, ptep, addr) pte_free((tlb)->mm, ptep)
-#define __pmd_free_tlb(tlb, pmdp, addr) pmd_free((tlb)->mm, pmdp)
-#define __pud_free_tlb(tlb, pudp, addr) pud_free((tlb)->mm, pudp)
-
-#include <asm-generic/tlb.h>
#if defined(CONFIG_CPU_SH4) || defined(CONFIG_SUPERH64)
extern void tlb_wire_entry(struct vm_area_struct *, unsigned long, pte_t);
@@ -35,8 +24,6 @@ static inline void tlb_unwire_entry(void)
}
#endif
-#else /* CONFIG_MMU */
-
#define __tlb_remove_tlb_entry(tlb, pte, address) do { } while (0)
#include <asm-generic/tlb.h>
diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
index 90a725c..571e2cf 100644
--- a/include/asm-generic/tlb.h
+++ b/include/asm-generic/tlb.h
@@ -18,6 +18,7 @@
#include <linux/swap.h>
#include <asm/pgalloc.h>
#include <asm/tlbflush.h>
+#include <asm/cacheflush.h>
static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page);
On 05/17/2012 10:07 AM, Steven Rostedt wrote:
> On Thu, 2012-05-17 at 08:43 +0800, Alex Shi wrote:
>
>>> I've seem to have misplaced my cross-compiler set, so I've only compiled
>>> x86-64 for now.
>>
>>
>> Oh, I also need a cross-compiler for other archs. Thanks reminder!
>
> Here:
>
> http://kernel.org/pub/tools/crosstool/
>
> Oh, and if you want to automate this. I attached a ktest.pl config that
> does it for you. I'll be pushing this config and others into a examples
> directory come the next merge window.
>
> Ktest is located in the Linux tree under tools/testing/ktest/
>
> You can run a bunch of cross compiles by doing:
>
> ktest.pl crosstests.conf
It works fine. :)
But does ktest only do one kind of config testing?
How it do randconfig testing?
>
> -- Steve
>
>
>
>