From: Borislav Petkov <[email protected]>
Hi all,
now that the Intel TLB subset flushing patches are in mainline, here's
the AMD side of the deal.
Those have been tested on all our families with the mprotect
microbenchmark. Any comments are appreciated.
Thanks.
From: Borislav Petkov <[email protected]>
The TLB characteristics appeared like this in dmesg:
[ 0.065817] Last level iTLB entries: 4KB 512, 2MB 1024, 4MB 512
[ 0.065817] Last level dTLB entries: 4KB 1024, 2MB 1024, 4MB 512
[ 0.065817] tlb_flushall_shift is 0xffffffff
where tlb_flushall_shift is actually -1 but dumped as a hex number.
However, the Kconfig option CONFIG_DEBUG_TLBFLUSH and the rest of the
code treats this as a signed decimal and states "If you set it to -1,
the code flushes the whole TLB unconditionally."
So, fix its formatting in accordance with the other references to it.
Signed-off-by: Borislav Petkov <[email protected]>
---
arch/x86/kernel/cpu/common.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 46d8786d655e..d239977f361f 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -474,7 +474,7 @@ void __cpuinit cpu_detect_tlb(struct cpuinfo_x86 *c)
printk(KERN_INFO "Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n" \
"Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d\n" \
- "tlb_flushall_shift is 0x%x\n",
+ "tlb_flushall_shift: %d\n",
tlb_lli_4k[ENTRIES], tlb_lli_2m[ENTRIES],
tlb_lli_4m[ENTRIES], tlb_lld_4k[ENTRIES],
tlb_lld_2m[ENTRIES], tlb_lld_4m[ENTRIES],
--
1.7.11.rc1
From: Borislav Petkov <[email protected]>
Read I- and DTLB entries count from CPUID on AMD. Handle all the
different family-specific cases.
Signed-off-by: Borislav Petkov <[email protected]>
---
arch/x86/kernel/cpu/amd.c | 51 +++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 51 insertions(+)
diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c
index 9d92e19039f0..48e9d33245d9 100644
--- a/arch/x86/kernel/cpu/amd.c
+++ b/arch/x86/kernel/cpu/amd.c
@@ -737,6 +737,56 @@ static unsigned int __cpuinit amd_size_cache(struct cpuinfo_x86 *c,
}
#endif
+static void __cpuinit cpu_detect_tlb_amd(struct cpuinfo_x86 *c)
+{
+ u32 ebx, eax, ecx, edx;
+ u16 mask = 0xfff;
+
+ if (c->x86 < 0xf)
+ return;
+
+ cpuid(0x80000006, &eax, &ebx, &ecx, &edx);
+
+ tlb_lld_4k[ENTRIES] = (ebx >> 16) & mask;
+ tlb_lli_4k[ENTRIES] = ebx & mask;
+
+ /*
+ * K8 doesn't have 2M/4M entries in the L2 TLB so read out the L1 TLB
+ * characteristics from the CPUID function 0x80000005 instead.
+ */
+ if (c->x86 == 0xf) {
+ cpuid(0x80000005, &eax, &ebx, &ecx, &edx);
+ mask = 0xff;
+ }
+
+ /* Handle DTLB 2M and 4M sizes, fall back to L1 if L2 is disabled */
+ if (!((eax >> 16) & mask)) {
+ u32 a, b, c, d;
+
+ cpuid(0x80000005, &a, &b, &c, &d);
+ tlb_lld_2m[ENTRIES] = (a >> 16) & 0xff;
+ } else {
+ tlb_lld_2m[ENTRIES] = (eax >> 16) & mask;
+ }
+
+ /* a 4M entry uses two 2M entries */
+ tlb_lld_4m[ENTRIES] = tlb_lld_2m[ENTRIES] >> 1;
+
+ /* Handle ITLB 2M and 4M sizes, fall back to L1 if L2 is disabled */
+ if (!(eax & mask)) {
+ /* Erratum 658 */
+ if (c->x86 == 0x15 && c->x86_model <= 0x1f) {
+ tlb_lli_2m[ENTRIES] = 1024;
+ } else {
+ cpuid(0x80000005, &eax, &ebx, &ecx, &edx);
+ tlb_lli_2m[ENTRIES] = eax & 0xff;
+ }
+ } else
+ tlb_lli_2m[ENTRIES] = eax & mask;
+
+ tlb_lli_4m[ENTRIES] = tlb_lli_2m[ENTRIES] >> 1;
+}
+
static const struct cpu_dev __cpuinitconst amd_cpu_dev = {
.c_vendor = "AMD",
.c_ident = { "AuthenticAMD" },
@@ -756,6 +806,7 @@ static const struct cpu_dev __cpuinitconst amd_cpu_dev = {
.c_size_cache = amd_size_cache,
#endif
.c_early_init = early_init_amd,
+ .c_detect_tlb = cpu_detect_tlb_amd,
.c_bsp_init = bsp_init_amd,
.c_init = init_amd,
.c_x86_vendor = X86_VENDOR_AMD,
--
1.7.11.rc1
From: Borislav Petkov <[email protected]>
TLB characteristics on AMD are in the extended CPUID level, leafs
0x8000000{5,6} so we need to check those before doing the detection.
Signed-off-by: Borislav Petkov <[email protected]>
---
arch/x86/kernel/cpu/common.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index d239977f361f..ebab0e77e691 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -940,7 +940,8 @@ void __init identify_boot_cpu(void)
#else
vgetcpu_set_mode();
#endif
- if (boot_cpu_data.cpuid_level >= 2)
+ if (boot_cpu_data.cpuid_level >= 2 ||
+ boot_cpu_data.extended_cpuid_level >= 0x80000006)
cpu_detect_tlb(&boot_cpu_data);
}
--
1.7.11.rc1
From: Borislav Petkov <[email protected]>
Run the mprotect.c microbenchmark on all our families >= K8 and preset
the flushall shift variable accordingly.
Signed-off-by: Borislav Petkov <[email protected]>
---
arch/x86/kernel/cpu/amd.c | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c
index 48e9d33245d9..2da9e1d9838b 100644
--- a/arch/x86/kernel/cpu/amd.c
+++ b/arch/x86/kernel/cpu/amd.c
@@ -737,6 +737,17 @@ static unsigned int __cpuinit amd_size_cache(struct cpuinfo_x86 *c,
}
#endif
+static void __cpuinit cpu_set_tlb_flushall_shift(struct cpuinfo_x86 *c)
+{
+ if (!cpu_has_invlpg)
+ return;
+
+ tlb_flushall_shift = 5;
+
+ if (c->x86 <= 0x11)
+ tlb_flushall_shift = 4;
+}
+
static void __cpuinit cpu_detect_tlb_amd(struct cpuinfo_x86 *c)
{
u32 ebx, eax, ecx, edx;
@@ -785,6 +796,8 @@ static void __cpuinit cpu_detect_tlb_amd(struct cpuinfo_x86 *c)
tlb_lli_2m[ENTRIES] = eax & mask;
tlb_lli_4m[ENTRIES] = tlb_lli_2m[ENTRIES] >> 1;
+
+ cpu_set_tlb_flushall_shift(c);
}
static const struct cpu_dev __cpuinitconst amd_cpu_dev = {
--
1.7.11.rc1
On 08/03/2012 09:37 AM, Borislav Petkov wrote:
> From: Borislav Petkov <[email protected]>
>
> TLB characteristics on AMD are in the extended CPUID level, leafs
> 0x8000000{5,6} so we need to check those before doing the detection.
>
> Signed-off-by: Borislav Petkov <[email protected]>
> ---
> arch/x86/kernel/cpu/common.c | 3 ++-
> 1 file changed, 2 insertions(+), 1 deletion(-)
>
> diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
> index d239977f361f..ebab0e77e691 100644
> --- a/arch/x86/kernel/cpu/common.c
> +++ b/arch/x86/kernel/cpu/common.c
> @@ -940,7 +940,8 @@ void __init identify_boot_cpu(void)
> #else
> vgetcpu_set_mode();
> #endif
> - if (boot_cpu_data.cpuid_level >= 2)
> + if (boot_cpu_data.cpuid_level >= 2 ||
> + boot_cpu_data.extended_cpuid_level >= 0x80000006)
> cpu_detect_tlb(&boot_cpu_data);
> }
>
This really belongs in cpu_detect_tlb() itself (or rather, in the
subfunctions called by it.) Can we just lose the general test here, please?
-hpa
On 08/04/2012 12:37 AM, Borislav Petkov wrote:
> From: Borislav Petkov <[email protected]>
>
> The TLB characteristics appeared like this in dmesg:
>
> [ 0.065817] Last level iTLB entries: 4KB 512, 2MB 1024, 4MB 512
> [ 0.065817] Last level dTLB entries: 4KB 1024, 2MB 1024, 4MB 512
> [ 0.065817] tlb_flushall_shift is 0xffffffff
>
> where tlb_flushall_shift is actually -1 but dumped as a hex number.
> However, the Kconfig option CONFIG_DEBUG_TLBFLUSH and the rest of the
> code treats this as a signed decimal and states "If you set it to -1,
> the code flushes the whole TLB unconditionally."
>
> So, fix its formatting in accordance with the other references to it.
>
> Signed-off-by: Borislav Petkov <[email protected]>
Acked-by Alex Shi <[email protected]>
On Fri, Aug 03, 2012 at 11:32:51AM -0700, H. Peter Anvin wrote:
> > - if (boot_cpu_data.cpuid_level >= 2)
> > + if (boot_cpu_data.cpuid_level >= 2 ||
> > + boot_cpu_data.extended_cpuid_level >= 0x80000006)
> > cpu_detect_tlb(&boot_cpu_data);
>
> This really belongs in cpu_detect_tlb() itself (or rather, in the
> subfunctions called by it.) Can we just lose the general test here,
> please?
Sure, will resend in a bit.
--
Regards/Gruss,
Boris.
Advanced Micro Devices GmbH
Einsteinring 24, 85609 Dornach
GM: Alberto Bozzo
Reg: Dornach, Landkreis Muenchen
HRB Nr. 43632 WEEE Registernr: 129 19551