Return-Path: Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S269686AbUIRXb6 (ORCPT ); Sat, 18 Sep 2004 19:31:58 -0400 Received: (majordomo@vger.kernel.org) by vger.kernel.org id S269684AbUIRXbP (ORCPT ); Sat, 18 Sep 2004 19:31:15 -0400 Received: from omx1-ext.sgi.com ([192.48.179.11]:50387 "EHLO omx1.americas.sgi.com") by vger.kernel.org with ESMTP id S269674AbUIRX3S (ORCPT ); Sat, 18 Sep 2004 19:29:18 -0400 Date: Sat, 18 Sep 2004 16:28:50 -0700 (PDT) From: Christoph Lameter X-X-Sender: clameter@schroedinger.engr.sgi.com To: akpm@osdl.org cc: "David S. Miller" , benh@kernel.crashing.org, wli@holomorphy.com, davem@redhat.com, raybry@sgi.com, ak@muc.de, manfred@colorfullife.com, linux-ia64@vger.kernel.org, linux-kernel@vger.kernel.org, vrajesh@umich.edu, hugh@veritas.com Subject: page fault scalability patch V8: [4/7] universally available cmpxchg on i386 In-Reply-To: Message-ID: References: <20040816143903.GY11200@holomorphy.com> <1094012689.6538.330.camel@gaston> <1094080164.4025.17.camel@gaston> <20040901215741.3538bbf4.davem@davemloft.net> <20040902131057.0341e337.davem@davemloft.net> <20040902140759.5f1003d5.davem@davemloft.net> MIME-Version: 1.0 Content-Type: TEXT/PLAIN; charset=US-ASCII Sender: linux-kernel-owner@vger.kernel.org X-Mailing-List: linux-kernel@vger.kernel.org Content-Length: 6992 Lines: 255 Changelog * Make cmpxchg and cmpxchg8b generally available on i386. * Provide emulation of cmpxchg suitable for UP if build and run on 386. * Provide emulation of cmpxchg8b suitable for UP if build and run on 386 or 486. Signed-off-by: Christoph Lameter Index: linus/include/asm-i386/system.h =================================================================== --- linus.orig/include/asm-i386/system.h 2004-09-18 14:25:23.000000000 -0700 +++ linus/include/asm-i386/system.h 2004-09-18 14:56:59.000000000 -0700 @@ -203,77 +203,6 @@ __set_64bit(ptr, (unsigned int)(value), (unsigned int)((value)>>32ULL) ) : \ __set_64bit(ptr, ll_low(value), ll_high(value)) ) -/* - * Note: no "lock" prefix even on SMP: xchg always implies lock anyway - * Note 2: xchg has side effect, so that attribute volatile is necessary, - * but generally the primitive is invalid, *ptr is output argument. --ANK - */ -static inline unsigned long __xchg(unsigned long x, volatile void * ptr, int size) -{ - switch (size) { - case 1: - __asm__ __volatile__("xchgb %b0,%1" - :"=q" (x) - :"m" (*__xg(ptr)), "0" (x) - :"memory"); - break; - case 2: - __asm__ __volatile__("xchgw %w0,%1" - :"=r" (x) - :"m" (*__xg(ptr)), "0" (x) - :"memory"); - break; - case 4: - __asm__ __volatile__("xchgl %0,%1" - :"=r" (x) - :"m" (*__xg(ptr)), "0" (x) - :"memory"); - break; - } - return x; -} - -/* - * Atomic compare and exchange. Compare OLD with MEM, if identical, - * store NEW in MEM. Return the initial value in MEM. Success is - * indicated by comparing RETURN with OLD. - */ - -#ifdef CONFIG_X86_CMPXCHG -#define __HAVE_ARCH_CMPXCHG 1 -#endif - -static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, - unsigned long new, int size) -{ - unsigned long prev; - switch (size) { - case 1: - __asm__ __volatile__(LOCK_PREFIX "cmpxchgb %b1,%2" - : "=a"(prev) - : "q"(new), "m"(*__xg(ptr)), "0"(old) - : "memory"); - return prev; - case 2: - __asm__ __volatile__(LOCK_PREFIX "cmpxchgw %w1,%2" - : "=a"(prev) - : "q"(new), "m"(*__xg(ptr)), "0"(old) - : "memory"); - return prev; - case 4: - __asm__ __volatile__(LOCK_PREFIX "cmpxchgl %1,%2" - : "=a"(prev) - : "q"(new), "m"(*__xg(ptr)), "0"(old) - : "memory"); - return prev; - } - return old; -} - -#define cmpxchg(ptr,o,n)\ - ((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\ - (unsigned long)(n),sizeof(*(ptr)))) - #ifdef __KERNEL__ struct alt_instr { __u8 *instr; /* original instruction */ Index: linus/arch/i386/Kconfig =================================================================== --- linus.orig/arch/i386/Kconfig 2004-09-18 14:25:23.000000000 -0700 +++ linus/arch/i386/Kconfig 2004-09-18 14:56:59.000000000 -0700 @@ -345,6 +345,11 @@ depends on !M386 default y +config X86_CMPXCHG8B + bool + depends on !M386 && !M486 + default y + config X86_XADD bool depends on !M386 Index: linus/include/asm-i386/processor.h =================================================================== --- linus.orig/include/asm-i386/processor.h 2004-09-18 14:25:23.000000000 -0700 +++ linus/include/asm-i386/processor.h 2004-09-18 14:56:59.000000000 -0700 @@ -657,4 +657,137 @@ #define cache_line_size() (boot_cpu_data.x86_cache_alignment) +/* + * Note: no "lock" prefix even on SMP: xchg always implies lock anyway + * Note 2: xchg has side effect, so that attribute volatile is necessary, + * but generally the primitive is invalid, *ptr is output argument. --ANK + */ +static inline unsigned long __xchg(unsigned long x, volatile void * ptr, int size) +{ + switch (size) { + case 1: + __asm__ __volatile__("xchgb %b0,%1" + :"=q" (x) + :"m" (*__xg(ptr)), "0" (x) + :"memory"); + break; + case 2: + __asm__ __volatile__("xchgw %w0,%1" + :"=r" (x) + :"m" (*__xg(ptr)), "0" (x) + :"memory"); + break; + case 4: + __asm__ __volatile__("xchgl %0,%1" + :"=r" (x) + :"m" (*__xg(ptr)), "0" (x) + :"memory"); + break; + } + return x; +} + +/* + * Atomic compare and exchange. Compare OLD with MEM, if identical, + * store NEW in MEM. Return the initial value in MEM. Success is + * indicated by comparing RETURN with OLD. + */ + +#ifdef CONFIG_X86_CMPXCHG +#define __HAVE_ARCH_CMPXCHG 1 +#endif + +static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, + unsigned long new, int size) +{ + unsigned long prev; +#ifndef CONFIG_X86_CMPXCHG + /* + * Check if the kernel was compiled for an old cpu but the + * currently running cpu can do cmpxchg after all + */ + unsigned long flags; + + /* All CPUs except 386 support CMPXCHG */ + if (cpu_data->x86 > 3) goto have_cmpxchg; + + /* Poor man's cmpxchg for 386. Unsuitable for SMP */ + local_irq_save(flags); + switch (size) { + case 1: + prev = * (u8 *)ptr; + if (prev == old) *(u8 *)ptr = new; + break; + case 2: + prev = * (u16 *)ptr; + if (prev == old) *(u16 *)ptr = new; + case 4: + prev = *(u32 *)ptr; + if (prev == old) *(u32 *)ptr = new; + break; + } + local_irq_restore(flags); + return prev; +have_cmpxchg: +#endif + switch (size) { + case 1: + __asm__ __volatile__(LOCK_PREFIX "cmpxchgb %b1,%2" + : "=a"(prev) + : "q"(new), "m"(*__xg(ptr)), "0"(old) + : "memory"); + return prev; + case 2: + __asm__ __volatile__(LOCK_PREFIX "cmpxchgw %w1,%2" + : "=a"(prev) + : "q"(new), "m"(*__xg(ptr)), "0"(old) + : "memory"); + return prev; + case 4: + __asm__ __volatile__(LOCK_PREFIX "cmpxchgl %1,%2" + : "=a"(prev) + : "q"(new), "m"(*__xg(ptr)), "0"(old) + : "memory"); + return prev; + } + return prev; +} + +static inline unsigned long long cmpxchg8b(volatile unsigned long long *ptr, + unsigned long long old, unsigned long long newv) +{ + unsigned long long prev; +#ifndef CONFIG_X86_CMPXCHG8B + unsigned long flags; + + /* + * Check if the kernel was compiled for an old cpu but + * we are running really on a cpu capable of cmpxchg8b + */ + + if (cpu_has(cpu_data, X86_FEATURE_CX8)) goto have_cmpxchg8b; + + /* Poor mans cmpxchg8b for 386 and 486. Not suitable for SMP */ + local_irq_save(flags); + prev = *ptr; + if (prev == old) *ptr = newv; + local_irq_restore(flags); + return prev; + +have_cmpxchg8b: +#endif + + __asm__ __volatile__( + LOCK_PREFIX "cmpxchg8b %4\n" + : "=A" (prev) + : "0" (old), "c" ((unsigned long)(newv >> 32)), + "b" ((unsigned long)(newv & 0xffffffffLL)), "m" (ptr) + : "memory"); + return prev ; +} + +#define cmpxchg(ptr,o,n)\ + ((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\ + (unsigned long)(n),sizeof(*(ptr)))) + #endif /* __ASM_I386_PROCESSOR_H */ - To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/