Introduce an atomic_cmpxchg operation. Implement this for i386 and ppc64.
Signed-off-by: Nick Piggin <[email protected]>
Index: linux-2.6/include/asm-i386/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-i386/atomic.h
+++ linux-2.6/include/asm-i386/atomic.h
@@ -215,6 +215,8 @@ static __inline__ int atomic_sub_return(
return atomic_add_return(-i,v);
}
+#define atomic_cmpxchg(v, old, new) ((int)cmpxchg(&((v)->counter), old, new))
+
#define atomic_inc_return(v) (atomic_add_return(1,v))
#define atomic_dec_return(v) (atomic_sub_return(1,v))
Index: linux-2.6/include/asm-ppc64/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-ppc64/atomic.h
+++ linux-2.6/include/asm-ppc64/atomic.h
@@ -162,6 +162,8 @@ static __inline__ int atomic_dec_return(
return t;
}
+#define atomic_cmpxchg(v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n)))
+
#define atomic_sub_and_test(a, v) (atomic_sub_return((a), (v)) == 0)
#define atomic_dec_and_test(v) (atomic_dec_return((v)) == 0)
Index: linux-2.6/include/asm-ia64/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-ia64/atomic.h
+++ linux-2.6/include/asm-ia64/atomic.h
@@ -88,6 +88,8 @@ ia64_atomic64_sub (__s64 i, atomic64_t *
return new;
}
+#define atomic_cmpxchg(v, old, new) ((int)cmpxchg(&((v)->counter), old, new))
+
#define atomic_add_return(i,v) \
({ \
int __ia64_aar_i = (i); \
Index: linux-2.6/include/asm-x86_64/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-x86_64/atomic.h
+++ linux-2.6/include/asm-x86_64/atomic.h
@@ -360,6 +360,8 @@ static __inline__ int atomic_sub_return(
return atomic_add_return(-i,v);
}
+#define atomic_cmpxchg(v, old, new) ((int)cmpxchg(&((v)->counter), old, new))
+
#define atomic_inc_return(v) (atomic_add_return(1,v))
#define atomic_dec_return(v) (atomic_sub_return(1,v))
Index: linux-2.6/Documentation/atomic_ops.txt
===================================================================
--- linux-2.6.orig/Documentation/atomic_ops.txt
+++ linux-2.6/Documentation/atomic_ops.txt
@@ -115,6 +115,21 @@ boolean is return which indicates whethe
is negative. It requires explicit memory barrier semantics around the
operation.
+Finally:
+
+ int atomic_cmpxchg(atomic_t *v, int old, int new);
+
+This performs an atomic compare exchange operation on the atomic value v,
+with the given old and new values. Like all atomic_xxx operations,
+atomic_cmpxchg will only satisfy its atomicity semantics as long as all
+other accesses of *v are performed through atomic_xxx operations.
+
+atomic_cmpxchg requires explicit memory barriers around the operation.
+
+The semantics for atomic_cmpxchg are the same as those defined for 'cas'
+below.
+
+
If a caller requires memory barrier semantics around an atomic_t
operation which does not return a value, a set of interfaces are
defined which accomplish this:
Index: linux-2.6/arch/sparc/lib/atomic32.c
===================================================================
--- linux-2.6.orig/arch/sparc/lib/atomic32.c
+++ linux-2.6/arch/sparc/lib/atomic32.c
@@ -38,6 +38,20 @@ int __atomic_add_return(int i, atomic_t
return ret;
}
+int atomic_cmpxchg(atomic_t *v, int old, int new)
+{
+ int ret;
+ unsigned long flags;
+ spin_lock_irqsave(ATOMIC_HASH(v), flags);
+
+ ret = v->counter;
+ if (likely(ret == old))
+ v->counter = new;
+
+ spin_unlock_irqrestore(ATOMIC_HASH(v), flags);
+ return ret;
+}
+
void atomic_set(atomic_t *v, int i)
{
unsigned long flags;
Index: linux-2.6/include/asm-sparc/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-sparc/atomic.h
+++ linux-2.6/include/asm-sparc/atomic.h
@@ -19,6 +19,7 @@ typedef struct { volatile int counter; }
#define ATOMIC_INIT(i) { (i) }
extern int __atomic_add_return(int, atomic_t *);
+extern int atomic_cmpxchg(atomic_t *, int, int);
extern void atomic_set(atomic_t *, int);
#define atomic_read(v) ((v)->counter)
Index: linux-2.6/include/asm-alpha/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-alpha/atomic.h
+++ linux-2.6/include/asm-alpha/atomic.h
@@ -171,6 +171,8 @@ static __inline__ long atomic64_sub_retu
return result;
}
+#define atomic_cmpxchg(v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n)))
+
#define atomic_dec_return(v) atomic_sub_return(1,(v))
#define atomic64_dec_return(v) atomic64_sub_return(1,(v))
Index: linux-2.6/include/asm-m68k/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-m68k/atomic.h
+++ linux-2.6/include/asm-m68k/atomic.h
@@ -139,6 +139,8 @@ static inline void atomic_set_mask(unsig
__asm__ __volatile__("orl %1,%0" : "+m" (*v) : "id" (mask));
}
+#define atomic_cmpxchg(v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n)))
+
/* Atomic operations are already serializing */
#define smp_mb__before_atomic_dec() barrier()
#define smp_mb__after_atomic_dec() barrier()
Index: linux-2.6/include/asm-m68knommu/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-m68knommu/atomic.h
+++ linux-2.6/include/asm-m68knommu/atomic.h
@@ -128,6 +128,8 @@ extern __inline__ int atomic_sub_return(
return temp;
}
+#define atomic_cmpxchg(v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n)))
+
#define atomic_dec_return(v) atomic_sub_return(1,(v))
#define atomic_inc_return(v) atomic_add_return(1,(v))
Index: linux-2.6/include/asm-mips/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-mips/atomic.h
+++ linux-2.6/include/asm-mips/atomic.h
@@ -267,6 +267,8 @@ static __inline__ int atomic_sub_if_posi
return result;
}
+#define atomic_cmpxchg(v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n)))
+
#define atomic_dec_return(v) atomic_sub_return(1,(v))
#define atomic_inc_return(v) atomic_add_return(1,(v))
Index: linux-2.6/include/asm-parisc/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-parisc/atomic.h
+++ linux-2.6/include/asm-parisc/atomic.h
@@ -164,6 +164,7 @@ static __inline__ int atomic_read(const
}
/* exported interface */
+#define atomic_cmpxchg(v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n)))
#define atomic_add(i,v) ((void)(__atomic_add_return( ((int)i),(v))))
#define atomic_sub(i,v) ((void)(__atomic_add_return(-((int)i),(v))))
Index: linux-2.6/include/asm-ppc/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-ppc/atomic.h
+++ linux-2.6/include/asm-ppc/atomic.h
@@ -177,6 +177,8 @@ static __inline__ int atomic_dec_return(
return t;
}
+#define atomic_cmpxchg(v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n)))
+
#define atomic_sub_and_test(a, v) (atomic_sub_return((a), (v)) == 0)
#define atomic_dec_and_test(v) (atomic_dec_return((v)) == 0)
Index: linux-2.6/include/asm-s390/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-s390/atomic.h
+++ linux-2.6/include/asm-s390/atomic.h
@@ -198,6 +198,8 @@ atomic_compare_and_swap(int expected_old
return retval;
}
+#define atomic_cmpxchg(v, o, n) (atomic_compare_and_swap((o), (n), &((v)->counter)))
+
#define smp_mb__before_atomic_dec() smp_mb()
#define smp_mb__after_atomic_dec() smp_mb()
#define smp_mb__before_atomic_inc() smp_mb()
Index: linux-2.6/include/asm-sparc64/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-sparc64/atomic.h
+++ linux-2.6/include/asm-sparc64/atomic.h
@@ -70,6 +70,8 @@ extern int atomic64_sub_ret(int, atomic6
#define atomic_add_negative(i, v) (atomic_add_ret(i, v) < 0)
#define atomic64_add_negative(i, v) (atomic64_add_ret(i, v) < 0)
+#define atomic_cmpxchg(v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n)))
+
/* Atomic operations are already serializing */
#ifdef CONFIG_SMP
#define smp_mb__before_atomic_dec() membar_storeload_loadload();
Index: linux-2.6/include/asm-arm26/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-arm26/atomic.h
+++ linux-2.6/include/asm-arm26/atomic.h
@@ -62,6 +62,20 @@ static inline int atomic_sub_return(int
return val;
}
+static inline int atomic_cmpxchg(atomic_t *v, int old, int new)
+{
+ int ret;
+ unsigned long flags;
+
+ local_irq_save(flags);
+ ret = v->counter;
+ if (likely(ret == old))
+ v->counter = new;
+ local_irq_restore(flags);
+
+ return ret;
+}
+
static inline void atomic_clear_mask(unsigned long mask, unsigned long *addr)
{
unsigned long flags;
Index: linux-2.6/include/asm-frv/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-frv/atomic.h
+++ linux-2.6/include/asm-frv/atomic.h
@@ -414,4 +414,6 @@ extern uint32_t __cmpxchg_32(uint32_t *v
#endif
+#define atomic_cmpxchg(v, old, new) ((int)cmpxchg(&((v)->counter), old, new))
+
#endif /* _ASM_ATOMIC_H */
Index: linux-2.6/include/asm-h8300/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-h8300/atomic.h
+++ linux-2.6/include/asm-h8300/atomic.h
@@ -82,6 +82,18 @@ static __inline__ int atomic_dec_and_tes
return ret == 0;
}
+static __inline__ int atomic_cmpxchg(atomic_t *v, int old, int new)
+{
+ int ret;
+ unsigned long flags;
+ local_irq_save(flags);
+ ret = v->counter;
+ if (likely(ret == old))
+ v->counter = new;
+ local_irq_restore(flags);
+ return ret;
+}
+
static __inline__ void atomic_clear_mask(unsigned long mask, unsigned long *v)
{
__asm__ __volatile__("stc ccr,r1l\n\t"
Index: linux-2.6/include/asm-sh64/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-sh64/atomic.h
+++ linux-2.6/include/asm-sh64/atomic.h
@@ -99,6 +99,20 @@ static __inline__ int atomic_sub_return(
#define atomic_inc(v) atomic_add(1,(v))
#define atomic_dec(v) atomic_sub(1,(v))
+static __inline__ int atomic_cmpxchg(atomic_t *v, int old, int new)
+{
+ int ret;
+ unsigned long flags;
+
+ local_irq_save(flags);
+ ret = v->counter;
+ if (likely(ret == old))
+ v->counter = new;
+ local_irq_restore(flags);
+
+ return ret;
+}
+
static __inline__ void atomic_clear_mask(unsigned int mask, atomic_t *v)
{
unsigned long flags;
Index: linux-2.6/include/asm-v850/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-v850/atomic.h
+++ linux-2.6/include/asm-v850/atomic.h
@@ -90,6 +90,20 @@ static __inline__ void atomic_clear_mask
#define atomic_dec_and_test(v) (atomic_sub_return (1, (v)) == 0)
#define atomic_add_negative(i,v) (atomic_add_return ((i), (v)) < 0)
+static __inline__ int atomic_cmpxchg(atomic_t *v, int old, int new)
+{
+ int ret;
+ unsigned long flags;
+
+ local_irq_save(flags);
+ ret = v->counter;
+ if (likely(ret == old))
+ v->counter = new;
+ local_irq_restore(flags);
+
+ return ret;
+}
+
/* Atomic operations are already serializing on ARM */
#define smp_mb__before_atomic_dec() barrier()
#define smp_mb__after_atomic_dec() barrier()
Index: linux-2.6/include/asm-xtensa/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-xtensa/atomic.h
+++ linux-2.6/include/asm-xtensa/atomic.h
@@ -223,6 +223,7 @@ static inline int atomic_sub_return(int
*/
#define atomic_add_negative(i,v) (atomic_add_return((i),(v)) < 0)
+#define atomic_cmpxchg(v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n)))
static inline void atomic_clear_mask(unsigned int mask, atomic_t *v)
{
Index: linux-2.6/include/asm-sh/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-sh/atomic.h
+++ linux-2.6/include/asm-sh/atomic.h
@@ -87,6 +87,20 @@ static __inline__ int atomic_sub_return(
#define atomic_inc(v) atomic_add(1,(v))
#define atomic_dec(v) atomic_sub(1,(v))
+static __inline__ int atomic_cmpxchg(atomic_t *v, int old, int new)
+{
+ int ret;
+ unsigned long flags;
+
+ local_irq_save(flags);
+ ret = v->counter;
+ if (likely(ret == old))
+ v->counter = new;
+ local_irq_restore(flags);
+
+ return ret;
+}
+
static __inline__ void atomic_clear_mask(unsigned int mask, atomic_t *v)
{
unsigned long flags;
Index: linux-2.6/include/asm-arm/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-arm/atomic.h
+++ linux-2.6/include/asm-arm/atomic.h
@@ -131,6 +131,21 @@ static inline int atomic_sub_return(int
return val;
}
+static inline int atomic_cmpxchg(atomic_t *v, int old, int new)
+{
+ int ret;
+ unsigned long flags;
+
+ local_irq_save(flags);
+ ret = v->counter;
+ if (likely(ret == old))
+ v->counter = new;
+ local_irq_restore(flags);
+
+ return ret;
+}
+
+static inline void atomic_clear_mask(unsigned long mask, unsigned long *addr)
static inline void atomic_clear_mask(unsigned long mask, unsigned long *addr)
{
unsigned long flags;
Index: linux-2.6/include/asm-cris/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-cris/atomic.h
+++ linux-2.6/include/asm-cris/atomic.h
@@ -123,6 +123,19 @@ extern __inline__ int atomic_inc_and_tes
return retval;
}
+static __inline__ int atomic_cmpxchg(atomic_t *v, int old, int new)
+{
+ int ret;
+ unsigned long flags;
+ cris_atomic_save(v, flags);
+ ret = v->counter;
+ if (likely(ret == old))
+ v->counter = new;
+ cris_atomic_restore(v, flags);
+ return ret;
+}
+
+static inline void atomic_clear_mask(unsigned long mask, unsigned long *addr)
/* Atomic operations are already serializing */
#define smp_mb__before_atomic_dec() barrier()
#define smp_mb__after_atomic_dec() barrier()
Index: linux-2.6/include/linux/rcuref.h
===================================================================
--- linux-2.6.orig/include/linux/rcuref.h
+++ /dev/null
@@ -1,220 +0,0 @@
-/*
- * rcuref.h
- *
- * Reference counting for elements of lists/arrays protected by
- * RCU.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
- *
- * Copyright (C) IBM Corporation, 2005
- *
- * Author: Dipankar Sarma <[email protected]>
- * Ravikiran Thirumalai <[email protected]>
- *
- * See Documentation/RCU/rcuref.txt for detailed user guide.
- *
- */
-
-#ifndef _RCUREF_H_
-#define _RCUREF_H_
-
-#ifdef __KERNEL__
-
-#include <linux/types.h>
-#include <linux/interrupt.h>
-#include <linux/spinlock.h>
-#include <asm/atomic.h>
-
-/*
- * These APIs work on traditional atomic_t counters used in the
- * kernel for reference counting. Under special circumstances
- * where a lock-free get() operation races with a put() operation
- * these APIs can be used. See Documentation/RCU/rcuref.txt.
- */
-
-#ifdef __HAVE_ARCH_CMPXCHG
-
-/**
- * rcuref_inc - increment refcount for object.
- * @rcuref: reference counter in the object in question.
- *
- * This should be used only for objects where we use RCU and
- * use the rcuref_inc_lf() api to acquire a reference
- * in a lock-free reader-side critical section.
- */
-static inline void rcuref_inc(atomic_t *rcuref)
-{
- atomic_inc(rcuref);
-}
-
-/**
- * rcuref_dec - decrement refcount for object.
- * @rcuref: reference counter in the object in question.
- *
- * This should be used only for objects where we use RCU and
- * use the rcuref_inc_lf() api to acquire a reference
- * in a lock-free reader-side critical section.
- */
-static inline void rcuref_dec(atomic_t *rcuref)
-{
- atomic_dec(rcuref);
-}
-
-/**
- * rcuref_dec_and_test - decrement refcount for object and test
- * @rcuref: reference counter in the object.
- * @release: pointer to the function that will clean up the object
- * when the last reference to the object is released.
- * This pointer is required.
- *
- * Decrement the refcount, and if 0, return 1. Else return 0.
- *
- * This should be used only for objects where we use RCU and
- * use the rcuref_inc_lf() api to acquire a reference
- * in a lock-free reader-side critical section.
- */
-static inline int rcuref_dec_and_test(atomic_t *rcuref)
-{
- return atomic_dec_and_test(rcuref);
-}
-
-/*
- * cmpxchg is needed on UP too, if deletions to the list/array can happen
- * in interrupt context.
- */
-
-/**
- * rcuref_inc_lf - Take reference to an object in a read-side
- * critical section protected by RCU.
- * @rcuref: reference counter in the object in question.
- *
- * Try and increment the refcount by 1. The increment might fail if
- * the reference counter has been through a 1 to 0 transition and
- * is no longer part of the lock-free list.
- * Returns non-zero on successful increment and zero otherwise.
- */
-static inline int rcuref_inc_lf(atomic_t *rcuref)
-{
- int c, old;
- c = atomic_read(rcuref);
- while (c && (old = cmpxchg(&rcuref->counter, c, c + 1)) != c)
- c = old;
- return c;
-}
-
-#else /* !__HAVE_ARCH_CMPXCHG */
-
-extern spinlock_t __rcuref_hash[];
-
-/*
- * Use a hash table of locks to protect the reference count
- * since cmpxchg is not available in this arch.
- */
-#ifdef CONFIG_SMP
-#define RCUREF_HASH_SIZE 4
-#define RCUREF_HASH(k) \
- (&__rcuref_hash[(((unsigned long)k)>>8) & (RCUREF_HASH_SIZE-1)])
-#else
-#define RCUREF_HASH_SIZE 1
-#define RCUREF_HASH(k) &__rcuref_hash[0]
-#endif /* CONFIG_SMP */
-
-/**
- * rcuref_inc - increment refcount for object.
- * @rcuref: reference counter in the object in question.
- *
- * This should be used only for objects where we use RCU and
- * use the rcuref_inc_lf() api to acquire a reference in a lock-free
- * reader-side critical section.
- */
-static inline void rcuref_inc(atomic_t *rcuref)
-{
- unsigned long flags;
- spin_lock_irqsave(RCUREF_HASH(rcuref), flags);
- rcuref->counter += 1;
- spin_unlock_irqrestore(RCUREF_HASH(rcuref), flags);
-}
-
-/**
- * rcuref_dec - decrement refcount for object.
- * @rcuref: reference counter in the object in question.
- *
- * This should be used only for objects where we use RCU and
- * use the rcuref_inc_lf() api to acquire a reference in a lock-free
- * reader-side critical section.
- */
-static inline void rcuref_dec(atomic_t *rcuref)
-{
- unsigned long flags;
- spin_lock_irqsave(RCUREF_HASH(rcuref), flags);
- rcuref->counter -= 1;
- spin_unlock_irqrestore(RCUREF_HASH(rcuref), flags);
-}
-
-/**
- * rcuref_dec_and_test - decrement refcount for object and test
- * @rcuref: reference counter in the object.
- * @release: pointer to the function that will clean up the object
- * when the last reference to the object is released.
- * This pointer is required.
- *
- * Decrement the refcount, and if 0, return 1. Else return 0.
- *
- * This should be used only for objects where we use RCU and
- * use the rcuref_inc_lf() api to acquire a reference in a lock-free
- * reader-side critical section.
- */
-static inline int rcuref_dec_and_test(atomic_t *rcuref)
-{
- unsigned long flags;
- spin_lock_irqsave(RCUREF_HASH(rcuref), flags);
- rcuref->counter--;
- if (!rcuref->counter) {
- spin_unlock_irqrestore(RCUREF_HASH(rcuref), flags);
- return 1;
- } else {
- spin_unlock_irqrestore(RCUREF_HASH(rcuref), flags);
- return 0;
- }
-}
-
-/**
- * rcuref_inc_lf - Take reference to an object of a lock-free collection
- * by traversing a lock-free list/array.
- * @rcuref: reference counter in the object in question.
- *
- * Try and increment the refcount by 1. The increment might fail if
- * the reference counter has been through a 1 to 0 transition and
- * object is no longer part of the lock-free list.
- * Returns non-zero on successful increment and zero otherwise.
- */
-static inline int rcuref_inc_lf(atomic_t *rcuref)
-{
- int ret;
- unsigned long flags;
- spin_lock_irqsave(RCUREF_HASH(rcuref), flags);
- if (rcuref->counter)
- ret = rcuref->counter++;
- else
- ret = 0;
- spin_unlock_irqrestore(RCUREF_HASH(rcuref), flags);
- return ret;
-}
-
-
-#endif /* !__HAVE_ARCH_CMPXCHG */
-
-#endif /* __KERNEL__ */
-#endif /* _RCUREF_H_ */
Index: linux-2.6/fs/aio.c
===================================================================
--- linux-2.6.orig/fs/aio.c
+++ linux-2.6/fs/aio.c
@@ -29,7 +29,6 @@
#include <linux/highmem.h>
#include <linux/workqueue.h>
#include <linux/security.h>
-#include <linux/rcuref.h>
#include <asm/kmap_types.h>
#include <asm/uaccess.h>
@@ -500,7 +499,7 @@ static int __aio_put_req(struct kioctx *
/* Must be done under the lock to serialise against cancellation.
* Call this aio_fput as it duplicates fput via the fput_work.
*/
- if (unlikely(rcuref_dec_and_test(&req->ki_filp->f_count))) {
+ if (unlikely(atomic_dec_and_test(&req->ki_filp->f_count))) {
get_ioctx(ctx);
spin_lock(&fput_lock);
list_add(&req->ki_list, &fput_head);
Index: linux-2.6/fs/file_table.c
===================================================================
--- linux-2.6.orig/fs/file_table.c
+++ linux-2.6/fs/file_table.c
@@ -117,7 +117,7 @@ EXPORT_SYMBOL(get_empty_filp);
void fastcall fput(struct file *file)
{
- if (rcuref_dec_and_test(&file->f_count))
+ if (atomic_dec_and_test(&file->f_count))
__fput(file);
}
@@ -166,7 +166,7 @@ struct file fastcall *fget(unsigned int
rcu_read_lock();
file = fcheck_files(files, fd);
if (file) {
- if (!rcuref_inc_lf(&file->f_count)) {
+ if (!atomic_inc_not_zero(&file->f_count)) {
/* File object ref couldn't be taken */
rcu_read_unlock();
return NULL;
@@ -198,7 +198,7 @@ struct file fastcall *fget_light(unsigne
rcu_read_lock();
file = fcheck_files(files, fd);
if (file) {
- if (rcuref_inc_lf(&file->f_count))
+ if (atomic_inc_not_zero(&file->f_count))
*fput_needed = 1;
else
/* Didn't get the reference, someone's freed */
@@ -213,7 +213,7 @@ struct file fastcall *fget_light(unsigne
void put_filp(struct file *file)
{
- if (rcuref_dec_and_test(&file->f_count)) {
+ if (atomic_dec_and_test(&file->f_count)) {
security_file_free(file);
file_kill(file);
file_free(file);
Index: linux-2.6/include/linux/fs.h
===================================================================
--- linux-2.6.orig/include/linux/fs.h
+++ linux-2.6/include/linux/fs.h
@@ -9,7 +9,6 @@
#include <linux/config.h>
#include <linux/limits.h>
#include <linux/ioctl.h>
-#include <linux/rcuref.h>
/*
* It's silly to have NR_OPEN bigger than NR_FILE, but you can change
@@ -604,7 +603,7 @@ extern spinlock_t files_lock;
#define file_list_lock() spin_lock(&files_lock);
#define file_list_unlock() spin_unlock(&files_lock);
-#define get_file(x) rcuref_inc(&(x)->f_count)
+#define get_file(x) atomic_inc(&(x)->f_count)
#define file_count(x) atomic_read(&(x)->f_count)
#define MAX_NON_LFS ((1UL<<31) - 1)
Index: linux-2.6/Documentation/RCU/rcuref.txt
===================================================================
--- linux-2.6.orig/Documentation/RCU/rcuref.txt
+++ linux-2.6/Documentation/RCU/rcuref.txt
@@ -31,9 +31,9 @@ release_referenced() delete()
If this list/array is made lock free using rcu as in changing the
write_lock in add() and delete() to spin_lock and changing read_lock
-in search_and_reference to rcu_read_lock(), the rcuref_get in
+in search_and_reference to rcu_read_lock(), the atomic_get in
search_and_reference could potentially hold reference to an element which
-has already been deleted from the list/array. rcuref_lf_get_rcu takes
+has already been deleted from the list/array. atomic_inc_not_zero takes
care of this scenario. search_and_reference should look as;
1. 2.
@@ -41,7 +41,7 @@ add() search_and_reference()
{ {
alloc_object rcu_read_lock();
... search_for_element
- atomic_set(&el->rc, 1); if (rcuref_inc_lf(&el->rc)) {
+ atomic_set(&el->rc, 1); if (atomic_inc_not_zero(&el->rc)) {
write_lock(&list_lock); rcu_read_unlock();
return FAIL;
add_element }
@@ -52,23 +52,23 @@ add() search_and_reference()
release_referenced() delete()
{ {
... write_lock(&list_lock);
- rcuref_dec(&el->rc, relfunc) ...
+ atomic_dec(&el->rc, relfunc) ...
... delete_element
} write_unlock(&list_lock);
...
- if (rcuref_dec_and_test(&el->rc))
+ if (atomic_dec_and_test(&el->rc))
call_rcu(&el->head, el_free);
...
}
Sometimes, reference to the element need to be obtained in the
-update (write) stream. In such cases, rcuref_inc_lf might be an overkill
-since the spinlock serialising list updates are held. rcuref_inc
+update (write) stream. In such cases, atomic_inc_not_zero might be an overkill
+since the spinlock serialising list updates are held. atomic_inc
is to be used in such cases.
-For arches which do not have cmpxchg rcuref_inc_lf
+For arches which do not have cmpxchg atomic_inc_not_zero
api uses a hashed spinlock implementation and the same hashed spinlock
-is acquired in all rcuref_xxx primitives to preserve atomicity.
-Note: Use rcuref_inc api only if you need to use rcuref_inc_lf on the
-refcounter atleast at one place. Mixing rcuref_inc and atomic_xxx api
-might lead to races. rcuref_inc_lf() must be used in lockfree
+is acquired in all atomic_xxx primitives to preserve atomicity.
+Note: Use atomic_inc api only if you need to use atomic_inc_not_zero on the
+refcounter atleast at one place. Mixing atomic_inc and atomic_xxx api
+might lead to races. atomic_inc_not_zero() must be used in lockfree
RCU critical sections only.
Index: linux-2.6/lib/dec_and_lock.c
===================================================================
--- linux-2.6.orig/lib/dec_and_lock.c
+++ linux-2.6/lib/dec_and_lock.c
@@ -3,10 +3,21 @@
#include <asm/atomic.h>
/*
- * This is an architecture-neutral, but slow,
- * implementation of the notion of "decrement
- * a reference count, and return locked if it
- * decremented to zero".
+ * Decrement v if it is not equal to one, and return non zero.
+ * Otherwise return zero.
+ */
+static int atomic_dec_not_one(atomic_t *v)
+{
+ int c, old;
+ c = atomic_read(v);
+ while (c > 1 && (old = atomic_cmpxchg((v), c, c - 1)) != c)
+ c = old;
+ return c > 1;
+}
+
+/*
+ * Implement an atomic "decrement a reference count, and
+ * return locked if it decremented to zero" operation.
*
* NOTE NOTE NOTE! This is _not_ equivalent to
*
@@ -19,14 +30,16 @@
* because the spin-lock and the decrement must be
* "atomic".
*
- * This slow version gets the spinlock unconditionally,
- * and releases it if it isn't needed. Architectures
- * are encouraged to come up with better approaches,
- * this is trivially done efficiently using a load-locked
- * store-conditional approach, for example.
+ * This version shortcuts the unconditional spin lock on
+ * CONFIG_SMP builds to avoid the expensive lock operation.
*/
int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock)
{
+#ifdef CONFIG_SMP
+ if (atomic_dec_not_one(atomic))
+ return 0;
+#endif
+
spin_lock(lock);
if (atomic_dec_and_test(atomic))
return 1;
Index: linux-2.6/arch/alpha/lib/dec_and_lock.c
===================================================================
--- linux-2.6.orig/arch/alpha/lib/dec_and_lock.c
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * arch/alpha/lib/dec_and_lock.c
- *
- * ll/sc version of atomic_dec_and_lock()
- *
- */
-
-#include <linux/spinlock.h>
-#include <asm/atomic.h>
-
- asm (".text \n\
- .global _atomic_dec_and_lock \n\
- .ent _atomic_dec_and_lock \n\
- .align 4 \n\
-_atomic_dec_and_lock: \n\
- .prologue 0 \n\
-1: ldl_l $1, 0($16) \n\
- subl $1, 1, $1 \n\
- beq $1, 2f \n\
- stl_c $1, 0($16) \n\
- beq $1, 4f \n\
- mb \n\
- clr $0 \n\
- ret \n\
-2: br $29, 3f \n\
-3: ldgp $29, 0($29) \n\
- br $atomic_dec_and_lock_1..ng \n\
- .subsection 2 \n\
-4: br 1b \n\
- .previous \n\
- .end _atomic_dec_and_lock");
-
-static int __attribute_used__
-atomic_dec_and_lock_1(atomic_t *atomic, spinlock_t *lock)
-{
- /* Slow path */
- spin_lock(lock);
- if (atomic_dec_and_test(atomic))
- return 1;
- spin_unlock(lock);
- return 0;
-}
Index: linux-2.6/arch/i386/lib/dec_and_lock.c
===================================================================
--- linux-2.6.orig/arch/i386/lib/dec_and_lock.c
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * x86 version of "atomic_dec_and_lock()" using
- * the atomic "cmpxchg" instruction.
- *
- * (For CPU's lacking cmpxchg, we use the slow
- * generic version, and this one never even gets
- * compiled).
- */
-
-#include <linux/spinlock.h>
-#include <linux/module.h>
-#include <asm/atomic.h>
-
-int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock)
-{
- int counter;
- int newcount;
-
-repeat:
- counter = atomic_read(atomic);
- newcount = counter-1;
-
- if (!newcount)
- goto slow_path;
-
- asm volatile("lock; cmpxchgl %1,%2"
- :"=a" (newcount)
- :"r" (newcount), "m" (atomic->counter), "0" (counter));
-
- /* If the above failed, "eax" will have changed */
- if (newcount != counter)
- goto repeat;
- return 0;
-
-slow_path:
- spin_lock(lock);
- if (atomic_dec_and_test(atomic))
- return 1;
- spin_unlock(lock);
- return 0;
-}
-EXPORT_SYMBOL(_atomic_dec_and_lock);
Index: linux-2.6/arch/ia64/lib/dec_and_lock.c
===================================================================
--- linux-2.6.orig/arch/ia64/lib/dec_and_lock.c
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright (C) 2003 Jerome Marchand, Bull S.A.
- * Cleaned up by David Mosberger-Tang <[email protected]>
- *
- * This file is released under the GPLv2, or at your option any later version.
- *
- * ia64 version of "atomic_dec_and_lock()" using the atomic "cmpxchg" instruction. This
- * code is an adaptation of the x86 version of "atomic_dec_and_lock()".
- */
-
-#include <linux/compiler.h>
-#include <linux/module.h>
-#include <linux/spinlock.h>
-#include <asm/atomic.h>
-
-/*
- * Decrement REFCOUNT and if the count reaches zero, acquire the spinlock. Both of these
- * operations have to be done atomically, so that the count doesn't drop to zero without
- * acquiring the spinlock first.
- */
-int
-_atomic_dec_and_lock (atomic_t *refcount, spinlock_t *lock)
-{
- int old, new;
-
- do {
- old = atomic_read(refcount);
- new = old - 1;
-
- if (unlikely (old == 1)) {
- /* oops, we may be decrementing to zero, do it the slow way... */
- spin_lock(lock);
- if (atomic_dec_and_test(refcount))
- return 1;
- spin_unlock(lock);
- return 0;
- }
- } while (cmpxchg(&refcount->counter, old, new) != old);
- return 0;
-}
-
-EXPORT_SYMBOL(_atomic_dec_and_lock);
Index: linux-2.6/arch/mips/lib/dec_and_lock.c
===================================================================
--- linux-2.6.orig/arch/mips/lib/dec_and_lock.c
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * MIPS version of atomic_dec_and_lock() using cmpxchg
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version
- * 2 of the License, or (at your option) any later version.
- */
-
-#include <linux/module.h>
-#include <linux/spinlock.h>
-#include <asm/atomic.h>
-#include <asm/system.h>
-
-/*
- * This is an implementation of the notion of "decrement a
- * reference count, and return locked if it decremented to zero".
- *
- * This implementation can be used on any architecture that
- * has a cmpxchg, and where atomic->value is an int holding
- * the value of the atomic (i.e. the high bits aren't used
- * for a lock or anything like that).
- */
-int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock)
-{
- int counter;
- int newcount;
-
- for (;;) {
- counter = atomic_read(atomic);
- newcount = counter - 1;
- if (!newcount)
- break; /* do it the slow way */
-
- newcount = cmpxchg(&atomic->counter, counter, newcount);
- if (newcount == counter)
- return 0;
- }
-
- spin_lock(lock);
- if (atomic_dec_and_test(atomic))
- return 1;
- spin_unlock(lock);
- return 0;
-}
-
-EXPORT_SYMBOL(_atomic_dec_and_lock);
Index: linux-2.6/arch/ppc/lib/dec_and_lock.c
===================================================================
--- linux-2.6.orig/arch/ppc/lib/dec_and_lock.c
+++ /dev/null
@@ -1,38 +0,0 @@
-#include <linux/module.h>
-#include <linux/spinlock.h>
-#include <asm/atomic.h>
-#include <asm/system.h>
-
-/*
- * This is an implementation of the notion of "decrement a
- * reference count, and return locked if it decremented to zero".
- *
- * This implementation can be used on any architecture that
- * has a cmpxchg, and where atomic->value is an int holding
- * the value of the atomic (i.e. the high bits aren't used
- * for a lock or anything like that).
- */
-int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock)
-{
- int counter;
- int newcount;
-
- for (;;) {
- counter = atomic_read(atomic);
- newcount = counter - 1;
- if (!newcount)
- break; /* do it the slow way */
-
- newcount = cmpxchg(&atomic->counter, counter, newcount);
- if (newcount == counter)
- return 0;
- }
-
- spin_lock(lock);
- if (atomic_dec_and_test(atomic))
- return 1;
- spin_unlock(lock);
- return 0;
-}
-
-EXPORT_SYMBOL(_atomic_dec_and_lock);
Index: linux-2.6/arch/ppc64/lib/dec_and_lock.c
===================================================================
--- linux-2.6.orig/arch/ppc64/lib/dec_and_lock.c
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * ppc64 version of atomic_dec_and_lock() using cmpxchg
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; either version
- * 2 of the License, or (at your option) any later version.
- */
-
-#include <linux/module.h>
-#include <linux/spinlock.h>
-#include <asm/atomic.h>
-#include <asm/system.h>
-
-/*
- * This is an implementation of the notion of "decrement a
- * reference count, and return locked if it decremented to zero".
- *
- * This implementation can be used on any architecture that
- * has a cmpxchg, and where atomic->value is an int holding
- * the value of the atomic (i.e. the high bits aren't used
- * for a lock or anything like that).
- */
-int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock)
-{
- int counter;
- int newcount;
-
- for (;;) {
- counter = atomic_read(atomic);
- newcount = counter - 1;
- if (!newcount)
- break; /* do it the slow way */
-
- newcount = cmpxchg(&atomic->counter, counter, newcount);
- if (newcount == counter)
- return 0;
- }
-
- spin_lock(lock);
- if (atomic_dec_and_test(atomic))
- return 1;
- spin_unlock(lock);
- return 0;
-}
-
-EXPORT_SYMBOL(_atomic_dec_and_lock);
Index: linux-2.6/arch/x86_64/lib/dec_and_lock.c
===================================================================
--- linux-2.6.orig/arch/x86_64/lib/dec_and_lock.c
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * x86 version of "atomic_dec_and_lock()" using
- * the atomic "cmpxchg" instruction.
- *
- * (For CPU's lacking cmpxchg, we use the slow
- * generic version, and this one never even gets
- * compiled).
- */
-
-#include <linux/spinlock.h>
-#include <asm/atomic.h>
-
-int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock)
-{
- int counter;
- int newcount;
-
-repeat:
- counter = atomic_read(atomic);
- newcount = counter-1;
-
- if (!newcount)
- goto slow_path;
-
- asm volatile("lock; cmpxchgl %1,%2"
- :"=a" (newcount)
- :"r" (newcount), "m" (atomic->counter), "0" (counter));
-
- /* If the above failed, "eax" will have changed */
- if (newcount != counter)
- goto repeat;
- return 0;
-
-slow_path:
- spin_lock(lock);
- if (atomic_dec_and_test(atomic))
- return 1;
- spin_unlock(lock);
- return 0;
-}
Index: linux-2.6/lib/Makefile
===================================================================
--- linux-2.6.orig/lib/Makefile
+++ linux-2.6/lib/Makefile
@@ -5,7 +5,7 @@
lib-y := errno.o ctype.o string.o vsprintf.o cmdline.o \
bust_spinlocks.o rbtree.o radix-tree.o dump_stack.o \
idr.o div64.o int_sqrt.o bitmap.o extable.o prio_tree.o \
- sha1.o
+ sha1.o dec_and_lock.o
lib-y += kobject.o kref.o kobject_uevent.o klist.o
@@ -24,10 +24,6 @@ lib-$(CONFIG_GENERIC_FIND_NEXT_BIT) += f
obj-$(CONFIG_LOCK_KERNEL) += kernel_lock.o
obj-$(CONFIG_DEBUG_PREEMPT) += smp_processor_id.o
-ifneq ($(CONFIG_HAVE_DEC_LOCK),y)
- lib-y += dec_and_lock.o
-endif
-
obj-$(CONFIG_CRC_CCITT) += crc-ccitt.o
obj-$(CONFIG_CRC16) += crc16.o
obj-$(CONFIG_CRC32) += crc32.o
Index: linux-2.6/arch/alpha/Kconfig
===================================================================
--- linux-2.6.orig/arch/alpha/Kconfig
+++ linux-2.6/arch/alpha/Kconfig
@@ -501,11 +501,6 @@ config SMP
If you don't know what to do here, say N.
-config HAVE_DEC_LOCK
- bool
- depends on SMP
- default y
-
config NR_CPUS
int "Maximum number of CPUs (2-64)"
range 2 64
Index: linux-2.6/arch/i386/Kconfig
===================================================================
--- linux-2.6.orig/arch/i386/Kconfig
+++ linux-2.6/arch/i386/Kconfig
@@ -908,11 +908,6 @@ config IRQBALANCE
The default yes will allow the kernel to do irq load balancing.
Saying no will keep the kernel from doing irq load balancing.
-config HAVE_DEC_LOCK
- bool
- depends on (SMP || PREEMPT) && X86_CMPXCHG
- default y
-
# turning this on wastes a bunch of space.
# Summit needs it only when NUMA is on
config BOOT_IOREMAP
Index: linux-2.6/arch/i386/lib/Makefile
===================================================================
--- linux-2.6.orig/arch/i386/lib/Makefile
+++ linux-2.6/arch/i386/lib/Makefile
@@ -7,4 +7,3 @@ lib-y = checksum.o delay.o usercopy.o ge
bitops.o
lib-$(CONFIG_X86_USE_3DNOW) += mmx.o
-lib-$(CONFIG_HAVE_DEC_LOCK) += dec_and_lock.o
Index: linux-2.6/arch/ia64/Kconfig
===================================================================
--- linux-2.6.orig/arch/ia64/Kconfig
+++ linux-2.6/arch/ia64/Kconfig
@@ -298,11 +298,6 @@ config PREEMPT
source "mm/Kconfig"
-config HAVE_DEC_LOCK
- bool
- depends on (SMP || PREEMPT)
- default y
-
config IA32_SUPPORT
bool "Support for Linux/x86 binaries"
help
Index: linux-2.6/arch/ia64/lib/Makefile
===================================================================
--- linux-2.6.orig/arch/ia64/lib/Makefile
+++ linux-2.6/arch/ia64/lib/Makefile
@@ -15,7 +15,6 @@ lib-$(CONFIG_ITANIUM) += copy_page.o cop
lib-$(CONFIG_MCKINLEY) += copy_page_mck.o memcpy_mck.o
lib-$(CONFIG_PERFMON) += carta_random.o
lib-$(CONFIG_MD_RAID5) += xor.o
-lib-$(CONFIG_HAVE_DEC_LOCK) += dec_and_lock.o
AFLAGS___divdi3.o =
AFLAGS___udivdi3.o = -DUNSIGNED
Index: linux-2.6/arch/m32r/Kconfig
===================================================================
--- linux-2.6.orig/arch/m32r/Kconfig
+++ linux-2.6/arch/m32r/Kconfig
@@ -220,11 +220,6 @@ config PREEMPT
Say Y here if you are building a kernel for a desktop, embedded
or real-time system. Say N if you are unsure.
-config HAVE_DEC_LOCK
- bool
- depends on (SMP || PREEMPT)
- default n
-
config SMP
bool "Symmetric multi-processing support"
---help---
Index: linux-2.6/arch/mips/Kconfig
===================================================================
--- linux-2.6.orig/arch/mips/Kconfig
+++ linux-2.6/arch/mips/Kconfig
@@ -1009,10 +1009,6 @@ config GENERIC_CALIBRATE_DELAY
bool
default y
-config HAVE_DEC_LOCK
- bool
- default y
-
#
# Select some configuration options automatically based on user selections.
#
Index: linux-2.6/arch/ppc/Kconfig
===================================================================
--- linux-2.6.orig/arch/ppc/Kconfig
+++ linux-2.6/arch/ppc/Kconfig
@@ -26,10 +26,6 @@ config GENERIC_CALIBRATE_DELAY
bool
default y
-config HAVE_DEC_LOCK
- bool
- default y
-
config PPC
bool
default y
Index: linux-2.6/arch/ppc64/Kconfig
===================================================================
--- linux-2.6.orig/arch/ppc64/Kconfig
+++ linux-2.6/arch/ppc64/Kconfig
@@ -28,10 +28,6 @@ config GENERIC_ISA_DMA
bool
default y
-config HAVE_DEC_LOCK
- bool
- default y
-
config EARLY_PRINTK
bool
default y
Index: linux-2.6/arch/sparc64/lib/Makefile
===================================================================
--- linux-2.6.orig/arch/sparc64/lib/Makefile
+++ linux-2.6/arch/sparc64/lib/Makefile
@@ -14,6 +14,4 @@ lib-y := PeeCeeI.o copy_page.o clear_pag
copy_in_user.o user_fixup.o memmove.o \
mcount.o ipcsum.o rwsem.o xor.o find_bit.o delay.o
-lib-$(CONFIG_HAVE_DEC_LOCK) += dec_and_lock.o
-
obj-y += iomap.o
Index: linux-2.6/arch/x86_64/Kconfig
===================================================================
--- linux-2.6.orig/arch/x86_64/Kconfig
+++ linux-2.6/arch/x86_64/Kconfig
@@ -277,11 +277,6 @@ source "mm/Kconfig"
config HAVE_ARCH_EARLY_PFN_TO_NID
def_bool y
-config HAVE_DEC_LOCK
- bool
- depends on SMP
- default y
-
config NR_CPUS
int "Maximum number of CPUs (2-256)"
range 2 256
Index: linux-2.6/arch/x86_64/lib/Makefile
===================================================================
--- linux-2.6.orig/arch/x86_64/lib/Makefile
+++ linux-2.6/arch/x86_64/lib/Makefile
@@ -10,5 +10,3 @@ lib-y := csum-partial.o csum-copy.o csum
usercopy.o getuser.o putuser.o \
thunk.o clear_page.o copy_page.o bitstr.o bitops.o
lib-y += memcpy.o memmove.o memset.o copy_user.o
-
-lib-$(CONFIG_HAVE_DEC_LOCK) += dec_and_lock.o
Index: linux-2.6/arch/xtensa/Kconfig
===================================================================
--- linux-2.6.orig/arch/xtensa/Kconfig
+++ linux-2.6/arch/xtensa/Kconfig
@@ -26,10 +26,6 @@ config RWSEM_XCHGADD_ALGORITHM
bool
default y
-config HAVE_DEC_LOCK
- bool
- default y
-
config GENERIC_HARDIRQS
bool
default y
Index: linux-2.6/arch/sparc64/lib/dec_and_lock.S
===================================================================
--- linux-2.6.orig/arch/sparc64/lib/dec_and_lock.S
+++ /dev/null
@@ -1,80 +0,0 @@
-/* $Id: dec_and_lock.S,v 1.5 2001/11/18 00:12:56 davem Exp $
- * dec_and_lock.S: Sparc64 version of "atomic_dec_and_lock()"
- * using cas and ldstub instructions.
- *
- * Copyright (C) 2000 David S. Miller ([email protected])
- */
-#include <linux/config.h>
-#include <asm/thread_info.h>
-
- .text
- .align 64
-
- /* CAS basically works like this:
- *
- * void CAS(MEM, REG1, REG2)
- * {
- * START_ATOMIC();
- * if (*(MEM) == REG1) {
- * TMP = *(MEM);
- * *(MEM) = REG2;
- * REG2 = TMP;
- * } else
- * REG2 = *(MEM);
- * END_ATOMIC();
- * }
- */
-
- .globl _atomic_dec_and_lock
-_atomic_dec_and_lock: /* %o0 = counter, %o1 = lock */
-loop1: lduw [%o0], %g2
- subcc %g2, 1, %g7
- be,pn %icc, start_to_zero
- nop
-nzero: cas [%o0], %g2, %g7
- cmp %g2, %g7
- bne,pn %icc, loop1
- mov 0, %g1
-
-out:
- membar #StoreLoad | #StoreStore
- retl
- mov %g1, %o0
-start_to_zero:
-#ifdef CONFIG_PREEMPT
- ldsw [%g6 + TI_PRE_COUNT], %g3
- add %g3, 1, %g3
- stw %g3, [%g6 + TI_PRE_COUNT]
-#endif
-to_zero:
- ldstub [%o1], %g3
- membar #StoreLoad | #StoreStore
- brnz,pn %g3, spin_on_lock
- nop
-loop2: cas [%o0], %g2, %g7 /* ASSERT(g7 == 0) */
- cmp %g2, %g7
-
- be,pt %icc, out
- mov 1, %g1
- lduw [%o0], %g2
- subcc %g2, 1, %g7
- be,pn %icc, loop2
- nop
- membar #StoreStore | #LoadStore
- stb %g0, [%o1]
-#ifdef CONFIG_PREEMPT
- ldsw [%g6 + TI_PRE_COUNT], %g3
- sub %g3, 1, %g3
- stw %g3, [%g6 + TI_PRE_COUNT]
-#endif
-
- b,pt %xcc, nzero
- nop
-spin_on_lock:
- ldub [%o1], %g3
- membar #LoadLoad
- brnz,pt %g3, spin_on_lock
- nop
- ba,pt %xcc, to_zero
- nop
- nop
Index: linux-2.6/arch/alpha/kernel/alpha_ksyms.c
===================================================================
--- linux-2.6.orig/arch/alpha/kernel/alpha_ksyms.c
+++ linux-2.6/arch/alpha/kernel/alpha_ksyms.c
@@ -184,7 +184,6 @@ EXPORT_SYMBOL(cpu_data);
EXPORT_SYMBOL(smp_num_cpus);
EXPORT_SYMBOL(smp_call_function);
EXPORT_SYMBOL(smp_call_function_on_cpu);
-EXPORT_SYMBOL(_atomic_dec_and_lock);
EXPORT_SYMBOL(cpu_present_mask);
#endif /* CONFIG_SMP */
Index: linux-2.6/arch/sparc64/kernel/sparc64_ksyms.c
===================================================================
--- linux-2.6.orig/arch/sparc64/kernel/sparc64_ksyms.c
+++ linux-2.6/arch/sparc64/kernel/sparc64_ksyms.c
@@ -163,9 +163,6 @@ EXPORT_SYMBOL(atomic64_add);
EXPORT_SYMBOL(atomic64_add_ret);
EXPORT_SYMBOL(atomic64_sub);
EXPORT_SYMBOL(atomic64_sub_ret);
-#ifdef CONFIG_SMP
-EXPORT_SYMBOL(_atomic_dec_and_lock);
-#endif
/* Atomic bit operations. */
EXPORT_SYMBOL(test_and_set_bit);
Index: linux-2.6/arch/x86_64/kernel/x8664_ksyms.c
===================================================================
--- linux-2.6.orig/arch/x86_64/kernel/x8664_ksyms.c
+++ linux-2.6/arch/x86_64/kernel/x8664_ksyms.c
@@ -178,10 +178,6 @@ EXPORT_SYMBOL(rwsem_down_write_failed_th
EXPORT_SYMBOL(empty_zero_page);
-#ifdef CONFIG_HAVE_DEC_LOCK
-EXPORT_SYMBOL(_atomic_dec_and_lock);
-#endif
-
EXPORT_SYMBOL(die_chain);
EXPORT_SYMBOL(register_die_notifier);
Index: linux-2.6/arch/alpha/lib/Makefile
===================================================================
--- linux-2.6.orig/arch/alpha/lib/Makefile
+++ linux-2.6/arch/alpha/lib/Makefile
@@ -40,8 +40,6 @@ lib-y = __divqu.o __remqu.o __divlu.o __
fpreg.o \
callback_srm.o srm_puts.o srm_printk.o
-lib-$(CONFIG_SMP) += dec_and_lock.o
-
# The division routines are built from single source, with different defines.
AFLAGS___divqu.o = -DDIV
AFLAGS___remqu.o = -DREM
Index: linux-2.6/arch/mips/lib/Makefile
===================================================================
--- linux-2.6.orig/arch/mips/lib/Makefile
+++ linux-2.6/arch/mips/lib/Makefile
@@ -2,8 +2,7 @@
# Makefile for MIPS-specific library files..
#
-lib-y += csum_partial_copy.o dec_and_lock.o memcpy.o promlib.o \
- strlen_user.o strncpy_user.o strnlen_user.o
+lib-y += csum_partial_copy.o memcpy.o promlib.o strlen_user.o strncpy_user.o strnlen_user.o
obj-y += iomap.o
Index: linux-2.6/arch/ppc/lib/Makefile
===================================================================
--- linux-2.6.orig/arch/ppc/lib/Makefile
+++ linux-2.6/arch/ppc/lib/Makefile
@@ -2,7 +2,7 @@
# Makefile for ppc-specific library files..
#
-obj-y := checksum.o string.o strcase.o dec_and_lock.o div64.o
+obj-y := checksum.o string.o strcase.o div64.o
obj-$(CONFIG_8xx) += rheap.o
obj-$(CONFIG_CPM2) += rheap.o
Index: linux-2.6/arch/ppc64/lib/Makefile
===================================================================
--- linux-2.6.orig/arch/ppc64/lib/Makefile
+++ linux-2.6/arch/ppc64/lib/Makefile
@@ -2,7 +2,7 @@
# Makefile for ppc64-specific library files..
#
-lib-y := checksum.o dec_and_lock.o string.o strcase.o
+lib-y := checksum.o string.o strcase.o
lib-y += copypage.o memcpy.o copyuser.o usercopy.o
# Lock primitives are defined as no-ops in include/linux/spinlock.h
Index: linux-2.6/arch/sparc64/Kconfig.debug
===================================================================
--- linux-2.6.orig/arch/sparc64/Kconfig.debug
+++ linux-2.6/arch/sparc64/Kconfig.debug
@@ -33,14 +33,6 @@ config DEBUG_BOOTMEM
depends on DEBUG_KERNEL
bool "Debug BOOTMEM initialization"
-# We have a custom atomic_dec_and_lock() implementation but it's not
-# compatible with spinlock debugging so we need to fall back on
-# the generic version in that case.
-config HAVE_DEC_LOCK
- bool
- depends on SMP && !DEBUG_SPINLOCK
- default y
-
config MCOUNT
bool
depends on STACK_DEBUG
Index: linux-2.6/kernel/rcupdate.c
===================================================================
--- linux-2.6.orig/kernel/rcupdate.c
+++ linux-2.6/kernel/rcupdate.c
@@ -45,7 +45,6 @@
#include <linux/percpu.h>
#include <linux/notifier.h>
#include <linux/rcupdate.h>
-#include <linux/rcuref.h>
#include <linux/cpu.h>
/* Definition for rcupdate control block. */
@@ -73,19 +72,6 @@ DEFINE_PER_CPU(struct rcu_data, rcu_bh_d
static DEFINE_PER_CPU(struct tasklet_struct, rcu_tasklet) = {NULL};
static int maxbatch = 10;
-#ifndef __HAVE_ARCH_CMPXCHG
-/*
- * We use an array of spinlocks for the rcurefs -- similar to ones in sparc
- * 32 bit atomic_t implementations, and a hash function similar to that
- * for our refcounting needs.
- * Can't help multiprocessors which donot have cmpxchg :(
- */
-
-spinlock_t __rcuref_hash[RCUREF_HASH_SIZE] = {
- [0 ... (RCUREF_HASH_SIZE-1)] = SPIN_LOCK_UNLOCKED
-};
-#endif
-
/**
* call_rcu - Queue an RCU callback for invocation after a grace period.
* @head: structure to be used for queueing the RCU updates.
Index: linux-2.6/include/asm-ppc64/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-ppc64/atomic.h
+++ linux-2.6/include/asm-ppc64/atomic.h
@@ -164,6 +164,23 @@ static __inline__ int atomic_dec_return(
#define atomic_cmpxchg(v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n)))
+/**
+ * atomic_inc_not_zero - increment if not zero
+ * @v: pointer of type atomic_t
+ *
+ * Atomically increments @v by 1, so long as it was not 0.
+ * Returns non-zero on successful increment and zero otherwise.
+ */
+#define atomic_inc_not_zero(v) \
+({ \
+ int c, old; \
+ c = atomic_read(v); \
+ while (c && (old = atomic_cmpxchg((v), c, c + 1)) != c) \
+ c = old; \
+ c; \
+})
+
+
#define atomic_sub_and_test(a, v) (atomic_sub_return((a), (v)) == 0)
#define atomic_dec_and_test(v) (atomic_dec_return((v)) == 0)
Index: linux-2.6/include/asm-alpha/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-alpha/atomic.h
+++ linux-2.6/include/asm-alpha/atomic.h
@@ -173,6 +173,15 @@ static __inline__ long atomic64_sub_retu
#define atomic_cmpxchg(v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n)))
+#define atomic_inc_not_zero(v) \
+({ \
+ int c, old; \
+ c = atomic_read(v); \
+ while (c && (old = atomic_cmpxchg((v), c, c + 1)) != c) \
+ c = old; \
+ c; \
+})
+
#define atomic_dec_return(v) atomic_sub_return(1,(v))
#define atomic64_dec_return(v) atomic64_sub_return(1,(v))
Index: linux-2.6/include/asm-arm26/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-arm26/atomic.h
+++ linux-2.6/include/asm-arm26/atomic.h
@@ -76,6 +76,20 @@ static inline int atomic_cmpxchg(atomic_
return ret;
}
+static inline int atomic_inc_not_zero(atomic_t *v)
+{
+ int ret;
+ unsigned long flags;
+
+ local_irq_save(flags);
+ ret = v->counter;
+ if (ret > 0)
+ v->counter++;
+ local_irq_restore(flags);
+
+ return ret;
+}
+
static inline void atomic_clear_mask(unsigned long mask, unsigned long *addr)
{
unsigned long flags;
Index: linux-2.6/include/asm-frv/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-frv/atomic.h
+++ linux-2.6/include/asm-frv/atomic.h
@@ -416,4 +416,13 @@ extern uint32_t __cmpxchg_32(uint32_t *v
#define atomic_cmpxchg(v, old, new) ((int)cmpxchg(&((v)->counter), old, new))
+#define atomic_inc_not_zero(v) \
+({ \
+ int c, old; \
+ c = atomic_read(v); \
+ while (c && (old = atomic_cmpxchg((v), c, c + 1)) != c) \
+ c = old; \
+ c; \
+})
+
#endif /* _ASM_ATOMIC_H */
Index: linux-2.6/include/asm-h8300/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-h8300/atomic.h
+++ linux-2.6/include/asm-h8300/atomic.h
@@ -94,6 +94,18 @@ static __inline__ int atomic_cmpxchg(ato
return ret;
}
+static __inline__ int atomic_inc_not_zero(atomic_t *v)
+{
+ int ret;
+ unsigned long flags;
+ local_irq_save(flags);
+ ret = v->counter;
+ if (ret > 0)
+ v->counter++;
+ local_irq_restore(flags);
+ return ret;
+}
+
static __inline__ void atomic_clear_mask(unsigned long mask, unsigned long *v)
{
__asm__ __volatile__("stc ccr,r1l\n\t"
Index: linux-2.6/include/asm-i386/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-i386/atomic.h
+++ linux-2.6/include/asm-i386/atomic.h
@@ -217,6 +217,22 @@ static __inline__ int atomic_sub_return(
#define atomic_cmpxchg(v, old, new) ((int)cmpxchg(&((v)->counter), old, new))
+/**
+ * atomic_inc_not_zero - increment if not zero
+ * @v: pointer of type atomic_t
+ *
+ * Atomically increments @v by 1, so long as it was not 0.
+ * Returns non-zero on successful increment and zero otherwise.
+ */
+#define atomic_inc_not_zero(v) \
+({ \
+ int c, old; \
+ c = atomic_read(v); \
+ while (c && (old = atomic_cmpxchg((v), c, c + 1)) != c) \
+ c = old; \
+ c; \
+})
+
#define atomic_inc_return(v) (atomic_add_return(1,v))
#define atomic_dec_return(v) (atomic_sub_return(1,v))
Index: linux-2.6/include/asm-ia64/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-ia64/atomic.h
+++ linux-2.6/include/asm-ia64/atomic.h
@@ -90,6 +90,15 @@ ia64_atomic64_sub (__s64 i, atomic64_t *
#define atomic_cmpxchg(v, old, new) ((int)cmpxchg(&((v)->counter), old, new))
+#define atomic_inc_not_zero(v) \
+({ \
+ int c, old; \
+ c = atomic_read(v); \
+ while (c && (old = atomic_cmpxchg((v), c, c + 1)) != c) \
+ c = old; \
+ c; \
+})
+
#define atomic_add_return(i,v) \
({ \
int __ia64_aar_i = (i); \
Index: linux-2.6/include/asm-m68k/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-m68k/atomic.h
+++ linux-2.6/include/asm-m68k/atomic.h
@@ -141,6 +141,15 @@ static inline void atomic_set_mask(unsig
#define atomic_cmpxchg(v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n)))
+#define atomic_inc_not_zero(v) \
+({ \
+ int c, old; \
+ c = atomic_read(v); \
+ while (c && (old = atomic_cmpxchg((v), c, c + 1)) != c) \
+ c = old; \
+ c; \
+})
+
/* Atomic operations are already serializing */
#define smp_mb__before_atomic_dec() barrier()
#define smp_mb__after_atomic_dec() barrier()
Index: linux-2.6/include/asm-m68knommu/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-m68knommu/atomic.h
+++ linux-2.6/include/asm-m68knommu/atomic.h
@@ -130,6 +130,15 @@ extern __inline__ int atomic_sub_return(
#define atomic_cmpxchg(v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n)))
+#define atomic_inc_not_zero(v) \
+({ \
+ int c, old; \
+ c = atomic_read(v); \
+ while (c && (old = atomic_cmpxchg((v), c, c + 1)) != c) \
+ c = old; \
+ c; \
+})
+
#define atomic_dec_return(v) atomic_sub_return(1,(v))
#define atomic_inc_return(v) atomic_add_return(1,(v))
Index: linux-2.6/include/asm-mips/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-mips/atomic.h
+++ linux-2.6/include/asm-mips/atomic.h
@@ -269,6 +269,22 @@ static __inline__ int atomic_sub_if_posi
#define atomic_cmpxchg(v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n)))
+/**
+ * atomic_inc_not_zero - increment if not zero
+ * @v: pointer of type atomic_t
+ *
+ * Atomically increments @v by 1, so long as it was not 0.
+ * Returns non-zero on successful increment and zero otherwise.
+ */
+#define atomic_inc_not_zero(v) \
+({ \
+ int c, old; \
+ c = atomic_read(v); \
+ while (c && (old = atomic_cmpxchg((v), c, c + 1)) != c) \
+ c = old; \
+ c; \
+})
+
#define atomic_dec_return(v) atomic_sub_return(1,(v))
#define atomic_inc_return(v) atomic_add_return(1,(v))
Index: linux-2.6/include/asm-parisc/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-parisc/atomic.h
+++ linux-2.6/include/asm-parisc/atomic.h
@@ -166,6 +166,22 @@ static __inline__ int atomic_read(const
/* exported interface */
#define atomic_cmpxchg(v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n)))
+/**
+ * atomic_inc_not_zero - increment if not zero
+ * @v: pointer of type atomic_t
+ *
+ * Atomically increments @v by 1, so long as it was not 0.
+ * Returns non-zero on successful increment and zero otherwise.
+ */
+#define atomic_inc_not_zero(v) \
+({ \
+ int c, old; \
+ c = atomic_read(v); \
+ while (c && (old = atomic_cmpxchg((v), c, c + 1)) != c) \
+ c = old; \
+ c; \
+})
+
#define atomic_add(i,v) ((void)(__atomic_add_return( ((int)i),(v))))
#define atomic_sub(i,v) ((void)(__atomic_add_return(-((int)i),(v))))
#define atomic_inc(v) ((void)(__atomic_add_return( 1,(v))))
Index: linux-2.6/include/asm-ppc/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-ppc/atomic.h
+++ linux-2.6/include/asm-ppc/atomic.h
@@ -179,6 +179,22 @@ static __inline__ int atomic_dec_return(
#define atomic_cmpxchg(v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n)))
+/**
+ * atomic_inc_not_zero - increment if not zero
+ * @v: pointer of type atomic_t
+ *
+ * Atomically increments @v by 1, so long as it was not 0.
+ * Returns non-zero on successful increment and zero otherwise.
+ */
+#define atomic_inc_not_zero(v) \
+({ \
+ int c, old; \
+ c = atomic_read(v); \
+ while (c && (old = atomic_cmpxchg((v), c, c + 1)) != c) \
+ c = old; \
+ c; \
+})
+
#define atomic_sub_and_test(a, v) (atomic_sub_return((a), (v)) == 0)
#define atomic_dec_and_test(v) (atomic_dec_return((v)) == 0)
Index: linux-2.6/include/asm-s390/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-s390/atomic.h
+++ linux-2.6/include/asm-s390/atomic.h
@@ -200,6 +200,15 @@ atomic_compare_and_swap(int expected_old
#define atomic_cmpxchg(v, o, n) (atomic_compare_and_swap((o), (n), &((v)->counter)))
+#define atomic_inc_not_zero(v) \
+({ \
+ int c, old; \
+ c = atomic_read(v); \
+ while (c && (old = atomic_cmpxchg((v), c, c + 1)) != c) \
+ c = old; \
+ c; \
+})
+
#define smp_mb__before_atomic_dec() smp_mb()
#define smp_mb__after_atomic_dec() smp_mb()
#define smp_mb__before_atomic_inc() smp_mb()
Index: linux-2.6/include/asm-sh/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-sh/atomic.h
+++ linux-2.6/include/asm-sh/atomic.h
@@ -101,6 +101,20 @@ static __inline__ int atomic_cmpxchg(ato
return ret;
}
+static __inline__ int atomic_inc_not_zero(atomic_t *v)
+{
+ int ret;
+ unsigned long flags;
+
+ local_irq_save(flags);
+ ret = v->counter;
+ if (ret > 0)
+ v->counter++;
+ local_irq_restore(flags);
+
+ return ret;
+}
+
static __inline__ void atomic_clear_mask(unsigned int mask, atomic_t *v)
{
unsigned long flags;
Index: linux-2.6/include/asm-sh64/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-sh64/atomic.h
+++ linux-2.6/include/asm-sh64/atomic.h
@@ -113,6 +113,20 @@ static __inline__ int atomic_cmpxchg(ato
return ret;
}
+static __inline__ int atomic_inc_not_zero(atomic_t *v)
+{
+ int ret;
+ unsigned long flags;
+
+ local_irq_save(flags);
+ ret = v->counter;
+ if (ret > 0)
+ v->counter++;
+ local_irq_restore(flags);
+
+ return ret;
+}
+
static __inline__ void atomic_clear_mask(unsigned int mask, atomic_t *v)
{
unsigned long flags;
Index: linux-2.6/include/asm-sparc/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-sparc/atomic.h
+++ linux-2.6/include/asm-sparc/atomic.h
@@ -20,6 +20,7 @@ typedef struct { volatile int counter; }
extern int __atomic_add_return(int, atomic_t *);
extern int atomic_cmpxchg(atomic_t *, int, int);
+extern int atomic_inc_not_zero(atomic_t *);
extern void atomic_set(atomic_t *, int);
#define atomic_read(v) ((v)->counter)
Index: linux-2.6/include/asm-sparc64/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-sparc64/atomic.h
+++ linux-2.6/include/asm-sparc64/atomic.h
@@ -72,6 +72,15 @@ extern int atomic64_sub_ret(int, atomic6
#define atomic_cmpxchg(v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n)))
+#define atomic_inc_not_zero(v) \
+({ \
+ int c, old; \
+ c = atomic_read(v); \
+ while (c && (old = atomic_cmpxchg((v), c, c + 1)) != c) \
+ c = old; \
+ c; \
+})
+
/* Atomic operations are already serializing */
#ifdef CONFIG_SMP
#define smp_mb__before_atomic_dec() membar_storeload_loadload();
Index: linux-2.6/include/asm-v850/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-v850/atomic.h
+++ linux-2.6/include/asm-v850/atomic.h
@@ -104,6 +104,20 @@ static __inline__ int atomic_cmpxchg(ato
return ret;
}
+static __inline__ int atomic_inc_not_zero(atomic_t *v)
+{
+ int ret;
+ unsigned long flags;
+
+ local_irq_save(flags);
+ ret = v->counter;
+ if (ret > 0)
+ v->counter++;
+ local_irq_restore(flags);
+
+ return ret;
+}
+
/* Atomic operations are already serializing on ARM */
#define smp_mb__before_atomic_dec() barrier()
#define smp_mb__after_atomic_dec() barrier()
Index: linux-2.6/include/asm-x86_64/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-x86_64/atomic.h
+++ linux-2.6/include/asm-x86_64/atomic.h
@@ -362,6 +362,22 @@ static __inline__ int atomic_sub_return(
#define atomic_cmpxchg(v, old, new) ((int)cmpxchg(&((v)->counter), old, new))
+/**
+ * atomic_inc_not_zero - increment if not zero
+ * @v: pointer of type atomic_t
+ *
+ * Atomically increments @v by 1, so long as it was not 0.
+ * Returns non-zero on successful increment and zero otherwise.
+ */
+#define atomic_inc_not_zero(v) \
+({ \
+ int c, old; \
+ c = atomic_read(v); \
+ while (c && (old = atomic_cmpxchg((v), c, c + 1)) != c) \
+ c = old; \
+ c; \
+})
+
#define atomic_inc_return(v) (atomic_add_return(1,v))
#define atomic_dec_return(v) (atomic_sub_return(1,v))
Index: linux-2.6/include/asm-xtensa/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-xtensa/atomic.h
+++ linux-2.6/include/asm-xtensa/atomic.h
@@ -225,6 +225,22 @@ static inline int atomic_sub_return(int
#define atomic_cmpxchg(v, o, n) ((int)cmpxchg(&((v)->counter), (o), (n)))
+/**
+ * atomic_inc_not_zero - increment if not zero
+ * @v: pointer of type atomic_t
+ *
+ * Atomically increments @v by 1, so long as it was not 0.
+ * Returns non-zero on successful increment and zero otherwise.
+ */
+#define atomic_inc_not_zero(v) \
+({ \
+ int c, old; \
+ c = atomic_read(v); \
+ while (c && (old = atomic_cmpxchg((v), c, c + 1)) != c) \
+ c = old; \
+ c; \
+})
+
static inline void atomic_clear_mask(unsigned int mask, atomic_t *v)
{
unsigned int all_f = -1;
Index: linux-2.6/include/asm-cris/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-cris/atomic.h
+++ linux-2.6/include/asm-cris/atomic.h
@@ -135,6 +135,18 @@ static __inline__ int atomic_cmpxchg(ato
return ret;
}
+static __inline__ int atomic_inc_not_zero(atomic_t *v)
+{
+ int ret;
+ unsigned long flags;
+ cris_atomic_save(v, flags);
+ ret = v->counter;
+ if (ret > 0)
+ v->counter++;
+ cris_atomic_restore(v, flags);
+ return ret;
+}
+
static inline void atomic_clear_mask(unsigned long mask, unsigned long *addr)
/* Atomic operations are already serializing */
#define smp_mb__before_atomic_dec() barrier()
Index: linux-2.6/arch/sparc/lib/atomic32.c
===================================================================
--- linux-2.6.orig/arch/sparc/lib/atomic32.c
+++ linux-2.6/arch/sparc/lib/atomic32.c
@@ -52,6 +52,20 @@ int atomic_cmpxchg(atomic_t *v, int old,
return ret;
}
+int atomic_inc_not_zero(atomic_t *v)
+{
+ int ret;
+ unsigned long flags;
+ spin_lock_irqsave(ATOMIC_HASH(v), flags);
+ ret = v->counter;
+ if (ret > 0)
+ v->counter++;
+ spin_unlock_irqrestore(ATOMIC_HASH(v), flags);
+ return ret;
+}
+
+static inline void atomic_clear_mask(unsigned long mask, unsigned long *addr)
+/* Atomic operations are already serializing */
void atomic_set(atomic_t *v, int i)
{
unsigned long flags;
Index: linux-2.6/Documentation/atomic_ops.txt
===================================================================
--- linux-2.6.orig/Documentation/atomic_ops.txt
+++ linux-2.6/Documentation/atomic_ops.txt
@@ -115,7 +115,7 @@ boolean is return which indicates whethe
is negative. It requires explicit memory barrier semantics around the
operation.
-Finally:
+Then:
int atomic_cmpxchg(atomic_t *v, int old, int new);
@@ -129,6 +129,16 @@ atomic_cmpxchg requires explicit memory
The semantics for atomic_cmpxchg are the same as those defined for 'cas'
below.
+Finally:
+
+ int atomic_inc_not_zero(atomic_t *v);
+
+If the atomic value v is not zero, this function increments v and returns
+non zero. If v is zero then it returns zero. This is done as an atomic
+operation.
+
+atomic_inc_not_zero requires explicit memory barriers around the operation.
+
If a caller requires memory barrier semantics around an atomic_t
operation which does not return a value, a set of interfaces are
Index: linux-2.6/include/asm-alpha/system.h
===================================================================
--- linux-2.6.orig/include/asm-alpha/system.h
+++ linux-2.6/include/asm-alpha/system.h
@@ -477,8 +477,6 @@ extern void __xchg_called_with_bad_point
* we don't need any memory barrier as far I can tell.
*/
-#define __HAVE_ARCH_CMPXCHG 1
-
static inline unsigned long
__cmpxchg_u8(volatile char *m, long old, long new)
{
Index: linux-2.6/include/asm-i386/system.h
===================================================================
--- linux-2.6.orig/include/asm-i386/system.h
+++ linux-2.6/include/asm-i386/system.h
@@ -257,10 +257,6 @@ static inline unsigned long __xchg(unsig
* indicated by comparing RETURN with OLD.
*/
-#ifdef CONFIG_X86_CMPXCHG
-#define __HAVE_ARCH_CMPXCHG 1
-#endif
-
static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
unsigned long new, int size)
{
Index: linux-2.6/include/asm-m68k/system.h
===================================================================
--- linux-2.6.orig/include/asm-m68k/system.h
+++ linux-2.6/include/asm-m68k/system.h
@@ -164,7 +164,6 @@ static inline unsigned long __xchg(unsig
* indicated by comparing RETURN with OLD.
*/
#ifdef CONFIG_RMW_INSNS
-#define __HAVE_ARCH_CMPXCHG 1
static inline unsigned long __cmpxchg(volatile void *p, unsigned long old,
unsigned long new, int size)
Index: linux-2.6/include/asm-m68knommu/system.h
===================================================================
--- linux-2.6.orig/include/asm-m68knommu/system.h
+++ linux-2.6/include/asm-m68knommu/system.h
@@ -195,7 +195,6 @@ static inline unsigned long __xchg(unsig
* store NEW in MEM. Return the initial value in MEM. Success is
* indicated by comparing RETURN with OLD.
*/
-#define __HAVE_ARCH_CMPXCHG 1
static __inline__ unsigned long
cmpxchg(volatile int *p, int old, int new)
Index: linux-2.6/include/asm-mips/system.h
===================================================================
--- linux-2.6.orig/include/asm-mips/system.h
+++ linux-2.6/include/asm-mips/system.h
@@ -277,8 +277,6 @@ static inline unsigned long __xchg(unsig
#define xchg(ptr,x) ((__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr))))
#define tas(ptr) (xchg((ptr),1))
-#define __HAVE_ARCH_CMPXCHG 1
-
static inline unsigned long __cmpxchg_u32(volatile int * m, unsigned long old,
unsigned long new)
{
Index: linux-2.6/include/asm-ppc/system.h
===================================================================
--- linux-2.6.orig/include/asm-ppc/system.h
+++ linux-2.6/include/asm-ppc/system.h
@@ -155,8 +155,6 @@ extern inline void * xchg_ptr(void * m,
}
-#define __HAVE_ARCH_CMPXCHG 1
-
static __inline__ unsigned long
__cmpxchg_u32(volatile unsigned int *p, unsigned int old, unsigned int new)
{
Index: linux-2.6/include/asm-ppc64/system.h
===================================================================
--- linux-2.6.orig/include/asm-ppc64/system.h
+++ linux-2.6/include/asm-ppc64/system.h
@@ -223,8 +223,6 @@ __xchg(volatile void *ptr, unsigned long
#define tas(ptr) (xchg((ptr),1))
-#define __HAVE_ARCH_CMPXCHG 1
-
static __inline__ unsigned long
__cmpxchg_u32(volatile unsigned int *p, unsigned long old, unsigned long new)
{
Index: linux-2.6/include/asm-s390/system.h
===================================================================
--- linux-2.6.orig/include/asm-s390/system.h
+++ linux-2.6/include/asm-s390/system.h
@@ -189,8 +189,6 @@ static inline unsigned long __xchg(unsig
* indicated by comparing RETURN with OLD.
*/
-#define __HAVE_ARCH_CMPXCHG 1
-
#define cmpxchg(ptr,o,n)\
((__typeof__(*(ptr)))__cmpxchg((ptr),(unsigned long)(o),\
(unsigned long)(n),sizeof(*(ptr))))
Index: linux-2.6/include/asm-sparc64/system.h
===================================================================
--- linux-2.6.orig/include/asm-sparc64/system.h
+++ linux-2.6/include/asm-sparc64/system.h
@@ -321,8 +321,6 @@ extern void die_if_kernel(char *str, str
* indicated by comparing RETURN with OLD.
*/
-#define __HAVE_ARCH_CMPXCHG 1
-
static __inline__ unsigned long
__cmpxchg_u32(volatile int *m, int old, int new)
{
Index: linux-2.6/include/asm-x86_64/system.h
===================================================================
--- linux-2.6.orig/include/asm-x86_64/system.h
+++ linux-2.6/include/asm-x86_64/system.h
@@ -237,8 +237,6 @@ static inline unsigned long __xchg(unsig
* indicated by comparing RETURN with OLD.
*/
-#define __HAVE_ARCH_CMPXCHG 1
-
static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
unsigned long new, int size)
{
Index: linux-2.6/arch/i386/kernel/acpi/boot.c
===================================================================
--- linux-2.6.orig/arch/i386/kernel/acpi/boot.c
+++ linux-2.6/arch/i386/kernel/acpi/boot.c
@@ -83,7 +83,7 @@ int acpi_skip_timer_override __initdata;
static u64 acpi_lapic_addr __initdata = APIC_DEFAULT_PHYS_BASE;
#endif
-#ifndef __HAVE_ARCH_CMPXCHG
+#ifndef CONFIG_X86_CMPXCHG
#warning ACPI uses CMPXCHG, i486 and later hardware
#endif
Index: linux-2.6/include/asm-i386/mc146818rtc.h
===================================================================
--- linux-2.6.orig/include/asm-i386/mc146818rtc.h
+++ linux-2.6/include/asm-i386/mc146818rtc.h
@@ -13,7 +13,7 @@
#define RTC_ALWAYS_BCD 1 /* RTC operates in binary mode */
#endif
-#ifdef __HAVE_ARCH_CMPXCHG
+#ifdef CONFIG_X86_CMPXCHG
/*
* This lock provides nmi access to the CMOS/RTC registers. It has some
* special properties. It is owned by a CPU and stores the index register
Index: linux-2.6/include/asm-ia64/intrinsics.h
===================================================================
--- linux-2.6.orig/include/asm-ia64/intrinsics.h
+++ linux-2.6/include/asm-ia64/intrinsics.h
@@ -111,8 +111,6 @@ extern void ia64_xchg_called_with_bad_po
* indicated by comparing RETURN with OLD.
*/
-#define __HAVE_ARCH_CMPXCHG 1
-
/*
* This function doesn't exist, so you'll get a linker error
* if something tries to do an invalid cmpxchg().
Index: linux-2.6/include/asm-parisc/atomic.h
===================================================================
--- linux-2.6.orig/include/asm-parisc/atomic.h
+++ linux-2.6/include/asm-parisc/atomic.h
@@ -98,8 +98,6 @@ static __inline__ unsigned long __xchg(u
((__typeof__(*(ptr)))__xchg((unsigned long)(x),(ptr),sizeof(*(ptr))))
-#define __HAVE_ARCH_CMPXCHG 1
-
/* bug catcher for when unsupported size is used - won't link */
extern void __cmpxchg_called_with_bad_pointer(void);
Index: linux-2.6/include/asm-um/system-i386.h
===================================================================
--- linux-2.6.orig/include/asm-um/system-i386.h
+++ linux-2.6/include/asm-um/system-i386.h
@@ -3,6 +3,4 @@
#include "asm/system-generic.h"
-#define __HAVE_ARCH_CMPXCHG 1
-
#endif
On Thu, Sep 15, 2005 at 12:48:05AM +1000, Nick Piggin wrote:
> This patch still needs work on arm (v6) and m32r. I would
> just be shooting in the dark if I attempted either myself.
ARMv6, something like:
int atomic_cmpxchg(atomic_t *ptr, int old, int new)
{
u32 oldval, res;
do {
asm(
"ldrex %1, [%2]\n\t"
"teq %1, %3\n\t"
"strexeq %0, %4, [%2]\n\t"
: "=&r" (res), "=&r" (oldval)
: "r" (&ptr->counter), "r" (old), "r" (new)
: "cc");
} while (res);
return oldval;
}
should do what you require.
--
Russell King
Linux kernel 2.6 ARM Linux - http://www.arm.linux.org.uk/
maintainer of: 2.6 Serial core
Nick Piggin wrote:
> Index: linux-2.6/kernel/rcupdate.c
> ===================================================================
> --- linux-2.6.orig/kernel/rcupdate.c
> +++ linux-2.6/kernel/rcupdate.c
Dang, this leaked in from 3/5, and 1/5 also has some funny
whitespace I introduced.
They'll all still apply just fine, and the end result will
look the same, so they're fine for testing and comments.
I'll submit cleaned up and signed off versions if/when we're
ready (speaking of which, David I should get you to sign
this as well if you agree with it)
--
SUSE Labs, Novell Inc.
Send instant messages to your online friends http://au.messenger.yahoo.com
Hi,
On Thu, 15 Sep 2005, Nick Piggin wrote:
> Also needs work on those same architectures. Other architectures
> might want to look at providing a more optimal implementation.
IMO a rather pointless primitive, unless there is a cpu architecture which
has a inc_not_zero instruction, otherwise it will always be the same as
using cmpxchg.
bye, Roman
Russell King wrote:
> On Thu, Sep 15, 2005 at 12:48:05AM +1000, Nick Piggin wrote:
>
>>This patch still needs work on arm (v6) and m32r. I would
>>just be shooting in the dark if I attempted either myself.
>
>
> ARMv6, something like:
>
Thanks very much, I've updated the patchset.
--
SUSE Labs, Novell Inc.
Send instant messages to your online friends http://au.messenger.yahoo.com
From: Nick Piggin <[email protected]>
Date: Thu, 15 Sep 2005 00:55:45 +1000
> I noticed David posted a patch to do something similar the
> other day. With a generic atomic_cmpxchg available, such a
> patch is no longer objectionable to those architectures
> that don't #define __HAVE_ARCH_CMPXCHG
>
> Do we want this David? Any other architecture have a super
> optimised atomic_dec_and_lock that surpasses this
> implementation?
I talked to Linus about my patch and he made the good point that the
Alpha optimization is very valid (it avoids loading the GP and other
stuff by doing that fancy assembler entry point). So we decided that
we'd make the cmpxchg() generic version, but make moving over to that
gradual instead of mandatorily forcing everyone to use the new thing.
So, for example, we'd convert sparc64 (because I want to) and the
architectures using the exact same ppc64 C code, but leave the others
alone. It'd be up to the remaining platform's maintainers to move
over if they wanted to.
I'll submit that updated patch to him later today.
Hi,
On Thu, 15 Sep 2005, Nick Piggin wrote:
> Is there any point in keeping this around?
Yes, for drivers which want to use it to synchronize with userspace.
Alternatively it could be changed into a Kconfig definition.
bye, Roman
Roman Zippel wrote:
> Hi,
>
> On Thu, 15 Sep 2005, Nick Piggin wrote:
>
>
>>Also needs work on those same architectures. Other architectures
>>might want to look at providing a more optimal implementation.
>
>
> IMO a rather pointless primitive, unless there is a cpu architecture which
> has a inc_not_zero instruction, otherwise it will always be the same as
> using cmpxchg.
>
It will always be *implemented* with cmpxchg you mean, which is a
bit different. But even then, no, you definitely don't need an
inc_not_zero instruction to make this primitive faster than the
cmpxchg version. Just look at all the !SMP architectures that just
turn off interrupts while doing the op. Look at the architectures
that use hashed spinlocks.
Or here is possible pseudo code for an architecture with ll/sc
instructions:
do {
tmp = load_locked(v);
if (!tmp)
break;
tmp++;
} while (!store_cond(v, tmp));
return tmp;
As opposed to using the cmpxchg version, which would have more
loads and conditional branches, AFAIKS.
--
SUSE Labs, Novell Inc.
Send instant messages to your online friends http://au.messenger.yahoo.com
David S. Miller wrote:
> I talked to Linus about my patch and he made the good point that the
> Alpha optimization is very valid (it avoids loading the GP and other
> stuff by doing that fancy assembler entry point). So we decided that
> we'd make the cmpxchg() generic version, but make moving over to that
> gradual instead of mandatorily forcing everyone to use the new thing.
>
> So, for example, we'd convert sparc64 (because I want to) and the
> architectures using the exact same ppc64 C code, but leave the others
> alone. It'd be up to the remaining platform's maintainers to move
> over if they wanted to.
>
> I'll submit that updated patch to him later today.
>
OK that's fair enough. I'll submit whatever cleanups are possible
on top of your patch after the atomic_cmpxchg patch goes in (if
it goes in). Thanks.
--
SUSE Labs, Novell Inc.
Send instant messages to your online friends http://au.messenger.yahoo.com
Hi,
On Thu, 15 Sep 2005, Nick Piggin wrote:
> Or here is possible pseudo code for an architecture with ll/sc
> instructions:
>
> do {
> tmp = load_locked(v);
> if (!tmp)
> break;
> tmp++;
> } while (!store_cond(v, tmp));
>
> return tmp;
>
> As opposed to using the cmpxchg version, which would have more
> loads and conditional branches, AFAIKS.
I'd prefer to generalize this construct, than polluting atomic.h with all
kinds of esoteric atomic operations.
So you would get:
do {
old = atomic_load_locked(v);
if (!old)
break;
new = old + 1;
} while (!atomic_store_lock(v, old, new));
bye, Roman
Roman Zippel wrote:
> Hi,
>
> On Thu, 15 Sep 2005, Nick Piggin wrote:
>
>
>>Is there any point in keeping this around?
>
>
> Yes, for drivers which want to use it to synchronize with userspace.
> Alternatively it could be changed into a Kconfig definition.
>
I think it already is. At least, I did grep for it and didn't
see anything.
I think userspace synchronization may be quite a valid use of
atomic cmpxchg, but Kconfig is a far better place to do it than
testing HAVE_ARCH_CMPXCHG.
--
SUSE Labs, Novell Inc.
Send instant messages to your online friends http://au.messenger.yahoo.com
Nick Piggin wrote:
> Roman Zippel wrote:
>
>> Hi,
>>
>> On Thu, 15 Sep 2005, Nick Piggin wrote:
>>
>>
>>> Also needs work on those same architectures. Other architectures
>>> might want to look at providing a more optimal implementation.
>>
>>
>>
>> IMO a rather pointless primitive, unless there is a cpu architecture
>> which has a inc_not_zero instruction, otherwise it will always be the
>> same as using cmpxchg.
>>
>
[snip]
But even supposing the cmpxchg variant was the highest
performing implementation available on any architecture, I
would still consider exporting the inc_not_zero instruction.
The reason is that cmpxchg is not nearly so readable as
inc_not_zero when used inline in the code.
--
SUSE Labs, Novell Inc.
Send instant messages to your online friends http://au.messenger.yahoo.com
From: Nick Piggin <[email protected]>
Date: Thu, 15 Sep 2005 02:49:58 +1000
> I think it already is. At least, I did grep for it and didn't
> see anything.
Things in this class, such as DRM, just happen to only get enabled on
platforms that have a cmpxchg() available, they don't actually check.
That being said, I agree that Kconfig is the place to do this and
then DRM's Kconfig can get the correct dependency.
On Wed, Sep 14, 2005 at 07:18:31PM +0200, Roman Zippel wrote:
> On Thu, 15 Sep 2005, Nick Piggin wrote:
> > Or here is possible pseudo code for an architecture with ll/sc
> > instructions:
> >
> > do {
> > tmp = load_locked(v);
> > if (!tmp)
> > break;
> > tmp++;
> > } while (!store_cond(v, tmp));
> >
> > return tmp;
> >
> > As opposed to using the cmpxchg version, which would have more
> > loads and conditional branches, AFAIKS.
>
> I'd prefer to generalize this construct, than polluting atomic.h with all
> kinds of esoteric atomic operations.
> So you would get:
>
> do {
> old = atomic_load_locked(v);
> if (!old)
> break;
> new = old + 1;
> } while (!atomic_store_lock(v, old, new));
How do you propose architectures which don't have locked loads implement
this, where the only atomic instruction is an unconditional atomic swap
between memory and CPU register?
--
Russell King
Linux kernel 2.6 ARM Linux - http://www.arm.linux.org.uk/
maintainer of: 2.6 Serial core
On Thu, Sep 15, 2005 at 02:49:58AM +1000, Nick Piggin wrote:
> Roman Zippel wrote:
> > Hi,
> >
> > On Thu, 15 Sep 2005, Nick Piggin wrote:
> >
> >
> >>Is there any point in keeping this around?
> >
> >
> > Yes, for drivers which want to use it to synchronize with userspace.
> > Alternatively it could be changed into a Kconfig definition.
> >
>
> I think it already is. At least, I did grep for it and didn't
> see anything.
>
> I think userspace synchronization may be quite a valid use of
> atomic cmpxchg, but Kconfig is a far better place to do it than
> testing HAVE_ARCH_CMPXCHG.
What business has userspace got of telling whether cmpxchg works on
an architecture by looking at kernel headers?
Even if an architecture provides an implementation of it, it might
rely on turning IRQs off, which may not be possible in userspace,
leading to the userspace version actually being non-atomic.
*Forget* kernel includes telling userspace what architecture
features are available. It's extremely buggy by design.
If you want to remove HAVE_ARCH_CMPXCHG that's fine. If userspace
complains, you've found a bug for them. 8)
--
Russell King
Linux kernel 2.6 ARM Linux - http://www.arm.linux.org.uk/
maintainer of: 2.6 Serial core
Hi,
On Wed, 14 Sep 2005, Russell King wrote:
> > do {
> > old = atomic_load_locked(v);
> > if (!old)
> > break;
> > new = old + 1;
> > } while (!atomic_store_lock(v, old, new));
>
> How do you propose architectures which don't have locked loads implement
> this, where the only atomic instruction is an unconditional atomic swap
> between memory and CPU register?
#define atomic_store_lock atomic_cmpxchg
bye, Roman
On Thu, Sep 15, 2005 at 12:10:56AM +0200, Roman Zippel wrote:
> Hi,
>
> On Wed, 14 Sep 2005, Russell King wrote:
>
> > > do {
> > > old = atomic_load_locked(v);
> > > if (!old)
> > > break;
> > > new = old + 1;
> > > } while (!atomic_store_lock(v, old, new));
> >
> > How do you propose architectures which don't have locked loads implement
> > this, where the only atomic instruction is an unconditional atomic swap
> > between memory and CPU register?
>
> #define atomic_store_lock atomic_cmpxchg
No. "unconditional atomic swap" does not mean cmpxchg - it means that
atomic_cmpxchg itself would have to be open coded, which is inefficient.
What you're asking architectures to do is:
retry:
load
operation
save interrupts
load
compare
store if equal
restore interrupts
goto retry if not equal
whereas they could have done the far simpler version of:
save interrupts
load
operation
store
restore interrupts
which they do today.
The whole point about architecture specific includes is not to provide
a frenzied feeding ground for folk who like to "clean code up" but to
allow architectures to do things in the most efficient way for them
without polluting the kernel too much.
It seems that aspect is being lost sight of here.
--
Russell King
Linux kernel 2.6 ARM Linux - http://www.arm.linux.org.uk/
maintainer of: 2.6 Serial core
From: Russell King <[email protected]>
Date: Wed, 14 Sep 2005 23:03:53 +0100
> What business has userspace got of telling whether cmpxchg works on
> an architecture by looking at kernel headers?
Russell, please don't fly off the handle like this.
Nick is talking about something slightly different.
Things, for example, like DRM assume there is a cmpxchg()
they can use in the kernel and userland for interlocking.
DRM doesn't actually _check_ anything to see if this is the case, it
just so happens to only get enabled on platforms where cmpxchg() is
available in this fashion.
What Nick is suggesting is to actually move HAVE_ARCH_CMPXCHG or
something like it into the Kconfig so that things like DRM can
actually do the correct dependency check. If you want something like
"HAVE_ARCH_CMPXCHG_WHICH_CAN_INTERFACE_WITH_USERSPACE" that DRM can
check too, all the better.
It's not about whether userspace can include some kernel header
and get cmpxchg(), it's whether there is some way that a cmpxchg()
shared semaphore between userspace and kernel is possible, which
things like DRM depend upon having available.
Russell King wrote:
> The whole point about architecture specific includes is not to provide
> a frenzied feeding ground for folk who like to "clean code up" but to
> allow architectures to do things in the most efficient way for them
> without polluting the kernel too much.
>
> It seems that aspect is being lost sight of here.
>
Yep. We've got atomic_add, atomic_sub, atomic_inc, atomic_dec,
atomic_inc_return, atomic_add_return, atomic_dec_return, atomic_sub_return
to start with.
Not only that, but we can probably emulate all the atomic_ operations
with atomic_cmpxchg, not just atomic_inc_not_zero.
Roman: any ideas about what you would prefer? You'll notice
atomic_inc_not_zero replaces rcuref_inc_lf, which is used several times
in the VFS.
Thanks,
Nick
--
SUSE Labs, Novell Inc.
Send instant messages to your online friends http://au.messenger.yahoo.com
On Wed, Sep 14, 2005 at 03:26:10PM -0700, David S. Miller wrote:
> From: Russell King <[email protected]>
> Date: Wed, 14 Sep 2005 23:03:53 +0100
>
> > What business has userspace got of telling whether cmpxchg works on
> > an architecture by looking at kernel headers?
>
> Russell, please don't fly off the handle like this.
Sigh, I wasn't. I was making a valid point. Maybe if you read the
bit I quoted you'd have realised that, which was:
> I think userspace synchronization may be quite a valid use of
^^^^^^^^^
> atomic cmpxchg, but Kconfig is a far better place to do it than
> testing HAVE_ARCH_CMPXCHG.
But maybe I'm not capable of interpreting the above english? To
me at least it's definitely talking about user space, not kernel
space.
--
Russell King
Linux kernel 2.6 ARM Linux - http://www.arm.linux.org.uk/
maintainer of: 2.6 Serial core
On Iau, 2005-09-15 at 01:01 +1000, Nick Piggin wrote:
> Is there any point in keeping this around?
>
Not all platforms have CMPXCHG and some code wants to know about that -
eg DRI
From: Nick Piggin <[email protected]>
Date: Thu, 15 Sep 2005 02:52:08 +1000
> OK that's fair enough. I'll submit whatever cleanups are possible
> on top of your patch after the atomic_cmpxchg patch goes in (if
> it goes in). Thanks.
The updated patch is in Linus's current tree, FYI.
The GIT commit is: 4db2ce0199f04b6e99999f22e28ef9a0ae5f0d2f
Hi,
On Wed, 14 Sep 2005, Russell King wrote:
> >
> > > > do {
> > > > old = atomic_load_locked(v);
> > > > if (!old)
> > > > break;
> > > > new = old + 1;
> > > > } while (!atomic_store_lock(v, old, new));
> > >
>
> What you're asking architectures to do is:
>
> retry:
> load
> operation
> save interrupts
> load
> compare
> store if equal
> restore interrupts
> goto retry if not equal
>
> whereas they could have done the far simpler version of:
>
> save interrupts
> load
> operation
> store
> restore interrupts
>
> which they do today.
So modify it this way:
atomic_lock(flags);
do {
old = atomic_load_locked(v);
if (!old)
break;
new = old + 1;
} while (!atomic_store_locked(v, old, new));
atomic_unlock(flags);
> The whole point about architecture specific includes is not to provide
> a frenzied feeding ground for folk who like to "clean code up" but to
> allow architectures to do things in the most efficient way for them
> without polluting the kernel too much.
In this case it's massively repeated code which only differs slighty. My
biggest problem here is the lack of gcc support to get the condition code
out of an asm. Especially architectures with locked load/store instruction
could then have just a single implementation in asm-generic.
A long time ago I was playing with something like this:
#define atomic_exchange(ptr, old, new) ({ \
old = *(ptr); \
asm volatile ("1:" \
: "=&a" (old), "=m" (*(ptr)) \
: "0" (old), "m" (*(ptr)) \
: "memory"); \
asm volatile (__LOCK "cmpxchg %2,%0; jne 1b" \
: "=m" (*(ptr)), "=&a" (old) \
: "r" (new), "m" (*(ptr)), "1" (old) \
: "memory"); \
})
so e.g. an atomic_add_return() would be atomic_exchange(ptr, old, old +
val), but I guess gcc people would kill me for that. :-)
bye, Roman
Hi,
On Thu, 15 Sep 2005, Nick Piggin wrote:
> Roman: any ideas about what you would prefer? You'll notice
> atomic_inc_not_zero replaces rcuref_inc_lf, which is used several times
> in the VFS.
In the larger picture I'm not completely happy with these scalibilty
patches, as they add extra overhead at the lower end. On a UP system in
general nothing beats:
spin_lock();
if (*ptr)
ptr += 1;
spin_unlock();
The main problem is here that the atomic functions are used in two basic
situation:
1) interrupt synchronization
2) multiprocessor synchronization
The atomic functions have to assume both, but on UP systems it often is
a lot cheaper if they don't have to synchronize with interrupts. So
replacing a spinlock with a few atomic operations can hurt UP performance.
bye, Roman
Roman Zippel <[email protected]> wrote:
>
> Hi,
>
> On Thu, 15 Sep 2005, Nick Piggin wrote:
>
> > Roman: any ideas about what you would prefer? You'll notice
> > atomic_inc_not_zero replaces rcuref_inc_lf, which is used several times
> > in the VFS.
>
> In the larger picture I'm not completely happy with these scalibilty
> patches, as they add extra overhead at the lower end. On a UP system in
> general nothing beats:
>
> spin_lock();
> if (*ptr)
> ptr += 1;
> spin_unlock();
>
> The main problem is here that the atomic functions are used in two basic
> situation:
>
> 1) interrupt synchronization
> 2) multiprocessor synchronization
>
> The atomic functions have to assume both, but on UP systems it often is
> a lot cheaper if they don't have to synchronize with interrupts. So
> replacing a spinlock with a few atomic operations can hurt UP performance.
>
Nope. On uniprocessor systems, atomic_foo() doesn't actually do the
buslocked atomic thing.
#ifdef CONFIG_SMP
#define LOCK "lock ; "
#else
#define LOCK ""
#endif
On x86, at least. Other architectures can do the same thing if they have
an atomic-wrt-IRQs add and sub.
From: Roman Zippel <[email protected]>
Date: Sat, 17 Sep 2005 02:59:50 +0200 (CEST)
> My biggest problem here is the lack of gcc support to get the
> condition code out of an asm.
I agree, this is the biggest deficiency in gcc inline assembly and I
run into it all the time.
Things like __builtin_trap() would never even be needed if we could
communicate condition code state into and out of inline asm
statements, for example.
Sparc32 and sparc64 both have "trap on condition code" instructions.
__builtin_trap() will do the right thing, _BUT_ I want to emit nice
bug table entries like ppc64 does in asm-ppc64/bug.h, but I can't do
that without generating really crappy code. And the whole limitation
comes from the fact that I can't tell ask gcc for "the condition codes
that result from test X" as the input for an asm.
So what would be great is something akin to:
__asm__ __volatile__("t%cc 0x5"
: "=C" (test));
or something like that. The "%cc" thing would be expand to the
appropriate two/three letter condition test type code, for example
"ne", "eq", "ge", "ltu" and the like. GCC could choose the best
comparison + %cc code, as it already does for conditional branch
generation.
It could even help on platforms like ppc64 where the trap condition
instruction does a test of a register against an immediate value.
Something like "BUG_ON(a != 2)" is probably emitting something like:
sub %a, 2, %tmp
tdnei %tmp, 0
whereas this could be:
tdnei %tmp, 2
if GCC has some inline asm condition code input facility like the
above.
From: Roman Zippel <[email protected]>
Date: Sat, 17 Sep 2005 03:15:29 +0200 (CEST)
> The main problem is here that the atomic functions are used in two basic
> situation:
>
> 1) interrupt synchronization
> 2) multiprocessor synchronization
3) preempt synchronization
On Sat, Sep 17, 2005 at 12:18:22AM -0700, David S. Miller wrote:
> > My biggest problem here is the lack of gcc support to get the
> > condition code out of an asm.
>
> I agree, this is the biggest deficiency in gcc inline assembly and I
> run into it all the time.
gcc did have some support to pass condition codes into assembly.
On ARM, you used to be able to do things like:
if (foo)
asm("blah%? whatever");
and gcc would replace %? with whatever condition was appropriate
for the current block of code. You can still write it as the
above.
However, this optimisation was disabled on ARM apparantly because
it was very hard to for people to get correct - if you forgot the
%?, you need to add a "cc" clobber, and if you forget that as well
you might get unconditional behaviour.
--
Russell King
Linux kernel 2.6 ARM Linux - http://www.arm.linux.org.uk/
maintainer of: 2.6 Serial core
On Sat, Sep 17, 2005 at 03:15:29AM +0200, Roman Zippel wrote:
> Hi,
>
> On Thu, 15 Sep 2005, Nick Piggin wrote:
>
> > Roman: any ideas about what you would prefer? You'll notice
> > atomic_inc_not_zero replaces rcuref_inc_lf, which is used several times
> > in the VFS.
>
> In the larger picture I'm not completely happy with these scalibilty
> patches, as they add extra overhead at the lower end. On a UP system in
> general nothing beats:
>
> spin_lock();
> if (*ptr)
> ptr += 1;
> spin_unlock();
>
> The main problem is here that the atomic functions are used in two basic
> situation:
Are you talking about the lock-free fdtable patches ? They don't replace
non-atomic locked critical sections by atomic operations. Reference counting
is already there to extend the life of objects beyond locked critical
setions.
Thanks
Dipankar
Hi,
On Fri, 16 Sep 2005, Andrew Morton wrote:
> Nope. On uniprocessor systems, atomic_foo() doesn't actually do the
> buslocked atomic thing.
>
> #ifdef CONFIG_SMP
> #define LOCK "lock ; "
> #else
> #define LOCK ""
> #endif
>
> On x86, at least. Other architectures can do the same thing if they have
> an atomic-wrt-IRQs add and sub.
That's true on x86, but if these functions have to be emulated using
spinlocks, they always have to disable interrupts, whether the caller
needs it or not. Also x86 has the lock attribute, which helps a lot, but
on other archs a cmpxchg instruction can be quite expensive, so it could
be cheaper by just using {disable,enable}_preempt.
bye, Roman
From: Russell King <[email protected]>
Date: Sat, 17 Sep 2005 08:27:36 +0100
> gcc did have some support to pass condition codes into assembly.
> On ARM, you used to be able to do things like:
>
> if (foo)
> asm("blah%? whatever");
>
> and gcc would replace %? with whatever condition was appropriate
> for the current block of code. You can still write it as the
> above.
>
> However, this optimisation was disabled on ARM apparantly because
> it was very hard to for people to get correct - if you forgot the
> %?, you need to add a "cc" clobber, and if you forget that as well
> you might get unconditional behaviour.
Yes, that is an error prone syntax to use, that's for sure.
That is, incidentally, why I said the condition test should
be an explicit input arg to the asm.
Roman Zippel wrote:
> Hi,
>
> On Thu, 15 Sep 2005, Nick Piggin wrote:
>
>
>>Roman: any ideas about what you would prefer? You'll notice
>>atomic_inc_not_zero replaces rcuref_inc_lf, which is used several times
>>in the VFS.
>
>
> In the larger picture I'm not completely happy with these scalibilty
> patches, as they add extra overhead at the lower end. On a UP system in
> general nothing beats:
>
> spin_lock();
> if (*ptr)
> ptr += 1;
> spin_unlock();
>
> The main problem is here that the atomic functions are used in two basic
> situation:
>
> 1) interrupt synchronization
> 2) multiprocessor synchronization
>
> The atomic functions have to assume both, but on UP systems it often is
> a lot cheaper if they don't have to synchronize with interrupts. So
> replacing a spinlock with a few atomic operations can hurt UP performance.
>
Maybe so, but what I'm doing is introducing a slightly better
implementation of what is currently in tree, and attempting to
follow current standards as far as possible. I don't think you
could say that is a bad thing.
Now I don't think anyone would be flat out opposed to 1 - reworking
the atomic.h code to allow some genericity (is that a word?); 2 -
reworking atomic.h code to allow combining of atomic ops, or allowing
interrupt unsafe ops...
Of course, neither is going to be merged unless done tastefully, and
I imagine both would be difficult to get right, with probably a low
cost/benefit ratio.
Thanks,
Nick
--
SUSE Labs, Novell Inc.
Send instant messages to your online friends http://au.messenger.yahoo.com