Hi Andrew,
This patch adds read_lock() to the list of lock services that drop
preemption while spinning.
This patch is against 2.5.59, so it re-applies the spinlock repair
work already in your trees but not yet pushed to Linus.
A 2.4 version of this patch has been running here on several test
stands for nearly a week. The 2.5 version has been boot-tested.
Regards,
Joe
PS: I also have 2.4 versions of irq lock services that drop the irq
while spinning. These have been running here since Monday. I can
do a 2.5 port if you are interested in taking a look at them.
diff -ur 2.5.59-orig/include/asm-i386/atomic.h 2.5.59-rdlcks/include/asm-i386/atomic.h
--- 2.5.59-orig/include/asm-i386/atomic.h 2003-01-22 06:02:30.000000000 -0500
+++ 2.5.59-rdlcks/include/asm-i386/atomic.h 2003-01-29 08:51:31.000000000 -0500
@@ -146,6 +146,25 @@
}
/**
+ * atomic_dec_positive - decrement and test if positive
+ * @v: pointer of type atomic_t
+ *
+ * Atomically decrements @v by 1 and returns true if the
+ * result is greater than zero. Note that the guaranteed
+ * useful range of an atomic_t is only 24 bits.
+ */
+static __inline__ int atomic_dec_positive(atomic_t *v)
+{
+ unsigned char c;
+
+ __asm__ __volatile__(
+ LOCK "decl %0; setg %1"
+ :"=m" (v->counter), "=qm" (c)
+ :"m" (v->counter) : "memory");
+ return c !=0;
+}
+
+/**
* atomic_inc_and_test - increment and test
* @v: pointer of type atomic_t
*
diff -ur 2.5.59-orig/include/asm-i386/spinlock.h 2.5.59-rdlcks/include/asm-i386/spinlock.h
--- 2.5.59-orig/include/asm-i386/spinlock.h 2003-01-22 06:02:29.000000000 -0500
+++ 2.5.59-rdlcks/include/asm-i386/spinlock.h 2003-01-29 18:03:21.000000000 -0500
@@ -159,6 +159,7 @@
#define rwlock_init(x) do { *(x) = RW_LOCK_UNLOCKED; } while(0)
#define rwlock_is_locked(x) ((x)->lock != RW_LOCK_BIAS)
+#define rwlock_is_wrlocked(x) ((x)->lock <= 0)
/*
* On x86, we implement read-write locks as a 32-bit counter
@@ -201,4 +202,13 @@
return 0;
}
+static inline int _raw_read_trylock(rwlock_t *lock)
+{
+ atomic_t *count = (atomic_t *)lock;
+ if (atomic_dec_positive(count))
+ return 1;
+ atomic_inc(count);
+ return 0;
+}
+
#endif /* __ASM_SPINLOCK_H */
diff -ur 2.5.59-orig/include/linux/spinlock.h 2.5.59-rdlcks/include/linux/spinlock.h
--- 2.5.59-orig/include/linux/spinlock.h 2003-01-22 06:02:37.000000000 -0500
+++ 2.5.59-rdlcks/include/linux/spinlock.h 2003-01-29 18:04:28.000000000 -0500
@@ -91,11 +91,13 @@
#define write_trylock(lock) ({preempt_disable();_raw_write_trylock(lock) ? \
1 : ({preempt_enable(); 0;});})
-/* Where's read_trylock? */
+#define read_trylock(lock) ({preempt_disable();_raw_read_trylock(lock) ? \
+ 1 : ({preempt_enable(); 0;});})
#if defined(CONFIG_SMP) && defined(CONFIG_PREEMPT)
void __preempt_spin_lock(spinlock_t *lock);
void __preempt_write_lock(rwlock_t *lock);
+void __preempt_read_lock(rwlock_t *lock);
#define spin_lock(lock) \
do { \
@@ -111,6 +113,13 @@
__preempt_write_lock(lock); \
} while (0)
+#define read_lock(lock) \
+do { \
+ preempt_disable(); \
+ if (unlikely(!_raw_read_trylock(lock))) \
+ __preempt_read_lock(lock); \
+} while (0)
+
#else
#define spin_lock(lock) \
do { \
@@ -123,13 +132,13 @@
preempt_disable(); \
_raw_write_lock(lock); \
} while(0)
-#endif
#define read_lock(lock) \
do { \
preempt_disable(); \
_raw_read_lock(lock); \
} while(0)
+#endif
#define spin_unlock(lock) \
do { \
diff -ur 2.5.59-orig/kernel/ksyms.c 2.5.59-rdlcks/kernel/ksyms.c
--- 2.5.59-orig/kernel/ksyms.c 2003-01-22 06:02:16.000000000 -0500
+++ 2.5.59-rdlcks/kernel/ksyms.c 2003-01-28 17:22:03.000000000 -0500
@@ -496,6 +496,7 @@
#if defined(CONFIG_SMP) && defined(CONFIG_PREEMPT)
EXPORT_SYMBOL(__preempt_spin_lock);
EXPORT_SYMBOL(__preempt_write_lock);
+EXPORT_SYMBOL(__preempt_read_lock);
#endif
#if !defined(__ia64__)
EXPORT_SYMBOL(loops_per_jiffy);
diff -ur 2.5.59-orig/kernel/sched.c 2.5.59-rdlcks/kernel/sched.c
--- 2.5.59-orig/kernel/sched.c 2003-01-22 06:02:55.000000000 -0500
+++ 2.5.59-rdlcks/kernel/sched.c 2003-01-29 18:04:59.000000000 -0500
@@ -2465,15 +2465,13 @@
_raw_spin_lock(lock);
return;
}
-
- while (!_raw_spin_trylock(lock)) {
- if (need_resched()) {
- preempt_enable_no_resched();
- __cond_resched();
- preempt_disable();
+ do {
+ preempt_enable();
+ while(spin_is_locked(lock)) {
+ cpu_relax();
}
- cpu_relax();
- }
+ preempt_disable();
+ } while (!_raw_spin_trylock(lock));
}
void __preempt_write_lock(rwlock_t *lock)
@@ -2482,14 +2480,27 @@
_raw_write_lock(lock);
return;
}
-
- while (!_raw_write_trylock(lock)) {
- if (need_resched()) {
- preempt_enable_no_resched();
- __cond_resched();
- preempt_disable();
+ do {
+ preempt_enable();
+ while(rwlock_is_locked(lock)) {
+ cpu_relax();
}
- cpu_relax();
+ preempt_disable();
+ } while (!_raw_write_trylock(lock));
+}
+
+void __preempt_read_lock(rwlock_t *lock)
+{
+ if (preempt_count() > 1) {
+ _raw_read_lock(lock);
+ return;
}
+ do {
+ preempt_enable();
+ while(rwlock_is_wrlocked(lock)) {
+ cpu_relax();
+ }
+ preempt_disable();
+ } while (!_raw_read_trylock(lock));
}
#endif