2011-05-11 15:47:25

by Shaohua Li

[permalink] [raw]
Subject: [patch v2 5/5] percpu_counter: preemptless __per_cpu_counter_add

From: Christoph Lameter <[email protected]>

Make percpu_counter_add hot path preemptless.

Signed-off-by: Christoph Lameter <[email protected]>
Signed-off-by: Shaohua Li <[email protected]>
---
lib/percpu_counter.c | 37 +++++++++++++++++++++++++------------
1 file changed, 25 insertions(+), 12 deletions(-)

Index: linux/lib/percpu_counter.c
===================================================================
--- linux.orig/lib/percpu_counter.c 2011-05-10 16:23:01.000000000 +0800
+++ linux/lib/percpu_counter.c 2011-05-10 16:23:01.000000000 +0800
@@ -67,21 +67,34 @@ void percpu_counter_set(struct percpu_co
}
EXPORT_SYMBOL(percpu_counter_set);

+static bool __percpu_counter_add_cmpxchg(struct percpu_counter *fbc,
+ s64 count, s64 new)
+{
+#ifdef CONFIG_PREEMPT
+ return this_cpu_cmpxchg(*fbc->counters, count, new) == count;
+#else
+ this_cpu_write(*fbc->counters, new);
+ return true;
+#endif
+}
+
void __percpu_counter_add(struct percpu_counter *fbc, s64 amount, s32 batch)
{
- s64 count;
+ s64 count, new;
+
+ do {
+ count = this_cpu_read(*fbc->counters);
+ new = count + amount;

- preempt_disable();
- count = __this_cpu_read(*fbc->counters) + amount;
- if (count >= batch || count <= -batch) {
- lg_local_lock(fbc->lglock);
- atomic64_add(count, &fbc->count);
- __this_cpu_write(*fbc->counters, 0);
- lg_local_unlock(fbc->lglock);
- } else {
- __this_cpu_write(*fbc->counters, count);
- }
- preempt_enable();
+ if (new >= batch || new <= -batch) {
+ lg_local_lock(fbc->lglock);
+ count = __this_cpu_read(*fbc->counters) + amount;
+ atomic64_add(count, &fbc->count);
+ __this_cpu_write(*fbc->counters, 0);
+ lg_local_unlock(fbc->lglock);
+ return;
+ }
+ } while (!__percpu_counter_add_cmpxchg(fbc, count, new));
}
EXPORT_SYMBOL(__percpu_counter_add);