Currently, percpu_ref uses the global percpu_ref_switch_lock to
protect the mode switching operation. When multiple percpu_ref
perform mode switching at the same time, the lock may become a
performance bottleneck.
This patch introduces per percpu_ref percpu_ref_switch_lock to
fixes this situation.
Signed-off-by: Qi Zheng <[email protected]>
---
include/linux/percpu-refcount.h | 2 ++
lib/percpu-refcount.c | 30 +++++++++++++++---------------
2 files changed, 17 insertions(+), 15 deletions(-)
diff --git a/include/linux/percpu-refcount.h b/include/linux/percpu-refcount.h
index 75844939a965..eb8695e578fd 100644
--- a/include/linux/percpu-refcount.h
+++ b/include/linux/percpu-refcount.h
@@ -110,6 +110,8 @@ struct percpu_ref {
*/
unsigned long percpu_count_ptr;
+ spinlock_t percpu_ref_switch_lock;
+
/*
* 'percpu_ref' is often embedded into user structure, and only
* 'percpu_count_ptr' is required in fast path, move other fields
diff --git a/lib/percpu-refcount.c b/lib/percpu-refcount.c
index 3a8906715e09..4336fd1bd77a 100644
--- a/lib/percpu-refcount.c
+++ b/lib/percpu-refcount.c
@@ -36,7 +36,6 @@
#define PERCPU_COUNT_BIAS (1LU << (BITS_PER_LONG - 1))
-static DEFINE_SPINLOCK(percpu_ref_switch_lock);
static DECLARE_WAIT_QUEUE_HEAD(percpu_ref_switch_waitq);
static unsigned long __percpu *percpu_count_ptr(struct percpu_ref *ref)
@@ -95,6 +94,7 @@ int percpu_ref_init(struct percpu_ref *ref, percpu_ref_func_t *release,
start_count++;
atomic_long_set(&data->count, start_count);
+ spin_lock_init(&ref->percpu_ref_switch_lock);
data->release = release;
data->confirm_switch = NULL;
@@ -137,11 +137,11 @@ void percpu_ref_exit(struct percpu_ref *ref)
if (!data)
return;
- spin_lock_irqsave(&percpu_ref_switch_lock, flags);
+ spin_lock_irqsave(&ref->percpu_ref_switch_lock, flags);
ref->percpu_count_ptr |= atomic_long_read(&ref->data->count) <<
__PERCPU_REF_FLAG_BITS;
ref->data = NULL;
- spin_unlock_irqrestore(&percpu_ref_switch_lock, flags);
+ spin_unlock_irqrestore(&ref->percpu_ref_switch_lock, flags);
kfree(data);
}
@@ -287,7 +287,7 @@ static void __percpu_ref_switch_mode(struct percpu_ref *ref,
{
struct percpu_ref_data *data = ref->data;
- lockdep_assert_held(&percpu_ref_switch_lock);
+ lockdep_assert_held(&ref->percpu_ref_switch_lock);
/*
* If the previous ATOMIC switching hasn't finished yet, wait for
@@ -295,7 +295,7 @@ static void __percpu_ref_switch_mode(struct percpu_ref *ref,
* isn't in progress, this function can be called from any context.
*/
wait_event_lock_irq(percpu_ref_switch_waitq, !data->confirm_switch,
- percpu_ref_switch_lock);
+ ref->percpu_ref_switch_lock);
if (data->force_atomic || percpu_ref_is_dying(ref))
__percpu_ref_switch_to_atomic(ref, confirm_switch, sync);
@@ -329,12 +329,12 @@ void percpu_ref_switch_to_atomic(struct percpu_ref *ref,
{
unsigned long flags;
- spin_lock_irqsave(&percpu_ref_switch_lock, flags);
+ spin_lock_irqsave(&ref->percpu_ref_switch_lock, flags);
ref->data->force_atomic = true;
__percpu_ref_switch_mode(ref, confirm_switch, sync);
- spin_unlock_irqrestore(&percpu_ref_switch_lock, flags);
+ spin_unlock_irqrestore(&ref->percpu_ref_switch_lock, flags);
}
EXPORT_SYMBOL_GPL(percpu_ref_switch_to_atomic);
@@ -376,12 +376,12 @@ void percpu_ref_switch_to_percpu(struct percpu_ref *ref)
{
unsigned long flags;
- spin_lock_irqsave(&percpu_ref_switch_lock, flags);
+ spin_lock_irqsave(&ref->percpu_ref_switch_lock, flags);
ref->data->force_atomic = false;
__percpu_ref_switch_mode(ref, NULL, false);
- spin_unlock_irqrestore(&percpu_ref_switch_lock, flags);
+ spin_unlock_irqrestore(&ref->percpu_ref_switch_lock, flags);
}
EXPORT_SYMBOL_GPL(percpu_ref_switch_to_percpu);
@@ -407,7 +407,7 @@ void percpu_ref_kill_and_confirm(struct percpu_ref *ref,
{
unsigned long flags;
- spin_lock_irqsave(&percpu_ref_switch_lock, flags);
+ spin_lock_irqsave(&ref->percpu_ref_switch_lock, flags);
WARN_ONCE(percpu_ref_is_dying(ref),
"%s called more than once on %ps!", __func__,
@@ -417,7 +417,7 @@ void percpu_ref_kill_and_confirm(struct percpu_ref *ref,
__percpu_ref_switch_mode(ref, confirm_kill, false);
percpu_ref_put(ref);
- spin_unlock_irqrestore(&percpu_ref_switch_lock, flags);
+ spin_unlock_irqrestore(&ref->percpu_ref_switch_lock, flags);
}
EXPORT_SYMBOL_GPL(percpu_ref_kill_and_confirm);
@@ -438,12 +438,12 @@ bool percpu_ref_is_zero(struct percpu_ref *ref)
return false;
/* protect us from being destroyed */
- spin_lock_irqsave(&percpu_ref_switch_lock, flags);
+ spin_lock_irqsave(&ref->percpu_ref_switch_lock, flags);
if (ref->data)
count = atomic_long_read(&ref->data->count);
else
count = ref->percpu_count_ptr >> __PERCPU_REF_FLAG_BITS;
- spin_unlock_irqrestore(&percpu_ref_switch_lock, flags);
+ spin_unlock_irqrestore(&ref->percpu_ref_switch_lock, flags);
return count == 0;
}
@@ -487,7 +487,7 @@ void percpu_ref_resurrect(struct percpu_ref *ref)
unsigned long __percpu *percpu_count;
unsigned long flags;
- spin_lock_irqsave(&percpu_ref_switch_lock, flags);
+ spin_lock_irqsave(&ref->percpu_ref_switch_lock, flags);
WARN_ON_ONCE(!percpu_ref_is_dying(ref));
WARN_ON_ONCE(__ref_is_percpu(ref, &percpu_count));
@@ -496,6 +496,6 @@ void percpu_ref_resurrect(struct percpu_ref *ref)
percpu_ref_get(ref);
__percpu_ref_switch_mode(ref, NULL, false);
- spin_unlock_irqrestore(&percpu_ref_switch_lock, flags);
+ spin_unlock_irqrestore(&ref->percpu_ref_switch_lock, flags);
}
EXPORT_SYMBOL_GPL(percpu_ref_resurrect);
--
2.20.1