Now that we have rcu_state_p, which references rcu_preempt_state for
TREE_PREEMPT_RCU and rcu_sched_state for TREE_RCU, we don't need a
separate rcu_sysidle_state variable. This commit therefore eliminates
rcu_preempt_state in favor of rcu_state_p.
Signed-off-by: Paul E. McKenney <[email protected]>
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index 24f5bd54ca9b..e31fa71c4d3a 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -2662,16 +2662,6 @@ static void __maybe_unused rcu_kick_nohz_cpu(int cpu)
#ifdef CONFIG_NO_HZ_FULL_SYSIDLE
-/*
- * Define RCU flavor that holds sysidle state. This needs to be the
- * most active flavor of RCU.
- */
-#ifdef CONFIG_PREEMPT_RCU
-static struct rcu_state *rcu_sysidle_state = &rcu_preempt_state;
-#else /* #ifdef CONFIG_PREEMPT_RCU */
-static struct rcu_state *rcu_sysidle_state = &rcu_sched_state;
-#endif /* #else #ifdef CONFIG_PREEMPT_RCU */
-
static int full_sysidle_state; /* Current system-idle state. */
#define RCU_SYSIDLE_NOT 0 /* Some CPU is not idle. */
#define RCU_SYSIDLE_SHORT 1 /* All CPUs idle for brief period. */
@@ -2813,7 +2803,7 @@ static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,
* not the flavor of RCU that tracks sysidle state, or if this
* is an offline or the timekeeping CPU, nothing to do.
*/
- if (!*isidle || rdp->rsp != rcu_sysidle_state ||
+ if (!*isidle || rdp->rsp != rcu_state_p ||
cpu_is_offline(rdp->cpu) || rdp->cpu == tick_do_timer_cpu)
return;
if (rcu_gp_in_progress(rdp->rsp))
@@ -2839,7 +2829,7 @@ static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,
*/
static bool is_sysidle_rcu_state(struct rcu_state *rsp)
{
- return rsp == rcu_sysidle_state;
+ return rsp == rcu_state_p;
}
/*
@@ -2917,7 +2907,7 @@ static void rcu_sysidle_cancel(void)
static void rcu_sysidle_report(struct rcu_state *rsp, int isidle,
unsigned long maxj, bool gpkt)
{
- if (rsp != rcu_sysidle_state)
+ if (rsp != rcu_state_p)
return; /* Wrong flavor, ignore. */
if (gpkt && nr_cpu_ids <= CONFIG_NO_HZ_FULL_SYSIDLE_SMALL)
return; /* Running state machine from timekeeping CPU. */
@@ -2986,13 +2976,12 @@ bool rcu_sys_is_idle(void)
/* Scan all the CPUs looking for nonidle CPUs. */
for_each_possible_cpu(cpu) {
- rdp = per_cpu_ptr(rcu_sysidle_state->rda, cpu);
+ rdp = per_cpu_ptr(rcu_state_p->rda, cpu);
rcu_sysidle_check_cpu(rdp, &isidle, &maxj);
if (!isidle)
break;
}
- rcu_sysidle_report(rcu_sysidle_state,
- isidle, maxj, false);
+ rcu_sysidle_report(rcu_state_p, isidle, maxj, false);
oldrss = rss;
rss = ACCESS_ONCE(full_sysidle_state);
}
@@ -3019,7 +3008,7 @@ bool rcu_sys_is_idle(void)
* provided by the memory allocator.
*/
if (nr_cpu_ids > CONFIG_NO_HZ_FULL_SYSIDLE_SMALL &&
- !rcu_gp_in_progress(rcu_sysidle_state) &&
+ !rcu_gp_in_progress(rcu_state_p) &&
!rsh.inuse && xchg(&rsh.inuse, 1) == 0)
call_rcu(&rsh.rh, rcu_sysidle_cb);
return false;
If there are no nohz_full= CPUs, then there is currently no reason to
track sysidle state. This commit therefore short-circuits this state
tracking if !tick_nohz_full_enabled().
Note that these checks will need to be revisited if nohz_full= state
can ever be changed at runtime.
Signed-off-by: Paul E. McKenney <[email protected]>
diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
index e31fa71c4d3a..c31eb28ae8bb 100644
--- a/kernel/rcu/tree_plugin.h
+++ b/kernel/rcu/tree_plugin.h
@@ -2679,6 +2679,10 @@ static void rcu_sysidle_enter(struct rcu_dynticks *rdtp, int irq)
{
unsigned long j;
+ /* If there are no nohz_full= CPUs, no need to track this. */
+ if (!tick_nohz_full_enabled())
+ return;
+
/* Adjust nesting, check for fully idle. */
if (irq) {
rdtp->dynticks_idle_nesting--;
@@ -2744,6 +2748,10 @@ void rcu_sysidle_force_exit(void)
*/
static void rcu_sysidle_exit(struct rcu_dynticks *rdtp, int irq)
{
+ /* If there are no nohz_full= CPUs, no need to track this. */
+ if (!tick_nohz_full_enabled())
+ return;
+
/* Adjust nesting, check for already non-idle. */
if (irq) {
rdtp->dynticks_idle_nesting++;
@@ -2798,6 +2806,10 @@ static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,
unsigned long j;
struct rcu_dynticks *rdtp = rdp->dynticks;
+ /* If there are no nohz_full= CPUs, don't check system-wide idleness. */
+ if (!tick_nohz_full_enabled())
+ return;
+
/*
* If some other CPU has already reported non-idle, if this is
* not the flavor of RCU that tracks sysidle state, or if this
@@ -2924,6 +2936,10 @@ static void rcu_sysidle_report(struct rcu_state *rsp, int isidle,
static void rcu_sysidle_report_gp(struct rcu_state *rsp, int isidle,
unsigned long maxj)
{
+ /* If there are no nohz_full= CPUs, no need to track this. */
+ if (!tick_nohz_full_enabled())
+ return;
+
rcu_sysidle_report(rsp, isidle, maxj, true);
}
@@ -2950,7 +2966,8 @@ static void rcu_sysidle_cb(struct rcu_head *rhp)
/*
* Check to see if the system is fully idle, other than the timekeeping CPU.
- * The caller must have disabled interrupts.
+ * The caller must have disabled interrupts. This is not intended to be
+ * called unless tick_nohz_full_enabled().
*/
bool rcu_sys_is_idle(void)
{
On Wed, Jul 23, 2014 at 12:28 PM, Paul E. McKenney
<[email protected]> wrote:
> Now that we have rcu_state_p, which references rcu_preempt_state for
> TREE_PREEMPT_RCU and rcu_sched_state for TREE_RCU, we don't need a
> separate rcu_sysidle_state variable. This commit therefore eliminates
> rcu_preempt_state in favor of rcu_state_p.
>
> Signed-off-by: Paul E. McKenney <[email protected]>
>
Reviewed-by: Pranith Kumar <[email protected]>
> diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
> index 24f5bd54ca9b..e31fa71c4d3a 100644
> --- a/kernel/rcu/tree_plugin.h
> +++ b/kernel/rcu/tree_plugin.h
> @@ -2662,16 +2662,6 @@ static void __maybe_unused rcu_kick_nohz_cpu(int cpu)
>
> #ifdef CONFIG_NO_HZ_FULL_SYSIDLE
>
> -/*
> - * Define RCU flavor that holds sysidle state. This needs to be the
> - * most active flavor of RCU.
> - */
> -#ifdef CONFIG_PREEMPT_RCU
> -static struct rcu_state *rcu_sysidle_state = &rcu_preempt_state;
> -#else /* #ifdef CONFIG_PREEMPT_RCU */
> -static struct rcu_state *rcu_sysidle_state = &rcu_sched_state;
> -#endif /* #else #ifdef CONFIG_PREEMPT_RCU */
> -
> static int full_sysidle_state; /* Current system-idle state. */
> #define RCU_SYSIDLE_NOT 0 /* Some CPU is not idle. */
> #define RCU_SYSIDLE_SHORT 1 /* All CPUs idle for brief period. */
> @@ -2813,7 +2803,7 @@ static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,
> * not the flavor of RCU that tracks sysidle state, or if this
> * is an offline or the timekeeping CPU, nothing to do.
> */
> - if (!*isidle || rdp->rsp != rcu_sysidle_state ||
> + if (!*isidle || rdp->rsp != rcu_state_p ||
> cpu_is_offline(rdp->cpu) || rdp->cpu == tick_do_timer_cpu)
> return;
> if (rcu_gp_in_progress(rdp->rsp))
> @@ -2839,7 +2829,7 @@ static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,
> */
> static bool is_sysidle_rcu_state(struct rcu_state *rsp)
> {
> - return rsp == rcu_sysidle_state;
> + return rsp == rcu_state_p;
> }
>
> /*
> @@ -2917,7 +2907,7 @@ static void rcu_sysidle_cancel(void)
> static void rcu_sysidle_report(struct rcu_state *rsp, int isidle,
> unsigned long maxj, bool gpkt)
> {
> - if (rsp != rcu_sysidle_state)
> + if (rsp != rcu_state_p)
> return; /* Wrong flavor, ignore. */
> if (gpkt && nr_cpu_ids <= CONFIG_NO_HZ_FULL_SYSIDLE_SMALL)
> return; /* Running state machine from timekeeping CPU. */
> @@ -2986,13 +2976,12 @@ bool rcu_sys_is_idle(void)
>
> /* Scan all the CPUs looking for nonidle CPUs. */
> for_each_possible_cpu(cpu) {
> - rdp = per_cpu_ptr(rcu_sysidle_state->rda, cpu);
> + rdp = per_cpu_ptr(rcu_state_p->rda, cpu);
> rcu_sysidle_check_cpu(rdp, &isidle, &maxj);
> if (!isidle)
> break;
> }
> - rcu_sysidle_report(rcu_sysidle_state,
> - isidle, maxj, false);
> + rcu_sysidle_report(rcu_state_p, isidle, maxj, false);
> oldrss = rss;
> rss = ACCESS_ONCE(full_sysidle_state);
> }
> @@ -3019,7 +3008,7 @@ bool rcu_sys_is_idle(void)
> * provided by the memory allocator.
> */
> if (nr_cpu_ids > CONFIG_NO_HZ_FULL_SYSIDLE_SMALL &&
> - !rcu_gp_in_progress(rcu_sysidle_state) &&
> + !rcu_gp_in_progress(rcu_state_p) &&
> !rsh.inuse && xchg(&rsh.inuse, 1) == 0)
> call_rcu(&rsh.rh, rcu_sysidle_cb);
> return false;
>
--
Pranith
On Wed, Jul 23, 2014 at 09:28:52AM -0700, Paul E. McKenney wrote:
> Now that we have rcu_state_p, which references rcu_preempt_state for
> TREE_PREEMPT_RCU and rcu_sched_state for TREE_RCU, we don't need a
> separate rcu_sysidle_state variable. This commit therefore eliminates
> rcu_preempt_state in favor of rcu_state_p.
>
> Signed-off-by: Paul E. McKenney <[email protected]>
Acked-by: Frederic Weisbecker <[email protected]>
>
> diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h
> index 24f5bd54ca9b..e31fa71c4d3a 100644
> --- a/kernel/rcu/tree_plugin.h
> +++ b/kernel/rcu/tree_plugin.h
> @@ -2662,16 +2662,6 @@ static void __maybe_unused rcu_kick_nohz_cpu(int cpu)
>
> #ifdef CONFIG_NO_HZ_FULL_SYSIDLE
>
> -/*
> - * Define RCU flavor that holds sysidle state. This needs to be the
> - * most active flavor of RCU.
> - */
> -#ifdef CONFIG_PREEMPT_RCU
> -static struct rcu_state *rcu_sysidle_state = &rcu_preempt_state;
> -#else /* #ifdef CONFIG_PREEMPT_RCU */
> -static struct rcu_state *rcu_sysidle_state = &rcu_sched_state;
> -#endif /* #else #ifdef CONFIG_PREEMPT_RCU */
> -
> static int full_sysidle_state; /* Current system-idle state. */
> #define RCU_SYSIDLE_NOT 0 /* Some CPU is not idle. */
> #define RCU_SYSIDLE_SHORT 1 /* All CPUs idle for brief period. */
> @@ -2813,7 +2803,7 @@ static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,
> * not the flavor of RCU that tracks sysidle state, or if this
> * is an offline or the timekeeping CPU, nothing to do.
> */
> - if (!*isidle || rdp->rsp != rcu_sysidle_state ||
> + if (!*isidle || rdp->rsp != rcu_state_p ||
> cpu_is_offline(rdp->cpu) || rdp->cpu == tick_do_timer_cpu)
> return;
> if (rcu_gp_in_progress(rdp->rsp))
> @@ -2839,7 +2829,7 @@ static void rcu_sysidle_check_cpu(struct rcu_data *rdp, bool *isidle,
> */
> static bool is_sysidle_rcu_state(struct rcu_state *rsp)
> {
> - return rsp == rcu_sysidle_state;
> + return rsp == rcu_state_p;
> }
>
> /*
> @@ -2917,7 +2907,7 @@ static void rcu_sysidle_cancel(void)
> static void rcu_sysidle_report(struct rcu_state *rsp, int isidle,
> unsigned long maxj, bool gpkt)
> {
> - if (rsp != rcu_sysidle_state)
> + if (rsp != rcu_state_p)
> return; /* Wrong flavor, ignore. */
> if (gpkt && nr_cpu_ids <= CONFIG_NO_HZ_FULL_SYSIDLE_SMALL)
> return; /* Running state machine from timekeeping CPU. */
> @@ -2986,13 +2976,12 @@ bool rcu_sys_is_idle(void)
>
> /* Scan all the CPUs looking for nonidle CPUs. */
> for_each_possible_cpu(cpu) {
> - rdp = per_cpu_ptr(rcu_sysidle_state->rda, cpu);
> + rdp = per_cpu_ptr(rcu_state_p->rda, cpu);
> rcu_sysidle_check_cpu(rdp, &isidle, &maxj);
> if (!isidle)
> break;
> }
> - rcu_sysidle_report(rcu_sysidle_state,
> - isidle, maxj, false);
> + rcu_sysidle_report(rcu_state_p, isidle, maxj, false);
> oldrss = rss;
> rss = ACCESS_ONCE(full_sysidle_state);
> }
> @@ -3019,7 +3008,7 @@ bool rcu_sys_is_idle(void)
> * provided by the memory allocator.
> */
> if (nr_cpu_ids > CONFIG_NO_HZ_FULL_SYSIDLE_SMALL &&
> - !rcu_gp_in_progress(rcu_sysidle_state) &&
> + !rcu_gp_in_progress(rcu_state_p) &&
> !rsh.inuse && xchg(&rsh.inuse, 1) == 0)
> call_rcu(&rsh.rh, rcu_sysidle_cb);
> return false;
>
On Wed, Jul 23, 2014 at 09:29:51AM -0700, Paul E. McKenney wrote:
> If there are no nohz_full= CPUs, then there is currently no reason to
> track sysidle state. This commit therefore short-circuits this state
> tracking if !tick_nohz_full_enabled().
>
> Note that these checks will need to be revisited if nohz_full= state
> can ever be changed at runtime.
>
> Signed-off-by: Paul E. McKenney <[email protected]>
Acked-by: Frederic Weisbecker <[email protected]>