This is V3.
V2 -> V3:
- use dummy wait node to eliminate cache misses during a full llist
scan to obtain a tail node;
- add CONFIG_RCU_SR_NORMAL_DEBUG_GP Kconfig for a GP incomplete debug;
- move synchronize_rcu()'s main control data under the rcu_state struct;
- add rcutree.rcu_normal_wake_from_gp parameter description to the
kernel-parameters.txt file;
- split functionality into several patches.
V2: https://lore.kernel.org/all/[email protected]/T/
V1: https://lore.kernel.org/lkml/[email protected]/T/
Neeraj Upadhyay (1):
rcu: Improve handling of synchronize_rcu() users
Uladzislau Rezki (Sony) (6):
rcu: Reduce synchronize_rcu() latency
rcu: Add a trace event for synchronize_rcu_normal()
doc: Add rcutree.rcu_normal_wake_from_gp to kernel-parameters.txt
rcu: Support direct wake-up of synchronize_rcu() users
rcu: Move sync related data to rcu_state structure
rcu: Add CONFIG_RCU_SR_NORMAL_DEBUG_GP
.../admin-guide/kernel-parameters.txt | 14 +
include/trace/events/rcu.h | 27 ++
kernel/rcu/Kconfig.debug | 12 +
kernel/rcu/tree.c | 354 +++++++++++++++++-
kernel/rcu/tree.h | 19 +
kernel/rcu/tree_exp.h | 2 +-
6 files changed, 426 insertions(+), 2 deletions(-)
--
2.39.2
Move synchronize_rcu() main control data under the rcu_state
structure. An access is done via "rcu_state" global variable.
Signed-off-by: Uladzislau Rezki (Sony) <[email protected]>
---
kernel/rcu/tree.c | 50 ++++++++++++++---------------------------------
kernel/rcu/tree.h | 19 ++++++++++++++++++
2 files changed, 34 insertions(+), 35 deletions(-)
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index 69663a6d5050..c0d3e46730e8 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -1384,19 +1384,6 @@ static void rcu_poll_gp_seq_end_unlocked(unsigned long *snap)
raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
}
-/*
- * A max threshold for synchronize_rcu() users which are
- * awaken directly by the rcu_gp_kthread(). Left part is
- * deferred to the main worker.
- */
-#define SR_MAX_USERS_WAKE_FROM_GP 5
-#define SR_NORMAL_GP_WAIT_HEAD_MAX 5
-
-struct sr_wait_node {
- atomic_t inuse;
- struct llist_node node;
-};
-
/*
* There is a single llist, which is used for handling
* synchronize_rcu() users' enqueued rcu_synchronize nodes.
@@ -1523,17 +1510,10 @@ struct sr_wait_node {
* +----------+ +--------+
*
*/
-static struct sr_normal_state {
- struct llist_head srs_next; /* request a GP users. */
- struct llist_node *srs_wait_tail; /* wait for GP users. */
- struct llist_node *srs_done_tail; /* ready for GP users. */
- struct sr_wait_node srs_wait_nodes[SR_NORMAL_GP_WAIT_HEAD_MAX];
-} sr;
-
static bool rcu_sr_is_wait_head(struct llist_node *node)
{
- return &(sr.srs_wait_nodes)[0].node <= node &&
- node <= &(sr.srs_wait_nodes)[SR_NORMAL_GP_WAIT_HEAD_MAX - 1].node;
+ return &(rcu_state.srs_wait_nodes)[0].node <= node &&
+ node <= &(rcu_state.srs_wait_nodes)[SR_NORMAL_GP_WAIT_HEAD_MAX - 1].node;
}
static struct llist_node *rcu_sr_get_wait_head(void)
@@ -1542,7 +1522,7 @@ static struct llist_node *rcu_sr_get_wait_head(void)
int i;
for (i = 0; i < SR_NORMAL_GP_WAIT_HEAD_MAX; i++) {
- sr_wn = &(sr.srs_wait_nodes)[i];
+ sr_wn = &(rcu_state.srs_wait_nodes)[i];
if (!atomic_cmpxchg_acquire(&sr_wn->inuse, 0, 1))
return &sr_wn->node;
@@ -1590,7 +1570,7 @@ static void rcu_sr_normal_gp_cleanup_work(struct work_struct *work)
* cannot execute concurrently by multiple kworkers,
* the done tail list manipulations are protected here.
*/
- done = smp_load_acquire(&sr.srs_done_tail);
+ done = smp_load_acquire(&rcu_state.srs_done_tail);
if (!done)
return;
@@ -1626,12 +1606,12 @@ static void rcu_sr_normal_gp_cleanup(void)
struct llist_node *wait_tail, *head, *rcu;
int done = 0;
- wait_tail = sr.srs_wait_tail;
+ wait_tail = rcu_state.srs_wait_tail;
if (wait_tail == NULL)
return;
- sr.srs_wait_tail = NULL;
- ASSERT_EXCLUSIVE_WRITER(sr.srs_wait_tail);
+ rcu_state.srs_wait_tail = NULL;
+ ASSERT_EXCLUSIVE_WRITER(rcu_state.srs_wait_tail);
WARN_ON_ONCE(!rcu_sr_is_wait_head(wait_tail));
head = wait_tail->next;
@@ -1662,8 +1642,8 @@ static void rcu_sr_normal_gp_cleanup(void)
}
// concurrent sr_normal_gp_cleanup work might observe this update.
- smp_store_release(&sr.srs_done_tail, wait_tail);
- ASSERT_EXCLUSIVE_WRITER(sr.srs_done_tail);
+ smp_store_release(&rcu_state.srs_done_tail, wait_tail);
+ ASSERT_EXCLUSIVE_WRITER(rcu_state.srs_done_tail);
if (wait_tail->next)
queue_work(system_highpri_wq, &sr_normal_gp_cleanup);
@@ -1678,7 +1658,7 @@ static bool rcu_sr_normal_gp_init(void)
struct llist_node *wait_head;
bool start_new_poll = false;
- first = READ_ONCE(sr.srs_next.first);
+ first = READ_ONCE(rcu_state.srs_next.first);
if (!first || rcu_sr_is_wait_head(first))
return start_new_poll;
@@ -1690,23 +1670,23 @@ static bool rcu_sr_normal_gp_init(void)
}
/* Inject a wait-dummy-node. */
- llist_add(wait_head, &sr.srs_next);
+ llist_add(wait_head, &rcu_state.srs_next);
/*
* A waiting list of rcu_synchronize nodes should be empty on
* this step, since a GP-kthread, rcu_gp_init() -> gp_cleanup(),
* rolls it over. If not, it is a BUG, warn a user.
*/
- WARN_ON_ONCE(sr.srs_wait_tail != NULL);
- sr.srs_wait_tail = wait_head;
- ASSERT_EXCLUSIVE_WRITER(sr.srs_wait_tail);
+ WARN_ON_ONCE(rcu_state.srs_wait_tail != NULL);
+ rcu_state.srs_wait_tail = wait_head;
+ ASSERT_EXCLUSIVE_WRITER(rcu_state.srs_wait_tail);
return start_new_poll;
}
static void rcu_sr_normal_add_req(struct rcu_synchronize *rs)
{
- llist_add((struct llist_node *) &rs->head, &sr.srs_next);
+ llist_add((struct llist_node *) &rs->head, &rcu_state.srs_next);
}
/*
diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
index 192536916f9a..f72166b5067a 100644
--- a/kernel/rcu/tree.h
+++ b/kernel/rcu/tree.h
@@ -316,6 +316,19 @@ do { \
__set_current_state(TASK_RUNNING); \
} while (0)
+/*
+ * A max threshold for synchronize_rcu() users which are
+ * awaken directly by the rcu_gp_kthread(). Left part is
+ * deferred to the main worker.
+ */
+#define SR_MAX_USERS_WAKE_FROM_GP 5
+#define SR_NORMAL_GP_WAIT_HEAD_MAX 5
+
+struct sr_wait_node {
+ atomic_t inuse;
+ struct llist_node node;
+};
+
/*
* RCU global state, including node hierarchy. This hierarchy is
* represented in "heap" form in a dense array. The root (first level)
@@ -397,6 +410,12 @@ struct rcu_state {
/* Synchronize offline with */
/* GP pre-initialization. */
int nocb_is_setup; /* nocb is setup from boot */
+
+ /* synchronize_rcu() part. */
+ struct llist_head srs_next; /* request a GP users. */
+ struct llist_node *srs_wait_tail; /* wait for GP users. */
+ struct llist_node *srs_done_tail; /* ready for GP users. */
+ struct sr_wait_node srs_wait_nodes[SR_NORMAL_GP_WAIT_HEAD_MAX];
};
/* Values for rcu_state structure's gp_flags field. */
--
2.39.2
This option enables additional debugging for detecting a grace
period incompletion for synchronize_rcu() users. If a GP is not
fully passed for any user, the warning message is emitted.
Signed-off-by: Uladzislau Rezki (Sony) <[email protected]>
---
kernel/rcu/Kconfig.debug | 12 ++++++++++++
kernel/rcu/tree.c | 7 +++++--
2 files changed, 17 insertions(+), 2 deletions(-)
diff --git a/kernel/rcu/Kconfig.debug b/kernel/rcu/Kconfig.debug
index 2984de629f74..3d44106ca1f0 100644
--- a/kernel/rcu/Kconfig.debug
+++ b/kernel/rcu/Kconfig.debug
@@ -143,4 +143,16 @@ config RCU_STRICT_GRACE_PERIOD
when looking for certain types of RCU usage bugs, for example,
too-short RCU read-side critical sections.
+config RCU_SR_NORMAL_DEBUG_GP
+ bool "Debug synchronize_rcu() callers for a grace period completion"
+ depends on DEBUG_KERNEL && RCU_EXPERT
+ default n
+ help
+ This option enables additional debugging for detecting a grace
+ period incompletion for synchronize_rcu() users. If a GP is not
+ fully passed for any user, the warning message is emitted.
+
+ Say Y here if you want to enable such debugging
+ Say N if you are unsure.
+
endmenu # "RCU Debugging"
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
index c0d3e46730e8..421bce4b8dd7 100644
--- a/kernel/rcu/tree.c
+++ b/kernel/rcu/tree.c
@@ -1547,7 +1547,8 @@ static void rcu_sr_normal_complete(struct llist_node *node)
(struct rcu_head *) node, struct rcu_synchronize, head);
unsigned long oldstate = (unsigned long) rs->head.func;
- WARN_ONCE(!rcu_gp_is_expedited() && !poll_state_synchronize_rcu(oldstate),
+ WARN_ONCE(IS_ENABLED(CONFIG_RCU_SR_NORMAL_DEBUG_GP) &&
+ !poll_state_synchronize_rcu(oldstate),
"A full grace period is not passed yet: %lu",
rcu_seq_diff(get_state_synchronize_rcu(), oldstate));
@@ -3822,7 +3823,9 @@ static void synchronize_rcu_normal(void)
* This code might be preempted, therefore take a GP
* snapshot before adding a request.
*/
- rs.head.func = (void *) get_state_synchronize_rcu();
+ if (IS_ENABLED(CONFIG_RCU_SR_NORMAL_DEBUG_GP))
+ rs.head.func = (void *) get_state_synchronize_rcu();
+
rcu_sr_normal_add_req(&rs);
/* Kick a GP and start waiting. */
--
2.39.2
This commit adds rcutree.rcu_normal_wake_from_gp description
to the kernel-parameters.txt file.
Signed-off-by: Uladzislau Rezki (Sony) <[email protected]>
---
Documentation/admin-guide/kernel-parameters.txt | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
index 0a1731a0f0ef..65bfbfb09522 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -4944,6 +4944,20 @@
this kernel boot parameter, forcibly setting it
to zero.
+ rcutree.rcu_normal_wake_from_gp= [KNL]
+ Reduces a latency of synchronize_rcu() call. This approach
+ maintains its own track of synchronize_rcu() callers, so it
+ does not interact with regular callbacks because it does not
+ use a call_rcu[_hurry]() path. Please note, this is for a
+ normal grace period.
+
+ How to enable it:
+
+ echo 1 > /sys/module/rcutree/parameters/rcu_normal_wake_from_gp
+ or pass a boot parameter "rcutree.rcu_normal_wake_from_gp=1"
+
+ Default is 0.
+
rcuscale.gp_async= [KNL]
Measure performance of asynchronous
grace-period primitives such as call_rcu().
--
2.39.2
On Tue, Nov 28, 2023 at 09:00:33AM +0100, Uladzislau Rezki (Sony) wrote:
> This option enables additional debugging for detecting a grace
> period incompletion for synchronize_rcu() users. If a GP is not
> fully passed for any user, the warning message is emitted.
>
> Signed-off-by: Uladzislau Rezki (Sony) <[email protected]>
Much better, thank you, as this avoids the possibility of false positives
in production. But to avoid potential bisection issues, could you please
fold this into the first patch?
Thanx, Paul
> ---
> kernel/rcu/Kconfig.debug | 12 ++++++++++++
> kernel/rcu/tree.c | 7 +++++--
> 2 files changed, 17 insertions(+), 2 deletions(-)
>
> diff --git a/kernel/rcu/Kconfig.debug b/kernel/rcu/Kconfig.debug
> index 2984de629f74..3d44106ca1f0 100644
> --- a/kernel/rcu/Kconfig.debug
> +++ b/kernel/rcu/Kconfig.debug
> @@ -143,4 +143,16 @@ config RCU_STRICT_GRACE_PERIOD
> when looking for certain types of RCU usage bugs, for example,
> too-short RCU read-side critical sections.
>
> +config RCU_SR_NORMAL_DEBUG_GP
> + bool "Debug synchronize_rcu() callers for a grace period completion"
> + depends on DEBUG_KERNEL && RCU_EXPERT
> + default n
> + help
> + This option enables additional debugging for detecting a grace
> + period incompletion for synchronize_rcu() users. If a GP is not
> + fully passed for any user, the warning message is emitted.
> +
> + Say Y here if you want to enable such debugging
> + Say N if you are unsure.
> +
> endmenu # "RCU Debugging"
> diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
> index c0d3e46730e8..421bce4b8dd7 100644
> --- a/kernel/rcu/tree.c
> +++ b/kernel/rcu/tree.c
> @@ -1547,7 +1547,8 @@ static void rcu_sr_normal_complete(struct llist_node *node)
> (struct rcu_head *) node, struct rcu_synchronize, head);
> unsigned long oldstate = (unsigned long) rs->head.func;
>
> - WARN_ONCE(!rcu_gp_is_expedited() && !poll_state_synchronize_rcu(oldstate),
> + WARN_ONCE(IS_ENABLED(CONFIG_RCU_SR_NORMAL_DEBUG_GP) &&
> + !poll_state_synchronize_rcu(oldstate),
> "A full grace period is not passed yet: %lu",
> rcu_seq_diff(get_state_synchronize_rcu(), oldstate));
>
> @@ -3822,7 +3823,9 @@ static void synchronize_rcu_normal(void)
> * This code might be preempted, therefore take a GP
> * snapshot before adding a request.
> */
> - rs.head.func = (void *) get_state_synchronize_rcu();
> + if (IS_ENABLED(CONFIG_RCU_SR_NORMAL_DEBUG_GP))
> + rs.head.func = (void *) get_state_synchronize_rcu();
> +
> rcu_sr_normal_add_req(&rs);
>
> /* Kick a GP and start waiting. */
> --
> 2.39.2
>
On Tue, Nov 28, 2023 at 09:00:29AM +0100, Uladzislau Rezki (Sony) wrote:
> This commit adds rcutree.rcu_normal_wake_from_gp description
> to the kernel-parameters.txt file.
>
> Signed-off-by: Uladzislau Rezki (Sony) <[email protected]>
Very good, but could you please fold this into the first patch, which
is the one that adds this kernel parameter?
Thanx, Paul
> ---
> Documentation/admin-guide/kernel-parameters.txt | 14 ++++++++++++++
> 1 file changed, 14 insertions(+)
>
> diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
> index 0a1731a0f0ef..65bfbfb09522 100644
> --- a/Documentation/admin-guide/kernel-parameters.txt
> +++ b/Documentation/admin-guide/kernel-parameters.txt
> @@ -4944,6 +4944,20 @@
> this kernel boot parameter, forcibly setting it
> to zero.
>
> + rcutree.rcu_normal_wake_from_gp= [KNL]
> + Reduces a latency of synchronize_rcu() call. This approach
> + maintains its own track of synchronize_rcu() callers, so it
> + does not interact with regular callbacks because it does not
> + use a call_rcu[_hurry]() path. Please note, this is for a
> + normal grace period.
> +
> + How to enable it:
> +
> + echo 1 > /sys/module/rcutree/parameters/rcu_normal_wake_from_gp
> + or pass a boot parameter "rcutree.rcu_normal_wake_from_gp=1"
> +
> + Default is 0.
> +
> rcuscale.gp_async= [KNL]
> Measure performance of asynchronous
> grace-period primitives such as call_rcu().
> --
> 2.39.2
>
On Tue, Nov 28, 2023 at 09:00:32AM +0100, Uladzislau Rezki (Sony) wrote:
> Move synchronize_rcu() main control data under the rcu_state
> structure. An access is done via "rcu_state" global variable.
>
> Signed-off-by: Uladzislau Rezki (Sony) <[email protected]>
Could we please fold this into the earlier patches? Your future self
might thank me. ;-)
Thanx, Paul
> ---
> kernel/rcu/tree.c | 50 ++++++++++++++---------------------------------
> kernel/rcu/tree.h | 19 ++++++++++++++++++
> 2 files changed, 34 insertions(+), 35 deletions(-)
>
> diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c
> index 69663a6d5050..c0d3e46730e8 100644
> --- a/kernel/rcu/tree.c
> +++ b/kernel/rcu/tree.c
> @@ -1384,19 +1384,6 @@ static void rcu_poll_gp_seq_end_unlocked(unsigned long *snap)
> raw_spin_unlock_irqrestore_rcu_node(rnp, flags);
> }
>
> -/*
> - * A max threshold for synchronize_rcu() users which are
> - * awaken directly by the rcu_gp_kthread(). Left part is
> - * deferred to the main worker.
> - */
> -#define SR_MAX_USERS_WAKE_FROM_GP 5
> -#define SR_NORMAL_GP_WAIT_HEAD_MAX 5
> -
> -struct sr_wait_node {
> - atomic_t inuse;
> - struct llist_node node;
> -};
> -
> /*
> * There is a single llist, which is used for handling
> * synchronize_rcu() users' enqueued rcu_synchronize nodes.
> @@ -1523,17 +1510,10 @@ struct sr_wait_node {
> * +----------+ +--------+
> *
> */
> -static struct sr_normal_state {
> - struct llist_head srs_next; /* request a GP users. */
> - struct llist_node *srs_wait_tail; /* wait for GP users. */
> - struct llist_node *srs_done_tail; /* ready for GP users. */
> - struct sr_wait_node srs_wait_nodes[SR_NORMAL_GP_WAIT_HEAD_MAX];
> -} sr;
> -
> static bool rcu_sr_is_wait_head(struct llist_node *node)
> {
> - return &(sr.srs_wait_nodes)[0].node <= node &&
> - node <= &(sr.srs_wait_nodes)[SR_NORMAL_GP_WAIT_HEAD_MAX - 1].node;
> + return &(rcu_state.srs_wait_nodes)[0].node <= node &&
> + node <= &(rcu_state.srs_wait_nodes)[SR_NORMAL_GP_WAIT_HEAD_MAX - 1].node;
> }
>
> static struct llist_node *rcu_sr_get_wait_head(void)
> @@ -1542,7 +1522,7 @@ static struct llist_node *rcu_sr_get_wait_head(void)
> int i;
>
> for (i = 0; i < SR_NORMAL_GP_WAIT_HEAD_MAX; i++) {
> - sr_wn = &(sr.srs_wait_nodes)[i];
> + sr_wn = &(rcu_state.srs_wait_nodes)[i];
>
> if (!atomic_cmpxchg_acquire(&sr_wn->inuse, 0, 1))
> return &sr_wn->node;
> @@ -1590,7 +1570,7 @@ static void rcu_sr_normal_gp_cleanup_work(struct work_struct *work)
> * cannot execute concurrently by multiple kworkers,
> * the done tail list manipulations are protected here.
> */
> - done = smp_load_acquire(&sr.srs_done_tail);
> + done = smp_load_acquire(&rcu_state.srs_done_tail);
> if (!done)
> return;
>
> @@ -1626,12 +1606,12 @@ static void rcu_sr_normal_gp_cleanup(void)
> struct llist_node *wait_tail, *head, *rcu;
> int done = 0;
>
> - wait_tail = sr.srs_wait_tail;
> + wait_tail = rcu_state.srs_wait_tail;
> if (wait_tail == NULL)
> return;
>
> - sr.srs_wait_tail = NULL;
> - ASSERT_EXCLUSIVE_WRITER(sr.srs_wait_tail);
> + rcu_state.srs_wait_tail = NULL;
> + ASSERT_EXCLUSIVE_WRITER(rcu_state.srs_wait_tail);
>
> WARN_ON_ONCE(!rcu_sr_is_wait_head(wait_tail));
> head = wait_tail->next;
> @@ -1662,8 +1642,8 @@ static void rcu_sr_normal_gp_cleanup(void)
> }
>
> // concurrent sr_normal_gp_cleanup work might observe this update.
> - smp_store_release(&sr.srs_done_tail, wait_tail);
> - ASSERT_EXCLUSIVE_WRITER(sr.srs_done_tail);
> + smp_store_release(&rcu_state.srs_done_tail, wait_tail);
> + ASSERT_EXCLUSIVE_WRITER(rcu_state.srs_done_tail);
>
> if (wait_tail->next)
> queue_work(system_highpri_wq, &sr_normal_gp_cleanup);
> @@ -1678,7 +1658,7 @@ static bool rcu_sr_normal_gp_init(void)
> struct llist_node *wait_head;
> bool start_new_poll = false;
>
> - first = READ_ONCE(sr.srs_next.first);
> + first = READ_ONCE(rcu_state.srs_next.first);
> if (!first || rcu_sr_is_wait_head(first))
> return start_new_poll;
>
> @@ -1690,23 +1670,23 @@ static bool rcu_sr_normal_gp_init(void)
> }
>
> /* Inject a wait-dummy-node. */
> - llist_add(wait_head, &sr.srs_next);
> + llist_add(wait_head, &rcu_state.srs_next);
>
> /*
> * A waiting list of rcu_synchronize nodes should be empty on
> * this step, since a GP-kthread, rcu_gp_init() -> gp_cleanup(),
> * rolls it over. If not, it is a BUG, warn a user.
> */
> - WARN_ON_ONCE(sr.srs_wait_tail != NULL);
> - sr.srs_wait_tail = wait_head;
> - ASSERT_EXCLUSIVE_WRITER(sr.srs_wait_tail);
> + WARN_ON_ONCE(rcu_state.srs_wait_tail != NULL);
> + rcu_state.srs_wait_tail = wait_head;
> + ASSERT_EXCLUSIVE_WRITER(rcu_state.srs_wait_tail);
>
> return start_new_poll;
> }
>
> static void rcu_sr_normal_add_req(struct rcu_synchronize *rs)
> {
> - llist_add((struct llist_node *) &rs->head, &sr.srs_next);
> + llist_add((struct llist_node *) &rs->head, &rcu_state.srs_next);
> }
>
> /*
> diff --git a/kernel/rcu/tree.h b/kernel/rcu/tree.h
> index 192536916f9a..f72166b5067a 100644
> --- a/kernel/rcu/tree.h
> +++ b/kernel/rcu/tree.h
> @@ -316,6 +316,19 @@ do { \
> __set_current_state(TASK_RUNNING); \
> } while (0)
>
> +/*
> + * A max threshold for synchronize_rcu() users which are
> + * awaken directly by the rcu_gp_kthread(). Left part is
> + * deferred to the main worker.
> + */
> +#define SR_MAX_USERS_WAKE_FROM_GP 5
> +#define SR_NORMAL_GP_WAIT_HEAD_MAX 5
> +
> +struct sr_wait_node {
> + atomic_t inuse;
> + struct llist_node node;
> +};
> +
> /*
> * RCU global state, including node hierarchy. This hierarchy is
> * represented in "heap" form in a dense array. The root (first level)
> @@ -397,6 +410,12 @@ struct rcu_state {
> /* Synchronize offline with */
> /* GP pre-initialization. */
> int nocb_is_setup; /* nocb is setup from boot */
> +
> + /* synchronize_rcu() part. */
> + struct llist_head srs_next; /* request a GP users. */
> + struct llist_node *srs_wait_tail; /* wait for GP users. */
> + struct llist_node *srs_done_tail; /* ready for GP users. */
> + struct sr_wait_node srs_wait_nodes[SR_NORMAL_GP_WAIT_HEAD_MAX];
> };
>
> /* Values for rcu_state structure's gp_flags field. */
> --
> 2.39.2
>
> On Tue, Nov 28, 2023 at 09:00:33AM +0100, Uladzislau Rezki (Sony) wrote:
> > This option enables additional debugging for detecting a grace
> > period incompletion for synchronize_rcu() users. If a GP is not
> > fully passed for any user, the warning message is emitted.
> >
> > Signed-off-by: Uladzislau Rezki (Sony) <[email protected]>
>
> Much better, thank you, as this avoids the possibility of false positives
> in production. But to avoid potential bisection issues, could you please
> fold this into the first patch?
>
Makes sense. I will fold it into the first one!
--
Uladzislau Rezki
On Tue, Dec 19, 2023 at 05:17:19PM -0800, Paul E. McKenney wrote:
> On Tue, Nov 28, 2023 at 09:00:29AM +0100, Uladzislau Rezki (Sony) wrote:
> > This commit adds rcutree.rcu_normal_wake_from_gp description
> > to the kernel-parameters.txt file.
> >
> > Signed-off-by: Uladzislau Rezki (Sony) <[email protected]>
>
> Very good, but could you please fold this into the first patch, which
> is the one that adds this kernel parameter?
>
No problem :)
--
Uladzislau Rezki
On Tue, Dec 19, 2023 at 05:47:47PM -0800, Paul E. McKenney wrote:
> On Tue, Nov 28, 2023 at 09:00:32AM +0100, Uladzislau Rezki (Sony) wrote:
> > Move synchronize_rcu() main control data under the rcu_state
> > structure. An access is done via "rcu_state" global variable.
> >
> > Signed-off-by: Uladzislau Rezki (Sony) <[email protected]>
>
> Could we please fold this into the earlier patches? Your future self
> might thank me. ;-)
>
OK. No problem, i will see which one is the most appropriate for this :)
--
Uladzislau Rezki