Current users of the rstat code can source root-level statistics from
the native counters of their respective subsystem, allowing them to
forego aggregation at the root level. This optimization is currently
implemented inside the generic rstat code, which doesn't track the
root cgroup and doesn't invoke the subsystem flush callbacks on it.
However, the memory controller cannot do this optimization, because
cgroup1 breaks out memory specifically for the local level, including
at the root level. In preparation for the memory controller switching
to rstat, move the optimization from rstat core to the controllers.
Afterwards, rstat will always track the root cgroup for changes and
invoke the subsystem callbacks on it; and it's up to the subsystem to
special-case and skip aggregation of the root cgroup if it can source
this information through other, cheaper means.
The extra cost of tracking the root cgroup is negligible: on stat
changes, we actually remove a branch that checks for the root. The
queueing for a flush touches only per-cpu data, and only the first
stat change since a flush requires a (per-cpu) lock.
Signed-off-by: Johannes Weiner <[email protected]>
---
block/blk-cgroup.c | 14 +++++++---
kernel/cgroup/rstat.c | 60 +++++++++++++++++++++++++------------------
2 files changed, 45 insertions(+), 29 deletions(-)
diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c
index 02ce2058c14b..76725e1cad7f 100644
--- a/block/blk-cgroup.c
+++ b/block/blk-cgroup.c
@@ -766,6 +766,10 @@ static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu)
struct blkcg *blkcg = css_to_blkcg(css);
struct blkcg_gq *blkg;
+ /* Root-level stats are sourced from system-wide IO stats */
+ if (!cgroup_parent(css->cgroup))
+ return;
+
rcu_read_lock();
hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
@@ -789,6 +793,7 @@ static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu)
u64_stats_update_end(&blkg->iostat.sync);
/* propagate global delta to parent */
+ /* XXX: could skip this if parent is root */
if (parent) {
u64_stats_update_begin(&parent->iostat.sync);
blkg_iostat_set(&delta, &blkg->iostat.cur);
@@ -803,10 +808,11 @@ static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu)
}
/*
- * The rstat algorithms intentionally don't handle the root cgroup to avoid
- * incurring overhead when no cgroups are defined. For that reason,
- * cgroup_rstat_flush in blkcg_print_stat does not actually fill out the
- * iostat in the root cgroup's blkcg_gq.
+ * We source root cgroup stats from the system-wide stats to avoid
+ * tracking the same information twice and incurring overhead when no
+ * cgroups are defined. For that reason, cgroup_rstat_flush in
+ * blkcg_print_stat does not actually fill out the iostat in the root
+ * cgroup's blkcg_gq.
*
* However, we would like to re-use the printing code between the root and
* non-root cgroups to the extent possible. For that reason, we simulate
diff --git a/kernel/cgroup/rstat.c b/kernel/cgroup/rstat.c
index faa767a870ba..6f50c199bf2a 100644
--- a/kernel/cgroup/rstat.c
+++ b/kernel/cgroup/rstat.c
@@ -25,13 +25,8 @@ static struct cgroup_rstat_cpu *cgroup_rstat_cpu(struct cgroup *cgrp, int cpu)
void cgroup_rstat_updated(struct cgroup *cgrp, int cpu)
{
raw_spinlock_t *cpu_lock = per_cpu_ptr(&cgroup_rstat_cpu_lock, cpu);
- struct cgroup *parent;
unsigned long flags;
- /* nothing to do for root */
- if (!cgroup_parent(cgrp))
- return;
-
/*
* Speculative already-on-list test. This may race leading to
* temporary inaccuracies, which is fine.
@@ -46,10 +41,10 @@ void cgroup_rstat_updated(struct cgroup *cgrp, int cpu)
raw_spin_lock_irqsave(cpu_lock, flags);
/* put @cgrp and all ancestors on the corresponding updated lists */
- for (parent = cgroup_parent(cgrp); parent;
- cgrp = parent, parent = cgroup_parent(cgrp)) {
+ while (true) {
struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);
- struct cgroup_rstat_cpu *prstatc = cgroup_rstat_cpu(parent, cpu);
+ struct cgroup *parent = cgroup_parent(cgrp);
+ struct cgroup_rstat_cpu *prstatc;
/*
* Both additions and removals are bottom-up. If a cgroup
@@ -58,8 +53,16 @@ void cgroup_rstat_updated(struct cgroup *cgrp, int cpu)
if (rstatc->updated_next)
break;
+ if (!parent) {
+ rstatc->updated_next = cgrp;
+ break;
+ }
+
+ prstatc = cgroup_rstat_cpu(parent, cpu);
rstatc->updated_next = prstatc->updated_children;
prstatc->updated_children = cgrp;
+
+ cgrp = parent;
}
raw_spin_unlock_irqrestore(cpu_lock, flags);
@@ -113,23 +116,26 @@ static struct cgroup *cgroup_rstat_cpu_pop_updated(struct cgroup *pos,
*/
if (rstatc->updated_next) {
struct cgroup *parent = cgroup_parent(pos);
- struct cgroup_rstat_cpu *prstatc = cgroup_rstat_cpu(parent, cpu);
- struct cgroup_rstat_cpu *nrstatc;
- struct cgroup **nextp;
-
- nextp = &prstatc->updated_children;
- while (true) {
- nrstatc = cgroup_rstat_cpu(*nextp, cpu);
- if (*nextp == pos)
- break;
-
- WARN_ON_ONCE(*nextp == parent);
- nextp = &nrstatc->updated_next;
+
+ if (parent) {
+ struct cgroup_rstat_cpu *prstatc;
+ struct cgroup **nextp;
+
+ prstatc = cgroup_rstat_cpu(parent, cpu);
+ nextp = &prstatc->updated_children;
+ while (true) {
+ struct cgroup_rstat_cpu *nrstatc;
+
+ nrstatc = cgroup_rstat_cpu(*nextp, cpu);
+ if (*nextp == pos)
+ break;
+ WARN_ON_ONCE(*nextp == parent);
+ nextp = &nrstatc->updated_next;
+ }
+ *nextp = rstatc->updated_next;
}
- *nextp = rstatc->updated_next;
rstatc->updated_next = NULL;
-
return pos;
}
@@ -309,11 +315,15 @@ static void cgroup_base_stat_sub(struct cgroup_base_stat *dst_bstat,
static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu)
{
- struct cgroup *parent = cgroup_parent(cgrp);
struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);
+ struct cgroup *parent = cgroup_parent(cgrp);
struct cgroup_base_stat cur, delta;
unsigned seq;
+ /* Root-level stats are sourced from system-wide CPU stats */
+ if (!parent)
+ return;
+
/* fetch the current per-cpu values */
do {
seq = __u64_stats_fetch_begin(&rstatc->bsync);
@@ -326,8 +336,8 @@ static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu)
cgroup_base_stat_add(&cgrp->bstat, &delta);
cgroup_base_stat_add(&rstatc->last_bstat, &delta);
- /* propagate global delta to parent */
- if (parent) {
+ /* propagate global delta to parent (unless that's root) */
+ if (cgroup_parent(parent)) {
delta = cgrp->bstat;
cgroup_base_stat_sub(&delta, &cgrp->last_bstat);
cgroup_base_stat_add(&parent->bstat, &delta);
--
2.30.0
Hello,
On Fri, Feb 05, 2021 at 01:28:03PM -0500, Johannes Weiner wrote:
> Current users of the rstat code can source root-level statistics from
> the native counters of their respective subsystem, allowing them to
> forego aggregation at the root level. This optimization is currently
> implemented inside the generic rstat code, which doesn't track the
> root cgroup and doesn't invoke the subsystem flush callbacks on it.
>
> However, the memory controller cannot do this optimization, because
> cgroup1 breaks out memory specifically for the local level, including
> at the root level. In preparation for the memory controller switching
> to rstat, move the optimization from rstat core to the controllers.
>
> Afterwards, rstat will always track the root cgroup for changes and
> invoke the subsystem callbacks on it; and it's up to the subsystem to
> special-case and skip aggregation of the root cgroup if it can source
> this information through other, cheaper means.
>
> The extra cost of tracking the root cgroup is negligible: on stat
> changes, we actually remove a branch that checks for the root. The
> queueing for a flush touches only per-cpu data, and only the first
> stat change since a flush requires a (per-cpu) lock.
>
> Signed-off-by: Johannes Weiner <[email protected]>
Generally looks good to me.
Acked-by: Tejun Heo <[email protected]>
A couple nits below.
> diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c
> index 02ce2058c14b..76725e1cad7f 100644
> --- a/block/blk-cgroup.c
> +++ b/block/blk-cgroup.c
> @@ -766,6 +766,10 @@ static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu)
> struct blkcg *blkcg = css_to_blkcg(css);
> struct blkcg_gq *blkg;
>
> + /* Root-level stats are sourced from system-wide IO stats */
> + if (!cgroup_parent(css->cgroup))
> + return;
> +
> rcu_read_lock();
>
> hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
> @@ -789,6 +793,7 @@ static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu)
> u64_stats_update_end(&blkg->iostat.sync);
>
> /* propagate global delta to parent */
> + /* XXX: could skip this if parent is root */
> if (parent) {
> u64_stats_update_begin(&parent->iostat.sync);
> blkg_iostat_set(&delta, &blkg->iostat.cur);
Might as well update this similar to cgroup_base_stat_flush()?
> @@ -58,8 +53,16 @@ void cgroup_rstat_updated(struct cgroup *cgrp, int cpu)
> if (rstatc->updated_next)
> break;
>
> + if (!parent) {
Maybe useful to note that the node is being marked busy but not added to the
non-existent parent.
> + rstatc->updated_next = cgrp;
> + break;
> + }
> +
> + prstatc = cgroup_rstat_cpu(parent, cpu);
> rstatc->updated_next = prstatc->updated_children;
> prstatc->updated_children = cgrp;
> +
> + cgrp = parent;
> }
>
> raw_spin_unlock_irqrestore(cpu_lock, flags);
...
> static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu)
> {
> - struct cgroup *parent = cgroup_parent(cgrp);
> struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);
> + struct cgroup *parent = cgroup_parent(cgrp);
Is this chunk intentional?
> struct cgroup_base_stat cur, delta;
> unsigned seq;
>
> + /* Root-level stats are sourced from system-wide CPU stats */
> + if (!parent)
> + return;
> +
> /* fetch the current per-cpu values */
> do {
> seq = __u64_stats_fetch_begin(&rstatc->bsync);
> @@ -326,8 +336,8 @@ static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu)
> cgroup_base_stat_add(&cgrp->bstat, &delta);
> cgroup_base_stat_add(&rstatc->last_bstat, &delta);
>
> - /* propagate global delta to parent */
> - if (parent) {
> + /* propagate global delta to parent (unless that's root) */
> + if (cgroup_parent(parent)) {
Yeah, this makes sense. Can you add a short while-at-it note in the patch
description?
Thanks.
--
tejun
On Fri, Feb 05, 2021 at 10:34:39PM -0500, Tejun Heo wrote:
> Hello,
>
> On Fri, Feb 05, 2021 at 01:28:03PM -0500, Johannes Weiner wrote:
> > Current users of the rstat code can source root-level statistics from
> > the native counters of their respective subsystem, allowing them to
> > forego aggregation at the root level. This optimization is currently
> > implemented inside the generic rstat code, which doesn't track the
> > root cgroup and doesn't invoke the subsystem flush callbacks on it.
> >
> > However, the memory controller cannot do this optimization, because
> > cgroup1 breaks out memory specifically for the local level, including
> > at the root level. In preparation for the memory controller switching
> > to rstat, move the optimization from rstat core to the controllers.
> >
> > Afterwards, rstat will always track the root cgroup for changes and
> > invoke the subsystem callbacks on it; and it's up to the subsystem to
> > special-case and skip aggregation of the root cgroup if it can source
> > this information through other, cheaper means.
> >
> > The extra cost of tracking the root cgroup is negligible: on stat
> > changes, we actually remove a branch that checks for the root. The
> > queueing for a flush touches only per-cpu data, and only the first
> > stat change since a flush requires a (per-cpu) lock.
> >
> > Signed-off-by: Johannes Weiner <[email protected]>
>
> Generally looks good to me.
>
> Acked-by: Tejun Heo <[email protected]>
Thanks!
> A couple nits below.
>
> > diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c
> > index 02ce2058c14b..76725e1cad7f 100644
> > --- a/block/blk-cgroup.c
> > +++ b/block/blk-cgroup.c
> > @@ -766,6 +766,10 @@ static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu)
> > struct blkcg *blkcg = css_to_blkcg(css);
> > struct blkcg_gq *blkg;
> >
> > + /* Root-level stats are sourced from system-wide IO stats */
> > + if (!cgroup_parent(css->cgroup))
> > + return;
> > +
> > rcu_read_lock();
> >
> > hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
> > @@ -789,6 +793,7 @@ static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu)
> > u64_stats_update_end(&blkg->iostat.sync);
> >
> > /* propagate global delta to parent */
> > + /* XXX: could skip this if parent is root */
> > if (parent) {
> > u64_stats_update_begin(&parent->iostat.sync);
> > blkg_iostat_set(&delta, &blkg->iostat.cur);
>
> Might as well update this similar to cgroup_base_stat_flush()?
I meant to revisit that, but I'm never 100% confident when it comes to
the interaction and lifetime of css, blkcg and blkg_gq.
IIUC, the blkg_gq->parent linkage always matches the css parent
linkage; it just exists as an optimization for ancestor walks, which
would otherwise have to do radix lookups when going through the css.
So with the cgroup_parent() check at the beginning of the function
making sure we're looking at a non-root group, blkg_gq->parent should
also never be NULL and I can do if (paren->parent) directly, right?
> > @@ -58,8 +53,16 @@ void cgroup_rstat_updated(struct cgroup *cgrp, int cpu)
> > if (rstatc->updated_next)
> > break;
> >
> > + if (!parent) {
>
> Maybe useful to note that the node is being marked busy but not added to the
> non-existent parent.
Makes sense, I'll add a comment.
> > + rstatc->updated_next = cgrp;
> > + break;
> > + }
> > +
> > + prstatc = cgroup_rstat_cpu(parent, cpu);
> > rstatc->updated_next = prstatc->updated_children;
> > prstatc->updated_children = cgrp;
> > +
> > + cgrp = parent;
> > }
> >
> > raw_spin_unlock_irqrestore(cpu_lock, flags);
> ...
> > static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu)
> > {
> > - struct cgroup *parent = cgroup_parent(cgrp);
> > struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);
> > + struct cgroup *parent = cgroup_parent(cgrp);
>
> Is this chunk intentional?
Yeah, it puts the local variable declarations into reverse christmas
tree ordering to make them a bit easier to read. It's a while-at-it
cleanup, mostly a force of habit. I can drop it if it bothers you.
> > struct cgroup_base_stat cur, delta;
> > unsigned seq;
> >
> > + /* Root-level stats are sourced from system-wide CPU stats */
> > + if (!parent)
> > + return;
> > +
> > /* fetch the current per-cpu values */
> > do {
> > seq = __u64_stats_fetch_begin(&rstatc->bsync);
> > @@ -326,8 +336,8 @@ static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu)
> > cgroup_base_stat_add(&cgrp->bstat, &delta);
> > cgroup_base_stat_add(&rstatc->last_bstat, &delta);
> >
> > - /* propagate global delta to parent */
> > - if (parent) {
> > + /* propagate global delta to parent (unless that's root) */
> > + if (cgroup_parent(parent)) {
>
> Yeah, this makes sense. Can you add a short while-at-it note in the patch
> description?
Will do.
Thanks for the review!
Hello,
On Mon, Feb 08, 2021 at 03:29:21PM -0500, Johannes Weiner wrote:
> > > @@ -789,6 +793,7 @@ static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu)
> > > u64_stats_update_end(&blkg->iostat.sync);
> > >
> > > /* propagate global delta to parent */
> > > + /* XXX: could skip this if parent is root */
> > > if (parent) {
> > > u64_stats_update_begin(&parent->iostat.sync);
> > > blkg_iostat_set(&delta, &blkg->iostat.cur);
> >
> > Might as well update this similar to cgroup_base_stat_flush()?
>
> I meant to revisit that, but I'm never 100% confident when it comes to
> the interaction and lifetime of css, blkcg and blkg_gq.
Yeah, it does get hairy.
> IIUC, the blkg_gq->parent linkage always matches the css parent
> linkage; it just exists as an optimization for ancestor walks, which
> would otherwise have to do radix lookups when going through the css.
But yes, at least this part is straight-forward.
> So with the cgroup_parent() check at the beginning of the function
> making sure we're looking at a non-root group, blkg_gq->parent should
> also never be NULL and I can do if (paren->parent) directly, right?
I think so.
> > > static void cgroup_base_stat_flush(struct cgroup *cgrp, int cpu)
> > > {
> > > - struct cgroup *parent = cgroup_parent(cgrp);
> > > struct cgroup_rstat_cpu *rstatc = cgroup_rstat_cpu(cgrp, cpu);
> > > + struct cgroup *parent = cgroup_parent(cgrp);
> >
> > Is this chunk intentional?
>
> Yeah, it puts the local variable declarations into reverse christmas
> tree ordering to make them a bit easier to read. It's a while-at-it
> cleanup, mostly a force of habit. I can drop it if it bothers you.
I don't mind either way. Was just wondering whether it was accidental.
Thanks.
--
tejun