Account bandwidth usage on the cfs_rq level versus the task_groups to which
they belong. Whether we are tracking bandwidht on a given cfs_rq is maintained
under cfs_rq->runtime_enabled.
cfs_rq's which belong to a bandwidth constrained task_group have their runtime
accounted via the update_curr() path, which withdraws bandwidth from the global
pool as desired. Updates involving the global pool are currently protected
under cfs_bandwidth->lock, local runtime is protected by rq->lock.
This patch only attempts to assign and track quota, no action is taken in the
case that cfs_rq->runtime_used exceeds cfs_rq->runtime_assigned.
Signed-off-by: Paul Turner <[email protected]>
Signed-off-by: Nikhil Rao <[email protected]>
Signed-off-by: Bharata B Rao <[email protected]>
---
include/linux/sched.h | 4 ++
kernel/sched.c | 2 +
kernel/sched_fair.c | 85 ++++++++++++++++++++++++++++++++++++++++++++++++--
kernel/sysctl.c | 8 ++++
4 files changed, 96 insertions(+), 3 deletions(-)
Index: tip/kernel/sched_fair.c
===================================================================
--- tip.orig/kernel/sched_fair.c
+++ tip/kernel/sched_fair.c
@@ -96,6 +96,15 @@ unsigned int __read_mostly sysctl_sched_
unsigned int sysctl_sched_cfs_bandwidth_consistent = 1;
#endif
+#ifdef CONFIG_CFS_BANDWIDTH
+/*
+ * amount of quota to allocate from global tg to local cfs_rq pool on each
+ * refresh
+ * default: 5ms, units: microseconds
+ */
+unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL;
+#endif
+
static const struct sched_class fair_sched_class;
/**************************************************************
@@ -312,6 +321,8 @@ find_matching_se(struct sched_entity **s
#endif /* CONFIG_FAIR_GROUP_SCHED */
+static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq,
+ unsigned long delta_exec);
/**************************************************************
* Scheduling class tree data structure manipulation methods:
@@ -605,6 +616,8 @@ static void update_curr(struct cfs_rq *c
cpuacct_charge(curtask, delta_exec);
account_group_exec_runtime(curtask, delta_exec);
}
+
+ account_cfs_rq_runtime(cfs_rq, delta_exec);
}
static inline void
@@ -1277,10 +1290,68 @@ static inline u64 default_cfs_period(voi
return 500000000ULL;
}
+static inline u64 sched_cfs_bandwidth_slice(void)
+{
+ return (u64)sysctl_sched_cfs_bandwidth_slice * NSEC_PER_USEC;
+}
+
+static void assign_cfs_rq_runtime(struct cfs_rq *cfs_rq)
+{
+ struct task_group *tg = cfs_rq->tg;
+ struct cfs_bandwidth *cfs_b = tg_cfs_bandwidth(tg);
+ u64 amount = 0, min_amount;
+
+ /* note: this is a positive sum, runtime_remaining <= 0 */
+ min_amount = sched_cfs_bandwidth_slice() - cfs_rq->runtime_remaining;
+
+ raw_spin_lock(&cfs_b->lock);
+ if (cfs_b->quota == RUNTIME_INF)
+ amount = min_amount;
+ else if (cfs_b->runtime > 0) {
+ amount = min(cfs_b->runtime, min_amount);
+ cfs_b->runtime -= amount;
+ }
+ cfs_b->idle = 0;
+ raw_spin_unlock(&cfs_b->lock);
+
+ cfs_rq->runtime_remaining += amount;
+}
+
+static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq,
+ unsigned long delta_exec)
+{
+ if (!cfs_rq->runtime_enabled)
+ return;
+
+ cfs_rq->runtime_remaining -= delta_exec;
+ if (cfs_rq->runtime_remaining > 0)
+ return;
+
+ assign_cfs_rq_runtime(cfs_rq);
+}
+
static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun)
{
- return 1;
+ u64 quota, runtime = 0;
+ int idle = 0;
+
+ raw_spin_lock(&cfs_b->lock);
+ quota = cfs_b->quota;
+
+ if (quota != RUNTIME_INF) {
+ runtime = quota;
+ cfs_b->runtime = runtime;
+
+ idle = cfs_b->idle;
+ cfs_b->idle = 1;
+ }
+ raw_spin_unlock(&cfs_b->lock);
+
+ return idle;
}
+#else
+static void account_cfs_rq_runtime(struct cfs_rq *cfs_rq,
+ unsigned long delta_exec) {}
#endif
/**************************************************
@@ -4222,8 +4293,16 @@ static void set_curr_task_fair(struct rq
{
struct sched_entity *se = &rq->curr->se;
- for_each_sched_entity(se)
- set_next_entity(cfs_rq_of(se), se);
+ for_each_sched_entity(se) {
+ struct cfs_rq *cfs_rq = cfs_rq_of(se);
+
+ set_next_entity(cfs_rq, se);
+ /*
+ * if bandwidth is enabled, make sure it is up-to-date or
+ * reschedule for the case of a move into a throttled cpu.
+ */
+ account_cfs_rq_runtime(cfs_rq, 0);
+ }
}
#ifdef CONFIG_FAIR_GROUP_SCHED
Index: tip/kernel/sysctl.c
===================================================================
--- tip.orig/kernel/sysctl.c
+++ tip/kernel/sysctl.c
@@ -377,6 +377,14 @@ static struct ctl_table kern_table[] = {
.extra1 = &zero,
.extra2 = &one,
},
+ {
+ .procname = "sched_cfs_bandwidth_slice_us",
+ .data = &sysctl_sched_cfs_bandwidth_slice,
+ .maxlen = sizeof(unsigned int),
+ .mode = 0644,
+ .proc_handler = proc_dointvec_minmax,
+ .extra1 = &one,
+ },
#endif
#ifdef CONFIG_SCHED_AUTOGROUP
{
Index: tip/include/linux/sched.h
===================================================================
--- tip.orig/include/linux/sched.h
+++ tip/include/linux/sched.h
@@ -1958,6 +1958,10 @@ int sched_cfs_consistent_handler(struct
loff_t *ppos);
#endif
+#ifdef CONFIG_CFS_BANDWIDTH
+extern unsigned int sysctl_sched_cfs_bandwidth_slice;
+#endif
+
#ifdef CONFIG_SCHED_AUTOGROUP
extern unsigned int sysctl_sched_autogroup_enabled;
Index: tip/kernel/sched.c
===================================================================
--- tip.orig/kernel/sched.c
+++ tip/kernel/sched.c
@@ -252,6 +252,7 @@ struct cfs_bandwidth {
raw_spinlock_t lock;
ktime_t period;
u64 quota;
+ u64 runtime;
s64 hierarchal_quota;
int idle;
@@ -426,6 +427,7 @@ static enum hrtimer_restart sched_cfs_pe
static void init_cfs_bandwidth(struct cfs_bandwidth *cfs_b)
{
raw_spin_lock_init(&cfs_b->lock);
+ cfs_b->runtime = 0;
cfs_b->quota = RUNTIME_INF;
cfs_b->period = ns_to_ktime(default_cfs_period());
(2011/05/03 18:28), Paul Turner wrote:
> Index: tip/include/linux/sched.h
> ===================================================================
> --- tip.orig/include/linux/sched.h
> +++ tip/include/linux/sched.h
> @@ -1958,6 +1958,10 @@ int sched_cfs_consistent_handler(struct
> loff_t *ppos);
> #endif
>
> +#ifdef CONFIG_CFS_BANDWIDTH
> +extern unsigned int sysctl_sched_cfs_bandwidth_slice;
> +#endif
> +
> #ifdef CONFIG_SCHED_AUTOGROUP
> extern unsigned int sysctl_sched_autogroup_enabled;
>
Nit: you can reuse ifdef just above here.
+#ifdef CONFIG_CFS_BANDWIDTH
+extern unsigned int sysctl_sched_cfs_bandwidth_consistent;
+
+int sched_cfs_consistent_handler(struct ctl_table *table, int write,
+ void __user *buffer, size_t *lenp,
+ loff_t *ppos);
+#endif
+
+#ifdef CONFIG_CFS_BANDWIDTH
+extern unsigned int sysctl_sched_cfs_bandwidth_slice;
+#endif
+
Reviewed-by: Hidetoshi Seto <[email protected]>
Thanks,
H.Seto
On Tue, May 10, 2011 at 12:22 AM, Hidetoshi Seto
<[email protected]> wrote:
> (2011/05/03 18:28), Paul Turner wrote:
>> Index: tip/include/linux/sched.h
>> ===================================================================
>> --- tip.orig/include/linux/sched.h
>> +++ tip/include/linux/sched.h
>> @@ -1958,6 +1958,10 @@ int sched_cfs_consistent_handler(struct
>> ? ? ? ? ? ? ? loff_t *ppos);
>> ?#endif
>>
>> +#ifdef CONFIG_CFS_BANDWIDTH
>> +extern unsigned int sysctl_sched_cfs_bandwidth_slice;
>> +#endif
>> +
>> ?#ifdef CONFIG_SCHED_AUTOGROUP
>> ?extern unsigned int sysctl_sched_autogroup_enabled;
>>
>
> Nit: you can reuse ifdef just above here.
Thanks! I think this was actually a quilt-mis-merge when I was
shuffling the order of things around. Definitely makes sense to
combine them.
>
> +#ifdef CONFIG_CFS_BANDWIDTH
> +extern unsigned int sysctl_sched_cfs_bandwidth_consistent;
> +
> +int sched_cfs_consistent_handler(struct ctl_table *table, int write,
> + ? ? ? ? ? ? ? void __user *buffer, size_t *lenp,
> + ? ? ? ? ? ? ? loff_t *ppos);
> +#endif
> +
> +#ifdef CONFIG_CFS_BANDWIDTH
> +extern unsigned int sysctl_sched_cfs_bandwidth_slice;
> +#endif
> +
>
> Reviewed-by: Hidetoshi Seto <[email protected]>
>
>
> Thanks,
> H.Seto
>
>
On Tue, 2011-05-03 at 02:28 -0700, Paul Turner wrote:
> +unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL;
What happens when the period is smaller than the slice?
On Tue, 2011-05-03 at 02:28 -0700, Paul Turner wrote:
> static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int
> overrun)
> {
> - return 1;
> + u64 quota, runtime = 0;
> + int idle = 0;
> +
> + raw_spin_lock(&cfs_b->lock);
> + quota = cfs_b->quota;
> +
> + if (quota != RUNTIME_INF) {
> + runtime = quota;
> + cfs_b->runtime = runtime;
> +
> + idle = cfs_b->idle;
> + cfs_b->idle = 1;
> + }
> + raw_spin_unlock(&cfs_b->lock);
> +
> + return idle;
> }
Shouldn't that also return 'idle' when quota is INF? No point in keeping
that timer ticking when there's no actual accounting anymore.
On Mon, May 16, 2011 at 3:27 AM, Peter Zijlstra <[email protected]> wrote:
> On Tue, 2011-05-03 at 02:28 -0700, Paul Turner wrote:
>> +unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL;
>
> What happens when the period is smaller than the slice?
>
We'll always take at most whatever's left in this case.
On Mon, 2011-05-16 at 05:59 -0700, Paul Turner wrote:
> On Mon, May 16, 2011 at 3:27 AM, Peter Zijlstra <[email protected]> wrote:
> > On Tue, 2011-05-03 at 02:28 -0700, Paul Turner wrote:
> >> +unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL;
> >
> > What happens when the period is smaller than the slice?
> >
>
> We'll always take at most whatever's left in this case.
Right, saw that, but it might be good to have a little comment
explaining the interaction between the slice and the period things.
On Tue, May 17, 2011 at 8:28 AM, Peter Zijlstra <[email protected]> wrote:
> On Mon, 2011-05-16 at 05:59 -0700, Paul Turner wrote:
>> On Mon, May 16, 2011 at 3:27 AM, Peter Zijlstra <[email protected]> wrote:
>> > On Tue, 2011-05-03 at 02:28 -0700, Paul Turner wrote:
>> >> +unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL;
>> >
>> > What happens when the period is smaller than the slice?
>> >
>>
>> We'll always take at most whatever's left in this case.
>
> Right, saw that, but it might be good to have a little comment
> explaining the interaction between the slice and the period things.
>
Oh, sure -- easy enough :)