Return-Path: Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1753163AbcD1KiZ (ORCPT ); Thu, 28 Apr 2016 06:38:25 -0400 Received: from mga02.intel.com ([134.134.136.20]:49305 "EHLO mga02.intel.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1753120AbcD1KiU (ORCPT ); Thu, 28 Apr 2016 06:38:20 -0400 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.24,546,1455004800"; d="scan'208";a="964534374" From: Yuyang Du To: peterz@infradead.org, mingo@kernel.org, linux-kernel@vger.kernel.org Cc: bsegall@google.com, pjt@google.com, morten.rasmussen@arm.com, vincent.guittot@linaro.org, dietmar.eggemann@arm.com, juri.lelli@arm.com, Yuyang Du Subject: [PATCH 2/6] sched/fair: Rename variable names for sched averages Date: Thu, 28 Apr 2016 10:56:09 +0800 Message-Id: <1461812173-32439-3-git-send-email-yuyang.du@intel.com> X-Mailer: git-send-email 1.7.9.5 In-Reply-To: <1461812173-32439-1-git-send-email-yuyang.du@intel.com> References: <1461812173-32439-1-git-send-email-yuyang.du@intel.com> Sender: linux-kernel-owner@vger.kernel.org List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Length: 21646 Lines: 551 The names of sched averages (including load_avg and util_avg) have been changed and added in the past a couple of years, some of the names are a bit confusing especially to people who first read them. This patch attempts to make the names more self-explaining. And some comments are updated too. Signed-off-by: Yuyang Du --- kernel/sched/fair.c | 192 ++++++++++++++++++++++++++------------------------- 1 file changed, 98 insertions(+), 94 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 6e0eec0..8d49276 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -660,13 +660,15 @@ static int select_idle_sibling(struct task_struct *p, int cpu); static unsigned long task_h_load(struct task_struct *p); /* - * We choose a half-life close to 1 scheduling period. - * Note: The tables runnable_avg_yN_inv and runnable_avg_yN_sum are - * dependent on this value. + * Note: everything in sched average calculation, including + * __decay_inv_multiply_N, __accumulated_sum_N, __accumulated_sum_N32, + * SCHED_AVG_MAX, and SCHED_AVG_MAX_N are all dependent on and only on + * (1) exponential decay, (2) a period of 1024*1024ns (~1ms), and (3) + * a half-life of 32 periods. */ -#define LOAD_AVG_PERIOD 32 -#define LOAD_AVG_MAX 47742 /* maximum possible load avg */ -#define LOAD_AVG_MAX_N 345 /* number of full periods to produce LOAD_AVG_MAX */ +#define SCHED_AVG_HALFLIFE 32 /* number of periods as a half-life */ +#define SCHED_AVG_MAX 47742 /* maximum possible sched avg */ +#define SCHED_AVG_MAX_N 345 /* number of full periods to produce SCHED_AVG_MAX */ /* Give new sched_entity start runnable values to heavy its load in infant time */ void init_entity_runnable_average(struct sched_entity *se) @@ -681,7 +683,7 @@ void init_entity_runnable_average(struct sched_entity *se) */ sa->period_contrib = 1023; sa->load_avg = scale_load_down(se->load.weight); - sa->load_sum = sa->load_avg * LOAD_AVG_MAX; + sa->load_sum = sa->load_avg * SCHED_AVG_MAX; /* * At this point, util_avg won't be used in select_task_rq_fair anyway */ @@ -731,7 +733,7 @@ void post_init_entity_util_avg(struct sched_entity *se) } else { sa->util_avg = cap; } - sa->util_sum = sa->util_avg * LOAD_AVG_MAX; + sa->util_sum = sa->util_avg * SCHED_AVG_MAX; } } @@ -1834,7 +1836,7 @@ static u64 numa_get_avg_runtime(struct task_struct *p, u64 *period) *period = now - p->last_task_numa_placement; } else { delta = p->se.avg.load_sum / p->se.load.weight; - *period = LOAD_AVG_MAX; + *period = SCHED_AVG_MAX; } p->last_sum_exec_runtime = runtime; @@ -2583,7 +2585,7 @@ static inline void update_cfs_shares(struct cfs_rq *cfs_rq) #ifdef CONFIG_SMP /* Precomputed fixed inverse multiplies for multiplication by y^n */ -static const u32 runnable_avg_yN_inv[] = { +static const u32 __decay_inv_multiply_N[] = { 0xffffffff, 0xfa83b2da, 0xf5257d14, 0xefe4b99a, 0xeac0c6e6, 0xe5b906e6, 0xe0ccdeeb, 0xdbfbb796, 0xd744fcc9, 0xd2a81d91, 0xce248c14, 0xc9b9bd85, 0xc5672a10, 0xc12c4cc9, 0xbd08a39e, 0xb8fbaf46, 0xb504f333, 0xb123f581, @@ -2596,7 +2598,7 @@ static const u32 runnable_avg_yN_inv[] = { * Precomputed \Sum y^k { 1<=k<=n }. These are floor(true_value) to prevent * over-estimates when re-combining. */ -static const u32 runnable_avg_yN_sum[] = { +static const u32 __accumulated_sum_N[] = { 0, 1002, 1982, 2941, 3880, 4798, 5697, 6576, 7437, 8279, 9103, 9909,10698,11470,12226,12966,13690,14398,15091,15769,16433,17082, 17718,18340,18949,19545,20128,20698,21256,21802,22336,22859,23371, @@ -2612,16 +2614,18 @@ static const u32 __accumulated_sum_N32[] = { }; /* - * Approximate: - * val * y^n, where y^32 ~= 0.5 (~1 scheduling period) + * val * y^n, where y^m ~= 0.5 + * + * n is the number of periods past; a period is ~1ms + * m is called half-life in exponential decay; here it is SCHED_AVG_HALFLIFE=32. */ -static __always_inline u64 decay_load(u64 val, u64 n) +static __always_inline u64 __decay_sum(u64 val, u64 n) { unsigned int local_n; if (!n) return val; - else if (unlikely(n > LOAD_AVG_PERIOD * 63)) + else if (unlikely(n > SCHED_AVG_HALFLIFE * 63)) return 0; /* after bounds checking we can collapse to 32-bit */ @@ -2634,36 +2638,36 @@ static __always_inline u64 decay_load(u64 val, u64 n) * * To achieve constant time decay_load. */ - if (unlikely(local_n >= LOAD_AVG_PERIOD)) { - val >>= local_n / LOAD_AVG_PERIOD; - local_n %= LOAD_AVG_PERIOD; + if (unlikely(local_n >= SCHED_AVG_HALFLIFE)) { + val >>= local_n / SCHED_AVG_HALFLIFE; + local_n %= SCHED_AVG_HALFLIFE; } - val = mul_u64_u32_shr(val, runnable_avg_yN_inv[local_n], 32); + val = mul_u64_u32_shr(val, __decay_inv_multiply_N[local_n], 32); return val; } /* - * For updates fully spanning n periods, the contribution to runnable - * average will be: \Sum 1024*y^n + * For updates fully spanning n periods, the accumulated contribution + * will be: \Sum 1024*y^n. * - * We can compute this reasonably efficiently by combining: - * y^PERIOD = 1/2 with precomputed \Sum 1024*y^n {for n = LOAD_AVG_MAX_N)) - return LOAD_AVG_MAX; + if (likely(n <= SCHED_AVG_HALFLIFE)) + return __accumulated_sum_N[n]; + else if (unlikely(n >= SCHED_AVG_MAX_N)) + return SCHED_AVG_MAX; - /* Since n < LOAD_AVG_MAX_N, n/LOAD_AVG_PERIOD < 11 */ - contrib = __accumulated_sum_N32[n>>5]; /* =n/LOAD_AVG_PERIOD */ - n %= LOAD_AVG_PERIOD; - contrib = decay_load(contrib, n); - return contrib + runnable_avg_yN_sum[n]; + /* Since n < SCHED_AVG_MAX_N, n/SCHED_AVG_HALFLIFE < 11 */ + contrib = __accumulated_sum_N32[n>>5]; /* =n/SCHED_AVG_HALFLIFE */ + n %= SCHED_AVG_HALFLIFE; + contrib = __decay_sum(contrib, n); + return contrib + __accumulated_sum_N[n]; } #if (SCHED_LOAD_SHIFT - SCHED_LOAD_RESOLUTION) != 10 || SCHED_CAPACITY_SHIFT != 10 @@ -2673,35 +2677,35 @@ static u32 __compute_runnable_contrib(u64 n) #define cap_scale(v, s) ((v)*(s) >> SCHED_CAPACITY_SHIFT) /* - * We can represent the historical contribution to runnable average as the - * coefficients of a geometric series. To do this we sub-divide our runnable - * history into segments of approximately 1ms (1024us); label the segment that - * occurred N-ms ago p_N, with p_0 corresponding to the current period, e.g. + * We can represent the historical contribution to sched average as the + * coefficients of a geometric series. To do this we divide the history + * into segments of approximately 1ms (1024*1024ns); label the segment that + * occurred N-1024us ago p_N, with p_0 corresponding to the current period, e.g. * * [<- 1024us ->|<- 1024us ->|<- 1024us ->| ... * p0 p1 p2 * (now) (~1ms ago) (~2ms ago) * - * Let u_i denote the fraction of p_i that the entity was runnable. + * Let u_i denote the fraction of p_i whose state (runnable/running) we count. * * We then designate the fractions u_i as our co-efficients, yielding the - * following representation of historical load: + * following representation of a sched metric: * u_0 + u_1*y + u_2*y^2 + u_3*y^3 + ... * - * We choose y based on the with of a reasonably scheduling period, fixing: - * y^32 = 0.5 + * We choose y based on a half-life of 32 periods (which is ~32ms): + * y^32 = 0.5 => y = (0.5)^(1/32) * - * This means that the contribution to load ~32ms ago (u_32) will be weighted - * approximately half as much as the contribution to load within the last ms - * (u_0). + * where 32 is the number of periods that a past period's contribution is + * halved. This means that the impact of a period every ~32ms ago will be + * as much as 50% of the previous value. * * When a period "rolls over" and we have new u_0`, multiplying the previous * sum again by y is sufficient to update: - * load_avg = u_0` + y*(u_0 + u_1*y + u_2*y^2 + ... ) - * = u_0 + u_1*y + u_2*y^2 + ... [re-labeling u_i --> u_{i+1}] + * avg = u_0` + y*(u_0 + u_1*y + u_2*y^2 + ... ) + * = u_0 + u_1*y + u_2*y^2 + ... [re-labeling u_i --> u_{i+1}] */ static __always_inline int -__update_load_avg(u64 now, int cpu, struct sched_avg *sa, +__update_sched_avg(u64 now, int cpu, struct sched_avg *sa, unsigned long weight, int running, struct cfs_rq *cfs_rq) { u64 delta, scaled_delta, periods; @@ -2762,15 +2766,15 @@ __update_load_avg(u64 now, int cpu, struct sched_avg *sa, periods = delta / 1024; delta %= 1024; - sa->load_sum = decay_load(sa->load_sum, periods + 1); + sa->load_sum = __decay_sum(sa->load_sum, periods + 1); if (cfs_rq) { cfs_rq->runnable_load_sum = - decay_load(cfs_rq->runnable_load_sum, periods + 1); + __decay_sum(cfs_rq->runnable_load_sum, periods + 1); } - sa->util_sum = decay_load((u64)(sa->util_sum), periods + 1); + sa->util_sum = __decay_sum((u64)(sa->util_sum), periods + 1); /* Efficiently calculate \sum (1..n_period) 1024*y^i */ - contrib = __compute_runnable_contrib(periods); + contrib = __accumulate_sum(periods); contrib = cap_scale(contrib, scale_freq); if (weight) { sa->load_sum += weight * contrib; @@ -2794,12 +2798,12 @@ __update_load_avg(u64 now, int cpu, struct sched_avg *sa, sa->period_contrib += delta; if (decayed) { - sa->load_avg = div_u64(sa->load_sum, LOAD_AVG_MAX); + sa->load_avg = div_u64(sa->load_sum, SCHED_AVG_MAX); if (cfs_rq) { cfs_rq->runnable_load_avg = - div_u64(cfs_rq->runnable_load_sum, LOAD_AVG_MAX); + div_u64(cfs_rq->runnable_load_sum, SCHED_AVG_MAX); } - sa->util_avg = sa->util_sum / LOAD_AVG_MAX; + sa->util_avg = sa->util_sum / SCHED_AVG_MAX; } return decayed; @@ -2867,7 +2871,7 @@ void set_task_rq_fair(struct sched_entity *se, p_last_update_time = prev->avg.last_update_time; n_last_update_time = next->avg.last_update_time; #endif - __update_load_avg(p_last_update_time, cpu_of(rq_of(prev)), + __update_sched_avg(p_last_update_time, cpu_of(rq_of(prev)), &se->avg, 0, 0, NULL); se->avg.last_update_time = n_last_update_time; } @@ -2879,7 +2883,7 @@ static inline void update_tg_load_avg(struct cfs_rq *cfs_rq, int force) {} static inline u64 cfs_rq_clock_task(struct cfs_rq *cfs_rq); /* Group cfs_rq's load_avg is used for task_h_load and update_cfs_share */ -static inline int update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq) +static inline int update_cfs_rq_sched_avg(u64 now, struct cfs_rq *cfs_rq) { struct sched_avg *sa = &cfs_rq->avg; int decayed, removed = 0; @@ -2887,17 +2891,17 @@ static inline int update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq) if (atomic_long_read(&cfs_rq->removed_load_avg)) { s64 r = atomic_long_xchg(&cfs_rq->removed_load_avg, 0); sa->load_avg = max_t(long, sa->load_avg - r, 0); - sa->load_sum = max_t(s64, sa->load_sum - r * LOAD_AVG_MAX, 0); + sa->load_sum = max_t(s64, sa->load_sum - r * SCHED_AVG_MAX, 0); removed = 1; } if (atomic_long_read(&cfs_rq->removed_util_avg)) { long r = atomic_long_xchg(&cfs_rq->removed_util_avg, 0); sa->util_avg = max_t(long, sa->util_avg - r, 0); - sa->util_sum = max_t(s32, sa->util_sum - r * LOAD_AVG_MAX, 0); + sa->util_sum = max_t(s32, sa->util_sum - r * SCHED_AVG_MAX, 0); } - decayed = __update_load_avg(now, cpu_of(rq_of(cfs_rq)), sa, + decayed = __update_sched_avg(now, cpu_of(rq_of(cfs_rq)), sa, scale_load_down(cfs_rq->load.weight), cfs_rq->curr != NULL, cfs_rq); #ifndef CONFIG_64BIT @@ -2909,7 +2913,7 @@ static inline int update_cfs_rq_load_avg(u64 now, struct cfs_rq *cfs_rq) } /* Update task and its cfs_rq load average */ -static inline void update_load_avg(struct sched_entity *se, int update_tg) +static inline void update_sched_avg(struct sched_entity *se, int update_tg) { struct cfs_rq *cfs_rq = cfs_rq_of(se); u64 now = cfs_rq_clock_task(cfs_rq); @@ -2920,11 +2924,11 @@ static inline void update_load_avg(struct sched_entity *se, int update_tg) * Track task load average for carrying it to new CPU after migrated, and * track group sched_entity load average for task_h_load calc in migration */ - __update_load_avg(now, cpu, &se->avg, + __update_sched_avg(now, cpu, &se->avg, se->on_rq * scale_load_down(se->load.weight), cfs_rq->curr == se, NULL); - if (update_cfs_rq_load_avg(now, cfs_rq) && update_tg) + if (update_cfs_rq_sched_avg(now, cfs_rq) && update_tg) update_tg_load_avg(cfs_rq, 0); if (cpu == smp_processor_id() && &rq->cfs == cfs_rq) { @@ -2951,7 +2955,7 @@ static inline void update_load_avg(struct sched_entity *se, int update_tg) } } -static void attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) +static void attach_entity_sched_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { if (!sched_feat(ATTACH_AGE_LOAD)) goto skip_aging; @@ -2961,7 +2965,7 @@ static void attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *s * have aged the average right before clearing @last_update_time. */ if (se->avg.last_update_time) { - __update_load_avg(cfs_rq->avg.last_update_time, cpu_of(rq_of(cfs_rq)), + __update_sched_avg(cfs_rq->avg.last_update_time, cpu_of(rq_of(cfs_rq)), &se->avg, 0, 0, NULL); /* @@ -2978,9 +2982,9 @@ skip_aging: cfs_rq->avg.util_sum += se->avg.util_sum; } -static void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) +static void detach_entity_sched_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { - __update_load_avg(cfs_rq->avg.last_update_time, cpu_of(rq_of(cfs_rq)), + __update_sched_avg(cfs_rq->avg.last_update_time, cpu_of(rq_of(cfs_rq)), &se->avg, se->on_rq * scale_load_down(se->load.weight), cfs_rq->curr == se, NULL); @@ -2992,7 +2996,7 @@ static void detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *s /* Add the load generated by se into cfs_rq's load average */ static inline void -enqueue_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) +enqueue_entity_sched_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { struct sched_avg *sa = &se->avg; u64 now = cfs_rq_clock_task(cfs_rq); @@ -3000,18 +3004,18 @@ enqueue_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) migrated = !sa->last_update_time; if (!migrated) { - __update_load_avg(now, cpu_of(rq_of(cfs_rq)), sa, + __update_sched_avg(now, cpu_of(rq_of(cfs_rq)), sa, se->on_rq * scale_load_down(se->load.weight), cfs_rq->curr == se, NULL); } - decayed = update_cfs_rq_load_avg(now, cfs_rq); + decayed = update_cfs_rq_sched_avg(now, cfs_rq); cfs_rq->runnable_load_avg += sa->load_avg; cfs_rq->runnable_load_sum += sa->load_sum; if (migrated) - attach_entity_load_avg(cfs_rq, se); + attach_entity_sched_avg(cfs_rq, se); if (decayed || migrated) update_tg_load_avg(cfs_rq, 0); @@ -3019,9 +3023,9 @@ enqueue_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) /* Remove the runnable load generated by se from cfs_rq's runnable load average */ static inline void -dequeue_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) +dequeue_entity_sched_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) { - update_load_avg(se, 1); + update_sched_avg(se, 1); cfs_rq->runnable_load_avg = max_t(long, cfs_rq->runnable_load_avg - se->avg.load_avg, 0); @@ -3054,7 +3058,7 @@ static inline u64 cfs_rq_last_update_time(struct cfs_rq *cfs_rq) * Task first catches up with cfs_rq, and then subtract * itself from the cfs_rq (task must be off the queue now). */ -void remove_entity_load_avg(struct sched_entity *se) +void remove_entity_sched_avg(struct sched_entity *se) { struct cfs_rq *cfs_rq = cfs_rq_of(se); u64 last_update_time; @@ -3068,7 +3072,7 @@ void remove_entity_load_avg(struct sched_entity *se) last_update_time = cfs_rq_last_update_time(cfs_rq); - __update_load_avg(last_update_time, cpu_of(rq_of(cfs_rq)), &se->avg, 0, 0, NULL); + __update_sched_avg(last_update_time, cpu_of(rq_of(cfs_rq)), &se->avg, 0, 0, NULL); atomic_long_add(se->avg.load_avg, &cfs_rq->removed_load_avg); atomic_long_add(se->avg.util_avg, &cfs_rq->removed_util_avg); } @@ -3087,17 +3091,17 @@ static int idle_balance(struct rq *this_rq); #else /* CONFIG_SMP */ -static inline void update_load_avg(struct sched_entity *se, int update_tg) {} +static inline void update_sched_avg(struct sched_entity *se, int update_tg) {} static inline void -enqueue_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {} +enqueue_entity_sched_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {} static inline void -dequeue_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {} -static inline void remove_entity_load_avg(struct sched_entity *se) {} +dequeue_entity_sched_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {} +static inline void remove_entity_sched_avg(struct sched_entity *se) {} static inline void -attach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {} +attach_entity_sched_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {} static inline void -detach_entity_load_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {} +detach_entity_sched_avg(struct cfs_rq *cfs_rq, struct sched_entity *se) {} static inline int idle_balance(struct rq *rq) { @@ -3257,7 +3261,7 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) if (renorm && !curr) se->vruntime += cfs_rq->min_vruntime; - enqueue_entity_load_avg(cfs_rq, se); + enqueue_entity_sched_avg(cfs_rq, se); account_entity_enqueue(cfs_rq, se); update_cfs_shares(cfs_rq); @@ -3336,7 +3340,7 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) * Update run-time statistics of the 'current'. */ update_curr(cfs_rq); - dequeue_entity_load_avg(cfs_rq, se); + dequeue_entity_sched_avg(cfs_rq, se); if (schedstat_enabled()) update_stats_dequeue(cfs_rq, se, flags); @@ -3416,7 +3420,7 @@ set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) if (schedstat_enabled()) update_stats_wait_end(cfs_rq, se); __dequeue_entity(cfs_rq, se); - update_load_avg(se, 1); + update_sched_avg(se, 1); } update_stats_curr_start(cfs_rq, se); @@ -3520,7 +3524,7 @@ static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev) /* Put 'current' back into the tree. */ __enqueue_entity(cfs_rq, prev); /* in !on_rq case, update occurred at dequeue */ - update_load_avg(prev, 0); + update_sched_avg(prev, 0); } cfs_rq->curr = NULL; } @@ -3536,7 +3540,7 @@ entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued) /* * Ensure that runnable average is periodically updated. */ - update_load_avg(curr, 1); + update_sched_avg(curr, 1); update_cfs_shares(cfs_rq); #ifdef CONFIG_SCHED_HRTICK @@ -4409,7 +4413,7 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) if (cfs_rq_throttled(cfs_rq)) break; - update_load_avg(se, 1); + update_sched_avg(se, 1); update_cfs_shares(cfs_rq); } @@ -4469,7 +4473,7 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) if (cfs_rq_throttled(cfs_rq)) break; - update_load_avg(se, 1); + update_sched_avg(se, 1); update_cfs_shares(cfs_rq); } @@ -5321,7 +5325,7 @@ static void migrate_task_rq_fair(struct task_struct *p) * will result in the wakee task is less decayed, but giving the wakee more * load sounds not bad. */ - remove_entity_load_avg(&p->se); + remove_entity_sched_avg(&p->se); /* Tell new CPU we are migrated */ p->se.avg.last_update_time = 0; @@ -5332,7 +5336,7 @@ static void migrate_task_rq_fair(struct task_struct *p) static void task_dead_fair(struct task_struct *p) { - remove_entity_load_avg(&p->se); + remove_entity_sched_avg(&p->se); } #endif /* CONFIG_SMP */ @@ -6213,7 +6217,7 @@ static void update_blocked_averages(int cpu) if (throttled_hierarchy(cfs_rq)) continue; - if (update_cfs_rq_load_avg(cfs_rq_clock_task(cfs_rq), cfs_rq)) + if (update_cfs_rq_sched_avg(cfs_rq_clock_task(cfs_rq), cfs_rq)) update_tg_load_avg(cfs_rq, 0); } raw_spin_unlock_irqrestore(&rq->lock, flags); @@ -6274,7 +6278,7 @@ static inline void update_blocked_averages(int cpu) raw_spin_lock_irqsave(&rq->lock, flags); update_rq_clock(rq); - update_cfs_rq_load_avg(cfs_rq_clock_task(cfs_rq), cfs_rq); + update_cfs_rq_sched_avg(cfs_rq_clock_task(cfs_rq), cfs_rq); raw_spin_unlock_irqrestore(&rq->lock, flags); } @@ -8302,7 +8306,7 @@ static void detach_task_cfs_rq(struct task_struct *p) } /* Catch up with the cfs_rq and remove our load when we leave */ - detach_entity_load_avg(cfs_rq, se); + detach_entity_sched_avg(cfs_rq, se); } static void attach_task_cfs_rq(struct task_struct *p) @@ -8319,7 +8323,7 @@ static void attach_task_cfs_rq(struct task_struct *p) #endif /* Synchronize task with its cfs_rq */ - attach_entity_load_avg(cfs_rq, se); + attach_entity_sched_avg(cfs_rq, se); if (!vruntime_normalized(p)) se->vruntime += cfs_rq->min_vruntime; @@ -8458,7 +8462,7 @@ void unregister_fair_sched_group(struct task_group *tg) for_each_possible_cpu(cpu) { if (tg->se[cpu]) - remove_entity_load_avg(tg->se[cpu]); + remove_entity_sched_avg(tg->se[cpu]); /* * Only empty task groups can be destroyed; so we can speculatively -- 1.7.9.5