When a sibling is forced-idle to match the core-cookie; search for
matching tasks to fill the core.
Signed-off-by: Peter Zijlstra (Intel) <[email protected]>
---
include/linux/sched.h | 1
kernel/sched/core.c | 131 +++++++++++++++++++++++++++++++++++++++++++++++++-
kernel/sched/idle.c | 1
kernel/sched/sched.h | 6 ++
4 files changed, 138 insertions(+), 1 deletion(-)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -640,6 +640,7 @@ struct task_struct {
#ifdef CONFIG_SCHED_CORE
struct rb_node core_node;
unsigned long core_cookie;
+ unsigned int core_occupation;
#endif
#ifdef CONFIG_CGROUP_SCHED
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -198,6 +198,21 @@ struct task_struct *sched_core_find(stru
return match;
}
+struct task_struct *sched_core_next(struct task_struct *p, unsigned long cookie)
+{
+ struct rb_node *node = &p->core_node;
+
+ node = rb_next(node);
+ if (!node)
+ return NULL;
+
+ p = container_of(node, struct task_struct, core_node);
+ if (p->core_cookie != cookie)
+ return NULL;
+
+ return p;
+}
+
/*
* The static-key + stop-machine variable are needed such that:
*
@@ -3650,7 +3665,7 @@ pick_next_task(struct rq *rq, struct tas
struct task_struct *next, *max = NULL;
const struct sched_class *class;
const struct cpumask *smt_mask;
- int i, j, cpu;
+ int i, j, cpu, occ = 0;
if (!sched_core_enabled(rq))
return __pick_next_task(rq, prev, rf);
@@ -3741,6 +3756,9 @@ pick_next_task(struct rq *rq, struct tas
goto done;
}
+ if (!is_idle_task(p))
+ occ++;
+
rq_i->core_pick = p;
/*
@@ -3764,6 +3782,7 @@ pick_next_task(struct rq *rq, struct tas
cpu_rq(j)->core_pick = NULL;
}
+ occ = 1;
goto again;
}
}
@@ -3786,6 +3805,8 @@ next_class:;
WARN_ON_ONCE(!rq_i->core_pick);
+ rq_i->core_pick->core_occupation = occ;
+
if (i == cpu)
continue;
@@ -3801,6 +3822,114 @@ next_class:;
return next;
}
+static bool try_steal_cookie(int this, int that)
+{
+ struct rq *dst = cpu_rq(this), *src = cpu_rq(that);
+ struct task_struct *p;
+ unsigned long cookie;
+ bool success = false;
+
+ local_irq_disable();
+ double_rq_lock(dst, src);
+
+ cookie = dst->core->core_cookie;
+ if (!cookie)
+ goto unlock;
+
+ if (dst->curr != dst->idle)
+ goto unlock;
+
+ p = sched_core_find(src, cookie);
+ if (p == src->idle)
+ goto unlock;
+
+ do {
+ if (p == src->core_pick || p == src->curr)
+ goto next;
+
+ if (!cpumask_test_cpu(this, &p->cpus_allowed))
+ goto next;
+
+ if (p->core_occupation > dst->idle->core_occupation)
+ goto next;
+
+ p->on_rq = TASK_ON_RQ_MIGRATING;
+ deactivate_task(src, p, 0);
+ set_task_cpu(p, this);
+ activate_task(dst, p, 0);
+ p->on_rq = TASK_ON_RQ_QUEUED;
+
+ resched_curr(dst);
+
+ success = true;
+ break;
+
+next:
+ p = sched_core_next(p, cookie);
+ } while (p);
+
+unlock:
+ double_rq_unlock(dst, src);
+ local_irq_enable();
+
+ return success;
+}
+
+static bool steal_cookie_task(int cpu, struct sched_domain *sd)
+{
+ int i;
+
+ for_each_cpu_wrap(i, sched_domain_span(sd), cpu) {
+ if (i == cpu)
+ continue;
+
+ if (need_resched())
+ break;
+
+ if (try_steal_cookie(cpu, i))
+ return true;
+ }
+
+ return false;
+}
+
+static void sched_core_balance(struct rq *rq)
+{
+ struct sched_domain *sd;
+ int cpu = cpu_of(rq);
+
+ rcu_read_lock();
+ raw_spin_unlock_irq(rq_lockp(rq));
+ for_each_domain(cpu, sd) {
+ if (!(sd->flags & SD_LOAD_BALANCE))
+ break;
+
+ if (need_resched())
+ break;
+
+ if (steal_cookie_task(cpu, sd))
+ break;
+ }
+ raw_spin_lock_irq(rq_lockp(rq));
+ rcu_read_unlock();
+}
+
+static DEFINE_PER_CPU(struct callback_head, core_balance_head);
+
+void queue_core_balance(struct rq *rq)
+{
+ if (!sched_core_enabled(rq))
+ return;
+
+ if (!rq->core->core_cookie)
+ return;
+
+ if (!rq->nr_running) /* not forced idle */
+ return;
+
+ queue_balance_callback(rq, &per_cpu(core_balance_head, rq->cpu), sched_core_balance);
+}
+
#else /* !CONFIG_SCHED_CORE */
static struct task_struct *
--- a/kernel/sched/idle.c
+++ b/kernel/sched/idle.c
@@ -387,6 +387,7 @@ static void set_next_task_idle(struct rq
{
update_idle_core(rq);
schedstat_inc(rq->sched_goidle);
+ queue_core_balance(rq);
}
static struct task_struct *
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -1013,6 +1013,8 @@ static inline raw_spinlock_t *rq_lockp(s
return &rq->__lock;
}
+extern void queue_core_balance(struct rq *rq);
+
#else /* !CONFIG_SCHED_CORE */
static inline bool sched_core_enabled(struct rq *rq)
@@ -1025,6 +1027,10 @@ static inline raw_spinlock_t *rq_lockp(s
return &rq->__lock;
}
+static inline void queue_core_balance(struct rq *rq)
+{
+}
+
#endif /* CONFIG_SCHED_CORE */
#ifdef CONFIG_SCHED_SMT
Hi,
On 18/02/2019 16:56, Peter Zijlstra wrote:
[...]
> +static bool try_steal_cookie(int this, int that)
> +{
> + struct rq *dst = cpu_rq(this), *src = cpu_rq(that);
> + struct task_struct *p;
> + unsigned long cookie;
> + bool success = false;
> +
> + local_irq_disable();
> + double_rq_lock(dst, src);
> +
> + cookie = dst->core->core_cookie;
> + if (!cookie)
> + goto unlock;
> +
> + if (dst->curr != dst->idle)
> + goto unlock;
> +
> + p = sched_core_find(src, cookie);
> + if (p == src->idle)
> + goto unlock;
> +
> + do {
> + if (p == src->core_pick || p == src->curr)
> + goto next;
> +
> + if (!cpumask_test_cpu(this, &p->cpus_allowed))
> + goto next;
> +
> + if (p->core_occupation > dst->idle->core_occupation)
> + goto next;
> +
IIUC, we're trying to find/steal tasks matching the core_cookie from other
rqs because dst has been cookie-forced-idle.
If the p we find isn't running, what's the meaning of core_occupation?
I would have expected it to be 0, but we don't seem to be clearing it when
resetting the state in pick_next_task().
If it is running, we prevent the stealing if the core it's on is running
more matching tasks than the core of the pulling rq. It feels to me as if
that's a balancing tweak to try to cram as many matching tasks as possible
in a single core, so to me this reads as "don't steal my tasks if I'm
running more than you are, but I will steal tasks from you if I'm given
the chance". Is that correct?
[...]
On Thu, Feb 21, 2019 at 04:19:46PM +0000, Valentin Schneider wrote:
> Hi,
>
> On 18/02/2019 16:56, Peter Zijlstra wrote:
> [...]
> > +static bool try_steal_cookie(int this, int that)
> > +{
> > + struct rq *dst = cpu_rq(this), *src = cpu_rq(that);
> > + struct task_struct *p;
> > + unsigned long cookie;
> > + bool success = false;
> > +
> > + local_irq_disable();
> > + double_rq_lock(dst, src);
> > +
> > + cookie = dst->core->core_cookie;
> > + if (!cookie)
> > + goto unlock;
> > +
> > + if (dst->curr != dst->idle)
> > + goto unlock;
> > +
> > + p = sched_core_find(src, cookie);
> > + if (p == src->idle)
> > + goto unlock;
> > +
> > + do {
> > + if (p == src->core_pick || p == src->curr)
> > + goto next;
> > +
> > + if (!cpumask_test_cpu(this, &p->cpus_allowed))
> > + goto next;
> > +
> > + if (p->core_occupation > dst->idle->core_occupation)
> > + goto next;
> > +
>
> IIUC, we're trying to find/steal tasks matching the core_cookie from other
> rqs because dst has been cookie-forced-idle.
>
> If the p we find isn't running, what's the meaning of core_occupation?
> I would have expected it to be 0, but we don't seem to be clearing it when
> resetting the state in pick_next_task().
Indeed. We preserve the occupation from the last time around; it's not
perfect but its better than nothing.
Consider there's two groups; and we just happen to run the other group.
Then our occopation, being what it was last, is still accurate. When
next we run, we'll again get that many siblings together.
> If it is running, we prevent the stealing if the core it's on is running
> more matching tasks than the core of the pulling rq. It feels to me as if
> that's a balancing tweak to try to cram as many matching tasks as possible
> in a single core, so to me this reads as "don't steal my tasks if I'm
> running more than you are, but I will steal tasks from you if I'm given
> the chance". Is that correct?
Correct, otherwise an SMT4 with 5 tasks could end up ping-ponging the
one task forever.
Note that a further condition a little up the callchain from here only
does this stealing if the thread was forced-idle -- ie. it had something
to run anyway. So under the condition where there simple aren't enough
tasks to keep all siblings busy, we'll not compact just cause.
On Thu, Feb 21, 2019 at 05:41:46PM +0100, Peter Zijlstra wrote:
> On Thu, Feb 21, 2019 at 04:19:46PM +0000, Valentin Schneider wrote:
> > Hi,
> >
> > On 18/02/2019 16:56, Peter Zijlstra wrote:
> > [...]
> > > +static bool try_steal_cookie(int this, int that)
> > > +{
> > > + struct rq *dst = cpu_rq(this), *src = cpu_rq(that);
> > > + struct task_struct *p;
> > > + unsigned long cookie;
> > > + bool success = false;
> > > +
> > > + local_irq_disable();
> > > + double_rq_lock(dst, src);
> > > +
> > > + cookie = dst->core->core_cookie;
> > > + if (!cookie)
> > > + goto unlock;
> > > +
> > > + if (dst->curr != dst->idle)
> > > + goto unlock;
> > > +
> > > + p = sched_core_find(src, cookie);
> > > + if (p == src->idle)
> > > + goto unlock;
> > > +
> > > + do {
> > > + if (p == src->core_pick || p == src->curr)
> > > + goto next;
> > > +
> > > + if (!cpumask_test_cpu(this, &p->cpus_allowed))
> > > + goto next;
> > > +
> > > + if (p->core_occupation > dst->idle->core_occupation)
> > > + goto next;
> > > +
> >
> > IIUC, we're trying to find/steal tasks matching the core_cookie from other
> > rqs because dst has been cookie-forced-idle.
> >
> > If the p we find isn't running, what's the meaning of core_occupation?
> > I would have expected it to be 0, but we don't seem to be clearing it when
> > resetting the state in pick_next_task().
>
> Indeed. We preserve the occupation from the last time around; it's not
> perfect but its better than nothing.
>
> Consider there's two groups; and we just happen to run the other group.
> Then our occopation, being what it was last, is still accurate. When
> next we run, we'll again get that many siblings together.
>
> > If it is running, we prevent the stealing if the core it's on is running
> > more matching tasks than the core of the pulling rq. It feels to me as if
> > that's a balancing tweak to try to cram as many matching tasks as possible
> > in a single core, so to me this reads as "don't steal my tasks if I'm
> > running more than you are, but I will steal tasks from you if I'm given
> > the chance". Is that correct?
>
> Correct, otherwise an SMT4 with 5 tasks could end up ping-ponging the
> one task forever.
>
> Note that a further condition a little up the callchain from here only
> does this stealing if the thread was forced-idle -- ie. it had something
> to run anyway. So under the condition where there simple aren't enough
> tasks to keep all siblings busy, we'll not compact just cause.
Better example; it will avoid stealing a task from a full SMT2 core to
fill another.
On 21/02/2019 16:47, Peter Zijlstra wrote:
[...]
>>> IIUC, we're trying to find/steal tasks matching the core_cookie from other
>>> rqs because dst has been cookie-forced-idle.
>>>
>>> If the p we find isn't running, what's the meaning of core_occupation?
>>> I would have expected it to be 0, but we don't seem to be clearing it when
>>> resetting the state in pick_next_task().
>>
>> Indeed. We preserve the occupation from the last time around; it's not
>> perfect but its better than nothing.
>>
>> Consider there's two groups; and we just happen to run the other group.
>> Then our occopation, being what it was last, is still accurate. When
>> next we run, we'll again get that many siblings together.
>>
>>> If it is running, we prevent the stealing if the core it's on is running
>>> more matching tasks than the core of the pulling rq. It feels to me as if
>>> that's a balancing tweak to try to cram as many matching tasks as possible
>>> in a single core, so to me this reads as "don't steal my tasks if I'm
>>> running more than you are, but I will steal tasks from you if I'm given
>>> the chance". Is that correct?
>>
>> Correct, otherwise an SMT4 with 5 tasks could end up ping-ponging the
>> one task forever.
>>
Wouldn't we want to move some tasks in those cases? If we're going newidle
we're guaranteed to have a thread for that extra task.
So
if (p->core_occupation == cpumask_weight(cpu_smt_mask(that))
we could want to steal, overriding the occupation comparison
(we already have a (p == src->core_pick) abort before). Kind of feels like
CFS stealing that steals when nr_running > 1.
>> Note that a further condition a little up the callchain from here only
>> does this stealing if the thread was forced-idle -- ie. it had something
>> to run anyway. So under the condition where there simple aren't enough
>> tasks to keep all siblings busy, we'll not compact just cause.
>
> Better example; it will avoid stealing a task from a full SMT2 core to
> fill another.
>
Aye, that's the scenario I was thinking of.
Thanks for clearing things up.
On Fri, Feb 22, 2019 at 12:42 AM Peter Zijlstra <[email protected]> wrote:
>
> On Thu, Feb 21, 2019 at 04:19:46PM +0000, Valentin Schneider wrote:
> > Hi,
> >
> > On 18/02/2019 16:56, Peter Zijlstra wrote:
> > [...]
> > > +static bool try_steal_cookie(int this, int that)
> > > +{
> > > + struct rq *dst = cpu_rq(this), *src = cpu_rq(that);
> > > + struct task_struct *p;
> > > + unsigned long cookie;
> > > + bool success = false;
> > > +
> > > + local_irq_disable();
> > > + double_rq_lock(dst, src);
Here, should we check dst and src's rq status before lock their rq?
if src is idle, it could be in the progress of load balance already?
Thanks,
-Aubrey
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 3e3162f..a1e0a6f 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -3861,6 +3861,13 @@ static bool try_steal_cookie(int this, int that)
unsigned long cookie;
bool success = false;
+ /*
+ * Don't steal if src is idle or has only one runnable task,
+ * or dst has more than one runnable task
+ */
+ if (src->nr_running <= 1 || unlikely(dst->nr_running >= 1))
+ return false;
+
local_irq_disable();
double_rq_lock(dst, src);
On Thu, Apr 4, 2019 at 4:31 PM Aubrey Li <[email protected]> wrote:
>
> On Fri, Feb 22, 2019 at 12:42 AM Peter Zijlstra <[email protected]> wrote:
> >
> > On Thu, Feb 21, 2019 at 04:19:46PM +0000, Valentin Schneider wrote:
> > > Hi,
> > >
> > > On 18/02/2019 16:56, Peter Zijlstra wrote:
> > > [...]
> > > > +static bool try_steal_cookie(int this, int that)
> > > > +{
> > > > + struct rq *dst = cpu_rq(this), *src = cpu_rq(that);
> > > > + struct task_struct *p;
> > > > + unsigned long cookie;
> > > > + bool success = false;
> > > > +
> > > > + local_irq_disable();
> > > > + double_rq_lock(dst, src);
>
> Here, should we check dst and src's rq status before lock their rq?
> if src is idle, it could be in the progress of load balance already?
>
> Thanks,
> -Aubrey
>
> diff --git a/kernel/sched/core.c b/kernel/sched/core.c
> index 3e3162f..a1e0a6f 100644
> --- a/kernel/sched/core.c
> +++ b/kernel/sched/core.c
> @@ -3861,6 +3861,13 @@ static bool try_steal_cookie(int this, int that)
> unsigned long cookie;
> bool success = false;
>
> + /*
> + * Don't steal if src is idle or has only one runnable task,
> + * or dst has more than one runnable task
> + */
> + if (src->nr_running <= 1 || unlikely(dst->nr_running >= 1))
> + return false;
> +
> local_irq_disable();
> double_rq_lock(dst, src);
This seems to eliminate a hard lockup on my side.
Thanks,
-Aubrey
[ 122.961909] NMI watchdog: Watchdog detected hard LOCKUP on cpu 0
[ 122.961910] Modules linked in: ipt_MASQUERADE xfrm_user xfrm_algo
iptable_nat nf_nat_ipv4 xt_addrtype iptable_filter ip_tables
xt_conntrack x_tables nf_nat nf_conntracki
[ 122.961940] irq event stamp: 8200
[ 122.961941] hardirqs last enabled at (8199): [<ffffffff81003829>]
trace_hardirqs_on_thunk+0x1a/0x1c
[ 122.961942] hardirqs last disabled at (8200): [<ffffffff81003845>]
trace_hardirqs_off_thunk+0x1a/0x1c
[ 122.961942] softirqs last enabled at (8192): [<ffffffff81e003a3>]
__do_softirq+0x3a3/0x3f2
[ 122.961943] softirqs last disabled at (8185): [<ffffffff81095ea1>]
irq_exit+0xc1/0xd0
[ 122.961944] CPU: 0 PID: 2704 Comm: schbench Tainted: G I
5.0.0-rc8-00544-gf24f5e9-dirty #20
[ 122.961945] Hardware name: Intel Corporation S2600CP/S2600CP, BIOS
SE5C600.86B.99.99.x058.082120120902 08/21/2012
[ 122.961945] RIP: 0010:native_queued_spin_lock_slowpath+0x5c/0x1d0
[ 122.961946] Code: ff ff ff 75 40 f0 0f ba 2f 08 0f 82 cd 00 00 00
8b 07 30 e4 09 c6 f7 c6 00 ff ff ff 75 1b 85 f6 74 0e 8b 07 84 c0 74
08 f3 90 <8b> 07 84 c0 75 f8 b8 05
[ 122.961947] RSP: 0000:ffff888c09c03e78 EFLAGS: 00000002
[ 122.961948] RAX: 0000000000740101 RBX: ffff888c0ade4400 RCX: 0000000000000540
[ 122.961948] RDX: 0000000000000002 RSI: 0000000000000001 RDI: ffff888c0ade4400
[ 122.961949] RBP: ffff888c0ade4400 R08: 0000000000000000 R09: 0000000000000001
[ 122.961950] R10: ffff888c09c03e20 R11: 0000000000000000 R12: 00000000001e4400
[ 122.961950] R13: 0000000000000000 R14: 0000000000000016 R15: ffff888be0340000
[ 122.961951] FS: 00007f21e17ea700(0000) GS:ffff888c09c00000(0000)
knlGS:0000000000000000
[ 122.961951] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 122.961952] CR2: 00007f21e17ea728 CR3: 0000000be0b6a002 CR4: 00000000000606f0
[ 122.961952] Call Trace:
[ 122.961953] <IRQ>
[ 122.961953] do_raw_spin_lock+0xb4/0xc0
[ 122.961954] _raw_spin_lock+0x4b/0x60
[ 122.961954] scheduler_tick+0x48/0x170
[ 122.961955] ? tick_sched_do_timer+0x60/0x60
[ 122.961955] update_process_times+0x40/0x50
[ 122.961956] tick_sched_handle+0x22/0x60
[ 122.961956] tick_sched_timer+0x37/0x70
[ 122.961957] __hrtimer_run_queues+0xed/0x3f0
[ 122.961957] hrtimer_interrupt+0x122/0x270
[ 122.961958] smp_apic_timer_interrupt+0x86/0x210
[ 122.961958] apic_timer_interrupt+0xf/0x20
[ 122.961959] </IRQ>
[ 122.961959] RIP: 0033:0x7fff855a2839
[ 122.961960] Code: 08 3b 15 6a c8 ff ff 75 df 31 c0 5d c3 b8 e4 00
00 00 5d 0f 05 c3 f3 90 eb ce 0f 1f 80 00 00 00 00 55 48 85 ff 48 89
e5 41 54 <49> 89 f4 53 74 30 48 8b
[ 122.961961] RSP: 002b:00007f21e17e9e38 EFLAGS: 00000206 ORIG_RAX:
ffffffffffffff13
[ 122.961962] RAX: 000000000002038b RBX: 00000000002dc6c0 RCX: 0000000000000000
[ 122.961962] RDX: 00007f21e17e9e60 RSI: 0000000000000000 RDI: 00007f21e17e9e50
[ 122.961963] RBP: 00007f21e17e9e40 R08: 0000000000000000 R09: 0000000000007a14
[ 122.961963] R10: 00007f21e17e9e30 R11: 0000000000000246 R12: 00007f21e17e9ed0
[ 122.961964] R13: 00007f2202716e6f R14: 0000000000000000 R15: 00007f21fc826b00
[ 122.961964] Kernel panic - not syncing: Hard LOCKUP
[ 122.961965] CPU: 0 PID: 2704 Comm: schbench Tainted: G I
5.0.0-rc8-00544-gf24f5e9-dirty #20
[ 122.961966] Hardware name: Intel Corporation S2600CP/S2600CP, BIOS
SE5C600.86B.99.99.x058.082120120902 08/21/2012
[ 122.961966] Call Trace:
[ 122.961967] <NMI>
[ 122.961967] dump_stack+0x7c/0xbb
[ 122.961968] panic+0x103/0x2c8
[ 122.961968] nmi_panic+0x35/0x40
[ 122.961969] watchdog_overflow_callback+0xfd/0x110
[ 122.961969] __perf_event_overflow+0x5a/0xe0
[ 122.961970] handle_pmi_common+0x1d1/0x280
[ 122.961970] ? intel_pmu_handle_irq+0xad/0x170
[ 122.961971] intel_pmu_handle_irq+0xad/0x170
[ 122.961971] perf_event_nmi_handler+0x2e/0x50
[ 122.961972] nmi_handle+0xc6/0x260
[ 122.961972] default_do_nmi+0xca/0x120
[ 122.961972] do_nmi+0x113/0x160
[ 122.961973] end_repeat_nmi+0x16/0x50
[ 122.961973] RIP: 0010:native_queued_spin_lock_slowpath+0x5c/0x1d0
[ 122.961974] Code: ff ff ff 75 40 f0 0f ba 2f 08 0f 82 cd 00 00 00
8b 07 30 e4 09 c6 f7 c6 00 ff ff ff 75 1b 85 f6 74 0e 8b 07 84 c0 74
08 f3 90 <8b> 07 84 c0 75 f8 b8 05
[ 122.961975] RSP: 0000:ffff888c09c03e78 EFLAGS: 00000002
[ 122.961976] RAX: 0000000000740101 RBX: ffff888c0ade4400 RCX: 0000000000000540
[ 122.961976] RDX: 0000000000000002 RSI: 0000000000000001 RDI: ffff888c0ade4400
[ 122.961977] RBP: ffff888c0ade4400 R08: 0000000000000000 R09: 0000000000000001
[ 122.961978] R10: ffff888c09c03e20 R11: 0000000000000000 R12: 00000000001e4400
[ 122.961978] R13: 0000000000000000 R14: 0000000000000016 R15: ffff888be0340000
[ 122.961979] ? native_queued_spin_lock_slowpath+0x5c/0x1d0
[ 122.961980] ? native_queued_spin_lock_slowpath+0x5c/0x1d0
[ 122.961980] </NMI>
[ 122.961981] <IRQ>
[ 122.961981] do_raw_spin_lock+0xb4/0xc0
[ 122.961982] _raw_spin_lock+0x4b/0x60
[ 122.961982] scheduler_tick+0x48/0x170
[ 122.961983] ? tick_sched_do_timer+0x60/0x60
[ 122.961983] update_process_times+0x40/0x50
[ 122.961984] tick_sched_handle+0x22/0x60
[ 122.961984] tick_sched_timer+0x37/0x70
[ 122.961985] __hrtimer_run_queues+0xed/0x3f0
[ 122.961985] hrtimer_interrupt+0x122/0x270
[ 122.961986] smp_apic_timer_interrupt+0x86/0x210
[ 122.961986] apic_timer_interrupt+0xf/0x20
[ 122.961987] </IRQ>
[ 122.961987] RIP: 0033:0x7fff855a2839
[ 122.961988] Code: 08 3b 15 6a c8 ff ff 75 df 31 c0 5d c3 b8 e4 00
00 00 5d 0f 05 c3 f3 90 eb ce 0f 1f 80 00 00 00 00 55 48 85 ff 48 89
e5 41 54 <49> 89 f4 53 74 30 48 8b
[ 122.961989] RSP: 002b:00007f21e17e9e38 EFLAGS: 00000206 ORIG_RAX:
ffffffffffffff13
[ 122.961990] RAX: 000000000002038b RBX: 00000000002dc6c0 RCX: 0000000000000000
[ 122.961990] RDX: 00007f21e17e9e60 RSI: 0000000000000000 RDI: 00007f21e17e9e50
[ 122.961991] RBP: 00007f21e17e9e40 R08: 0000000000000000 R09: 0000000000007a14
[ 122.961991] R10: 00007f21e17e9e30 R11: 0000000000000246 R12: 00007f21e17e9ed0
[ 122.961992] R13: 00007f2202716e6f R14: 0000000000000000 R15: 00007f21fc826b00
[ 124.022169] Shutting down cpus with NMI
[ 124.022170] Kernel Offset: disabled
[ 124.022170] NMI watchdog: Watchdog detected hard LOCKUP on cpu 1
[ 124.022171] Modules linked in: ipt_MASQUERADE xfrm_user xfrm_algo
iptable_nat nf_nat_ipv4 xt_addrtype iptable_filter ip_tables
xt_conntrack x_tables nf_nat nf_conntracki
[ 124.022200] irq event stamp: 148640
[ 124.022200] hardirqs last enabled at (148639):
[<ffffffff810c7544>] sched_core_balance+0x164/0x5d0
[ 124.022209] hardirqs last disabled at (148640):
[<ffffffff810c75b5>] sched_core_balance+0x1d5/0x5d0
[ 124.022218] softirqs last enabled at (148598):
[<ffffffff81e003a3>] __do_softirq+0x3a3/0x3f2
[ 124.022226] softirqs last disabled at (148563):
[<ffffffff81095ea1>] irq_exit+0xc1/0xd0
[ 124.022227] CPU: 1 PID: 0 Comm: swapper/1 Tainted: G I
5.0.0-rc8-00544-gf24f5e9-dirty #20
[ 124.022228] Hardware name: Intel Corporation S2600CP/S2600CP, BIOS
SE5C600.86B.99.99.x058.082120120902 08/21/2012
[ 124.022229] RIP: 0010:native_queued_spin_lock_slowpath+0x17e/0x1d0
[ 124.022230] Code: 34 c5 40 48 39 82 48 89 16 8b 42 08 85 c0 75 09
f3 90 8b 42 08 85 c0 74 f7 48 8b 32 48 85 f6 74 07 0f 18 0e eb 02 f3
90 8b 07 <66> 85 c0 75 f7 41 89 c7
[ 124.022230] RSP: 0000:ffffc9000634bcf8 EFLAGS: 00000002
[ 124.022231] RAX: 00000000001c0101 RBX: ffff888c0abe4400 RCX: 0000000000080000
[ 124.022232] RDX: ffff888c09fe5180 RSI: 0000000000000000 RDI: ffff888c0abe4400
[ 124.022232] RBP: ffff888c0abe4400 R08: 0000000000080000 R09: 0000000000000001
[ 124.022233] R10: ffffc9000634bca0 R11: 0000000000000000 R12: 0000000000000001
[ 124.022233] R13: 0000000000000000 R14: ffff888c09fe4400 R15: 00000000001e4400
[ 124.022234] FS: 0000000000000000(0000) GS:ffff888c09e00000(0000)
knlGS:0000000000000000
[ 124.022235] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 124.022235] CR2: 00007f21f1ffae00 CR3: 0000000be0b6a002 CR4: 00000000000606e0
[ 124.022236] Call Trace:
[ 124.022237] do_raw_spin_lock+0xb4/0xc0
[ 124.022237] _raw_spin_lock_nested+0x49/0x60
[ 124.022238] sched_core_balance+0x106/0x5d0
[ 124.022238] __balance_callback+0x49/0xa0
[ 124.022239] __schedule+0x12eb/0x1660
[ 124.022239] ? _raw_spin_unlock_irqrestore+0x4e/0x60
[ 124.022240] ? hrtimer_start_range_ns+0x1b7/0x340
[ 124.022240] schedule_idle+0x28/0x40
[ 124.022241] do_idle+0x169/0x2a0
[ 124.022241] cpu_startup_entry+0x19/0x20
[ 124.022242] start_secondary+0x17f/0x1d0
[ 124.022242] secondary_startup_64+0xa4/0xb0
[ 124.022243] NMI watchdog: Watchdog detected hard LOCKUP on cpu 2
[ 124.022243] Modules linked in: ipt_MASQUERADE xfrm_user xfrm_algo
iptable_nat nf_nat_ipv4 xt_addrtype iptable_filter ip_tables
xt_conntrack x_tables nf_nat nf_conntracki
[ 124.022272] irq event stamp: 154392
[ 124.022273] hardirqs last enabled at (154391):
[<ffffffff810c7544>] sched_core_balance+0x164/0x5d0
[ 124.022273] hardirqs last disabled at (154392):
[<ffffffff810c75b5>] sched_core_balance+0x1d5/0x5d0
[ 124.022274] softirqs last enabled at (154374):
[<ffffffff81e003a3>] __do_softirq+0x3a3/0x3f2
[ 124.022275] softirqs last disabled at (154309):
[<ffffffff81095ea1>] irq_exit+0xc1/0xd0
[ 124.022275] CPU: 2 PID: 0 Comm: swapper/2 Tainted: G I
5.0.0-rc8-00544-gf24f5e9-dirty #20
[ 124.022276] Hardware name: Intel Corporation S2600CP/S2600CP, BIOS
SE5C600.86B.99.99.x058.082120120902 08/21/2012
[ 124.022276] RIP: 0010:native_queued_spin_lock_slowpath+0x166/0x1d0
[ 124.022277] Code: c1 e8 12 48 c1 ee 0b 83 e8 01 83 e6 60 48 98 48
81 c6 80 51 1e 00 48 03 34 c5 40 48 39 82 48 89 16 8b 42 08 85 c0 75
09 f3 90 <8b> 42 08 85 c0 74 f7 40
[ 124.022278] RSP: 0000:ffffc90006353cf8 EFLAGS: 00000046
[ 124.022279] RAX: 0000000000000000 RBX: ffff888c0a3e4400 RCX: 00000000000c0000
[ 124.022279] RDX: ffff888c0a1e5180 RSI: ffff888c0a3e5180 RDI: ffff888c0a3e4400
[ 124.022280] RBP: ffff888c0a3e4400 R08: 00000000000c0000 R09: 0000000000000001
[ 124.022281] R10: ffffc90006353ca0 R11: 0000000000000000 R12: 0000000000000002
[ 124.022281] R13: 0000000000000000 R14: ffff888c0a1e4400 R15: 00000000001e4400
[ 124.022282] FS: 0000000000000000(0000) GS:ffff888c0a000000(0000)
knlGS:0000000000000000
[ 124.022282] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 124.022283] CR2: 00007f21e1feb728 CR3: 0000000be0b6a005 CR4: 00000000000606e0
[ 124.022283] Call Trace:
[ 124.022284] do_raw_spin_lock+0xb4/0xc0
[ 124.022284] _raw_spin_lock_nested+0x49/0x60
[ 124.022285] sched_core_balance+0x106/0x5d0
[ 124.022285] __balance_callback+0x49/0xa0
[ 124.022286] __schedule+0x12eb/0x1660
[ 124.022286] ? enqueue_entity+0x112/0x6f0
[ 124.022287] ? sched_clock+0x5/0x10
[ 124.022287] ? sched_clock_cpu+0xc/0xa0
[ 124.022288] ? lockdep_hardirqs_on+0x11d/0x190
[ 124.022288] schedule_idle+0x28/0x40
[ 124.022289] do_idle+0x169/0x2a0
[ 124.022289] cpu_startup_entry+0x19/0x20
[ 124.022290] start_secondary+0x17f/0x1d0
[ 124.022290] secondary_startup_64+0xa4/0xb0
[ 124.022291] NMI watchdog: Watchdog detected hard LOCKUP on cpu 4
[ 124.022292] Modules linked in: ipt_MASQUERADE xfrm_user xfrm_algo
iptable_nat nf_nat_ipv4 xt_addrtype iptable_filter ip_tables
xt_conntrack x_tables nf_nat nf_conntracki
[ 124.022321] irq event stamp: 13957
[ 124.022321] hardirqs last enabled at (13956): [<ffffffff81a10892>]
_raw_spin_unlock_irqrestore+0x32/0x60
[ 124.022322] hardirqs last disabled at (13957): [<ffffffff81a10f32>]
_raw_spin_lock_irqsave+0x22/0x80
[ 124.022323] softirqs last enabled at (13946): [<ffffffff81e003a3>]
__do_softirq+0x3a3/0x3f2
[ 124.022323] softirqs last disabled at (13949): [<ffffffff81095ea1>]
irq_exit+0xc1/0xd0
[ 124.022324] CPU: 4 PID: 2698 Comm: schbench Tainted: G I
5.0.0-rc8-00544-gf24f5e9-dirty #20
[ 124.022325] Hardware name: Intel Corporation S2600CP/S2600CP, BIOS
SE5C600.86B.99.99.x058.082120120902 08/21/2012
[ 124.022325] RIP: 0010:native_queued_spin_lock_slowpath+0x17e/0x1d0
[ 124.022326] Code: 34 c5 40 48 39 82 48 89 16 8b 42 08 85 c0 75 09
f3 90 8b 42 08 85 c0 74 f7 48 8b 32 48 85 f6 74 07 0f 18 0e eb 02 f3
90 8b 07 <66> 85 c0 75 f7 41 89 c7
[ 124.022327] RSP: 0000:ffff888c0a403da8 EFLAGS: 00000002
[ 124.022328] RAX: 0000000000540101 RBX: ffff888c0a1e4400 RCX: 0000000000140000
[ 124.022328] RDX: ffff888c0a5e5180 RSI: 0000000000000000 RDI: ffff888c0a1e4400
[ 124.022329] RBP: ffff888c0a1e4400 R08: 0000000000140000 R09: 0000000000000001
[ 124.022329] R10: ffff888c0a403d50 R11: 0000000000000000 R12: 0000000000000202
[ 124.022330] R13: 000000000000003b R14: 0000000000000001 R15: ffff888c0b1e4400
[ 124.022331] FS: 00007f21e47f0700(0000) GS:ffff888c0a400000(0000)
knlGS:0000000000000000
[ 124.022331] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 124.022332] CR2: 00007f21f17fa728 CR3: 0000000be0b6a004 CR4: 00000000000606e0
[ 124.022332] Call Trace:
[ 124.022332] <IRQ>
[ 124.022333] do_raw_spin_lock+0xb4/0xc0
[ 124.022333] _raw_spin_lock_irqsave+0x63/0x80
[ 124.022334] load_balance+0x358/0xde0
[ 124.022334] rebalance_domains+0x239/0x320
[ 124.022335] ? lockdep_hardirqs_on+0xa3/0x190
[ 124.022335] __do_softirq+0xd6/0x3f2
[ 124.022336] irq_exit+0xc1/0xd0
[ 124.022336] smp_apic_timer_interrupt+0xac/0x210
[ 124.022337] apic_timer_interrupt+0xf/0x20
[ 124.022337] </IRQ>
[ 124.022338] RIP: 0033:0x7fff855a26dd
[ 124.022339] Code: e2 20 48 09 c2 48 85 d2 49 8b 40 08 48 8b 0d c2
c9 ff ff 78 9b 48 39 d1 73 10 48 29 ca 8b 0d c2 c9 ff ff 48 0f af d1
48 01 d0 <8b> 0d b9 c9 ff ff 4d 8f
[ 124.022339] RSP: 002b:00007f21e47efe00 EFLAGS: 00000206 ORIG_RAX:
ffffffffffffff13
[ 124.022340] RAX: 001c0b86d5d09d9a RBX: 00007f21e47efe50 RCX: 00000000005f0b1d
[ 124.022341] RDX: 000063752327ca12 RSI: 00007f21e47efe50 RDI: 0000000000000000
[ 124.022341] RBP: 00007f21e47efe10 R08: 00007fff8559f0a0 R09: 0000000000007f96
[ 124.022342] R10: 00007f21e47efe30 R11: 0000000000000246 R12: 0000000000000000
[ 124.022342] R13: 00007f2202716e6f R14: 0000000000000000 R15: 00007f21fc518628
[ 124.022343] NMI watchdog: Watchdog detected hard LOCKUP on cpu 5
[ 124.022344] Modules linked in: ipt_MASQUERADE xfrm_user xfrm_algo
iptable_nat nf_nat_ipv4 xt_addrtype iptable_filter ip_tables
xt_conntrack x_tables nf_nat nf_conntracki
[ 124.022372] irq event stamp: 132866
[ 124.022373] hardirqs last enabled at (132865):
[<ffffffff810c7544>] sched_core_balance+0x164/0x5d0
[ 124.022374] hardirqs last disabled at (132866):
[<ffffffff810c75b5>] sched_core_balance+0x1d5/0x5d0
[ 124.022375] softirqs last enabled at (132834):
[<ffffffff81e003a3>] __do_softirq+0x3a3/0x3f2
[ 124.022375] softirqs last disabled at (132825):
[<ffffffff81095ea1>] irq_exit+0xc1/0xd0
[ 124.022376] CPU: 5 PID: 0 Comm: swapper/5 Tainted: G I
5.0.0-rc8-00544-gf24f5e9-dirty #20
[ 124.022377] Hardware name: Intel Corporation S2600CP/S2600CP, BIOS
SE5C600.86B.99.99.x058.082120120902 08/21/2012
[ 124.022377] RIP: 0010:native_queued_spin_lock_slowpath+0x169/0x1d0
[ 124.022378] Code: 48 c1 ee 0b 83 e8 01 83 e6 60 48 98 48 81 c6 80
51 1e 00 48 03 34 c5 40 48 39 82 48 89 16 8b 42 08 85 c0 75 09 f3 90
8b 42 08 <85> c0 74 f7 48 8b 32 46
[ 124.022379] RSP: 0000:ffffc9000636bcf8 EFLAGS: 00000046
[ 124.022380] RAX: 0000000000000000 RBX: ffff888c0abe4400 RCX: 0000000000180000
[ 124.022380] RDX: ffff888c0a7e5180 RSI: ffff888c0abe5180 RDI: ffff888c0abe4400
[ 124.022381] RBP: ffff888c0abe4400 R08: 0000000000180000 R09: 0000000000000001
[ 124.022381] R10: ffffc9000636bca0 R11: 0000000000000000 R12: 0000000000000005
[ 124.022382] R13: 0000000000000000 R14: ffff888c0a7e4400 R15: 00000000001e4400
[ 124.022382] FS: 0000000000000000(0000) GS:ffff888c0a600000(0000)
knlGS:0000000000000000
[ 124.022383] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 124.022384] CR2: 00007f7956376889 CR3: 0000000be0b6a001 CR4: 00000000000606e0
[ 124.022384] Call Trace:
[ 124.022384] do_raw_spin_lock+0xb4/0xc0
[ 124.022385] _raw_spin_lock_nested+0x49/0x60
[ 124.022385] sched_core_balance+0x106/0x5d0
[ 124.022386] __balance_callback+0x49/0xa0
[ 124.022386] __schedule+0x12eb/0x1660
[ 124.022387] ? enqueue_entity+0x112/0x6f0
[ 124.022387] ? sched_clock+0x5/0x10
[ 124.022388] ? sched_clock_cpu+0xc/0xa0
[ 124.022388] ? lockdep_hardirqs_on+0x11d/0x190
[ 124.022389] schedule_idle+0x28/0x40
[ 124.022389] do_idle+0x169/0x2a0
[ 124.022390] cpu_startup_entry+0x19/0x20
[ 124.022390] start_secondary+0x17f/0x1d0
[ 124.022391] secondary_startup_64+0xa4/0xb0
[ 124.022392] NMI watchdog: Watchdog detected hard LOCKUP on cpu 7
[ 124.022392] Modules linked in: ipt_MASQUERADE xfrm_user xfrm_algo
iptable_nat nf_nat_ipv4 xt_addrtype iptable_filter ip_tables
xt_conntrack x_tables nf_nat nf_conntracki
[ 124.022421] irq event stamp: 8164
[ 124.022422] hardirqs last enabled at (8163): [<ffffffff81003829>]
trace_hardirqs_on_thunk+0x1a/0x1c
[ 124.022422] hardirqs last disabled at (8164): [<ffffffff81003845>]
trace_hardirqs_off_thunk+0x1a/0x1c
[ 124.022423] softirqs last enabled at (8138): [<ffffffff81e003a3>]
__do_softirq+0x3a3/0x3f2
[ 124.022424] softirqs last disabled at (8129): [<ffffffff81095ea1>]
irq_exit+0xc1/0xd0
[ 124.022424] CPU: 7 PID: 2733 Comm: schbench Tainted: G I
5.0.0-rc8-00544-gf24f5e9-dirty #20
[ 124.022425] Hardware name: Intel Corporation S2600CP/S2600CP, BIOS
SE5C600.86B.99.99.x058.082120120902 08/21/2012
[ 124.022426] RIP: 0010:native_queued_spin_lock_slowpath+0x169/0x1d0
[ 124.022427] Code: 48 c1 ee 0b 83 e8 01 83 e6 60 48 98 48 81 c6 80
51 1e 00 48 03 34 c5 40 48 39 82 48 89 16 8b 42 08 85 c0 75 09 f3 90
8b 42 08 <85> c0 74 f7 48 8b 32 46
[ 124.022427] RSP: 0000:ffff888c0aa03e78 EFLAGS: 00000046
[ 124.022428] RAX: 0000000000000000 RBX: ffff888c0abe4400 RCX: 0000000000200000
[ 124.022429] RDX: ffff888c0abe5180 RSI: ffff888c09fe5180 RDI: ffff888c0abe4400
[ 124.022429] RBP: ffff888c0abe4400 R08: 0000000000200000 R09: 0000000000000001
[ 124.022430] R10: ffff888c0aa03e20 R11: 0000000000000000 R12: 00000000001e4400
[ 124.022430] R13: 0000000000000007 R14: 0000000000000002 R15: ffff888be038d480
[ 124.022431] FS: 00007f21d47d0700(0000) GS:ffff888c0aa00000(0000)
knlGS:0000000000000000
[ 124.022432] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 124.022432] CR2: 00007f21e07e8728 CR3: 0000000be0b6a001 CR4: 00000000000606e0
[ 124.022433] Call Trace:
[ 124.022433] <IRQ>
[ 124.022433] do_raw_spin_lock+0xb4/0xc0
[ 124.022434] _raw_spin_lock+0x4b/0x60
[ 124.022434] scheduler_tick+0x48/0x170
[ 124.022435] ? tick_sched_do_timer+0x60/0x60
[ 124.022435] update_process_times+0x40/0x50
[ 124.022436] tick_sched_handle+0x22/0x60
[ 124.022436] tick_sched_timer+0x37/0x70
[ 124.022437] __hrtimer_run_queues+0xed/0x3f0
[ 124.022437] hrtimer_interrupt+0x122/0x270
[ 124.022438] smp_apic_timer_interrupt+0x86/0x210
[ 124.022438] apic_timer_interrupt+0xf/0x20
[ 124.022439] </IRQ>
[ 124.022439] RIP: 0033:0x7fff855a262b
[ 124.022440] Code: 41 0e 10 86 02 48 0d 06 60 c6 0c 07 08 00 00 4c
8d 54 24 08 48 63 ff 48 83 e4 f0 4c 8d 47 02 48 8d 05 59 ca ff ff 41
ff 72 f8 <55> 49 c1 e0 04 48 89 e1
[ 124.022441] RSP: 002b:00007f21d47cfe18 EFLAGS: 00000202 ORIG_RAX:
ffffffffffffff13
[ 124.022442] RAX: 00007fff8559f080 RBX: 00007f21d47cfe50 RCX: 0000000000000000
[ 124.022442] RDX: 00007f21d47cfe60 RSI: 00007f21d47cfe50 RDI: 0000000000000000
[ 124.022443] RBP: 00007f21d47cfe40 R08: 0000000000000002 R09: 00000000000079f0
[ 124.022443] R10: 00007f21d47cfe30 R11: 0000000000000246 R12: 0000000000000000
[ 124.022444] R13: 00007f2201714e6f R14: 0000000000000000 R15: 00007f21f8b34fd8
[ 124.022446] NMI watchdog: Watchdog detected hard LOCKUP on cpu 16
[ 124.022447] Modules linked in: ipt_MASQUERADE xfrm_user xfrm_algo
iptable_nat nf_nat_ipv4 xt_addrtype iptable_filter ip_tables
xt_conntrack x_tables nf_nat nf_conntracki
[ 124.022475] irq event stamp: 134706
[ 124.022476] hardirqs last enabled at (134705):
[<ffffffff810c7544>] sched_core_balance+0x164/0x5d0
[ 124.022476] hardirqs last disabled at (134706):
[<ffffffff810c75b5>] sched_core_balance+0x1d5/0x5d0
[ 124.022477] softirqs last enabled at (134694):
[<ffffffff81e003a3>] __do_softirq+0x3a3/0x3f2
[ 124.022478] softirqs last disabled at (134683):
[<ffffffff81095ea1>] irq_exit+0xc1/0xd0
[ 124.022479] CPU: 16 PID: 0 Comm: swapper/16 Tainted: G I
5.0.0-rc8-00544-gf24f5e9-dirty #20
[ 124.022479] Hardware name: Intel Corporation S2600CP/S2600CP, BIOS
SE5C600.86B.99.99.x058.082120120902 08/21/2012
[ 124.022480] RIP: 0010:native_queued_spin_lock_slowpath+0x169/0x1d0
[ 124.022481] Code: 48 c1 ee 0b 83 e8 01 83 e6 60 48 98 48 81 c6 80
51 1e 00 48 03 34 c5 40 48 39 82 48 89 16 8b 42 08 85 c0 75 09 f3 90
8b 42 08 <85> c0 74 f7 48 8b 32 46
[ 124.022481] RSP: 0000:ffffc900063c3cf8 EFLAGS: 00000046
[ 124.022482] RAX: 0000000000000000 RBX: ffff888c09fe4400 RCX: 0000000000440000
[ 124.022483] RDX: ffff888c0ade5180 RSI: ffff88980b3e5180 RDI: ffff888c09fe4400
[ 124.022483] RBP: ffff888c09fe4400 R08: 0000000000440000 R09: 0000000000000001
[ 124.022484] R10: ffffc900063c3ca0 R11: 0000000000000000 R12: 0000000000000010
[ 124.022485] R13: 0000000000000000 R14: ffff888c0ade4400 R15: 00000000001e4400
[ 124.022485] FS: 0000000000000000(0000) GS:ffff888c0ac00000(0000)
knlGS:0000000000000000
[ 124.022486] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 124.022486] CR2: 00007f21e77f5e30 CR3: 0000000be0b6a003 CR4: 00000000000606e0
[ 124.022487] Call Trace:
[ 124.022487] do_raw_spin_lock+0xb4/0xc0
[ 124.022488] _raw_spin_lock_nested+0x49/0x60
[ 124.022488] sched_core_balance+0x106/0x5d0
[ 124.022489] __balance_callback+0x49/0xa0
[ 124.022489] __schedule+0x12eb/0x1660
[ 124.022490] ? _raw_spin_unlock_irqrestore+0x4e/0x60
[ 124.022490] ? hrtimer_start_range_ns+0x1b7/0x340
[ 124.022491] schedule_idle+0x28/0x40
[ 124.022491] do_idle+0x169/0x2a0
[ 124.022492] cpu_startup_entry+0x19/0x20
[ 124.022492] start_secondary+0x17f/0x1d0
[ 124.022493] secondary_startup_64+0xa4/0xb0
[ 124.022493] NMI watchdog: Watchdog detected hard LOCKUP on cpu 17
[ 124.022494] Modules linked in: ipt_MASQUERADE xfrm_user xfrm_algo
iptable_nat nf_nat_ipv4 xt_addrtype iptable_filter ip_tables
xt_conntrack x_tables nf_nat nf_conntracki
[ 124.022522] irq event stamp: 10038
[ 124.022523] hardirqs last enabled at (10037): [<ffffffff81003829>]
trace_hardirqs_on_thunk+0x1a/0x1c
[ 124.022524] hardirqs last disabled at (10038): [<ffffffff81003845>]
trace_hardirqs_off_thunk+0x1a/0x1c
[ 124.022524] softirqs last enabled at (10012): [<ffffffff81e003a3>]
__do_softirq+0x3a3/0x3f2
[ 124.022525] softirqs last disabled at (10001): [<ffffffff81095ea1>]
irq_exit+0xc1/0xd0
[ 124.022525] CPU: 17 PID: 2681 Comm: schbench Tainted: G I
5.0.0-rc8-00544-gf24f5e9-dirty #20
[ 124.022526] Hardware name: Intel Corporation S2600CP/S2600CP, BIOS
SE5C600.86B.99.99.x058.082120120902 08/21/2012
[ 124.022527] RIP: 0010:native_queued_spin_lock_slowpath+0x169/0x1d0
[ 124.022528] Code: 48 c1 ee 0b 83 e8 01 83 e6 60 48 98 48 81 c6 80
51 1e 00 48 03 34 c5 40 48 39 82 48 89 16 8b 42 08 85 c0 75 09 f3 90
8b 42 08 <85> c0 74 f7 48 8b 32 46
[ 124.022528] RSP: 0000:ffff888c0ae03e78 EFLAGS: 00000046
[ 124.022529] RAX: 0000000000000000 RBX: ffff888c09fe4400 RCX: 0000000000480000
[ 124.022530] RDX: ffff888c0afe5180 RSI: ffff88980a1e5180 RDI: ffff888c09fe4400
[ 124.022531] RBP: ffff888c09fe4400 R08: 0000000000480000 R09: 0000000000000001
[ 124.022531] R10: ffff888c0ae03e20 R11: 0000000000000000 R12: 00000000001e4400
[ 124.022532] R13: 0000000000000011 R14: 0000000000000002 R15: ffff888be006d480
[ 124.022533] FS: 00007f21f1ffb700(0000) GS:ffff888c0ae00000(0000)
knlGS:0000000000000000
[ 124.022533] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 124.022534] CR2: 00007f21e4ff1728 CR3: 0000000be0b6a004 CR4: 00000000000606e0
[ 124.022534] Call Trace:
[ 124.022535] <IRQ>
[ 124.022535] do_raw_spin_lock+0xb4/0xc0
[ 124.022536] _raw_spin_lock+0x4b/0x60
[ 124.022536] scheduler_tick+0x48/0x170
[ 124.022537] ? tick_sched_do_timer+0x60/0x60
[ 124.022537] update_process_times+0x40/0x50
[ 124.022538] tick_sched_handle+0x22/0x60
[ 124.022538] tick_sched_timer+0x37/0x70
[ 124.022538] __hrtimer_run_queues+0xed/0x3f0
[ 124.022539] hrtimer_interrupt+0x122/0x270
[ 124.022540] smp_apic_timer_interrupt+0x86/0x210
[ 124.022540] apic_timer_interrupt+0xf/0x20
[ 124.022540] </IRQ>
[ 124.022541] RIP: 0033:0x7fff855a26af
[ 124.022542] Code: ff 4c 8b 1d 83 e9 ff ff 0f 01 f9 66 90 8b 0d 68
e9 ff ff 41 39 ca 75 d6 48 c1 e2 20 48 09 d0 48 f7 e3 4c 01 da eb 0c
0f 01 f9 <66> 90 48 c1 e2 20 48 0f
[ 124.022542] RSP: 002b:00007f21f1ffae00 EFLAGS: 00000246 ORIG_RAX:
ffffffffffffff13
[ 124.022543] RAX: 00000000e4479e7b RBX: 00007f21f1ffae50 RCX: 0000000000000011
[ 124.022544] RDX: 0000000000000068 RSI: 00007f21f1ffae50 RDI: 0000000000000000
[ 124.022545] RBP: 00007f21f1ffae10 R08: 00007fff8559f0a0 R09: 00000000000079f0
[ 124.022545] R10: 00007f21f1ffae30 R11: 0000000000000246 R12: 0000000000000000
[ 124.022546] R13: 00007f2201f15e6f R14: 0000000000000000 R15: 00007f21f4105508
[ 124.022546] NMI watchdog: Watchdog detected hard LOCKUP on cpu 18
[ 124.022547] Modules linked in: ipt_MASQUERADE xfrm_user xfrm_algo
iptable_nat nf_nat_ipv4 xt_addrtype iptable_filter ip_tables
xt_conntrack x_tables nf_nat nf_conntracki
[ 124.022575] irq event stamp: 8788
[ 124.022576] hardirqs last enabled at (8787): [<ffffffff81003829>]
trace_hardirqs_on_thunk+0x1a/0x1c
[ 124.022577] hardirqs last disabled at (8788): [<ffffffff81003845>]
trace_hardirqs_off_thunk+0x1a/0x1c
[ 124.022577] softirqs last enabled at (8782): [<ffffffff81e003a3>]
__do_softirq+0x3a3/0x3f2
[ 124.022578] softirqs last disabled at (8775): [<ffffffff81095ea1>]
irq_exit+0xc1/0xd0
[ 124.022579] CPU: 18 PID: 2741 Comm: schbench Tainted: G I
5.0.0-rc8-00544-gf24f5e9-dirty #20
[ 124.022579] Hardware name: Intel Corporation S2600CP/S2600CP, BIOS
SE5C600.86B.99.99.x058.082120120902 08/21/2012
[ 124.022580] RIP: 0010:native_queued_spin_lock_slowpath+0x5c/0x1d0
[ 124.022581] Code: ff ff ff 75 40 f0 0f ba 2f 08 0f 82 cd 00 00 00
8b 07 30 e4 09 c6 f7 c6 00 ff ff ff 75 1b 85 f6 74 0e 8b 07 84 c0 74
08 f3 90 <8b> 07 84 c0 75 f8 b8 05
[ 124.022581] RSP: 0000:ffff888c0b003e78 EFLAGS: 00000002
[ 124.022582] RAX: 0000000000540101 RBX: ffff888c0a1e4400 RCX: 0000000000000540
[ 124.022583] RDX: 0000000000000002 RSI: 0000000000000001 RDI: ffff888c0a1e4400
[ 124.022584] RBP: ffff888c0a1e4400 R08: 0000000000000000 R09: 0000000000000001
[ 124.022584] R10: ffff888c0b003e20 R11: 0000000000000000 R12: 00000000001e4400
[ 124.022585] R13: 0000000000000012 R14: 0000000000000002 R15: ffff888be03a2a40
[ 124.022585] FS: 00007f21d07c8700(0000) GS:ffff888c0b000000(0000)
knlGS:0000000000000000
[ 124.022586] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 124.022586] CR2: 00007f21df7e6728 CR3: 0000000be0b6a003 CR4: 00000000000606e0
[ 124.022587] Call Trace:
[ 124.022587] <IRQ>
[ 124.022588] do_raw_spin_lock+0xb4/0xc0
[ 124.022588] _raw_spin_lock+0x4b/0x60
[ 124.022589] scheduler_tick+0x48/0x170
[ 124.022589] ? tick_sched_do_timer+0x60/0x60
[ 124.022590] update_process_times+0x40/0x50
[ 124.022590] tick_sched_handle+0x22/0x60
[ 124.022591] tick_sched_timer+0x37/0x70
[ 124.022591] __hrtimer_run_queues+0xed/0x3f0
[ 124.022592] hrtimer_interrupt+0x122/0x270
[ 124.022592] smp_apic_timer_interrupt+0x86/0x210
[ 124.022593] apic_timer_interrupt+0xf/0x20
[ 124.022593] </IRQ>
[ 124.022593] RIP: 0033:0x402050
[ 124.022594] Code: 48 83 ec 40 48 8d 7c 24 10 64 48 8b 04 25 28 00
00 00 48 89 44 24 38 31 c0 e8 2c eb ff ff eb 0c 66 2e 0f 1f 84 00 00
00 00 00 <f3> 90 31 f6 48 89 e7 e0
[ 124.022595] RSP: 002b:00007f21d07c7e50 EFLAGS: 00000283 ORIG_RAX:
ffffffffffffff13
[ 124.022596] RAX: 0000000000051d87 RBX: 00000000002dc6c0 RCX: 0000000000000000
[ 124.022596] RDX: 00007f21d07c7e60 RSI: 00007f21d07c7e50 RDI: 00007f21d07c7e70
[ 124.022597] RBP: 0000000000000002 R08: 0000000000000000 R09: 0000000000007a06
[ 124.022598] R10: 00007f21d07c7e30 R11: 0000000000000246 R12: 00007f21d07c7ed0
[ 124.022598] R13: 00007f2201714e6f R14: 0000000000000000 R15: 00007f21f8f480f8
[ 124.022599] NMI watchdog: Watchdog detected hard LOCKUP on cpu 19
[ 124.022599] Modules linked in: ipt_MASQUERADE xfrm_user xfrm_algo
iptable_nat nf_nat_ipv4 xt_addrtype iptable_filter ip_tables
xt_conntrack x_tables nf_nat nf_conntracki
[ 124.022628] irq event stamp: 132622
[ 124.022628] hardirqs last enabled at (132621):
[<ffffffff810c7544>] sched_core_balance+0x164/0x5d0
[ 124.022629] hardirqs last disabled at (132622):
[<ffffffff810c75b5>] sched_core_balance+0x1d5/0x5d0
[ 124.022630] softirqs last enabled at (132576):
[<ffffffff81095dde>] irq_enter+0x5e/0x60
[ 124.022630] softirqs last disabled at (132575):
[<ffffffff81095dc3>] irq_enter+0x43/0x60
[ 124.022631] CPU: 19 PID: 0 Comm: swapper/19 Tainted: G I
5.0.0-rc8-00544-gf24f5e9-dirty #20
[ 124.022632] Hardware name: Intel Corporation S2600CP/S2600CP, BIOS
SE5C600.86B.99.99.x058.082120120902 08/21/2012
[ 124.022632] RIP: 0010:native_queued_spin_lock_slowpath+0x5e/0x1d0
[ 124.022633] Code: ff 75 40 f0 0f ba 2f 08 0f 82 cd 00 00 00 8b 07
30 e4 09 c6 f7 c6 00 ff ff ff 75 1b 85 f6 74 0e 8b 07 84 c0 74 08 f3
90 8b 07 <84> c0 75 f8 b8 01 00 06
[ 124.022634] RSP: 0000:ffffc900063dbcf8 EFLAGS: 00000002
[ 124.022635] RAX: 00000000000c0101 RBX: ffff888c0a3e4400 RCX: 000000000000c980
[ 124.022635] RDX: 0000000000000002 RSI: 0000000000000001 RDI: ffff888c0a3e4400
[ 124.022636] RBP: ffff888c0a3e4400 R08: 0000000000000000 R09: 0000000000000001
[ 124.022636] R10: ffffc900063dbca0 R11: 0000000000000000 R12: 0000000000000013
[ 124.022637] R13: 0000000000000000 R14: ffff888c0b3e4400 R15: 00000000001e4400
[ 124.022638] FS: 0000000000000000(0000) GS:ffff888c0b200000(0000)
knlGS:0000000000000000
[ 124.022638] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 124.022639] CR2: 00007f21e7ff7728 CR3: 0000000be0b6a003 CR4: 00000000000606e0
[ 124.022639] Call Trace:
[ 124.022640] do_raw_spin_lock+0xb4/0xc0
[ 124.022640] _raw_spin_lock_nested+0x49/0x60
[ 124.022641] sched_core_balance+0x106/0x5d0
[ 124.022641] __balance_callback+0x49/0xa0
[ 124.022642] __schedule+0x12eb/0x1660
[ 124.022642] ? enqueue_entity+0x112/0x6f0
[ 124.022643] ? sched_clock+0x5/0x10
[ 124.022643] ? sched_clock_cpu+0xc/0xa0
[ 124.022643] ? lockdep_hardirqs_on+0x11d/0x190
[ 124.022644] schedule_idle+0x28/0x40
[ 124.022644] do_idle+0x169/0x2a0
[ 124.022645] cpu_startup_entry+0x19/0x20
[ 124.022645] start_secondary+0x17f/0x1d0
[ 124.022646] secondary_startup_64+0xa4/0xb0
[ 124.022647] NMI watchdog: Watchdog detected hard LOCKUP on cpu 20
[ 124.022647] Modules linked in: ipt_MASQUERADE xfrm_user xfrm_algo
iptable_nat nf_nat_ipv4 xt_addrtype iptable_filter ip_tables
xt_conntrack x_tables nf_nat nf_conntracki
[ 124.022676] irq event stamp: 12904
[ 124.022676] hardirqs last enabled at (12903): [<ffffffff81004552>]
do_syscall_64+0x12/0x1b0
[ 124.022677] hardirqs last disabled at (12904): [<ffffffff81a07e92>]
__schedule+0xe2/0x1660
[ 124.022677] softirqs last enabled at (12890): [<ffffffff81e003a3>]
__do_softirq+0x3a3/0x3f2
[ 124.022678] softirqs last disabled at (12883): [<ffffffff81095ea1>]
irq_exit+0xc1/0xd0
[ 124.022679] CPU: 20 PID: 2688 Comm: schbench Tainted: G I
5.0.0-rc8-00544-gf24f5e9-dirty #20
[ 124.022679] Hardware name: Intel Corporation S2600CP/S2600CP, BIOS
SE5C600.86B.99.99.x058.082120120902 08/21/2012
[ 124.022680] RIP: 0010:native_queued_spin_lock_slowpath+0x169/0x1d0
[ 124.022681] Code: 48 c1 ee 0b 83 e8 01 83 e6 60 48 98 48 81 c6 80
51 1e 00 48 03 34 c5 40 48 39 82 48 89 16 8b 42 08 85 c0 75 09 f3 90
8b 42 08 <85> c0 74 f7 48 8b 32 46
[ 124.022681] RSP: 0018:ffffc9000a1c39b8 EFLAGS: 00000046
[ 124.022682] RAX: 0000000000000000 RBX: ffff888c0a1e4400 RCX: 0000000000540000
[ 124.022683] RDX: ffff888c0b5e5180 RSI: ffff888c0a5e5180 RDI: ffff888c0a1e4400
[ 124.022683] RBP: ffff888c0a1e4400 R08: 0000000000540000 R09: 0000000000000001
[ 124.022684] R10: ffffc9000a1c3960 R11: 0000000000000000 R12: 0000000000000002
[ 124.022685] R13: 000000000000003b R14: 0000000000000001 R15: ffff888c0b1e4400
[ 124.022685] FS: 00007f21ea7fc700(0000) GS:ffff888c0b400000(0000)
knlGS:0000000000000000
[ 124.022686] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 124.022686] CR2: 00007f795684d2a0 CR3: 0000000be0b6a003 CR4: 00000000000606e0
[ 124.022687] Call Trace:
[ 124.022687] do_raw_spin_lock+0xb4/0xc0
[ 124.022688] _raw_spin_lock_irqsave+0x63/0x80
[ 124.022688] load_balance+0x358/0xde0
[ 124.022689] ? update_dl_rq_load_avg+0x121/0x280
[ 124.022689] newidle_balance+0x1d0/0x580
[ 124.022690] __schedule+0x120f/0x1660
[ 124.022690] ? lock_acquire+0xab/0x180
[ 124.022691] ? attach_entity_load_avg+0x140/0x170
[ 124.022691] ? sched_clock+0x5/0x10
[ 124.022692] ? sched_clock_cpu+0xc/0xa0
[ 124.022692] schedule+0x32/0x70
[ 124.022692] futex_wait_queue_me+0xbf/0x130
[ 124.022693] futex_wait+0xf6/0x250
[ 124.022693] ? __switch_to_asm+0x34/0x70
[ 124.022694] ? __switch_to_asm+0x40/0x70
[ 124.022694] ? __switch_to_asm+0x34/0x70
[ 124.022695] ? __switch_to_asm+0x40/0x70
[ 124.022695] ? __switch_to_asm+0x34/0x70
[ 124.022696] ? __switch_to_asm+0x40/0x70
[ 124.022696] do_futex+0x311/0xb80
[ 124.022697] ? __schedule+0xde3/0x1660
[ 124.022697] __x64_sys_futex+0x88/0x180
[ 124.022698] ? trace_hardirqs_off_thunk+0x1a/0x1c
[ 124.022698] do_syscall_64+0x60/0x1b0
[ 124.022699] entry_SYSCALL_64_after_hwframe+0x49/0xbe
[ 124.022699] RIP: 0033:0x7f2202c2d4d9
[ 124.022700] Code: 00 f3 c3 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 40
00 48 89 f8 48 89 f7 48 89 d6 48 89 ca 4d 89 c2 4d 89 c8 4c 8b 4c 24
08 0f 05 <48> 3d 01 f0 ff ff 73 08
[ 124.022701] RSP: 002b:00007f21ea7fbe68 EFLAGS: 00000246 ORIG_RAX:
00000000000000ca
[ 124.022702] RAX: ffffffffffffffda RBX: 00007f21fc20a180 RCX: 00007f2202c2d4d9
[ 124.022702] RDX: 0000000000000000 RSI: 0000000000000080 RDI: 00007f21fc20a180
[ 124.022703] RBP: 0000000000000001 R08: 0000000000000000 R09: 0000000000000000
[ 124.022704] R10: 0000000000000000 R11: 0000000000000246 R12: 00007f21ea7fbed0
[ 124.022704] R13: 00007f2202716e6f R14: 0000000000000000 R15: 00007f21fc20a150
[ 124.022705] NMI watchdog: Watchdog detected hard LOCKUP on cpu 21
[ 124.022705] Modules linked in: ipt_MASQUERADE xfrm_user xfrm_algo
iptable_nat nf_nat_ipv4 xt_addrtype iptable_filter ip_tables
xt_conntrack x_tables nf_nat nf_conntracki
[ 124.022734] irq event stamp: 7710
[ 124.022734] hardirqs last enabled at (7709): [<ffffffff81003829>]
trace_hardirqs_on_thunk+0x1a/0x1c
[ 124.022735] hardirqs last disabled at (7710): [<ffffffff81003845>]
trace_hardirqs_off_thunk+0x1a/0x1c
[ 124.022736] softirqs last enabled at (7704): [<ffffffff81e003a3>]
__do_softirq+0x3a3/0x3f2
[ 124.022736] softirqs last disabled at (7697): [<ffffffff81095ea1>]
irq_exit+0xc1/0xd0
[ 124.022737] CPU: 21 PID: 2719 Comm: schbench Tainted: G I
5.0.0-rc8-00544-gf24f5e9-dirty #20
[ 124.022737] Hardware name: Intel Corporation S2600CP/S2600CP, BIOS
SE5C600.86B.99.99.x058.082120120902 08/21/2012
[ 124.022738] RIP: 0010:native_queued_spin_lock_slowpath+0x5c/0x1d0
[ 124.022739] Code: ff ff ff 75 40 f0 0f ba 2f 08 0f 82 cd 00 00 00
8b 07 30 e4 09 c6 f7 c6 00 ff ff ff 75 1b 85 f6 74 0e 8b 07 84 c0 74
08 f3 90 <8b> 07 84 c0 75 f8 b8 05
[ 124.022739] RSP: 0000:ffff888c0b603e78 EFLAGS: 00000002
[ 124.022740] RAX: 00000000006c0101 RBX: ffff888c0a7e4400 RCX: 0000000000000540
[ 124.022741] RDX: 0000000000000002 RSI: 0000000000000001 RDI: ffff888c0a7e4400
[ 124.022742] RBP: ffff888c0a7e4400 R08: 0000000000000000 R09: 0000000000000001
[ 124.022742] R10: ffff888c0b603e20 R11: 0000000000000000 R12: 00000000001e4400
[ 124.022743] R13: 0000000000000015 R14: 0000000000000002 R15: ffff888be0362a40
[ 124.022743] FS: 00007f21dafdd700(0000) GS:ffff888c0b600000(0000)
knlGS:0000000000000000
[ 124.022744] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 124.022744] CR2: 00007f21dafdd728 CR3: 0000000be0b6a002 CR4: 00000000000606e0
[ 124.022745] Call Trace:
[ 124.022745] <IRQ>
[ 124.022746] do_raw_spin_lock+0xb4/0xc0
[ 124.022746] _raw_spin_lock+0x4b/0x60
[ 124.022747] scheduler_tick+0x48/0x170
[ 124.022747] ? tick_sched_do_timer+0x60/0x60
[ 124.022748] update_process_times+0x40/0x50
[ 124.022748] tick_sched_handle+0x22/0x60
[ 124.022749] tick_sched_timer+0x37/0x70
[ 124.022749] __hrtimer_run_queues+0xed/0x3f0
[ 124.022750] hrtimer_interrupt+0x122/0x270
[ 124.022750] smp_apic_timer_interrupt+0x86/0x210
[ 124.022751] apic_timer_interrupt+0xf/0x20
[ 124.022751] </IRQ>
[ 124.022752] RIP: 0033:0x400b70
[ 124.022752] Code: 68 03 00 00 00 e9 b0 ff ff ff ff 25 e2 34 20 00
68 04 00 00 00 e9 a0 ff ff ff ff 25 da 34 20 00 68 05 00 00 00 e9 90
ff ff ff <ff> 25 d2 34 20 00 68 00
[ 124.022753] RSP: 002b:00007f21dafdce48 EFLAGS: 00000246 ORIG_RAX:
ffffffffffffff13
[ 124.022754] RAX: 000000000001bd7b RBX: 00000000002dc6c0 RCX: 0000000000000000
[ 124.022755] RDX: 00007f21dafdce60 RSI: 0000000000000000 RDI: 00007f21dafdce50
[ 124.022755] RBP: 0000000000000002 R08: 0000000000000000 R09: 0000000000007a06
[ 124.022756] R10: 00007f21dafdce30 R11: 0000000000000246 R12: 00007f21dafdced0
[ 124.022756] R13: 00007f2202716e6f R14: 0000000000000000 R15: 00007f21fcf480f8
[ 124.022757] NMI watchdog: Watchdog detected hard LOCKUP on cpu 22
[ 124.022758] Modules linked in: ipt_MASQUERADE xfrm_user xfrm_algo
iptable_nat nf_nat_ipv4 xt_addrtype iptable_filter ip_tables
xt_conntrack x_tables nf_nat nf_conntracki
[ 124.022786] irq event stamp: 14310
[ 124.022787] hardirqs last enabled at (14309): [<ffffffff81003829>]
trace_hardirqs_on_thunk+0x1a/0x1c
[ 124.022787] hardirqs last disabled at (14310): [<ffffffff81003845>]
trace_hardirqs_off_thunk+0x1a/0x1c
[ 124.022788] softirqs last enabled at (14308): [<ffffffff81e003a3>]
__do_softirq+0x3a3/0x3f2
[ 124.022788] softirqs last disabled at (14301): [<ffffffff81095ea1>]
irq_exit+0xc1/0xd0
[ 124.022789] CPU: 22 PID: 2702 Comm: schbench Tainted: G I
5.0.0-rc8-00544-gf24f5e9-dirty #20
[ 124.022790] Hardware name: Intel Corporation S2600CP/S2600CP, BIOS
SE5C600.86B.99.99.x058.082120120902 08/21/2012
[ 124.022790] RIP: 0010:native_queued_spin_lock_slowpath+0x5e/0x1d0
[ 124.022791] Code: ff 75 40 f0 0f ba 2f 08 0f 82 cd 00 00 00 8b 07
30 e4 09 c6 f7 c6 00 ff ff ff 75 1b 85 f6 74 0e 8b 07 84 c0 74 08 f3
90 8b 07 <84> c0 75 f8 b8 01 00 06
[ 124.022792] RSP: 0000:ffff888c0b803e78 EFLAGS: 00000002
[ 124.022793] RAX: 0000000000000101 RBX: ffff888c0a9e4400 RCX: 0000000000000540
[ 124.022793] RDX: 0000000000000002 RSI: 0000000000000001 RDI: ffff888c0a9e4400
[ 124.022794] RBP: ffff888c0a9e4400 R08: 0000000000000000 R09: 0000000000000001
[ 124.022794] R10: ffff888c0b803e20 R11: 0000000000000000 R12: 00000000001e4400
[ 124.022795] R13: 0000000000000016 R14: 0000000000000006 R15: ffff888be0335480
[ 124.022795] FS: 00007f21e27ec700(0000) GS:ffff888c0b800000(0000)
knlGS:0000000000000000
[ 124.022796] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 124.022797] CR2: 00007f21e27ec728 CR3: 0000000be0b6a003 CR4: 00000000000606e0
[ 124.022797] Call Trace:
[ 124.022797] <IRQ>
[ 124.022798] do_raw_spin_lock+0xb4/0xc0
[ 124.022798] _raw_spin_lock+0x4b/0x60
[ 124.022799] scheduler_tick+0x48/0x170
[ 124.022799] ? tick_sched_do_timer+0x60/0x60
[ 124.022800] update_process_times+0x40/0x50
[ 124.022800] tick_sched_handle+0x22/0x60
[ 124.022801] tick_sched_timer+0x37/0x70
[ 124.022801] __hrtimer_run_queues+0xed/0x3f0
[ 124.022802] hrtimer_interrupt+0x122/0x270
[ 124.022802] smp_apic_timer_interrupt+0x86/0x210
[ 124.022803] apic_timer_interrupt+0xf/0x20
[ 124.022804] </IRQ>
[ 124.022804] RIP: 0010:smp_call_function_many+0x20e/0x270
[ 124.022805] Code: f5 89 00 3b 05 e7 ed 66 01 0f 83 7e fe ff ff 48
63 d0 48 8b 0b 48 03 0c d5 40 48 39 82 8b 51 18 83 e2 01 74 0a f3 90
8b 51 18 <83> e2 01 75 f6 eb c8 09
[ 124.022806] RSP: 0000:ffffc9000a237bc0 EFLAGS: 00000202 ORIG_RAX:
ffffffffffffff13
[ 124.022807] RAX: 0000000000000000 RBX: ffff888c0b9e5400 RCX: ffff888c09dead60
[ 124.022807] RDX: 0000000000000003 RSI: 0000000000000200 RDI: ffff888c0b9e5408
[ 124.022808] RBP: ffff888c0b9e5408 R08: 0000000000000001 R09: 0000000000000000
[ 124.022808] R10: 0000000000000000 R11: 0000000000000000 R12: ffffffff8107d070
[ 124.022809] R13: ffffc9000a237cc0 R14: 0000000000000001 R15: 0000000000000200
[ 124.022810] ? flush_tlb_func_common+0x300/0x300
[ 124.022810] ? flush_tlb_func_common+0x300/0x300
[ 124.022811] ? flush_tlb_func_common+0x300/0x300
[ 124.022811] on_each_cpu_mask+0x25/0x90
[ 124.022812] ? x86_configure_nx+0x50/0x50
[ 124.022812] on_each_cpu_cond_mask+0x97/0xd0
[ 124.022813] flush_tlb_mm_range+0xd5/0x140
[ 124.022813] ? sched_clock_cpu+0xc/0xa0
[ 124.022814] ? change_protection+0xa7b/0xaf0
[ 124.022814] ? _cond_resched+0x16/0x40
[ 124.022814] change_protection+0xa7b/0xaf0
[ 124.022815] change_prot_numa+0x18/0x30
[ 124.022815] task_numa_work+0x217/0x330
[ 124.022816] task_work_run+0x7e/0xa0
[ 124.022816] exit_to_usermode_loop+0xe0/0xf0
[ 124.022817] prepare_exit_to_usermode+0x9f/0xd0
[ 124.022817] retint_user+0x8/0x18
[ 124.022818] RIP: 0033:0x7fff855a2615
[ 124.022819] Code: 0c 07 08 00 00 1c 00 00 00 fc 00 00 00 c8 02 00
00 2a 00 00 00 00 41 0e 10 86 02 48 0d 06 60 c6 0c 07 08 00 00 4c 8d
54 24 08 <48> 63 ff 48 83 e4 f0 48
[ 124.022819] RSP: 002b:00007f21e27ebe28 EFLAGS: 00000246 ORIG_RAX:
ffffffffffffff13
[ 124.022820] RAX: 00000000001e2ec1 RBX: 00007f21e27ebe50 RCX: 0000000000000001
[ 124.022821] RDX: 00007f21e27ebe60 RSI: 00007f21e27ebe50 RDI: 0000000000000000
[ 124.022821] RBP: 00007f21e27ebe40 R08: 0000000000000001 R09: 0000000000007daa
[ 124.022822] R10: 00007f21e27ebe30 R11: 0000000000000246 R12: 0000000000000000
[ 124.022823] R13: 00007f2202716e6f R14: 0000000000000000 R15: 00007f21fc721eb8
[ 124.022823] NMI watchdog: Watchdog detected hard LOCKUP on cpu 23
[ 124.022824] Modules linked in: ipt_MASQUERADE xfrm_user xfrm_algo
iptable_nat nf_nat_ipv4 xt_addrtype iptable_filter ip_tables
xt_conntrack x_tables nf_nat nf_conntracki
[ 124.022852] irq event stamp: 141222
[ 124.022853] hardirqs last enabled at (141221):
[<ffffffff810c7544>] sched_core_balance+0x164/0x5d0
[ 124.022854] hardirqs last disabled at (141222):
[<ffffffff810c75b5>] sched_core_balance+0x1d5/0x5d0
[ 124.022854] softirqs last enabled at (141112):
[<ffffffff81e003a3>] __do_softirq+0x3a3/0x3f2
[ 124.022855] softirqs last disabled at (141101):
[<ffffffff81095ea1>] irq_exit+0xc1/0xd0
[ 124.022856] CPU: 23 PID: 0 Comm: swapper/23 Tainted: G I
5.0.0-rc8-00544-gf24f5e9-dirty #20
[ 124.022857] Hardware name: Intel Corporation S2600CP/S2600CP, BIOS
SE5C600.86B.99.99.x058.082120120902 08/21/2012
[ 124.022857] RIP: 0010:native_queued_spin_lock_slowpath+0x5e/0x1d0
[ 124.022858] Code: ff 75 40 f0 0f ba 2f 08 0f 82 cd 00 00 00 8b 07
30 e4 09 c6 f7 c6 00 ff ff ff 75 1b 85 f6 74 0e 8b 07 84 c0 74 08 f3
90 8b 07 <84> c0 75 f8 b8 01 00 06
[ 124.022859] RSP: 0000:ffffc900063fbcf8 EFLAGS: 00000002
[ 124.022860] RAX: 00000000001c0101 RBX: ffff888c0abe4400 RCX: 000000000000c980
[ 124.022860] RDX: 0000000000000002 RSI: 0000000000000001 RDI: ffff888c0abe4400
[ 124.022861] RBP: ffff888c0abe4400 R08: 0000000000000000 R09: 0000000000000001
[ 124.022861] R10: ffffc900063fbca0 R11: 0000000000000000 R12: 0000000000000017
[ 124.022862] R13: 0000000000000000 R14: ffff888c0bbe4400 R15: 00000000001e4400
[ 124.022863] FS: 0000000000000000(0000) GS:ffff888c0ba00000(0000)
knlGS:0000000000000000
[ 124.022863] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 124.022864] CR2: 00007f21eb7fe728 CR3: 0000000be0b6a002 CR4: 00000000000606e0
[ 124.022864] Call Trace:
[ 124.022865] do_raw_spin_lock+0xb4/0xc0
[ 124.022865] _raw_spin_lock_nested+0x49/0x60
[ 124.022866] sched_core_balance+0x106/0x5d0
[ 124.022866] __balance_callback+0x49/0xa0
[ 124.022867] __schedule+0x12eb/0x1660
[ 124.022867] ? _raw_spin_unlock_irqrestore+0x4e/0x60
[ 124.022868] ? hrtimer_start_range_ns+0x1b7/0x340
[ 124.022868] schedule_idle+0x28/0x40
[ 124.022868] do_idle+0x169/0x2a0
[ 124.022869] cpu_startup_entry+0x19/0x20
[ 124.022869] start_secondary+0x17f/0x1d0
[ 124.022870] secondary_startup_64+0xa4/0xb0
[ 129.616038] ---[ end Kernel panic - not syncing: Hard LOCKUP ]---