Currently, when lockdep detects a possible deadlock scenario that involves 3
or more levels, it just shows the chain, and a CPU sequence order of the
first and last part of the scenario, leaving out the middle level and this
can take a bit of effort to understand. By adding a third level, it becomes
easier to see where the deadlock is.
The current output displays:
[For an AB BC CA scenario:]
Chain exists of:
lockC --> lockA --> lockB
Possible unsafe locking scenario:
CPU0 CPU1
---- ----
lock(lockB);
lock(lockA);
lock(lockB);
lock(lockC);
*** DEADLOCK ***
This change, now shows:
[For an AB BC CA scenario:]
Chain exists of:
lockC --> lockA --> lockB
Possible unsafe locking scenario:
CPU0 CPU1 CPU2
---- ---- ----
lock(lockB);
lock(lockA);
lock(lockC);
lock(lockA);
lock(lockB);
lock(lockC);
*** DEADLOCK ***
Much easier to see where the deadlock happened.
This also updates the interrupt scenario:
Old way:
Chain exists of:
lockA --> lockB --> lockC
Possible interrupt unsafe locking scenario:
CPU0 CPU1
---- ----
lock(lockC);
local_irq_disable();
lock(lockA);
lock(lockB);
<Interrupt>
lock(lockA);
*** DEADLOCK ***
New way:
Chain exists of:
lockA --> lockB --> lockC
Possible interrupt unsafe locking scenario:
CPU0 CPU1 CPU2
---- ---- ----
lock(lockC);
local_irq_disable();
lock(lockB);
local_irq_disable();
lock(lockA);
lock(lockB);
lock(lockC);
<Interrupt>
lock(lockA);
*** DEADLOCK ***
As well as for completions:
Old way:
Chain exists of:
mutexB --> mutexA --> (completion)&comp
Possible unsafe locking scenario by crosslock:
CPU0 CPU1
---- ----
lock(mutexA);
lock((completion)&comp);
lock(mutexB);
unlock((completion)&comp);
*** DEADLOCK ***
New way:
Chain exists of:
mutexB --> mutexA --> (completion)&comp
Possible unsafe locking scenario by crosslock:
CPU0 CPU1 CPU2
---- ---- ----
lock(mutexA);
lock((completion)&comp);
lock(mutexB);
lock(mutexA);
lock(mutexB);
unlock((completion)&comp);
*** DEADLOCK ***
Signed-off-by: Steven Rostedt (VMware) <[email protected]>
---
kernel/locking/lockdep.c | 72 ++++++++++++++++++++++++++++++++++++++++--------
1 file changed, 61 insertions(+), 11 deletions(-)
diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c
index db933d063bfc..be15f86d47f0 100644
--- a/kernel/locking/lockdep.c
+++ b/kernel/locking/lockdep.c
@@ -1128,6 +1128,7 @@ print_circular_lock_scenario(struct held_lock *src,
struct lock_class *source = hlock_class(src);
struct lock_class *target = hlock_class(tgt);
struct lock_class *parent = prt->class;
+ const char *spaces = "";
/*
* A direct locking problem where unsafe_class lock is taken
@@ -1154,31 +1155,61 @@ print_circular_lock_scenario(struct held_lock *src,
if (cross_lock(tgt->instance)) {
printk(" Possible unsafe locking scenario by crosslock:\n\n");
- printk(" CPU0 CPU1\n");
- printk(" ---- ----\n");
+ printk(" CPU0 CPU1");
+ if (parent != source)
+ printk(KERN_CONT " CPU2");
+ printk(KERN_CONT "\n");
+ printk(" ---- ----");
+ if (parent != source)
+ printk(KERN_CONT " ----");
+ printk(KERN_CONT "\n");
printk(" lock(");
__print_lock_name(parent);
printk(KERN_CONT ");\n");
printk(" lock(");
__print_lock_name(target);
printk(KERN_CONT ");\n");
- printk(" lock(");
+ if (parent != source) {
+ printk(" lock(");
+ __print_lock_name(source);
+ printk(KERN_CONT ");\n");
+ printk(" lock(");
+ __print_lock_name(parent);
+ printk(KERN_CONT ");\n");
+ spaces = " ";
+ }
+ printk("%s lock(", spaces);
__print_lock_name(source);
printk(KERN_CONT ");\n");
- printk(" unlock(");
+ printk("%s unlock(",spaces);
__print_lock_name(target);
printk(KERN_CONT ");\n");
printk("\n *** DEADLOCK ***\n\n");
} else {
printk(" Possible unsafe locking scenario:\n\n");
- printk(" CPU0 CPU1\n");
- printk(" ---- ----\n");
+ printk(" CPU0 CPU1");
+ if (parent != source)
+ printk(KERN_CONT " CPU2");
+ printk(KERN_CONT "\n");
+ printk(" ---- ----");
+ if (parent != source)
+ printk(KERN_CONT " ----");
+ printk(KERN_CONT "\n");
printk(" lock(");
__print_lock_name(target);
printk(KERN_CONT ");\n");
printk(" lock(");
__print_lock_name(parent);
printk(KERN_CONT ");\n");
+ if (parent != source) {
+ spaces = " ";
+ printk("%s lock(", spaces);
+ __print_lock_name(source);
+ printk(KERN_CONT ");\n");
+ printk("%s lock(", spaces);
+ __print_lock_name(parent);
+ printk(KERN_CONT ");\n");
+ }
printk(" lock(");
__print_lock_name(target);
printk(KERN_CONT ");\n");
@@ -1500,6 +1531,7 @@ print_irq_lock_scenario(struct lock_list *safe_entry,
struct lock_class *safe_class = safe_entry->class;
struct lock_class *unsafe_class = unsafe_entry->class;
struct lock_class *middle_class = prev_class;
+ const char *spaces = "";
if (middle_class == safe_class)
middle_class = next_class;
@@ -1528,18 +1560,36 @@ print_irq_lock_scenario(struct lock_list *safe_entry,
}
printk(" Possible interrupt unsafe locking scenario:\n\n");
- printk(" CPU0 CPU1\n");
- printk(" ---- ----\n");
+ printk(" CPU0 CPU1");
+ if (middle_class != unsafe_class)
+ printk(KERN_CONT " CPU2");
+ printk(KERN_CONT "\n");
+ printk(" ---- ----");
+ if (middle_class != unsafe_class)
+ printk(KERN_CONT " ----");
+ printk(KERN_CONT "\n");
printk(" lock(");
__print_lock_name(unsafe_class);
printk(KERN_CONT ");\n");
- printk(" local_irq_disable();\n");
- printk(" lock(");
+ if (middle_class != unsafe_class) {
+ spaces = " ";
+ printk(" local_irq_disable();\n");
+ printk(" lock(");
+ __print_lock_name(middle_class);
+ printk(KERN_CONT ");\n");
+ }
+ printk("%s local_irq_disable();\n", spaces);
+ printk("%s lock(", spaces);
__print_lock_name(safe_class);
printk(KERN_CONT ");\n");
- printk(" lock(");
+ printk("%s lock(", spaces);
__print_lock_name(middle_class);
printk(KERN_CONT ");\n");
+ if (middle_class != unsafe_class) {
+ printk(" lock(");
+ __print_lock_name(unsafe_class);
+ printk(KERN_CONT ");\n");
+ }
printk(" <Interrupt>\n");
printk(" lock(");
__print_lock_name(safe_class);
--
2.13.6
On Thu, Dec 14, 2017 at 12:38:52PM -0500, Steven Rostedt wrote:
>
> Currently, when lockdep detects a possible deadlock scenario that involves 3
> or more levels, it just shows the chain, and a CPU sequence order of the
> first and last part of the scenario, leaving out the middle level and this
> can take a bit of effort to understand. By adding a third level, it becomes
> easier to see where the deadlock is.
So is anybody actually using this? This (together with the callchain for
#0) is always the first thing of the lockdep output I throw away.
On Thu, 14 Dec 2017 18:59:31 +0100
Peter Zijlstra <[email protected]> wrote:
> On Thu, Dec 14, 2017 at 12:38:52PM -0500, Steven Rostedt wrote:
> >
> > Currently, when lockdep detects a possible deadlock scenario that involves 3
> > or more levels, it just shows the chain, and a CPU sequence order of the
> > first and last part of the scenario, leaving out the middle level and this
> > can take a bit of effort to understand. By adding a third level, it becomes
> > easier to see where the deadlock is.
>
> So is anybody actually using this? This (together with the callchain for
> #0) is always the first thing of the lockdep output I throw away.
Um, most people that post lockdep issues do (including myself). You are
unique and understand lockdep inside and out, so this doesn't help you.
I've had talks accepted on how to read lockdep output (having a talk on
how to read output shows it's not trivial at all to do so). Most people
have no idea what the lockdep output means. This is the only part
that "normal" people appear to understand from it. I've seen this
part of the output posted many times on the mailing list to discuss
deadlocks. I've been asked by many people to add this change, I just
never had time to implement it.
-- Steve
On 2017-12-14 12:59 PM, Peter Zijlstra wrote:
> On Thu, Dec 14, 2017 at 12:38:52PM -0500, Steven Rostedt wrote:
>>
>> Currently, when lockdep detects a possible deadlock scenario that involves 3
>> or more levels, it just shows the chain, and a CPU sequence order of the
>> first and last part of the scenario, leaving out the middle level and this
>> can take a bit of effort to understand. By adding a third level, it becomes
>> easier to see where the deadlock is.
>
> So is anybody actually using this? This (together with the callchain for
> #0) is always the first thing of the lockdep output I throw away.
>
Yes :-). The other stuff is unreadable to people not you.
Dhaval
On Tue, Dec 19, 2017 at 11:30:29AM -0500, Dhaval Giani wrote:
> On 2017-12-14 12:59 PM, Peter Zijlstra wrote:
> > On Thu, Dec 14, 2017 at 12:38:52PM -0500, Steven Rostedt wrote:
> >>
> >> Currently, when lockdep detects a possible deadlock scenario that involves 3
> >> or more levels, it just shows the chain, and a CPU sequence order of the
> >> first and last part of the scenario, leaving out the middle level and this
> >> can take a bit of effort to understand. By adding a third level, it becomes
> >> easier to see where the deadlock is.
> >
> > So is anybody actually using this? This (together with the callchain for
> > #0) is always the first thing of the lockdep output I throw away.
> >
>
> Yes :-). The other stuff is unreadable to people not you.
It really isn't that hard, Its mostly a question of TL;DR.
#0 is useless and should be thrown out
#1 shows where we take #1 while holding #0
..
#n shows where we take #n while holding #n-1
And the bottom callstack shows where we take #0 while holding #n. Which
gets you a nice circle in your graph, which spells deadlock.
Plenty people have shown they get this stuff.
On Tue, 19 Dec 2017 17:46:19 +0100
Peter Zijlstra <[email protected]> wrote:
> It really isn't that hard, Its mostly a question of TL;DR.
>
> #0 is useless and should be thrown out
> #1 shows where we take #1 while holding #0
> ..
> #n shows where we take #n while holding #n-1
>
> And the bottom callstack shows where we take #0 while holding #n. Which
> gets you a nice circle in your graph, which spells deadlock.
>
> Plenty people have shown they get this stuff.
Then I suggest that you can either take my patch to improve the
visual or remove the visual completely, as nobody cares about it.
-- Steve
On 2017-12-19 11:52 AM, Steven Rostedt wrote:
> On Tue, 19 Dec 2017 17:46:19 +0100
> Peter Zijlstra <[email protected]> wrote:
>
>
>> It really isn't that hard, Its mostly a question of TL;DR.
>>
>> #0 is useless and should be thrown out
>> #1 shows where we take #1 while holding #0
>> ..
>> #n shows where we take #n while holding #n-1
>>
>> And the bottom callstack shows where we take #0 while holding #n. Which
>> gets you a nice circle in your graph, which spells deadlock.
>>
>> Plenty people have shown they get this stuff.
>
>
> Then I suggest that you can either take my patch to improve the
> visual or remove the visual completely, as nobody cares about it.
>
I prefer the former. As Steven has mentioned elsewhere, people find
lockdep output hard to follow (enough that he has given talks :) )
Dhaval
On Tue, Dec 19, 2017 at 11:52:38AM -0500, Steven Rostedt wrote:
> Then I suggest that you can either take my patch to improve the
> visual or remove the visual completely, as nobody cares about it.
Doesn't apply as is; but can you at least make it shut up if the chain
is longer than you support?
It's really annoying if it shows the AB-BA graphics when you're staring
at a 5-6 lock scenario.
On Tue, 19 Dec 2017 11:54:10 -0500
Dhaval Giani <[email protected]> wrote:
> On 2017-12-19 11:52 AM, Steven Rostedt wrote:
> > On Tue, 19 Dec 2017 17:46:19 +0100
> > Peter Zijlstra <[email protected]> wrote:
> >
> >
> >> It really isn't that hard, Its mostly a question of TL;DR.
> >>
> >> #0 is useless and should be thrown out
> >> #1 shows where we take #1 while holding #0
> >> ..
> >> #n shows where we take #n while holding #n-1
> >>
> >> And the bottom callstack shows where we take #0 while holding #n. Which
> >> gets you a nice circle in your graph, which spells deadlock.
> >>
> >> Plenty people have shown they get this stuff.
> >
> >
> > Then I suggest that you can either take my patch to improve the
> > visual or remove the visual completely, as nobody cares about it.
> >
>
> I prefer the former. As Steven has mentioned elsewhere, people find
> lockdep output hard to follow (enough that he has given talks :) )
>
Not to mention. There's commit logs that throw everything out except
for this information. See commits:
692b48258dda
5acb3cc2c2e9
7b7622bb95eb5
478fe3037b227
fdaf0a51bad49
1ddd45f8d76f0
63aea0dbab90a
1215e51edad12
f159b3c7cd45c
And those only go back to March of this year.
-- Steve
On Tue, 19 Dec 2017 18:07:16 +0100
Peter Zijlstra <[email protected]> wrote:
> On Tue, Dec 19, 2017 at 11:52:38AM -0500, Steven Rostedt wrote:
> > Then I suggest that you can either take my patch to improve the
> > visual or remove the visual completely, as nobody cares about it.
>
> Doesn't apply as is; but can you at least make it shut up if the chain
> is longer than you support?
I can add that.
>
> It's really annoying if it shows the AB-BA graphics when you're staring
> at a 5-6 lock scenario.
I could also make it dynamic, to handle the entire chain. But it will
make it longer.
-- Steve
On Tue, Dec 19, 2017 at 12:11:24PM -0500, Steven Rostedt wrote:
> On Tue, 19 Dec 2017 18:07:16 +0100
> Peter Zijlstra <[email protected]> wrote:
>
> > On Tue, Dec 19, 2017 at 11:52:38AM -0500, Steven Rostedt wrote:
> > > Then I suggest that you can either take my patch to improve the
> > > visual or remove the visual completely, as nobody cares about it.
> >
> > Doesn't apply as is; but can you at least make it shut up if the chain
> > is longer than you support?
>
> I can add that.
>
> >
> > It's really annoying if it shows the AB-BA graphics when you're staring
> > at a 5-6 lock scenario.
>
> I could also make it dynamic, to handle the entire chain. But it will
> make it longer.
Makes it too wide I think, people already have a problem quoting the
thing in emails (wrap mangles like stupid).
Take the below while you're poking at it.. that should hopefully make
the #0 stacktrace go away.
diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c
index 5fa1324a4f29..6aa431c4ad99 100644
--- a/kernel/locking/lockdep.c
+++ b/kernel/locking/lockdep.c
@@ -1085,7 +1085,8 @@ print_circular_bug_entry(struct lock_list *target, int depth)
printk("\n-> #%u", depth);
print_lock_name(target->class);
printk(KERN_CONT ":\n");
- print_stack_trace(&target->trace, 6);
+ if (depth)
+ print_stack_trace(&target->trace, 6);
return 0;
}