nmi_enter() does lockdep_off() and hence lockdep ignores everything.
And NMI context makes it impossible to do full IN-NMI tracking like we
do IN-HARDIRQ, that could result in graph_lock recursion.
However, since look_up_lock_class() is lockless, we can find the class
of a lock that has prior use and detect IN-NMI after USED, just not
USED after IN-NMI.
NOTE: By shifting the lockdep_off() recursion count to bit-16, we can
easily differentiate between actual recursion and off.
Signed-off-by: Peter Zijlstra (Intel) <[email protected]>
---
kernel/locking/lockdep.c | 53 ++++++++++++++++++++++++++++++++++++++++++++---
1 file changed, 50 insertions(+), 3 deletions(-)
--- a/kernel/locking/lockdep.c
+++ b/kernel/locking/lockdep.c
@@ -379,13 +379,13 @@ void lockdep_init_task(struct task_struc
void lockdep_off(void)
{
- current->lockdep_recursion++;
+ current->lockdep_recursion += BIT(16);
}
EXPORT_SYMBOL(lockdep_off);
void lockdep_on(void)
{
- current->lockdep_recursion--;
+ current->lockdep_recursion -= BIT(16);
}
EXPORT_SYMBOL(lockdep_on);
@@ -575,6 +575,7 @@ static const char *usage_str[] =
#include "lockdep_states.h"
#undef LOCKDEP_STATE
[LOCK_USED] = "INITIAL USE",
+ [LOCK_USAGE_STATES] = "IN-NMI",
};
#endif
@@ -787,6 +788,7 @@ static int count_matching_names(struct l
return count + 1;
}
+/* used from NMI context -- must be lockless */
static inline struct lock_class *
look_up_lock_class(const struct lockdep_map *lock, unsigned int subclass)
{
@@ -4463,6 +4465,34 @@ void lock_downgrade(struct lockdep_map *
}
EXPORT_SYMBOL_GPL(lock_downgrade);
+/* NMI context !!! */
+static void verify_lock_unused(struct lockdep_map *lock, struct held_lock *hlock, int subclass)
+{
+ struct lock_class *class = look_up_lock_class(lock, subclass);
+
+ /* if it doesn't have a class (yet), it certainly hasn't been used yet */
+ if (!class)
+ return;
+
+ if (!(class->usage_mask & LOCK_USED))
+ return;
+
+ hlock->class_idx = class - lock_classes;
+
+ print_usage_bug(current, hlock, LOCK_USED, LOCK_USAGE_STATES);
+}
+
+static bool lockdep_nmi(void)
+{
+ if (current->lockdep_recursion & 0xFFFF)
+ return false;
+
+ if (!in_nmi())
+ return false;
+
+ return true;
+}
+
/*
* We are not always called with irqs disabled - do that here,
* and also avoid lockdep recursion:
@@ -4473,8 +4503,25 @@ void lock_acquire(struct lockdep_map *lo
{
unsigned long flags;
- if (unlikely(current->lockdep_recursion))
+ if (unlikely(current->lockdep_recursion)) {
+ /* XXX allow trylock from NMI ?!? */
+ if (lockdep_nmi() && !trylock) {
+ struct held_lock hlock;
+
+ hlock.acquire_ip = ip;
+ hlock.instance = lock;
+ hlock.nest_lock = nest_lock;
+ hlock.irq_context = 2; // XXX
+ hlock.trylock = trylock;
+ hlock.read = read;
+ hlock.check = check;
+ hlock.hardirqs_off = true;
+ hlock.references = 0;
+
+ verify_lock_unused(lock, &hlock, subclass);
+ }
return;
+ }
raw_local_irq_save(flags);
check_flags(flags);
On Fri, 21 Feb 2020 14:34:17 +0100
Peter Zijlstra <[email protected]> wrote:
> --- a/kernel/locking/lockdep.c
> +++ b/kernel/locking/lockdep.c
> @@ -379,13 +379,13 @@ void lockdep_init_task(struct task_struc
>
> void lockdep_off(void)
> {
> - current->lockdep_recursion++;
> + current->lockdep_recursion += BIT(16);
> }
> EXPORT_SYMBOL(lockdep_off);
>
> void lockdep_on(void)
> {
> - current->lockdep_recursion--;
> + current->lockdep_recursion -= BIT(16);
> }
> EXPORT_SYMBOL(lockdep_on);
>
> +
> +static bool lockdep_nmi(void)
> +{
> + if (current->lockdep_recursion & 0xFFFF)
Nitpick, but the association with bit 16 and this mask really should be
defined as a macro somewhere and not have hard coded numbers.
-- Steve
> + return false;
> +
> + if (!in_nmi())
> + return false;
> +
> + return true;
> +}
> +
On Fri, Feb 21, 2020 at 10:08:55AM -0500, Steven Rostedt wrote:
> On Fri, 21 Feb 2020 14:34:17 +0100
> Peter Zijlstra <[email protected]> wrote:
>
> > --- a/kernel/locking/lockdep.c
> > +++ b/kernel/locking/lockdep.c
> > @@ -379,13 +379,13 @@ void lockdep_init_task(struct task_struc
> >
> > void lockdep_off(void)
> > {
> > - current->lockdep_recursion++;
> > + current->lockdep_recursion += BIT(16);
> > }
> > EXPORT_SYMBOL(lockdep_off);
> >
> > void lockdep_on(void)
> > {
> > - current->lockdep_recursion--;
> > + current->lockdep_recursion -= BIT(16);
> > }
> > EXPORT_SYMBOL(lockdep_on);
> >
>
> > +
> > +static bool lockdep_nmi(void)
> > +{
> > + if (current->lockdep_recursion & 0xFFFF)
>
> Nitpick, but the association with bit 16 and this mask really should be
> defined as a macro somewhere and not have hard coded numbers.
Right, I suppose I can do something like:
#define LOCKDEP_RECURSION_BITS 16
#define LOCKDEP_OFF (1U << LOCKDEP_RECURSION_BITS)
#define LOCKDEP_RECURSION_MASK (LOCKDEP_OFF - 1)
On Fri, 21 Feb 2020 21:25:11 +0100
Peter Zijlstra <[email protected]> wrote:
> > Nitpick, but the association with bit 16 and this mask really should be
> > defined as a macro somewhere and not have hard coded numbers.
>
> Right, I suppose I can do something like:
>
> #define LOCKDEP_RECURSION_BITS 16
> #define LOCKDEP_OFF (1U << LOCKDEP_RECURSION_BITS)
> #define LOCKDEP_RECURSION_MASK (LOCKDEP_OFF - 1)
LGTM
-- Steve
On Fri, Feb 21, 2020 at 09:25:11PM +0100, Peter Zijlstra wrote:
> On Fri, Feb 21, 2020 at 10:08:55AM -0500, Steven Rostedt wrote:
> > On Fri, 21 Feb 2020 14:34:17 +0100
> > Peter Zijlstra <[email protected]> wrote:
> >
> > > --- a/kernel/locking/lockdep.c
> > > +++ b/kernel/locking/lockdep.c
> > > @@ -379,13 +379,13 @@ void lockdep_init_task(struct task_struc
> > >
> > > void lockdep_off(void)
> > > {
> > > - current->lockdep_recursion++;
> > > + current->lockdep_recursion += BIT(16);
> > > }
> > > EXPORT_SYMBOL(lockdep_off);
> > >
> > > void lockdep_on(void)
> > > {
> > > - current->lockdep_recursion--;
> > > + current->lockdep_recursion -= BIT(16);
> > > }
> > > EXPORT_SYMBOL(lockdep_on);
> > >
> >
> > > +
> > > +static bool lockdep_nmi(void)
> > > +{
> > > + if (current->lockdep_recursion & 0xFFFF)
> >
> > Nitpick, but the association with bit 16 and this mask really should be
> > defined as a macro somewhere and not have hard coded numbers.
>
> Right, I suppose I can do something like:
>
> #define LOCKDEP_RECURSION_BITS 16
> #define LOCKDEP_OFF (1U << LOCKDEP_RECURSION_BITS)
> #define LOCKDEP_RECURSION_MASK (LOCKDEP_OFF - 1)
With that I'd say
Reviewed-by: Frederic Weisbecker <[email protected]>
On Fri, Feb 21, 2020 at 02:34:17PM +0100, Peter Zijlstra wrote:
> nmi_enter() does lockdep_off() and hence lockdep ignores everything.
>
> And NMI context makes it impossible to do full IN-NMI tracking like we
> do IN-HARDIRQ, that could result in graph_lock recursion.
The patch makes sense to me.
Reviewed-by: Joel Fernandes (Google) <[email protected]>
NOTE:
Also, I was wondering if we can detect the graph_lock recursion case and
avoid doing anything bad, that way we enable more of the lockdep
functionality for NMI where possible. Not sure if the suggestion makes sense
though!
thanks,
- Joel
> However, since look_up_lock_class() is lockless, we can find the class
> of a lock that has prior use and detect IN-NMI after USED, just not
> USED after IN-NMI.
>
> NOTE: By shifting the lockdep_off() recursion count to bit-16, we can
> easily differentiate between actual recursion and off.
>
> Signed-off-by: Peter Zijlstra (Intel) <[email protected]>
> ---
> kernel/locking/lockdep.c | 53 ++++++++++++++++++++++++++++++++++++++++++++---
> 1 file changed, 50 insertions(+), 3 deletions(-)
>
> --- a/kernel/locking/lockdep.c
> +++ b/kernel/locking/lockdep.c
> @@ -379,13 +379,13 @@ void lockdep_init_task(struct task_struc
>
> void lockdep_off(void)
> {
> - current->lockdep_recursion++;
> + current->lockdep_recursion += BIT(16);
> }
> EXPORT_SYMBOL(lockdep_off);
>
> void lockdep_on(void)
> {
> - current->lockdep_recursion--;
> + current->lockdep_recursion -= BIT(16);
> }
> EXPORT_SYMBOL(lockdep_on);
>
> @@ -575,6 +575,7 @@ static const char *usage_str[] =
> #include "lockdep_states.h"
> #undef LOCKDEP_STATE
> [LOCK_USED] = "INITIAL USE",
> + [LOCK_USAGE_STATES] = "IN-NMI",
> };
> #endif
>
> @@ -787,6 +788,7 @@ static int count_matching_names(struct l
> return count + 1;
> }
>
> +/* used from NMI context -- must be lockless */
> static inline struct lock_class *
> look_up_lock_class(const struct lockdep_map *lock, unsigned int subclass)
> {
> @@ -4463,6 +4465,34 @@ void lock_downgrade(struct lockdep_map *
> }
> EXPORT_SYMBOL_GPL(lock_downgrade);
>
> +/* NMI context !!! */
> +static void verify_lock_unused(struct lockdep_map *lock, struct held_lock *hlock, int subclass)
> +{
> + struct lock_class *class = look_up_lock_class(lock, subclass);
> +
> + /* if it doesn't have a class (yet), it certainly hasn't been used yet */
> + if (!class)
> + return;
> +
> + if (!(class->usage_mask & LOCK_USED))
> + return;
> +
> + hlock->class_idx = class - lock_classes;
> +
> + print_usage_bug(current, hlock, LOCK_USED, LOCK_USAGE_STATES);
> +}
> +
> +static bool lockdep_nmi(void)
> +{
> + if (current->lockdep_recursion & 0xFFFF)
> + return false;
> +
> + if (!in_nmi())
> + return false;
> +
> + return true;
> +}
> +
> /*
> * We are not always called with irqs disabled - do that here,
> * and also avoid lockdep recursion:
> @@ -4473,8 +4503,25 @@ void lock_acquire(struct lockdep_map *lo
> {
> unsigned long flags;
>
> - if (unlikely(current->lockdep_recursion))
> + if (unlikely(current->lockdep_recursion)) {
> + /* XXX allow trylock from NMI ?!? */
> + if (lockdep_nmi() && !trylock) {
> + struct held_lock hlock;
> +
> + hlock.acquire_ip = ip;
> + hlock.instance = lock;
> + hlock.nest_lock = nest_lock;
> + hlock.irq_context = 2; // XXX
> + hlock.trylock = trylock;
> + hlock.read = read;
> + hlock.check = check;
> + hlock.hardirqs_off = true;
> + hlock.references = 0;
> +
> + verify_lock_unused(lock, &hlock, subclass);
> + }
> return;
> + }
>
> raw_local_irq_save(flags);
> check_flags(flags);
>
>
On Fri, Feb 21, 2020 at 10:08:43PM -0500, Joel Fernandes wrote:
> On Fri, Feb 21, 2020 at 02:34:17PM +0100, Peter Zijlstra wrote:
> > nmi_enter() does lockdep_off() and hence lockdep ignores everything.
> >
> > And NMI context makes it impossible to do full IN-NMI tracking like we
> > do IN-HARDIRQ, that could result in graph_lock recursion.
>
> The patch makes sense to me.
>
> Reviewed-by: Joel Fernandes (Google) <[email protected]>
>
> NOTE:
> Also, I was wondering if we can detect the graph_lock recursion case and
> avoid doing anything bad, that way we enable more of the lockdep
> functionality for NMI where possible. Not sure if the suggestion makes sense
> though!
Yeah, I considered playing trylock games, but figured I shouldn't make
it more complicated that it needs to be.
On Mon, Feb 24, 2020 at 11:10:50AM +0100, Peter Zijlstra wrote:
> On Fri, Feb 21, 2020 at 10:08:43PM -0500, Joel Fernandes wrote:
> > On Fri, Feb 21, 2020 at 02:34:17PM +0100, Peter Zijlstra wrote:
> > > nmi_enter() does lockdep_off() and hence lockdep ignores everything.
> > >
> > > And NMI context makes it impossible to do full IN-NMI tracking like we
> > > do IN-HARDIRQ, that could result in graph_lock recursion.
> >
> > The patch makes sense to me.
> >
> > Reviewed-by: Joel Fernandes (Google) <[email protected]>
> >
> > NOTE:
> > Also, I was wondering if we can detect the graph_lock recursion case and
> > avoid doing anything bad, that way we enable more of the lockdep
> > functionality for NMI where possible. Not sure if the suggestion makes sense
> > though!
>
> Yeah, I considered playing trylock games, but figured I shouldn't make
> it more complicated that it needs to be.
Yes, I agree with you. Thanks.
- Joel
The following commit has been merged into the locking/core branch of tip:
Commit-ID: f6f48e18040402136874a6a71611e081b4d0788a
Gitweb: https://git.kernel.org/tip/f6f48e18040402136874a6a71611e081b4d0788a
Author: Peter Zijlstra <[email protected]>
AuthorDate: Thu, 20 Feb 2020 09:45:02 +01:00
Committer: Peter Zijlstra <[email protected]>
CommitterDate: Fri, 20 Mar 2020 13:06:25 +01:00
lockdep: Teach lockdep about "USED" <- "IN-NMI" inversions
nmi_enter() does lockdep_off() and hence lockdep ignores everything.
And NMI context makes it impossible to do full IN-NMI tracking like we
do IN-HARDIRQ, that could result in graph_lock recursion.
However, since look_up_lock_class() is lockless, we can find the class
of a lock that has prior use and detect IN-NMI after USED, just not
USED after IN-NMI.
NOTE: By shifting the lockdep_off() recursion count to bit-16, we can
easily differentiate between actual recursion and off.
Signed-off-by: Peter Zijlstra (Intel) <[email protected]>
Reviewed-by: Frederic Weisbecker <[email protected]>
Reviewed-by: Joel Fernandes (Google) <[email protected]>
Link: https://lkml.kernel.org/r/[email protected]
---
kernel/locking/lockdep.c | 62 +++++++++++++++++++++++++++++++++++++--
1 file changed, 59 insertions(+), 3 deletions(-)
diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c
index 47e3acb..4c3b1cc 100644
--- a/kernel/locking/lockdep.c
+++ b/kernel/locking/lockdep.c
@@ -393,15 +393,22 @@ void lockdep_init_task(struct task_struct *task)
task->lockdep_recursion = 0;
}
+/*
+ * Split the recrursion counter in two to readily detect 'off' vs recursion.
+ */
+#define LOCKDEP_RECURSION_BITS 16
+#define LOCKDEP_OFF (1U << LOCKDEP_RECURSION_BITS)
+#define LOCKDEP_RECURSION_MASK (LOCKDEP_OFF - 1)
+
void lockdep_off(void)
{
- current->lockdep_recursion++;
+ current->lockdep_recursion += LOCKDEP_OFF;
}
EXPORT_SYMBOL(lockdep_off);
void lockdep_on(void)
{
- current->lockdep_recursion--;
+ current->lockdep_recursion -= LOCKDEP_OFF;
}
EXPORT_SYMBOL(lockdep_on);
@@ -597,6 +604,7 @@ static const char *usage_str[] =
#include "lockdep_states.h"
#undef LOCKDEP_STATE
[LOCK_USED] = "INITIAL USE",
+ [LOCK_USAGE_STATES] = "IN-NMI",
};
#endif
@@ -809,6 +817,7 @@ static int count_matching_names(struct lock_class *new_class)
return count + 1;
}
+/* used from NMI context -- must be lockless */
static inline struct lock_class *
look_up_lock_class(const struct lockdep_map *lock, unsigned int subclass)
{
@@ -4720,6 +4729,36 @@ void lock_downgrade(struct lockdep_map *lock, unsigned long ip)
}
EXPORT_SYMBOL_GPL(lock_downgrade);
+/* NMI context !!! */
+static void verify_lock_unused(struct lockdep_map *lock, struct held_lock *hlock, int subclass)
+{
+#ifdef CONFIG_PROVE_LOCKING
+ struct lock_class *class = look_up_lock_class(lock, subclass);
+
+ /* if it doesn't have a class (yet), it certainly hasn't been used yet */
+ if (!class)
+ return;
+
+ if (!(class->usage_mask & LOCK_USED))
+ return;
+
+ hlock->class_idx = class - lock_classes;
+
+ print_usage_bug(current, hlock, LOCK_USED, LOCK_USAGE_STATES);
+#endif
+}
+
+static bool lockdep_nmi(void)
+{
+ if (current->lockdep_recursion & LOCKDEP_RECURSION_MASK)
+ return false;
+
+ if (!in_nmi())
+ return false;
+
+ return true;
+}
+
/*
* We are not always called with irqs disabled - do that here,
* and also avoid lockdep recursion:
@@ -4730,8 +4769,25 @@ void lock_acquire(struct lockdep_map *lock, unsigned int subclass,
{
unsigned long flags;
- if (unlikely(current->lockdep_recursion))
+ if (unlikely(current->lockdep_recursion)) {
+ /* XXX allow trylock from NMI ?!? */
+ if (lockdep_nmi() && !trylock) {
+ struct held_lock hlock;
+
+ hlock.acquire_ip = ip;
+ hlock.instance = lock;
+ hlock.nest_lock = nest_lock;
+ hlock.irq_context = 2; // XXX
+ hlock.trylock = trylock;
+ hlock.read = read;
+ hlock.check = check;
+ hlock.hardirqs_off = true;
+ hlock.references = 0;
+
+ verify_lock_unused(lock, &hlock, subclass);
+ }
return;
+ }
raw_local_irq_save(flags);
check_flags(flags);