Hi,
Arnaldo reported that "perf test sigtrap" fails on PREEMPT_RT. Sending
the signal gets delayed until event_sched_out() which then uses
task_work_add() for its delivery. This breaks on PREEMPT_RT because the
signal is delivered with disabled preemption.
While looking at this, I also stumbled upon __perf_pending_irq() which
requires disabled interrupts but this is not the case on PREEMPT_RT.
This series aim to address both issues while not introducing a new issue
at the same time ;)
Any testing is appreciated.
Sebastian
Only if perf_event::pending_sigtrap is zero, the irq_work accounted by
increminging perf_event::nr_pending. The member perf_event::pending_addr
might be overwritten by a subsequent event if the signal was not yet
delivered and is expected. The irq_work will not be enqeueued again
because it has a check to be only enqueued once.
Move irq_work_queue() to where the counter is incremented and
perf_event::pending_sigtrap is set to make it more obvious that the
irq_work is scheduled once.
Signed-off-by: Sebastian Andrzej Siewior <[email protected]>
---
kernel/events/core.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/kernel/events/core.c b/kernel/events/core.c
index f0f0f71213a1d..c7a0274c662c8 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -9595,6 +9595,7 @@ static int __perf_event_overflow(struct perf_event *event,
if (!event->pending_sigtrap) {
event->pending_sigtrap = pending_id;
local_inc(&event->ctx->nr_pending);
+ irq_work_queue(&event->pending_irq);
} else if (event->attr.exclude_kernel && valid_sample) {
/*
* Should not be able to return to user space without
@@ -9614,7 +9615,6 @@ static int __perf_event_overflow(struct perf_event *event,
event->pending_addr = 0;
if (valid_sample && (data->sample_flags & PERF_SAMPLE_ADDR))
event->pending_addr = data->addr;
- irq_work_queue(&event->pending_irq);
}
READ_ONCE(event->overflow_handler)(event, data, regs);
--
2.43.0
A signal is delivered by raising irq_work() which works from any context
including NMI. irq_work() can be delayed if the architecture does not
provide an interrupt vector. In order not to lose a signal, the signal
is injected via task_work during event_sched_out().
Instead going via irq_work, the signal could be added directly via
task_work. The signal is sent to current and can be enqueued on its
return path to userland instead triggering irq_work.
Queue signal via task_work. Remove perf_event::pending_sigtrap and
and use perf_event::pending_work instead.
Signed-off-by: Sebastian Andrzej Siewior <[email protected]>
---
include/linux/perf_event.h | 3 +--
kernel/events/core.c | 31 ++++++-------------------------
2 files changed, 7 insertions(+), 27 deletions(-)
diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index d2a15c0c6f8a9..24ac6765146c7 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -781,7 +781,6 @@ struct perf_event {
unsigned int pending_wakeup;
unsigned int pending_kill;
unsigned int pending_disable;
- unsigned int pending_sigtrap;
unsigned long pending_addr; /* SIGTRAP */
struct irq_work pending_irq;
struct callback_head pending_task;
@@ -959,7 +958,7 @@ struct perf_event_context {
struct rcu_head rcu_head;
/*
- * Sum (event->pending_sigtrap + event->pending_work)
+ * Sum (event->pending_work + event->pending_work)
*
* The SIGTRAP is targeted at ctx->task, as such it won't do changing
* that until the signal is delivered.
diff --git a/kernel/events/core.c b/kernel/events/core.c
index c7a0274c662c8..e4a85d6d2d685 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -2283,21 +2283,6 @@ event_sched_out(struct perf_event *event, struct perf_event_context *ctx)
state = PERF_EVENT_STATE_OFF;
}
- if (event->pending_sigtrap) {
- bool dec = true;
-
- event->pending_sigtrap = 0;
- if (state != PERF_EVENT_STATE_OFF &&
- !event->pending_work) {
- event->pending_work = 1;
- dec = false;
- WARN_ON_ONCE(!atomic_long_inc_not_zero(&event->refcount));
- task_work_add(current, &event->pending_task, TWA_RESUME);
- }
- if (dec)
- local_dec(&event->ctx->nr_pending);
- }
-
perf_event_set_state(event, state);
if (!is_software_event(event))
@@ -6741,11 +6726,6 @@ static void __perf_pending_irq(struct perf_event *event)
* Yay, we hit home and are in the context of the event.
*/
if (cpu == smp_processor_id()) {
- if (event->pending_sigtrap) {
- event->pending_sigtrap = 0;
- perf_sigtrap(event);
- local_dec(&event->ctx->nr_pending);
- }
if (event->pending_disable) {
event->pending_disable = 0;
perf_event_disable_local(event);
@@ -9592,14 +9572,15 @@ static int __perf_event_overflow(struct perf_event *event,
if (regs)
pending_id = hash32_ptr((void *)instruction_pointer(regs)) ?: 1;
- if (!event->pending_sigtrap) {
- event->pending_sigtrap = pending_id;
+ if (!event->pending_work) {
+ event->pending_work = pending_id;
local_inc(&event->ctx->nr_pending);
- irq_work_queue(&event->pending_irq);
+ WARN_ON_ONCE(!atomic_long_inc_not_zero(&event->refcount));
+ task_work_add(current, &event->pending_task, TWA_RESUME);
} else if (event->attr.exclude_kernel && valid_sample) {
/*
* Should not be able to return to user space without
- * consuming pending_sigtrap; with exceptions:
+ * consuming pending_work; with exceptions:
*
* 1. Where !exclude_kernel, events can overflow again
* in the kernel without returning to user space.
@@ -9609,7 +9590,7 @@ static int __perf_event_overflow(struct perf_event *event,
* To approximate progress (with false negatives),
* check 32-bit hash of the current IP.
*/
- WARN_ON_ONCE(event->pending_sigtrap != pending_id);
+ WARN_ON_ONCE(event->pending_work != pending_id);
}
event->pending_addr = 0;
--
2.43.0
perf_swevent_get_recursion_context() is supposed to avoid recursion.
This requires to remain on the same CPU in order to decrement/ increment
the same counter. This is done by using preempt_disable(). Having
preemption disabled while sending a signal leads to locking problems on
PREEMPT_RT because sighand, a spinlock_t, becomes a sleeping lock.
This callback runs in task context and currently delivers only a signal
to "itself". Any kind of recusrion protection in this context is not
required.
Remove recursion protection in perf_pending_task().
Signed-off-by: Sebastian Andrzej Siewior <[email protected]>
---
kernel/events/core.c | 12 ------------
1 file changed, 12 deletions(-)
diff --git a/kernel/events/core.c b/kernel/events/core.c
index e4a85d6d2d685..240e9b7dfd3e9 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -6785,14 +6785,6 @@ static void perf_pending_irq(struct irq_work *entry)
static void perf_pending_task(struct callback_head *head)
{
struct perf_event *event = container_of(head, struct perf_event, pending_task);
- int rctx;
-
- /*
- * If we 'fail' here, that's OK, it means recursion is already disabled
- * and we won't recurse 'further'.
- */
- preempt_disable_notrace();
- rctx = perf_swevent_get_recursion_context();
if (event->pending_work) {
event->pending_work = 0;
@@ -6800,10 +6792,6 @@ static void perf_pending_task(struct callback_head *head)
local_dec(&event->ctx->nr_pending);
}
- if (rctx >= 0)
- perf_swevent_put_recursion_context(rctx);
- preempt_enable_notrace();
-
put_event(event);
}
--
2.43.0
perf_pending_irq() invokes perf_event_wakeup() and __perf_pending_irq().
The former is in charge of waking any tasks which wait to be woken up
while the latter disables perf-events.
The irq_work perf_pending_irq(), while this an irq_work, the callback
is invoked in thread context on PREEMPT_RT. This is needed because all
the waking functions (wake_up_all(), kill_fasync()) acquire sleep locks
which must not be used with disabled interrupts.
Disabling events, as done by __perf_pending_irq(), expects a hardirq
context and disabled interrupts. This requirement is not fulfilled on
PREEMPT_RT.
Split functionality based on perf_event::pending_disable into irq_work
named `pending_disable_irq' and invoke it in hardirq context on
PREEMPT_RT. Rename the split out callback to perf_pending_disable().
Signed-off-by: Sebastian Andrzej Siewior <[email protected]>
---
include/linux/perf_event.h | 1 +
kernel/events/core.c | 29 ++++++++++++++++++++++-------
2 files changed, 23 insertions(+), 7 deletions(-)
diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index 24ac6765146c7..c1c6600541657 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -783,6 +783,7 @@ struct perf_event {
unsigned int pending_disable;
unsigned long pending_addr; /* SIGTRAP */
struct irq_work pending_irq;
+ struct irq_work pending_disable_irq;
struct callback_head pending_task;
unsigned int pending_work;
diff --git a/kernel/events/core.c b/kernel/events/core.c
index 240e9b7dfd3e9..eef7df33eea4d 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -2449,7 +2449,7 @@ static void __perf_event_disable(struct perf_event *event,
* hold the top-level event's child_mutex, so any descendant that
* goes to exit will block in perf_event_exit_event().
*
- * When called from perf_pending_irq it's OK because event->ctx
+ * When called from perf_pending_disable it's OK because event->ctx
* is the current context on this CPU and preemption is disabled,
* hence we can't get into perf_event_task_sched_out for this context.
*/
@@ -2489,7 +2489,7 @@ EXPORT_SYMBOL_GPL(perf_event_disable);
void perf_event_disable_inatomic(struct perf_event *event)
{
event->pending_disable = 1;
- irq_work_queue(&event->pending_irq);
+ irq_work_queue(&event->pending_disable_irq);
}
#define MAX_INTERRUPTS (~0ULL)
@@ -5175,6 +5175,7 @@ static void perf_addr_filters_splice(struct perf_event *event,
static void _free_event(struct perf_event *event)
{
irq_work_sync(&event->pending_irq);
+ irq_work_sync(&event->pending_disable_irq);
unaccount_event(event);
@@ -6711,7 +6712,7 @@ static void perf_sigtrap(struct perf_event *event)
/*
* Deliver the pending work in-event-context or follow the context.
*/
-static void __perf_pending_irq(struct perf_event *event)
+static void __perf_pending_disable(struct perf_event *event)
{
int cpu = READ_ONCE(event->oncpu);
@@ -6749,11 +6750,26 @@ static void __perf_pending_irq(struct perf_event *event)
* irq_work_queue(); // FAILS
*
* irq_work_run()
- * perf_pending_irq()
+ * perf_pending_disable()
*
* But the event runs on CPU-B and wants disabling there.
*/
- irq_work_queue_on(&event->pending_irq, cpu);
+ irq_work_queue_on(&event->pending_disable_irq, cpu);
+}
+
+static void perf_pending_disable(struct irq_work *entry)
+{
+ struct perf_event *event = container_of(entry, struct perf_event, pending_disable_irq);
+ int rctx;
+
+ /*
+ * If we 'fail' here, that's OK, it means recursion is already disabled
+ * and we won't recurse 'further'.
+ */
+ rctx = perf_swevent_get_recursion_context();
+ __perf_pending_disable(event);
+ if (rctx >= 0)
+ perf_swevent_put_recursion_context(rctx);
}
static void perf_pending_irq(struct irq_work *entry)
@@ -6776,8 +6792,6 @@ static void perf_pending_irq(struct irq_work *entry)
perf_event_wakeup(event);
}
- __perf_pending_irq(event);
-
if (rctx >= 0)
perf_swevent_put_recursion_context(rctx);
}
@@ -11904,6 +11918,7 @@ perf_event_alloc(struct perf_event_attr *attr, int cpu,
init_waitqueue_head(&event->waitq);
init_irq_work(&event->pending_irq, perf_pending_irq);
+ event->pending_disable_irq = IRQ_WORK_INIT_HARD(perf_pending_disable);
init_task_work(&event->pending_task, perf_pending_task);
mutex_init(&event->mmap_mutex);
--
2.43.0
On Fri, Mar 08, 2024 at 06:51PM +0100, Sebastian Andrzej Siewior wrote:
> Hi,
>
> Arnaldo reported that "perf test sigtrap" fails on PREEMPT_RT. Sending
> the signal gets delayed until event_sched_out() which then uses
> task_work_add() for its delivery. This breaks on PREEMPT_RT because the
> signal is delivered with disabled preemption.
>
> While looking at this, I also stumbled upon __perf_pending_irq() which
> requires disabled interrupts but this is not the case on PREEMPT_RT.
>
> This series aim to address both issues while not introducing a new issue
> at the same time ;)
> Any testing is appreciated.
Unfortunately there's a bug in it somewhere. It can be reproduced with
tools/testing/selftests/perf_events/remove_on_exec.
(FWIW, the kselftests in that directory are more aggressive test cases
that I never figured out how to properly port to the 'perf test'
framework: sigtrap_threads is a more aggressive version of the 'sigtrap'
perf test, and remove_on_exec has no perf test counterpart.)
$ make headers_install
$ cd tools/testing/selftests/perf_events && make
$ ./remove_on_exec
The test appears to pass, but the kernel log shows a few warnings:
[ 40.304858] ------------[ cut here ]------------
[ 40.307147] unexpected event refcount: 2; ptr=ffffa026c7d94100
[ 40.325434] WARNING: CPU: 1 PID: 404 at kernel/events/core.c:5242 perf_event_exit_event+0x1aa/0x1e0
[ 40.329603] Modules linked in:
[ 40.331401] CPU: 1 PID: 404 Comm: remove_on_exec Not tainted 6.8.0-00004-g852781317836 #1
[ 40.336013] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.16.3-debian-1.16.3-2 04/01/2014
[ 40.340997] RIP: 0010:perf_event_exit_event+0x1aa/0x1e0
[ 40.343374] Code: 5c 41 5d 41 5e 41 5f 5d e9 c3 b6 fe ff e8 3e d5 f7 ff 90 49 8b b6 38 02 00 00 48 c7 c7 6c eb d3 b4 4c 89 f2 e8 f7 66 e0 ff 90 <0f> 0b 90 90 f0 48 ff 8b 38 02 00 00 74 a7 e8 13 d5 f7 ff 5b 41 5c
[ 40.352207] RSP: 0000:ffffa4c4c2173c60 EFLAGS: 00010246
[ 40.354952] RAX: 35f6eccd09146400 RBX: ffffa026c7d92f48 RCX: ffffa026c3eba200
[ 40.358539] RDX: 0000000000000000 RSI: ffffffffb4d389e9 RDI: 00000000ffffffff
[ 40.362858] RBP: 00000000ffffffff R08: 0000000000000001 R09: 0000000000000003
[ 40.366295] R10: ffffa026c3eba200 R11: ffffffffb3d5caa0 R12: ffffa026c3a31538
[ 40.369540] R13: 0000000000000000 R14: ffffa026c7d94100 R15: ffffa026c3a31400
[ 40.373569] FS: 0000000000000000(0000) GS:ffffa029efc40000(0000) knlGS:0000000000000000
[ 40.377495] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 40.380214] CR2: 000055bd7562b73f CR3: 000000010970e001 CR4: 0000000000770ef0
[ 40.383313] DR0: 000055a5212ab828 DR1: 0000000000000000 DR2: 0000000000000000
[ 40.386447] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000600
[ 40.389492] PKRU: 55555554
[ 40.390588] Call Trace:
[ 40.391568] <TASK>
[ 40.392432] ? __warn+0xf6/0x290
[ 40.393736] ? perf_event_exit_event+0x1aa/0x1e0
[ 40.395544] ? report_bug+0x13f/0x1e0
[ 40.397039] ? handle_bug+0x3e/0x70
[ 40.398432] ? exc_invalid_op+0x1a/0x50
[ 40.399960] ? asm_exc_invalid_op+0x1a/0x20
[ 40.401561] ? __pfx_write_msg+0x10/0x10
[ 40.403016] ? perf_event_exit_event+0x1aa/0x1e0
[ 40.404707] perf_event_exec+0x3f5/0x700
[ 40.406172] begin_new_exec+0x720/0x850
[ 40.407589] ? load_elf_phdrs+0x84/0x100
[ 40.409030] load_elf_binary+0x3ba/0xf40
[ 40.410457] ? lock_release+0x23b/0x320
[ 40.411795] ? bprm_execve+0x4f7/0x960
[ 40.413101] bprm_execve+0x502/0x960
[ 40.414341] do_execveat_common+0x23d/0x2a0
[ 40.415699] __x64_sys_execve+0x3a/0x50
[ 40.416948] do_syscall_64+0xf2/0x1d0
[ 40.418281] ? lockdep_hardirqs_on+0x9c/0x150
[ 40.419699] entry_SYSCALL_64_after_hwframe+0x6f/0x77
[ 40.421267] RIP: 0033:0x7f9ccc6749fb
[ 40.422400] Code: Unable to access opcode bytes at 0x7f9ccc6749d1.
[ 40.424252] RSP: 002b:00007ffd2371f358 EFLAGS: 00000246 ORIG_RAX: 000000000000003b
[ 40.426519] RAX: ffffffffffffffda RBX: 00007ffd2371f798 RCX: 00007f9ccc6749fb
[ 40.428643] RDX: 00007ffd2371f7a8 RSI: 00007ffd2371f360 RDI: 00005653e364e74a
[ 40.430783] RBP: 00007ffd2371f3c0 R08: 00007ffd2371f370 R09: 00007ffd2371f390
[ 40.432924] R10: 00005653e364e73f R11: 0000000000000246 R12: 0000000000000000
[ 40.434964] R13: 00007ffd2371f7a8 R14: 00007f9ccc7ba000 R15: 00005653e3650dd8
[ 40.437052] </TASK>
[ 40.437686] irq event stamp: 2323
[ 40.438787] hardirqs last enabled at (2345): [<ffffffffb2f6725c>] console_unlock+0xec/0x1a0
[ 40.441125] hardirqs last disabled at (2356): [<ffffffffb2f67241>] console_unlock+0xd1/0x1a0
[ 40.443386] softirqs last enabled at (2340): [<ffffffffb2ec680f>] __irq_exit_rcu+0x6f/0x100
[ 40.445669] softirqs last disabled at (2333): [<ffffffffb2ec680f>] __irq_exit_rcu+0x6f/0x100
[ 40.447976] ---[ end trace 0000000000000000 ]---
[ 40.474464] ------------[ cut here ]------------
[ 40.474467] WARNING: CPU: 6 PID: 390 at kernel/events/core.c:9595 __perf_event_overflow+0x252/0x270
[ 40.474483] Modules linked in:
[ 40.474486] CPU: 6 PID: 390 Comm: remove_on_exec Tainted: G W 6.8.0-00004-g852781317836 #1
[ 40.474492] Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.16.3-debian-1.16.3-2 04/01/2014
[ 40.474496] RIP: 0010:__perf_event_overflow+0x252/0x270
[ 40.474504] Code: 41 5d 41 5e 41 5f 5d c3 cc cc cc cc cc e8 c6 b4 f8 ff 31 ed eb e5 e8 bd b4 f8 ff 90 0f 0b 90 e9 24 ff ff ff e8 af b4 f8 ff 90 <0f> 0b 90 e9 2f ff ff ff e8 a1 b4 f8 ff eb e2 66 66 66 66 66 66 2e
[ 40.474510] RSP: 0000:ffffa4c4c20dbc88 EFLAGS: 00010046
[ 40.474514] RAX: ffffffffb30a65b1 RBX: ffffa026c7d90008 RCX: ffffa026c7948040
[ 40.474517] RDX: 0000000080110000 RSI: ffffffffb4e127f5 RDI: ffffffffb4d7dc9a
[ 40.474521] RBP: 0000000000000000 R08: 0000000000000000 R09: 0000000000000000
[ 40.474524] R10: ffffa4c4c20dbd00 R11: ffffffffb2e071c0 R12: 0000003000000063
[ 40.474528] R13: 0000002000000001 R14: ffffa4c4c20dbf58 R15: ffffa4c4c20dbd00
[ 40.474531] FS: 00007f9ccc77d680(0000) GS:ffffa029efd80000(0000) knlGS:0000000000000000
[ 40.474536] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 40.474540] CR2: 00007f9ccc77dfa8 CR3: 0000000104fd8004 CR4: 0000000000770ef0
[ 40.474546] DR0: 000055a5212ab828 DR1: 0000000000000000 DR2: 0000000000000000
[ 40.474549] DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000600
[ 40.474552] PKRU: 55555554
[ 40.474554] Call Trace:
[ 40.474556] <TASK>
[ 40.474558] ? __warn+0xf6/0x290
[ 40.474570] ? __perf_event_overflow+0x252/0x270
[ 40.474578] ? report_bug+0x13f/0x1e0
[ 40.474588] ? handle_bug+0x3e/0x70
[ 40.474598] ? exc_invalid_op+0x1a/0x50
[ 40.474608] ? asm_exc_invalid_op+0x1a/0x20
[ 40.474622] ? __pfx_perf_event_nmi_handler+0x10/0x10
[ 40.474633] ? __perf_event_overflow+0x251/0x270
[ 40.474641] ? __perf_event_overflow+0x252/0x270
[ 40.474648] ? __perf_event_overflow+0x251/0x270
[ 40.474656] intel_pmu_handle_irq+0x3ab/0xb20
[ 40.474680] ? verify_lock_unused+0xb/0x70
[ 40.474691] ? lock_acquire+0x1ab/0x250
[ 40.474700] ? nmi_handle+0x30/0x310
[ 40.474709] perf_event_nmi_handler+0x2d/0x60
[ 40.474719] ? nmi_handle+0x30/0x310
[ 40.474724] nmi_handle+0xf9/0x310
[ 40.474730] ? trace_rcu_dyntick+0x41/0xe0
[ 40.474738] ? nmi_handle+0x30/0x310
[ 40.474746] default_do_nmi+0x63/0x160
[ 40.474757] exc_nmi+0xbc/0x110
[ 40.474768] asm_exc_nmi+0xb6/0xff
[ 40.474773] RIP: 0033:0x5653e364d7c5
[ 40.474777] Code: d4 83 45 c4 01 8b 45 c4 83 f8 1d 0f 86 28 fe ff ff 8b 05 2e 3f 00 00 89 85 2c ff ff ff 90 8b 15 21 3f 00 00 8b 85 2c ff ff ff <39> c2 74 f0 90 90 c9 c3 55 48 89 e5 48 81 ec b0 00 00 00 48 8d 95
[ 40.474782] RSP: 002b:00007ffd2371f3d0 EFLAGS: 00000246
[ 40.474786] RAX: 0000000000000005 RBX: 00007ffd2371f798 RCX: 00007f9ccc5d894b
[ 40.474789] RDX: 0000000000000005 RSI: 0000000000000009 RDI: 00000000000001a4
[ 40.474793] RBP: 00007ffd2371f4d0 R08: 0000000000000000 R09: 0000000000000000
[ 40.474796] R10: 0000000000000000 R11: 0000000000000246 R12: 0000000000000000
[ 40.474799] R13: 00007ffd2371f7a8 R14: 00007f9ccc7ba000 R15: 00005653e3650dd8
[ 40.474809] </TASK>
[ 40.474811] irq event stamp: 27946
[ 40.474812] hardirqs last enabled at (27945): [<ffffffffb458ed78>] exc_nmi+0xc8/0x110
[ 40.474824] hardirqs last disabled at (27946): [<ffffffffb458ed52>] exc_nmi+0xa2/0x110
[ 40.474834] softirqs last enabled at (27854): [<ffffffffb2e59aa9>] fpu__restore_sig+0x229/0x870
[ 40.474844] softirqs last disabled at (27852): [<ffffffffb2e59aa9>] fpu__restore_sig+0x229/0x870
[ 40.474853] ---[ end trace 0000000000000000 ]---
Thanks,
-- Marco
On 2024-03-11 11:34:18 [+0100], Marco Elver wrote:
> Unfortunately there's a bug in it somewhere. It can be reproduced with
> tools/testing/selftests/perf_events/remove_on_exec.
>
> (FWIW, the kselftests in that directory are more aggressive test cases
> that I never figured out how to properly port to the 'perf test'
> framework: sigtrap_threads is a more aggressive version of the 'sigtrap'
> perf test, and remove_on_exec has no perf test counterpart.)
You posted to warnings, I can only reproduce the second one. Based on the
backtrace it is obvious what happens and I dare to say that I know why…
Let me try address them.
> Thanks,
> -- Marco
Sebastian
On Mon, 11 Mar 2024 at 16:59, Sebastian Andrzej Siewior
<[email protected]> wrote:
>
> On 2024-03-11 11:34:18 [+0100], Marco Elver wrote:
> > Unfortunately there's a bug in it somewhere. It can be reproduced with
> > tools/testing/selftests/perf_events/remove_on_exec.
> >
> > (FWIW, the kselftests in that directory are more aggressive test cases
> > that I never figured out how to properly port to the 'perf test'
> > framework: sigtrap_threads is a more aggressive version of the 'sigtrap'
> > perf test, and remove_on_exec has no perf test counterpart.)
>
> You posted to warnings, I can only reproduce the second one. Based on the
> backtrace it is obvious what happens and I dare to say that I know why…
> Let me try address them.
I guess I got lucky with hitting 2 warnings instantly. It might be
necessary to run remove_on_exec several times, or even start several
instances of it concurrently (something like "for x in {0..100}; do
(./remove_on_exec &); done" might do the trick).
On Mon, Mar 11, 2024 at 05:31:09PM +0100, Marco Elver wrote:
> On Mon, 11 Mar 2024 at 16:59, Sebastian Andrzej Siewior
> <[email protected]> wrote:
> >
> > On 2024-03-11 11:34:18 [+0100], Marco Elver wrote:
> > > Unfortunately there's a bug in it somewhere. It can be reproduced with
> > > tools/testing/selftests/perf_events/remove_on_exec.
> > >
> > > (FWIW, the kselftests in that directory are more aggressive test cases
> > > that I never figured out how to properly port to the 'perf test'
> > > framework: sigtrap_threads is a more aggressive version of the 'sigtrap'
> > > perf test, and remove_on_exec has no perf test counterpart.)
> >
> > You posted to warnings, I can only reproduce the second one. Based on the
> > backtrace it is obvious what happens and I dare to say that I know why…
> > Let me try address them.
>
> I guess I got lucky with hitting 2 warnings instantly. It might be
> necessary to run remove_on_exec several times, or even start several
> instances of it concurrently (something like "for x in {0..100}; do
> (./remove_on_exec &); done" might do the trick).
Doing this parallel remove_on_exec selftest, without Sebastian's patchs I
eventually hit, which is the original, the following output, then I
managed to replicate Marco's two different splats, see below.
[ 275.644373] BUG: scheduling while atomic: remove_on_exec/10716/0x00000002
[ 275.644378] Modules linked in: xfs libcrc32c i915 i2c_algo_bit drm_buddy ttm intel_gtt drm_display_helper drm_kms_helper sr_mod sd_mod cdrom t10_pi sg drm sdhci_pci cqhci ahci crct10dif_pclmul crc32_pclmul libahci crc32c_intel sdhci e1000e libata ghash_clmulni_intel mmc_core cec video wmi serio_raw dm_mirror dm_region_hash dm_log dm_mod fuse
[ 275.644402] Preemption disabled at:
[ 275.644402] [<0000000000000000>] 0x0
[ 275.644407] CPU: 0 PID: 10716 Comm: remove_on_exec Kdump: loaded Tainted: G W 6.8.0-rc7-rt6 #1
[ 275.644410] Hardware name: LENOVO 427623U/427623U, BIOS 8BET45WW (1.25 ) 05/18/2011
[ 275.644411] Call Trace:
[ 275.644414] <TASK>
[ 275.644416] dump_stack_lvl+0x51/0x70
[ 275.644421] __schedule_bug+0x89/0xa0
[ 275.644426] schedule_debug.constprop.0+0xd1/0x120
[ 275.644430] __schedule+0x50/0x680
[ 275.644433] ? _raw_spin_lock+0x13/0x40
[ 275.644436] ? __sigqueue_alloc+0x81/0x120
[ 275.644440] schedule_rtlock+0x1e/0x40
[ 275.644443] rtlock_slowlock_locked+0xcd/0x270
[ 275.644446] rt_spin_lock+0x41/0x60
[ 275.644448] obj_cgroup_charge+0x3c/0x1d0
[ 275.644451] kmem_cache_alloc+0x255/0x2a0
[ 275.644455] __sigqueue_alloc+0x81/0x120
[ 275.644459] __send_signal_locked+0x25c/0x3c0
[ 275.644461] ? send_signal_locked+0x3f/0x130
[ 275.644464] do_send_sig_info+0x54/0xb0
[ 275.644467] send_sig_perf+0x70/0x90
[ 275.644470] perf_pending_task+0x89/0xa0
[ 275.644473] task_work_run+0x59/0x90
[ 275.644476] irqentry_exit_to_user_mode+0x1dd/0x1e0
[ 275.644480] asm_sysvec_irq_work+0x16/0x20
[ 275.644483] RIP: 0033:0x403376
[ 275.644485] Code: 48 89 bd c8 fe ff ff 48 89 b5 c0 fe ff ff 48 89 95 b8 fe ff ff e8 1a df ff ff 89 85 24 ff ff ff 83 bd 24 ff ff ff 00 75 15 90 <8b> 05 84 53 00 00 85 c0 74 f6 bf 2a 00 00 00 e8 e6 dc ff ff 48 8d
[ 275.644487] RSP: 002b:00007fffc462fde0 EFLAGS: 00000246
[ 275.644489] RAX: 0000000000000000 RBX: 0000000000000000 RCX: 00007f442bd9a018
[ 275.644490] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000001
[ 275.644491] RBP: 00007fffc462ff30 R08: 0000000000000000 R09: 0000000000000000
[ 275.644493] R10: 00007f442bd9a008 R11: 0000000000000246 R12: 00007fffc46301f8
[ 275.644494] R13: 00000000004046cf R14: 0000000000407e08 R15: 00007f442bd99000
[ 275.644497] </TASK>
[ 275.681832] BUG: using smp_processor_id() in preemptible [00000000] code: remove_on_exec/10716
[ 275.681838] caller is perf_pending_task+0x34/0xa0
Now with Sebastian's patch series:
612.723013] ------------[ cut here ]------------
[ 612.723016] WARNING: CPU: 3 PID: 8655 at kernel/events/core.c:9595 __perf_event_overflow+0x21a/0x230
[ 612.723026] Modules linked in: nft_fib_inet nft_fib_ipv4 nft_fib_ipv6 nft_fib nft_reject_inet nf_reject_ipv4 nf_reject_ipv6 nft_reject nft_ct nft_chain_nat nf_nat nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 ip_set nf_tables nfnetlink bnep sunrpc iwldvm intel_rapl_msr intel_rapl_common x86_pkg_temp_thermal intel_powerclamp mac80211 snd_hda_codec_hdmi snd_ctl_led snd_hda_codec_conexant coretemp snd_hda_codec_generic kvm_intel snd_hda_intel snd_intel_dspcfg snd_intel_sdw_acpi libarc4 snd_hda_codec kvm snd_hda_core iwlwifi snd_hwdep uvcvideo snd_seq uvc btusb snd_seq_device videobuf2_vmalloc btrtl snd_pcm videobuf2_memops btintel btbcm videobuf2_v4l2 btmtk videodev mei_wdt irqbypass thinkpad_acpi snd_timer rapl think_lmi iTCO_wdt cfg80211 bluetooth iTCO_vendor_support intel_cstate snd videobuf2_common i2c_i801 firmware_attributes_class mei_me mc pcspkr intel_uncore wmi_bmof ledtrig_audio i2c_smbus mei soundcore lpc_ich platform_profile rfkill joydev xfs libcrc32c i915 i2c_algo_bit drm_buddy ttm intel_gtt drm_display_helper
[ 612.723090] drm_kms_helper sr_mod sd_mod cdrom t10_pi sg drm crct10dif_pclmul ahci crc32_pclmul sdhci_pci libahci crc32c_intel e1000e cqhci libata sdhci ghash_clmulni_intel mmc_core cec video wmi serio_raw dm_mirror dm_region_hash dm_log dm_mod fuse
[ 612.723111] CPU: 3 PID: 8655 Comm: remove_on_exec Kdump: loaded Not tainted 6.8.0-rc7.sebastian-rt6+ #1
[ 612.723114] Hardware name: LENOVO 427623U/427623U, BIOS 8BET45WW (1.25 ) 05/18/2011
[ 612.723115] RIP: 0010:__perf_event_overflow+0x21a/0x230
[ 612.723120] Code: 45 31 ed 5b 5d 44 89 e8 41 5c 41 5d 41 5e c3 cc cc cc cc 48 c7 83 40 03 00 00 00 00 00 00 e9 03 ff ff ff 0f 0b e9 c8 fe ff ff <0f> 0b e9 77 ff ff ff 66 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 40 00
[ 612.723123] RSP: 0000:ffffb81e833abc50 EFLAGS: 00010087
[ 612.723125] RAX: 00000000004045c1 RBX: ffff93e80107ec48 RCX: 00000000004045bb
[ 612.723127] RDX: 0000000000000020 RSI: ffffffffac25de92 RDI: ffffffffac1f45ee
[ 612.723128] RBP: ffffb81e833abcc0 R08: 0000000000000002 R09: ffffb81e833abcc0
[ 612.723130] R10: 0000000000000000 R11: 0000000000000000 R12: ffffb81e833abf58
[ 612.723131] R13: 0000000000000000 R14: 0000000000000001 R15: 0000000000000020
[ 612.723132] FS: 00007f6a22d87600(0000) GS:ffff93eb1e2c0000(0000) knlGS:0000000000000000
[ 612.723135] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 612.723136] CR2: 0000000000405730 CR3: 000000011687c003 CR4: 00000000000606f0
[ 612.723138] Call Trace:
[ 612.723139] <TASK>
[ 612.723141] ? __warn+0x85/0x140
[ 612.723145] ? __perf_event_overflow+0x21a/0x230
[ 612.723149] ? report_bug+0x1c3/0x1d0
[ 612.723155] ? handle_bug+0x3c/0x70
[ 612.723159] ? exc_invalid_op+0x14/0x70
[ 612.723163] ? asm_exc_invalid_op+0x16/0x20
[ 612.723170] ? __perf_event_overflow+0x21a/0x230
[ 612.723174] handle_pmi_common+0x18b/0x400
[ 612.723178] ? kick_process+0x46/0xc0
[ 612.723183] ? migrate_enable+0xaa/0xf0
[ 612.723188] ? rt_spin_unlock+0x13/0x40
[ 612.723191] ? do_send_sig_info+0x63/0xb0
[ 612.723195] ? kill_pid_info+0x9f/0xb0
[ 612.723199] ? kill_something_info+0x170/0x1a0
[ 612.723202] ? rt_read_unlock+0x31/0x1a0
[ 612.723205] ? __x64_sys_kill+0x7f/0xb0
[ 612.723209] intel_pmu_handle_irq+0x112/0x2a0
[ 612.723213] perf_event_nmi_handler+0x28/0x50
[ 612.723217] nmi_handle+0x5e/0x150
[ 612.723222] default_do_nmi+0x6b/0x180
[ 612.723225] exc_nmi+0x10f/0x190
[ 612.723227] asm_exc_nmi+0xb1/0xfa
[ 612.723230] RIP: 0033:0x4045c1
[ 612.723233] Code: 8b 40 2c 85 c0 75 d4 83 45 c4 01 8b 45 c4 83 f8 1d 0f 86 3e fe ff ff 8b 05 4c 41 00 00 89 85 2c ff ff ff 90 8b 15 3f 41 00 00 <8b> 85 2c ff ff ff 39 c2 74 f0 90 90 c9 c3 55 48 89 e5 48 81 ec b0
[ 612.723234] RSP: 002b:00007ffc40fd6c40 EFLAGS: 00000246
[ 612.723236] RAX: 0000000000000005 RBX: 0000000000000000 RCX: 00007f6a22ba676b
[ 612.723238] RDX: 0000000000000005 RSI: 0000000000000009 RDI: 00000000000021ed
[ 612.723239] RBP: 00007ffc40fd6d40 R08: 00000000000021ed R09: 0000000000000000
[ 612.723241] R10: 0000000000000000 R11: 0000000000000246 R12: 00007ffc40fd7008
[ 612.723242] R13: 00000000004046cf R14: 0000000000407e08 R15: 00007f6a22dca000
[ 612.723246] </TASK>
[ 612.723247] ---[ end trace 0000000000000000 ]---
[ 617.892290] perf: interrupt took too long (2509 > 2500), lowering kernel.perf_event_max_sample_rate to 79000
[ 618.992995] ------------[ cut here ]------------
[ 618.992999] unexpected event refcount: 2; ptr=0000000022c05d43
[ 618.993007] WARNING: CPU: 2 PID: 8967 at kernel/events/core.c:5240 free_event+0x36/0x40
[ 618.993015] Modules linked in: nft_fib_inet nft_fib_ipv4 nft_fib_ipv6 nft_fib nft_reject_inet nf_reject_ipv4 nf_reject_ipv6 nft_reject nft_ct nft_chain_nat nf_nat nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 ip_set nf_tables nfnetlink bnep sunrpc iwldvm intel_rapl_msr intel_rapl_common x86_pkg_temp_thermal intel_powerclamp mac80211 snd_hda_codec_hdmi snd_ctl_led snd_hda_codec_conexant coretemp snd_hda_codec_generic kvm_intel snd_hda_intel snd_intel_dspcfg snd_intel_sdw_acpi libarc4 snd_hda_codec kvm snd_hda_core iwlwifi snd_hwdep uvcvideo snd_seq uvc btusb snd_seq_device videobuf2_vmalloc btrtl snd_pcm videobuf2_memops btintel btbcm videobuf2_v4l2 btmtk videodev mei_wdt irqbypass thinkpad_acpi snd_timer rapl think_lmi iTCO_wdt cfg80211 bluetooth iTCO_vendor_support intel_cstate snd videobuf2_common i2c_i801 firmware_attributes_class mei_me mc pcspkr intel_uncore wmi_bmof ledtrig_audio i2c_smbus mei soundcore lpc_ich platform_profile rfkill joydev xfs libcrc32c i915 i2c_algo_bit drm_buddy ttm intel_gtt drm_display_helper
[ 618.993068] drm_kms_helper sr_mod sd_mod cdrom t10_pi sg drm crct10dif_pclmul ahci crc32_pclmul sdhci_pci libahci crc32c_intel e1000e cqhci libata sdhci ghash_clmulni_intel mmc_core cec video wmi serio_raw dm_mirror dm_region_hash dm_log dm_mod fuse
[ 618.993086] CPU: 2 PID: 8967 Comm: exe Kdump: loaded Tainted: G W 6.8.0-rc7.sebastian-rt6+ #1
[ 618.993089] Hardware name: LENOVO 427623U/427623U, BIOS 8BET45WW (1.25 ) 05/18/2011
[ 618.993090] RIP: 0010:free_event+0x36/0x40
[ 618.993093] Code: f0 48 0f b1 97 38 02 00 00 48 83 f8 01 75 05 e9 a0 fd ff ff 48 8b b7 38 02 00 00 48 89 fa 48 c7 c7 20 9c 1b ac e8 6a 52 dd ff <0f> 0b c3 cc cc cc cc 0f 1f 00 90 90 90 90 90 90 90 90 90 90 90 90
[ 618.993095] RSP: 0018:ffffb81e83c8bd30 EFLAGS: 00010282
[ 618.993097] RAX: 0000000000000000 RBX: dead000000000122 RCX: 0000000000000000
[ 618.993099] RDX: 0000000000000002 RSI: ffffffffac1dcc6a RDI: 00000000ffffffff
[ 618.993100] RBP: ffff93e8010799c8 R08: 0000000000000000 R09: ffffb81e83c8bb28
[ 618.993102] R10: ffffb81e83c8bb20 R11: ffffffffacdefe48 R12: ffff93e80104e1f8
[ 618.993103] R13: ffff93e82d021284 R14: ffffb81e83c8bad8 R15: dead000000000100
[ 618.993104] FS: 0000000000000000(0000) GS:ffff93eb1e280000(0000) knlGS:0000000000000000
[ 618.993106] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 618.993108] CR2: 00007ffce8826e40 CR3: 0000000116464004 CR4: 00000000000606f0
[ 618.993110] Call Trace:
[ 618.993111] <TASK>
[ 618.993113] ? __warn+0x85/0x140
[ 618.993116] ? free_event+0x36/0x40
[ 618.993118] ? report_bug+0x1c3/0x1d0
[ 618.993124] ? handle_bug+0x3c/0x70
[ 618.993127] ? exc_invalid_op+0x14/0x70
[ 618.993130] ? asm_exc_invalid_op+0x16/0x20
[ 618.993136] ? free_event+0x36/0x40
[ 618.993138] perf_event_release_kernel+0x255/0x2f0
[ 618.993142] perf_release+0x11/0x20
[ 618.993144] __fput+0x96/0x2c0
[ 618.993147] task_work_run+0x59/0x90
[ 618.993152] get_signal+0x91/0x900
[ 618.993156] arch_do_signal_or_restart+0x25/0x100
[ 618.993160] ? do_execveat_common.isra.0+0x1ae/0x230
[ 618.993164] ? syscall_exit_work+0x103/0x130
[ 618.993166] syscall_exit_to_user_mode+0x1ae/0x1f0
[ 618.993170] do_syscall_64+0x90/0x170
[ 618.993173] ? do_user_addr_fault+0x31b/0x690
[ 618.993177] entry_SYSCALL_64_after_hwframe+0x6e/0x76
[ 618.993180] RIP: 0033:0x7f573a46eed0
[ 618.993185] Code: Unable to access opcode bytes at 0x7f573a46eea6.
[ 618.993186] RSP: 002b:00007ffce8826e40 EFLAGS: 00000200 ORIG_RAX: 000000000000003b
[ 618.993188] RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000000000000
[ 618.993189] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000000
[ 618.993190] RBP: 0000000000000000 R08: 0000000000000000 R09: 0000000000000000
[ 618.993191] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000000
[ 618.993192] R13: 0000000000000000 R14: 0000000000000000 R15: 0000000000000000
[ 618.993195] </TASK>
[ 618.993195] ---[ end trace 0000000000000000 ]---
[ 619.727029] ------------[ cut here ]------------
[ 619.727033] unexpected event refcount: 2; ptr=00000000b09e0f42
[ 619.727042] WARNING: CPU: 2 PID: 9017 at kernel/events/core.c:5240 free_event+0x36/0x40
[ 619.727048] Modules linked in: nft_fib_inet nft_fib_ipv4 nft_fib_ipv6 nft_fib nft_reject_inet nf_reject_ipv4 nf_reject_ipv6 nft_reject nft_ct nft_chain_nat nf_nat nf_conntrack nf_defrag_ipv6 nf_defrag_ipv4 ip_set nf_tables nfnetlink bnep sunrpc iwldvm intel_rapl_msr intel_rapl_common x86_pkg_temp_thermal intel_powerclamp mac80211 snd_hda_codec_hdmi snd_ctl_led snd_hda_codec_conexant coretemp snd_hda_codec_generic kvm_intel snd_hda_intel snd_intel_dspcfg snd_intel_sdw_acpi libarc4 snd_hda_codec kvm snd_hda_core iwlwifi snd_hwdep uvcvideo snd_seq uvc btusb snd_seq_device videobuf2_vmalloc btrtl snd_pcm videobuf2_memops btintel btbcm videobuf2_v4l2 btmtk videodev mei_wdt irqbypass thinkpad_acpi snd_timer rapl think_lmi iTCO_wdt cfg80211 bluetooth iTCO_vendor_support intel_cstate snd videobuf2_common i2c_i801 firmware_attributes_class mei_me mc pcspkr intel_uncore wmi_bmof ledtrig_audio i2c_smbus mei soundcore lpc_ich platform_profile rfkill joydev xfs libcrc32c i915 i2c_algo_bit drm_buddy ttm intel_gtt drm_display_helper
[ 619.727102] drm_kms_helper sr_mod sd_mod cdrom t10_pi sg drm crct10dif_pclmul ahci crc32_pclmul sdhci_pci libahci crc32c_intel e1000e cqhci libata sdhci ghash_clmulni_intel mmc_core cec video wmi serio_raw dm_mirror dm_region_hash dm_log dm_mod fuse
[ 619.727119] CPU: 2 PID: 9017 Comm: exe Kdump: loaded Tainted: G W 6.8.0-rc7.sebastian-rt6+ #1
[ 619.727122] Hardware name: LENOVO 427623U/427623U, BIOS 8BET45WW (1.25 ) 05/18/2011
[ 619.727123] RIP: 0010:free_event+0x36/0x40
[ 619.727126] Code: f0 48 0f b1 97 38 02 00 00 48 83 f8 01 75 05 e9 a0 fd ff ff 48 8b b7 38 02 00 00 48 89 fa 48 c7 c7 20 9c 1b ac e8 6a 52 dd ff <0f> 0b c3 cc cc cc cc 0f 1f 00 90 90 90 90 90 90 90 90 90 90 90 90
[ 619.727128] RSP: 0018:ffffb81e83d5bca8 EFLAGS: 00010282
[ 619.727130] RAX: 0000000000000000 RBX: dead000000000122 RCX: 0000000000000000
[ 619.727132] RDX: 0000000000000002 RSI: ffffffffac1dcc6a RDI: 00000000ffffffff
[ 619.727133] RBP: ffff93e801023390 R08: 0000000000000000 R09: ffffb81e83d5baa0
[ 619.727134] R10: ffffb81e83d5ba98 R11: ffffffffacdefe48 R12: ffff93e8010261f8
[ 619.727136] R13: ffff93e8054b7d84 R14: ffffb81e83d5ba50 R15: dead000000000100
[ 619.727137] FS: 0000000000000000(0000) GS:ffff93eb1e280000(0000) knlGS:0000000000000000
[ 619.727139] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033
[ 619.727140] CR2: 00007ffe58ed17b9 CR3: 00000001058da003 CR4: 00000000000606f0
[ 619.727142] Call Trace:
[ 619.727143] <TASK>
[ 619.727145] ? __warn+0x85/0x140
[ 619.727148] ? free_event+0x36/0x40
[ 619.727150] ? report_bug+0x1c3/0x1d0
[ 619.727156] ? handle_bug+0x3c/0x70
[ 619.727159] ? exc_invalid_op+0x14/0x70
[ 619.727162] ? asm_exc_invalid_op+0x16/0x20
[ 619.727169] ? free_event+0x36/0x40
[ 619.727171] ? free_event+0x36/0x40
[ 619.727172] perf_event_release_kernel+0x255/0x2f0
[ 619.727176] perf_release+0x11/0x20
[ 619.727178] __fput+0x96/0x2c0
[ 619.727182] task_work_run+0x59/0x90
[ 619.727185] get_signal+0x91/0x900
[ 619.727190] ? migrate_enable+0xaa/0xf0
[ 619.727194] arch_do_signal_or_restart+0x25/0x100
[ 619.727197] ? do_execveat_common.isra.0+0x1ae/0x230
[ 619.727200] ? syscall_exit_work+0x103/0x130
[ 619.727202] syscall_exit_to_user_mode+0x1ae/0x1f0
[ 619.727206] do_syscall_64+0x90/0x170
[ 619.727209] ? __count_memcg_events+0x60/0x100
[ 619.727213] ? mm_account_fault+0x6c/0x100
[ 619.727217] ? handle_mm_fault+0xd8/0x210
[ 619.727220] ? do_user_addr_fault+0x31b/0x690
[ 619.727224] entry_SYSCALL_64_after_hwframe+0x6e/0x76
[ 619.727227] RIP: 0033:0x7ffb0bcffed0
[ 619.727232] Code: Unable to access opcode bytes at 0x7ffb0bcffea6.
[ 619.727233] RSP: 002b:00007ffe58ed14b0 EFLAGS: 00000200 ORIG_RAX: 000000000000003b
[ 619.727236] RAX: 0000000000000000 RBX: 0000000000000000 RCX: 0000000000000000
[ 619.727237] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000000
[ 619.727238] RBP: 0000000000000000 R08: 0000000000000000 R09: 0000000000000000
[ 619.727239] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000000
[ 619.727240] R13: 0000000000000000 R14: 0000000000000000 R15: 0000000000000000
[ 619.727242] </TASK>
[ 619.727243] ---[ end trace 0000000000000000 ]---
[acme@nine perf_events]$