2022-01-24 17:06:34

by Anshuman Khandual

[permalink] [raw]
Subject: [RFC V1 00/11] arm64/perf: Enable branch stack sampling

This series enables perf branch stack sampling support on arm64 platform
via a new arch feature called Branch Record Buffer Extension (BRBE). All
relevant register definitions could be accessed here.

https://developer.arm.com/documentation/ddi0601/2021-12/AArch64-Registers

The last two patches extend the perf ABI to accommodate additional branch
information that can be captured in BRBE. This series applies on v5.17-rc1.

Cc: Catalin Marinas <[email protected]>
Cc: Will Deacon <[email protected]>
Cc: Mark Rutland <[email protected]>
Cc: Peter Zijlstra <[email protected]>
Cc: Ingo Molnar <[email protected]>
Cc: Arnaldo Carvalho de Melo <[email protected]>
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]

Anshuman Khandual (11):
perf: Consolidate branch sample filter helpers
arm64/perf: Add register definitions for BRBE
arm64/perf: Update struct arm_pmu for BRBE
arm64/perf: Update struct pmu_hw_events for BRBE
arm64/perf: Detect support for BRBE
arm64/perf: Drive BRBE from perf event states
arm64/perf: Add BRBE driver
arm64/perf: Enable branch stack sampling
perf: Add more generic branch types
perf: Expand perf_branch_entry.type
perf: Capture branch privilege information

arch/arm64/include/asm/sysreg.h | 216 ++++++++++++
arch/arm64/kernel/perf_event.c | 48 +++
arch/x86/events/intel/lbr.c | 4 +-
drivers/perf/Kconfig | 11 +
drivers/perf/Makefile | 1 +
drivers/perf/arm_pmu.c | 65 +++-
drivers/perf/arm_pmu_brbe.c | 432 +++++++++++++++++++++++
drivers/perf/arm_pmu_brbe.h | 259 ++++++++++++++
drivers/perf/arm_pmu_platform.c | 34 ++
include/linux/perf/arm_pmu.h | 49 +++
include/linux/perf_event.h | 24 ++
include/uapi/linux/perf_event.h | 26 +-
kernel/events/core.c | 9 +-
tools/include/uapi/linux/perf_event.h | 26 +-
tools/perf/Documentation/perf-record.txt | 1 +
tools/perf/util/branch.c | 13 +-
tools/perf/util/parse-branch-options.c | 1 +
17 files changed, 1202 insertions(+), 17 deletions(-)
create mode 100644 drivers/perf/arm_pmu_brbe.c
create mode 100644 drivers/perf/arm_pmu_brbe.h

--
2.25.1


2022-01-24 17:06:45

by Anshuman Khandual

[permalink] [raw]
Subject: [RFC V1 07/11] arm64/perf: Add BRBE driver

This adds a BRBE driver which implements all the required helper functions
for struct arm_pmu. Following functions are defined by this driver which
will configure, enable, capture, reset and disable BRBE buffer HW as and
when requested via perf branch stack sampling framework.

- arm64_pmu_brbe_filter()
- arm64_pmu_brbe_enable()
- arm64_pmu_brbe_disable()
- arm64_pmu_brbe_read()
- arm64_pmu_brbe_probe()
- arm64_pmu_brbe_reset()
- arm64_pmu_brbe_supported()

Cc: Peter Zijlstra <[email protected]>
Cc: Ingo Molnar <[email protected]>
Cc: Arnaldo Carvalho de Melo <[email protected]>
Cc: Mark Rutland <[email protected]>
Cc: Will Deacon <[email protected]>
Cc: Catalin Marinas <[email protected]>
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Signed-off-by: Anshuman Khandual <[email protected]>
---
arch/arm64/kernel/perf_event.c | 8 +-
drivers/perf/Kconfig | 11 +
drivers/perf/Makefile | 1 +
drivers/perf/arm_pmu_brbe.c | 396 +++++++++++++++++++++++++++++++++
drivers/perf/arm_pmu_brbe.h | 259 +++++++++++++++++++++
include/linux/perf/arm_pmu.h | 20 ++
6 files changed, 694 insertions(+), 1 deletion(-)
create mode 100644 drivers/perf/arm_pmu_brbe.c
create mode 100644 drivers/perf/arm_pmu_brbe.h

diff --git a/arch/arm64/kernel/perf_event.c b/arch/arm64/kernel/perf_event.c
index 11c82c8f2eec..4c805b3f01fc 100644
--- a/arch/arm64/kernel/perf_event.c
+++ b/arch/arm64/kernel/perf_event.c
@@ -1023,31 +1023,37 @@ static int armv8pmu_filter_match(struct perf_event *event)

static void armv8pmu_brbe_filter(struct pmu_hw_events *hw_event, struct perf_event *event)
{
+ arm64_pmu_brbe_filter(hw_event, event);
}

static void armv8pmu_brbe_enable(struct pmu_hw_events *hw_event)
{
+ arm64_pmu_brbe_enable(hw_event);
}

static void armv8pmu_brbe_disable(struct pmu_hw_events *hw_event)
{
+ arm64_pmu_brbe_disable(hw_event);
}

static void armv8pmu_brbe_read(struct pmu_hw_events *hw_event, struct perf_event *event)
{
+ arm64_pmu_brbe_read(hw_event, event);
}

static void armv8pmu_brbe_probe(struct pmu_hw_events *hw_event)
{
+ arm64_pmu_brbe_probe(hw_event);
}

static void armv8pmu_brbe_reset(struct pmu_hw_events *hw_event)
{
+ arm64_pmu_brbe_reset(hw_event);
}

static bool armv8pmu_brbe_supported(struct perf_event *event)
{
- return false;
+ return arm64_pmu_brbe_supported(event);
}

static void armv8pmu_reset(void *info)
diff --git a/drivers/perf/Kconfig b/drivers/perf/Kconfig
index e1a0c44bc686..284fba623c54 100644
--- a/drivers/perf/Kconfig
+++ b/drivers/perf/Kconfig
@@ -146,6 +146,17 @@ config MARVELL_CN10K_TAD_PMU
Provides support for Last-Level cache Tag-and-data Units (LLC-TAD)
performance monitors on CN10K family silicons.

+config ARM_BRBE_PMU
+ tristate "Enable support for Branch Record Buffer Extension (BRBE)"
+ depends on ARM64 && ARM_PMU
+ default y
+ help
+ Enable perf support for Branch Record Buffer Extension (BRBE) which
+ records all branches taken in an execution path. This supports some
+ branch types and privilege based filtering. It captured additional
+ relevant information such as cycle count, misprediction and branch
+ type, branch privilege level etc.
+
source "drivers/perf/hisilicon/Kconfig"

endmenu
diff --git a/drivers/perf/Makefile b/drivers/perf/Makefile
index 2db5418d5b0a..fd985570a63b 100644
--- a/drivers/perf/Makefile
+++ b/drivers/perf/Makefile
@@ -15,3 +15,4 @@ obj-$(CONFIG_XGENE_PMU) += xgene_pmu.o
obj-$(CONFIG_ARM_SPE_PMU) += arm_spe_pmu.o
obj-$(CONFIG_ARM_DMC620_PMU) += arm_dmc620_pmu.o
obj-$(CONFIG_MARVELL_CN10K_TAD_PMU) += marvell_cn10k_tad_pmu.o
+obj-$(CONFIG_ARM_BRBE_PMU) += arm_pmu_brbe.o
diff --git a/drivers/perf/arm_pmu_brbe.c b/drivers/perf/arm_pmu_brbe.c
new file mode 100644
index 000000000000..8144514b9997
--- /dev/null
+++ b/drivers/perf/arm_pmu_brbe.c
@@ -0,0 +1,396 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Branch Record Buffer Extension Driver.
+ *
+ * Copyright (C) 2021 ARM Limited
+ *
+ * Author: Anshuman Khandual <[email protected]>
+ */
+#include "arm_pmu_brbe.h"
+
+#define BRBE_FCR_MASK (BRBFCR_BRANCH_ALL)
+#define BRBE_CR_MASK (BRBCR_EXCEPTION | BRBCR_ERTN | BRBCR_CC | \
+ BRBCR_MPRED | BRBCR_E1BRE | BRBCR_E0BRE)
+
+static void set_brbe_disabled(struct pmu_hw_events *cpuc)
+{
+ cpuc->brbe_nr = 0;
+}
+
+static bool brbe_disabled(struct pmu_hw_events *cpuc)
+{
+ return !cpuc->brbe_nr;
+}
+
+bool arm64_pmu_brbe_supported(struct perf_event *event)
+{
+ struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
+ struct pmu_hw_events *hw_events = per_cpu_ptr(armpmu->hw_events, event->cpu);
+
+ if (event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_KERNEL) {
+ if (!perfmon_capable()) {
+ pr_warn_once("does not have permission for kernel branch filter\n");
+ return false;
+ }
+ }
+
+ /*
+ * If the event does not have at least one of the privilege
+ * branch filters as in PERF_SAMPLE_BRANCH_PLM_ALL, the core
+ * perf will adjust its value based on perf event's existing
+ * privilege level via attr.exclude_[user|kernel|hv].
+ *
+ * As event->attr.branch_sample_type might have been changed
+ * when the event reaches here, it is not possible to figure
+ * out whether the event originally had HV privilege request
+ * or got added via the core perf. Just report this situation
+ * once and continue ignoring if there are other instances.
+ */
+ if (event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_HV)
+ pr_warn_once("does not support hypervisor privilege branch filter\n");
+
+ if (event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_ABORT_TX) {
+ pr_warn_once("does not support aborted transaction branch filter\n");
+ return false;
+ }
+
+ if (event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_NO_TX) {
+ pr_warn_once("does not support non transaction branch filter\n");
+ return false;
+ }
+
+ if (event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_IN_TX) {
+ pr_warn_once("does not support in transaction branch filter\n");
+ return false;
+ }
+ return !brbe_disabled(hw_events);
+}
+
+void arm64_pmu_brbe_probe(struct pmu_hw_events *cpuc)
+{
+ u64 aa64dfr0, brbidr;
+ unsigned int brbe, format, cpu = smp_processor_id();
+
+ aa64dfr0 = read_sysreg_s(SYS_ID_AA64DFR0_EL1);
+ brbe = cpuid_feature_extract_unsigned_field(aa64dfr0, ID_AA64DFR0_BRBE_SHIFT);
+ if (!brbe) {
+ pr_info("no implementation found on cpu %d\n", cpu);
+ set_brbe_disabled(cpuc);
+ return;
+ } else if (brbe == ID_AA64DFR0_BRBE) {
+ pr_info("implementation found on cpu %d\n", cpu);
+ cpuc->v1p1 = false;
+ } else if (brbe == ID_AA64DFR0_BRBE_V1P1) {
+ pr_info("implementation (v1p1) found on cpu %d\n", cpu);
+ cpuc->v1p1 = true;
+ }
+
+ brbidr = read_sysreg_s(SYS_BRBIDR0_EL1);
+ format = brbe_fetch_format(brbidr);
+ if (format != BRBIDR0_FORMAT_0) {
+ pr_warn("format 0 not implemented\n");
+ set_brbe_disabled(cpuc);
+ return;
+ }
+
+ cpuc->brbe_cc = brbe_fetch_cc_bits(brbidr);
+ if (cpuc->brbe_cc != BRBIDR0_CC_20_BIT) {
+ pr_warn("20-bit counter not implemented\n");
+ set_brbe_disabled(cpuc);
+ return;
+ }
+
+ cpuc->brbe_nr = brbe_fetch_numrec(brbidr);
+ if (!valid_brbe_nr(cpuc->brbe_nr)) {
+ pr_warn("invalid number of records\n");
+ set_brbe_disabled(cpuc);
+ return;
+ }
+}
+
+void arm64_pmu_brbe_enable(struct pmu_hw_events *cpuc)
+{
+ u64 brbfcr, brbcr;
+
+ if (brbe_disabled(cpuc))
+ return;
+
+ brbfcr = read_sysreg_s(SYS_BRBFCR_EL1);
+ brbfcr &= ~(BRBFCR_BANK_MASK << BRBFCR_BANK_SHIFT);
+ brbfcr &= ~(BRBFCR_ENL | BRBFCR_PAUSED | BRBE_FCR_MASK);
+ brbfcr |= (cpuc->brbfcr & BRBE_FCR_MASK);
+ write_sysreg_s(brbfcr, SYS_BRBFCR_EL1);
+ isb();
+
+ brbcr = read_sysreg_s(SYS_BRBCR_EL1);
+ brbcr &= ~BRBE_CR_MASK;
+ brbcr |= BRBCR_FZP;
+ brbcr |= (BRBCR_TS_PHYSICAL << BRBCR_TS_SHIFT);
+ brbcr |= (cpuc->brbcr & BRBE_CR_MASK);
+ write_sysreg_s(brbcr, SYS_BRBCR_EL1);
+ isb();
+}
+
+void arm64_pmu_brbe_disable(struct pmu_hw_events *cpuc)
+{
+ u64 brbcr;
+
+ if (brbe_disabled(cpuc))
+ return;
+
+ brbcr = read_sysreg_s(SYS_BRBCR_EL1);
+ brbcr &= ~(BRBCR_E0BRE | BRBCR_E1BRE);
+ write_sysreg_s(brbcr, SYS_BRBCR_EL1);
+ isb();
+}
+
+static void perf_branch_to_brbfcr(struct pmu_hw_events *cpuc, int branch_type)
+{
+ cpuc->brbfcr = 0;
+
+ if (branch_type & PERF_SAMPLE_BRANCH_ANY) {
+ cpuc->brbfcr |= BRBFCR_BRANCH_ALL;
+ return;
+ }
+
+ if (branch_type & PERF_SAMPLE_BRANCH_ANY_CALL)
+ cpuc->brbfcr |= (BRBFCR_INDCALL | BRBFCR_DIRCALL);
+
+ if (branch_type & PERF_SAMPLE_BRANCH_ANY_RETURN)
+ cpuc->brbfcr |= BRBFCR_RTN;
+
+ if (branch_type & PERF_SAMPLE_BRANCH_IND_CALL)
+ cpuc->brbfcr |= BRBFCR_INDCALL;
+
+ if (branch_type & PERF_SAMPLE_BRANCH_COND)
+ cpuc->brbfcr |= BRBFCR_CONDDIR;
+
+ if (branch_type & PERF_SAMPLE_BRANCH_IND_JUMP)
+ cpuc->brbfcr |= BRBFCR_INDIRECT;
+
+ if (branch_type & PERF_SAMPLE_BRANCH_CALL)
+ cpuc->brbfcr |= BRBFCR_DIRCALL;
+}
+
+static void perf_branch_to_brbcr(struct pmu_hw_events *cpuc, int branch_type)
+{
+ cpuc->brbcr = (BRBCR_CC | BRBCR_MPRED);
+
+ if (branch_type & PERF_SAMPLE_BRANCH_USER)
+ cpuc->brbcr |= BRBCR_E0BRE;
+
+ if (branch_type & PERF_SAMPLE_BRANCH_KERNEL) {
+ /*
+ * This should have been verified earlier.
+ */
+ WARN_ON(!perfmon_capable());
+ cpuc->brbcr |= BRBCR_E1BRE;
+ }
+
+ if (branch_type & PERF_SAMPLE_BRANCH_NO_CYCLES)
+ cpuc->brbcr &= ~BRBCR_CC;
+
+ if (branch_type & PERF_SAMPLE_BRANCH_NO_FLAGS)
+ cpuc->brbcr &= ~BRBCR_MPRED;
+
+ if (!perfmon_capable())
+ return;
+
+ if (branch_type & PERF_SAMPLE_BRANCH_ANY) {
+ cpuc->brbcr |= BRBCR_EXCEPTION;
+ cpuc->brbcr |= BRBCR_ERTN;
+ return;
+ }
+
+ if (branch_type & PERF_SAMPLE_BRANCH_ANY_CALL)
+ cpuc->brbcr |= BRBCR_EXCEPTION;
+
+ if (branch_type & PERF_SAMPLE_BRANCH_ANY_RETURN)
+ cpuc->brbcr |= BRBCR_ERTN;
+}
+
+
+void arm64_pmu_brbe_filter(struct pmu_hw_events *cpuc, struct perf_event *event)
+{
+ u64 branch_type = event->attr.branch_sample_type;
+
+ if (brbe_disabled(cpuc))
+ return;
+
+ perf_branch_to_brbfcr(cpuc, branch_type);
+ perf_branch_to_brbcr(cpuc, branch_type);
+}
+
+static int brbe_fetch_perf_type(u64 brbinf)
+{
+ int brbe_type = brbe_fetch_type(brbinf);
+
+ switch (brbe_type) {
+ case BRBINF_TYPE_UNCOND_DIR:
+ return PERF_BR_UNCOND;
+ case BRBINF_TYPE_INDIR:
+ return PERF_BR_IND;
+ case BRBINF_TYPE_DIR_LINK:
+ return PERF_BR_CALL;
+ case BRBINF_TYPE_INDIR_LINK:
+ return PERF_BR_IND_CALL;
+ case BRBINF_TYPE_RET_SUB:
+ return PERF_BR_RET;
+ case BRBINF_TYPE_COND_DIR:
+ return PERF_BR_COND;
+ case BRBINF_TYPE_CALL:
+ return PERF_BR_CALL;
+ case BRBINF_TYPE_TRAP:
+ return PERF_BR_SYSCALL;
+ case BRBINF_TYPE_RET_EXCPT:
+ return PERF_BR_UNKNOWN;
+ case BRBINF_TYPE_IRQ:
+ return PERF_BR_UNKNOWN;
+ case BRBINF_TYPE_DEBUG_HALT:
+ case BRBINF_TYPE_SERROR:
+ case BRBINF_TYPE_INST_DEBUG:
+ case BRBINF_TYPE_DATA_DEBUG:
+ case BRBINF_TYPE_ALGN_FAULT:
+ case BRBINF_TYPE_INST_FAULT:
+ case BRBINF_TYPE_DATA_FAULT:
+ case BRBINF_TYPE_FIQ:
+ case BRBINF_TYPE_DEBUG_EXIT:
+ return PERF_BR_UNKNOWN;
+ default:
+ pr_warn("unknown branch type captured\n");
+ return PERF_BR_UNKNOWN;
+ }
+}
+
+static void capture_brbe_flags(struct pmu_hw_events *cpuc, struct perf_event *event,
+ u64 brbinf, int idx)
+{
+ int type = brbe_record_valid(brbinf);
+
+ if (!branch_sample_no_cycles(event))
+ cpuc->brbe_entries[idx].cycles = brbe_fetch_cycles(brbinf);
+
+ if (branch_sample_type(event))
+ cpuc->brbe_entries[idx].type = brbe_fetch_perf_type(brbinf);
+
+ if (!branch_sample_no_flags(event)) {
+ /*
+ * BRBINF_LASTFAILED does not indicate that the last transaction
+ * got failed or aborted during the current branch record itself.
+ * Rather, this indicates that all the branch records which were
+ * in transaction until the curret branch record have failed. So
+ * the entire BRBE buffer needs to be processed later on to find
+ * all branch records which might have failed.
+ */
+ cpuc->brbe_entries[idx].abort = brbinf & BRBINF_LASTFAILED;
+
+ /*
+ * All these information (i.e transaction state and mispredicts)
+ * are not available for target only branch records.
+ */
+ if (type != BRBINF_VALID_TARGET) {
+ cpuc->brbe_entries[idx].mispred = brbinf & BRBINF_MPRED;
+ cpuc->brbe_entries[idx].predicted = !(brbinf & BRBINF_MPRED);
+ cpuc->brbe_entries[idx].in_tx = brbinf & BRBINF_TX;
+ }
+ }
+}
+
+/*
+ * A branch record with BRBINF_EL1.LASTFAILED set, implies that all
+ * preceding consecutive branch records, that were in a transaction
+ * (i.e their BRBINF_EL1.TX set) have been aborted.
+ *
+ * Similarly BRBFCR_EL1.LASTFAILED set, indicate that all preceding
+ * consecutive branch records upto the last record, which were in a
+ * transaction (i.e their BRBINF_EL1.TX set) have been aborted.
+ *
+ * --------------------------------- -------------------
+ * | 00 | BRBSRC | BRBTGT | BRBINF | | TX = 1 | LF = 0 | [TX success]
+ * --------------------------------- -------------------
+ * | 01 | BRBSRC | BRBTGT | BRBINF | | TX = 1 | LF = 0 | [TX success]
+ * --------------------------------- -------------------
+ * | 02 | BRBSRC | BRBTGT | BRBINF | | TX = 0 | LF = 0 |
+ * --------------------------------- -------------------
+ * | 03 | BRBSRC | BRBTGT | BRBINF | | TX = 1 | LF = 0 | [TX failed]
+ * --------------------------------- -------------------
+ * | 04 | BRBSRC | BRBTGT | BRBINF | | TX = 1 | LF = 0 | [TX failed]
+ * --------------------------------- -------------------
+ * | 05 | BRBSRC | BRBTGT | BRBINF | | TX = 0 | LF = 1 |
+ * --------------------------------- -------------------
+ * | .. | BRBSRC | BRBTGT | BRBINF | | TX = 0 | LF = 0 |
+ * --------------------------------- -------------------
+ * | 61 | BRBSRC | BRBTGT | BRBINF | | TX = 1 | LF = 0 | [TX failed]
+ * --------------------------------- -------------------
+ * | 62 | BRBSRC | BRBTGT | BRBINF | | TX = 1 | LF = 0 | [TX failed]
+ * --------------------------------- -------------------
+ * | 63 | BRBSRC | BRBTGT | BRBINF | | TX = 1 | LF = 0 | [TX failed]
+ * --------------------------------- -------------------
+ *
+ * BRBFCR_EL1.LASTFAILED == 1
+ *
+ * Here BRBFCR_EL1.LASTFAILED failes all those consecutive and also
+ * in transaction branches near the end of the BRBE buffer.
+ */
+static void process_branch_aborts(struct pmu_hw_events *cpuc)
+{
+ u64 brbfcr = read_sysreg_s(SYS_BRBFCR_EL1);
+ bool lastfailed = !!(brbfcr & BRBFCR_LASTFAILED);
+ int idx = cpuc->brbe_nr - 1;
+
+ do {
+ if (cpuc->brbe_entries[idx].in_tx) {
+ cpuc->brbe_entries[idx].abort = lastfailed;
+ } else {
+ lastfailed = cpuc->brbe_entries[idx].abort;
+ cpuc->brbe_entries[idx].abort = false;
+ }
+ } while (idx--, idx >= 0);
+}
+
+void arm64_pmu_brbe_read(struct pmu_hw_events *cpuc, struct perf_event *event)
+{
+ u64 brbinf;
+ int idx;
+
+ if (brbe_disabled(cpuc))
+ return;
+
+ set_brbe_paused();
+ for (idx = 0; idx < cpuc->brbe_nr; idx++) {
+ select_brbe_bank_index(idx);
+ brbinf = get_brbinf_reg(idx);
+ /*
+ * There are no valid entries anymore on the buffer.
+ * Abort the branch record processing to save some
+ * cycles and also reduce the capture/process load
+ * for the user space as well.
+ */
+ if (brbe_invalid(brbinf))
+ break;
+
+ if (brbe_valid(brbinf)) {
+ cpuc->brbe_entries[idx].from = get_brbsrc_reg(idx);
+ cpuc->brbe_entries[idx].to = get_brbtgt_reg(idx);
+ } else if (brbe_source(brbinf)) {
+ cpuc->brbe_entries[idx].from = get_brbsrc_reg(idx);
+ cpuc->brbe_entries[idx].to = 0;
+ } else if (brbe_target(brbinf)) {
+ cpuc->brbe_entries[idx].from = 0;
+ cpuc->brbe_entries[idx].to = get_brbtgt_reg(idx);
+ }
+ capture_brbe_flags(cpuc, event, brbinf, idx);
+ }
+ cpuc->brbe_stack.nr = idx;
+ cpuc->brbe_stack.hw_idx = -1ULL;
+ process_branch_aborts(cpuc);
+}
+
+void arm64_pmu_brbe_reset(struct pmu_hw_events *cpuc)
+{
+ if (brbe_disabled(cpuc))
+ return;
+
+ asm volatile(BRB_IALL);
+ isb();
+}
diff --git a/drivers/perf/arm_pmu_brbe.h b/drivers/perf/arm_pmu_brbe.h
new file mode 100644
index 000000000000..f04975cdc242
--- /dev/null
+++ b/drivers/perf/arm_pmu_brbe.h
@@ -0,0 +1,259 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Branch Record Buffer Extension Helpers.
+ *
+ * Copyright (C) 2021 ARM Limited
+ *
+ * Author: Anshuman Khandual <[email protected]>
+ */
+#define pr_fmt(fmt) "brbe: " fmt
+
+#include <linux/perf/arm_pmu.h>
+
+/*
+ * BRBE Instructions
+ *
+ * BRB_IALL : Invalidate the entire buffer
+ * BRB_INJ : Inject latest branch record derived from [BRBSRCINJ, BRBTGTINJ, BRBINFINJ]
+ */
+#define BRB_IALL __emit_inst(0xD5000000 | sys_insn(1, 1, 7, 2, 4) | (0x1f))
+#define BRB_INJ __emit_inst(0xD5000000 | sys_insn(1, 1, 7, 2, 5) | (0x1f))
+
+/*
+ * BRBE Buffer Organization
+ *
+ * BRBE buffer is arranged as multiple banks of 32 branch record
+ * entries each. An indivdial branch record in a given bank could
+ * be accessedi, after selecting the bank in BRBFCR_EL1.BANK and
+ * accessing the registers i.e [BRBSRC, BRBTGT, BRBINF] set with
+ * indices [0..31].
+ *
+ * Bank 0
+ *
+ * --------------------------------- ------
+ * | 00 | BRBSRC | BRBTGT | BRBINF | | 00 |
+ * --------------------------------- ------
+ * | 01 | BRBSRC | BRBTGT | BRBINF | | 01 |
+ * --------------------------------- ------
+ * | .. | BRBSRC | BRBTGT | BRBINF | | .. |
+ * --------------------------------- ------
+ * | 31 | BRBSRC | BRBTGT | BRBINF | | 31 |
+ * --------------------------------- ------
+ *
+ * Bank 1
+ *
+ * --------------------------------- ------
+ * | 32 | BRBSRC | BRBTGT | BRBINF | | 00 |
+ * --------------------------------- ------
+ * | 33 | BRBSRC | BRBTGT | BRBINF | | 01 |
+ * --------------------------------- ------
+ * | .. | BRBSRC | BRBTGT | BRBINF | | .. |
+ * --------------------------------- ------
+ * | 63 | BRBSRC | BRBTGT | BRBINF | | 31 |
+ * --------------------------------- ------
+ */
+#define BRBE_BANK0_IDX_MIN 0
+#define BRBE_BANK0_IDX_MAX 31
+#define BRBE_BANK1_IDX_MIN 32
+#define BRBE_BANK1_IDX_MAX 63
+
+#define RETURN_READ_BRBSRCN(n) \
+ read_sysreg_s(SYS_BRBSRC##n##_EL1)
+
+#define RETURN_READ_BRBTGTN(n) \
+ read_sysreg_s(SYS_BRBTGT##n##_EL1)
+
+#define RETURN_READ_BRBINFN(n) \
+ read_sysreg_s(SYS_BRBINF##n##_EL1)
+
+#define BRBE_REGN_CASE(n, case_macro) \
+ case n: return case_macro(n); break
+
+#define BRBE_REGN_SWITCH(x, case_macro) \
+ do { \
+ switch (x) { \
+ BRBE_REGN_CASE(0, case_macro); \
+ BRBE_REGN_CASE(1, case_macro); \
+ BRBE_REGN_CASE(2, case_macro); \
+ BRBE_REGN_CASE(3, case_macro); \
+ BRBE_REGN_CASE(4, case_macro); \
+ BRBE_REGN_CASE(5, case_macro); \
+ BRBE_REGN_CASE(6, case_macro); \
+ BRBE_REGN_CASE(7, case_macro); \
+ BRBE_REGN_CASE(8, case_macro); \
+ BRBE_REGN_CASE(9, case_macro); \
+ BRBE_REGN_CASE(10, case_macro); \
+ BRBE_REGN_CASE(11, case_macro); \
+ BRBE_REGN_CASE(12, case_macro); \
+ BRBE_REGN_CASE(13, case_macro); \
+ BRBE_REGN_CASE(14, case_macro); \
+ BRBE_REGN_CASE(15, case_macro); \
+ BRBE_REGN_CASE(16, case_macro); \
+ BRBE_REGN_CASE(17, case_macro); \
+ BRBE_REGN_CASE(18, case_macro); \
+ BRBE_REGN_CASE(19, case_macro); \
+ BRBE_REGN_CASE(20, case_macro); \
+ BRBE_REGN_CASE(21, case_macro); \
+ BRBE_REGN_CASE(22, case_macro); \
+ BRBE_REGN_CASE(23, case_macro); \
+ BRBE_REGN_CASE(24, case_macro); \
+ BRBE_REGN_CASE(25, case_macro); \
+ BRBE_REGN_CASE(26, case_macro); \
+ BRBE_REGN_CASE(27, case_macro); \
+ BRBE_REGN_CASE(28, case_macro); \
+ BRBE_REGN_CASE(29, case_macro); \
+ BRBE_REGN_CASE(30, case_macro); \
+ BRBE_REGN_CASE(31, case_macro); \
+ default: \
+ pr_warn("unknown register index\n"); \
+ return -1; \
+ } \
+ } while (0)
+
+static inline int buffer_to_brbe_idx(int buffer_idx)
+{
+ return buffer_idx % 32;
+}
+
+static inline u64 get_brbsrc_reg(int buffer_idx)
+{
+ int brbe_idx = buffer_to_brbe_idx(buffer_idx);
+
+ BRBE_REGN_SWITCH(brbe_idx, RETURN_READ_BRBSRCN);
+}
+
+static inline u64 get_brbtgt_reg(int buffer_idx)
+{
+ int brbe_idx = buffer_to_brbe_idx(buffer_idx);
+
+ BRBE_REGN_SWITCH(brbe_idx, RETURN_READ_BRBTGTN);
+}
+
+static inline u64 get_brbinf_reg(int buffer_idx)
+{
+ int brbe_idx = buffer_to_brbe_idx(buffer_idx);
+
+ BRBE_REGN_SWITCH(brbe_idx, RETURN_READ_BRBINFN);
+}
+
+static inline u64 brbe_record_valid(u64 brbinf)
+{
+ return brbinf & (BRBINF_VALID_MASK << BRBINF_VALID_SHIFT);
+}
+
+static inline bool brbe_invalid(u64 brbinf)
+{
+ return brbe_record_valid(brbinf) == BRBINF_VALID_INVALID;
+}
+
+static inline bool brbe_valid(u64 brbinf)
+{
+ return brbe_record_valid(brbinf) == BRBINF_VALID_ALL;
+}
+
+static inline bool brbe_source(u64 brbinf)
+{
+ return brbe_record_valid(brbinf) == BRBINF_VALID_SOURCE;
+}
+
+static inline bool brbe_target(u64 brbinf)
+{
+ return brbe_record_valid(brbinf) == BRBINF_VALID_TARGET;
+}
+
+static inline int brbe_fetch_cycles(u64 brbinf)
+{
+ /*
+ * Captured cycle count is unknown and hence
+ * should not be passed on the user space.
+ */
+ if (brbinf & BRBINF_CCU)
+ return 0;
+
+ return (brbinf >> BRBINF_CC_SHIFT) & BRBINF_CC_MASK;
+}
+
+static inline int brbe_fetch_type(u64 brbinf)
+{
+ return (brbinf >> BRBINF_TYPE_SHIFT) & BRBINF_TYPE_MASK;
+}
+
+static inline int brbe_fetch_el(u64 brbinf)
+{
+ return (brbinf >> BRBINF_EL_SHIFT) & BRBINF_EL_MASK;
+}
+
+static inline int brbe_fetch_numrec(u64 brbidr)
+{
+ return (brbidr >> BRBIDR0_NUMREC_SHIFT) & BRBIDR0_NUMREC_MASK;
+}
+
+static inline int brbe_fetch_format(u64 brbidr)
+{
+ return (brbidr >> BRBIDR0_FORMAT_SHIFT) & BRBIDR0_FORMAT_MASK;
+}
+
+static inline int brbe_fetch_cc_bits(u64 brbidr)
+{
+ return (brbidr >> BRBIDR0_CC_SHIFT) & BRBIDR0_CC_MASK;
+}
+
+static inline void select_brbe_bank(int bank)
+{
+ static int brbe_current_bank = -1;
+ u64 brbfcr;
+
+ if (brbe_current_bank == bank)
+ return;
+
+ WARN_ON(bank > 1);
+ brbfcr = read_sysreg_s(SYS_BRBFCR_EL1);
+ brbfcr &= ~(BRBFCR_BANK_MASK << BRBFCR_BANK_SHIFT);
+ brbfcr |= ((bank & BRBFCR_BANK_MASK) << BRBFCR_BANK_SHIFT);
+ write_sysreg_s(brbfcr, SYS_BRBFCR_EL1);
+ isb();
+ brbe_current_bank = bank;
+}
+
+static inline void select_brbe_bank_index(int buffer_idx)
+{
+ switch (buffer_idx) {
+ case BRBE_BANK0_IDX_MIN ... BRBE_BANK0_IDX_MAX:
+ select_brbe_bank(0);
+ break;
+ case BRBE_BANK1_IDX_MIN ... BRBE_BANK1_IDX_MAX:
+ select_brbe_bank(1);
+ break;
+ default:
+ pr_warn("unsupported BRBE index\n");
+ }
+}
+
+static inline bool valid_brbe_nr(int brbe_nr)
+{
+ switch (brbe_nr) {
+ case BRBIDR0_NUMREC_8:
+ case BRBIDR0_NUMREC_16:
+ case BRBIDR0_NUMREC_32:
+ case BRBIDR0_NUMREC_64:
+ return true;
+ default:
+ pr_warn("unsupported BRBE entries\n");
+ return false;
+ }
+}
+
+static inline bool brbe_paused(void)
+{
+ u64 brbfcr = read_sysreg_s(SYS_BRBFCR_EL1);
+
+ return brbfcr & BRBFCR_PAUSED;
+}
+
+static inline void set_brbe_paused(void)
+{
+ u64 brbfcr = read_sysreg_s(SYS_BRBFCR_EL1);
+
+ write_sysreg_s(brbfcr | BRBFCR_PAUSED, SYS_BRBFCR_EL1);
+ isb();
+}
diff --git a/include/linux/perf/arm_pmu.h b/include/linux/perf/arm_pmu.h
index c4a705d3d054..d401771e7767 100644
--- a/include/linux/perf/arm_pmu.h
+++ b/include/linux/perf/arm_pmu.h
@@ -146,6 +146,26 @@ struct arm_pmu {
unsigned long acpi_cpuid;
};

+#ifdef CONFIG_ARM_BRBE_PMU
+void arm64_pmu_brbe_filter(struct pmu_hw_events *hw_events, struct perf_event *event);
+void arm64_pmu_brbe_read(struct pmu_hw_events *cpuc, struct perf_event *event);
+void arm64_pmu_brbe_disable(struct pmu_hw_events *cpuc);
+void arm64_pmu_brbe_enable(struct pmu_hw_events *cpuc);
+void arm64_pmu_brbe_probe(struct pmu_hw_events *cpuc);
+void arm64_pmu_brbe_reset(struct pmu_hw_events *cpuc);
+bool arm64_pmu_brbe_supported(struct perf_event *event);
+#else
+static inline void arm64_pmu_brbe_filter(struct pmu_hw_events *hw_events, struct perf_event *event)
+{
+}
+static inline void arm64_pmu_brbe_read(struct pmu_hw_events *cpuc, struct perf_event *event) { }
+static inline void arm64_pmu_brbe_disable(struct pmu_hw_events *cpuc) { }
+static inline void arm64_pmu_brbe_enable(struct pmu_hw_events *cpuc) { }
+static inline void arm64_pmu_brbe_probe(struct pmu_hw_events *cpuc) { }
+static inline void arm64_pmu_brbe_reset(struct pmu_hw_events *cpuc) { }
+static inline bool arm64_pmu_brbe_supported(struct perf_event *event) {return false; }
+#endif
+
#define to_arm_pmu(p) (container_of(p, struct arm_pmu, pmu))

u64 armpmu_event_update(struct perf_event *event);
--
2.25.1

2022-01-24 17:06:47

by Anshuman Khandual

[permalink] [raw]
Subject: [RFC V1 08/11] arm64/perf: Enable branch stack sampling

Now that all the required pieces are already in place, just enable the perf
branch stack sampling support on arm64 platform, by removing the gate which
blocks it in armpmu_event_init().

Cc: Mark Rutland <[email protected]>
Cc: Will Deacon <[email protected]>
Cc: Catalin Marinas <[email protected]>
Cc: [email protected]
Cc: [email protected]
Signed-off-by: Anshuman Khandual <[email protected]>
---
drivers/perf/arm_pmu.c | 25 ++++++++++++++++++++++---
1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/drivers/perf/arm_pmu.c b/drivers/perf/arm_pmu.c
index 0800c8858ed8..2117bf5d3232 100644
--- a/drivers/perf/arm_pmu.c
+++ b/drivers/perf/arm_pmu.c
@@ -537,9 +537,28 @@ static int armpmu_event_init(struct perf_event *event)
!cpumask_test_cpu(event->cpu, &armpmu->supported_cpus))
return -ENOENT;

- /* does not support taken branch sampling */
- if (has_branch_stack(event))
- return -EOPNOTSUPP;
+ if (has_branch_stack(event)) {
+ /*
+ * BRBE support is absent. Select CONFIG_ARM_BRBE_PMU
+ * in the config, before branch stack sampling events
+ * can be requested.
+ */
+ if (!IS_ENABLED(CONFIG_ARM_BRBE_PMU)) {
+ pr_warn_once("BRBE is disabled, select CONFIG_ARM_BRBE_PMU\n");
+ return -EOPNOTSUPP;
+ }
+
+ /*
+ * Branch stack sampling event can not be supported in
+ * case either the required driver itself is absent or
+ * BRBE buffer, is not supported. Besides checking for
+ * the callback prevents a crash in case it's absent.
+ */
+ if (!armpmu->brbe_supported || !armpmu->brbe_supported(event)) {
+ pr_warn_once("BRBE is not supported\n");
+ return -EOPNOTSUPP;
+ }
+ }

if (armpmu->map_event(event) == -ENOENT)
return -ENOENT;
--
2.25.1

2022-01-24 17:06:49

by Anshuman Khandual

[permalink] [raw]
Subject: [RFC V1 01/11] perf: Consolidate branch sample filter helpers

Besides the branch type filtering requests, 'event.attr.branch_sample_type'
also contains various flags indicating which additional information should
be captured, along with the base branch record. These flags help configure
the underlying hardware, and capture the branch records appropriately when
required e.g after PMU interrupt. But first, this moves an existing helper
perf_sample_save_hw_index() into the header before adding some more helpers
for other branch sample filter flags.

Cc: Peter Zijlstra <[email protected]>
Cc: Ingo Molnar <[email protected]>
Cc: Arnaldo Carvalho de Melo <[email protected]>
Cc: [email protected]
Cc: [email protected]
Signed-off-by: Anshuman Khandual <[email protected]>
---
include/linux/perf_event.h | 19 +++++++++++++++++++
kernel/events/core.c | 9 ++-------
2 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index 117f230bcdfd..916ce5102b33 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -1669,4 +1669,23 @@ typedef int (perf_snapshot_branch_stack_t)(struct perf_branch_entry *entries,
unsigned int cnt);
DECLARE_STATIC_CALL(perf_snapshot_branch_stack, perf_snapshot_branch_stack_t);

+static inline bool branch_sample_no_flags(const struct perf_event *event)
+{
+ return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_NO_FLAGS;
+}
+
+static inline bool branch_sample_no_cycles(const struct perf_event *event)
+{
+ return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_NO_CYCLES;
+}
+
+static inline bool branch_sample_type(const struct perf_event *event)
+{
+ return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_TYPE_SAVE;
+}
+
+static inline bool branch_sample_hw_index(const struct perf_event *event)
+{
+ return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_HW_INDEX;
+}
#endif /* _LINUX_PERF_EVENT_H */
diff --git a/kernel/events/core.c b/kernel/events/core.c
index fc18664f49b0..740d0d829b24 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -6984,11 +6984,6 @@ static void perf_output_read(struct perf_output_handle *handle,
perf_output_read_one(handle, event, enabled, running);
}

-static inline bool perf_sample_save_hw_index(struct perf_event *event)
-{
- return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_HW_INDEX;
-}
-
void perf_output_sample(struct perf_output_handle *handle,
struct perf_event_header *header,
struct perf_sample_data *data,
@@ -7077,7 +7072,7 @@ void perf_output_sample(struct perf_output_handle *handle,
* sizeof(struct perf_branch_entry);

perf_output_put(handle, data->br_stack->nr);
- if (perf_sample_save_hw_index(event))
+ if (branch_sample_hw_index(event))
perf_output_put(handle, data->br_stack->hw_idx);
perf_output_copy(handle, data->br_stack->entries, size);
} else {
@@ -7371,7 +7366,7 @@ void perf_prepare_sample(struct perf_event_header *header,
if (sample_type & PERF_SAMPLE_BRANCH_STACK) {
int size = sizeof(u64); /* nr */
if (data->br_stack) {
- if (perf_sample_save_hw_index(event))
+ if (branch_sample_hw_index(event))
size += sizeof(u64);

size += data->br_stack->nr
--
2.25.1

2022-01-24 17:06:59

by Anshuman Khandual

[permalink] [raw]
Subject: [RFC V1 09/11] perf: Add more generic branch types

This expands generic branch type classification by adding some more entries
, that can still be represented with the existing 4 bit 'type' field. While
here this also updates the x86 implementation with these new branch types.

Cc: Peter Zijlstra <[email protected]>
Cc: Ingo Molnar <[email protected]>
Cc: Arnaldo Carvalho de Melo <[email protected]>
Cc: Mark Rutland <[email protected]>
Cc: Alexander Shishkin <[email protected]>
Cc: Jiri Olsa <[email protected]>
Cc: Namhyung Kim <[email protected]>
Cc: Will Deacon <[email protected]>
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Signed-off-by: Anshuman Khandual <[email protected]>
---
arch/x86/events/intel/lbr.c | 4 ++--
drivers/perf/arm_pmu_brbe.c | 11 +++++++----
include/uapi/linux/perf_event.h | 5 +++++
tools/include/uapi/linux/perf_event.h | 5 +++++
tools/perf/util/branch.c | 7 ++++++-
5 files changed, 25 insertions(+), 7 deletions(-)

diff --git a/arch/x86/events/intel/lbr.c b/arch/x86/events/intel/lbr.c
index 8043213b75a5..9f86fac8c6a5 100644
--- a/arch/x86/events/intel/lbr.c
+++ b/arch/x86/events/intel/lbr.c
@@ -1336,10 +1336,10 @@ static int branch_map[X86_BR_TYPE_MAP_MAX] = {
PERF_BR_SYSCALL, /* X86_BR_SYSCALL */
PERF_BR_SYSRET, /* X86_BR_SYSRET */
PERF_BR_UNKNOWN, /* X86_BR_INT */
- PERF_BR_UNKNOWN, /* X86_BR_IRET */
+ PERF_BR_EXPT_RET, /* X86_BR_IRET */
PERF_BR_COND, /* X86_BR_JCC */
PERF_BR_UNCOND, /* X86_BR_JMP */
- PERF_BR_UNKNOWN, /* X86_BR_IRQ */
+ PERF_BR_IRQ, /* X86_BR_IRQ */
PERF_BR_IND_CALL, /* X86_BR_IND_CALL */
PERF_BR_UNKNOWN, /* X86_BR_ABORT */
PERF_BR_UNKNOWN, /* X86_BR_IN_TX */
diff --git a/drivers/perf/arm_pmu_brbe.c b/drivers/perf/arm_pmu_brbe.c
index 8144514b9997..8d27ad868359 100644
--- a/drivers/perf/arm_pmu_brbe.c
+++ b/drivers/perf/arm_pmu_brbe.c
@@ -243,18 +243,21 @@ static int brbe_fetch_perf_type(u64 brbinf)
case BRBINF_TYPE_TRAP:
return PERF_BR_SYSCALL;
case BRBINF_TYPE_RET_EXCPT:
- return PERF_BR_UNKNOWN;
+ return PERF_BR_EXPT_RET;
case BRBINF_TYPE_IRQ:
- return PERF_BR_UNKNOWN;
+ return PERF_BR_IRQ;
+ case BRBINF_TYPE_FIQ:
+ return PERF_BR_FIQ;
case BRBINF_TYPE_DEBUG_HALT:
+ return PERF_BR_DEBUG_HALT;
+ case BRBINF_TYPE_DEBUG_EXIT:
+ return PERF_BR_DEBUG_EXIT;
case BRBINF_TYPE_SERROR:
case BRBINF_TYPE_INST_DEBUG:
case BRBINF_TYPE_DATA_DEBUG:
case BRBINF_TYPE_ALGN_FAULT:
case BRBINF_TYPE_INST_FAULT:
case BRBINF_TYPE_DATA_FAULT:
- case BRBINF_TYPE_FIQ:
- case BRBINF_TYPE_DEBUG_EXIT:
return PERF_BR_UNKNOWN;
default:
pr_warn("unknown branch type captured\n");
diff --git a/include/uapi/linux/perf_event.h b/include/uapi/linux/perf_event.h
index 1b65042ab1db..b91d0f575d0c 100644
--- a/include/uapi/linux/perf_event.h
+++ b/include/uapi/linux/perf_event.h
@@ -251,6 +251,11 @@ enum {
PERF_BR_SYSRET = 8, /* syscall return */
PERF_BR_COND_CALL = 9, /* conditional function call */
PERF_BR_COND_RET = 10, /* conditional function return */
+ PERF_BR_EXPT_RET = 11, /* exception return */
+ PERF_BR_IRQ = 12, /* irq */
+ PERF_BR_FIQ = 13, /* fiq */
+ PERF_BR_DEBUG_HALT = 14, /* debug halt */
+ PERF_BR_DEBUG_EXIT = 15, /* debug exit */
PERF_BR_MAX,
};

diff --git a/tools/include/uapi/linux/perf_event.h b/tools/include/uapi/linux/perf_event.h
index 4cd39aaccbe7..1882054e8684 100644
--- a/tools/include/uapi/linux/perf_event.h
+++ b/tools/include/uapi/linux/perf_event.h
@@ -251,6 +251,11 @@ enum {
PERF_BR_SYSRET = 8, /* syscall return */
PERF_BR_COND_CALL = 9, /* conditional function call */
PERF_BR_COND_RET = 10, /* conditional function return */
+ PERF_BR_EXPT_RET = 11, /* exception return */
+ PERF_BR_IRQ = 12, /* irq */
+ PERF_BR_FIQ = 13, /* fiq */
+ PERF_BR_DEBUG_HALT = 14, /* debug halt */
+ PERF_BR_DEBUG_EXIT = 15, /* debug exit */
PERF_BR_MAX,
};

diff --git a/tools/perf/util/branch.c b/tools/perf/util/branch.c
index 2285b1eb3128..74e5e67b1779 100644
--- a/tools/perf/util/branch.c
+++ b/tools/perf/util/branch.c
@@ -49,7 +49,12 @@ const char *branch_type_name(int type)
"SYSCALL",
"SYSRET",
"COND_CALL",
- "COND_RET"
+ "COND_RET",
+ "EXPT_RET",
+ "IRQ",
+ "FIQ",
+ "DEBUG_HALT",
+ "DEBUG_EXIT"
};

if (type >= 0 && type < PERF_BR_MAX)
--
2.25.1

2022-01-24 17:07:04

by Anshuman Khandual

[permalink] [raw]
Subject: [RFC V1 10/11] perf: Expand perf_branch_entry.type

Current perf_branch_entry.type is a 4 bits field just enough to accommodate
16 generic branch types. This is insufficient to accommodate platforms like
arm64 which has much more branch types. Lets just expands this field into a
6 bits one, which can now hold 64 generic branch types. This also adds more
generic branch types and updates the BRBE driver as required.

Cc: Peter Zijlstra <[email protected]>
Cc: Ingo Molnar <[email protected]>
Cc: Arnaldo Carvalho de Melo <[email protected]>
Cc: Mark Rutland <[email protected]>
Cc: Alexander Shishkin <[email protected]>
Cc: Jiri Olsa <[email protected]>
Cc: Namhyung Kim <[email protected]>
Cc: Will Deacon <[email protected]>
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Signed-off-by: Anshuman Khandual <[email protected]>
---
drivers/perf/arm_pmu_brbe.c | 7 ++++++-
include/uapi/linux/perf_event.h | 10 ++++++++--
tools/include/uapi/linux/perf_event.h | 10 ++++++++--
tools/perf/util/branch.c | 8 +++++++-
4 files changed, 29 insertions(+), 6 deletions(-)

diff --git a/drivers/perf/arm_pmu_brbe.c b/drivers/perf/arm_pmu_brbe.c
index 8d27ad868359..7cd1208c6c58 100644
--- a/drivers/perf/arm_pmu_brbe.c
+++ b/drivers/perf/arm_pmu_brbe.c
@@ -253,12 +253,17 @@ static int brbe_fetch_perf_type(u64 brbinf)
case BRBINF_TYPE_DEBUG_EXIT:
return PERF_BR_DEBUG_EXIT;
case BRBINF_TYPE_SERROR:
+ return PERF_BR_SERROR;
case BRBINF_TYPE_INST_DEBUG:
+ return PERF_BR_DEBUG_INST;
case BRBINF_TYPE_DATA_DEBUG:
+ return PERF_BR_DEBUG_DATA;
case BRBINF_TYPE_ALGN_FAULT:
+ return PERF_BR_FAULT_ALGN;
case BRBINF_TYPE_INST_FAULT:
+ return PERF_BR_FAULT_INST;
case BRBINF_TYPE_DATA_FAULT:
- return PERF_BR_UNKNOWN;
+ return PERF_BR_FAULT_DATA;
default:
pr_warn("unknown branch type captured\n");
return PERF_BR_UNKNOWN;
diff --git a/include/uapi/linux/perf_event.h b/include/uapi/linux/perf_event.h
index b91d0f575d0c..361fdc6b87a0 100644
--- a/include/uapi/linux/perf_event.h
+++ b/include/uapi/linux/perf_event.h
@@ -256,6 +256,12 @@ enum {
PERF_BR_FIQ = 13, /* fiq */
PERF_BR_DEBUG_HALT = 14, /* debug halt */
PERF_BR_DEBUG_EXIT = 15, /* debug exit */
+ PERF_BR_DEBUG_INST = 16, /* instruciton debug */
+ PERF_BR_DEBUG_DATA = 17, /* data debug */
+ PERF_BR_FAULT_ALGN = 18, /* alignment fault */
+ PERF_BR_FAULT_DATA = 19, /* data fault */
+ PERF_BR_FAULT_INST = 20, /* instruction fault */
+ PERF_BR_SERROR = 21, /* system error */
PERF_BR_MAX,
};

@@ -1370,8 +1376,8 @@ struct perf_branch_entry {
in_tx:1, /* in transaction */
abort:1, /* transaction abort */
cycles:16, /* cycle count to last branch */
- type:4, /* branch type */
- reserved:40;
+ type:6, /* branch type */
+ reserved:38;
};

union perf_sample_weight {
diff --git a/tools/include/uapi/linux/perf_event.h b/tools/include/uapi/linux/perf_event.h
index 1882054e8684..9a82b8aaed93 100644
--- a/tools/include/uapi/linux/perf_event.h
+++ b/tools/include/uapi/linux/perf_event.h
@@ -256,6 +256,12 @@ enum {
PERF_BR_FIQ = 13, /* fiq */
PERF_BR_DEBUG_HALT = 14, /* debug halt */
PERF_BR_DEBUG_EXIT = 15, /* debug exit */
+ PERF_BR_DEBUG_INST = 16, /* instruciton debug */
+ PERF_BR_DEBUG_DATA = 17, /* data debug */
+ PERF_BR_FAULT_ALGN = 18, /* alignment fault */
+ PERF_BR_FAULT_DATA = 19, /* data fault */
+ PERF_BR_FAULT_INST = 20, /* instruction fault */
+ PERF_BR_SERROR = 21, /* system error */
PERF_BR_MAX,
};

@@ -1370,8 +1376,8 @@ struct perf_branch_entry {
in_tx:1, /* in transaction */
abort:1, /* transaction abort */
cycles:16, /* cycle count to last branch */
- type:4, /* branch type */
- reserved:40;
+ type:6, /* branch type */
+ reserved:38;
};

union perf_sample_weight {
diff --git a/tools/perf/util/branch.c b/tools/perf/util/branch.c
index 74e5e67b1779..1e216ea2e2a8 100644
--- a/tools/perf/util/branch.c
+++ b/tools/perf/util/branch.c
@@ -54,7 +54,13 @@ const char *branch_type_name(int type)
"IRQ",
"FIQ",
"DEBUG_HALT",
- "DEBUG_EXIT"
+ "DEBUG_EXIT",
+ "DEBUG_INST",
+ "DEBUG_DATA",
+ "FAULT_ALGN",
+ "FAULT_DATA",
+ "FAULT_INST",
+ "SERROR"
};

if (type >= 0 && type < PERF_BR_MAX)
--
2.25.1

2022-01-24 17:07:10

by Anshuman Khandual

[permalink] [raw]
Subject: [RFC V1 11/11] perf: Capture branch privilege information

Platforms like arm64 could capture privilege level information for all the
branch records. Hence this adds a new element in the struct branch_entry to
record the privilege level information, which could be requested through a
new event.attr.branch_sample_type flag PERF_SAMPLE_BRANCH_PRIV_SAVE. While
here, update the BRBE driver as required.

Cc: Will Deacon <[email protected]>
Cc: Mark Rutland <[email protected]>
Cc: Peter Zijlstra <[email protected]>
Cc: Ingo Molnar <[email protected]>
Cc: Arnaldo Carvalho de Melo <[email protected]>
Cc: Jiri Olsa <[email protected]>
Cc: Namhyung Kim <[email protected]>
Cc: [email protected]
Cc: [email protected]
Cc: [email protected]
Signed-off-by: Anshuman Khandual <[email protected]>
---
drivers/perf/arm_pmu_brbe.c | 28 ++++++++++++++++++++++++
include/linux/perf_event.h | 5 +++++
include/uapi/linux/perf_event.h | 13 ++++++++++-
tools/include/uapi/linux/perf_event.h | 13 ++++++++++-
tools/perf/Documentation/perf-record.txt | 1 +
tools/perf/util/parse-branch-options.c | 1 +
6 files changed, 59 insertions(+), 2 deletions(-)

diff --git a/drivers/perf/arm_pmu_brbe.c b/drivers/perf/arm_pmu_brbe.c
index 7cd1208c6c58..d4cbea74c148 100644
--- a/drivers/perf/arm_pmu_brbe.c
+++ b/drivers/perf/arm_pmu_brbe.c
@@ -270,6 +270,25 @@ static int brbe_fetch_perf_type(u64 brbinf)
}
}

+static int brbe_fetch_perf_priv(u64 brbinf)
+{
+ int brbe_el = brbe_fetch_el(brbinf);
+
+ switch (brbe_el) {
+ case BRBINF_EL_EL0:
+ return PERF_BR_USER;
+ case BRBINF_EL_EL1:
+ return PERF_BR_KERNEL;
+ case BRBINF_EL_EL2:
+ if (is_kernel_in_hyp_mode())
+ return PERF_BR_KERNEL;
+ return PERF_BR_HV;
+ default:
+ pr_warn("unknown branch privilege captured\n");
+ return -1;
+ }
+}
+
static void capture_brbe_flags(struct pmu_hw_events *cpuc, struct perf_event *event,
u64 brbinf, int idx)
{
@@ -302,6 +321,15 @@ static void capture_brbe_flags(struct pmu_hw_events *cpuc, struct perf_event *ev
cpuc->brbe_entries[idx].in_tx = brbinf & BRBINF_TX;
}
}
+
+ if (branch_sample_priv(event)) {
+ /*
+ * All these information (i.e branch privilege level) are not
+ * available for source only branch records.
+ */
+ if (type != BRBINF_VALID_SOURCE)
+ cpuc->brbe_entries[idx].priv = brbe_fetch_perf_priv(brbinf);
+ }
}

/*
diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
index 916ce5102b33..8021b6a30d86 100644
--- a/include/linux/perf_event.h
+++ b/include/linux/perf_event.h
@@ -1688,4 +1688,9 @@ static inline bool branch_sample_hw_index(const struct perf_event *event)
{
return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_HW_INDEX;
}
+
+static inline bool branch_sample_priv(const struct perf_event *event)
+{
+ return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_PRIV_SAVE;
+}
#endif /* _LINUX_PERF_EVENT_H */
diff --git a/include/uapi/linux/perf_event.h b/include/uapi/linux/perf_event.h
index 361fdc6b87a0..4d77710f7a4e 100644
--- a/include/uapi/linux/perf_event.h
+++ b/include/uapi/linux/perf_event.h
@@ -204,6 +204,8 @@ enum perf_branch_sample_type_shift {

PERF_SAMPLE_BRANCH_HW_INDEX_SHIFT = 17, /* save low level index of raw branch records */

+ PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT = 18, /* save privillege mode */
+
PERF_SAMPLE_BRANCH_MAX_SHIFT /* non-ABI */
};

@@ -233,6 +235,8 @@ enum perf_branch_sample_type {

PERF_SAMPLE_BRANCH_HW_INDEX = 1U << PERF_SAMPLE_BRANCH_HW_INDEX_SHIFT,

+ PERF_SAMPLE_BRANCH_PRIV_SAVE = 1U << PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT,
+
PERF_SAMPLE_BRANCH_MAX = 1U << PERF_SAMPLE_BRANCH_MAX_SHIFT,
};

@@ -265,6 +269,12 @@ enum {
PERF_BR_MAX,
};

+enum {
+ PERF_BR_USER = 0,
+ PERF_BR_KERNEL = 1,
+ PERF_BR_HV = 2,
+};
+
#define PERF_SAMPLE_BRANCH_PLM_ALL \
(PERF_SAMPLE_BRANCH_USER|\
PERF_SAMPLE_BRANCH_KERNEL|\
@@ -1377,7 +1387,8 @@ struct perf_branch_entry {
abort:1, /* transaction abort */
cycles:16, /* cycle count to last branch */
type:6, /* branch type */
- reserved:38;
+ priv:2, /* privilege level */
+ reserved:36;
};

union perf_sample_weight {
diff --git a/tools/include/uapi/linux/perf_event.h b/tools/include/uapi/linux/perf_event.h
index 9a82b8aaed93..a2208400b0b9 100644
--- a/tools/include/uapi/linux/perf_event.h
+++ b/tools/include/uapi/linux/perf_event.h
@@ -204,6 +204,8 @@ enum perf_branch_sample_type_shift {

PERF_SAMPLE_BRANCH_HW_INDEX_SHIFT = 17, /* save low level index of raw branch records */

+ PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT = 18, /* save privillege mode */
+
PERF_SAMPLE_BRANCH_MAX_SHIFT /* non-ABI */
};

@@ -233,6 +235,8 @@ enum perf_branch_sample_type {

PERF_SAMPLE_BRANCH_HW_INDEX = 1U << PERF_SAMPLE_BRANCH_HW_INDEX_SHIFT,

+ PERF_SAMPLE_BRANCH_PRIV_SAVE = 1U << PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT,
+
PERF_SAMPLE_BRANCH_MAX = 1U << PERF_SAMPLE_BRANCH_MAX_SHIFT,
};

@@ -265,6 +269,12 @@ enum {
PERF_BR_MAX,
};

+enum {
+ PERF_BR_USER = 0,
+ PERF_BR_KERNEL = 1,
+ PERF_BR_HV = 2,
+};
+
#define PERF_SAMPLE_BRANCH_PLM_ALL \
(PERF_SAMPLE_BRANCH_USER|\
PERF_SAMPLE_BRANCH_KERNEL|\
@@ -1377,7 +1387,8 @@ struct perf_branch_entry {
abort:1, /* transaction abort */
cycles:16, /* cycle count to last branch */
type:6, /* branch type */
- reserved:38;
+ priv:2, /* privilege level */
+ reserved:36;
};

union perf_sample_weight {
diff --git a/tools/perf/Documentation/perf-record.txt b/tools/perf/Documentation/perf-record.txt
index 9ccc75935bc5..3e33686977a1 100644
--- a/tools/perf/Documentation/perf-record.txt
+++ b/tools/perf/Documentation/perf-record.txt
@@ -387,6 +387,7 @@ following filters are defined:
- abort_tx: only when the target is a hardware transaction abort
- cond: conditional branches
- save_type: save branch type during sampling in case binary is not available later
+ - priv: save privilege state during sampling in case binary is not available later

+
The option requires at least one branch type among any, any_call, any_ret, ind_call, cond.
diff --git a/tools/perf/util/parse-branch-options.c b/tools/perf/util/parse-branch-options.c
index bb4aa88c50a8..00588b9db474 100644
--- a/tools/perf/util/parse-branch-options.c
+++ b/tools/perf/util/parse-branch-options.c
@@ -32,6 +32,7 @@ static const struct branch_mode branch_modes[] = {
BRANCH_OPT("call", PERF_SAMPLE_BRANCH_CALL),
BRANCH_OPT("save_type", PERF_SAMPLE_BRANCH_TYPE_SAVE),
BRANCH_OPT("stack", PERF_SAMPLE_BRANCH_CALL_STACK),
+ BRANCH_OPT("priv", PERF_SAMPLE_BRANCH_PRIV_SAVE),
BRANCH_END
};

--
2.25.1

2022-01-24 19:48:48

by James Clark

[permalink] [raw]
Subject: Re: [RFC V1 08/11] arm64/perf: Enable branch stack sampling



On 24/01/2022 04:30, Anshuman Khandual wrote:
> Now that all the required pieces are already in place, just enable the perf
> branch stack sampling support on arm64 platform, by removing the gate which
> blocks it in armpmu_event_init().
>
> Cc: Mark Rutland <[email protected]>
> Cc: Will Deacon <[email protected]>
> Cc: Catalin Marinas <[email protected]>
> Cc: [email protected]
> Cc: [email protected]
> Signed-off-by: Anshuman Khandual <[email protected]>
> ---
> drivers/perf/arm_pmu.c | 25 ++++++++++++++++++++++---
> 1 file changed, 22 insertions(+), 3 deletions(-)
>
> diff --git a/drivers/perf/arm_pmu.c b/drivers/perf/arm_pmu.c
> index 0800c8858ed8..2117bf5d3232 100644
> --- a/drivers/perf/arm_pmu.c
> +++ b/drivers/perf/arm_pmu.c
> @@ -537,9 +537,28 @@ static int armpmu_event_init(struct perf_event *event)
> !cpumask_test_cpu(event->cpu, &armpmu->supported_cpus))
> return -ENOENT;
>
> - /* does not support taken branch sampling */
> - if (has_branch_stack(event))
> - return -EOPNOTSUPP;
> + if (has_branch_stack(event)) {
> + /*
> + * BRBE support is absent. Select CONFIG_ARM_BRBE_PMU
> + * in the config, before branch stack sampling events
> + * can be requested.
> + */
> + if (!IS_ENABLED(CONFIG_ARM_BRBE_PMU)) {
> + pr_warn_once("BRBE is disabled, select CONFIG_ARM_BRBE_PMU\n");
> + return -EOPNOTSUPP;
> + }
> +
> + /*
> + * Branch stack sampling event can not be supported in
> + * case either the required driver itself is absent or
> + * BRBE buffer, is not supported. Besides checking for
> + * the callback prevents a crash in case it's absent.
> + */
> + if (!armpmu->brbe_supported || !armpmu->brbe_supported(event)) {
> + pr_warn_once("BRBE is not supported\n");
> + return -EOPNOTSUPP;

brbe_supported() returns false for one permission case, rather than a "not supported" case
so EOPNOTSUPP is the wrong thing to return here for this case otherwise it makes perf
print confusing error messages.

The brbe_supported() function needs to be split into two, one that handles support and one
that handles permissions and different errors need to be reported. Here is the permission
bit from that function:

+ if (event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_KERNEL) {
+ if (!perfmon_capable()) {
+ pr_warn_once("does not have permission for kernel branch filter\n");
+ return false;
+ }
+ }

> + }
> + }
>
> if (armpmu->map_event(event) == -ENOENT)
> return -ENOENT;
>

2022-01-24 19:50:52

by James Clark

[permalink] [raw]
Subject: Re: [RFC V1 07/11] arm64/perf: Add BRBE driver



On 24/01/2022 04:30, Anshuman Khandual wrote:
> This adds a BRBE driver which implements all the required helper functions
> for struct arm_pmu. Following functions are defined by this driver which
> will configure, enable, capture, reset and disable BRBE buffer HW as and
> when requested via perf branch stack sampling framework.
>
> - arm64_pmu_brbe_filter()
> - arm64_pmu_brbe_enable()
> - arm64_pmu_brbe_disable()
> - arm64_pmu_brbe_read()
> - arm64_pmu_brbe_probe()
> - arm64_pmu_brbe_reset()
> - arm64_pmu_brbe_supported()
>
> Cc: Peter Zijlstra <[email protected]>
> Cc: Ingo Molnar <[email protected]>
> Cc: Arnaldo Carvalho de Melo <[email protected]>
> Cc: Mark Rutland <[email protected]>
> Cc: Will Deacon <[email protected]>
> Cc: Catalin Marinas <[email protected]>
> Cc: [email protected]
> Cc: [email protected]
> Cc: [email protected]
> Signed-off-by: Anshuman Khandual <[email protected]>
> ---
[...]
> +static void perf_branch_to_brbcr(struct pmu_hw_events *cpuc, int branch_type)
> +{
> + cpuc->brbcr = (BRBCR_CC | BRBCR_MPRED);
> +
> + if (branch_type & PERF_SAMPLE_BRANCH_USER)
> + cpuc->brbcr |= BRBCR_E0BRE;
> +
> + if (branch_type & PERF_SAMPLE_BRANCH_KERNEL) {
> + /*
> + * This should have been verified earlier.
> + */
> + WARN_ON(!perfmon_capable());

If it's verified earlier than that value should be saved. At the moment this can
change depending on whichever process happens to be current when this is called.

I don't think perfmon_capable() should be called outside of the init function
of a PMU or outside of the perf core code. This function gets called on a path
from the PMU add function, where any process can be current.

See https://lore.kernel.org/lkml/20220118100702.GB16547@willie-the-truck/T/#m42df3688e8afa299a7b8dfb7c3f4a785c52bcbed

> + cpuc->brbcr |= BRBCR_E1BRE;
> + }
> +
> + if (branch_type & PERF_SAMPLE_BRANCH_NO_CYCLES)
> + cpuc->brbcr &= ~BRBCR_CC;
> +
> + if (branch_type & PERF_SAMPLE_BRANCH_NO_FLAGS)
> + cpuc->brbcr &= ~BRBCR_MPRED;
> +
> + if (!perfmon_capable())
> + return;
> +

This will return here randomly depending on whether a user process or a kernel
process is 'current', resulting in exception samples randomly being enabled
and disabled.

> + if (branch_type & PERF_SAMPLE_BRANCH_ANY) {
> + cpuc->brbcr |= BRBCR_EXCEPTION;
> + cpuc->brbcr |= BRBCR_ERTN;
> + return;
> + }
> +
> + if (branch_type & PERF_SAMPLE_BRANCH_ANY_CALL)
> + cpuc->brbcr |= BRBCR_EXCEPTION;
> +
> + if (branch_type & PERF_SAMPLE_BRANCH_ANY_RETURN)
> + cpuc->brbcr |= BRBCR_ERTN;
> +}
> +
> +
> +void arm64_pmu_brbe_filter(struct pmu_hw_events *cpuc, struct perf_event *event)
> +{
> + u64 branch_type = event->attr.branch_sample_type;
> +
> + if (brbe_disabled(cpuc))
> + return;
> +
> + perf_branch_to_brbfcr(cpuc, branch_type);
> + perf_branch_to_brbcr(cpuc, branch_type);
> +}
> +
> +static int brbe_fetch_perf_type(u64 brbinf)
> +{
> + int brbe_type = brbe_fetch_type(brbinf);
> +
> + switch (brbe_type) {
> + case BRBINF_TYPE_UNCOND_DIR:
> + return PERF_BR_UNCOND;
> + case BRBINF_TYPE_INDIR:
> + return PERF_BR_IND;
> + case BRBINF_TYPE_DIR_LINK:
> + return PERF_BR_CALL;
> + case BRBINF_TYPE_INDIR_LINK:
> + return PERF_BR_IND_CALL;
> + case BRBINF_TYPE_RET_SUB:
> + return PERF_BR_RET;
> + case BRBINF_TYPE_COND_DIR:
> + return PERF_BR_COND;
> + case BRBINF_TYPE_CALL:
> + return PERF_BR_CALL;
> + case BRBINF_TYPE_TRAP:
> + return PERF_BR_SYSCALL;
> + case BRBINF_TYPE_RET_EXCPT:
> + return PERF_BR_UNKNOWN;
> + case BRBINF_TYPE_IRQ:
> + return PERF_BR_UNKNOWN;
> + case BRBINF_TYPE_DEBUG_HALT:
> + case BRBINF_TYPE_SERROR:
> + case BRBINF_TYPE_INST_DEBUG:
> + case BRBINF_TYPE_DATA_DEBUG:
> + case BRBINF_TYPE_ALGN_FAULT:
> + case BRBINF_TYPE_INST_FAULT:
> + case BRBINF_TYPE_DATA_FAULT:
> + case BRBINF_TYPE_FIQ:
> + case BRBINF_TYPE_DEBUG_EXIT:
> + return PERF_BR_UNKNOWN;
> + default:
> + pr_warn("unknown branch type captured\n");
> + return PERF_BR_UNKNOWN;
> + }
> +}
> +
> +static void capture_brbe_flags(struct pmu_hw_events *cpuc, struct perf_event *event,
> + u64 brbinf, int idx)
> +{
> + int type = brbe_record_valid(brbinf);
> +
> + if (!branch_sample_no_cycles(event))
> + cpuc->brbe_entries[idx].cycles = brbe_fetch_cycles(brbinf);
> +
> + if (branch_sample_type(event))
> + cpuc->brbe_entries[idx].type = brbe_fetch_perf_type(brbinf);
> +
> + if (!branch_sample_no_flags(event)) {
> + /*
> + * BRBINF_LASTFAILED does not indicate that the last transaction
> + * got failed or aborted during the current branch record itself.
> + * Rather, this indicates that all the branch records which were
> + * in transaction until the curret branch record have failed. So
> + * the entire BRBE buffer needs to be processed later on to find
> + * all branch records which might have failed.
> + */
> + cpuc->brbe_entries[idx].abort = brbinf & BRBINF_LASTFAILED;
> +
> + /*
> + * All these information (i.e transaction state and mispredicts)
> + * are not available for target only branch records.
> + */
> + if (type != BRBINF_VALID_TARGET) {
> + cpuc->brbe_entries[idx].mispred = brbinf & BRBINF_MPRED;
> + cpuc->brbe_entries[idx].predicted = !(brbinf & BRBINF_MPRED);
> + cpuc->brbe_entries[idx].in_tx = brbinf & BRBINF_TX;
> + }
> + }
> +}
> +
> +/*
> + * A branch record with BRBINF_EL1.LASTFAILED set, implies that all
> + * preceding consecutive branch records, that were in a transaction
> + * (i.e their BRBINF_EL1.TX set) have been aborted.
> + *
> + * Similarly BRBFCR_EL1.LASTFAILED set, indicate that all preceding
> + * consecutive branch records upto the last record, which were in a
> + * transaction (i.e their BRBINF_EL1.TX set) have been aborted.
> + *
> + * --------------------------------- -------------------
> + * | 00 | BRBSRC | BRBTGT | BRBINF | | TX = 1 | LF = 0 | [TX success]
> + * --------------------------------- -------------------
> + * | 01 | BRBSRC | BRBTGT | BRBINF | | TX = 1 | LF = 0 | [TX success]
> + * --------------------------------- -------------------
> + * | 02 | BRBSRC | BRBTGT | BRBINF | | TX = 0 | LF = 0 |
> + * --------------------------------- -------------------
> + * | 03 | BRBSRC | BRBTGT | BRBINF | | TX = 1 | LF = 0 | [TX failed]
> + * --------------------------------- -------------------
> + * | 04 | BRBSRC | BRBTGT | BRBINF | | TX = 1 | LF = 0 | [TX failed]
> + * --------------------------------- -------------------
> + * | 05 | BRBSRC | BRBTGT | BRBINF | | TX = 0 | LF = 1 |
> + * --------------------------------- -------------------
> + * | .. | BRBSRC | BRBTGT | BRBINF | | TX = 0 | LF = 0 |
> + * --------------------------------- -------------------
> + * | 61 | BRBSRC | BRBTGT | BRBINF | | TX = 1 | LF = 0 | [TX failed]
> + * --------------------------------- -------------------
> + * | 62 | BRBSRC | BRBTGT | BRBINF | | TX = 1 | LF = 0 | [TX failed]
> + * --------------------------------- -------------------
> + * | 63 | BRBSRC | BRBTGT | BRBINF | | TX = 1 | LF = 0 | [TX failed]
> + * --------------------------------- -------------------
> + *
> + * BRBFCR_EL1.LASTFAILED == 1
> + *
> + * Here BRBFCR_EL1.LASTFAILED failes all those consecutive and also
> + * in transaction branches near the end of the BRBE buffer.
> + */
> +static void process_branch_aborts(struct pmu_hw_events *cpuc)
> +{
> + u64 brbfcr = read_sysreg_s(SYS_BRBFCR_EL1);
> + bool lastfailed = !!(brbfcr & BRBFCR_LASTFAILED);
> + int idx = cpuc->brbe_nr - 1;
> +
> + do {
> + if (cpuc->brbe_entries[idx].in_tx) {
> + cpuc->brbe_entries[idx].abort = lastfailed;
> + } else {
> + lastfailed = cpuc->brbe_entries[idx].abort;
> + cpuc->brbe_entries[idx].abort = false;
> + }
> + } while (idx--, idx >= 0);
> +}
> +
> +void arm64_pmu_brbe_read(struct pmu_hw_events *cpuc, struct perf_event *event)
> +{
> + u64 brbinf;
> + int idx;
> +
> + if (brbe_disabled(cpuc))
> + return;
> +
> + set_brbe_paused();
> + for (idx = 0; idx < cpuc->brbe_nr; idx++) {
> + select_brbe_bank_index(idx);
> + brbinf = get_brbinf_reg(idx);
> + /*
> + * There are no valid entries anymore on the buffer.
> + * Abort the branch record processing to save some
> + * cycles and also reduce the capture/process load
> + * for the user space as well.
> + */
> + if (brbe_invalid(brbinf))
> + break;
> +
> + if (brbe_valid(brbinf)) {
> + cpuc->brbe_entries[idx].from = get_brbsrc_reg(idx);
> + cpuc->brbe_entries[idx].to = get_brbtgt_reg(idx);
> + } else if (brbe_source(brbinf)) {
> + cpuc->brbe_entries[idx].from = get_brbsrc_reg(idx);
> + cpuc->brbe_entries[idx].to = 0;
> + } else if (brbe_target(brbinf)) {
> + cpuc->brbe_entries[idx].from = 0;
> + cpuc->brbe_entries[idx].to = get_brbtgt_reg(idx);
> + }
> + capture_brbe_flags(cpuc, event, brbinf, idx);
> + }
> + cpuc->brbe_stack.nr = idx;
> + cpuc->brbe_stack.hw_idx = -1ULL;
> + process_branch_aborts(cpuc);
> +}
> +
> +void arm64_pmu_brbe_reset(struct pmu_hw_events *cpuc)
> +{
> + if (brbe_disabled(cpuc))
> + return;
> +
> + asm volatile(BRB_IALL);
> + isb();
> +}
> diff --git a/drivers/perf/arm_pmu_brbe.h b/drivers/perf/arm_pmu_brbe.h
> new file mode 100644
> index 000000000000..f04975cdc242
> --- /dev/null
> +++ b/drivers/perf/arm_pmu_brbe.h
> @@ -0,0 +1,259 @@
> +/* SPDX-License-Identifier: GPL-2.0-only */
> +/*
> + * Branch Record Buffer Extension Helpers.
> + *
> + * Copyright (C) 2021 ARM Limited
> + *
> + * Author: Anshuman Khandual <[email protected]>
> + */
> +#define pr_fmt(fmt) "brbe: " fmt
> +
> +#include <linux/perf/arm_pmu.h>
> +
> +/*
> + * BRBE Instructions
> + *
> + * BRB_IALL : Invalidate the entire buffer
> + * BRB_INJ : Inject latest branch record derived from [BRBSRCINJ, BRBTGTINJ, BRBINFINJ]
> + */
> +#define BRB_IALL __emit_inst(0xD5000000 | sys_insn(1, 1, 7, 2, 4) | (0x1f))
> +#define BRB_INJ __emit_inst(0xD5000000 | sys_insn(1, 1, 7, 2, 5) | (0x1f))
> +
> +/*
> + * BRBE Buffer Organization
> + *
> + * BRBE buffer is arranged as multiple banks of 32 branch record
> + * entries each. An indivdial branch record in a given bank could
> + * be accessedi, after selecting the bank in BRBFCR_EL1.BANK and
> + * accessing the registers i.e [BRBSRC, BRBTGT, BRBINF] set with
> + * indices [0..31].
> + *
> + * Bank 0
> + *
> + * --------------------------------- ------
> + * | 00 | BRBSRC | BRBTGT | BRBINF | | 00 |
> + * --------------------------------- ------
> + * | 01 | BRBSRC | BRBTGT | BRBINF | | 01 |
> + * --------------------------------- ------
> + * | .. | BRBSRC | BRBTGT | BRBINF | | .. |
> + * --------------------------------- ------
> + * | 31 | BRBSRC | BRBTGT | BRBINF | | 31 |
> + * --------------------------------- ------
> + *
> + * Bank 1
> + *
> + * --------------------------------- ------
> + * | 32 | BRBSRC | BRBTGT | BRBINF | | 00 |
> + * --------------------------------- ------
> + * | 33 | BRBSRC | BRBTGT | BRBINF | | 01 |
> + * --------------------------------- ------
> + * | .. | BRBSRC | BRBTGT | BRBINF | | .. |
> + * --------------------------------- ------
> + * | 63 | BRBSRC | BRBTGT | BRBINF | | 31 |
> + * --------------------------------- ------
> + */
> +#define BRBE_BANK0_IDX_MIN 0
> +#define BRBE_BANK0_IDX_MAX 31
> +#define BRBE_BANK1_IDX_MIN 32
> +#define BRBE_BANK1_IDX_MAX 63
> +
> +#define RETURN_READ_BRBSRCN(n) \
> + read_sysreg_s(SYS_BRBSRC##n##_EL1)
> +
> +#define RETURN_READ_BRBTGTN(n) \
> + read_sysreg_s(SYS_BRBTGT##n##_EL1)
> +
> +#define RETURN_READ_BRBINFN(n) \
> + read_sysreg_s(SYS_BRBINF##n##_EL1)
> +
> +#define BRBE_REGN_CASE(n, case_macro) \
> + case n: return case_macro(n); break
> +
> +#define BRBE_REGN_SWITCH(x, case_macro) \
> + do { \
> + switch (x) { \
> + BRBE_REGN_CASE(0, case_macro); \
> + BRBE_REGN_CASE(1, case_macro); \
> + BRBE_REGN_CASE(2, case_macro); \
> + BRBE_REGN_CASE(3, case_macro); \
> + BRBE_REGN_CASE(4, case_macro); \
> + BRBE_REGN_CASE(5, case_macro); \
> + BRBE_REGN_CASE(6, case_macro); \
> + BRBE_REGN_CASE(7, case_macro); \
> + BRBE_REGN_CASE(8, case_macro); \
> + BRBE_REGN_CASE(9, case_macro); \
> + BRBE_REGN_CASE(10, case_macro); \
> + BRBE_REGN_CASE(11, case_macro); \
> + BRBE_REGN_CASE(12, case_macro); \
> + BRBE_REGN_CASE(13, case_macro); \
> + BRBE_REGN_CASE(14, case_macro); \
> + BRBE_REGN_CASE(15, case_macro); \
> + BRBE_REGN_CASE(16, case_macro); \
> + BRBE_REGN_CASE(17, case_macro); \
> + BRBE_REGN_CASE(18, case_macro); \
> + BRBE_REGN_CASE(19, case_macro); \
> + BRBE_REGN_CASE(20, case_macro); \
> + BRBE_REGN_CASE(21, case_macro); \
> + BRBE_REGN_CASE(22, case_macro); \
> + BRBE_REGN_CASE(23, case_macro); \
> + BRBE_REGN_CASE(24, case_macro); \
> + BRBE_REGN_CASE(25, case_macro); \
> + BRBE_REGN_CASE(26, case_macro); \
> + BRBE_REGN_CASE(27, case_macro); \
> + BRBE_REGN_CASE(28, case_macro); \
> + BRBE_REGN_CASE(29, case_macro); \
> + BRBE_REGN_CASE(30, case_macro); \
> + BRBE_REGN_CASE(31, case_macro); \
> + default: \
> + pr_warn("unknown register index\n"); \
> + return -1; \
> + } \
> + } while (0)
> +
> +static inline int buffer_to_brbe_idx(int buffer_idx)
> +{
> + return buffer_idx % 32;
> +}
> +
> +static inline u64 get_brbsrc_reg(int buffer_idx)
> +{
> + int brbe_idx = buffer_to_brbe_idx(buffer_idx);
> +
> + BRBE_REGN_SWITCH(brbe_idx, RETURN_READ_BRBSRCN);
> +}
> +
> +static inline u64 get_brbtgt_reg(int buffer_idx)
> +{
> + int brbe_idx = buffer_to_brbe_idx(buffer_idx);
> +
> + BRBE_REGN_SWITCH(brbe_idx, RETURN_READ_BRBTGTN);
> +}
> +
> +static inline u64 get_brbinf_reg(int buffer_idx)
> +{
> + int brbe_idx = buffer_to_brbe_idx(buffer_idx);
> +
> + BRBE_REGN_SWITCH(brbe_idx, RETURN_READ_BRBINFN);
> +}
> +
> +static inline u64 brbe_record_valid(u64 brbinf)
> +{
> + return brbinf & (BRBINF_VALID_MASK << BRBINF_VALID_SHIFT);
> +}
> +
> +static inline bool brbe_invalid(u64 brbinf)
> +{
> + return brbe_record_valid(brbinf) == BRBINF_VALID_INVALID;
> +}
> +
> +static inline bool brbe_valid(u64 brbinf)
> +{
> + return brbe_record_valid(brbinf) == BRBINF_VALID_ALL;
> +}
> +
> +static inline bool brbe_source(u64 brbinf)
> +{
> + return brbe_record_valid(brbinf) == BRBINF_VALID_SOURCE;
> +}
> +
> +static inline bool brbe_target(u64 brbinf)
> +{
> + return brbe_record_valid(brbinf) == BRBINF_VALID_TARGET;
> +}
> +
> +static inline int brbe_fetch_cycles(u64 brbinf)
> +{
> + /*
> + * Captured cycle count is unknown and hence
> + * should not be passed on the user space.
> + */
> + if (brbinf & BRBINF_CCU)
> + return 0;
> +
> + return (brbinf >> BRBINF_CC_SHIFT) & BRBINF_CC_MASK;
> +}
> +
> +static inline int brbe_fetch_type(u64 brbinf)
> +{
> + return (brbinf >> BRBINF_TYPE_SHIFT) & BRBINF_TYPE_MASK;
> +}
> +
> +static inline int brbe_fetch_el(u64 brbinf)
> +{
> + return (brbinf >> BRBINF_EL_SHIFT) & BRBINF_EL_MASK;
> +}
> +
> +static inline int brbe_fetch_numrec(u64 brbidr)
> +{
> + return (brbidr >> BRBIDR0_NUMREC_SHIFT) & BRBIDR0_NUMREC_MASK;
> +}
> +
> +static inline int brbe_fetch_format(u64 brbidr)
> +{
> + return (brbidr >> BRBIDR0_FORMAT_SHIFT) & BRBIDR0_FORMAT_MASK;
> +}
> +
> +static inline int brbe_fetch_cc_bits(u64 brbidr)
> +{
> + return (brbidr >> BRBIDR0_CC_SHIFT) & BRBIDR0_CC_MASK;
> +}
> +
> +static inline void select_brbe_bank(int bank)
> +{
> + static int brbe_current_bank = -1;
> + u64 brbfcr;
> +
> + if (brbe_current_bank == bank)
> + return;
> +
> + WARN_ON(bank > 1);
> + brbfcr = read_sysreg_s(SYS_BRBFCR_EL1);
> + brbfcr &= ~(BRBFCR_BANK_MASK << BRBFCR_BANK_SHIFT);
> + brbfcr |= ((bank & BRBFCR_BANK_MASK) << BRBFCR_BANK_SHIFT);
> + write_sysreg_s(brbfcr, SYS_BRBFCR_EL1);
> + isb();
> + brbe_current_bank = bank;
> +}
> +
> +static inline void select_brbe_bank_index(int buffer_idx)
> +{
> + switch (buffer_idx) {
> + case BRBE_BANK0_IDX_MIN ... BRBE_BANK0_IDX_MAX:
> + select_brbe_bank(0);
> + break;
> + case BRBE_BANK1_IDX_MIN ... BRBE_BANK1_IDX_MAX:
> + select_brbe_bank(1);
> + break;
> + default:
> + pr_warn("unsupported BRBE index\n");
> + }
> +}
> +
> +static inline bool valid_brbe_nr(int brbe_nr)
> +{
> + switch (brbe_nr) {
> + case BRBIDR0_NUMREC_8:
> + case BRBIDR0_NUMREC_16:
> + case BRBIDR0_NUMREC_32:
> + case BRBIDR0_NUMREC_64:
> + return true;
> + default:
> + pr_warn("unsupported BRBE entries\n");
> + return false;
> + }
> +}
> +
> +static inline bool brbe_paused(void)
> +{
> + u64 brbfcr = read_sysreg_s(SYS_BRBFCR_EL1);
> +
> + return brbfcr & BRBFCR_PAUSED;
> +}
> +
> +static inline void set_brbe_paused(void)
> +{
> + u64 brbfcr = read_sysreg_s(SYS_BRBFCR_EL1);
> +
> + write_sysreg_s(brbfcr | BRBFCR_PAUSED, SYS_BRBFCR_EL1);
> + isb();
> +}
> diff --git a/include/linux/perf/arm_pmu.h b/include/linux/perf/arm_pmu.h
> index c4a705d3d054..d401771e7767 100644
> --- a/include/linux/perf/arm_pmu.h
> +++ b/include/linux/perf/arm_pmu.h
> @@ -146,6 +146,26 @@ struct arm_pmu {
> unsigned long acpi_cpuid;
> };
>
> +#ifdef CONFIG_ARM_BRBE_PMU
> +void arm64_pmu_brbe_filter(struct pmu_hw_events *hw_events, struct perf_event *event);
> +void arm64_pmu_brbe_read(struct pmu_hw_events *cpuc, struct perf_event *event);
> +void arm64_pmu_brbe_disable(struct pmu_hw_events *cpuc);
> +void arm64_pmu_brbe_enable(struct pmu_hw_events *cpuc);
> +void arm64_pmu_brbe_probe(struct pmu_hw_events *cpuc);
> +void arm64_pmu_brbe_reset(struct pmu_hw_events *cpuc);
> +bool arm64_pmu_brbe_supported(struct perf_event *event);
> +#else
> +static inline void arm64_pmu_brbe_filter(struct pmu_hw_events *hw_events, struct perf_event *event)
> +{
> +}
> +static inline void arm64_pmu_brbe_read(struct pmu_hw_events *cpuc, struct perf_event *event) { }
> +static inline void arm64_pmu_brbe_disable(struct pmu_hw_events *cpuc) { }
> +static inline void arm64_pmu_brbe_enable(struct pmu_hw_events *cpuc) { }
> +static inline void arm64_pmu_brbe_probe(struct pmu_hw_events *cpuc) { }
> +static inline void arm64_pmu_brbe_reset(struct pmu_hw_events *cpuc) { }
> +static inline bool arm64_pmu_brbe_supported(struct perf_event *event) {return false; }
> +#endif
> +
> #define to_arm_pmu(p) (container_of(p, struct arm_pmu, pmu))
>
> u64 armpmu_event_update(struct perf_event *event);
>

2022-01-24 19:50:56

by James Clark

[permalink] [raw]
Subject: Re: [RFC V1 07/11] arm64/perf: Add BRBE driver



On 24/01/2022 04:30, Anshuman Khandual wrote:
> This adds a BRBE driver which implements all the required helper functions
> for struct arm_pmu. Following functions are defined by this driver which
> will configure, enable, capture, reset and disable BRBE buffer HW as and
> when requested via perf branch stack sampling framework.
>
> - arm64_pmu_brbe_filter()
> - arm64_pmu_brbe_enable()
> - arm64_pmu_brbe_disable()
> - arm64_pmu_brbe_read()
> - arm64_pmu_brbe_probe()
> - arm64_pmu_brbe_reset()
> - arm64_pmu_brbe_supported()
>
> Cc: Peter Zijlstra <[email protected]>
> Cc: Ingo Molnar <[email protected]>
> Cc: Arnaldo Carvalho de Melo <[email protected]>
> Cc: Mark Rutland <[email protected]>
> Cc: Will Deacon <[email protected]>
> Cc: Catalin Marinas <[email protected]>
> Cc: [email protected]
> Cc: [email protected]
> Cc: [email protected]
> Signed-off-by: Anshuman Khandual <[email protected]>
> ---
[...]
> +bool arm64_pmu_brbe_supported(struct perf_event *event)
> +{
> + struct arm_pmu *armpmu = to_arm_pmu(event->pmu);
> + struct pmu_hw_events *hw_events = per_cpu_ptr(armpmu->hw_events, event->cpu);
> +
> + if (event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_KERNEL) {
> + if (!perfmon_capable()) {
> + pr_warn_once("does not have permission for kernel branch filter\n");
> + return false;

Why not check perf_event_paranoid too? I would expect kernel sampling to be allowed for non
root users with paranoid <= 0.

2022-01-25 22:51:07

by James Clark

[permalink] [raw]
Subject: Re: [RFC V1 11/11] perf: Capture branch privilege information



On 24/01/2022 04:30, Anshuman Khandual wrote:
> Platforms like arm64 could capture privilege level information for all the
> branch records. Hence this adds a new element in the struct branch_entry to
> record the privilege level information, which could be requested through a
> new event.attr.branch_sample_type flag PERF_SAMPLE_BRANCH_PRIV_SAVE. While
> here, update the BRBE driver as required.
>
> Cc: Will Deacon <[email protected]>
> Cc: Mark Rutland <[email protected]>
> Cc: Peter Zijlstra <[email protected]>
> Cc: Ingo Molnar <[email protected]>
> Cc: Arnaldo Carvalho de Melo <[email protected]>
> Cc: Jiri Olsa <[email protected]>
> Cc: Namhyung Kim <[email protected]>
> Cc: [email protected]
> Cc: [email protected]
> Cc: [email protected]
> Signed-off-by: Anshuman Khandual <[email protected]>
> ---
> drivers/perf/arm_pmu_brbe.c | 28 ++++++++++++++++++++++++
> include/linux/perf_event.h | 5 +++++
> include/uapi/linux/perf_event.h | 13 ++++++++++-
> tools/include/uapi/linux/perf_event.h | 13 ++++++++++-
> tools/perf/Documentation/perf-record.txt | 1 +
> tools/perf/util/parse-branch-options.c | 1 +
> 6 files changed, 59 insertions(+), 2 deletions(-)
>
[...]
> diff --git a/include/uapi/linux/perf_event.h b/include/uapi/linux/perf_event.h
> index 361fdc6b87a0..4d77710f7a4e 100644
> --- a/include/uapi/linux/perf_event.h
> +++ b/include/uapi/linux/perf_event.h
> @@ -204,6 +204,8 @@ enum perf_branch_sample_type_shift {
>
> PERF_SAMPLE_BRANCH_HW_INDEX_SHIFT = 17, /* save low level index of raw branch records */
>
> + PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT = 18, /* save privillege mode */

privillege -> privilege

> +
> PERF_SAMPLE_BRANCH_MAX_SHIFT /* non-ABI */
> };
>
> @@ -233,6 +235,8 @@ enum perf_branch_sample_type {
>
> PERF_SAMPLE_BRANCH_HW_INDEX = 1U << PERF_SAMPLE_BRANCH_HW_INDEX_SHIFT,
>
> + PERF_SAMPLE_BRANCH_PRIV_SAVE = 1U << PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT,
> +

Can you also add the new entry to the perf verbose printer otherwise it looks like
it's not being set on branch_sample_type
(as in when using ./perf record -j any_call,u,priv -vvv):

static void __p_branch_sample_type(char *buf, size_t size, u64 value)
{
#define bit_name(n) { PERF_SAMPLE_BRANCH_##n, #n }
struct bit_names bits[] = {
bit_name(USER), bit_name(KERNEL), bit_name(HV), bit_name(ANY),
bit_name(ANY_CALL), bit_name(ANY_RETURN), bit_name(IND_CALL),
bit_name(ABORT_TX), bit_name(IN_TX), bit_name(NO_TX),
bit_name(COND), bit_name(CALL_STACK), bit_name(IND_JUMP),
bit_name(CALL), bit_name(NO_FLAGS), bit_name(NO_CYCLES),
bit_name(HW_INDEX),
{ .name = NULL, }
};
#undef bit_name
__p_bits(buf, size, value, bits);
}

PERF_SAMPLE_BRANCH_TYPE_SAVE_SHIFT is also missing so it probably makes sense to add it
at the same time as that was expanded for BRBE.

> PERF_SAMPLE_BRANCH_MAX = 1U << PERF_SAMPLE_BRANCH_MAX_SHIFT,
> };
>
> @@ -265,6 +269,12 @@ enum {
> PERF_BR_MAX,
> };
>
> +enum {
> + PERF_BR_USER = 0,
> + PERF_BR_KERNEL = 1,
> + PERF_BR_HV = 2,
> +};
> +
> #define PERF_SAMPLE_BRANCH_PLM_ALL \
> (PERF_SAMPLE_BRANCH_USER|\
> PERF_SAMPLE_BRANCH_KERNEL|\
> @@ -1377,7 +1387,8 @@ struct perf_branch_entry {
> abort:1, /* transaction abort */
> cycles:16, /* cycle count to last branch */
> type:6, /* branch type */
> - reserved:38;
> + priv:2, /* privilege level */
> + reserved:36;
> };
>
> union perf_sample_weight {
> diff --git a/tools/include/uapi/linux/perf_event.h b/tools/include/uapi/linux/perf_event.h
> index 9a82b8aaed93..a2208400b0b9 100644
> --- a/tools/include/uapi/linux/perf_event.h
> +++ b/tools/include/uapi/linux/perf_event.h
> @@ -204,6 +204,8 @@ enum perf_branch_sample_type_shift {
>
> PERF_SAMPLE_BRANCH_HW_INDEX_SHIFT = 17, /* save low level index of raw branch records */
>
> + PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT = 18, /* save privillege mode */
> +
> PERF_SAMPLE_BRANCH_MAX_SHIFT /* non-ABI */
> };
>
> @@ -233,6 +235,8 @@ enum perf_branch_sample_type {
>
> PERF_SAMPLE_BRANCH_HW_INDEX = 1U << PERF_SAMPLE_BRANCH_HW_INDEX_SHIFT,
>
> + PERF_SAMPLE_BRANCH_PRIV_SAVE = 1U << PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT,
> +
> PERF_SAMPLE_BRANCH_MAX = 1U << PERF_SAMPLE_BRANCH_MAX_SHIFT,
> };
>
> @@ -265,6 +269,12 @@ enum {
> PERF_BR_MAX,
> };
>
> +enum {
> + PERF_BR_USER = 0,
> + PERF_BR_KERNEL = 1,
> + PERF_BR_HV = 2,
> +};
> +
> #define PERF_SAMPLE_BRANCH_PLM_ALL \
> (PERF_SAMPLE_BRANCH_USER|\
> PERF_SAMPLE_BRANCH_KERNEL|\
> @@ -1377,7 +1387,8 @@ struct perf_branch_entry {
> abort:1, /* transaction abort */
> cycles:16, /* cycle count to last branch */
> type:6, /* branch type */
> - reserved:38;
> + priv:2, /* privilege level */
> + reserved:36;
> };
>
> union perf_sample_weight {
> diff --git a/tools/perf/Documentation/perf-record.txt b/tools/perf/Documentation/perf-record.txt
> index 9ccc75935bc5..3e33686977a1 100644
> --- a/tools/perf/Documentation/perf-record.txt
> +++ b/tools/perf/Documentation/perf-record.txt
> @@ -387,6 +387,7 @@ following filters are defined:
> - abort_tx: only when the target is a hardware transaction abort
> - cond: conditional branches
> - save_type: save branch type during sampling in case binary is not available later
> + - priv: save privilege state during sampling in case binary is not available later
>
> +
> The option requires at least one branch type among any, any_call, any_ret, ind_call, cond.
> diff --git a/tools/perf/util/parse-branch-options.c b/tools/perf/util/parse-branch-options.c
> index bb4aa88c50a8..00588b9db474 100644
> --- a/tools/perf/util/parse-branch-options.c
> +++ b/tools/perf/util/parse-branch-options.c
> @@ -32,6 +32,7 @@ static const struct branch_mode branch_modes[] = {
> BRANCH_OPT("call", PERF_SAMPLE_BRANCH_CALL),
> BRANCH_OPT("save_type", PERF_SAMPLE_BRANCH_TYPE_SAVE),
> BRANCH_OPT("stack", PERF_SAMPLE_BRANCH_CALL_STACK),
> + BRANCH_OPT("priv", PERF_SAMPLE_BRANCH_PRIV_SAVE),
> BRANCH_END
> };
>
>

2022-01-25 23:04:12

by German Gomez

[permalink] [raw]
Subject: [PATCH 0/1] perf test: Add branch stack sampling tests for ARM64

Adds testing of branch stack sampling on ARM64. Branch stack sampling is
supported by the Branch Record Buffer Extension (BRBE). In order to run
the tests, the Kernel must have BRBE support enabled.

At the time of writing, BRBE support in the Kernel is provided by the
patches in RFC [1]. If BRBE support is not detected, the tests will be
skipped.

Information about BRBE can be found in [2].

[1]: https://lore.kernel.org/all/[email protected]/
[2]: https://developer.arm.com/documentation/ddi0608/latest

German Gomez (1):
perf test: Add branch stack sampling tests for ARM64

.../perf/tests/shell/test_arm_brbe_kernel.sh | 42 ++++++++++
.../tests/shell/test_arm_brbe_userspace.sh | 80 +++++++++++++++++++
2 files changed, 122 insertions(+)
create mode 100755 tools/perf/tests/shell/test_arm_brbe_kernel.sh
create mode 100755 tools/perf/tests/shell/test_arm_brbe_userspace.sh

--
2.25.1

2022-01-25 23:04:25

by German Gomez

[permalink] [raw]
Subject: [PATCH 1/1] perf test: Add branch stack sampling tests for ARM64

Adds two shell script tests in order to test branch stack sampling on
ARM64 plarforms. This functionality is enabled by the Branch Record
Buffer Extension (BRBE) on Arm.

Information about BRBE can be found in [1] chapter F1.

[1]: https://developer.arm.com/documentation/ddi0608/latest

Signed-off-by: German Gomez <[email protected]>
---
.../perf/tests/shell/test_arm_brbe_kernel.sh | 42 ++++++++++
.../tests/shell/test_arm_brbe_userspace.sh | 80 +++++++++++++++++++
2 files changed, 122 insertions(+)
create mode 100755 tools/perf/tests/shell/test_arm_brbe_kernel.sh
create mode 100755 tools/perf/tests/shell/test_arm_brbe_userspace.sh

diff --git a/tools/perf/tests/shell/test_arm_brbe_kernel.sh b/tools/perf/tests/shell/test_arm_brbe_kernel.sh
new file mode 100755
index 000000000..dc5a2238f
--- /dev/null
+++ b/tools/perf/tests/shell/test_arm_brbe_kernel.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+# Arm64 BRBE kernel branches
+
+# SPDX-License-Identifier: GPL-2.0
+# German Gomez <[email protected]>, 2022
+
+PERF_DATA=$(mktemp /tmp/__perf_test.perf.data.XXXXX)
+
+cleanup_files() {
+ rm -f $PERF_DATA*
+}
+
+trap cleanup_files exit term int
+
+test_brbe_kernel() {
+ lscpu | grep -q "aarch64" || return $?
+ perf record -o $PERF_DATA --branch-filter any,k -- true > /dev/null 2>&1
+}
+
+test_brbe_kernel || exit 2
+
+# example perf-script output:
+# 0xffffffff9a80dd5e/0xffffffff9a87c5c0/P/-/-/1
+# 0xffff8000080ac20c/0xffff800008e99720/P/-/-/2
+# 0xffff8000080ac20c/0xffff800008e99720/P/-/-/3
+
+# kernel addresses always have the upper 16 bits set (https://lwn.net/Articles/718895/)
+KERNEL_ADDRESS_REGEX="0xffff[0-9a-f]{12}"
+
+perf record -o $PERF_DATA --branch-filter any,k -a -- sleep 1
+perf script -i $PERF_DATA --fields brstack | egrep "(0x0|$KERNEL_ADDRESS_REGEX)\/(0x0|$KERNEL_ADDRESS_REGEX)\/" > /dev/null
+err=$?
+
+echo -n "BRB kernel branches: "
+if [ $err != 0 ]; then
+ echo "FAIL"
+ exit 1
+else
+ echo "PASS"
+fi
+
+exit 0
diff --git a/tools/perf/tests/shell/test_arm_brbe_userspace.sh b/tools/perf/tests/shell/test_arm_brbe_userspace.sh
new file mode 100755
index 000000000..4f0bdc03a
--- /dev/null
+++ b/tools/perf/tests/shell/test_arm_brbe_userspace.sh
@@ -0,0 +1,80 @@
+#!/bin/bash
+# Arm64 BRBE userspace branches
+
+# SPDX-License-Identifier: GPL-2.0
+# German Gomez <[email protected]>, 2022
+
+PERF_DATA=$(mktemp /tmp/__perf_test.perf.data.XXXXX)
+TEST_PROGRAM_SOURCE=$(mktemp /tmp/brbe_test_program.XXXXX.c)
+TEST_PROGRAM=$(mktemp /tmp/brbe_test_program.XXXXX)
+
+cleanup_files() {
+ rm -f $PERF_DATA*
+ rm -f $TEST_PROGRAM_SOURCE
+ rm -f $TEST_PROGRAM
+}
+
+trap cleanup_files exit term int
+
+test_brbe_user() {
+ lscpu | grep -q "aarch64" || return $?
+ perf record -o $PERF_DATA --branch-filter any,u -- true > /dev/null 2>&1
+}
+
+test_brbe_user || exit 2
+
+# Skip if there's no compiler
+# We need it to compile the test program
+if ! [ -x "$(command -v cc)" ]; then
+ echo "failed: no compiler, install gcc"
+ exit 2
+fi
+
+script_has_branch() {
+ local from="$1\+0x[0-9a-f]+"
+ local to="$2\+0x[0-9a-f]+"
+ perf script -i $PERF_DATA --fields brstacksym | egrep -qm 1 " +$from\/$to\/"
+}
+
+# compile test program
+cat << EOF > $TEST_PROGRAM_SOURCE
+void f2() {
+}
+void f1() {
+ f2();
+}
+void f0() {
+ f1();
+ f2();
+}
+int main() {
+ while(1) {
+ f0();
+ f1();
+ }
+ return 0;
+}
+EOF
+
+CFLAGS="-O0 -fno-inline -static"
+cc $CFLAGS $TEST_PROGRAM_SOURCE -o $TEST_PROGRAM || exit 1
+
+perf record -o $PERF_DATA --branch-filter any,u -- timeout 1 $TEST_PROGRAM
+
+script_has_branch "main" "f0" &&
+ script_has_branch "main" "f1" &&
+ script_has_branch "f0" "f1" &&
+ script_has_branch "f0" "f2" &&
+ script_has_branch "f1" "f2" &&
+ script_has_branch "main" "main"
+err=$?
+
+echo -n "BRB user branches: "
+if [ $err != 0 ]; then
+ echo "FAIL"
+ exit 1
+else
+ echo "PASS"
+fi
+
+exit 0
--
2.25.1

2022-01-26 01:21:43

by James Clark

[permalink] [raw]
Subject: Re: [RFC V1 10/11] perf: Expand perf_branch_entry.type



On 24/01/2022 04:30, Anshuman Khandual wrote:
> Current perf_branch_entry.type is a 4 bits field just enough to accommodate
> 16 generic branch types. This is insufficient to accommodate platforms like
> arm64 which has much more branch types. Lets just expands this field into a
> 6 bits one, which can now hold 64 generic branch types. This also adds more
> generic branch types and updates the BRBE driver as required.
>
> Cc: Peter Zijlstra <[email protected]>
> Cc: Ingo Molnar <[email protected]>
> Cc: Arnaldo Carvalho de Melo <[email protected]>
> Cc: Mark Rutland <[email protected]>
> Cc: Alexander Shishkin <[email protected]>
> Cc: Jiri Olsa <[email protected]>
> Cc: Namhyung Kim <[email protected]>
> Cc: Will Deacon <[email protected]>
> Cc: [email protected]
> Cc: [email protected]
> Cc: [email protected]
> Signed-off-by: Anshuman Khandual <[email protected]>
> ---
> drivers/perf/arm_pmu_brbe.c | 7 ++++++-
> include/uapi/linux/perf_event.h | 10 ++++++++--
> tools/include/uapi/linux/perf_event.h | 10 ++++++++--
> tools/perf/util/branch.c | 8 +++++++-
> 4 files changed, 29 insertions(+), 6 deletions(-)
>
> diff --git a/drivers/perf/arm_pmu_brbe.c b/drivers/perf/arm_pmu_brbe.c
> index 8d27ad868359..7cd1208c6c58 100644
> --- a/drivers/perf/arm_pmu_brbe.c
> +++ b/drivers/perf/arm_pmu_brbe.c
> @@ -253,12 +253,17 @@ static int brbe_fetch_perf_type(u64 brbinf)
> case BRBINF_TYPE_DEBUG_EXIT:
> return PERF_BR_DEBUG_EXIT;
> case BRBINF_TYPE_SERROR:
> + return PERF_BR_SERROR;
> case BRBINF_TYPE_INST_DEBUG:
> + return PERF_BR_DEBUG_INST;
> case BRBINF_TYPE_DATA_DEBUG:
> + return PERF_BR_DEBUG_DATA;
> case BRBINF_TYPE_ALGN_FAULT:
> + return PERF_BR_FAULT_ALGN;
> case BRBINF_TYPE_INST_FAULT:
> + return PERF_BR_FAULT_INST;
> case BRBINF_TYPE_DATA_FAULT:
> - return PERF_BR_UNKNOWN;
> + return PERF_BR_FAULT_DATA;
> default:
> pr_warn("unknown branch type captured\n");
> return PERF_BR_UNKNOWN;
> diff --git a/include/uapi/linux/perf_event.h b/include/uapi/linux/perf_event.h
> index b91d0f575d0c..361fdc6b87a0 100644
> --- a/include/uapi/linux/perf_event.h
> +++ b/include/uapi/linux/perf_event.h
> @@ -256,6 +256,12 @@ enum {
> PERF_BR_FIQ = 13, /* fiq */
> PERF_BR_DEBUG_HALT = 14, /* debug halt */
> PERF_BR_DEBUG_EXIT = 15, /* debug exit */
> + PERF_BR_DEBUG_INST = 16, /* instruciton debug */
> + PERF_BR_DEBUG_DATA = 17, /* data debug */
> + PERF_BR_FAULT_ALGN = 18, /* alignment fault */
> + PERF_BR_FAULT_DATA = 19, /* data fault */
> + PERF_BR_FAULT_INST = 20, /* instruction fault */
> + PERF_BR_SERROR = 21, /* system error */
> PERF_BR_MAX,
> };
>
> @@ -1370,8 +1376,8 @@ struct perf_branch_entry {
> in_tx:1, /* in transaction */
> abort:1, /* transaction abort */
> cycles:16, /* cycle count to last branch */
> - type:4, /* branch type */
> - reserved:40;
> + type:6, /* branch type */
> + reserved:38;
> };
>
> union perf_sample_weight {
> diff --git a/tools/include/uapi/linux/perf_event.h b/tools/include/uapi/linux/perf_event.h
> index 1882054e8684..9a82b8aaed93 100644
> --- a/tools/include/uapi/linux/perf_event.h
> +++ b/tools/include/uapi/linux/perf_event.h
> @@ -256,6 +256,12 @@ enum {
> PERF_BR_FIQ = 13, /* fiq */
> PERF_BR_DEBUG_HALT = 14, /* debug halt */
> PERF_BR_DEBUG_EXIT = 15, /* debug exit */
> + PERF_BR_DEBUG_INST = 16, /* instruciton debug */
> + PERF_BR_DEBUG_DATA = 17, /* data debug */
> + PERF_BR_FAULT_ALGN = 18, /* alignment fault */
> + PERF_BR_FAULT_DATA = 19, /* data fault */
> + PERF_BR_FAULT_INST = 20, /* instruction fault */
> + PERF_BR_SERROR = 21, /* system error */
> PERF_BR_MAX,
> };
>
> @@ -1370,8 +1376,8 @@ struct perf_branch_entry {
> in_tx:1, /* in transaction */
> abort:1, /* transaction abort */
> cycles:16, /* cycle count to last branch */
> - type:4, /* branch type */
> - reserved:40;
> + type:6, /* branch type */
> + reserved:38;
> };

There's another copy of this struct in branch.h that is used to access the same data in
perf which also needs updating:

struct branch_flags {
union {
u64 value;
struct {
u64 mispred:1;
u64 predicted:1;
u64 in_tx:1;
u64 abort:1;
u64 cycles:16;
u64 type:4;
u64 reserved:40;
};
};
};

It's never assigned directly but there is some casting stuff going on in
evsel__parse_sample() and it eventually ends up being used to access branch
records. Same applies to the privilege data change.

>
> union perf_sample_weight {
> diff --git a/tools/perf/util/branch.c b/tools/perf/util/branch.c
> index 74e5e67b1779..1e216ea2e2a8 100644
> --- a/tools/perf/util/branch.c
> +++ b/tools/perf/util/branch.c
> @@ -54,7 +54,13 @@ const char *branch_type_name(int type)
> "IRQ",
> "FIQ",
> "DEBUG_HALT",
> - "DEBUG_EXIT"
> + "DEBUG_EXIT",
> + "DEBUG_INST",
> + "DEBUG_DATA",
> + "FAULT_ALGN",
> + "FAULT_DATA",
> + "FAULT_INST",
> + "SERROR"
> };
>
> if (type >= 0 && type < PERF_BR_MAX)
>

2022-01-26 22:19:04

by Rob Herring

[permalink] [raw]
Subject: Re: [RFC V1 10/11] perf: Expand perf_branch_entry.type

On Mon, Jan 24, 2022 at 10:00:52AM +0530, Anshuman Khandual wrote:
> Current perf_branch_entry.type is a 4 bits field just enough to accommodate
> 16 generic branch types. This is insufficient to accommodate platforms like
> arm64 which has much more branch types. Lets just expands this field into a
> 6 bits one, which can now hold 64 generic branch types. This also adds more
> generic branch types and updates the BRBE driver as required.
>
> Cc: Peter Zijlstra <[email protected]>
> Cc: Ingo Molnar <[email protected]>
> Cc: Arnaldo Carvalho de Melo <[email protected]>
> Cc: Mark Rutland <[email protected]>
> Cc: Alexander Shishkin <[email protected]>
> Cc: Jiri Olsa <[email protected]>
> Cc: Namhyung Kim <[email protected]>
> Cc: Will Deacon <[email protected]>
> Cc: [email protected]
> Cc: [email protected]
> Cc: [email protected]
> Signed-off-by: Anshuman Khandual <[email protected]>
> ---
> drivers/perf/arm_pmu_brbe.c | 7 ++++++-

This patch should come before the BRBE support so that it can be applied
sooner rather than later. If another field is added to the end before
expanding this field, then you will be stuck with assembling bits from 2
fields.

> include/uapi/linux/perf_event.h | 10 ++++++++--
> tools/include/uapi/linux/perf_event.h | 10 ++++++++--
> tools/perf/util/branch.c | 8 +++++++-
> 4 files changed, 29 insertions(+), 6 deletions(-)

2022-01-26 22:24:13

by James Clark

[permalink] [raw]
Subject: Re: [RFC V1 11/11] perf: Capture branch privilege information



On 24/01/2022 04:30, Anshuman Khandual wrote:
> Platforms like arm64 could capture privilege level information for all the
> branch records. Hence this adds a new element in the struct branch_entry to
> record the privilege level information, which could be requested through a
> new event.attr.branch_sample_type flag PERF_SAMPLE_BRANCH_PRIV_SAVE. While
> here, update the BRBE driver as required.
>
> Cc: Will Deacon <[email protected]>
> Cc: Mark Rutland <[email protected]>
> Cc: Peter Zijlstra <[email protected]>
> Cc: Ingo Molnar <[email protected]>
> Cc: Arnaldo Carvalho de Melo <[email protected]>
> Cc: Jiri Olsa <[email protected]>
> Cc: Namhyung Kim <[email protected]>
> Cc: [email protected]
> Cc: [email protected]
> Cc: [email protected]
> Signed-off-by: Anshuman Khandual <[email protected]>
> ---
> drivers/perf/arm_pmu_brbe.c | 28 ++++++++++++++++++++++++
> include/linux/perf_event.h | 5 +++++
> include/uapi/linux/perf_event.h | 13 ++++++++++-
> tools/include/uapi/linux/perf_event.h | 13 ++++++++++-
> tools/perf/Documentation/perf-record.txt | 1 +
> tools/perf/util/parse-branch-options.c | 1 +
> 6 files changed, 59 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/perf/arm_pmu_brbe.c b/drivers/perf/arm_pmu_brbe.c
> index 7cd1208c6c58..d4cbea74c148 100644
> --- a/drivers/perf/arm_pmu_brbe.c
> +++ b/drivers/perf/arm_pmu_brbe.c
> @@ -270,6 +270,25 @@ static int brbe_fetch_perf_type(u64 brbinf)
> }
> }
>
> +static int brbe_fetch_perf_priv(u64 brbinf)
> +{
> + int brbe_el = brbe_fetch_el(brbinf);
> +
> + switch (brbe_el) {
> + case BRBINF_EL_EL0:
> + return PERF_BR_USER;
> + case BRBINF_EL_EL1:
> + return PERF_BR_KERNEL;
> + case BRBINF_EL_EL2:
> + if (is_kernel_in_hyp_mode())
> + return PERF_BR_KERNEL;
> + return PERF_BR_HV;
> + default:
> + pr_warn("unknown branch privilege captured\n");
> + return -1;
> + }
> +}
> +
> static void capture_brbe_flags(struct pmu_hw_events *cpuc, struct perf_event *event,
> u64 brbinf, int idx)
> {
> @@ -302,6 +321,15 @@ static void capture_brbe_flags(struct pmu_hw_events *cpuc, struct perf_event *ev
> cpuc->brbe_entries[idx].in_tx = brbinf & BRBINF_TX;
> }
> }
> +
> + if (branch_sample_priv(event)) {
> + /*
> + * All these information (i.e branch privilege level) are not
> + * available for source only branch records.
> + */
> + if (type != BRBINF_VALID_SOURCE)
> + cpuc->brbe_entries[idx].priv = brbe_fetch_perf_priv(brbinf);
> + }
> }
>
> /*
> diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
> index 916ce5102b33..8021b6a30d86 100644
> --- a/include/linux/perf_event.h
> +++ b/include/linux/perf_event.h
> @@ -1688,4 +1688,9 @@ static inline bool branch_sample_hw_index(const struct perf_event *event)
> {
> return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_HW_INDEX;
> }
> +
> +static inline bool branch_sample_priv(const struct perf_event *event)
> +{
> + return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_PRIV_SAVE;
> +}
> #endif /* _LINUX_PERF_EVENT_H */
> diff --git a/include/uapi/linux/perf_event.h b/include/uapi/linux/perf_event.h
> index 361fdc6b87a0..4d77710f7a4e 100644
> --- a/include/uapi/linux/perf_event.h
> +++ b/include/uapi/linux/perf_event.h
> @@ -204,6 +204,8 @@ enum perf_branch_sample_type_shift {
>
> PERF_SAMPLE_BRANCH_HW_INDEX_SHIFT = 17, /* save low level index of raw branch records */
>
> + PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT = 18, /* save privillege mode */
> +
> PERF_SAMPLE_BRANCH_MAX_SHIFT /* non-ABI */
> };
>
> @@ -233,6 +235,8 @@ enum perf_branch_sample_type {
>
> PERF_SAMPLE_BRANCH_HW_INDEX = 1U << PERF_SAMPLE_BRANCH_HW_INDEX_SHIFT,
>
> + PERF_SAMPLE_BRANCH_PRIV_SAVE = 1U << PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT,
> +
> PERF_SAMPLE_BRANCH_MAX = 1U << PERF_SAMPLE_BRANCH_MAX_SHIFT,
> };
>
> @@ -265,6 +269,12 @@ enum {
> PERF_BR_MAX,
> };
>
> +enum {
> + PERF_BR_USER = 0,
> + PERF_BR_KERNEL = 1,
> + PERF_BR_HV = 2,
> +};
> +

Can we have 0 as "UNKNOWN". It's going to be difficult to parse files when privilege information
isn't saved and get accurate results without that. For example if it's not set then presumably
the field would be 0 (PERF_BR_USER), but that doesn't mean the samples are user in that case.

I know you might be able to go backwards and look at what arguments were passed to the kernel but
it's not guaranteed that the kernel honored the request anyway. There are also other platforms
to think about etc.

If you look at the branch type definitions above they start at 0 (PERF_BR_UNKNOWN) which I think
works out quite nicely in the userspace code.

> #define PERF_SAMPLE_BRANCH_PLM_ALL \
> (PERF_SAMPLE_BRANCH_USER|\
> PERF_SAMPLE_BRANCH_KERNEL|\
> @@ -1377,7 +1387,8 @@ struct perf_branch_entry {
> abort:1, /* transaction abort */
> cycles:16, /* cycle count to last branch */
> type:6, /* branch type */
> - reserved:38;
> + priv:2, /* privilege level */> + reserved:36;
> };
>
> union perf_sample_weight {
> diff --git a/tools/include/uapi/linux/perf_event.h b/tools/include/uapi/linux/perf_event.h
> index 9a82b8aaed93..a2208400b0b9 100644
> --- a/tools/include/uapi/linux/perf_event.h
> +++ b/tools/include/uapi/linux/perf_event.h
> @@ -204,6 +204,8 @@ enum perf_branch_sample_type_shift {
>
> PERF_SAMPLE_BRANCH_HW_INDEX_SHIFT = 17, /* save low level index of raw branch records */
>
> + PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT = 18, /* save privillege mode */
> +
> PERF_SAMPLE_BRANCH_MAX_SHIFT /* non-ABI */
> };
>
> @@ -233,6 +235,8 @@ enum perf_branch_sample_type {
>
> PERF_SAMPLE_BRANCH_HW_INDEX = 1U << PERF_SAMPLE_BRANCH_HW_INDEX_SHIFT,
>
> + PERF_SAMPLE_BRANCH_PRIV_SAVE = 1U << PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT,
> +
> PERF_SAMPLE_BRANCH_MAX = 1U << PERF_SAMPLE_BRANCH_MAX_SHIFT,
> };
>
> @@ -265,6 +269,12 @@ enum {
> PERF_BR_MAX,
> };
>
> +enum {
> + PERF_BR_USER = 0,
> + PERF_BR_KERNEL = 1,
> + PERF_BR_HV = 2,
> +};
> +
> #define PERF_SAMPLE_BRANCH_PLM_ALL \
> (PERF_SAMPLE_BRANCH_USER|\
> PERF_SAMPLE_BRANCH_KERNEL|\
> @@ -1377,7 +1387,8 @@ struct perf_branch_entry {
> abort:1, /* transaction abort */
> cycles:16, /* cycle count to last branch */
> type:6, /* branch type */
> - reserved:38;
> + priv:2, /* privilege level */
> + reserved:36;
> };
>
> union perf_sample_weight {
> diff --git a/tools/perf/Documentation/perf-record.txt b/tools/perf/Documentation/perf-record.txt
> index 9ccc75935bc5..3e33686977a1 100644
> --- a/tools/perf/Documentation/perf-record.txt
> +++ b/tools/perf/Documentation/perf-record.txt
> @@ -387,6 +387,7 @@ following filters are defined:
> - abort_tx: only when the target is a hardware transaction abort
> - cond: conditional branches
> - save_type: save branch type during sampling in case binary is not available later
> + - priv: save privilege state during sampling in case binary is not available later
>
> +
> The option requires at least one branch type among any, any_call, any_ret, ind_call, cond.
> diff --git a/tools/perf/util/parse-branch-options.c b/tools/perf/util/parse-branch-options.c
> index bb4aa88c50a8..00588b9db474 100644
> --- a/tools/perf/util/parse-branch-options.c
> +++ b/tools/perf/util/parse-branch-options.c
> @@ -32,6 +32,7 @@ static const struct branch_mode branch_modes[] = {
> BRANCH_OPT("call", PERF_SAMPLE_BRANCH_CALL),
> BRANCH_OPT("save_type", PERF_SAMPLE_BRANCH_TYPE_SAVE),
> BRANCH_OPT("stack", PERF_SAMPLE_BRANCH_CALL_STACK),
> + BRANCH_OPT("priv", PERF_SAMPLE_BRANCH_PRIV_SAVE),
> BRANCH_END
> };
>
>

2022-01-27 18:00:12

by Anshuman Khandual

[permalink] [raw]
Subject: Re: [RFC V1 10/11] perf: Expand perf_branch_entry.type



On 1/26/22 10:17 PM, Rob Herring wrote:
> On Mon, Jan 24, 2022 at 10:00:52AM +0530, Anshuman Khandual wrote:
>> Current perf_branch_entry.type is a 4 bits field just enough to accommodate
>> 16 generic branch types. This is insufficient to accommodate platforms like
>> arm64 which has much more branch types. Lets just expands this field into a
>> 6 bits one, which can now hold 64 generic branch types. This also adds more
>> generic branch types and updates the BRBE driver as required.
>>
>> Cc: Peter Zijlstra <[email protected]>
>> Cc: Ingo Molnar <[email protected]>
>> Cc: Arnaldo Carvalho de Melo <[email protected]>
>> Cc: Mark Rutland <[email protected]>
>> Cc: Alexander Shishkin <[email protected]>
>> Cc: Jiri Olsa <[email protected]>
>> Cc: Namhyung Kim <[email protected]>
>> Cc: Will Deacon <[email protected]>
>> Cc: [email protected]
>> Cc: [email protected]
>> Cc: [email protected]
>> Signed-off-by: Anshuman Khandual <[email protected]>
>> ---
>> drivers/perf/arm_pmu_brbe.c | 7 ++++++-
>
> This patch should come before the BRBE support so that it can be applied
> sooner rather than later. If another field is added to the end before
> expanding this field, then you will be stuck with assembling bits from 2
> fields.

Should I just split this patch from the series and post it separately
right away, to be included in perf while BRBE support gets reviewed ?

>
>> include/uapi/linux/perf_event.h | 10 ++++++++--
>> tools/include/uapi/linux/perf_event.h | 10 ++++++++--
>> tools/perf/util/branch.c | 8 +++++++-
>> 4 files changed, 29 insertions(+), 6 deletions(-)

2022-01-29 14:50:52

by Anshuman Khandual

[permalink] [raw]
Subject: Re: [RFC V1 10/11] perf: Expand perf_branch_entry.type



On 1/25/22 10:28 PM, James Clark wrote:
>
> On 24/01/2022 04:30, Anshuman Khandual wrote:
>> Current perf_branch_entry.type is a 4 bits field just enough to accommodate
>> 16 generic branch types. This is insufficient to accommodate platforms like
>> arm64 which has much more branch types. Lets just expands this field into a
>> 6 bits one, which can now hold 64 generic branch types. This also adds more
>> generic branch types and updates the BRBE driver as required.
>>
>> Cc: Peter Zijlstra <[email protected]>
>> Cc: Ingo Molnar <[email protected]>
>> Cc: Arnaldo Carvalho de Melo <[email protected]>
>> Cc: Mark Rutland <[email protected]>
>> Cc: Alexander Shishkin <[email protected]>
>> Cc: Jiri Olsa <[email protected]>
>> Cc: Namhyung Kim <[email protected]>
>> Cc: Will Deacon <[email protected]>
>> Cc: [email protected]
>> Cc: [email protected]
>> Cc: [email protected]
>> Signed-off-by: Anshuman Khandual <[email protected]>
>> ---
>> drivers/perf/arm_pmu_brbe.c | 7 ++++++-
>> include/uapi/linux/perf_event.h | 10 ++++++++--
>> tools/include/uapi/linux/perf_event.h | 10 ++++++++--
>> tools/perf/util/branch.c | 8 +++++++-
>> 4 files changed, 29 insertions(+), 6 deletions(-)
>>
>> diff --git a/drivers/perf/arm_pmu_brbe.c b/drivers/perf/arm_pmu_brbe.c
>> index 8d27ad868359..7cd1208c6c58 100644
>> --- a/drivers/perf/arm_pmu_brbe.c
>> +++ b/drivers/perf/arm_pmu_brbe.c
>> @@ -253,12 +253,17 @@ static int brbe_fetch_perf_type(u64 brbinf)
>> case BRBINF_TYPE_DEBUG_EXIT:
>> return PERF_BR_DEBUG_EXIT;
>> case BRBINF_TYPE_SERROR:
>> + return PERF_BR_SERROR;
>> case BRBINF_TYPE_INST_DEBUG:
>> + return PERF_BR_DEBUG_INST;
>> case BRBINF_TYPE_DATA_DEBUG:
>> + return PERF_BR_DEBUG_DATA;
>> case BRBINF_TYPE_ALGN_FAULT:
>> + return PERF_BR_FAULT_ALGN;
>> case BRBINF_TYPE_INST_FAULT:
>> + return PERF_BR_FAULT_INST;
>> case BRBINF_TYPE_DATA_FAULT:
>> - return PERF_BR_UNKNOWN;
>> + return PERF_BR_FAULT_DATA;
>> default:
>> pr_warn("unknown branch type captured\n");
>> return PERF_BR_UNKNOWN;
>> diff --git a/include/uapi/linux/perf_event.h b/include/uapi/linux/perf_event.h
>> index b91d0f575d0c..361fdc6b87a0 100644
>> --- a/include/uapi/linux/perf_event.h
>> +++ b/include/uapi/linux/perf_event.h
>> @@ -256,6 +256,12 @@ enum {
>> PERF_BR_FIQ = 13, /* fiq */
>> PERF_BR_DEBUG_HALT = 14, /* debug halt */
>> PERF_BR_DEBUG_EXIT = 15, /* debug exit */
>> + PERF_BR_DEBUG_INST = 16, /* instruciton debug */
>> + PERF_BR_DEBUG_DATA = 17, /* data debug */
>> + PERF_BR_FAULT_ALGN = 18, /* alignment fault */
>> + PERF_BR_FAULT_DATA = 19, /* data fault */
>> + PERF_BR_FAULT_INST = 20, /* instruction fault */
>> + PERF_BR_SERROR = 21, /* system error */
>> PERF_BR_MAX,
>> };
>>
>> @@ -1370,8 +1376,8 @@ struct perf_branch_entry {
>> in_tx:1, /* in transaction */
>> abort:1, /* transaction abort */
>> cycles:16, /* cycle count to last branch */
>> - type:4, /* branch type */
>> - reserved:40;
>> + type:6, /* branch type */
>> + reserved:38;
>> };
>>
>> union perf_sample_weight {
>> diff --git a/tools/include/uapi/linux/perf_event.h b/tools/include/uapi/linux/perf_event.h
>> index 1882054e8684..9a82b8aaed93 100644
>> --- a/tools/include/uapi/linux/perf_event.h
>> +++ b/tools/include/uapi/linux/perf_event.h
>> @@ -256,6 +256,12 @@ enum {
>> PERF_BR_FIQ = 13, /* fiq */
>> PERF_BR_DEBUG_HALT = 14, /* debug halt */
>> PERF_BR_DEBUG_EXIT = 15, /* debug exit */
>> + PERF_BR_DEBUG_INST = 16, /* instruciton debug */
>> + PERF_BR_DEBUG_DATA = 17, /* data debug */
>> + PERF_BR_FAULT_ALGN = 18, /* alignment fault */
>> + PERF_BR_FAULT_DATA = 19, /* data fault */
>> + PERF_BR_FAULT_INST = 20, /* instruction fault */
>> + PERF_BR_SERROR = 21, /* system error */
>> PERF_BR_MAX,
>> };
>>
>> @@ -1370,8 +1376,8 @@ struct perf_branch_entry {
>> in_tx:1, /* in transaction */
>> abort:1, /* transaction abort */
>> cycles:16, /* cycle count to last branch */
>> - type:4, /* branch type */
>> - reserved:40;
>> + type:6, /* branch type */
>> + reserved:38;
>> };
> There's another copy of this struct in branch.h that is used to access the same data in
> perf which also needs updating:
>
> struct branch_flags {
> union {
> u64 value;
> struct {
> u64 mispred:1;
> u64 predicted:1;
> u64 in_tx:1;
> u64 abort:1;
> u64 cycles:16;
> u64 type:4;
> u64 reserved:40;
> };
> };
> };

Sure, thanks for the catch. Will fix it.

>
> It's never assigned directly but there is some casting stuff going on in
> evsel__parse_sample() and it eventually ends up being used to access branch
> records. Same applies to the privilege data change.
>

Okay, will do the necessary changes in the privilege data patch.

2022-02-04 09:28:42

by Anshuman Khandual

[permalink] [raw]
Subject: Re: [RFC V1 11/11] perf: Capture branch privilege information



On 1/25/22 9:09 PM, James Clark wrote:
>
>
> On 24/01/2022 04:30, Anshuman Khandual wrote:
>> Platforms like arm64 could capture privilege level information for all the
>> branch records. Hence this adds a new element in the struct branch_entry to
>> record the privilege level information, which could be requested through a
>> new event.attr.branch_sample_type flag PERF_SAMPLE_BRANCH_PRIV_SAVE. While
>> here, update the BRBE driver as required.
>>
>> Cc: Will Deacon <[email protected]>
>> Cc: Mark Rutland <[email protected]>
>> Cc: Peter Zijlstra <[email protected]>
>> Cc: Ingo Molnar <[email protected]>
>> Cc: Arnaldo Carvalho de Melo <[email protected]>
>> Cc: Jiri Olsa <[email protected]>
>> Cc: Namhyung Kim <[email protected]>
>> Cc: [email protected]
>> Cc: [email protected]
>> Cc: [email protected]
>> Signed-off-by: Anshuman Khandual <[email protected]>
>> ---
>> drivers/perf/arm_pmu_brbe.c | 28 ++++++++++++++++++++++++
>> include/linux/perf_event.h | 5 +++++
>> include/uapi/linux/perf_event.h | 13 ++++++++++-
>> tools/include/uapi/linux/perf_event.h | 13 ++++++++++-
>> tools/perf/Documentation/perf-record.txt | 1 +
>> tools/perf/util/parse-branch-options.c | 1 +
>> 6 files changed, 59 insertions(+), 2 deletions(-)
>>
> [...]
>> diff --git a/include/uapi/linux/perf_event.h b/include/uapi/linux/perf_event.h
>> index 361fdc6b87a0..4d77710f7a4e 100644
>> --- a/include/uapi/linux/perf_event.h
>> +++ b/include/uapi/linux/perf_event.h
>> @@ -204,6 +204,8 @@ enum perf_branch_sample_type_shift {
>>
>> PERF_SAMPLE_BRANCH_HW_INDEX_SHIFT = 17, /* save low level index of raw branch records */
>>
>> + PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT = 18, /* save privillege mode */
>
> privillege -> privilege

Fixed.

>
>> +
>> PERF_SAMPLE_BRANCH_MAX_SHIFT /* non-ABI */
>> };
>>
>> @@ -233,6 +235,8 @@ enum perf_branch_sample_type {
>>
>> PERF_SAMPLE_BRANCH_HW_INDEX = 1U << PERF_SAMPLE_BRANCH_HW_INDEX_SHIFT,
>>
>> + PERF_SAMPLE_BRANCH_PRIV_SAVE = 1U << PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT,
>> +
>
> Can you also add the new entry to the perf verbose printer otherwise it looks like
> it's not being set on branch_sample_type
> (as in when using ./perf record -j any_call,u,priv -vvv):

Sure, will do.

>
> static void __p_branch_sample_type(char *buf, size_t size, u64 value)
> {
> #define bit_name(n) { PERF_SAMPLE_BRANCH_##n, #n }
> struct bit_names bits[] = {
> bit_name(USER), bit_name(KERNEL), bit_name(HV), bit_name(ANY),
> bit_name(ANY_CALL), bit_name(ANY_RETURN), bit_name(IND_CALL),
> bit_name(ABORT_TX), bit_name(IN_TX), bit_name(NO_TX),
> bit_name(COND), bit_name(CALL_STACK), bit_name(IND_JUMP),
> bit_name(CALL), bit_name(NO_FLAGS), bit_name(NO_CYCLES),
> bit_name(HW_INDEX),
> { .name = NULL, }
> };
> #undef bit_name
> __p_bits(buf, size, value, bits);
> }
>
> PERF_SAMPLE_BRANCH_TYPE_SAVE_SHIFT is also missing so it probably makes sense to add it
> at the same time as that was expanded for BRBE.

Posted a patch for PERF_SAMPLE_BRANCH_TYPE_SAVE_SHIFT here but will update
for the new 'priv' request via this series.

https://lore.kernel.org/all/[email protected]/

2022-03-14 14:48:09

by Anshuman Khandual

[permalink] [raw]
Subject: Re: [RFC V1 11/11] perf: Capture branch privilege information



On 1/26/22 22:57, James Clark wrote:
>
> On 24/01/2022 04:30, Anshuman Khandual wrote:
>> Platforms like arm64 could capture privilege level information for all the
>> branch records. Hence this adds a new element in the struct branch_entry to
>> record the privilege level information, which could be requested through a
>> new event.attr.branch_sample_type flag PERF_SAMPLE_BRANCH_PRIV_SAVE. While
>> here, update the BRBE driver as required.
>>
>> Cc: Will Deacon <[email protected]>
>> Cc: Mark Rutland <[email protected]>
>> Cc: Peter Zijlstra <[email protected]>
>> Cc: Ingo Molnar <[email protected]>
>> Cc: Arnaldo Carvalho de Melo <[email protected]>
>> Cc: Jiri Olsa <[email protected]>
>> Cc: Namhyung Kim <[email protected]>
>> Cc: [email protected]
>> Cc: [email protected]
>> Cc: [email protected]
>> Signed-off-by: Anshuman Khandual <[email protected]>
>> ---
>> drivers/perf/arm_pmu_brbe.c | 28 ++++++++++++++++++++++++
>> include/linux/perf_event.h | 5 +++++
>> include/uapi/linux/perf_event.h | 13 ++++++++++-
>> tools/include/uapi/linux/perf_event.h | 13 ++++++++++-
>> tools/perf/Documentation/perf-record.txt | 1 +
>> tools/perf/util/parse-branch-options.c | 1 +
>> 6 files changed, 59 insertions(+), 2 deletions(-)
>>
>> diff --git a/drivers/perf/arm_pmu_brbe.c b/drivers/perf/arm_pmu_brbe.c
>> index 7cd1208c6c58..d4cbea74c148 100644
>> --- a/drivers/perf/arm_pmu_brbe.c
>> +++ b/drivers/perf/arm_pmu_brbe.c
>> @@ -270,6 +270,25 @@ static int brbe_fetch_perf_type(u64 brbinf)
>> }
>> }
>>
>> +static int brbe_fetch_perf_priv(u64 brbinf)
>> +{
>> + int brbe_el = brbe_fetch_el(brbinf);
>> +
>> + switch (brbe_el) {
>> + case BRBINF_EL_EL0:
>> + return PERF_BR_USER;
>> + case BRBINF_EL_EL1:
>> + return PERF_BR_KERNEL;
>> + case BRBINF_EL_EL2:
>> + if (is_kernel_in_hyp_mode())
>> + return PERF_BR_KERNEL;
>> + return PERF_BR_HV;
>> + default:
>> + pr_warn("unknown branch privilege captured\n");
>> + return -1;
>> + }
>> +}
>> +
>> static void capture_brbe_flags(struct pmu_hw_events *cpuc, struct perf_event *event,
>> u64 brbinf, int idx)
>> {
>> @@ -302,6 +321,15 @@ static void capture_brbe_flags(struct pmu_hw_events *cpuc, struct perf_event *ev
>> cpuc->brbe_entries[idx].in_tx = brbinf & BRBINF_TX;
>> }
>> }
>> +
>> + if (branch_sample_priv(event)) {
>> + /*
>> + * All these information (i.e branch privilege level) are not
>> + * available for source only branch records.
>> + */
>> + if (type != BRBINF_VALID_SOURCE)
>> + cpuc->brbe_entries[idx].priv = brbe_fetch_perf_priv(brbinf);
>> + }
>> }
>>
>> /*
>> diff --git a/include/linux/perf_event.h b/include/linux/perf_event.h
>> index 916ce5102b33..8021b6a30d86 100644
>> --- a/include/linux/perf_event.h
>> +++ b/include/linux/perf_event.h
>> @@ -1688,4 +1688,9 @@ static inline bool branch_sample_hw_index(const struct perf_event *event)
>> {
>> return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_HW_INDEX;
>> }
>> +
>> +static inline bool branch_sample_priv(const struct perf_event *event)
>> +{
>> + return event->attr.branch_sample_type & PERF_SAMPLE_BRANCH_PRIV_SAVE;
>> +}
>> #endif /* _LINUX_PERF_EVENT_H */
>> diff --git a/include/uapi/linux/perf_event.h b/include/uapi/linux/perf_event.h
>> index 361fdc6b87a0..4d77710f7a4e 100644
>> --- a/include/uapi/linux/perf_event.h
>> +++ b/include/uapi/linux/perf_event.h
>> @@ -204,6 +204,8 @@ enum perf_branch_sample_type_shift {
>>
>> PERF_SAMPLE_BRANCH_HW_INDEX_SHIFT = 17, /* save low level index of raw branch records */
>>
>> + PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT = 18, /* save privillege mode */
>> +
>> PERF_SAMPLE_BRANCH_MAX_SHIFT /* non-ABI */
>> };
>>
>> @@ -233,6 +235,8 @@ enum perf_branch_sample_type {
>>
>> PERF_SAMPLE_BRANCH_HW_INDEX = 1U << PERF_SAMPLE_BRANCH_HW_INDEX_SHIFT,
>>
>> + PERF_SAMPLE_BRANCH_PRIV_SAVE = 1U << PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT,
>> +
>> PERF_SAMPLE_BRANCH_MAX = 1U << PERF_SAMPLE_BRANCH_MAX_SHIFT,
>> };
>>
>> @@ -265,6 +269,12 @@ enum {
>> PERF_BR_MAX,
>> };
>>
>> +enum {
>> + PERF_BR_USER = 0,
>> + PERF_BR_KERNEL = 1,
>> + PERF_BR_HV = 2,
>> +};
>> +
> Can we have 0 as "UNKNOWN". It's going to be difficult to parse files when privilege information
> isn't saved and get accurate results without that. For example if it's not set then presumably
> the field would be 0 (PERF_BR_USER), but that doesn't mean the samples are user in that case.
>
> I know you might be able to go backwards and look at what arguments were passed to the kernel but
> it's not guaranteed that the kernel honored the request anyway. There are also other platforms
> to think about etc.
>
> If you look at the branch type definitions above they start at 0 (PERF_BR_UNKNOWN) which I think
> works out quite nicely in the userspace code.

This is being taken care in the new BRBE related perf ABI changes series (V3).

https://lore.kernel.org/all/[email protected]/