Metrics in the perf tool come in via json. Json doesn't allow
comments, line breaks, etc. making it an inconvenient way to write
metrics. Further, it is useful to detect when writing a metric that
the event specified is supported within the event json for a model.
These patches introduce infrastructure and fixes for the addition of
metrics written in python for Arm64, AMD Zen and Intel CPUs. Later
patches will introduce the metrics split apart by the vendor.
v3. Some code tidying, make the input directory a command line
argument, but no other functional or output changes.
v2. Fixes two type issues in the python code but no functional or
output changes.
Ian Rogers (12):
perf jevents: Allow multiple metricgroups.json files
perf jevents: Update metric constraint support
perf jevents: Add descriptions to metricgroup abstraction
perf jevents: Allow metric groups not to be named
perf jevents: Support parsing negative exponents
perf jevents: Term list fix in event parsing
perf jevents: Add threshold expressions to Metric
perf jevents: Move json encoding to its own functions
perf jevents: Drop duplicate pending metrics
perf jevents: Skip optional metrics in metric group list
perf jevents: Build support for generating metrics from python
perf jevents: Add load event json to verify and allow fallbacks
tools/perf/.gitignore | 2 +
tools/perf/Makefile.perf | 17 ++-
tools/perf/pmu-events/Build | 60 ++++++++-
tools/perf/pmu-events/amd_metrics.py | 42 +++++++
tools/perf/pmu-events/arm64_metrics.py | 43 +++++++
tools/perf/pmu-events/intel_metrics.py | 42 +++++++
tools/perf/pmu-events/jevents.py | 6 +-
tools/perf/pmu-events/metric.py | 162 +++++++++++++++++++++----
tools/perf/pmu-events/metric_test.py | 4 +
9 files changed, 342 insertions(+), 36 deletions(-)
create mode 100755 tools/perf/pmu-events/amd_metrics.py
create mode 100755 tools/perf/pmu-events/arm64_metrics.py
create mode 100755 tools/perf/pmu-events/intel_metrics.py
--
2.44.0.278.ge034bb2e1d-goog
Allow multiple metricgroups.json files by handling any file ending
with metricgroups.json as a metricgroups file.
Signed-off-by: Ian Rogers <[email protected]>
---
tools/perf/pmu-events/jevents.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/tools/perf/pmu-events/jevents.py b/tools/perf/pmu-events/jevents.py
index fc704a96ba0b..f68b0a60cdef 100755
--- a/tools/perf/pmu-events/jevents.py
+++ b/tools/perf/pmu-events/jevents.py
@@ -608,7 +608,7 @@ def preprocess_one_file(parents: Sequence[str], item: os.DirEntry) -> None:
if not item.is_file() or not item.name.endswith('.json'):
return
- if item.name == 'metricgroups.json':
+ if item.name.endswith('metricgroups.json'):
metricgroup_descriptions = json.load(open(item.path))
for mgroup in metricgroup_descriptions:
assert len(mgroup) > 1, parents
@@ -658,7 +658,7 @@ def process_one_file(parents: Sequence[str], item: os.DirEntry) -> None:
# Ignore other directories. If the file name does not have a .json
# extension, ignore it. It could be a readme.txt for instance.
- if not item.is_file() or not item.name.endswith('.json') or item.name == 'metricgroups.json':
+ if not item.is_file() or not item.name.endswith('.json') or item.name.endswith('metricgroups.json'):
return
add_events_table_entries(item, get_topic(item.name))
--
2.44.0.278.ge034bb2e1d-goog
Previous metric constraints were binary, either none or don't group
when the NMI watchdog is present. Update to match the definitions in
'enum metric_event_groups' in pmu-events.h.
Signed-off-by: Ian Rogers <[email protected]>
---
tools/perf/pmu-events/metric.py | 14 ++++++++++----
1 file changed, 10 insertions(+), 4 deletions(-)
diff --git a/tools/perf/pmu-events/metric.py b/tools/perf/pmu-events/metric.py
index 92acd89ed97a..8a718dd4b1fe 100644
--- a/tools/perf/pmu-events/metric.py
+++ b/tools/perf/pmu-events/metric.py
@@ -4,8 +4,14 @@ import ast
import decimal
import json
import re
+from enum import Enum
from typing import Dict, List, Optional, Set, Tuple, Union
+class MetricConstraint(Enum):
+ GROUPED_EVENTS = 0
+ NO_GROUP_EVENTS = 1
+ NO_GROUP_EVENTS_NMI = 2
+ NO_GROUP_EVENTS_SMT = 3
class Expression:
"""Abstract base class of elements in a metric expression."""
@@ -423,14 +429,14 @@ class Metric:
groups: Set[str]
expr: Expression
scale_unit: str
- constraint: bool
+ constraint: MetricConstraint
def __init__(self,
name: str,
description: str,
expr: Expression,
scale_unit: str,
- constraint: bool = False):
+ constraint: MetricConstraint = MetricConstraint.GROUPED_EVENTS):
self.name = name
self.description = description
self.expr = expr.Simplify()
@@ -464,8 +470,8 @@ class Metric:
'MetricExpr': self.expr.ToPerfJson(),
'ScaleUnit': self.scale_unit
}
- if self.constraint:
- result['MetricConstraint'] = 'NO_NMI_WATCHDOG'
+ if self.constraint != MetricConstraint.GROUPED_EVENTS:
+ result['MetricConstraint'] = self.constraint.name
return result
--
2.44.0.278.ge034bb2e1d-goog
It can be convenient to have unnamed metric groups for the sake of
organizing other metrics and metric groups. An unspecified name
shouldn't contribute to the MetricGroup json value, so don't record
it.
Signed-off-by: Ian Rogers <[email protected]>
---
tools/perf/pmu-events/metric.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/tools/perf/pmu-events/metric.py b/tools/perf/pmu-events/metric.py
index 1de4fb72c75e..847b614d40d5 100644
--- a/tools/perf/pmu-events/metric.py
+++ b/tools/perf/pmu-events/metric.py
@@ -455,7 +455,8 @@ class Metric:
def AddToMetricGroup(self, group):
"""Callback used when being added to a MetricGroup."""
- self.groups.add(group.name)
+ if group.name:
+ self.groups.add(group.name)
def Flatten(self) -> Set['Metric']:
"""Return a leaf metric."""
--
2.44.0.278.ge034bb2e1d-goog
Support negative exponents when parsing from a json metric string by
making the numbers after the 'e' optional in the 'Event' insertion fix
up.
Signed-off-by: Ian Rogers <[email protected]>
---
tools/perf/pmu-events/metric.py | 2 +-
tools/perf/pmu-events/metric_test.py | 4 ++++
2 files changed, 5 insertions(+), 1 deletion(-)
diff --git a/tools/perf/pmu-events/metric.py b/tools/perf/pmu-events/metric.py
index 847b614d40d5..31eea2f45152 100644
--- a/tools/perf/pmu-events/metric.py
+++ b/tools/perf/pmu-events/metric.py
@@ -573,7 +573,7 @@ def ParsePerfJson(orig: str) -> Expression:
# a double by the Bison parser
py = re.sub(r'0Event\(r"[xX]([0-9a-fA-F]*)"\)', r'Event("0x\1")', py)
# Convert accidentally converted scientific notation constants back
- py = re.sub(r'([0-9]+)Event\(r"(e[0-9]+)"\)', r'\1\2', py)
+ py = re.sub(r'([0-9]+)Event\(r"(e[0-9]*)"\)', r'\1\2', py)
# Convert all the known keywords back from events to just the keyword
keywords = ['if', 'else', 'min', 'max', 'd_ratio', 'source_count', 'has_event', 'strcmp_cpuid_str']
for kw in keywords:
diff --git a/tools/perf/pmu-events/metric_test.py b/tools/perf/pmu-events/metric_test.py
index ee22ff43ddd7..8acfe4652b55 100755
--- a/tools/perf/pmu-events/metric_test.py
+++ b/tools/perf/pmu-events/metric_test.py
@@ -61,6 +61,10 @@ class TestMetricExpressions(unittest.TestCase):
after = before
self.assertEqual(ParsePerfJson(before).ToPerfJson(), after)
+ before = r'a + 3e-12 + b'
+ after = before
+ self.assertEqual(ParsePerfJson(before).ToPerfJson(), after)
+
def test_IfElseTests(self):
# if-else needs rewriting to Select and back.
before = r'Event1 if #smt_on else Event2'
--
2.44.0.278.ge034bb2e1d-goog
Add a function to recursively generate metric group descriptions.
Signed-off-by: Ian Rogers <[email protected]>
---
tools/perf/pmu-events/metric.py | 14 ++++++++++++--
1 file changed, 12 insertions(+), 2 deletions(-)
diff --git a/tools/perf/pmu-events/metric.py b/tools/perf/pmu-events/metric.py
index 8a718dd4b1fe..1de4fb72c75e 100644
--- a/tools/perf/pmu-events/metric.py
+++ b/tools/perf/pmu-events/metric.py
@@ -475,6 +475,8 @@ class Metric:
return result
+ def ToMetricGroupDescriptions(self, root: bool = True) -> Dict[str, str]:
+ return {}
class _MetricJsonEncoder(json.JSONEncoder):
"""Special handling for Metric objects."""
@@ -493,10 +495,12 @@ class MetricGroup:
which can facilitate arrangements similar to trees.
"""
- def __init__(self, name: str, metric_list: List[Union[Metric,
- 'MetricGroup']]):
+ def __init__(self, name: str,
+ metric_list: List[Union[Metric, 'MetricGroup']],
+ description: Optional[str] = None):
self.name = name
self.metric_list = metric_list
+ self.description = description
for metric in metric_list:
metric.AddToMetricGroup(self)
@@ -516,6 +520,12 @@ class MetricGroup:
def ToPerfJson(self) -> str:
return json.dumps(sorted(self.Flatten()), indent=2, cls=_MetricJsonEncoder)
+ def ToMetricGroupDescriptions(self, root: bool = True) -> Dict[str, str]:
+ result = {self.name: self.description} if self.description else {}
+ for x in self.metric_list:
+ result.update(x.ToMetricGroupDescriptions(False))
+ return result
+
def __str__(self) -> str:
return self.ToPerfJson()
--
2.44.0.278.ge034bb2e1d-goog
Fix events seemingly broken apart at a comma.
Signed-off-by: Ian Rogers <[email protected]>
---
tools/perf/pmu-events/metric.py | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/tools/perf/pmu-events/metric.py b/tools/perf/pmu-events/metric.py
index 31eea2f45152..0f4e67e5cfea 100644
--- a/tools/perf/pmu-events/metric.py
+++ b/tools/perf/pmu-events/metric.py
@@ -568,6 +568,12 @@ def ParsePerfJson(orig: str) -> Expression:
r'Event(r"\1")', py)
# If it started with a # it should have been a literal, rather than an event name
py = re.sub(r'#Event\(r"([^"]*)"\)', r'Literal("#\1")', py)
+ # Fix events wrongly broken at a ','
+ while True:
+ prev_py = py
+ py = re.sub(r'Event\(r"([^"]*)"\),Event\(r"([^"]*)"\)', r'Event(r"\1,\2")', py)
+ if py == prev_py:
+ break
# Convert accidentally converted hex constants ("0Event(r"xDEADBEEF)"") back to a constant,
# but keep it wrapped in Event(), otherwise Python drops the 0x prefix and it gets interpreted as
# a double by the Bison parser
@@ -586,7 +592,6 @@ def ParsePerfJson(orig: str) -> Expression:
parsed = ast.fix_missing_locations(parsed)
return _Constify(eval(compile(parsed, orig, 'eval')))
-
def RewriteMetricsInTermsOfOthers(metrics: List[Tuple[str, str, Expression]]
)-> Dict[Tuple[str, str], Expression]:
"""Shorten metrics by rewriting in terms of others.
--
2.44.0.278.ge034bb2e1d-goog
Allow threshold expressions for metrics to be generated.
Signed-off-by: Ian Rogers <[email protected]>
---
tools/perf/pmu-events/metric.py | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/tools/perf/pmu-events/metric.py b/tools/perf/pmu-events/metric.py
index 0f4e67e5cfea..e81fed2e29b5 100644
--- a/tools/perf/pmu-events/metric.py
+++ b/tools/perf/pmu-events/metric.py
@@ -430,13 +430,15 @@ class Metric:
expr: Expression
scale_unit: str
constraint: MetricConstraint
+ threshold: Optional[Expression]
def __init__(self,
name: str,
description: str,
expr: Expression,
scale_unit: str,
- constraint: MetricConstraint = MetricConstraint.GROUPED_EVENTS):
+ constraint: MetricConstraint = MetricConstraint.GROUPED_EVENTS,
+ threshold: Optional[Expression] = None):
self.name = name
self.description = description
self.expr = expr.Simplify()
@@ -447,6 +449,7 @@ class Metric:
else:
self.scale_unit = f'1{scale_unit}'
self.constraint = constraint
+ self.threshold = threshold
self.groups = set()
def __lt__(self, other):
@@ -473,6 +476,8 @@ class Metric:
}
if self.constraint != MetricConstraint.GROUPED_EVENTS:
result['MetricConstraint'] = self.constraint.name
+ if self.threshold:
+ result['MetricThreshold'] = self.threshold.ToPerfJson()
return result
--
2.44.0.278.ge034bb2e1d-goog
Have dedicate encode functions rather than having them embedded in
MetricGroup. This is to provide some uniformity in the Metric ToXXX
routines.
Signed-off-by: Ian Rogers <[email protected]>
---
tools/perf/pmu-events/metric.py | 34 +++++++++++++++++++++------------
1 file changed, 22 insertions(+), 12 deletions(-)
diff --git a/tools/perf/pmu-events/metric.py b/tools/perf/pmu-events/metric.py
index e81fed2e29b5..b39189182608 100644
--- a/tools/perf/pmu-events/metric.py
+++ b/tools/perf/pmu-events/metric.py
@@ -484,15 +484,6 @@ class Metric:
def ToMetricGroupDescriptions(self, root: bool = True) -> Dict[str, str]:
return {}
-class _MetricJsonEncoder(json.JSONEncoder):
- """Special handling for Metric objects."""
-
- def default(self, o):
- if isinstance(o, Metric):
- return o.ToPerfJson()
- return json.JSONEncoder.default(self, o)
-
-
class MetricGroup:
"""A group of metrics.
@@ -523,8 +514,11 @@ class MetricGroup:
return result
- def ToPerfJson(self) -> str:
- return json.dumps(sorted(self.Flatten()), indent=2, cls=_MetricJsonEncoder)
+ def ToPerfJson(self) -> List[Dict[str, str]]:
+ result = []
+ for x in sorted(self.Flatten()):
+ result.append(x.ToPerfJson())
+ return result
def ToMetricGroupDescriptions(self, root: bool = True) -> Dict[str, str]:
result = {self.name: self.description} if self.description else {}
@@ -533,7 +527,23 @@ class MetricGroup:
return result
def __str__(self) -> str:
- return self.ToPerfJson()
+ return str(self.ToPerfJson())
+
+
+def JsonEncodeMetric(x: MetricGroup):
+ class MetricJsonEncoder(json.JSONEncoder):
+ """Special handling for Metric objects."""
+
+ def default(self, o):
+ if isinstance(o, Metric) or isinstance(o, MetricGroup):
+ return o.ToPerfJson()
+ return json.JSONEncoder.default(self, o)
+
+ return json.dumps(x, indent=2, cls=MetricJsonEncoder)
+
+
+def JsonEncodeMetricGroupDescriptions(x: MetricGroup):
+ return json.dumps(x.ToMetricGroupDescriptions(), indent=2)
class _RewriteIfExpToSelect(ast.NodeTransformer):
--
2.44.0.278.ge034bb2e1d-goog
Drop adding a pending metric if there is an existing one.
Signed-off-by: Ian Rogers <[email protected]>
---
tools/perf/pmu-events/jevents.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tools/perf/pmu-events/jevents.py b/tools/perf/pmu-events/jevents.py
index f68b0a60cdef..23f9809b3850 100755
--- a/tools/perf/pmu-events/jevents.py
+++ b/tools/perf/pmu-events/jevents.py
@@ -473,7 +473,7 @@ def add_events_table_entries(item: os.DirEntry, topic: str) -> None:
for e in read_json_events(item.path, topic):
if e.name:
_pending_events.append(e)
- if e.metric_name:
+ if e.metric_name and not any(e.metric_name == x.metric_name for x in _pending_metrics):
_pending_metrics.append(e)
--
2.44.0.278.ge034bb2e1d-goog
For metric groups, skip metrics in the list that are None. This allows
functions to better optionally return metrics.
Signed-off-by: Ian Rogers <[email protected]>
---
tools/perf/pmu-events/metric.py | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/tools/perf/pmu-events/metric.py b/tools/perf/pmu-events/metric.py
index b39189182608..dd8fd06940e6 100644
--- a/tools/perf/pmu-events/metric.py
+++ b/tools/perf/pmu-events/metric.py
@@ -493,13 +493,15 @@ class MetricGroup:
"""
def __init__(self, name: str,
- metric_list: List[Union[Metric, 'MetricGroup']],
+ metric_list: List[Union[Optional[Metric], Optional['MetricGroup']]],
description: Optional[str] = None):
self.name = name
- self.metric_list = metric_list
+ self.metric_list = []
self.description = description
for metric in metric_list:
- metric.AddToMetricGroup(self)
+ if metric:
+ self.metric_list.append(metric)
+ metric.AddToMetricGroup(self)
def AddToMetricGroup(self, group):
"""Callback used when a MetricGroup is added into another."""
--
2.44.0.278.ge034bb2e1d-goog
Generate extra-metrics.json and extra-metricgroups.json from python
architecture specific scripts. The metrics themselves will be added in
later patches.
If a build takes place in tools/perf/ then extra-metrics.json and
extra-metricgroups.json are generated in that directory and so added
to .gitignore. If there is an OUTPUT directory then the
tools/perf/pmu-events/arch files are copied to it so the generated
extra-metrics.json and extra-metricgroups.json can be added/generated
there.
Signed-off-by: Ian Rogers <[email protected]>
---
tools/perf/.gitignore | 2 +
tools/perf/Makefile.perf | 17 ++++++--
tools/perf/pmu-events/Build | 60 ++++++++++++++++++++++++--
tools/perf/pmu-events/amd_metrics.py | 38 ++++++++++++++++
tools/perf/pmu-events/arm64_metrics.py | 39 +++++++++++++++++
tools/perf/pmu-events/intel_metrics.py | 38 ++++++++++++++++
6 files changed, 187 insertions(+), 7 deletions(-)
create mode 100755 tools/perf/pmu-events/amd_metrics.py
create mode 100755 tools/perf/pmu-events/arm64_metrics.py
create mode 100755 tools/perf/pmu-events/intel_metrics.py
diff --git a/tools/perf/.gitignore b/tools/perf/.gitignore
index f5b81d439387..c9a8da5bfc56 100644
--- a/tools/perf/.gitignore
+++ b/tools/perf/.gitignore
@@ -39,6 +39,8 @@ trace/beauty/generated/
pmu-events/pmu-events.c
pmu-events/jevents
pmu-events/metric_test.log
+pmu-events/arch/**/extra-metrics.json
+pmu-events/arch/**/extra-metricgroups.json
tests/shell/*.shellcheck_log
tests/shell/coresight/*.shellcheck_log
tests/shell/lib/*.shellcheck_log
diff --git a/tools/perf/Makefile.perf b/tools/perf/Makefile.perf
index f8774a9b1377..e8e70d02ee92 100644
--- a/tools/perf/Makefile.perf
+++ b/tools/perf/Makefile.perf
@@ -1169,7 +1169,20 @@ endif # CONFIG_PERF_BPF_SKEL
bpf-skel-clean:
$(call QUIET_CLEAN, bpf-skel) $(RM) -r $(SKEL_TMP_OUT) $(SKELETONS)
-clean:: $(LIBAPI)-clean $(LIBBPF)-clean $(LIBSUBCMD)-clean $(LIBSYMBOL)-clean $(LIBPERF)-clean arm64-sysreg-defs-clean fixdep-clean python-clean bpf-skel-clean tests-coresight-targets-clean
+pmu-events-clean:
+ifeq ($(OUTPUT),)
+ $(call QUIET_CLEAN, pmu-events) $(RM) \
+ pmu-events/pmu-events.c \
+ pmu-events/metric_test.log
+ $(Q)find pmu-events/arch -name 'extra-metrics.json' -delete -o \
+ -name 'extra-metricgroups.json' -delete
+else
+ $(call QUIET_CLEAN, pmu-events) $(RM) -r $(OUTPUT)pmu-events/arch \
+ $(OUTPUT)pmu-events/pmu-events.c \
+ $(OUTPUT)pmu-events/metric_test.log
+endif
+
+clean:: $(LIBAPI)-clean $(LIBBPF)-clean $(LIBSUBCMD)-clean $(LIBSYMBOL)-clean $(LIBPERF)-clean arm64-sysreg-defs-clean fixdep-clean python-clean bpf-skel-clean tests-coresight-targets-clean pmu-events-clean
$(call QUIET_CLEAN, core-objs) $(RM) $(LIBPERF_A) $(OUTPUT)perf-archive $(OUTPUT)perf-iostat $(LANG_BINDINGS)
$(Q)find $(or $(OUTPUT),.) -name '*.o' -delete -o -name '\.*.cmd' -delete -o -name '\.*.d' -delete -o -name '*.shellcheck_log' -delete
$(Q)$(RM) $(OUTPUT).config-detected
@@ -1177,8 +1190,6 @@ clean:: $(LIBAPI)-clean $(LIBBPF)-clean $(LIBSUBCMD)-clean $(LIBSYMBOL)-clean $(
$(call QUIET_CLEAN, core-gen) $(RM) *.spec *.pyc *.pyo */*.pyc */*.pyo $(OUTPUT)common-cmds.h TAGS tags cscope* $(OUTPUT)PERF-VERSION-FILE $(OUTPUT)FEATURE-DUMP $(OUTPUT)util/*-bison* $(OUTPUT)util/*-flex* \
$(OUTPUT)util/intel-pt-decoder/inat-tables.c \
$(OUTPUT)tests/llvm-src-{base,kbuild,prologue,relocation}.c \
- $(OUTPUT)pmu-events/pmu-events.c \
- $(OUTPUT)pmu-events/metric_test.log \
$(OUTPUT)$(fadvise_advice_array) \
$(OUTPUT)$(fsconfig_arrays) \
$(OUTPUT)$(fsmount_arrays) \
diff --git a/tools/perf/pmu-events/Build b/tools/perf/pmu-events/Build
index 1d18bb89402e..e2db33577707 100644
--- a/tools/perf/pmu-events/Build
+++ b/tools/perf/pmu-events/Build
@@ -1,7 +1,6 @@
pmu-events-y += pmu-events.o
JDIR = pmu-events/arch/$(SRCARCH)
-JSON = $(shell [ -d $(JDIR) ] && \
- find $(JDIR) -name '*.json' -o -name 'mapfile.csv')
+JSON = $(shell find pmu-events/arch -name *.json -o -name *.csv)
JDIR_TEST = pmu-events/arch/test
JSON_TEST = $(shell [ -d $(JDIR_TEST) ] && \
find $(JDIR_TEST) -name '*.json')
@@ -27,13 +26,66 @@ $(PMU_EVENTS_C): $(EMPTY_PMU_EVENTS_C)
$(call rule_mkdir)
$(Q)$(call echo-cmd,gen)cp $< $@
else
+# Extract the model from a extra-metrics.json or extra-metricgroups.json path
+model_name = $(shell echo $(1)|sed -e 's@.\+/\(.*\)/extra-metric.*\.json@\1@')
+vendor_name = $(shell echo $(1)|sed -e 's@.\+/\(.*\)/[^/]*/extra-metric.*\.json@\1@')
+
+# Copy checked-in json for generation.
+$(OUTPUT)pmu-events/arch/%: pmu-events/arch/%
+ $(call rule_mkdir)
+ $(Q)$(call echo-cmd,gen)cp $< $@
+
+# Generate AMD Json
+ZENS = $(shell ls -d pmu-events/arch/x86/amdzen*)
+ZEN_METRICS = $(foreach x,$(ZENS),$(OUTPUT)$(x)/extra-metrics.json)
+ZEN_METRICGROUPS = $(foreach x,$(ZENS),$(OUTPUT)$(x)/extra-metricgroups.json)
+
+$(ZEN_METRICS): pmu-events/amd_metrics.py
+ $(call rule_mkdir)
+ $(Q)$(call echo-cmd,gen)$(PYTHON) $< $(call model_name,$@) arch > $@
+
+$(ZEN_METRICGROUPS): pmu-events/amd_metrics.py
+ $(call rule_mkdir)
+ $(Q)$(call echo-cmd,gen)$(PYTHON) $< -metricgroups $(call model_name,$@) arch > $@
+
+# Generate ARM Json
+ARMS = $(shell ls -d pmu-events/arch/arm64/arm/*)
+ARM_METRICS = $(foreach x,$(ARMS),$(OUTPUT)$(x)/extra-metrics.json)
+ARM_METRICGROUPS = $(foreach x,$(ARMS),$(OUTPUT)$(x)/extra-metricgroups.json)
+
+$(ARM_METRICS): pmu-events/arm64_metrics.py
+ $(call rule_mkdir)
+ $(Q)$(call echo-cmd,gen)$(PYTHON) $< $(call vendor_name,$@) $(call model_name,$@) arch > $@
+
+$(ARM_METRICGROUPS): pmu-events/arm64_metrics.py
+ $(call rule_mkdir)
+ $(Q)$(call echo-cmd,gen)$(PYTHON) $< -metricgroups $(call vendor_name,$@) $(call model_name,$@) arch > $@
+
+# Generate Intel Json
+INTELS = $(shell ls -d pmu-events/arch/x86/*|grep -v amdzen|grep -v mapfile.csv)
+INTEL_METRICS = $(foreach x,$(INTELS),$(OUTPUT)$(x)/extra-metrics.json)
+INTEL_METRICGROUPS = $(foreach x,$(INTELS),$(OUTPUT)$(x)/extra-metricgroups.json)
+
+$(INTEL_METRICS): pmu-events/intel_metrics.py
+ $(call rule_mkdir)
+ $(Q)$(call echo-cmd,gen)$(PYTHON) $< $(call model_name,$@) arch > $@
+
+$(INTEL_METRICGROUPS): pmu-events/intel_metrics.py
+ $(call rule_mkdir)
+ $(Q)$(call echo-cmd,gen)$(PYTHON) $< -metricgroups $(call model_name,$@) arch > $@
+
+GEN_JSON = $(patsubst %,$(OUTPUT)%,$(JSON)) \
+ $(ZEN_METRICS) $(ZEN_METRICGROUPS) \
+ $(ARM_METRICS) $(ARM_METRICGROUPS) \
+ $(INTEL_METRICS) $(INTEL_METRICGROUPS)
+
$(METRIC_TEST_LOG): $(METRIC_TEST_PY) $(METRIC_PY)
$(call rule_mkdir)
$(Q)$(call echo-cmd,test)$(PYTHON) $< 2> $@ || (cat $@ && false)
-$(PMU_EVENTS_C): $(JSON) $(JSON_TEST) $(JEVENTS_PY) $(METRIC_PY) $(METRIC_TEST_LOG)
+$(PMU_EVENTS_C): $(GEN_JSON) $(JSON_TEST) $(JEVENTS_PY) $(METRIC_PY) $(METRIC_TEST_LOG)
$(call rule_mkdir)
- $(Q)$(call echo-cmd,gen)$(PYTHON) $(JEVENTS_PY) $(JEVENTS_ARCH) $(JEVENTS_MODEL) pmu-events/arch $@
+ $(Q)$(call echo-cmd,gen)$(PYTHON) $(JEVENTS_PY) $(JEVENTS_ARCH) $(JEVENTS_MODEL) $(OUTPUT)pmu-events/arch $@
endif
# pmu-events.c file is generated in the OUTPUT directory so it needs a
diff --git a/tools/perf/pmu-events/amd_metrics.py b/tools/perf/pmu-events/amd_metrics.py
new file mode 100755
index 000000000000..7ab2ee4fdb17
--- /dev/null
+++ b/tools/perf/pmu-events/amd_metrics.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
+from metric import (JsonEncodeMetric, JsonEncodeMetricGroupDescriptions, MetricGroup)
+import argparse
+import json
+import os
+
+# Global command line arguments.
+_args = None
+
+def main() -> None:
+ global _args
+
+ def dir_path(path: str) -> str:
+ """Validate path is a directory for argparse."""
+ if os.path.isdir(path):
+ return path
+ raise argparse.ArgumentTypeError(f'\'{path}\' is not a valid directory')
+
+ parser = argparse.ArgumentParser(description="AMD perf json generator")
+ parser.add_argument("-metricgroups", help="Generate metricgroups data", action='store_true')
+ parser.add_argument("model", help="e.g. amdzen[123]")
+ parser.add_argument(
+ 'events_path',
+ type=dir_path,
+ help='Root of tree containing architecture directories containing json files'
+ )
+ _args = parser.parse_args()
+
+ all_metrics = MetricGroup("",[])
+
+ if _args.metricgroups:
+ print(JsonEncodeMetricGroupDescriptions(all_metrics))
+ else:
+ print(JsonEncodeMetric(all_metrics))
+
+if __name__ == '__main__':
+ main()
diff --git a/tools/perf/pmu-events/arm64_metrics.py b/tools/perf/pmu-events/arm64_metrics.py
new file mode 100755
index 000000000000..a9f0e6bc751b
--- /dev/null
+++ b/tools/perf/pmu-events/arm64_metrics.py
@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
+from metric import (JsonEncodeMetric, JsonEncodeMetricGroupDescriptions, MetricGroup)
+import argparse
+import json
+import os
+
+# Global command line arguments.
+_args = None
+
+def main() -> None:
+ global _args
+
+ def dir_path(path: str) -> str:
+ """Validate path is a directory for argparse."""
+ if os.path.isdir(path):
+ return path
+ raise argparse.ArgumentTypeError(f'\'{path}\' is not a valid directory')
+
+ parser = argparse.ArgumentParser(description="ARM perf json generator")
+ parser.add_argument("-metricgroups", help="Generate metricgroups data", action='store_true')
+ parser.add_argument("vendor", help="e.g. arm")
+ parser.add_argument("model", help="e.g. neoverse-n1")
+ parser.add_argument(
+ 'events_path',
+ type=dir_path,
+ help='Root of tree containing architecture directories containing json files'
+ )
+ _args = parser.parse_args()
+
+ all_metrics = MetricGroup("",[])
+
+ if _args.metricgroups:
+ print(JsonEncodeMetricGroupDescriptions(all_metrics))
+ else:
+ print(JsonEncodeMetric(all_metrics))
+
+if __name__ == '__main__':
+ main()
diff --git a/tools/perf/pmu-events/intel_metrics.py b/tools/perf/pmu-events/intel_metrics.py
new file mode 100755
index 000000000000..f004c27640d2
--- /dev/null
+++ b/tools/perf/pmu-events/intel_metrics.py
@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
+from metric import (JsonEncodeMetric, JsonEncodeMetricGroupDescriptions, MetricGroup)
+import argparse
+import json
+import os
+
+# Global command line arguments.
+_args = None
+
+def main() -> None:
+ global _args
+
+ def dir_path(path: str) -> str:
+ """Validate path is a directory for argparse."""
+ if os.path.isdir(path):
+ return path
+ raise argparse.ArgumentTypeError(f'\'{path}\' is not a valid directory')
+
+ parser = argparse.ArgumentParser(description="Intel perf json generator")
+ parser.add_argument("-metricgroups", help="Generate metricgroups data", action='store_true')
+ parser.add_argument("model", help="e.g. skylakex")
+ parser.add_argument(
+ 'events_path',
+ type=dir_path,
+ help='Root of tree containing architecture directories containing json files'
+ )
+ _args = parser.parse_args()
+
+ all_metrics = MetricGroup("",[])
+
+ if _args.metricgroups:
+ print(JsonEncodeMetricGroupDescriptions(all_metrics))
+ else:
+ print(JsonEncodeMetric(all_metrics))
+
+if __name__ == '__main__':
+ main()
--
2.44.0.278.ge034bb2e1d-goog
Add a LoadEvents function that loads all event json files in a
directory. In the Event constructor ensure all events are defined in
the event json except for legacy events like "cycles". If the initial
event isn't found then legacy_event1 is used, and if that isn't found
legacy_event2 is used. This allows a single Event to have multiple
event names as models will often rename the same event over time. If
the event doesn't exist an exception is raised.
So that references to metrics can be added, add the MetricRef
class. This doesn't validate as an event name and so provides an
escape hatch for metrics to refer to each other.
Signed-off-by: Ian Rogers <[email protected]>
---
tools/perf/pmu-events/Build | 12 ++--
tools/perf/pmu-events/amd_metrics.py | 6 +-
tools/perf/pmu-events/arm64_metrics.py | 6 +-
tools/perf/pmu-events/intel_metrics.py | 6 +-
tools/perf/pmu-events/metric.py | 77 +++++++++++++++++++++++++-
5 files changed, 95 insertions(+), 12 deletions(-)
diff --git a/tools/perf/pmu-events/Build b/tools/perf/pmu-events/Build
index e2db33577707..a2c5c04e5c46 100644
--- a/tools/perf/pmu-events/Build
+++ b/tools/perf/pmu-events/Build
@@ -42,11 +42,11 @@ ZEN_METRICGROUPS = $(foreach x,$(ZENS),$(OUTPUT)$(x)/extra-metricgroups.json)
$(ZEN_METRICS): pmu-events/amd_metrics.py
$(call rule_mkdir)
- $(Q)$(call echo-cmd,gen)$(PYTHON) $< $(call model_name,$@) arch > $@
+ $(Q)$(call echo-cmd,gen)$(PYTHON) $< $(call model_name,$@) pmu-events/arch > $@
$(ZEN_METRICGROUPS): pmu-events/amd_metrics.py
$(call rule_mkdir)
- $(Q)$(call echo-cmd,gen)$(PYTHON) $< -metricgroups $(call model_name,$@) arch > $@
+ $(Q)$(call echo-cmd,gen)$(PYTHON) $< -metricgroups $(call model_name,$@) pmu-events/arch > $@
# Generate ARM Json
ARMS = $(shell ls -d pmu-events/arch/arm64/arm/*)
@@ -55,11 +55,11 @@ ARM_METRICGROUPS = $(foreach x,$(ARMS),$(OUTPUT)$(x)/extra-metricgroups.json)
$(ARM_METRICS): pmu-events/arm64_metrics.py
$(call rule_mkdir)
- $(Q)$(call echo-cmd,gen)$(PYTHON) $< $(call vendor_name,$@) $(call model_name,$@) arch > $@
+ $(Q)$(call echo-cmd,gen)$(PYTHON) $< $(call vendor_name,$@) $(call model_name,$@) pmu-events/arch > $@
$(ARM_METRICGROUPS): pmu-events/arm64_metrics.py
$(call rule_mkdir)
- $(Q)$(call echo-cmd,gen)$(PYTHON) $< -metricgroups $(call vendor_name,$@) $(call model_name,$@) arch > $@
+ $(Q)$(call echo-cmd,gen)$(PYTHON) $< -metricgroups $(call vendor_name,$@) $(call model_name,$@) pmu-events/arch > $@
# Generate Intel Json
INTELS = $(shell ls -d pmu-events/arch/x86/*|grep -v amdzen|grep -v mapfile.csv)
@@ -68,11 +68,11 @@ INTEL_METRICGROUPS = $(foreach x,$(INTELS),$(OUTPUT)$(x)/extra-metricgroups.json
$(INTEL_METRICS): pmu-events/intel_metrics.py
$(call rule_mkdir)
- $(Q)$(call echo-cmd,gen)$(PYTHON) $< $(call model_name,$@) arch > $@
+ $(Q)$(call echo-cmd,gen)$(PYTHON) $< $(call model_name,$@) pmu-events/arch > $@
$(INTEL_METRICGROUPS): pmu-events/intel_metrics.py
$(call rule_mkdir)
- $(Q)$(call echo-cmd,gen)$(PYTHON) $< -metricgroups $(call model_name,$@) arch > $@
+ $(Q)$(call echo-cmd,gen)$(PYTHON) $< -metricgroups $(call model_name,$@) pmu-events/arch > $@
GEN_JSON = $(patsubst %,$(OUTPUT)%,$(JSON)) \
$(ZEN_METRICS) $(ZEN_METRICGROUPS) \
diff --git a/tools/perf/pmu-events/amd_metrics.py b/tools/perf/pmu-events/amd_metrics.py
index 7ab2ee4fdb17..4f728e7aae4a 100755
--- a/tools/perf/pmu-events/amd_metrics.py
+++ b/tools/perf/pmu-events/amd_metrics.py
@@ -1,6 +1,7 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
-from metric import (JsonEncodeMetric, JsonEncodeMetricGroupDescriptions, MetricGroup)
+from metric import (JsonEncodeMetric, JsonEncodeMetricGroupDescriptions, LoadEvents,
+ MetricGroup)
import argparse
import json
import os
@@ -27,6 +28,9 @@ def main() -> None:
)
_args = parser.parse_args()
+ directory = f"{_args.events_path}/x86/{_args.model}/"
+ LoadEvents(directory)
+
all_metrics = MetricGroup("",[])
if _args.metricgroups:
diff --git a/tools/perf/pmu-events/arm64_metrics.py b/tools/perf/pmu-events/arm64_metrics.py
index a9f0e6bc751b..c9aa2d827a82 100755
--- a/tools/perf/pmu-events/arm64_metrics.py
+++ b/tools/perf/pmu-events/arm64_metrics.py
@@ -1,6 +1,7 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
-from metric import (JsonEncodeMetric, JsonEncodeMetricGroupDescriptions, MetricGroup)
+from metric import (JsonEncodeMetric, JsonEncodeMetricGroupDescriptions, LoadEvents,
+ MetricGroup)
import argparse
import json
import os
@@ -30,6 +31,9 @@ def main() -> None:
all_metrics = MetricGroup("",[])
+ directory = f"{_args.events_path}/arm64/{_args.vendor}/{_args.model}/"
+ LoadEvents(directory)
+
if _args.metricgroups:
print(JsonEncodeMetricGroupDescriptions(all_metrics))
else:
diff --git a/tools/perf/pmu-events/intel_metrics.py b/tools/perf/pmu-events/intel_metrics.py
index f004c27640d2..04a19d05c6c1 100755
--- a/tools/perf/pmu-events/intel_metrics.py
+++ b/tools/perf/pmu-events/intel_metrics.py
@@ -1,6 +1,7 @@
#!/usr/bin/env python3
# SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
-from metric import (JsonEncodeMetric, JsonEncodeMetricGroupDescriptions, MetricGroup)
+from metric import (JsonEncodeMetric, JsonEncodeMetricGroupDescriptions, LoadEvents,
+ MetricGroup)
import argparse
import json
import os
@@ -27,6 +28,9 @@ def main() -> None:
)
_args = parser.parse_args()
+ directory = f"{_args.events_path}/x86/{_args.model}/"
+ LoadEvents(directory)
+
all_metrics = MetricGroup("",[])
if _args.metricgroups:
diff --git a/tools/perf/pmu-events/metric.py b/tools/perf/pmu-events/metric.py
index dd8fd06940e6..03312cd6d491 100644
--- a/tools/perf/pmu-events/metric.py
+++ b/tools/perf/pmu-events/metric.py
@@ -3,10 +3,50 @@
import ast
import decimal
import json
+import os
import re
from enum import Enum
from typing import Dict, List, Optional, Set, Tuple, Union
+all_events = set()
+
+def LoadEvents(directory: str) -> None:
+ """Populate a global set of all known events for the purpose of validating Event names"""
+ global all_events
+ all_events = {
+ "context\-switches",
+ "cycles",
+ "duration_time",
+ "instructions",
+ "l2_itlb_misses",
+ }
+ for file in os.listdir(os.fsencode(directory)):
+ filename = os.fsdecode(file)
+ if filename.endswith(".json"):
+ for x in json.load(open(f"{directory}/{filename}")):
+ if "EventName" in x:
+ all_events.add(x["EventName"])
+ elif "ArchStdEvent" in x:
+ all_events.add(x["ArchStdEvent"])
+
+
+def CheckEvent(name: str) -> bool:
+ """Check the event name exists in the set of all loaded events"""
+ global all_events
+ if len(all_events) == 0:
+ # No events loaded so assume any event is good.
+ return True
+
+ if ':' in name:
+ # Remove trailing modifier.
+ name = name[:name.find(':')]
+ elif '/' in name:
+ # Name could begin with a PMU or an event, for now assume it is good.
+ return True
+
+ return name in all_events
+
+
class MetricConstraint(Enum):
GROUPED_EVENTS = 0
NO_GROUP_EVENTS = 1
@@ -317,9 +357,18 @@ def _FixEscapes(s: str) -> str:
class Event(Expression):
"""An event in an expression."""
- def __init__(self, name: str, legacy_name: str = ''):
- self.name = _FixEscapes(name)
- self.legacy_name = _FixEscapes(legacy_name)
+ def __init__(self, *args: str):
+ error = ""
+ for name in args:
+ if CheckEvent(name):
+ self.name = _FixEscapes(name)
+ return
+ if error:
+ error += " or " + name
+ else:
+ error = name
+ global all_events
+ raise Exception(f"No event {error} in:\n{all_events}")
def ToPerfJson(self):
result = re.sub('/', '@', self.name)
@@ -338,6 +387,28 @@ class Event(Expression):
return self
+class MetricRef(Expression):
+ """A metric reference in an expression."""
+
+ def __init__(self, name: str):
+ self.name = _FixEscapes(name)
+
+ def ToPerfJson(self):
+ return self.name
+
+ def ToPython(self):
+ return f'MetricRef(r"{self.name}")'
+
+ def Simplify(self) -> Expression:
+ return self
+
+ def Equals(self, other: Expression) -> bool:
+ return isinstance(other, MetricRef) and self.name == other.name
+
+ def Substitute(self, name: str, expression: Expression) -> Expression:
+ return self
+
+
class Constant(Expression):
"""A constant within the expression tree."""
--
2.44.0.278.ge034bb2e1d-goog
Hello,
kernel test robot noticed "perf-test.perf.make.fail" on:
commit: d3ed44a7bee70b097709fe06f339c468489f16c9 ("[PATCH v3 12/12] perf jevents: Add load event json to verify and allow fallbacks")
url: https://github.com/intel-lab-lkp/linux/commits/Ian-Rogers/perf-jevents-Allow-multiple-metricgroups-json-files/20240314-135507
base: v6.8
patch link: https://lore.kernel.org/all/[email protected]/
patch subject: [PATCH v3 12/12] perf jevents: Add load event json to verify and allow fallbacks
in testcase: perf-test
version: perf-test-x86_64-git-1_20240223
with following parameters:
type: lkp
group: group-01
compiler: gcc-12
test machine: 224 threads 2 sockets Intel(R) Xeon(R) Platinum 8480+ (Sapphire Rapids) with 256G memory
(please refer to attached dmesg/kmsg for entire log/backtrace)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-lkp/[email protected]
please check attached stderr, more materials are in link [1]
Traceback (most recent call last):
File "/usr/src/perf_selftests-x86_64-rhel-8.3-bpf-d3ed44a7bee70b097709fe06f339c468489f16c9/tools/perf/pmu-events/amd_metrics.py", line 42, in <module>
main()
File "/usr/src/perf_selftests-x86_64-rhel-8.3-bpf-d3ed44a7bee70b097709fe06f339c468489f16c9/tools/perf/pmu-events/amd_metrics.py", line 32, in main
LoadEvents(directory)
File "/usr/src/perf_selftests-x86_64-rhel-8.3-bpf-d3ed44a7bee70b097709fe06f339c468489f16c9/tools/perf/pmu-events/metric.py", line 26, in LoadEvents
for x in json.load(open(f"{directory}/{filename}")):
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.11/json/__init__.py", line 293, in load
return loads(fp.read(),
^^^^^^^^^^^^^^^^
File "/usr/lib/python3.11/json/__init__.py", line 346, in loads
return _default_decoder.decode(s)
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.11/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.11/json/decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
make[3]: *** [pmu-events/Build:45: pmu-events/arch/x86/amdzen1/extra-metrics.json] Error 1
make[3]: *** Deleting file 'pmu-events/arch/x86/amdzen1/extra-metrics.json'
make[3]: *** Waiting for unfinished jobs....
Traceback (most recent call last):
File "/usr/src/perf_selftests-x86_64-rhel-8.3-bpf-d3ed44a7bee70b097709fe06f339c468489f16c9/tools/perf/pmu-events/amd_metrics.py", line 42, in <module>
main()
File "/usr/src/perf_selftests-x86_64-rhel-8.3-bpf-d3ed44a7bee70b097709fe06f339c468489f16c9/tools/perf/pmu-events/amd_metrics.py", line 32, in main
LoadEvents(directory)
File "/usr/src/perf_selftests-x86_64-rhel-8.3-bpf-d3ed44a7bee70b097709fe06f339c468489f16c9/tools/perf/pmu-events/metric.py", line 26, in LoadEvents
for x in json.load(open(f"{directory}/{filename}")):
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.11/json/__init__.py", line 293, in load
return loads(fp.read(),
^^^^^^^^^^^^^^^^
File "/usr/lib/python3.11/json/__init__.py", line 346, in loads
return _default_decoder.decode(s)
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.11/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.11/json/decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
make[3]: *** [pmu-events/Build:45: pmu-events/arch/x86/amdzen2/extra-metrics.json] Error 1
make[3]: *** Deleting file 'pmu-events/arch/x86/amdzen2/extra-metrics.json'
make[2]: *** [Makefile.perf:706: pmu-events/pmu-events-in.o] Error 2
make[2]: *** Waiting for unfinished jobs....
make[1]: *** [Makefile.perf:261: sub-make] Error 2
make: *** [Makefile:70: all] Error 2
[1]
The kernel config and materials to reproduce are available at:
https://download.01.org/0day-ci/archive/20240322/[email protected]
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki
On 2024-03-14 1:50 a.m., Ian Rogers wrote:
> Add a LoadEvents function that loads all event json files in a
> directory. In the Event constructor ensure all events are defined in
> the event json except for legacy events like "cycles". If the initial
> event isn't found then legacy_event1 is used, and if that isn't found
> legacy_event2 is used. This allows a single Event to have multiple
> event names as models will often rename the same event over time. If
> the event doesn't exist an exception is raised.
>
> So that references to metrics can be added, add the MetricRef
> class. This doesn't validate as an event name and so provides an
> escape hatch for metrics to refer to each other.
>
> Signed-off-by: Ian Rogers <[email protected]>
> ---
> tools/perf/pmu-events/Build | 12 ++--
> tools/perf/pmu-events/amd_metrics.py | 6 +-
> tools/perf/pmu-events/arm64_metrics.py | 6 +-
> tools/perf/pmu-events/intel_metrics.py | 6 +-
> tools/perf/pmu-events/metric.py | 77 +++++++++++++++++++++++++-
> 5 files changed, 95 insertions(+), 12 deletions(-)
>
make fails.
GEN pmu-events/arch/arm64/arm/cortex-a710/extra-metricgroups.json
Traceback (most recent call last):
File
"/home/kan/tmp/perf-tools-next-test/tools/perf/pmu-events/arm64_metrics.py",
line 43, in <module>
main()
File
"/home/kan/tmp/perf-tools-next-test/tools/perf/pmu-events/arm64_metrics.py",
line 35, in main
LoadEvents(directory)
File
"/home/kan/tmp/perf-tools-next-test/tools/perf/pmu-events/metric.py",
line 26, in LoadEvents
for x in json.load(open(f"{directory}/{filename}")):
File "/usr/lib64/python3.10/json/__init__.py", line 293, in load
return loads(fp.read(),
File "/usr/lib64/python3.10/json/__init__.py", line 346, in loads
return _default_decoder.decode(s)
File "/usr/lib64/python3.10/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/lib64/python3.10/json/decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
GEN pmu-events/arch/arm64/arm/cortex-a73/extra-metrics.json
make[3]: Circular pmu-events/arch/arm64/arm/cortex-a75/branch.json <-
pmu-events/arch/arm64/arm/cortex-a75/branch.json dependency dropped.
make[3]: Circular pmu-events/arch/arm64/arm/cortex-a75/bus.json <-
pmu-events/arch/arm64/arm/cortex-a75/bus.json dependency dropped.
make[3]: Circular pmu-events/arch/arm64/arm/cortex-a75/cache.json <-
pmu-events/arch/arm64/arm/cortex-a75/cache.json dependency dropped.
make[3]: Circular pmu-events/arch/arm64/arm/cortex-a75/etm.json <-
pmu-events/arch/arm64/arm/cortex-a75/etm.json dependency dropped.
make[3]: Circular pmu-events/arch/arm64/arm/cortex-a75/exception.json <-
pmu-events/arch/arm64/arm/cortex-a75/exception.json dependency dropped.
make[3]: Circular pmu-events/arch/arm64/arm/cortex-a75/instruction.json
<- pmu-events/arch/arm64/arm/cortex-a75/instruction.json dependency dropped.
make[3]: Circular pmu-events/arch/arm64/arm/cortex-a75/memory.json <-
pmu-events/arch/arm64/arm/cortex-a75/memory.json dependency dropped.
make[3]: Circular pmu-events/arch/arm64/arm/cortex-a75/mmu.json <-
pmu-events/arch/arm64/arm/cortex-a75/mmu.json dependency dropped.
make[3]: Circular pmu-events/arch/arm64/arm/cortex-a75/pipeline.json <-
pmu-events/arch/arm64/arm/cortex-a75/pipeline.json dependency dropped.
GEN pmu-events/arch/arm64/arm/cortex-a73/extra-metricgroups.json
make[3]: *** [pmu-events/Build:58:
pmu-events/arch/arm64/arm/cortex-a57-a72/extra-metrics.json] Error 1
make[3]: *** Deleting file
'pmu-events/arch/arm64/arm/cortex-a57-a72/extra-metrics.json'
make[3]: *** Waiting for unfinished jobs....
Traceback (most recent call last):
File
"/home/kan/tmp/perf-tools-next-test/tools/perf/pmu-events/arm64_metrics.py",
line 43, in <module>
main()
File
"/home/kan/tmp/perf-tools-next-test/tools/perf/pmu-events/arm64_metrics.py",
line 35, in main
LoadEvents(directory)
File
"/home/kan/tmp/perf-tools-next-test/tools/perf/pmu-events/metric.py",
line 26, in LoadEvents
for x in json.load(open(f"{directory}/{filename}")):
File "/usr/lib64/python3.10/json/__init__.py", line 293, in load
return loads(fp.read(),
File "/usr/lib64/python3.10/json/__init__.py", line 346, in loads
return _default_decoder.decode(s)
File "/usr/lib64/python3.10/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/lib64/python3.10/json/decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
make[3]: *** [pmu-events/Build:58:
pmu-events/arch/arm64/arm/cortex-a65-e1/extra-metrics.json] Error 1
make[3]: *** Deleting file
'pmu-events/arch/arm64/arm/cortex-a65-e1/extra-metrics.json'
Traceback (most recent call last):
File
"/home/kan/tmp/perf-tools-next-test/tools/perf/pmu-events/arm64_metrics.py",
line 43, in <module>
main()
File
"/home/kan/tmp/perf-tools-next-test/tools/perf/pmu-events/arm64_metrics.py",
line 35, in main
LoadEvents(directory)
File
"/home/kan/tmp/perf-tools-next-test/tools/perf/pmu-events/metric.py",
line 26, in LoadEvents
for x in json.load(open(f"{directory}/{filename}")):
File "/usr/lib64/python3.10/json/__init__.py", line 293, in load
return loads(fp.read(),
File "/usr/lib64/python3.10/json/__init__.py", line 346, in loads
return _default_decoder.decode(s)
File "/usr/lib64/python3.10/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/lib64/python3.10/json/decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
make[3]: *** [pmu-events/Build:62:
pmu-events/arch/arm64/arm/cortex-a57-a72/extra-metricgroups.json] Error 1
make[3]: *** Deleting file
'pmu-events/arch/arm64/arm/cortex-a57-a72/extra-metricgroups.json'
Traceback (most recent call last):
File
"/home/kan/tmp/perf-tools-next-test/tools/perf/pmu-events/arm64_metrics.py",
line 43, in <module>
main()
File
"/home/kan/tmp/perf-tools-next-test/tools/perf/pmu-events/arm64_metrics.py",
line 35, in main
LoadEvents(directory)
File
"/home/kan/tmp/perf-tools-next-test/tools/perf/pmu-events/metric.py",
line 26, in LoadEvents
for x in json.load(open(f"{directory}/{filename}")):
File "/usr/lib64/python3.10/json/__init__.py", line 293, in load
return loads(fp.read(),
File "/usr/lib64/python3.10/json/__init__.py", line 346, in loads
return _default_decoder.decode(s)
File "/usr/lib64/python3.10/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/lib64/python3.10/json/decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
make[3]: *** [pmu-events/Build:62:
pmu-events/arch/arm64/arm/cortex-a65-e1/extra-metricgroups.json] Error 1
make[3]: *** Deleting file
'pmu-events/arch/arm64/arm/cortex-a65-e1/extra-metricgroups.json'
Traceback (most recent call last):
File
"/home/kan/tmp/perf-tools-next-test/tools/perf/pmu-events/arm64_metrics.py",
line 43, in <module>
main()
File
"/home/kan/tmp/perf-tools-next-test/tools/perf/pmu-events/arm64_metrics.py",
line 35, in main
LoadEvents(directory)
File
"/home/kan/tmp/perf-tools-next-test/tools/perf/pmu-events/metric.py",
line 26, in LoadEvents
for x in json.load(open(f"{directory}/{filename}")):
File "/usr/lib64/python3.10/json/__init__.py", line 293, in load
return loads(fp.read(),
File "/usr/lib64/python3.10/json/__init__.py", line 346, in loads
Traceback (most recent call last):
File
"/home/kan/tmp/perf-tools-next-test/tools/perf/pmu-events/arm64_metrics.py",
line 43, in <module>
return _default_decoder.decode(s)
File "/usr/lib64/python3.10/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/lib64/python3.10/json/decoder.py", line 355, in raw_decode
main()
File
"/home/kan/tmp/perf-tools-next-test/tools/perf/pmu-events/arm64_metrics.py",
line 35, in main
LoadEvents(directory)
File
"/home/kan/tmp/perf-tools-next-test/tools/perf/pmu-events/metric.py",
line 26, in LoadEvents
raise JSONDecodeError("Expecting value", s, err.value) from None
for x in json.load(open(f"{directory}/{filename}")):
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
File "/usr/lib64/python3.10/json/__init__.py", line 293, in load
return loads(fp.read(),
File "/usr/lib64/python3.10/json/__init__.py", line 346, in loads
return _default_decoder.decode(s)
File "/usr/lib64/python3.10/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/lib64/python3.10/json/decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
Traceback (most recent call last):
File
"/home/kan/tmp/perf-tools-next-test/tools/perf/pmu-events/arm64_metrics.py",
line 43, in <module>
main()
File
"/home/kan/tmp/perf-tools-next-test/tools/perf/pmu-events/arm64_metrics.py",
line 35, in main
LoadEvents(directory)
File
"/home/kan/tmp/perf-tools-next-test/tools/perf/pmu-events/metric.py",
line 26, in LoadEvents
for x in json.load(open(f"{directory}/{filename}")):
File "/usr/lib64/python3.10/json/__init__.py", line 293, in load
return loads(fp.read(),
File "/usr/lib64/python3.10/json/__init__.py", line 346, in loads
return _default_decoder.decode(s)
File "/usr/lib64/python3.10/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/lib64/python3.10/json/decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
Traceback (most recent call last):
File
"/home/kan/tmp/perf-tools-next-test/tools/perf/pmu-events/arm64_metrics.py",
line 43, in <module>
main()
File
"/home/kan/tmp/perf-tools-next-test/tools/perf/pmu-events/arm64_metrics.py",
line 35, in main
LoadEvents(directory)
File
"/home/kan/tmp/perf-tools-next-test/tools/perf/pmu-events/metric.py",
line 26, in LoadEvents
for x in json.load(open(f"{directory}/{filename}")):
File "/usr/lib64/python3.10/json/__init__.py", line 293, in load
return loads(fp.read(),
File "/usr/lib64/python3.10/json/__init__.py", line 346, in loads
return _default_decoder.decode(s)
File "/usr/lib64/python3.10/json/decoder.py", line 337, in decode
obj, end = self.raw_decode(s, idx=_w(s, 0).end())
File "/usr/lib64/python3.10/json/decoder.py", line 355, in raw_decode
raise JSONDecodeError("Expecting value", s, err.value) from None
json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)
make[3]: *** [pmu-events/Build:62:
pmu-events/arch/arm64/arm/cortex-a710/extra-metricgroups.json] Error 1
make[3]: *** Deleting file
'pmu-events/arch/arm64/arm/cortex-a710/extra-metricgroups.json'
make[3]: *** [pmu-events/Build:58:
pmu-events/arch/arm64/arm/cortex-a710/extra-metrics.json] Error 1
make[3]: *** Deleting file
'pmu-events/arch/arm64/arm/cortex-a710/extra-metrics.json'
make[3]: *** [pmu-events/Build:62:
pmu-events/arch/arm64/arm/cortex-a73/extra-metricgroups.json] Error 1
make[3]: *** Deleting file
'pmu-events/arch/arm64/arm/cortex-a73/extra-metricgroups.json'
make[2]: *** [Makefile.perf:730: pmu-events/pmu-events-in.o] Error 2
make[1]: *** [Makefile.perf:264: sub-make] Error 2
make: *** [Makefile:70: all] Error 2
Thanks,
Kan
> diff --git a/tools/perf/pmu-events/Build b/tools/perf/pmu-events/Build
> index e2db33577707..a2c5c04e5c46 100644
> --- a/tools/perf/pmu-events/Build
> +++ b/tools/perf/pmu-events/Build
> @@ -42,11 +42,11 @@ ZEN_METRICGROUPS = $(foreach x,$(ZENS),$(OUTPUT)$(x)/extra-metricgroups.json)
>
> $(ZEN_METRICS): pmu-events/amd_metrics.py
> $(call rule_mkdir)
> - $(Q)$(call echo-cmd,gen)$(PYTHON) $< $(call model_name,$@) arch > $@
> + $(Q)$(call echo-cmd,gen)$(PYTHON) $< $(call model_name,$@) pmu-events/arch > $@
>
> $(ZEN_METRICGROUPS): pmu-events/amd_metrics.py
> $(call rule_mkdir)
> - $(Q)$(call echo-cmd,gen)$(PYTHON) $< -metricgroups $(call model_name,$@) arch > $@
> + $(Q)$(call echo-cmd,gen)$(PYTHON) $< -metricgroups $(call model_name,$@) pmu-events/arch > $@
>
> # Generate ARM Json
> ARMS = $(shell ls -d pmu-events/arch/arm64/arm/*)
> @@ -55,11 +55,11 @@ ARM_METRICGROUPS = $(foreach x,$(ARMS),$(OUTPUT)$(x)/extra-metricgroups.json)
>
> $(ARM_METRICS): pmu-events/arm64_metrics.py
> $(call rule_mkdir)
> - $(Q)$(call echo-cmd,gen)$(PYTHON) $< $(call vendor_name,$@) $(call model_name,$@) arch > $@
> + $(Q)$(call echo-cmd,gen)$(PYTHON) $< $(call vendor_name,$@) $(call model_name,$@) pmu-events/arch > $@
>
> $(ARM_METRICGROUPS): pmu-events/arm64_metrics.py
> $(call rule_mkdir)
> - $(Q)$(call echo-cmd,gen)$(PYTHON) $< -metricgroups $(call vendor_name,$@) $(call model_name,$@) arch > $@
> + $(Q)$(call echo-cmd,gen)$(PYTHON) $< -metricgroups $(call vendor_name,$@) $(call model_name,$@) pmu-events/arch > $@
>
> # Generate Intel Json
> INTELS = $(shell ls -d pmu-events/arch/x86/*|grep -v amdzen|grep -v mapfile.csv)
> @@ -68,11 +68,11 @@ INTEL_METRICGROUPS = $(foreach x,$(INTELS),$(OUTPUT)$(x)/extra-metricgroups.json
>
> $(INTEL_METRICS): pmu-events/intel_metrics.py
> $(call rule_mkdir)
> - $(Q)$(call echo-cmd,gen)$(PYTHON) $< $(call model_name,$@) arch > $@
> + $(Q)$(call echo-cmd,gen)$(PYTHON) $< $(call model_name,$@) pmu-events/arch > $@
>
> $(INTEL_METRICGROUPS): pmu-events/intel_metrics.py
> $(call rule_mkdir)
> - $(Q)$(call echo-cmd,gen)$(PYTHON) $< -metricgroups $(call model_name,$@) arch > $@
> + $(Q)$(call echo-cmd,gen)$(PYTHON) $< -metricgroups $(call model_name,$@) pmu-events/arch > $@
>
> GEN_JSON = $(patsubst %,$(OUTPUT)%,$(JSON)) \
> $(ZEN_METRICS) $(ZEN_METRICGROUPS) \
> diff --git a/tools/perf/pmu-events/amd_metrics.py b/tools/perf/pmu-events/amd_metrics.py
> index 7ab2ee4fdb17..4f728e7aae4a 100755
> --- a/tools/perf/pmu-events/amd_metrics.py
> +++ b/tools/perf/pmu-events/amd_metrics.py
> @@ -1,6 +1,7 @@
> #!/usr/bin/env python3
> # SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
> -from metric import (JsonEncodeMetric, JsonEncodeMetricGroupDescriptions, MetricGroup)
> +from metric import (JsonEncodeMetric, JsonEncodeMetricGroupDescriptions, LoadEvents,
> + MetricGroup)
> import argparse
> import json
> import os
> @@ -27,6 +28,9 @@ def main() -> None:
> )
> _args = parser.parse_args()
>
> + directory = f"{_args.events_path}/x86/{_args.model}/"
> + LoadEvents(directory)
> +
> all_metrics = MetricGroup("",[])
>
> if _args.metricgroups:
> diff --git a/tools/perf/pmu-events/arm64_metrics.py b/tools/perf/pmu-events/arm64_metrics.py
> index a9f0e6bc751b..c9aa2d827a82 100755
> --- a/tools/perf/pmu-events/arm64_metrics.py
> +++ b/tools/perf/pmu-events/arm64_metrics.py
> @@ -1,6 +1,7 @@
> #!/usr/bin/env python3
> # SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
> -from metric import (JsonEncodeMetric, JsonEncodeMetricGroupDescriptions, MetricGroup)
> +from metric import (JsonEncodeMetric, JsonEncodeMetricGroupDescriptions, LoadEvents,
> + MetricGroup)
> import argparse
> import json
> import os
> @@ -30,6 +31,9 @@ def main() -> None:
>
> all_metrics = MetricGroup("",[])
>
> + directory = f"{_args.events_path}/arm64/{_args.vendor}/{_args.model}/"
> + LoadEvents(directory)
> +
> if _args.metricgroups:
> print(JsonEncodeMetricGroupDescriptions(all_metrics))
> else:
> diff --git a/tools/perf/pmu-events/intel_metrics.py b/tools/perf/pmu-events/intel_metrics.py
> index f004c27640d2..04a19d05c6c1 100755
> --- a/tools/perf/pmu-events/intel_metrics.py
> +++ b/tools/perf/pmu-events/intel_metrics.py
> @@ -1,6 +1,7 @@
> #!/usr/bin/env python3
> # SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
> -from metric import (JsonEncodeMetric, JsonEncodeMetricGroupDescriptions, MetricGroup)
> +from metric import (JsonEncodeMetric, JsonEncodeMetricGroupDescriptions, LoadEvents,
> + MetricGroup)
> import argparse
> import json
> import os
> @@ -27,6 +28,9 @@ def main() -> None:
> )
> _args = parser.parse_args()
>
> + directory = f"{_args.events_path}/x86/{_args.model}/"
> + LoadEvents(directory)
> +
> all_metrics = MetricGroup("",[])
>
> if _args.metricgroups:
> diff --git a/tools/perf/pmu-events/metric.py b/tools/perf/pmu-events/metric.py
> index dd8fd06940e6..03312cd6d491 100644
> --- a/tools/perf/pmu-events/metric.py
> +++ b/tools/perf/pmu-events/metric.py
> @@ -3,10 +3,50 @@
> import ast
> import decimal
> import json
> +import os
> import re
> from enum import Enum
> from typing import Dict, List, Optional, Set, Tuple, Union
>
> +all_events = set()
> +
> +def LoadEvents(directory: str) -> None:
> + """Populate a global set of all known events for the purpose of validating Event names"""
> + global all_events
> + all_events = {
> + "context\-switches",
> + "cycles",
> + "duration_time",
> + "instructions",
> + "l2_itlb_misses",
> + }
> + for file in os.listdir(os.fsencode(directory)):
> + filename = os.fsdecode(file)
> + if filename.endswith(".json"):
> + for x in json.load(open(f"{directory}/{filename}")):
> + if "EventName" in x:
> + all_events.add(x["EventName"])
> + elif "ArchStdEvent" in x:
> + all_events.add(x["ArchStdEvent"])
> +
> +
> +def CheckEvent(name: str) -> bool:
> + """Check the event name exists in the set of all loaded events"""
> + global all_events
> + if len(all_events) == 0:
> + # No events loaded so assume any event is good.
> + return True
> +
> + if ':' in name:
> + # Remove trailing modifier.
> + name = name[:name.find(':')]
> + elif '/' in name:
> + # Name could begin with a PMU or an event, for now assume it is good.
> + return True
> +
> + return name in all_events
> +
> +
> class MetricConstraint(Enum):
> GROUPED_EVENTS = 0
> NO_GROUP_EVENTS = 1
> @@ -317,9 +357,18 @@ def _FixEscapes(s: str) -> str:
> class Event(Expression):
> """An event in an expression."""
>
> - def __init__(self, name: str, legacy_name: str = ''):
> - self.name = _FixEscapes(name)
> - self.legacy_name = _FixEscapes(legacy_name)
> + def __init__(self, *args: str):
> + error = ""
> + for name in args:
> + if CheckEvent(name):
> + self.name = _FixEscapes(name)
> + return
> + if error:
> + error += " or " + name
> + else:
> + error = name
> + global all_events
> + raise Exception(f"No event {error} in:\n{all_events}")
>
> def ToPerfJson(self):
> result = re.sub('/', '@', self.name)
> @@ -338,6 +387,28 @@ class Event(Expression):
> return self
>
>
> +class MetricRef(Expression):
> + """A metric reference in an expression."""
> +
> + def __init__(self, name: str):
> + self.name = _FixEscapes(name)
> +
> + def ToPerfJson(self):
> + return self.name
> +
> + def ToPython(self):
> + return f'MetricRef(r"{self.name}")'
> +
> + def Simplify(self) -> Expression:
> + return self
> +
> + def Equals(self, other: Expression) -> bool:
> + return isinstance(other, MetricRef) and self.name == other.name
> +
> + def Substitute(self, name: str, expression: Expression) -> Expression:
> + return self
> +
> +
> class Constant(Expression):
> """A constant within the expression tree."""
>