Received-SPF: pass (google.com: domain of linux-kernel+bounces-40116-linux.lists.archive=gmail.com@vger.kernel.org designates 2604:1380:45d1:ec00::1 as permitted sender) client-ip=2604:1380:45d1:ec00::1;
From: Haifeng Xu <haifeng.xu@shopee.com>
To: reinette.chatre@intel.com
Cc: fenghua.yu@intel.com,
	babu.moger@amd.com,
	peternewman@google.com,
	james.morse@arm.com,
	x86@kernel.org,
	linux-kernel@vger.kernel.org,
	Haifeng Xu <haifeng.xu@shopee.com>
Subject: [PATCH] x86/resctrl: Add tracepoint for llc_occupancy tracking
Date: Fri, 26 Jan 2024 13:02:13 +0000
Message-Id: <20240126130213.159339-1-haifeng.xu@shopee.com>
Precedence: bulk
MIME-Version: 1.0
Content-Transfer-Encoding: 8bit

If llc_occupany is enabled, the rmid may not be freed immediately unless
its llc_occupany is less than the resctrl_rmid_realloc_threshold.

In our production environment, those unused rmids get stuck in the limbo
list forever because their llc_occupancy are larger than the threshold.
After turning it up, we can successfully free unused rmids and create
new monitor groups. In order to acquire the llc_occupancy of rmids in
each rdt domain, we use perf tool to track and filter the log manually.

It's not efficient enough. Therefore, we can add a new tracepoint that
shows the llc_occupancy of busy rmids when scanning the limbo list. It
can help us to adjust the resctrl_rmid_realloc_threshold to a reasonable
value.

Signed-off-by: Haifeng Xu <haifeng.xu@shopee.com>
---
 arch/x86/kernel/cpu/resctrl/Makefile        |  1 +
 arch/x86/kernel/cpu/resctrl/monitor.c       |  5 ++++
 arch/x86/kernel/cpu/resctrl/monitor_event.h | 30 +++++++++++++++++++++
 3 files changed, 36 insertions(+)
 create mode 100644 arch/x86/kernel/cpu/resctrl/monitor_event.h

diff --git a/arch/x86/kernel/cpu/resctrl/Makefile b/arch/x86/kernel/cpu/resctrl/Makefile
index 4a06c37b9cf1..0d3031850d26 100644
--- a/arch/x86/kernel/cpu/resctrl/Makefile
+++ b/arch/x86/kernel/cpu/resctrl/Makefile
@@ -1,4 +1,5 @@
 # SPDX-License-Identifier: GPL-2.0
 obj-$(CONFIG_X86_CPU_RESCTRL)	+= core.o rdtgroup.o monitor.o
 obj-$(CONFIG_X86_CPU_RESCTRL)	+= ctrlmondata.o pseudo_lock.o
+CFLAGS_monitor.o = -I$(src)
 CFLAGS_pseudo_lock.o = -I$(src)
diff --git a/arch/x86/kernel/cpu/resctrl/monitor.c b/arch/x86/kernel/cpu/resctrl/monitor.c
index f136ac046851..a6f94fcae174 100644
--- a/arch/x86/kernel/cpu/resctrl/monitor.c
+++ b/arch/x86/kernel/cpu/resctrl/monitor.c
@@ -24,6 +24,10 @@
 
 #include "internal.h"
 
+#define CREATE_TRACE_POINTS
+#include "monitor_event.h"
+#undef CREATE_TRACE_POINTS
+
 struct rmid_entry {
 	u32				rmid;
 	int				busy;
@@ -302,6 +306,7 @@ void __check_limbo(struct rdt_domain *d, bool force_free)
 			}
 		}
 		crmid = nrmid + 1;
+		trace_rmid_llc_occupancy(nrmid, d->id, val);
 	}
 }
 
diff --git a/arch/x86/kernel/cpu/resctrl/monitor_event.h b/arch/x86/kernel/cpu/resctrl/monitor_event.h
new file mode 100644
index 000000000000..91265a2dd2c9
--- /dev/null
+++ b/arch/x86/kernel/cpu/resctrl/monitor_event.h
@@ -0,0 +1,30 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#undef TRACE_SYSTEM
+#define TRACE_SYSTEM resctrl
+
+#if !defined(_TRACE_MONITOR_H) || defined(TRACE_HEADER_MULTI_READ)
+#define _TRACE_MONITOR_H
+
+#include <linux/tracepoint.h>
+
+TRACE_EVENT(rmid_llc_occupancy,
+	    TP_PROTO(u32 rmid, int id, u64 occupancy),
+	    TP_ARGS(rmid, id, occupancy),
+	    TP_STRUCT__entry(__field(u32, rmid)
+			     __field(int, id)
+			     __field(u64, occupancy)),
+	    TP_fast_assign(__entry->rmid = rmid;
+			   __entry->id = id;
+			   __entry->occupancy = occupancy;),
+	    TP_printk("rmid=%u domain=%d occupancy=%llu",
+		      __entry->rmid, __entry->id, __entry->occupancy)
+	   );
+
+#endif /* _TRACE_MONITOR_H */
+
+#undef TRACE_INCLUDE_PATH
+#define TRACE_INCLUDE_PATH .
+#define TRACE_INCLUDE_FILE monitor_event
+
+/* This part must be outside protection */
+#include <trace/define_trace.h>
-- 
2.25.1