On Fam19h server CPUs from AMD an HSMP (Host System Management Port)
mailbox interface is added to monitor and manage the CPU parameters.
Export hsmp_send_message() function which takes a socket number and
a messages structure as inputs, returns 0 on success and negative
error otherwise. The API can be used by other kernel components to
access system management features.
Eg: hwmon based k10temp driver can be modified to report power metrics
Signed-off-by: Naveen Krishna Chatradhi <[email protected]>
Cc: Guenter Roeck <[email protected]>
Cc: Ingo Molnar <[email protected]>
Cc: Borislav Petkov <[email protected]>
Cc: Jean Delvare <[email protected]>
Cc: x86-ml <[email protected]>
Cc: Yazen Ghannam <[email protected]>
---
arch/x86/include/asm/amd_nb.h | 40 +++++++
arch/x86/kernel/amd_nb.c | 217 ++++++++++++++++++++++++++++++++++
2 files changed, 257 insertions(+)
diff --git a/arch/x86/include/asm/amd_nb.h b/arch/x86/include/asm/amd_nb.h
index 455066a06f60..2640429114ee 100644
--- a/arch/x86/include/asm/amd_nb.h
+++ b/arch/x86/include/asm/amd_nb.h
@@ -66,6 +66,7 @@ struct amd_northbridge {
struct pci_dev *link;
struct amd_l3_cache l3_cache;
struct threshold_bank *bank4;
+ struct semaphore hsmp_sem_lock;
};
struct amd_northbridge_info {
@@ -77,6 +78,7 @@ struct amd_northbridge_info {
#define AMD_NB_GART BIT(0)
#define AMD_NB_L3_INDEX_DISABLE BIT(1)
#define AMD_NB_L3_PARTITIONING BIT(2)
+#define AMD_NB_HSMP BIT(3)
#ifdef CONFIG_AMD_NB
@@ -123,5 +125,43 @@ static inline bool amd_gart_present(void)
#endif
+/*
+ * HSMP Message types supported
+ */
+enum hsmp_message_ids {
+ HSMP_TEST = 1,
+ HSMP_GET_SMU_VER,
+ HSMP_GET_PROTO_VER,
+ HSMP_GET_SOCKET_POWER,
+ HSMP_SET_SOCKET_POWER_LIMIT,
+ HSMP_GET_SOCKET_POWER_LIMIT,
+ HSMP_GET_SOCKET_POWER_LIMIT_MAX,
+ HSMP_SET_BOOST_LIMIT,
+ HSMP_SET_BOOST_LIMIT_SOCKET,
+ HSMP_GET_BOOST_LIMIT,
+ HSMP_GET_PROC_HOT,
+ HSMP_SET_XGMI_LINK_WIDTH,
+ HSMP_SET_DF_PSTATE,
+ HSMP_AUTO_DF_PSTATE,
+ HSMP_GET_FCLK_MCLK,
+ HSMP_GET_CCLK_THROTTLE_LIMIT,
+ HSMP_GET_C0_PERCENT,
+ HSMP_SET_NBIO_DPM_LEVEL,
+ HSMP_RESERVED,
+ HSMP_GET_DDR_BANDWIDTH,
+ HSMP_MSG_ID_MAX,
+};
+
+#define HSMP_MAX_MSG_LEN 8
+
+struct hsmp_message {
+ u32 msg_id; /* Message ID */
+ u16 num_args; /* Number of arguments in message */
+ u16 response_sz; /* Number of expected response words */
+ u32 args[HSMP_MAX_MSG_LEN]; /* Argument(s) */
+ u32 response[HSMP_MAX_MSG_LEN]; /* Response word(s) */
+};
+
+int hsmp_send_message(int socket_id, struct hsmp_message *msg);
#endif /* _ASM_X86_AMD_NB_H */
diff --git a/arch/x86/kernel/amd_nb.c b/arch/x86/kernel/amd_nb.c
index c92c9c774c0e..2bba59c1350e 100644
--- a/arch/x86/kernel/amd_nb.c
+++ b/arch/x86/kernel/amd_nb.c
@@ -6,6 +6,7 @@
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
+#include <linux/delay.h>
#include <linux/types.h>
#include <linux/slab.h>
#include <linux/init.h>
@@ -29,6 +30,22 @@
#define PCI_DEVICE_ID_AMD_19H_M40H_DF_F4 0x167d
#define PCI_DEVICE_ID_AMD_19H_M50H_DF_F4 0x166e
+/*
+ * HSMP Status / Error codes
+ */
+#define HSMP_STATUS_NOT_READY 0x00
+#define HSMP_STATUS_OK 0x01
+#define HSMP_ERR_INVALID_MSG 0xFE
+#define HSMP_ERR_INVALID_INPUT 0xFF
+
+/* To access specific HSMP mailbox register, s/w writes the SMN address of HSMP mailbox
+ * register into the SMN_INDEX register, and reads/writes the SMN_DATA reg.
+ * Below are required SMN address for HSMP Mailbox register offsets in SMU address space
+ */
+#define SMN_HSMP_MSG_ID 0x3B10534
+#define SMN_HSMP_MSG_RESP 0x3B10980
+#define SMN_HSMP_MSG_DATA 0x3B109E0
+
/* Protect the PCI config register pairs used for SMN and DF indirect access. */
static DEFINE_MUTEX(smn_mutex);
@@ -108,6 +125,9 @@ const struct amd_nb_bus_dev_range amd_nb_bus_dev_ranges[] __initconst = {
static struct amd_northbridge_info amd_northbridges;
+/* Timeout in millsec */
+#define HSMP_MSG_TIMEOUT 100
+
u16 amd_nb_num(void)
{
return amd_northbridges.num;
@@ -182,6 +202,194 @@ int amd_smn_write(u16 node, u32 address, u32 value)
}
EXPORT_SYMBOL_GPL(amd_smn_write);
+#define HSMP_WR true
+#define HSMP_RD false
+
+static int __amd_hsmp_rdwr(struct pci_dev *root, u32 address,
+ u32 *value, bool write)
+{
+ int err;
+
+ err = pci_write_config_dword(root, 0xc4, address);
+ if (err) {
+ pr_warn("Error programming SMN address 0x%x.\n", address);
+ return err;
+ }
+
+ err = (write ? pci_write_config_dword(root, 0xc8, *value)
+ : pci_read_config_dword(root, 0xc8, value));
+ if (err)
+ pr_warn("Error %s SMN address 0x%x.\n",
+ (write ? "writing to" : "reading from"), address);
+
+ return err;
+}
+
+/*
+ * Send a message to the HSMP port via PCI-e config space registers.
+ *
+ * The caller is expected to zero out any unused arguments.
+ * If a response is expected, the number of response words should be greater than 0.
+ *
+ * Returns 0 for success and populates the requested number of arguments.
+ * Returns a negative error code for failure.
+ */
+static int __hsmp_send_message(struct pci_dev *root, struct hsmp_message *msg)
+{
+ u64 timeout = HSMP_MSG_TIMEOUT;
+ u32 mbox_status;
+ u32 arg_num;
+ int err;
+
+ /* Clear the status register */
+ mbox_status = HSMP_STATUS_NOT_READY;
+ err = __amd_hsmp_rdwr(root, SMN_HSMP_MSG_RESP, &mbox_status, HSMP_WR);
+ if (err) {
+ pr_err("Error %d clearing mailbox status register.\n", err);
+ return err;
+ }
+
+ arg_num = 0;
+ /* Write any message arguments */
+ while (arg_num < msg->num_args) {
+ err = __amd_hsmp_rdwr(root, SMN_HSMP_MSG_DATA + (arg_num << 2),
+ &msg->args[arg_num], HSMP_WR);
+ if (err) {
+ pr_err("Error %d writing message argument %d\n",
+ err, arg_num);
+ return err;
+ }
+ arg_num++;
+ }
+
+ /* Write the message ID which starts the operation */
+ err = __amd_hsmp_rdwr(root, SMN_HSMP_MSG_ID, &msg->msg_id, HSMP_WR);
+ if (err) {
+ pr_err("Error %d writing message ID %u\n", err, msg->msg_id);
+ return err;
+ }
+
+ /*
+ * Depending on when the trigger write completes relative to the SMU
+ * firmware 1 ms cycle, the operation may take from tens of us to 1 ms
+ * to complete. Some operations may take more. Therefore we will try
+ * a few short duration sleeps and switch to long sleeps if we don't
+ * succeed quickly.
+ */
+ do {
+ if (likely(timeout > 90))
+ usleep_range(50, 100);
+ else
+ usleep_range(100, 500);
+
+ err = __amd_hsmp_rdwr(root, SMN_HSMP_MSG_RESP, &mbox_status, HSMP_RD);
+ if (err) {
+ pr_err("Error %d reading mailbox status\n", err);
+ return err;
+ }
+
+ if (mbox_status != HSMP_STATUS_NOT_READY)
+ break;
+
+ } while (--timeout);
+
+ if (unlikely(mbox_status == HSMP_ERR_INVALID_MSG)) {
+ pr_err("Invalid message ID %u\n", msg->msg_id);
+ err = -ENOMSG;
+ return err;
+ } else if (unlikely(mbox_status == HSMP_ERR_INVALID_INPUT)) {
+ pr_err("Invalid arguments for %d\n", msg->msg_id);
+ err = -EINVAL;
+ return err;
+ } else if (unlikely(mbox_status != HSMP_STATUS_OK)) {
+ pr_err("Message ID %u unknown failure (status = 0x%X), timedout\n",
+ msg->msg_id, mbox_status);
+ err = -ETIMEDOUT;
+ return err;
+ }
+
+ /* SMU has responded OK. Read response data */
+ arg_num = 0;
+ while (arg_num < msg->response_sz) {
+ err = __amd_hsmp_rdwr(root, SMN_HSMP_MSG_DATA + (arg_num << 2),
+ &msg->response[arg_num], HSMP_RD);
+ if (err) {
+ pr_err("Error %d reading response %u for message ID:%u\n",
+ err, arg_num, msg->msg_id);
+ break;
+ }
+ arg_num++;
+ }
+
+ return err;
+}
+
+/* Verify the HSMP test message */
+static bool amd_hsmp_support(void)
+{
+ struct hsmp_message msg = { 0 };
+ int err = 0;
+ struct pci_dev *root;
+
+ /*
+ * Check HSMP support on socket 0. the test message takes one argument
+ * and returns the value of that argument + 1.
+ */
+ root = node_to_amd_nb(0)->root;
+ if (!root)
+ return false;
+
+ msg.args[0] = 0xDEADBEEF;
+ msg.response_sz = 1;
+ msg.msg_id = HSMP_TEST;
+ msg.num_args = 1;
+
+ err = __hsmp_send_message(root, &msg);
+ if (err)
+ return false;
+
+ if (msg.response[0] != msg.args[0] + 1)
+ return false;
+
+ return true;
+}
+
+int hsmp_send_message(int node, struct hsmp_message *msg)
+{
+ struct pci_dev *root;
+ int err;
+
+ if (!amd_nb_has_feature(AMD_NB_HSMP))
+ return -ENODEV;
+
+ root = node_to_amd_nb(node)->root;
+ if (!root || !msg)
+ return -ENODEV;
+
+ if (msg->msg_id < HSMP_TEST || msg->msg_id >= HSMP_MSG_ID_MAX)
+ return -EINVAL;
+
+ if (msg->num_args > HSMP_MAX_MSG_LEN || msg->response_sz > HSMP_MAX_MSG_LEN)
+ return -EINVAL;
+
+ /*
+ * The time taken by smu operation to complete is between
+ * 10us to 1ms. Sometime it may take more time.
+ * In SMP system timeout of 100 millisecs should
+ * be enough for the previous thread to finish the operation
+ */
+ err = down_timeout(&(node_to_amd_nb(node)->hsmp_sem_lock),
+ msecs_to_jiffies(HSMP_MSG_TIMEOUT));
+ if (err < 0)
+ return err;
+
+ err = __hsmp_send_message(root, msg);
+ up(&(node_to_amd_nb(node)->hsmp_sem_lock));
+
+ return err;
+}
+EXPORT_SYMBOL_GPL(hsmp_send_message);
+
/*
* Data Fabric Indirect Access uses FICAA/FICAD.
*
@@ -307,6 +515,15 @@ int amd_cache_northbridges(void)
if (amd_gart_present())
amd_northbridges.flags |= AMD_NB_GART;
+ if (boot_cpu_data.x86 >= 0x19 && amd_hsmp_support()) {
+ amd_northbridges.flags |= AMD_NB_HSMP;
+
+ /* Protect the PCI config register pairs used for HSMP mailbox access, */
+ /* by initialising semaphore for each socket */
+ for (i = 0; i < amd_northbridges.num; i++)
+ sema_init(&(node_to_amd_nb(i)->hsmp_sem_lock), 1);
+ }
+
/*
* Check for L3 cache presence.
*/
--
2.17.1
On Thu, Sep 02, 2021 at 11:11:53PM +0530, Naveen Krishna Chatradhi wrote:
> On Fam19h server CPUs from AMD an HSMP (Host System Management Port)
> mailbox interface is added to monitor and manage the CPU parameters.
Mailbox? Hmm, that rings a bell. There's drivers/mailbox/ with a bunch
of drivers in there. Those are
"Mailbox is a framework to control hardware communication between
on-chip processors through queued messages and interrupt driven
signals."
I wonder if that HSMP thing can be wrapped like a mailbox driver...
If not, I still don't like it being slapped in amd_nb.c where it would
be built in on *all* AMD hardware which is kinda too much.
You could make it a separate driver module called amd_hsmp.ko which
loads only on the appropriate hw and uses amd_nb.c for detection only
like the other drivers, for example.
--
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquette
HI Boris,
On 9/2/2021 11:28 PM, Borislav Petkov wrote:
> [CAUTION: External Email]
>
> On Thu, Sep 02, 2021 at 11:11:53PM +0530, Naveen Krishna Chatradhi wrote:
>> On Fam19h server CPUs from AMD an HSMP (Host System Management Port)
>> mailbox interface is added to monitor and manage the CPU parameters.
> Mailbox? Hmm, that rings a bell. There's drivers/mailbox/ with a bunch
> of drivers in there. Those are
>
> "Mailbox is a framework to control hardware communication between
> on-chip processors through queued messages and interrupt driven
> signals."
>
> I wonder if that HSMP thing can be wrapped like a mailbox driver...
Sure, I will look into the mailbox framework.
>
> If not, I still don't like it being slapped in amd_nb.c where it would
> be built in on *all* AMD hardware which is kinda too much.
In all the future server platforms, AMD's direction is the support HSMP
interface, which exposes system management knobs.
>
> You could make it a separate driver module called amd_hsmp.ko which
> loads only on the appropriate hw and uses amd_nb.c for detection only
> like the other drivers, for example.
How about, creating a module under drivers/platform/x86/ (lets say
amd_hsmp.c) export an API from here and it can be extended to support
all the knobs that does not fit in an existing frameworks (such as
hwmon, etc) and provide a user space access.
I can see similar references in the drivers/platform/x86/ directory.
>
> --
> Regards/Gruss,
> Boris.
>
> https://nam11.safelinks.protection.outlook.com/?url=https%3A%2F%2Fpeople.kernel.org%2Ftglx%2Fnotes-about-netiquette&data=04%7C01%7CNaveenKrishna.Chatradhi%40amd.com%7C52aacf76c0d74c526ade08d96e3b3765%7C3dd8961fe4884e608e11a82d994e183d%7C0%7C0%7C637662022905004162%7CUnknown%7CTWFpbGZsb3d8eyJWIjoiMC4wLjAwMDAiLCJQIjoiV2luMzIiLCJBTiI6Ik1haWwiLCJXVCI6Mn0%3D%7C1000&sdata=DYcj%2BcjedWpUaJYMl7yPDy%2BB%2FIe6YDucA00JcJtfLFk%3D&reserved=0
On Wed, Sep 08, 2021 at 10:41:20PM +0530, Chatradhi, Naveen Krishna wrote:
> In all the future server platforms, AMD's direction is the support HSMP
> interface, which exposes system management knobs.
I know you all think about the future only but there's the past too and
there's a bunch of AMD hardware out there which doesn't have that.
> > You could make it a separate driver module called amd_hsmp.ko which
> > loads only on the appropriate hw and uses amd_nb.c for detection only
> > like the other drivers, for example.
>
> How about, creating a module under drivers/platform/x86/ (lets say
> amd_hsmp.c) export an API from here and it can be extended to support all
> the knobs that does not fit in an existing frameworks (such as hwmon, etc)
> and provide a user space access.
>
> I can see similar references in the drivers/platform/x86/ directory.
That sounds ok to me too. There's also arch/x86/platform/, btw, and I
still have to find out what the difference is. :-)
Lemme add the platform drivers folks.
Thread begins at:
https://lkml.kernel.org/r/[email protected]
Thx.
--
Regards/Gruss,
Boris.
https://people.kernel.org/tglx/notes-about-netiquette