v3->v2, fixed the issues Matthew Wilcox raised.
PCI Express ASPM defines a protocol for PCI Express components in the D0
state to reduce Link power by placing their Links into a low power state
and instructing the other end of the Link to do likewise. This
capability allows hardware-autonomous, dynamic Link power reduction
beyond what is achievable by software-only controlled power management.
However, The device should be configured by software appropriately.
Enabling ASPM will save power, but will introduce device latency.
This patch adds ASPM support in Linux. It introduces a global policy for
ASPM, a sysfs file /sys/module/pcie_aspm/parameters/policy can control
it. The interface can be used as a boot option too. Currently we have
below setting:
-default, BIOS default setting
-powersave, highest power saving mode, enable all available ASPM
state
and clock power management
-performance, highest performance, disable ASPM and clock power
management
By default, the 'default' policy is used currently.
In my test, power difference between powersave mode and performance mode
is about 1.3w in a system with 3 PCIE links.
please review, any comments will be appreciated.
Signed-off-by: Shaohua Li <[email protected]>
---
drivers/pci/pci-sysfs.c | 5
drivers/pci/pci.c | 4
drivers/pci/pcie/Kconfig | 20 +
drivers/pci/pcie/Makefile | 3
drivers/pci/pcie/aspm.c | 812 ++++++++++++++++++++++++++++++++++++++++++++++
drivers/pci/probe.c | 5
drivers/pci/remove.c | 4
include/linux/aspm.h | 44 ++
include/linux/pci.h | 4
include/linux/pci_regs.h | 8
10 files changed, 909 insertions(+)
Index: linux/drivers/pci/pcie/Makefile
===================================================================
--- linux.orig/drivers/pci/pcie/Makefile 2008-01-16 15:59:42.000000000 +0800
+++ linux/drivers/pci/pcie/Makefile 2008-01-18 09:41:20.000000000 +0800
@@ -2,6 +2,9 @@
# Makefile for PCI-Express PORT Driver
#
+# Build PCI Express ASPM if needed
+obj-$(CONFIG_PCIEASPM) += aspm.o
+
pcieportdrv-y := portdrv_core.o portdrv_pci.o portdrv_bus.o
obj-$(CONFIG_PCIEPORTBUS) += pcieportdrv.o
Index: linux/drivers/pci/pcie/aspm.c
===================================================================
--- /dev/null 1970-01-01 00:00:00.000000000 +0000
+++ linux/drivers/pci/pcie/aspm.c 2008-01-18 09:42:24.000000000 +0800
@@ -0,0 +1,812 @@
+/*
+ * File: drivers/pci/pcie/aspm.c
+ * Enabling PCIE link L0s/L1 state and Clock Power Management
+ *
+ * Copyright (C) 2007 Intel
+ * Copyright (C) Zhang Yanmin ([email protected])
+ * Copyright (C) Shaohua Li ([email protected])
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/pci.h>
+#include <linux/pci_regs.h>
+#include <linux/errno.h>
+#include <linux/pm.h>
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <linux/aspm.h>
+#include <acpi/acpi_bus.h>
+#include <linux/pci-acpi.h>
+#include "../pci.h"
+
+#ifdef MODULE_PARAM_PREFIX
+#undef MODULE_PARAM_PREFIX
+#endif
+#define MODULE_PARAM_PREFIX "pcie_aspm."
+
+/* only for downstream port */
+struct link_state {
+ struct list_head sibiling;
+ struct pci_dev *pdev;
+
+ /* ASPM state */
+ unsigned int support_state;
+ unsigned int enabled_state;
+ unsigned int bios_aspm_state;
+ /* upstream component */
+ unsigned int l0s_upper_latency;
+ unsigned int l1_upper_latency;
+ /* downstream component */
+ unsigned int l0s_down_latency;
+ unsigned int l1_down_latency;
+ /* Clock PM state*/
+ unsigned int clk_pm_capable;
+ unsigned int clk_pm_enabled;
+ unsigned int bios_clk_state;
+
+};
+
+/* Only for endpoint */
+struct endpoint_state {
+ unsigned int l0s_acceptable_latency;
+ unsigned int l1_acceptable_latency;
+};
+
+static int aspm_disabled;
+static DEFINE_MUTEX(aspm_lock);
+static LIST_HEAD(link_list);
+
+#define POLICY_DEFAULT 0 /* BIOS default setting */
+#define POLICY_PERFORMANCE 1 /* high performance */
+#define POLICY_POWERSAVE 2 /* high power saving */
+static int aspm_policy;
+static const char* policy_str[] = {
+ [POLICY_DEFAULT] = "default",
+ [POLICY_PERFORMANCE] = "performance",
+ [POLICY_POWERSAVE] = "powersave"
+};
+
+static int policy_to_aspm_state(struct pci_dev *pdev)
+{
+ struct link_state *link_state = pdev->link_state;
+
+ switch (aspm_policy) {
+ case POLICY_PERFORMANCE:
+ /* Disable ASPM and Clock PM */
+ return 0;
+ case POLICY_POWERSAVE:
+ /* Enable ASPM L0s/L1 */
+ return PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1;
+ case POLICY_DEFAULT:
+ return link_state->bios_aspm_state;
+ }
+ return 0;
+}
+
+static int policy_to_clkpm_state(struct pci_dev *pdev)
+{
+ struct link_state *link_state = pdev->link_state;
+
+ switch (aspm_policy) {
+ case POLICY_PERFORMANCE:
+ /* Disable ASPM and Clock PM */
+ return 0;
+ case POLICY_POWERSAVE:
+ /* Disable Clock PM */
+ return 1;
+ case POLICY_DEFAULT:
+ return link_state->bios_clk_state;
+ }
+ return 0;
+}
+
+static void pcie_set_clock_pm(struct pci_dev *pdev, int enable)
+{
+ struct pci_dev *child_dev;
+ int pos;
+ u16 reg16;
+ struct link_state *link_state = pdev->link_state;
+
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+ if (!pos)
+ return;
+ pci_read_config_word(child_dev, pos + PCI_EXP_LNKCTL, ®16);
+ if (enable)
+ reg16 |= PCI_EXP_LNKCTL_CLKREQ_EN;
+ else
+ reg16 &= ~PCI_EXP_LNKCTL_CLKREQ_EN;
+ pci_write_config_word(child_dev, pos + PCI_EXP_LNKCTL, reg16);
+ }
+ link_state->clk_pm_enabled = !!enable;
+}
+
+static void pcie_check_clock_pm(struct pci_dev *pdev)
+{
+ int pos;
+ u32 reg32;
+ u16 reg16;
+ int capable = 1, enabled = 1;
+ struct pci_dev *child_dev;
+ struct link_state *link_state = pdev->link_state;
+
+ /* All functions should have the same cap and state, take the worst */
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+ if (!pos)
+ return;
+ pci_read_config_dword(child_dev, pos + PCI_EXP_LNKCAP, ®32);
+ if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) {
+ capable = 0;
+ enabled = 0;
+ break;
+ }
+ pci_read_config_word(child_dev, pos + PCI_EXP_LNKCTL, ®16);
+ if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN))
+ enabled = 0;
+ }
+ link_state->clk_pm_capable = capable;
+ link_state->clk_pm_enabled = enabled;
+ link_state->bios_clk_state = enabled;
+ pcie_set_clock_pm(pdev, policy_to_clkpm_state(pdev));
+}
+
+/*
+ * pcie_aspm_configure_common_clock: check if the 2 ends of a link
+ * could use common clock. If they are, configure them to use the
+ * common clock. That will reduce the ASPM state exit latency.
+ */
+static void pcie_aspm_configure_common_clock(struct pci_dev *pdev)
+{
+ int pos, child_pos;
+ u16 reg16 = 0;
+ struct pci_dev *child_dev;
+ int same_clock = 1;
+
+ /*
+ * all functions of a slot should have the same Slot Clock
+ * Configuration, so just check one function
+ * */
+ child_dev = list_entry(pdev->subordinate->devices.next, struct pci_dev,
+ bus_list);
+ BUG_ON(!child_dev->is_pcie);
+
+ /* Check downstream component if bit Slot Clock Configuration is 1 */
+ child_pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+ pci_read_config_word(child_dev, child_pos + PCI_EXP_LNKSTA, ®16);
+ if (!(reg16 & PCI_EXP_LNKSTA_SLC))
+ same_clock = 0;
+
+ /* Check upstream component if bit Slot Clock Configuration is 1 */
+ pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
+ pci_read_config_word(pdev, pos + PCI_EXP_LNKSTA, ®16);
+ if (!(reg16 & PCI_EXP_LNKSTA_SLC))
+ same_clock = 0;
+
+ /* Configure downstream component, all functions */
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ child_pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+ pci_read_config_word(child_dev, child_pos + PCI_EXP_LNKCTL, ®16);
+ if (same_clock)
+ reg16 |= PCI_EXP_LNKCTL_CCC;
+ else
+ reg16 &= ~PCI_EXP_LNKCTL_CCC;
+ pci_write_config_word(child_dev, child_pos + PCI_EXP_LNKCTL, reg16);
+ }
+
+ /* Configure upstream component */
+ pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16);
+ if (same_clock)
+ reg16 |= PCI_EXP_LNKCTL_CCC;
+ else
+ reg16 &= ~PCI_EXP_LNKCTL_CCC;
+ pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16);
+
+ /* retrain link */
+ reg16 |= PCI_EXP_LNKCTL_RL;
+ pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16);
+
+ /* Wait for link training end */
+ while (1) {
+ pci_read_config_word(pdev, pos + PCI_EXP_LNKSTA, ®16);
+ if (!(reg16 & PCI_EXP_LNKSTA_LT))
+ break;
+ cpu_relax();
+ }
+}
+
+/*
+ * calc_L0S_latency: Convert L0s latency encoding to ns
+ */
+static unsigned int calc_L0S_latency(unsigned int latency_encoding, int ac)
+{
+ unsigned int ns = 64;
+
+ if (latency_encoding == 0x7) {
+ if (ac)
+ ns = -1U;
+ else
+ ns = 5*1000; /* > 4us */
+ } else
+ ns *= (1 << latency_encoding);
+ return ns;
+}
+
+/*
+ * calc_L1_latency: Convert L1 latency encoding to ns
+ */
+static unsigned int calc_L1_latency(unsigned int latency_encoding, int ac)
+{
+ unsigned int ns = 1000;
+
+ if (latency_encoding == 0x7) {
+ if (ac)
+ ns = -1U;
+ else
+ ns = 65*1000; /* > 64us */
+ } else
+ ns *= (1 << latency_encoding);
+ return ns;
+}
+
+static void pcie_aspm_get_cap_device(struct pci_dev *pdev, u32 *state,
+ unsigned int *l0s, unsigned int *l1, unsigned int *enabled)
+{
+ int pos;
+ u16 reg16;
+ u32 reg32;
+ unsigned int latency;
+
+ pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
+ pci_read_config_dword(pdev, pos + PCI_EXP_LNKCAP, ®32);
+ *state = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10;
+ if (*state != PCIE_LINK_STATE_L0S && *state != (PCIE_LINK_STATE_L1|PCIE_LINK_STATE_L0S))
+ *state = 0;
+ if (*state == 0)
+ return;
+
+ latency = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12;
+ *l0s = calc_L0S_latency(latency, 0);
+ if (*state & PCIE_LINK_STATE_L1) {
+ latency = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15;
+ *l1 = calc_L1_latency(latency, 0);
+ }
+ pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16);
+ *enabled = reg16 & (PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1);
+}
+
+static void pcie_aspm_cap_init(struct pci_dev *pdev)
+{
+ struct pci_dev *child_dev;
+ u32 state, tmp;
+ struct link_state *link_state = pdev->link_state;
+
+ /* upstream component states */
+ pcie_aspm_get_cap_device(pdev, &link_state->support_state,
+ &link_state->l0s_upper_latency,
+ &link_state->l1_upper_latency,
+ &link_state->enabled_state);
+ /* downstream component states, all functions have the same setting */
+ child_dev = list_entry(pdev->subordinate->devices.next, struct pci_dev,
+ bus_list);
+ pcie_aspm_get_cap_device(child_dev, &state,
+ &link_state->l0s_down_latency,
+ &link_state->l1_down_latency,
+ &tmp);
+ link_state->support_state &= state;
+ if (!link_state->support_state)
+ return;
+ link_state->enabled_state &= link_state->support_state;
+ link_state->bios_aspm_state = link_state->enabled_state;
+
+ /* ENDPOINT states*/
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ int pos;
+ u32 reg32;
+ unsigned int latency;
+ struct endpoint_state *ep_state;
+
+ if (child_dev->pcie_type != PCI_EXP_TYPE_ENDPOINT &&
+ child_dev->pcie_type != PCI_EXP_TYPE_LEG_END)
+ continue;
+
+ ep_state = child_dev->link_state;
+ pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+ pci_read_config_dword(child_dev, pos + PCI_EXP_DEVCAP, ®32);
+ latency = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6;
+ latency = calc_L0S_latency(latency, 1);
+ ep_state->l0s_acceptable_latency = latency;
+ if (link_state->support_state & PCIE_LINK_STATE_L1) {
+ latency = (reg32 & PCI_EXP_DEVCAP_L1) >> 9;
+ latency = calc_L1_latency(latency, 1);
+ ep_state->l1_acceptable_latency = latency;
+ }
+ }
+}
+
+static unsigned int __pcie_aspm_check_state_one(struct pci_dev *pdev,
+ unsigned int state)
+{
+ struct pci_dev *parent_dev, *tmp_dev;
+ unsigned int latency, l1_latency = 0;
+ struct link_state *link_state;
+ struct endpoint_state *ep_state = pdev->link_state;
+
+ parent_dev = pdev->bus->self;
+ link_state = parent_dev->link_state;
+ state &= link_state->support_state;
+ if (state == 0)
+ return 0;
+
+ /*
+ * Check latency for endpoint device.
+ * TBD: The latency from the endpoint to root complex vary per
+ * switch's upstream link state above the device. Here we just do a
+ * simple check which assumes all links above the device can be in L1
+ * state, that is we just consider the worst case. If switch's upstream
+ * link can't be put into L0S/L1, then our check is too strictly.
+ */
+ tmp_dev = pdev;
+ while (state & (PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1)) {
+ parent_dev = tmp_dev->bus->self;
+ link_state = parent_dev->link_state;
+ if (state & PCIE_LINK_STATE_L0S) {
+ latency = max_t(unsigned int,
+ link_state->l0s_upper_latency,
+ link_state->l0s_down_latency);
+ if (latency > ep_state->l0s_acceptable_latency)
+ state &= ~PCIE_LINK_STATE_L0S;
+ }
+ if (state & PCIE_LINK_STATE_L1) {
+ latency = max_t(unsigned int,
+ link_state->l1_upper_latency,
+ link_state->l1_down_latency);
+ if (latency + l1_latency >
+ ep_state->l1_acceptable_latency)
+ state &= ~PCIE_LINK_STATE_L1;
+ }
+ if (!parent_dev->bus->self) /* parent_dev is a root port */
+ break;
+ else {
+ /*
+ * parent_dev is the downstream port of a switch, make
+ * tmp_dev the upstream port of the switch
+ */
+ tmp_dev = parent_dev->bus->self;
+ /*
+ * every switch on the path to root complex need 1 more
+ * microsecond for L1. Spec doesn't mention L0S.
+ */
+ if (state & PCIE_LINK_STATE_L1)
+ l1_latency += 1000;
+ }
+ }
+ return state;
+}
+
+static unsigned int pcie_aspm_check_state(struct pci_dev *pdev,
+ unsigned int state)
+{
+ struct pci_dev *child_dev;
+
+ /* If no child, disable the link */
+ if (list_empty(&pdev->subordinate->devices))
+ return 0;
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ if (child_dev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE) {
+ /*
+ * If downstream component of a link is pci bridge, we
+ * disable ASPM for now for the link
+ * */
+ state = 0;
+ break;
+ }
+ if ((child_dev->pcie_type != PCI_EXP_TYPE_ENDPOINT &&
+ child_dev->pcie_type != PCI_EXP_TYPE_LEG_END) ||
+ !child_dev->link_state)
+ continue;
+ /* Device not in D0 doesn't need check latency */
+ if (child_dev->current_state == PCI_D1 ||
+ child_dev->current_state == PCI_D2 ||
+ child_dev->current_state == PCI_D3hot ||
+ child_dev->current_state == PCI_D3cold)
+ continue;
+ state = __pcie_aspm_check_state_one(child_dev, state);
+ }
+ return state;
+}
+
+static void __pcie_aspm_config_one_dev(struct pci_dev *pdev, unsigned int state)
+{
+ u16 reg16;
+ int pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
+
+ pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16);
+ reg16 &= ~0x3;
+ reg16 |= state;
+ pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16);
+}
+
+static void __pcie_aspm_config_link(struct pci_dev *pdev, unsigned int state)
+{
+ struct pci_dev *child_dev;
+ int valid = 1;
+ struct link_state *link_state = pdev->link_state;
+
+ /*
+ * if the downstream component has pci bridge function, don't do ASPM
+ * now
+ */
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ if (child_dev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE){
+ valid = 0;
+ break;
+ }
+ }
+ if (!valid)
+ return;
+
+ /*
+ * spec 2.0 suggests all functions should be configured the same
+ * setting for ASPM. Enabling ASPM L1 should be done in upstream
+ * component first and then downstream, and vice versa for disabling
+ * ASPM L1. Spec doesn't mention L0S.
+ */
+ if (state & PCIE_LINK_STATE_L1)
+ __pcie_aspm_config_one_dev(pdev, state);
+
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list)
+ __pcie_aspm_config_one_dev(child_dev, state);
+
+ if (!(state & PCIE_LINK_STATE_L1))
+ __pcie_aspm_config_one_dev(pdev, state);
+
+ link_state->enabled_state = state;
+}
+
+static void __pcie_aspm_configure_link_state(struct pci_dev *pdev, unsigned int state)
+{
+ struct link_state *link_state = pdev->link_state;
+
+ if (link_state->support_state == 0)
+ return;
+ state &= PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1;
+
+ /* state 0 means disabling aspm */
+ state = pcie_aspm_check_state(pdev, state);
+ if (link_state->enabled_state == state)
+ return;
+ __pcie_aspm_config_link(pdev, state);
+}
+
+/*
+ * pcie_aspm_configure_link_state: enable/disable PCI express link state
+ * @pdev: the root port or switch downstream port
+ */
+void pcie_aspm_configure_link_state(struct pci_dev *pdev, unsigned int state)
+{
+ down_read(&pci_bus_sem);
+ mutex_lock(&aspm_lock);
+ __pcie_aspm_configure_link_state(pdev, state);
+ mutex_unlock(&aspm_lock);
+ up_read(&pci_bus_sem);
+}
+
+static void free_link_state(struct pci_dev *pdev)
+{
+ struct pci_dev *child_dev;
+
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ kfree(child_dev->link_state);
+ child_dev->link_state = NULL;
+ }
+ kfree(pdev->link_state);
+ pdev->link_state = NULL;
+}
+
+/*
+ * pcie_aspm_init_link_state: Initiate PCI express link state.
+ * It is called after the pcie and its children devices are scaned.
+ * @pdev: the root port or switch downstream port
+ */
+void pcie_aspm_init_link_state(struct pci_dev *pdev)
+{
+ unsigned int state;
+ struct link_state *link_state;
+ struct pci_dev *child_dev;
+ struct endpoint_state *ep_state;
+ int error = 0;
+
+ if (aspm_disabled || !pdev->is_pcie || pdev->link_state)
+ return;
+ if (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
+ pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
+ return;
+ down_read(&pci_bus_sem);
+ if (list_empty(&pdev->subordinate->devices))
+ goto out;
+
+ mutex_lock(&aspm_lock);
+
+ link_state = kzalloc(sizeof(*link_state), GFP_KERNEL);
+ if (!link_state)
+ goto unlock_out;
+ pdev->link_state = link_state;
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ ep_state = kzalloc(sizeof(*ep_state), GFP_KERNEL);
+ if (!ep_state) {
+ error = 1;
+ goto unlock_out;
+ }
+ child_dev->link_state = ep_state;
+ }
+
+ pcie_aspm_configure_common_clock(pdev);
+
+ pcie_aspm_cap_init(pdev);
+
+ /* config link state to avoid BIOS error */
+ state = pcie_aspm_check_state(pdev, policy_to_aspm_state(pdev));
+ __pcie_aspm_config_link(pdev, state);
+
+ pcie_check_clock_pm(pdev);
+
+ link_state->pdev = pdev;
+ list_add(&link_state->sibiling, &link_list);
+
+unlock_out:
+ if (error)
+ free_link_state(pdev);
+ mutex_unlock(&aspm_lock);
+out:
+ up_read(&pci_bus_sem);
+}
+
+/* @pdev: the endpoint device */
+void pcie_aspm_exit_link_state(struct pci_dev *pdev)
+{
+ struct pci_dev *parent = pdev->bus->self;
+ struct link_state *link_state = parent->link_state;
+
+ if (aspm_disabled || !pdev->is_pcie || !parent || !link_state)
+ return;
+ if (parent->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
+ parent->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
+ return;
+ down_read(&pci_bus_sem);
+ mutex_lock(&aspm_lock);
+
+ /*
+ * All PCIe functions are in one slot, remove one function will remove
+ * the the whole slot, so just wait
+ */
+ if (!list_empty(&parent->subordinate->devices)) {
+ kfree(pdev->link_state);
+ pdev->link_state = NULL;
+ goto out;
+ }
+
+ /* All functions are removed, so just disable ASPM for the link */
+ __pcie_aspm_config_one_dev(parent, 0);
+ list_del(&link_state->sibiling);
+ /* Clock PM is for endpoint device */
+
+ free_link_state(parent);
+out:
+ mutex_unlock(&aspm_lock);
+ up_read(&pci_bus_sem);
+}
+
+/* @pdev: the root port or switch downstream port */
+void pcie_aspm_pm_state_change(struct pci_dev *pdev)
+{
+ struct link_state *link_state = pdev->link_state;
+
+ if (aspm_disabled || !pdev->is_pcie || !pdev->link_state)
+ return;
+ if (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
+ pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
+ return;
+ /*
+ * devices changed PM state, we should recheck if latency meets all
+ * functions' requirement
+ */
+ pcie_aspm_configure_link_state(pdev, link_state->enabled_state);
+}
+
+/*
+ * pci_disable_link_state - disable pci device's link state, so the link will
+ * never enter specific states
+ */
+void pci_disable_link_state(struct pci_dev *pdev, int state)
+{
+ struct pci_dev *parent = pdev->bus->self;
+ struct link_state *link_state;
+
+ if (aspm_disabled || !pdev->is_pcie || !pdev->link_state)
+ return;
+ if (pdev->pcie_type == PCI_EXP_TYPE_ROOT_PORT ||
+ pdev->pcie_type == PCI_EXP_TYPE_DOWNSTREAM)
+ parent = pdev;
+ if (!parent)
+ return;
+
+ down_read(&pci_bus_sem);
+ mutex_lock(&aspm_lock);
+ link_state = parent->link_state;
+ link_state->support_state &= ~(state & (PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1));
+ if (state & PCIE_LINK_STATE_CLKPM)
+ link_state->clk_pm_capable = 0;
+
+ __pcie_aspm_configure_link_state(parent, link_state->enabled_state);
+ if (!link_state->clk_pm_capable && link_state->clk_pm_enabled)
+ pcie_set_clock_pm(parent, 0);
+ mutex_unlock(&aspm_lock);
+ up_read(&pci_bus_sem);
+}
+EXPORT_SYMBOL(pci_disable_link_state);
+
+static int pcie_aspm_set_policy(const char *val, struct kernel_param *kp)
+{
+ int i;
+ struct pci_dev *pdev;
+ struct link_state *link_state;
+
+ for (i = 0; i < ARRAY_SIZE(policy_str); i++)
+ if (!strncmp(val, policy_str[i], strlen(policy_str[i])))
+ break;
+ if (i >= ARRAY_SIZE(policy_str))
+ return -EINVAL;
+ if (i == aspm_policy)
+ return 0;
+
+ down_read(&pci_bus_sem);
+ mutex_lock(&aspm_lock);
+ aspm_policy = i;
+ list_for_each_entry(link_state, &link_list, sibiling) {
+ pdev = link_state->pdev;
+ __pcie_aspm_configure_link_state(pdev,
+ policy_to_aspm_state(pdev));
+ if (link_state->clk_pm_capable &&
+ link_state->clk_pm_enabled != policy_to_clkpm_state(pdev))
+ pcie_set_clock_pm(pdev, policy_to_clkpm_state(pdev));
+
+ }
+ mutex_unlock(&aspm_lock);
+ up_read(&pci_bus_sem);
+ return 0;
+}
+
+static int pcie_aspm_get_policy(char *buffer, struct kernel_param *kp)
+{
+ int i, cnt = 0;
+ for (i = 0; i < ARRAY_SIZE(policy_str); i++)
+ if (i == aspm_policy)
+ cnt += sprintf(buffer + cnt, "[%s] ", policy_str[i]);
+ else
+ cnt += sprintf(buffer + cnt, "%s ", policy_str[i]);
+ return cnt;
+}
+
+module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy,
+ NULL, 0644);
+
+#ifdef CONFIG_PCIEASPM_DEBUG
+static ssize_t link_state_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct pci_dev *pci_device = to_pci_dev(dev);
+ struct link_state *link_state = pci_device->link_state;
+
+ return sprintf(buf, "%d\n", link_state->enabled_state);
+}
+
+static ssize_t link_state_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf,
+ size_t n)
+{
+ struct pci_dev *pci_device = to_pci_dev(dev);
+ int state;
+
+ if (n < 1)
+ return -EINVAL;
+ state = buf[0]-'0';
+ if (state >= 0 && state <= 3) {
+ /* setup link aspm state */
+ pcie_aspm_configure_link_state(pci_device, state);
+ return n;
+ }
+
+ return -EINVAL;
+}
+
+static ssize_t clk_ctl_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct pci_dev *pci_device = to_pci_dev(dev);
+ struct link_state *link_state = pci_device->link_state;
+
+ return sprintf(buf, "%d\n", link_state->clk_pm_enabled);
+}
+
+static ssize_t clk_ctl_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf,
+ size_t n)
+{
+ struct pci_dev *pci_device = to_pci_dev(dev);
+ int state;
+
+ if (n < 1)
+ return -EINVAL;
+ state = buf[0]-'0';
+
+ down_read(&pci_bus_sem);
+ mutex_lock(&aspm_lock);
+ pcie_set_clock_pm(pci_device, !!state);
+ mutex_unlock(&aspm_lock);
+ up_read(&pci_bus_sem);
+
+ return n;
+}
+
+static DEVICE_ATTR(link_state, 0644, link_state_show, link_state_store);
+static DEVICE_ATTR(clk_ctl, 0644, clk_ctl_show, clk_ctl_store);
+
+static char power_group[] = "power";
+void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev)
+{
+ struct link_state *link_state = pdev->link_state;
+
+ if (!pdev->is_pcie || (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
+ pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM))
+ return;
+
+ if (link_state->support_state)
+ sysfs_add_file_to_group(&pdev->dev.kobj,
+ &dev_attr_link_state.attr, power_group);
+ if (link_state->clk_pm_capable)
+ sysfs_add_file_to_group(&pdev->dev.kobj,
+ &dev_attr_clk_ctl.attr, power_group);
+}
+
+void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev)
+{
+ struct link_state *link_state = pdev->link_state;
+
+ if (!pdev->is_pcie || (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
+ pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM))
+ return;
+
+ if (link_state->support_state)
+ sysfs_remove_file_from_group(&pdev->dev.kobj,
+ &dev_attr_link_state.attr, power_group);
+ if (link_state->clk_pm_capable)
+ sysfs_remove_file_from_group(&pdev->dev.kobj,
+ &dev_attr_clk_ctl.attr, power_group);
+}
+#endif
+
+static int __init pcie_aspm_disable(char *str)
+{
+ aspm_disabled = 1;
+ return 1;
+}
+
+__setup("pcie_noaspm", pcie_aspm_disable);
+
+static int __init pcie_aspm_init(void)
+{
+ if (aspm_disabled)
+ return 0;
+ pci_osc_support_set(OSC_ACTIVE_STATE_PWR_SUPPORT|
+ OSC_CLOCK_PWR_CAPABILITY_SUPPORT);
+ return 0;
+}
+
+fs_initcall(pcie_aspm_init);
Index: linux/include/linux/aspm.h
===================================================================
--- /dev/null 1970-01-01 00:00:00.000000000 +0000
+++ linux/include/linux/aspm.h 2008-01-18 09:43:02.000000000 +0800
@@ -0,0 +1,44 @@
+/*
+ * aspm.h
+ *
+ * PCI Express ASPM defines and function prototypes
+ *
+ * Copyright (C) 2007 Intel Corp.
+ * Zhang Yanmin ([email protected])
+ * Shaohua Li ([email protected])
+ *
+ * For more information, please consult the following manuals (look at
+ * http://www.pcisig.com/ for how to get them):
+ *
+ * PCI Express Specification
+ */
+
+#ifndef LINUX_ASPM_H
+#define LINUX_ASPM_H
+
+#include <linux/pci.h>
+
+#define PCIE_LINK_STATE_L0S 1
+#define PCIE_LINK_STATE_L1 2
+#define PCIE_LINK_STATE_CLKPM 4
+
+#ifdef CONFIG_PCIEASPM
+extern void pcie_aspm_init_link_state(struct pci_dev *pdev);
+extern void pcie_aspm_exit_link_state(struct pci_dev *pdev);
+extern void pcie_aspm_pm_state_change(struct pci_dev *pdev);
+extern void pci_disable_link_state(struct pci_dev *pdev, int state);
+#else
+#define pcie_aspm_init_link_state(pdev) do {} while (0)
+#define pcie_aspm_exit_link_state(pdev) do {} while (0)
+#define pcie_aspm_pm_state_change(pdev) do {} while (0)
+#define pci_disable_link_state(pdev, state) do {} while (0)
+#endif
+
+#ifdef CONFIG_PCIEASPM_DEBUG /* this depends on CONFIG_PCIEASPM */
+extern void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev);
+extern void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev);
+#else
+#define pcie_aspm_create_sysfs_dev_files(pdev) do {} while (0)
+#define pcie_aspm_remove_sysfs_dev_files(pdev) do {} while (0)
+#endif
+#endif /* LINUX_ASPM_H */
Index: linux/drivers/pci/pcie/Kconfig
===================================================================
--- linux.orig/drivers/pci/pcie/Kconfig 2008-01-16 15:59:42.000000000 +0800
+++ linux/drivers/pci/pcie/Kconfig 2008-01-18 09:41:20.000000000 +0800
@@ -26,3 +26,23 @@ config HOTPLUG_PCI_PCIE
When in doubt, say N.
source "drivers/pci/pcie/aer/Kconfig"
+
+#
+# PCI Express ASPM
+#
+config PCIEASPM
+ bool "PCI Express ASPM support(Experimental)"
+ depends on PCI && EXPERIMENTAL
+ default y
+ help
+ This enables PCI Express ASPM (Active State Power Management) and
+ Clock Power Management. ASPM supports state L0/L0s/L1.
+
+ When in doubt, say N.
+config PCIEASPM_DEBUG
+ bool "Debug PCI Express ASPM"
+ depends on PCIEASPM
+ default n
+ help
+ This enables PCI Express ASPM debug support. It will add per-device
+ interface to control ASPM.
Index: linux/include/linux/pci.h
===================================================================
--- linux.orig/include/linux/pci.h 2008-01-16 15:59:42.000000000 +0800
+++ linux/include/linux/pci.h 2008-01-18 09:41:20.000000000 +0800
@@ -164,6 +164,10 @@ struct pci_dev {
this is D0-D3, D0 being fully functional,
and D3 being off. */
+#ifdef CONFIG_PCIEASPM
+ void *link_state; /* ASPM link state. */
+#endif
+
pci_channel_state_t error_state; /* current connectivity state */
struct device dev; /* Generic device interface */
Index: linux/include/linux/pci_regs.h
===================================================================
--- linux.orig/include/linux/pci_regs.h 2008-01-16 15:59:42.000000000 +0800
+++ linux/include/linux/pci_regs.h 2008-01-18 09:41:20.000000000 +0800
@@ -395,9 +395,17 @@
#define PCI_EXP_DEVSTA_AUXPD 0x10 /* AUX Power Detected */
#define PCI_EXP_DEVSTA_TRPND 0x20 /* Transactions Pending */
#define PCI_EXP_LNKCAP 12 /* Link Capabilities */
+#define PCI_EXP_LNKCAP_ASPMS 0xc00 /* ASPM Support */
+#define PCI_EXP_LNKCAP_L0SEL 0x7000 /* L0s Exit Latency */
+#define PCI_EXP_LNKCAP_L1EL 0x38000 /* L1 Exit Latency */
+#define PCI_EXP_LNKCAP_CLKPM 0x40000 /* L1 Clock Power Management */
#define PCI_EXP_LNKCTL 16 /* Link Control */
+#define PCI_EXP_LNKCTL_RL 0x20 /* Retrain Link */
+#define PCI_EXP_LNKCTL_CCC 0x40 /* Common Clock COnfiguration */
#define PCI_EXP_LNKCTL_CLKREQ_EN 0x100 /* Enable clkreq */
#define PCI_EXP_LNKSTA 18 /* Link Status */
+#define PCI_EXP_LNKSTA_LT 0x800 /* Link Training */
+#define PCI_EXP_LNKSTA_SLC 0x1000 /* Slot Clock Configuration */
#define PCI_EXP_SLTCAP 20 /* Slot Capabilities */
#define PCI_EXP_SLTCTL 24 /* Slot Control */
#define PCI_EXP_SLTSTA 26 /* Slot Status */
Index: linux/drivers/pci/probe.c
===================================================================
--- linux.orig/drivers/pci/probe.c 2008-01-16 15:59:42.000000000 +0800
+++ linux/drivers/pci/probe.c 2008-01-18 09:41:20.000000000 +0800
@@ -9,6 +9,7 @@
#include <linux/slab.h>
#include <linux/module.h>
#include <linux/cpumask.h>
+#include <linux/aspm.h>
#include "pci.h"
#define CARDBUS_LATENCY_TIMER 176 /* secondary latency timer */
@@ -1011,6 +1012,10 @@ int pci_scan_slot(struct pci_bus *bus, i
break;
}
}
+
+ if (bus->self)
+ pcie_aspm_init_link_state(bus->self);
+
return nr;
}
Index: linux/drivers/pci/remove.c
===================================================================
--- linux.orig/drivers/pci/remove.c 2008-01-16 15:59:42.000000000 +0800
+++ linux/drivers/pci/remove.c 2008-01-18 09:41:20.000000000 +0800
@@ -1,5 +1,6 @@
#include <linux/pci.h>
#include <linux/module.h>
+#include <linux/aspm.h>
#include "pci.h"
static void pci_free_resources(struct pci_dev *dev)
@@ -30,6 +31,9 @@ static void pci_stop_dev(struct pci_dev
dev->global_list.next = dev->global_list.prev = NULL;
up_write(&pci_bus_sem);
}
+
+ if (dev->bus->self)
+ pcie_aspm_exit_link_state(dev);
}
static void pci_destroy_dev(struct pci_dev *dev)
Index: linux/drivers/pci/pci.c
===================================================================
--- linux.orig/drivers/pci/pci.c 2008-01-18 09:41:18.000000000 +0800
+++ linux/drivers/pci/pci.c 2008-01-18 09:41:20.000000000 +0800
@@ -18,6 +18,7 @@
#include <linux/spinlock.h>
#include <linux/string.h>
#include <linux/log2.h>
+#include <linux/aspm.h>
#include <asm/dma.h> /* isa_dma_bridge_buggy */
#include "pci.h"
@@ -501,6 +502,9 @@ pci_set_power_state(struct pci_dev *dev,
if (need_restore)
pci_restore_bars(dev);
+ if (dev->bus->self)
+ pcie_aspm_pm_state_change(dev->bus->self);
+
return 0;
}
Index: linux/drivers/pci/pci-sysfs.c
===================================================================
--- linux.orig/drivers/pci/pci-sysfs.c 2008-01-16 15:59:42.000000000 +0800
+++ linux/drivers/pci/pci-sysfs.c 2008-01-18 09:41:20.000000000 +0800
@@ -21,6 +21,7 @@
#include <linux/topology.h>
#include <linux/mm.h>
#include <linux/capability.h>
+#include <linux/aspm.h>
#include "pci.h"
static int sysfs_initialized; /* = 0 */
@@ -650,6 +651,8 @@ int __must_check pci_create_sysfs_dev_fi
if (pcibios_add_platform_entries(pdev))
goto err_rom_file;
+ pcie_aspm_create_sysfs_dev_files(pdev);
+
return 0;
err_rom_file:
@@ -679,6 +682,8 @@ void pci_remove_sysfs_dev_files(struct p
if (!sysfs_initialized)
return;
+ pcie_aspm_remove_sysfs_dev_files(pdev);
+
if (pdev->cfg_size < 4096)
sysfs_remove_bin_file(&pdev->dev.kobj, &pci_config_attr);
else
On Fri, Jan 18, 2008 at 09:56:28AM +0800, Shaohua Li wrote:
> v3->v2, fixed the issues Matthew Wilcox raised.
>
> PCI Express ASPM defines a protocol for PCI Express components in the D0
> state to reduce Link power by placing their Links into a low power state
> and instructing the other end of the Link to do likewise. This
> capability allows hardware-autonomous, dynamic Link power reduction
> beyond what is achievable by software-only controlled power management.
> However, The device should be configured by software appropriately.
> Enabling ASPM will save power, but will introduce device latency.
>
> This patch adds ASPM support in Linux. It introduces a global policy for
> ASPM, a sysfs file /sys/module/pcie_aspm/parameters/policy can control
> it. The interface can be used as a boot option too. Currently we have
> below setting:
> -default, BIOS default setting
> -powersave, highest power saving mode, enable all available ASPM
> state
> and clock power management
> -performance, highest performance, disable ASPM and clock power
> management
> By default, the 'default' policy is used currently.
>
> In my test, power difference between powersave mode and performance mode
> is about 1.3w in a system with 3 PCIE links.
>
> please review, any comments will be appreciated.
Can you please fix up all of the warnings that checkpatch.pl and sparse
produce from this patch?
Also, one small thing:
> --- linux.orig/include/linux/pci.h 2008-01-16 15:59:42.000000000 +0800
> +++ linux/include/linux/pci.h 2008-01-18 09:41:20.000000000 +0800
> @@ -164,6 +164,10 @@ struct pci_dev {
> this is D0-D3, D0 being fully functional,
> and D3 being off. */
>
> +#ifdef CONFIG_PCIEASPM
> + void *link_state; /* ASPM link state. */
> +#endif
Can we make this a "real" pointer to a structure? I note that you use
two different structures here in this pointer, should you really do
that? It's good to get type-checks whereever possible.
thanks,
greg k-h
On Tue, 2008-01-22 at 14:58 -0800, Greg KH wrote:
> On Fri, Jan 18, 2008 at 09:56:28AM +0800, Shaohua Li wrote:
> > v3->v2, fixed the issues Matthew Wilcox raised.
> >
> > PCI Express ASPM defines a protocol for PCI Express components in the D0
> > state to reduce Link power by placing their Links into a low power state
> > and instructing the other end of the Link to do likewise. This
> > capability allows hardware-autonomous, dynamic Link power reduction
> > beyond what is achievable by software-only controlled power management.
> > However, The device should be configured by software appropriately.
> > Enabling ASPM will save power, but will introduce device latency.
> >
> > This patch adds ASPM support in Linux. It introduces a global policy for
> > ASPM, a sysfs file /sys/module/pcie_aspm/parameters/policy can control
> > it. The interface can be used as a boot option too. Currently we have
> > below setting:
> > -default, BIOS default setting
> > -powersave, highest power saving mode, enable all available ASPM
> > state
> > and clock power management
> > -performance, highest performance, disable ASPM and clock power
> > management
> > By default, the 'default' policy is used currently.
> >
> > In my test, power difference between powersave mode and performance mode
> > is about 1.3w in a system with 3 PCIE links.
> >
> > please review, any comments will be appreciated.
>
> Can you please fix up all of the warnings that checkpatch.pl and sparse
> produce from this patch?
>
> Also, one small thing:
>
> > --- linux.orig/include/linux/pci.h 2008-01-16 15:59:42.000000000 +0800
> > +++ linux/include/linux/pci.h 2008-01-18 09:41:20.000000000 +0800
> > @@ -164,6 +164,10 @@ struct pci_dev {
> > this is D0-D3, D0 being fully functional,
> > and D3 being off. */
> >
> > +#ifdef CONFIG_PCIEASPM
> > + void *link_state; /* ASPM link state. */
> > +#endif
>
> Can we make this a "real" pointer to a structure? I note that you use
> two different structures here in this pointer, should you really do
> that? It's good to get type-checks whereever possible.
The structure is just for internal use of ASPM, just don't want make it
global.
Fixed, now sparse and checkpatch.pl haven't warning.
Signed-off-by: Shaohua Li <[email protected]>
---
drivers/pci/pci-sysfs.c | 5
drivers/pci/pci.c | 4
drivers/pci/pcie/Kconfig | 20 +
drivers/pci/pcie/Makefile | 3
drivers/pci/pcie/aspm.c | 818 ++++++++++++++++++++++++++++++++++++++++++++++
drivers/pci/probe.c | 5
drivers/pci/remove.c | 4
include/linux/aspm.h | 44 ++
include/linux/pci.h | 4
include/linux/pci_regs.h | 8
10 files changed, 915 insertions(+)
Index: linux/drivers/pci/pcie/Makefile
===================================================================
--- linux.orig/drivers/pci/pcie/Makefile 2008-01-23 10:14:14.000000000 +0800
+++ linux/drivers/pci/pcie/Makefile 2008-01-23 10:14:46.000000000 +0800
@@ -2,6 +2,9 @@
# Makefile for PCI-Express PORT Driver
#
+# Build PCI Express ASPM if needed
+obj-$(CONFIG_PCIEASPM) += aspm.o
+
pcieportdrv-y := portdrv_core.o portdrv_pci.o portdrv_bus.o
obj-$(CONFIG_PCIEPORTBUS) += pcieportdrv.o
Index: linux/drivers/pci/pcie/aspm.c
===================================================================
--- /dev/null 1970-01-01 00:00:00.000000000 +0000
+++ linux/drivers/pci/pcie/aspm.c 2008-01-23 10:14:46.000000000 +0800
@@ -0,0 +1,818 @@
+/*
+ * File: drivers/pci/pcie/aspm.c
+ * Enabling PCIE link L0s/L1 state and Clock Power Management
+ *
+ * Copyright (C) 2007 Intel
+ * Copyright (C) Zhang Yanmin ([email protected])
+ * Copyright (C) Shaohua Li ([email protected])
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/pci.h>
+#include <linux/pci_regs.h>
+#include <linux/errno.h>
+#include <linux/pm.h>
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <linux/aspm.h>
+#include <acpi/acpi_bus.h>
+#include <linux/pci-acpi.h>
+#include "../pci.h"
+
+#ifdef MODULE_PARAM_PREFIX
+#undef MODULE_PARAM_PREFIX
+#endif
+#define MODULE_PARAM_PREFIX "pcie_aspm."
+
+/* only for downstream port */
+struct link_state {
+ struct list_head sibiling;
+ struct pci_dev *pdev;
+
+ /* ASPM state */
+ unsigned int support_state;
+ unsigned int enabled_state;
+ unsigned int bios_aspm_state;
+ /* upstream component */
+ unsigned int l0s_upper_latency;
+ unsigned int l1_upper_latency;
+ /* downstream component */
+ unsigned int l0s_down_latency;
+ unsigned int l1_down_latency;
+ /* Clock PM state*/
+ unsigned int clk_pm_capable;
+ unsigned int clk_pm_enabled;
+ unsigned int bios_clk_state;
+
+};
+
+/* Only for endpoint */
+struct endpoint_state {
+ unsigned int l0s_acceptable_latency;
+ unsigned int l1_acceptable_latency;
+};
+
+static int aspm_disabled;
+static DEFINE_MUTEX(aspm_lock);
+static LIST_HEAD(link_list);
+
+#define POLICY_DEFAULT 0 /* BIOS default setting */
+#define POLICY_PERFORMANCE 1 /* high performance */
+#define POLICY_POWERSAVE 2 /* high power saving */
+static int aspm_policy;
+static const char *policy_str[] = {
+ [POLICY_DEFAULT] = "default",
+ [POLICY_PERFORMANCE] = "performance",
+ [POLICY_POWERSAVE] = "powersave"
+};
+
+static int policy_to_aspm_state(struct pci_dev *pdev)
+{
+ struct link_state *link_state = pdev->link_state;
+
+ switch (aspm_policy) {
+ case POLICY_PERFORMANCE:
+ /* Disable ASPM and Clock PM */
+ return 0;
+ case POLICY_POWERSAVE:
+ /* Enable ASPM L0s/L1 */
+ return PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1;
+ case POLICY_DEFAULT:
+ return link_state->bios_aspm_state;
+ }
+ return 0;
+}
+
+static int policy_to_clkpm_state(struct pci_dev *pdev)
+{
+ struct link_state *link_state = pdev->link_state;
+
+ switch (aspm_policy) {
+ case POLICY_PERFORMANCE:
+ /* Disable ASPM and Clock PM */
+ return 0;
+ case POLICY_POWERSAVE:
+ /* Disable Clock PM */
+ return 1;
+ case POLICY_DEFAULT:
+ return link_state->bios_clk_state;
+ }
+ return 0;
+}
+
+static void pcie_set_clock_pm(struct pci_dev *pdev, int enable)
+{
+ struct pci_dev *child_dev;
+ int pos;
+ u16 reg16;
+ struct link_state *link_state = pdev->link_state;
+
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+ if (!pos)
+ return;
+ pci_read_config_word(child_dev, pos + PCI_EXP_LNKCTL, ®16);
+ if (enable)
+ reg16 |= PCI_EXP_LNKCTL_CLKREQ_EN;
+ else
+ reg16 &= ~PCI_EXP_LNKCTL_CLKREQ_EN;
+ pci_write_config_word(child_dev, pos + PCI_EXP_LNKCTL, reg16);
+ }
+ link_state->clk_pm_enabled = !!enable;
+}
+
+static void pcie_check_clock_pm(struct pci_dev *pdev)
+{
+ int pos;
+ u32 reg32;
+ u16 reg16;
+ int capable = 1, enabled = 1;
+ struct pci_dev *child_dev;
+ struct link_state *link_state = pdev->link_state;
+
+ /* All functions should have the same cap and state, take the worst */
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+ if (!pos)
+ return;
+ pci_read_config_dword(child_dev, pos + PCI_EXP_LNKCAP, ®32);
+ if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) {
+ capable = 0;
+ enabled = 0;
+ break;
+ }
+ pci_read_config_word(child_dev, pos + PCI_EXP_LNKCTL, ®16);
+ if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN))
+ enabled = 0;
+ }
+ link_state->clk_pm_capable = capable;
+ link_state->clk_pm_enabled = enabled;
+ link_state->bios_clk_state = enabled;
+ pcie_set_clock_pm(pdev, policy_to_clkpm_state(pdev));
+}
+
+/*
+ * pcie_aspm_configure_common_clock: check if the 2 ends of a link
+ * could use common clock. If they are, configure them to use the
+ * common clock. That will reduce the ASPM state exit latency.
+ */
+static void pcie_aspm_configure_common_clock(struct pci_dev *pdev)
+{
+ int pos, child_pos;
+ u16 reg16 = 0;
+ struct pci_dev *child_dev;
+ int same_clock = 1;
+
+ /*
+ * all functions of a slot should have the same Slot Clock
+ * Configuration, so just check one function
+ * */
+ child_dev = list_entry(pdev->subordinate->devices.next, struct pci_dev,
+ bus_list);
+ BUG_ON(!child_dev->is_pcie);
+
+ /* Check downstream component if bit Slot Clock Configuration is 1 */
+ child_pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+ pci_read_config_word(child_dev, child_pos + PCI_EXP_LNKSTA, ®16);
+ if (!(reg16 & PCI_EXP_LNKSTA_SLC))
+ same_clock = 0;
+
+ /* Check upstream component if bit Slot Clock Configuration is 1 */
+ pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
+ pci_read_config_word(pdev, pos + PCI_EXP_LNKSTA, ®16);
+ if (!(reg16 & PCI_EXP_LNKSTA_SLC))
+ same_clock = 0;
+
+ /* Configure downstream component, all functions */
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ child_pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+ pci_read_config_word(child_dev, child_pos + PCI_EXP_LNKCTL,
+ ®16);
+ if (same_clock)
+ reg16 |= PCI_EXP_LNKCTL_CCC;
+ else
+ reg16 &= ~PCI_EXP_LNKCTL_CCC;
+ pci_write_config_word(child_dev, child_pos + PCI_EXP_LNKCTL,
+ reg16);
+ }
+
+ /* Configure upstream component */
+ pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16);
+ if (same_clock)
+ reg16 |= PCI_EXP_LNKCTL_CCC;
+ else
+ reg16 &= ~PCI_EXP_LNKCTL_CCC;
+ pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16);
+
+ /* retrain link */
+ reg16 |= PCI_EXP_LNKCTL_RL;
+ pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16);
+
+ /* Wait for link training end */
+ while (1) {
+ pci_read_config_word(pdev, pos + PCI_EXP_LNKSTA, ®16);
+ if (!(reg16 & PCI_EXP_LNKSTA_LT))
+ break;
+ cpu_relax();
+ }
+}
+
+/*
+ * calc_L0S_latency: Convert L0s latency encoding to ns
+ */
+static unsigned int calc_L0S_latency(unsigned int latency_encoding, int ac)
+{
+ unsigned int ns = 64;
+
+ if (latency_encoding == 0x7) {
+ if (ac)
+ ns = -1U;
+ else
+ ns = 5*1000; /* > 4us */
+ } else
+ ns *= (1 << latency_encoding);
+ return ns;
+}
+
+/*
+ * calc_L1_latency: Convert L1 latency encoding to ns
+ */
+static unsigned int calc_L1_latency(unsigned int latency_encoding, int ac)
+{
+ unsigned int ns = 1000;
+
+ if (latency_encoding == 0x7) {
+ if (ac)
+ ns = -1U;
+ else
+ ns = 65*1000; /* > 64us */
+ } else
+ ns *= (1 << latency_encoding);
+ return ns;
+}
+
+static void pcie_aspm_get_cap_device(struct pci_dev *pdev, u32 *state,
+ unsigned int *l0s, unsigned int *l1, unsigned int *enabled)
+{
+ int pos;
+ u16 reg16;
+ u32 reg32;
+ unsigned int latency;
+
+ pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
+ pci_read_config_dword(pdev, pos + PCI_EXP_LNKCAP, ®32);
+ *state = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10;
+ if (*state != PCIE_LINK_STATE_L0S &&
+ *state != (PCIE_LINK_STATE_L1|PCIE_LINK_STATE_L0S))
+ * state = 0;
+ if (*state == 0)
+ return;
+
+ latency = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12;
+ *l0s = calc_L0S_latency(latency, 0);
+ if (*state & PCIE_LINK_STATE_L1) {
+ latency = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15;
+ *l1 = calc_L1_latency(latency, 0);
+ }
+ pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16);
+ *enabled = reg16 & (PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1);
+}
+
+static void pcie_aspm_cap_init(struct pci_dev *pdev)
+{
+ struct pci_dev *child_dev;
+ u32 state, tmp;
+ struct link_state *link_state = pdev->link_state;
+
+ /* upstream component states */
+ pcie_aspm_get_cap_device(pdev, &link_state->support_state,
+ &link_state->l0s_upper_latency,
+ &link_state->l1_upper_latency,
+ &link_state->enabled_state);
+ /* downstream component states, all functions have the same setting */
+ child_dev = list_entry(pdev->subordinate->devices.next, struct pci_dev,
+ bus_list);
+ pcie_aspm_get_cap_device(child_dev, &state,
+ &link_state->l0s_down_latency,
+ &link_state->l1_down_latency,
+ &tmp);
+ link_state->support_state &= state;
+ if (!link_state->support_state)
+ return;
+ link_state->enabled_state &= link_state->support_state;
+ link_state->bios_aspm_state = link_state->enabled_state;
+
+ /* ENDPOINT states*/
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ int pos;
+ u32 reg32;
+ unsigned int latency;
+ struct endpoint_state *ep_state;
+
+ if (child_dev->pcie_type != PCI_EXP_TYPE_ENDPOINT &&
+ child_dev->pcie_type != PCI_EXP_TYPE_LEG_END)
+ continue;
+
+ ep_state = child_dev->link_state;
+ pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+ pci_read_config_dword(child_dev, pos + PCI_EXP_DEVCAP, ®32);
+ latency = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6;
+ latency = calc_L0S_latency(latency, 1);
+ ep_state->l0s_acceptable_latency = latency;
+ if (link_state->support_state & PCIE_LINK_STATE_L1) {
+ latency = (reg32 & PCI_EXP_DEVCAP_L1) >> 9;
+ latency = calc_L1_latency(latency, 1);
+ ep_state->l1_acceptable_latency = latency;
+ }
+ }
+}
+
+static unsigned int __pcie_aspm_check_state_one(struct pci_dev *pdev,
+ unsigned int state)
+{
+ struct pci_dev *parent_dev, *tmp_dev;
+ unsigned int latency, l1_latency = 0;
+ struct link_state *link_state;
+ struct endpoint_state *ep_state = pdev->link_state;
+
+ parent_dev = pdev->bus->self;
+ link_state = parent_dev->link_state;
+ state &= link_state->support_state;
+ if (state == 0)
+ return 0;
+
+ /*
+ * Check latency for endpoint device.
+ * TBD: The latency from the endpoint to root complex vary per
+ * switch's upstream link state above the device. Here we just do a
+ * simple check which assumes all links above the device can be in L1
+ * state, that is we just consider the worst case. If switch's upstream
+ * link can't be put into L0S/L1, then our check is too strictly.
+ */
+ tmp_dev = pdev;
+ while (state & (PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1)) {
+ parent_dev = tmp_dev->bus->self;
+ link_state = parent_dev->link_state;
+ if (state & PCIE_LINK_STATE_L0S) {
+ latency = max_t(unsigned int,
+ link_state->l0s_upper_latency,
+ link_state->l0s_down_latency);
+ if (latency > ep_state->l0s_acceptable_latency)
+ state &= ~PCIE_LINK_STATE_L0S;
+ }
+ if (state & PCIE_LINK_STATE_L1) {
+ latency = max_t(unsigned int,
+ link_state->l1_upper_latency,
+ link_state->l1_down_latency);
+ if (latency + l1_latency >
+ ep_state->l1_acceptable_latency)
+ state &= ~PCIE_LINK_STATE_L1;
+ }
+ if (!parent_dev->bus->self) /* parent_dev is a root port */
+ break;
+ else {
+ /*
+ * parent_dev is the downstream port of a switch, make
+ * tmp_dev the upstream port of the switch
+ */
+ tmp_dev = parent_dev->bus->self;
+ /*
+ * every switch on the path to root complex need 1 more
+ * microsecond for L1. Spec doesn't mention L0S.
+ */
+ if (state & PCIE_LINK_STATE_L1)
+ l1_latency += 1000;
+ }
+ }
+ return state;
+}
+
+static unsigned int pcie_aspm_check_state(struct pci_dev *pdev,
+ unsigned int state)
+{
+ struct pci_dev *child_dev;
+
+ /* If no child, disable the link */
+ if (list_empty(&pdev->subordinate->devices))
+ return 0;
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ if (child_dev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE) {
+ /*
+ * If downstream component of a link is pci bridge, we
+ * disable ASPM for now for the link
+ * */
+ state = 0;
+ break;
+ }
+ if ((child_dev->pcie_type != PCI_EXP_TYPE_ENDPOINT &&
+ child_dev->pcie_type != PCI_EXP_TYPE_LEG_END) ||
+ !child_dev->link_state)
+ continue;
+ /* Device not in D0 doesn't need check latency */
+ if (child_dev->current_state == PCI_D1 ||
+ child_dev->current_state == PCI_D2 ||
+ child_dev->current_state == PCI_D3hot ||
+ child_dev->current_state == PCI_D3cold)
+ continue;
+ state = __pcie_aspm_check_state_one(child_dev, state);
+ }
+ return state;
+}
+
+static void __pcie_aspm_config_one_dev(struct pci_dev *pdev, unsigned int state)
+{
+ u16 reg16;
+ int pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
+
+ pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16);
+ reg16 &= ~0x3;
+ reg16 |= state;
+ pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16);
+}
+
+static void __pcie_aspm_config_link(struct pci_dev *pdev, unsigned int state)
+{
+ struct pci_dev *child_dev;
+ int valid = 1;
+ struct link_state *link_state = pdev->link_state;
+
+ /*
+ * if the downstream component has pci bridge function, don't do ASPM
+ * now
+ */
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ if (child_dev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE) {
+ valid = 0;
+ break;
+ }
+ }
+ if (!valid)
+ return;
+
+ /*
+ * spec 2.0 suggests all functions should be configured the same
+ * setting for ASPM. Enabling ASPM L1 should be done in upstream
+ * component first and then downstream, and vice versa for disabling
+ * ASPM L1. Spec doesn't mention L0S.
+ */
+ if (state & PCIE_LINK_STATE_L1)
+ __pcie_aspm_config_one_dev(pdev, state);
+
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list)
+ __pcie_aspm_config_one_dev(child_dev, state);
+
+ if (!(state & PCIE_LINK_STATE_L1))
+ __pcie_aspm_config_one_dev(pdev, state);
+
+ link_state->enabled_state = state;
+}
+
+static void __pcie_aspm_configure_link_state(struct pci_dev *pdev,
+ unsigned int state)
+{
+ struct link_state *link_state = pdev->link_state;
+
+ if (link_state->support_state == 0)
+ return;
+ state &= PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1;
+
+ /* state 0 means disabling aspm */
+ state = pcie_aspm_check_state(pdev, state);
+ if (link_state->enabled_state == state)
+ return;
+ __pcie_aspm_config_link(pdev, state);
+}
+
+/*
+ * pcie_aspm_configure_link_state: enable/disable PCI express link state
+ * @pdev: the root port or switch downstream port
+ */
+static void pcie_aspm_configure_link_state(struct pci_dev *pdev,
+ unsigned int state)
+{
+ down_read(&pci_bus_sem);
+ mutex_lock(&aspm_lock);
+ __pcie_aspm_configure_link_state(pdev, state);
+ mutex_unlock(&aspm_lock);
+ up_read(&pci_bus_sem);
+}
+
+static void free_link_state(struct pci_dev *pdev)
+{
+ struct pci_dev *child_dev;
+
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ kfree(child_dev->link_state);
+ child_dev->link_state = NULL;
+ }
+ kfree(pdev->link_state);
+ pdev->link_state = NULL;
+}
+
+/*
+ * pcie_aspm_init_link_state: Initiate PCI express link state.
+ * It is called after the pcie and its children devices are scaned.
+ * @pdev: the root port or switch downstream port
+ */
+void pcie_aspm_init_link_state(struct pci_dev *pdev)
+{
+ unsigned int state;
+ struct link_state *link_state;
+ struct pci_dev *child_dev;
+ struct endpoint_state *ep_state;
+ int error = 0;
+
+ if (aspm_disabled || !pdev->is_pcie || pdev->link_state)
+ return;
+ if (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
+ pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
+ return;
+ down_read(&pci_bus_sem);
+ if (list_empty(&pdev->subordinate->devices))
+ goto out;
+
+ mutex_lock(&aspm_lock);
+
+ link_state = kzalloc(sizeof(*link_state), GFP_KERNEL);
+ if (!link_state)
+ goto unlock_out;
+ pdev->link_state = link_state;
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ ep_state = kzalloc(sizeof(*ep_state), GFP_KERNEL);
+ if (!ep_state) {
+ error = 1;
+ goto unlock_out;
+ }
+ child_dev->link_state = ep_state;
+ }
+
+ pcie_aspm_configure_common_clock(pdev);
+
+ pcie_aspm_cap_init(pdev);
+
+ /* config link state to avoid BIOS error */
+ state = pcie_aspm_check_state(pdev, policy_to_aspm_state(pdev));
+ __pcie_aspm_config_link(pdev, state);
+
+ pcie_check_clock_pm(pdev);
+
+ link_state->pdev = pdev;
+ list_add(&link_state->sibiling, &link_list);
+
+unlock_out:
+ if (error)
+ free_link_state(pdev);
+ mutex_unlock(&aspm_lock);
+out:
+ up_read(&pci_bus_sem);
+}
+
+/* @pdev: the endpoint device */
+void pcie_aspm_exit_link_state(struct pci_dev *pdev)
+{
+ struct pci_dev *parent = pdev->bus->self;
+ struct link_state *link_state = parent->link_state;
+
+ if (aspm_disabled || !pdev->is_pcie || !parent || !link_state)
+ return;
+ if (parent->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
+ parent->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
+ return;
+ down_read(&pci_bus_sem);
+ mutex_lock(&aspm_lock);
+
+ /*
+ * All PCIe functions are in one slot, remove one function will remove
+ * the the whole slot, so just wait
+ */
+ if (!list_empty(&parent->subordinate->devices)) {
+ kfree(pdev->link_state);
+ pdev->link_state = NULL;
+ goto out;
+ }
+
+ /* All functions are removed, so just disable ASPM for the link */
+ __pcie_aspm_config_one_dev(parent, 0);
+ list_del(&link_state->sibiling);
+ /* Clock PM is for endpoint device */
+
+ free_link_state(parent);
+out:
+ mutex_unlock(&aspm_lock);
+ up_read(&pci_bus_sem);
+}
+
+/* @pdev: the root port or switch downstream port */
+void pcie_aspm_pm_state_change(struct pci_dev *pdev)
+{
+ struct link_state *link_state = pdev->link_state;
+
+ if (aspm_disabled || !pdev->is_pcie || !pdev->link_state)
+ return;
+ if (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
+ pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
+ return;
+ /*
+ * devices changed PM state, we should recheck if latency meets all
+ * functions' requirement
+ */
+ pcie_aspm_configure_link_state(pdev, link_state->enabled_state);
+}
+
+/*
+ * pci_disable_link_state - disable pci device's link state, so the link will
+ * never enter specific states
+ */
+void pci_disable_link_state(struct pci_dev *pdev, int state)
+{
+ struct pci_dev *parent = pdev->bus->self;
+ struct link_state *link_state;
+
+ if (aspm_disabled || !pdev->is_pcie || !pdev->link_state)
+ return;
+ if (pdev->pcie_type == PCI_EXP_TYPE_ROOT_PORT ||
+ pdev->pcie_type == PCI_EXP_TYPE_DOWNSTREAM)
+ parent = pdev;
+ if (!parent)
+ return;
+
+ down_read(&pci_bus_sem);
+ mutex_lock(&aspm_lock);
+ link_state = parent->link_state;
+ link_state->support_state &=
+ ~(state & (PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1));
+ if (state & PCIE_LINK_STATE_CLKPM)
+ link_state->clk_pm_capable = 0;
+
+ __pcie_aspm_configure_link_state(parent, link_state->enabled_state);
+ if (!link_state->clk_pm_capable && link_state->clk_pm_enabled)
+ pcie_set_clock_pm(parent, 0);
+ mutex_unlock(&aspm_lock);
+ up_read(&pci_bus_sem);
+}
+EXPORT_SYMBOL(pci_disable_link_state);
+
+static int pcie_aspm_set_policy(const char *val, struct kernel_param *kp)
+{
+ int i;
+ struct pci_dev *pdev;
+ struct link_state *link_state;
+
+ for (i = 0; i < ARRAY_SIZE(policy_str); i++)
+ if (!strncmp(val, policy_str[i], strlen(policy_str[i])))
+ break;
+ if (i >= ARRAY_SIZE(policy_str))
+ return -EINVAL;
+ if (i == aspm_policy)
+ return 0;
+
+ down_read(&pci_bus_sem);
+ mutex_lock(&aspm_lock);
+ aspm_policy = i;
+ list_for_each_entry(link_state, &link_list, sibiling) {
+ pdev = link_state->pdev;
+ __pcie_aspm_configure_link_state(pdev,
+ policy_to_aspm_state(pdev));
+ if (link_state->clk_pm_capable &&
+ link_state->clk_pm_enabled != policy_to_clkpm_state(pdev))
+ pcie_set_clock_pm(pdev, policy_to_clkpm_state(pdev));
+
+ }
+ mutex_unlock(&aspm_lock);
+ up_read(&pci_bus_sem);
+ return 0;
+}
+
+static int pcie_aspm_get_policy(char *buffer, struct kernel_param *kp)
+{
+ int i, cnt = 0;
+ for (i = 0; i < ARRAY_SIZE(policy_str); i++)
+ if (i == aspm_policy)
+ cnt += sprintf(buffer + cnt, "[%s] ", policy_str[i]);
+ else
+ cnt += sprintf(buffer + cnt, "%s ", policy_str[i]);
+ return cnt;
+}
+
+module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy,
+ NULL, 0644);
+
+#ifdef CONFIG_PCIEASPM_DEBUG
+static ssize_t link_state_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct pci_dev *pci_device = to_pci_dev(dev);
+ struct link_state *link_state = pci_device->link_state;
+
+ return sprintf(buf, "%d\n", link_state->enabled_state);
+}
+
+static ssize_t link_state_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf,
+ size_t n)
+{
+ struct pci_dev *pci_device = to_pci_dev(dev);
+ int state;
+
+ if (n < 1)
+ return -EINVAL;
+ state = buf[0]-'0';
+ if (state >= 0 && state <= 3) {
+ /* setup link aspm state */
+ pcie_aspm_configure_link_state(pci_device, state);
+ return n;
+ }
+
+ return -EINVAL;
+}
+
+static ssize_t clk_ctl_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct pci_dev *pci_device = to_pci_dev(dev);
+ struct link_state *link_state = pci_device->link_state;
+
+ return sprintf(buf, "%d\n", link_state->clk_pm_enabled);
+}
+
+static ssize_t clk_ctl_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf,
+ size_t n)
+{
+ struct pci_dev *pci_device = to_pci_dev(dev);
+ int state;
+
+ if (n < 1)
+ return -EINVAL;
+ state = buf[0]-'0';
+
+ down_read(&pci_bus_sem);
+ mutex_lock(&aspm_lock);
+ pcie_set_clock_pm(pci_device, !!state);
+ mutex_unlock(&aspm_lock);
+ up_read(&pci_bus_sem);
+
+ return n;
+}
+
+static DEVICE_ATTR(link_state, 0644, link_state_show, link_state_store);
+static DEVICE_ATTR(clk_ctl, 0644, clk_ctl_show, clk_ctl_store);
+
+static char power_group[] = "power";
+void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev)
+{
+ struct link_state *link_state = pdev->link_state;
+
+ if (!pdev->is_pcie || (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
+ pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM))
+ return;
+
+ if (link_state->support_state)
+ sysfs_add_file_to_group(&pdev->dev.kobj,
+ &dev_attr_link_state.attr, power_group);
+ if (link_state->clk_pm_capable)
+ sysfs_add_file_to_group(&pdev->dev.kobj,
+ &dev_attr_clk_ctl.attr, power_group);
+}
+
+void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev)
+{
+ struct link_state *link_state = pdev->link_state;
+
+ if (!pdev->is_pcie || (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
+ pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM))
+ return;
+
+ if (link_state->support_state)
+ sysfs_remove_file_from_group(&pdev->dev.kobj,
+ &dev_attr_link_state.attr, power_group);
+ if (link_state->clk_pm_capable)
+ sysfs_remove_file_from_group(&pdev->dev.kobj,
+ &dev_attr_clk_ctl.attr, power_group);
+}
+#endif
+
+static int __init pcie_aspm_disable(char *str)
+{
+ aspm_disabled = 1;
+ return 1;
+}
+
+__setup("pcie_noaspm", pcie_aspm_disable);
+
+static int __init pcie_aspm_init(void)
+{
+ if (aspm_disabled)
+ return 0;
+ pci_osc_support_set(OSC_ACTIVE_STATE_PWR_SUPPORT|
+ OSC_CLOCK_PWR_CAPABILITY_SUPPORT);
+ return 0;
+}
+
+fs_initcall(pcie_aspm_init);
Index: linux/include/linux/aspm.h
===================================================================
--- /dev/null 1970-01-01 00:00:00.000000000 +0000
+++ linux/include/linux/aspm.h 2008-01-23 10:14:46.000000000 +0800
@@ -0,0 +1,44 @@
+/*
+ * aspm.h
+ *
+ * PCI Express ASPM defines and function prototypes
+ *
+ * Copyright (C) 2007 Intel Corp.
+ * Zhang Yanmin ([email protected])
+ * Shaohua Li ([email protected])
+ *
+ * For more information, please consult the following manuals (look at
+ * http://www.pcisig.com/ for how to get them):
+ *
+ * PCI Express Specification
+ */
+
+#ifndef LINUX_ASPM_H
+#define LINUX_ASPM_H
+
+#include <linux/pci.h>
+
+#define PCIE_LINK_STATE_L0S 1
+#define PCIE_LINK_STATE_L1 2
+#define PCIE_LINK_STATE_CLKPM 4
+
+#ifdef CONFIG_PCIEASPM
+extern void pcie_aspm_init_link_state(struct pci_dev *pdev);
+extern void pcie_aspm_exit_link_state(struct pci_dev *pdev);
+extern void pcie_aspm_pm_state_change(struct pci_dev *pdev);
+extern void pci_disable_link_state(struct pci_dev *pdev, int state);
+#else
+#define pcie_aspm_init_link_state(pdev) do {} while (0)
+#define pcie_aspm_exit_link_state(pdev) do {} while (0)
+#define pcie_aspm_pm_state_change(pdev) do {} while (0)
+#define pci_disable_link_state(pdev, state) do {} while (0)
+#endif
+
+#ifdef CONFIG_PCIEASPM_DEBUG /* this depends on CONFIG_PCIEASPM */
+extern void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev);
+extern void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev);
+#else
+#define pcie_aspm_create_sysfs_dev_files(pdev) do {} while (0)
+#define pcie_aspm_remove_sysfs_dev_files(pdev) do {} while (0)
+#endif
+#endif /* LINUX_ASPM_H */
Index: linux/drivers/pci/pcie/Kconfig
===================================================================
--- linux.orig/drivers/pci/pcie/Kconfig 2008-01-23 10:14:14.000000000 +0800
+++ linux/drivers/pci/pcie/Kconfig 2008-01-23 10:14:46.000000000 +0800
@@ -26,3 +26,23 @@ config HOTPLUG_PCI_PCIE
When in doubt, say N.
source "drivers/pci/pcie/aer/Kconfig"
+
+#
+# PCI Express ASPM
+#
+config PCIEASPM
+ bool "PCI Express ASPM support(Experimental)"
+ depends on PCI && EXPERIMENTAL
+ default y
+ help
+ This enables PCI Express ASPM (Active State Power Management) and
+ Clock Power Management. ASPM supports state L0/L0s/L1.
+
+ When in doubt, say N.
+config PCIEASPM_DEBUG
+ bool "Debug PCI Express ASPM"
+ depends on PCIEASPM
+ default n
+ help
+ This enables PCI Express ASPM debug support. It will add per-device
+ interface to control ASPM.
Index: linux/include/linux/pci.h
===================================================================
--- linux.orig/include/linux/pci.h 2008-01-23 10:14:14.000000000 +0800
+++ linux/include/linux/pci.h 2008-01-23 10:14:46.000000000 +0800
@@ -164,6 +164,10 @@ struct pci_dev {
this is D0-D3, D0 being fully functional,
and D3 being off. */
+#ifdef CONFIG_PCIEASPM
+ void *link_state; /* ASPM link state. */
+#endif
+
pci_channel_state_t error_state; /* current connectivity state */
struct device dev; /* Generic device interface */
Index: linux/include/linux/pci_regs.h
===================================================================
--- linux.orig/include/linux/pci_regs.h 2008-01-23 10:14:14.000000000 +0800
+++ linux/include/linux/pci_regs.h 2008-01-23 10:14:46.000000000 +0800
@@ -395,9 +395,17 @@
#define PCI_EXP_DEVSTA_AUXPD 0x10 /* AUX Power Detected */
#define PCI_EXP_DEVSTA_TRPND 0x20 /* Transactions Pending */
#define PCI_EXP_LNKCAP 12 /* Link Capabilities */
+#define PCI_EXP_LNKCAP_ASPMS 0xc00 /* ASPM Support */
+#define PCI_EXP_LNKCAP_L0SEL 0x7000 /* L0s Exit Latency */
+#define PCI_EXP_LNKCAP_L1EL 0x38000 /* L1 Exit Latency */
+#define PCI_EXP_LNKCAP_CLKPM 0x40000 /* L1 Clock Power Management */
#define PCI_EXP_LNKCTL 16 /* Link Control */
+#define PCI_EXP_LNKCTL_RL 0x20 /* Retrain Link */
+#define PCI_EXP_LNKCTL_CCC 0x40 /* Common Clock COnfiguration */
#define PCI_EXP_LNKCTL_CLKREQ_EN 0x100 /* Enable clkreq */
#define PCI_EXP_LNKSTA 18 /* Link Status */
+#define PCI_EXP_LNKSTA_LT 0x800 /* Link Training */
+#define PCI_EXP_LNKSTA_SLC 0x1000 /* Slot Clock Configuration */
#define PCI_EXP_SLTCAP 20 /* Slot Capabilities */
#define PCI_EXP_SLTCTL 24 /* Slot Control */
#define PCI_EXP_SLTSTA 26 /* Slot Status */
Index: linux/drivers/pci/probe.c
===================================================================
--- linux.orig/drivers/pci/probe.c 2008-01-23 10:14:14.000000000 +0800
+++ linux/drivers/pci/probe.c 2008-01-23 10:14:46.000000000 +0800
@@ -9,6 +9,7 @@
#include <linux/slab.h>
#include <linux/module.h>
#include <linux/cpumask.h>
+#include <linux/aspm.h>
#include "pci.h"
#define CARDBUS_LATENCY_TIMER 176 /* secondary latency timer */
@@ -1011,6 +1012,10 @@ int pci_scan_slot(struct pci_bus *bus, i
break;
}
}
+
+ if (bus->self)
+ pcie_aspm_init_link_state(bus->self);
+
return nr;
}
Index: linux/drivers/pci/remove.c
===================================================================
--- linux.orig/drivers/pci/remove.c 2008-01-23 10:14:14.000000000 +0800
+++ linux/drivers/pci/remove.c 2008-01-23 10:14:46.000000000 +0800
@@ -1,5 +1,6 @@
#include <linux/pci.h>
#include <linux/module.h>
+#include <linux/aspm.h>
#include "pci.h"
static void pci_free_resources(struct pci_dev *dev)
@@ -30,6 +31,9 @@ static void pci_stop_dev(struct pci_dev
dev->global_list.next = dev->global_list.prev = NULL;
up_write(&pci_bus_sem);
}
+
+ if (dev->bus->self)
+ pcie_aspm_exit_link_state(dev);
}
static void pci_destroy_dev(struct pci_dev *dev)
Index: linux/drivers/pci/pci.c
===================================================================
--- linux.orig/drivers/pci/pci.c 2008-01-23 10:14:14.000000000 +0800
+++ linux/drivers/pci/pci.c 2008-01-23 10:14:46.000000000 +0800
@@ -18,6 +18,7 @@
#include <linux/spinlock.h>
#include <linux/string.h>
#include <linux/log2.h>
+#include <linux/aspm.h>
#include <asm/dma.h> /* isa_dma_bridge_buggy */
#include "pci.h"
@@ -501,6 +502,9 @@ pci_set_power_state(struct pci_dev *dev,
if (need_restore)
pci_restore_bars(dev);
+ if (dev->bus->self)
+ pcie_aspm_pm_state_change(dev->bus->self);
+
return 0;
}
Index: linux/drivers/pci/pci-sysfs.c
===================================================================
--- linux.orig/drivers/pci/pci-sysfs.c 2008-01-23 10:14:14.000000000 +0800
+++ linux/drivers/pci/pci-sysfs.c 2008-01-23 10:14:46.000000000 +0800
@@ -21,6 +21,7 @@
#include <linux/topology.h>
#include <linux/mm.h>
#include <linux/capability.h>
+#include <linux/aspm.h>
#include "pci.h"
static int sysfs_initialized; /* = 0 */
@@ -650,6 +651,8 @@ int __must_check pci_create_sysfs_dev_fi
if (pcibios_add_platform_entries(pdev))
goto err_rom_file;
+ pcie_aspm_create_sysfs_dev_files(pdev);
+
return 0;
err_rom_file:
@@ -679,6 +682,8 @@ void pci_remove_sysfs_dev_files(struct p
if (!sysfs_initialized)
return;
+ pcie_aspm_remove_sysfs_dev_files(pdev);
+
if (pdev->cfg_size < 4096)
sysfs_remove_bin_file(&pdev->dev.kobj, &pci_config_attr);
else
On Wed, Jan 23, 2008 at 10:20:54AM +0800, Shaohua Li wrote:
>
> On Tue, 2008-01-22 at 14:58 -0800, Greg KH wrote:
> > On Fri, Jan 18, 2008 at 09:56:28AM +0800, Shaohua Li wrote:
> > > v3->v2, fixed the issues Matthew Wilcox raised.
> > >
> > > PCI Express ASPM defines a protocol for PCI Express components in the D0
> > > state to reduce Link power by placing their Links into a low power state
> > > and instructing the other end of the Link to do likewise. This
> > > capability allows hardware-autonomous, dynamic Link power reduction
> > > beyond what is achievable by software-only controlled power management.
> > > However, The device should be configured by software appropriately.
> > > Enabling ASPM will save power, but will introduce device latency.
> > >
> > > This patch adds ASPM support in Linux. It introduces a global policy for
> > > ASPM, a sysfs file /sys/module/pcie_aspm/parameters/policy can control
> > > it. The interface can be used as a boot option too. Currently we have
> > > below setting:
> > > -default, BIOS default setting
> > > -powersave, highest power saving mode, enable all available ASPM
> > > state
> > > and clock power management
> > > -performance, highest performance, disable ASPM and clock power
> > > management
> > > By default, the 'default' policy is used currently.
> > >
> > > In my test, power difference between powersave mode and performance mode
> > > is about 1.3w in a system with 3 PCIE links.
> > >
> > > please review, any comments will be appreciated.
> >
> > Can you please fix up all of the warnings that checkpatch.pl and sparse
> > produce from this patch?
> >
> > Also, one small thing:
> >
> > > --- linux.orig/include/linux/pci.h 2008-01-16 15:59:42.000000000 +0800
> > > +++ linux/include/linux/pci.h 2008-01-18 09:41:20.000000000 +0800
> > > @@ -164,6 +164,10 @@ struct pci_dev {
> > > this is D0-D3, D0 being fully functional,
> > > and D3 being off. */
> > >
> > > +#ifdef CONFIG_PCIEASPM
> > > + void *link_state; /* ASPM link state. */
> > > +#endif
> >
> > Can we make this a "real" pointer to a structure? I note that you use
> > two different structures here in this pointer, should you really do
> > that? It's good to get type-checks whereever possible.
> The structure is just for internal use of ASPM, just don't want make it
> global.
Yes, you don't need to expose the structure type, just name it, and then
define it in the code itself.
But using a void pointer as you have here, allows you to assign two
different types of structures to this pointer. Are you sure that you
always get this right? :)
Please, let's try to inforce type-saftey and set this to be a specific
type of pointer to a structure. That will require you to possibly merge
the two structures, which will require some code changes.
thanks,
greg k-h
On Wed, 2008-01-23 at 10:26 -0800, Greg KH wrote:
> On Wed, Jan 23, 2008 at 10:20:54AM +0800, Shaohua Li wrote:
> >
> > On Tue, 2008-01-22 at 14:58 -0800, Greg KH wrote:
> > > On Fri, Jan 18, 2008 at 09:56:28AM +0800, Shaohua Li wrote:
> > > > v3->v2, fixed the issues Matthew Wilcox raised.
> > > >
> > > > PCI Express ASPM defines a protocol for PCI Express components in the D0
> > > > state to reduce Link power by placing their Links into a low power state
> > > > and instructing the other end of the Link to do likewise. This
> > > > capability allows hardware-autonomous, dynamic Link power reduction
> > > > beyond what is achievable by software-only controlled power management.
> > > > However, The device should be configured by software appropriately.
> > > > Enabling ASPM will save power, but will introduce device latency.
> > > >
> > > > This patch adds ASPM support in Linux. It introduces a global policy for
> > > > ASPM, a sysfs file /sys/module/pcie_aspm/parameters/policy can control
> > > > it. The interface can be used as a boot option too. Currently we have
> > > > below setting:
> > > > -default, BIOS default setting
> > > > -powersave, highest power saving mode, enable all available ASPM
> > > > state
> > > > and clock power management
> > > > -performance, highest performance, disable ASPM and clock power
> > > > management
> > > > By default, the 'default' policy is used currently.
> > > >
> > > > In my test, power difference between powersave mode and performance mode
> > > > is about 1.3w in a system with 3 PCIE links.
> > > >
> > > > please review, any comments will be appreciated.
> > >
> > > Can you please fix up all of the warnings that checkpatch.pl and sparse
> > > produce from this patch?
> > >
> > > Also, one small thing:
> > >
> > > > --- linux.orig/include/linux/pci.h 2008-01-16 15:59:42.000000000 +0800
> > > > +++ linux/include/linux/pci.h 2008-01-18 09:41:20.000000000 +0800
> > > > @@ -164,6 +164,10 @@ struct pci_dev {
> > > > this is D0-D3, D0 being fully functional,
> > > > and D3 being off. */
> > > >
> > > > +#ifdef CONFIG_PCIEASPM
> > > > + void *link_state; /* ASPM link state. */
> > > > +#endif
> > >
> > > Can we make this a "real" pointer to a structure? I note that you use
> > > two different structures here in this pointer, should you really do
> > > that? It's good to get type-checks whereever possible.
> > The structure is just for internal use of ASPM, just don't want make it
> > global.
>
> Yes, you don't need to expose the structure type, just name it, and then
> define it in the code itself.
>
> But using a void pointer as you have here, allows you to assign two
> different types of structures to this pointer. Are you sure that you
> always get this right? :)
>
> Please, let's try to inforce type-saftey and set this to be a specific
> type of pointer to a structure. That will require you to possibly merge
> the two structures, which will require some code changes.
Ok, fixed.
PCI Express ASPM defines a protocol for PCI Express components in the D0
state to reduce Link power by placing their Links into a low power state
and instructing the other end of the Link to do likewise. This
capability allows hardware-autonomous, dynamic Link power reduction
beyond what is achievable by software-only controlled power management.
However, The device should be configured by software appropriately.
Enabling ASPM will save power, but will introduce device latency.
This patch adds ASPM support in Linux. It introduces a global policy for
ASPM, a sysfs file /sys/module/pcie_aspm/parameters/policy can control
it. The interface can be used as a boot option too. Currently we have
below setting:
-default, BIOS default setting
-powersave, highest power saving mode, enable all available ASPM
state
and clock power management
-performance, highest performance, disable ASPM and clock power
management
By default, the 'default' policy is used currently.
In my test, power difference between powersave mode and performance mode
is about 1.3w in a system with 3 PCIE links.
Signed-off-by: Shaohua Li <[email protected]>
---
drivers/pci/pci-sysfs.c | 5
drivers/pci/pci.c | 4
drivers/pci/pcie/Kconfig | 20 +
drivers/pci/pcie/Makefile | 3
drivers/pci/pcie/aspm.c | 802 ++++++++++++++++++++++++++++++++++++++++++++++
drivers/pci/probe.c | 5
drivers/pci/remove.c | 4
include/linux/aspm.h | 44 ++
include/linux/pci.h | 5
include/linux/pci_regs.h | 8
10 files changed, 900 insertions(+)
Index: linux/drivers/pci/pcie/Makefile
===================================================================
--- linux.orig/drivers/pci/pcie/Makefile 2008-01-23 10:22:14.000000000 +0800
+++ linux/drivers/pci/pcie/Makefile 2008-01-24 09:16:38.000000000 +0800
@@ -2,6 +2,9 @@
# Makefile for PCI-Express PORT Driver
#
+# Build PCI Express ASPM if needed
+obj-$(CONFIG_PCIEASPM) += aspm.o
+
pcieportdrv-y := portdrv_core.o portdrv_pci.o portdrv_bus.o
obj-$(CONFIG_PCIEPORTBUS) += pcieportdrv.o
Index: linux/drivers/pci/pcie/aspm.c
===================================================================
--- /dev/null 1970-01-01 00:00:00.000000000 +0000
+++ linux/drivers/pci/pcie/aspm.c 2008-01-24 10:04:39.000000000 +0800
@@ -0,0 +1,802 @@
+/*
+ * File: drivers/pci/pcie/aspm.c
+ * Enabling PCIE link L0s/L1 state and Clock Power Management
+ *
+ * Copyright (C) 2007 Intel
+ * Copyright (C) Zhang Yanmin ([email protected])
+ * Copyright (C) Shaohua Li ([email protected])
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/pci.h>
+#include <linux/pci_regs.h>
+#include <linux/errno.h>
+#include <linux/pm.h>
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <linux/aspm.h>
+#include <acpi/acpi_bus.h>
+#include <linux/pci-acpi.h>
+#include "../pci.h"
+
+#ifdef MODULE_PARAM_PREFIX
+#undef MODULE_PARAM_PREFIX
+#endif
+#define MODULE_PARAM_PREFIX "pcie_aspm."
+
+struct endpoint_state {
+ unsigned int l0s_acceptable_latency;
+ unsigned int l1_acceptable_latency;
+};
+
+struct pcie_link_state {
+ struct list_head sibiling;
+ struct pci_dev *pdev;
+
+ /* ASPM state */
+ unsigned int support_state;
+ unsigned int enabled_state;
+ unsigned int bios_aspm_state;
+ /* upstream component */
+ unsigned int l0s_upper_latency;
+ unsigned int l1_upper_latency;
+ /* downstream component */
+ unsigned int l0s_down_latency;
+ unsigned int l1_down_latency;
+ /* Clock PM state*/
+ unsigned int clk_pm_capable;
+ unsigned int clk_pm_enabled;
+ unsigned int bios_clk_state;
+
+ /*
+ * A pcie downstream port only has one slot under it, so at most there
+ * are 8 functions
+ */
+ struct endpoint_state endpoints[8];
+};
+
+static int aspm_disabled;
+static DEFINE_MUTEX(aspm_lock);
+static LIST_HEAD(link_list);
+
+#define POLICY_DEFAULT 0 /* BIOS default setting */
+#define POLICY_PERFORMANCE 1 /* high performance */
+#define POLICY_POWERSAVE 2 /* high power saving */
+static int aspm_policy;
+static const char *policy_str[] = {
+ [POLICY_DEFAULT] = "default",
+ [POLICY_PERFORMANCE] = "performance",
+ [POLICY_POWERSAVE] = "powersave"
+};
+
+static int policy_to_aspm_state(struct pci_dev *pdev)
+{
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ switch (aspm_policy) {
+ case POLICY_PERFORMANCE:
+ /* Disable ASPM and Clock PM */
+ return 0;
+ case POLICY_POWERSAVE:
+ /* Enable ASPM L0s/L1 */
+ return PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1;
+ case POLICY_DEFAULT:
+ return link_state->bios_aspm_state;
+ }
+ return 0;
+}
+
+static int policy_to_clkpm_state(struct pci_dev *pdev)
+{
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ switch (aspm_policy) {
+ case POLICY_PERFORMANCE:
+ /* Disable ASPM and Clock PM */
+ return 0;
+ case POLICY_POWERSAVE:
+ /* Disable Clock PM */
+ return 1;
+ case POLICY_DEFAULT:
+ return link_state->bios_clk_state;
+ }
+ return 0;
+}
+
+static void pcie_set_clock_pm(struct pci_dev *pdev, int enable)
+{
+ struct pci_dev *child_dev;
+ int pos;
+ u16 reg16;
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+ if (!pos)
+ return;
+ pci_read_config_word(child_dev, pos + PCI_EXP_LNKCTL, ®16);
+ if (enable)
+ reg16 |= PCI_EXP_LNKCTL_CLKREQ_EN;
+ else
+ reg16 &= ~PCI_EXP_LNKCTL_CLKREQ_EN;
+ pci_write_config_word(child_dev, pos + PCI_EXP_LNKCTL, reg16);
+ }
+ link_state->clk_pm_enabled = !!enable;
+}
+
+static void pcie_check_clock_pm(struct pci_dev *pdev)
+{
+ int pos;
+ u32 reg32;
+ u16 reg16;
+ int capable = 1, enabled = 1;
+ struct pci_dev *child_dev;
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ /* All functions should have the same cap and state, take the worst */
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+ if (!pos)
+ return;
+ pci_read_config_dword(child_dev, pos + PCI_EXP_LNKCAP, ®32);
+ if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) {
+ capable = 0;
+ enabled = 0;
+ break;
+ }
+ pci_read_config_word(child_dev, pos + PCI_EXP_LNKCTL, ®16);
+ if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN))
+ enabled = 0;
+ }
+ link_state->clk_pm_capable = capable;
+ link_state->clk_pm_enabled = enabled;
+ link_state->bios_clk_state = enabled;
+ pcie_set_clock_pm(pdev, policy_to_clkpm_state(pdev));
+}
+
+/*
+ * pcie_aspm_configure_common_clock: check if the 2 ends of a link
+ * could use common clock. If they are, configure them to use the
+ * common clock. That will reduce the ASPM state exit latency.
+ */
+static void pcie_aspm_configure_common_clock(struct pci_dev *pdev)
+{
+ int pos, child_pos;
+ u16 reg16 = 0;
+ struct pci_dev *child_dev;
+ int same_clock = 1;
+
+ /*
+ * all functions of a slot should have the same Slot Clock
+ * Configuration, so just check one function
+ * */
+ child_dev = list_entry(pdev->subordinate->devices.next, struct pci_dev,
+ bus_list);
+ BUG_ON(!child_dev->is_pcie);
+
+ /* Check downstream component if bit Slot Clock Configuration is 1 */
+ child_pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+ pci_read_config_word(child_dev, child_pos + PCI_EXP_LNKSTA, ®16);
+ if (!(reg16 & PCI_EXP_LNKSTA_SLC))
+ same_clock = 0;
+
+ /* Check upstream component if bit Slot Clock Configuration is 1 */
+ pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
+ pci_read_config_word(pdev, pos + PCI_EXP_LNKSTA, ®16);
+ if (!(reg16 & PCI_EXP_LNKSTA_SLC))
+ same_clock = 0;
+
+ /* Configure downstream component, all functions */
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ child_pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+ pci_read_config_word(child_dev, child_pos + PCI_EXP_LNKCTL,
+ ®16);
+ if (same_clock)
+ reg16 |= PCI_EXP_LNKCTL_CCC;
+ else
+ reg16 &= ~PCI_EXP_LNKCTL_CCC;
+ pci_write_config_word(child_dev, child_pos + PCI_EXP_LNKCTL,
+ reg16);
+ }
+
+ /* Configure upstream component */
+ pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16);
+ if (same_clock)
+ reg16 |= PCI_EXP_LNKCTL_CCC;
+ else
+ reg16 &= ~PCI_EXP_LNKCTL_CCC;
+ pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16);
+
+ /* retrain link */
+ reg16 |= PCI_EXP_LNKCTL_RL;
+ pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16);
+
+ /* Wait for link training end */
+ while (1) {
+ pci_read_config_word(pdev, pos + PCI_EXP_LNKSTA, ®16);
+ if (!(reg16 & PCI_EXP_LNKSTA_LT))
+ break;
+ cpu_relax();
+ }
+}
+
+/*
+ * calc_L0S_latency: Convert L0s latency encoding to ns
+ */
+static unsigned int calc_L0S_latency(unsigned int latency_encoding, int ac)
+{
+ unsigned int ns = 64;
+
+ if (latency_encoding == 0x7) {
+ if (ac)
+ ns = -1U;
+ else
+ ns = 5*1000; /* > 4us */
+ } else
+ ns *= (1 << latency_encoding);
+ return ns;
+}
+
+/*
+ * calc_L1_latency: Convert L1 latency encoding to ns
+ */
+static unsigned int calc_L1_latency(unsigned int latency_encoding, int ac)
+{
+ unsigned int ns = 1000;
+
+ if (latency_encoding == 0x7) {
+ if (ac)
+ ns = -1U;
+ else
+ ns = 65*1000; /* > 64us */
+ } else
+ ns *= (1 << latency_encoding);
+ return ns;
+}
+
+static void pcie_aspm_get_cap_device(struct pci_dev *pdev, u32 *state,
+ unsigned int *l0s, unsigned int *l1, unsigned int *enabled)
+{
+ int pos;
+ u16 reg16;
+ u32 reg32;
+ unsigned int latency;
+
+ pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
+ pci_read_config_dword(pdev, pos + PCI_EXP_LNKCAP, ®32);
+ *state = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10;
+ if (*state != PCIE_LINK_STATE_L0S &&
+ *state != (PCIE_LINK_STATE_L1|PCIE_LINK_STATE_L0S))
+ * state = 0;
+ if (*state == 0)
+ return;
+
+ latency = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12;
+ *l0s = calc_L0S_latency(latency, 0);
+ if (*state & PCIE_LINK_STATE_L1) {
+ latency = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15;
+ *l1 = calc_L1_latency(latency, 0);
+ }
+ pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16);
+ *enabled = reg16 & (PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1);
+}
+
+static void pcie_aspm_cap_init(struct pci_dev *pdev)
+{
+ struct pci_dev *child_dev;
+ u32 state, tmp;
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ /* upstream component states */
+ pcie_aspm_get_cap_device(pdev, &link_state->support_state,
+ &link_state->l0s_upper_latency,
+ &link_state->l1_upper_latency,
+ &link_state->enabled_state);
+ /* downstream component states, all functions have the same setting */
+ child_dev = list_entry(pdev->subordinate->devices.next, struct pci_dev,
+ bus_list);
+ pcie_aspm_get_cap_device(child_dev, &state,
+ &link_state->l0s_down_latency,
+ &link_state->l1_down_latency,
+ &tmp);
+ link_state->support_state &= state;
+ if (!link_state->support_state)
+ return;
+ link_state->enabled_state &= link_state->support_state;
+ link_state->bios_aspm_state = link_state->enabled_state;
+
+ /* ENDPOINT states*/
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ int pos;
+ u32 reg32;
+ unsigned int latency;
+ struct endpoint_state *ep_state =
+ &link_state->endpoints[PCI_FUNC(child_dev->devfn)];
+
+ if (child_dev->pcie_type != PCI_EXP_TYPE_ENDPOINT &&
+ child_dev->pcie_type != PCI_EXP_TYPE_LEG_END)
+ continue;
+
+ pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+ pci_read_config_dword(child_dev, pos + PCI_EXP_DEVCAP, ®32);
+ latency = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6;
+ latency = calc_L0S_latency(latency, 1);
+ ep_state->l0s_acceptable_latency = latency;
+ if (link_state->support_state & PCIE_LINK_STATE_L1) {
+ latency = (reg32 & PCI_EXP_DEVCAP_L1) >> 9;
+ latency = calc_L1_latency(latency, 1);
+ ep_state->l1_acceptable_latency = latency;
+ }
+ }
+}
+
+static unsigned int __pcie_aspm_check_state_one(struct pci_dev *pdev,
+ unsigned int state)
+{
+ struct pci_dev *parent_dev, *tmp_dev;
+ unsigned int latency, l1_latency = 0;
+ struct pcie_link_state *link_state;
+ struct endpoint_state *ep_state;
+
+ parent_dev = pdev->bus->self;
+ link_state = parent_dev->link_state;
+ state &= link_state->support_state;
+ if (state == 0)
+ return 0;
+ ep_state = &link_state->endpoints[PCI_FUNC(pdev->devfn)];
+
+ /*
+ * Check latency for endpoint device.
+ * TBD: The latency from the endpoint to root complex vary per
+ * switch's upstream link state above the device. Here we just do a
+ * simple check which assumes all links above the device can be in L1
+ * state, that is we just consider the worst case. If switch's upstream
+ * link can't be put into L0S/L1, then our check is too strictly.
+ */
+ tmp_dev = pdev;
+ while (state & (PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1)) {
+ parent_dev = tmp_dev->bus->self;
+ link_state = parent_dev->link_state;
+ if (state & PCIE_LINK_STATE_L0S) {
+ latency = max_t(unsigned int,
+ link_state->l0s_upper_latency,
+ link_state->l0s_down_latency);
+ if (latency > ep_state->l0s_acceptable_latency)
+ state &= ~PCIE_LINK_STATE_L0S;
+ }
+ if (state & PCIE_LINK_STATE_L1) {
+ latency = max_t(unsigned int,
+ link_state->l1_upper_latency,
+ link_state->l1_down_latency);
+ if (latency + l1_latency >
+ ep_state->l1_acceptable_latency)
+ state &= ~PCIE_LINK_STATE_L1;
+ }
+ if (!parent_dev->bus->self) /* parent_dev is a root port */
+ break;
+ else {
+ /*
+ * parent_dev is the downstream port of a switch, make
+ * tmp_dev the upstream port of the switch
+ */
+ tmp_dev = parent_dev->bus->self;
+ /*
+ * every switch on the path to root complex need 1 more
+ * microsecond for L1. Spec doesn't mention L0S.
+ */
+ if (state & PCIE_LINK_STATE_L1)
+ l1_latency += 1000;
+ }
+ }
+ return state;
+}
+
+static unsigned int pcie_aspm_check_state(struct pci_dev *pdev,
+ unsigned int state)
+{
+ struct pci_dev *child_dev;
+
+ /* If no child, disable the link */
+ if (list_empty(&pdev->subordinate->devices))
+ return 0;
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ if (child_dev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE) {
+ /*
+ * If downstream component of a link is pci bridge, we
+ * disable ASPM for now for the link
+ * */
+ state = 0;
+ break;
+ }
+ if ((child_dev->pcie_type != PCI_EXP_TYPE_ENDPOINT &&
+ child_dev->pcie_type != PCI_EXP_TYPE_LEG_END))
+ continue;
+ /* Device not in D0 doesn't need check latency */
+ if (child_dev->current_state == PCI_D1 ||
+ child_dev->current_state == PCI_D2 ||
+ child_dev->current_state == PCI_D3hot ||
+ child_dev->current_state == PCI_D3cold)
+ continue;
+ state = __pcie_aspm_check_state_one(child_dev, state);
+ }
+ return state;
+}
+
+static void __pcie_aspm_config_one_dev(struct pci_dev *pdev, unsigned int state)
+{
+ u16 reg16;
+ int pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
+
+ pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16);
+ reg16 &= ~0x3;
+ reg16 |= state;
+ pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16);
+}
+
+static void __pcie_aspm_config_link(struct pci_dev *pdev, unsigned int state)
+{
+ struct pci_dev *child_dev;
+ int valid = 1;
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ /*
+ * if the downstream component has pci bridge function, don't do ASPM
+ * now
+ */
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ if (child_dev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE) {
+ valid = 0;
+ break;
+ }
+ }
+ if (!valid)
+ return;
+
+ /*
+ * spec 2.0 suggests all functions should be configured the same
+ * setting for ASPM. Enabling ASPM L1 should be done in upstream
+ * component first and then downstream, and vice versa for disabling
+ * ASPM L1. Spec doesn't mention L0S.
+ */
+ if (state & PCIE_LINK_STATE_L1)
+ __pcie_aspm_config_one_dev(pdev, state);
+
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list)
+ __pcie_aspm_config_one_dev(child_dev, state);
+
+ if (!(state & PCIE_LINK_STATE_L1))
+ __pcie_aspm_config_one_dev(pdev, state);
+
+ link_state->enabled_state = state;
+}
+
+static void __pcie_aspm_configure_link_state(struct pci_dev *pdev,
+ unsigned int state)
+{
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ if (link_state->support_state == 0)
+ return;
+ state &= PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1;
+
+ /* state 0 means disabling aspm */
+ state = pcie_aspm_check_state(pdev, state);
+ if (link_state->enabled_state == state)
+ return;
+ __pcie_aspm_config_link(pdev, state);
+}
+
+/*
+ * pcie_aspm_configure_link_state: enable/disable PCI express link state
+ * @pdev: the root port or switch downstream port
+ */
+static void pcie_aspm_configure_link_state(struct pci_dev *pdev,
+ unsigned int state)
+{
+ down_read(&pci_bus_sem);
+ mutex_lock(&aspm_lock);
+ __pcie_aspm_configure_link_state(pdev, state);
+ mutex_unlock(&aspm_lock);
+ up_read(&pci_bus_sem);
+}
+
+static void free_link_state(struct pci_dev *pdev)
+{
+ kfree(pdev->link_state);
+ pdev->link_state = NULL;
+}
+
+/*
+ * pcie_aspm_init_link_state: Initiate PCI express link state.
+ * It is called after the pcie and its children devices are scaned.
+ * @pdev: the root port or switch downstream port
+ */
+void pcie_aspm_init_link_state(struct pci_dev *pdev)
+{
+ unsigned int state;
+ struct pcie_link_state *link_state;
+ int error = 0;
+
+ if (aspm_disabled || !pdev->is_pcie || pdev->link_state)
+ return;
+ if (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
+ pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
+ return;
+ down_read(&pci_bus_sem);
+ if (list_empty(&pdev->subordinate->devices))
+ goto out;
+
+ mutex_lock(&aspm_lock);
+
+ link_state = kzalloc(sizeof(*link_state), GFP_KERNEL);
+ if (!link_state)
+ goto unlock_out;
+ pdev->link_state = link_state;
+
+ pcie_aspm_configure_common_clock(pdev);
+
+ pcie_aspm_cap_init(pdev);
+
+ /* config link state to avoid BIOS error */
+ state = pcie_aspm_check_state(pdev, policy_to_aspm_state(pdev));
+ __pcie_aspm_config_link(pdev, state);
+
+ pcie_check_clock_pm(pdev);
+
+ link_state->pdev = pdev;
+ list_add(&link_state->sibiling, &link_list);
+
+unlock_out:
+ if (error)
+ free_link_state(pdev);
+ mutex_unlock(&aspm_lock);
+out:
+ up_read(&pci_bus_sem);
+}
+
+/* @pdev: the endpoint device */
+void pcie_aspm_exit_link_state(struct pci_dev *pdev)
+{
+ struct pci_dev *parent = pdev->bus->self;
+ struct pcie_link_state *link_state = parent->link_state;
+
+ if (aspm_disabled || !pdev->is_pcie || !parent || !link_state)
+ return;
+ if (parent->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
+ parent->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
+ return;
+ down_read(&pci_bus_sem);
+ mutex_lock(&aspm_lock);
+
+ /*
+ * All PCIe functions are in one slot, remove one function will remove
+ * the the whole slot, so just wait
+ */
+ if (!list_empty(&parent->subordinate->devices))
+ goto out;
+
+ /* All functions are removed, so just disable ASPM for the link */
+ __pcie_aspm_config_one_dev(parent, 0);
+ list_del(&link_state->sibiling);
+ /* Clock PM is for endpoint device */
+
+ free_link_state(parent);
+out:
+ mutex_unlock(&aspm_lock);
+ up_read(&pci_bus_sem);
+}
+
+/* @pdev: the root port or switch downstream port */
+void pcie_aspm_pm_state_change(struct pci_dev *pdev)
+{
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ if (aspm_disabled || !pdev->is_pcie || !pdev->link_state)
+ return;
+ if (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
+ pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
+ return;
+ /*
+ * devices changed PM state, we should recheck if latency meets all
+ * functions' requirement
+ */
+ pcie_aspm_configure_link_state(pdev, link_state->enabled_state);
+}
+
+/*
+ * pci_disable_link_state - disable pci device's link state, so the link will
+ * never enter specific states
+ */
+void pci_disable_link_state(struct pci_dev *pdev, int state)
+{
+ struct pci_dev *parent = pdev->bus->self;
+ struct pcie_link_state *link_state;
+
+ if (aspm_disabled || !pdev->is_pcie)
+ return;
+ if (pdev->pcie_type == PCI_EXP_TYPE_ROOT_PORT ||
+ pdev->pcie_type == PCI_EXP_TYPE_DOWNSTREAM)
+ parent = pdev;
+ if (!parent)
+ return;
+
+ down_read(&pci_bus_sem);
+ mutex_lock(&aspm_lock);
+ link_state = parent->link_state;
+ link_state->support_state &=
+ ~(state & (PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1));
+ if (state & PCIE_LINK_STATE_CLKPM)
+ link_state->clk_pm_capable = 0;
+
+ __pcie_aspm_configure_link_state(parent, link_state->enabled_state);
+ if (!link_state->clk_pm_capable && link_state->clk_pm_enabled)
+ pcie_set_clock_pm(parent, 0);
+ mutex_unlock(&aspm_lock);
+ up_read(&pci_bus_sem);
+}
+EXPORT_SYMBOL(pci_disable_link_state);
+
+static int pcie_aspm_set_policy(const char *val, struct kernel_param *kp)
+{
+ int i;
+ struct pci_dev *pdev;
+ struct pcie_link_state *link_state;
+
+ for (i = 0; i < ARRAY_SIZE(policy_str); i++)
+ if (!strncmp(val, policy_str[i], strlen(policy_str[i])))
+ break;
+ if (i >= ARRAY_SIZE(policy_str))
+ return -EINVAL;
+ if (i == aspm_policy)
+ return 0;
+
+ down_read(&pci_bus_sem);
+ mutex_lock(&aspm_lock);
+ aspm_policy = i;
+ list_for_each_entry(link_state, &link_list, sibiling) {
+ pdev = link_state->pdev;
+ __pcie_aspm_configure_link_state(pdev,
+ policy_to_aspm_state(pdev));
+ if (link_state->clk_pm_capable &&
+ link_state->clk_pm_enabled != policy_to_clkpm_state(pdev))
+ pcie_set_clock_pm(pdev, policy_to_clkpm_state(pdev));
+
+ }
+ mutex_unlock(&aspm_lock);
+ up_read(&pci_bus_sem);
+ return 0;
+}
+
+static int pcie_aspm_get_policy(char *buffer, struct kernel_param *kp)
+{
+ int i, cnt = 0;
+ for (i = 0; i < ARRAY_SIZE(policy_str); i++)
+ if (i == aspm_policy)
+ cnt += sprintf(buffer + cnt, "[%s] ", policy_str[i]);
+ else
+ cnt += sprintf(buffer + cnt, "%s ", policy_str[i]);
+ return cnt;
+}
+
+module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy,
+ NULL, 0644);
+
+#ifdef CONFIG_PCIEASPM_DEBUG
+static ssize_t link_state_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct pci_dev *pci_device = to_pci_dev(dev);
+ struct pcie_link_state *link_state = pci_device->link_state;
+
+ return sprintf(buf, "%d\n", link_state->enabled_state);
+}
+
+static ssize_t link_state_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf,
+ size_t n)
+{
+ struct pci_dev *pci_device = to_pci_dev(dev);
+ int state;
+
+ if (n < 1)
+ return -EINVAL;
+ state = buf[0]-'0';
+ if (state >= 0 && state <= 3) {
+ /* setup link aspm state */
+ pcie_aspm_configure_link_state(pci_device, state);
+ return n;
+ }
+
+ return -EINVAL;
+}
+
+static ssize_t clk_ctl_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct pci_dev *pci_device = to_pci_dev(dev);
+ struct pcie_link_state *link_state = pci_device->link_state;
+
+ return sprintf(buf, "%d\n", link_state->clk_pm_enabled);
+}
+
+static ssize_t clk_ctl_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf,
+ size_t n)
+{
+ struct pci_dev *pci_device = to_pci_dev(dev);
+ int state;
+
+ if (n < 1)
+ return -EINVAL;
+ state = buf[0]-'0';
+
+ down_read(&pci_bus_sem);
+ mutex_lock(&aspm_lock);
+ pcie_set_clock_pm(pci_device, !!state);
+ mutex_unlock(&aspm_lock);
+ up_read(&pci_bus_sem);
+
+ return n;
+}
+
+static DEVICE_ATTR(link_state, 0644, link_state_show, link_state_store);
+static DEVICE_ATTR(clk_ctl, 0644, clk_ctl_show, clk_ctl_store);
+
+static char power_group[] = "power";
+void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev)
+{
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ if (!pdev->is_pcie || (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
+ pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM))
+ return;
+
+ if (link_state->support_state)
+ sysfs_add_file_to_group(&pdev->dev.kobj,
+ &dev_attr_link_state.attr, power_group);
+ if (link_state->clk_pm_capable)
+ sysfs_add_file_to_group(&pdev->dev.kobj,
+ &dev_attr_clk_ctl.attr, power_group);
+}
+
+void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev)
+{
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ if (!pdev->is_pcie || (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
+ pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM))
+ return;
+
+ if (link_state->support_state)
+ sysfs_remove_file_from_group(&pdev->dev.kobj,
+ &dev_attr_link_state.attr, power_group);
+ if (link_state->clk_pm_capable)
+ sysfs_remove_file_from_group(&pdev->dev.kobj,
+ &dev_attr_clk_ctl.attr, power_group);
+}
+#endif
+
+static int __init pcie_aspm_disable(char *str)
+{
+ aspm_disabled = 1;
+ return 1;
+}
+
+__setup("pcie_noaspm", pcie_aspm_disable);
+
+static int __init pcie_aspm_init(void)
+{
+ if (aspm_disabled)
+ return 0;
+ pci_osc_support_set(OSC_ACTIVE_STATE_PWR_SUPPORT|
+ OSC_CLOCK_PWR_CAPABILITY_SUPPORT);
+ return 0;
+}
+
+fs_initcall(pcie_aspm_init);
Index: linux/include/linux/aspm.h
===================================================================
--- /dev/null 1970-01-01 00:00:00.000000000 +0000
+++ linux/include/linux/aspm.h 2008-01-24 09:16:38.000000000 +0800
@@ -0,0 +1,44 @@
+/*
+ * aspm.h
+ *
+ * PCI Express ASPM defines and function prototypes
+ *
+ * Copyright (C) 2007 Intel Corp.
+ * Zhang Yanmin ([email protected])
+ * Shaohua Li ([email protected])
+ *
+ * For more information, please consult the following manuals (look at
+ * http://www.pcisig.com/ for how to get them):
+ *
+ * PCI Express Specification
+ */
+
+#ifndef LINUX_ASPM_H
+#define LINUX_ASPM_H
+
+#include <linux/pci.h>
+
+#define PCIE_LINK_STATE_L0S 1
+#define PCIE_LINK_STATE_L1 2
+#define PCIE_LINK_STATE_CLKPM 4
+
+#ifdef CONFIG_PCIEASPM
+extern void pcie_aspm_init_link_state(struct pci_dev *pdev);
+extern void pcie_aspm_exit_link_state(struct pci_dev *pdev);
+extern void pcie_aspm_pm_state_change(struct pci_dev *pdev);
+extern void pci_disable_link_state(struct pci_dev *pdev, int state);
+#else
+#define pcie_aspm_init_link_state(pdev) do {} while (0)
+#define pcie_aspm_exit_link_state(pdev) do {} while (0)
+#define pcie_aspm_pm_state_change(pdev) do {} while (0)
+#define pci_disable_link_state(pdev, state) do {} while (0)
+#endif
+
+#ifdef CONFIG_PCIEASPM_DEBUG /* this depends on CONFIG_PCIEASPM */
+extern void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev);
+extern void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev);
+#else
+#define pcie_aspm_create_sysfs_dev_files(pdev) do {} while (0)
+#define pcie_aspm_remove_sysfs_dev_files(pdev) do {} while (0)
+#endif
+#endif /* LINUX_ASPM_H */
Index: linux/drivers/pci/pcie/Kconfig
===================================================================
--- linux.orig/drivers/pci/pcie/Kconfig 2008-01-23 10:22:14.000000000 +0800
+++ linux/drivers/pci/pcie/Kconfig 2008-01-24 09:16:38.000000000 +0800
@@ -26,3 +26,23 @@ config HOTPLUG_PCI_PCIE
When in doubt, say N.
source "drivers/pci/pcie/aer/Kconfig"
+
+#
+# PCI Express ASPM
+#
+config PCIEASPM
+ bool "PCI Express ASPM support(Experimental)"
+ depends on PCI && EXPERIMENTAL
+ default y
+ help
+ This enables PCI Express ASPM (Active State Power Management) and
+ Clock Power Management. ASPM supports state L0/L0s/L1.
+
+ When in doubt, say N.
+config PCIEASPM_DEBUG
+ bool "Debug PCI Express ASPM"
+ depends on PCIEASPM
+ default n
+ help
+ This enables PCI Express ASPM debug support. It will add per-device
+ interface to control ASPM.
Index: linux/include/linux/pci.h
===================================================================
--- linux.orig/include/linux/pci.h 2008-01-23 10:22:14.000000000 +0800
+++ linux/include/linux/pci.h 2008-01-24 09:34:40.000000000 +0800
@@ -129,6 +129,7 @@ struct pci_cap_saved_state {
u32 data[0];
};
+struct pcie_link_state;
/*
* The pci_dev structure is used to describe PCI devices.
*/
@@ -164,6 +165,10 @@ struct pci_dev {
this is D0-D3, D0 being fully functional,
and D3 being off. */
+#ifdef CONFIG_PCIEASPM
+ struct pcie_link_state *link_state; /* ASPM link state. */
+#endif
+
pci_channel_state_t error_state; /* current connectivity state */
struct device dev; /* Generic device interface */
Index: linux/include/linux/pci_regs.h
===================================================================
--- linux.orig/include/linux/pci_regs.h 2008-01-23 10:22:14.000000000 +0800
+++ linux/include/linux/pci_regs.h 2008-01-24 09:16:38.000000000 +0800
@@ -395,9 +395,17 @@
#define PCI_EXP_DEVSTA_AUXPD 0x10 /* AUX Power Detected */
#define PCI_EXP_DEVSTA_TRPND 0x20 /* Transactions Pending */
#define PCI_EXP_LNKCAP 12 /* Link Capabilities */
+#define PCI_EXP_LNKCAP_ASPMS 0xc00 /* ASPM Support */
+#define PCI_EXP_LNKCAP_L0SEL 0x7000 /* L0s Exit Latency */
+#define PCI_EXP_LNKCAP_L1EL 0x38000 /* L1 Exit Latency */
+#define PCI_EXP_LNKCAP_CLKPM 0x40000 /* L1 Clock Power Management */
#define PCI_EXP_LNKCTL 16 /* Link Control */
+#define PCI_EXP_LNKCTL_RL 0x20 /* Retrain Link */
+#define PCI_EXP_LNKCTL_CCC 0x40 /* Common Clock COnfiguration */
#define PCI_EXP_LNKCTL_CLKREQ_EN 0x100 /* Enable clkreq */
#define PCI_EXP_LNKSTA 18 /* Link Status */
+#define PCI_EXP_LNKSTA_LT 0x800 /* Link Training */
+#define PCI_EXP_LNKSTA_SLC 0x1000 /* Slot Clock Configuration */
#define PCI_EXP_SLTCAP 20 /* Slot Capabilities */
#define PCI_EXP_SLTCTL 24 /* Slot Control */
#define PCI_EXP_SLTSTA 26 /* Slot Status */
Index: linux/drivers/pci/probe.c
===================================================================
--- linux.orig/drivers/pci/probe.c 2008-01-23 10:22:14.000000000 +0800
+++ linux/drivers/pci/probe.c 2008-01-24 09:16:38.000000000 +0800
@@ -9,6 +9,7 @@
#include <linux/slab.h>
#include <linux/module.h>
#include <linux/cpumask.h>
+#include <linux/aspm.h>
#include "pci.h"
#define CARDBUS_LATENCY_TIMER 176 /* secondary latency timer */
@@ -1011,6 +1012,10 @@ int pci_scan_slot(struct pci_bus *bus, i
break;
}
}
+
+ if (bus->self)
+ pcie_aspm_init_link_state(bus->self);
+
return nr;
}
Index: linux/drivers/pci/remove.c
===================================================================
--- linux.orig/drivers/pci/remove.c 2008-01-23 10:22:14.000000000 +0800
+++ linux/drivers/pci/remove.c 2008-01-24 09:16:38.000000000 +0800
@@ -1,5 +1,6 @@
#include <linux/pci.h>
#include <linux/module.h>
+#include <linux/aspm.h>
#include "pci.h"
static void pci_free_resources(struct pci_dev *dev)
@@ -30,6 +31,9 @@ static void pci_stop_dev(struct pci_dev
dev->global_list.next = dev->global_list.prev = NULL;
up_write(&pci_bus_sem);
}
+
+ if (dev->bus->self)
+ pcie_aspm_exit_link_state(dev);
}
static void pci_destroy_dev(struct pci_dev *dev)
Index: linux/drivers/pci/pci.c
===================================================================
--- linux.orig/drivers/pci/pci.c 2008-01-24 09:16:36.000000000 +0800
+++ linux/drivers/pci/pci.c 2008-01-24 09:16:38.000000000 +0800
@@ -18,6 +18,7 @@
#include <linux/spinlock.h>
#include <linux/string.h>
#include <linux/log2.h>
+#include <linux/aspm.h>
#include <asm/dma.h> /* isa_dma_bridge_buggy */
#include "pci.h"
@@ -501,6 +502,9 @@ pci_set_power_state(struct pci_dev *dev,
if (need_restore)
pci_restore_bars(dev);
+ if (dev->bus->self)
+ pcie_aspm_pm_state_change(dev->bus->self);
+
return 0;
}
Index: linux/drivers/pci/pci-sysfs.c
===================================================================
--- linux.orig/drivers/pci/pci-sysfs.c 2008-01-23 10:22:14.000000000 +0800
+++ linux/drivers/pci/pci-sysfs.c 2008-01-24 09:16:38.000000000 +0800
@@ -21,6 +21,7 @@
#include <linux/topology.h>
#include <linux/mm.h>
#include <linux/capability.h>
+#include <linux/aspm.h>
#include "pci.h"
static int sysfs_initialized; /* = 0 */
@@ -650,6 +651,8 @@ int __must_check pci_create_sysfs_dev_fi
if (pcibios_add_platform_entries(pdev))
goto err_rom_file;
+ pcie_aspm_create_sysfs_dev_files(pdev);
+
return 0;
err_rom_file:
@@ -679,6 +682,8 @@ void pci_remove_sysfs_dev_files(struct p
if (!sysfs_initialized)
return;
+ pcie_aspm_remove_sysfs_dev_files(pdev);
+
if (pdev->cfg_size < 4096)
sysfs_remove_bin_file(&pdev->dev.kobj, &pci_config_attr);
else
Hi!
> v3->v2, fixed the issues Matthew Wilcox raised.
>
> PCI Express ASPM defines a protocol for PCI Express components in the D0
> state to reduce Link power by placing their Links into a low power state
> and instructing the other end of the Link to do likewise. This
> capability allows hardware-autonomous, dynamic Link power reduction
> beyond what is achievable by software-only controlled power management.
> However, The device should be configured by software appropriately.
> Enabling ASPM will save power, but will introduce device latency.
How big is the latency? 1msec? 10msec? 100usec?
--
(english) http://www.livejournal.com/~pavelmachek
(cesky, pictures) http://atrey.karlin.mff.cuni.cz/~pavel/picture/horses/blog.html
>
>
>Hi!
>
>> v3->v2, fixed the issues Matthew Wilcox raised.
>>
>> PCI Express ASPM defines a protocol for PCI Express components in the
D0
>> state to reduce Link power by placing their Links into a low power
state
>> and instructing the other end of the Link to do likewise. This
>> capability allows hardware-autonomous, dynamic Link power reduction
>> beyond what is achievable by software-only controlled power
management.
>> However, The device should be configured by software appropriately.
>> Enabling ASPM will save power, but will introduce device latency.
>
>How big is the latency? 1msec? 10msec? 100usec?
Haven't accurate number, but in one device, it declaims L0s latency is <
128ns, L1 latency is < 64us.
Thanks,
Shaohua
This is a note to let you know that I've just added the patch titled
Subject: PCI: PCIE ASPM support
to my gregkh-2.6 tree. Its filename is
pci-pcie-aspm-support.patch
This tree can be found at
http://www.kernel.org/pub/linux/kernel/people/gregkh/gregkh-2.6/patches/
>From [email protected] Fri Jan 25 10:10:52 2008
From: Shaohua Li <[email protected]>
Date: Thu, 24 Jan 2008 10:21:57 +0800
Subject: PCI: PCIE ASPM support
To: Greg KH <[email protected]>
Cc: lkml <[email protected]>, linux-pci <[email protected]>, "Pallipadi, Venkatesh" <[email protected]>, "Kok, Auke" <[email protected]>, Matthew Wilcox <[email protected]>
Message-ID: <[email protected]>
PCI Express ASPM defines a protocol for PCI Express components in the D0
state to reduce Link power by placing their Links into a low power state
and instructing the other end of the Link to do likewise. This
capability allows hardware-autonomous, dynamic Link power reduction
beyond what is achievable by software-only controlled power management.
However, The device should be configured by software appropriately.
Enabling ASPM will save power, but will introduce device latency.
This patch adds ASPM support in Linux. It introduces a global policy for
ASPM, a sysfs file /sys/module/pcie_aspm/parameters/policy can control
it. The interface can be used as a boot option too. Currently we have
below setting:
-default, BIOS default setting
-powersave, highest power saving mode, enable all available ASPM
state
and clock power management
-performance, highest performance, disable ASPM and clock power
management
By default, the 'default' policy is used currently.
In my test, power difference between powersave mode and performance mode
is about 1.3w in a system with 3 PCIE links.
Signed-off-by: Shaohua Li <[email protected]>
Signed-off-by: Greg Kroah-Hartman <[email protected]>
---
drivers/pci/pci-sysfs.c | 5
drivers/pci/pci.c | 4
drivers/pci/pcie/Kconfig | 20 +
drivers/pci/pcie/Makefile | 3
drivers/pci/pcie/aspm.c | 802 ++++++++++++++++++++++++++++++++++++++++++++++
drivers/pci/probe.c | 5
drivers/pci/remove.c | 4
include/linux/aspm.h | 44 ++
include/linux/pci.h | 5
include/linux/pci_regs.h | 8
10 files changed, 900 insertions(+)
--- a/drivers/pci/pci-sysfs.c
+++ b/drivers/pci/pci-sysfs.c
@@ -21,6 +21,7 @@
#include <linux/topology.h>
#include <linux/mm.h>
#include <linux/capability.h>
+#include <linux/aspm.h>
#include "pci.h"
static int sysfs_initialized; /* = 0 */
@@ -681,6 +682,8 @@ int __must_check pci_create_sysfs_dev_fi
if (pcibios_add_platform_entries(pdev))
goto err_rom_file;
+ pcie_aspm_create_sysfs_dev_files(pdev);
+
return 0;
err_rom_file:
@@ -710,6 +713,8 @@ void pci_remove_sysfs_dev_files(struct p
if (!sysfs_initialized)
return;
+ pcie_aspm_remove_sysfs_dev_files(pdev);
+
if (pdev->cfg_size < 4096)
sysfs_remove_bin_file(&pdev->dev.kobj, &pci_config_attr);
else
--- a/drivers/pci/pci.c
+++ b/drivers/pci/pci.c
@@ -18,6 +18,7 @@
#include <linux/spinlock.h>
#include <linux/string.h>
#include <linux/log2.h>
+#include <linux/aspm.h>
#include <asm/dma.h> /* isa_dma_bridge_buggy */
#include "pci.h"
@@ -519,6 +520,9 @@ pci_set_power_state(struct pci_dev *dev,
if (need_restore)
pci_restore_bars(dev);
+ if (dev->bus->self)
+ pcie_aspm_pm_state_change(dev->bus->self);
+
return 0;
}
--- a/drivers/pci/pcie/Kconfig
+++ b/drivers/pci/pcie/Kconfig
@@ -26,3 +26,23 @@ config HOTPLUG_PCI_PCIE
When in doubt, say N.
source "drivers/pci/pcie/aer/Kconfig"
+
+#
+# PCI Express ASPM
+#
+config PCIEASPM
+ bool "PCI Express ASPM support(Experimental)"
+ depends on PCI && EXPERIMENTAL
+ default y
+ help
+ This enables PCI Express ASPM (Active State Power Management) and
+ Clock Power Management. ASPM supports state L0/L0s/L1.
+
+ When in doubt, say N.
+config PCIEASPM_DEBUG
+ bool "Debug PCI Express ASPM"
+ depends on PCIEASPM
+ default n
+ help
+ This enables PCI Express ASPM debug support. It will add per-device
+ interface to control ASPM.
--- a/drivers/pci/pcie/Makefile
+++ b/drivers/pci/pcie/Makefile
@@ -2,6 +2,9 @@
# Makefile for PCI-Express PORT Driver
#
+# Build PCI Express ASPM if needed
+obj-$(CONFIG_PCIEASPM) += aspm.o
+
pcieportdrv-y := portdrv_core.o portdrv_pci.o portdrv_bus.o
obj-$(CONFIG_PCIEPORTBUS) += pcieportdrv.o
--- /dev/null
+++ b/drivers/pci/pcie/aspm.c
@@ -0,0 +1,802 @@
+/*
+ * File: drivers/pci/pcie/aspm.c
+ * Enabling PCIE link L0s/L1 state and Clock Power Management
+ *
+ * Copyright (C) 2007 Intel
+ * Copyright (C) Zhang Yanmin ([email protected])
+ * Copyright (C) Shaohua Li ([email protected])
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/pci.h>
+#include <linux/pci_regs.h>
+#include <linux/errno.h>
+#include <linux/pm.h>
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <linux/aspm.h>
+#include <acpi/acpi_bus.h>
+#include <linux/pci-acpi.h>
+#include "../pci.h"
+
+#ifdef MODULE_PARAM_PREFIX
+#undef MODULE_PARAM_PREFIX
+#endif
+#define MODULE_PARAM_PREFIX "pcie_aspm."
+
+struct endpoint_state {
+ unsigned int l0s_acceptable_latency;
+ unsigned int l1_acceptable_latency;
+};
+
+struct pcie_link_state {
+ struct list_head sibiling;
+ struct pci_dev *pdev;
+
+ /* ASPM state */
+ unsigned int support_state;
+ unsigned int enabled_state;
+ unsigned int bios_aspm_state;
+ /* upstream component */
+ unsigned int l0s_upper_latency;
+ unsigned int l1_upper_latency;
+ /* downstream component */
+ unsigned int l0s_down_latency;
+ unsigned int l1_down_latency;
+ /* Clock PM state*/
+ unsigned int clk_pm_capable;
+ unsigned int clk_pm_enabled;
+ unsigned int bios_clk_state;
+
+ /*
+ * A pcie downstream port only has one slot under it, so at most there
+ * are 8 functions
+ */
+ struct endpoint_state endpoints[8];
+};
+
+static int aspm_disabled;
+static DEFINE_MUTEX(aspm_lock);
+static LIST_HEAD(link_list);
+
+#define POLICY_DEFAULT 0 /* BIOS default setting */
+#define POLICY_PERFORMANCE 1 /* high performance */
+#define POLICY_POWERSAVE 2 /* high power saving */
+static int aspm_policy;
+static const char *policy_str[] = {
+ [POLICY_DEFAULT] = "default",
+ [POLICY_PERFORMANCE] = "performance",
+ [POLICY_POWERSAVE] = "powersave"
+};
+
+static int policy_to_aspm_state(struct pci_dev *pdev)
+{
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ switch (aspm_policy) {
+ case POLICY_PERFORMANCE:
+ /* Disable ASPM and Clock PM */
+ return 0;
+ case POLICY_POWERSAVE:
+ /* Enable ASPM L0s/L1 */
+ return PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1;
+ case POLICY_DEFAULT:
+ return link_state->bios_aspm_state;
+ }
+ return 0;
+}
+
+static int policy_to_clkpm_state(struct pci_dev *pdev)
+{
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ switch (aspm_policy) {
+ case POLICY_PERFORMANCE:
+ /* Disable ASPM and Clock PM */
+ return 0;
+ case POLICY_POWERSAVE:
+ /* Disable Clock PM */
+ return 1;
+ case POLICY_DEFAULT:
+ return link_state->bios_clk_state;
+ }
+ return 0;
+}
+
+static void pcie_set_clock_pm(struct pci_dev *pdev, int enable)
+{
+ struct pci_dev *child_dev;
+ int pos;
+ u16 reg16;
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+ if (!pos)
+ return;
+ pci_read_config_word(child_dev, pos + PCI_EXP_LNKCTL, ®16);
+ if (enable)
+ reg16 |= PCI_EXP_LNKCTL_CLKREQ_EN;
+ else
+ reg16 &= ~PCI_EXP_LNKCTL_CLKREQ_EN;
+ pci_write_config_word(child_dev, pos + PCI_EXP_LNKCTL, reg16);
+ }
+ link_state->clk_pm_enabled = !!enable;
+}
+
+static void pcie_check_clock_pm(struct pci_dev *pdev)
+{
+ int pos;
+ u32 reg32;
+ u16 reg16;
+ int capable = 1, enabled = 1;
+ struct pci_dev *child_dev;
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ /* All functions should have the same cap and state, take the worst */
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+ if (!pos)
+ return;
+ pci_read_config_dword(child_dev, pos + PCI_EXP_LNKCAP, ®32);
+ if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) {
+ capable = 0;
+ enabled = 0;
+ break;
+ }
+ pci_read_config_word(child_dev, pos + PCI_EXP_LNKCTL, ®16);
+ if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN))
+ enabled = 0;
+ }
+ link_state->clk_pm_capable = capable;
+ link_state->clk_pm_enabled = enabled;
+ link_state->bios_clk_state = enabled;
+ pcie_set_clock_pm(pdev, policy_to_clkpm_state(pdev));
+}
+
+/*
+ * pcie_aspm_configure_common_clock: check if the 2 ends of a link
+ * could use common clock. If they are, configure them to use the
+ * common clock. That will reduce the ASPM state exit latency.
+ */
+static void pcie_aspm_configure_common_clock(struct pci_dev *pdev)
+{
+ int pos, child_pos;
+ u16 reg16 = 0;
+ struct pci_dev *child_dev;
+ int same_clock = 1;
+
+ /*
+ * all functions of a slot should have the same Slot Clock
+ * Configuration, so just check one function
+ * */
+ child_dev = list_entry(pdev->subordinate->devices.next, struct pci_dev,
+ bus_list);
+ BUG_ON(!child_dev->is_pcie);
+
+ /* Check downstream component if bit Slot Clock Configuration is 1 */
+ child_pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+ pci_read_config_word(child_dev, child_pos + PCI_EXP_LNKSTA, ®16);
+ if (!(reg16 & PCI_EXP_LNKSTA_SLC))
+ same_clock = 0;
+
+ /* Check upstream component if bit Slot Clock Configuration is 1 */
+ pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
+ pci_read_config_word(pdev, pos + PCI_EXP_LNKSTA, ®16);
+ if (!(reg16 & PCI_EXP_LNKSTA_SLC))
+ same_clock = 0;
+
+ /* Configure downstream component, all functions */
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ child_pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+ pci_read_config_word(child_dev, child_pos + PCI_EXP_LNKCTL,
+ ®16);
+ if (same_clock)
+ reg16 |= PCI_EXP_LNKCTL_CCC;
+ else
+ reg16 &= ~PCI_EXP_LNKCTL_CCC;
+ pci_write_config_word(child_dev, child_pos + PCI_EXP_LNKCTL,
+ reg16);
+ }
+
+ /* Configure upstream component */
+ pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16);
+ if (same_clock)
+ reg16 |= PCI_EXP_LNKCTL_CCC;
+ else
+ reg16 &= ~PCI_EXP_LNKCTL_CCC;
+ pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16);
+
+ /* retrain link */
+ reg16 |= PCI_EXP_LNKCTL_RL;
+ pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16);
+
+ /* Wait for link training end */
+ while (1) {
+ pci_read_config_word(pdev, pos + PCI_EXP_LNKSTA, ®16);
+ if (!(reg16 & PCI_EXP_LNKSTA_LT))
+ break;
+ cpu_relax();
+ }
+}
+
+/*
+ * calc_L0S_latency: Convert L0s latency encoding to ns
+ */
+static unsigned int calc_L0S_latency(unsigned int latency_encoding, int ac)
+{
+ unsigned int ns = 64;
+
+ if (latency_encoding == 0x7) {
+ if (ac)
+ ns = -1U;
+ else
+ ns = 5*1000; /* > 4us */
+ } else
+ ns *= (1 << latency_encoding);
+ return ns;
+}
+
+/*
+ * calc_L1_latency: Convert L1 latency encoding to ns
+ */
+static unsigned int calc_L1_latency(unsigned int latency_encoding, int ac)
+{
+ unsigned int ns = 1000;
+
+ if (latency_encoding == 0x7) {
+ if (ac)
+ ns = -1U;
+ else
+ ns = 65*1000; /* > 64us */
+ } else
+ ns *= (1 << latency_encoding);
+ return ns;
+}
+
+static void pcie_aspm_get_cap_device(struct pci_dev *pdev, u32 *state,
+ unsigned int *l0s, unsigned int *l1, unsigned int *enabled)
+{
+ int pos;
+ u16 reg16;
+ u32 reg32;
+ unsigned int latency;
+
+ pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
+ pci_read_config_dword(pdev, pos + PCI_EXP_LNKCAP, ®32);
+ *state = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10;
+ if (*state != PCIE_LINK_STATE_L0S &&
+ *state != (PCIE_LINK_STATE_L1|PCIE_LINK_STATE_L0S))
+ * state = 0;
+ if (*state == 0)
+ return;
+
+ latency = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12;
+ *l0s = calc_L0S_latency(latency, 0);
+ if (*state & PCIE_LINK_STATE_L1) {
+ latency = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15;
+ *l1 = calc_L1_latency(latency, 0);
+ }
+ pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16);
+ *enabled = reg16 & (PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1);
+}
+
+static void pcie_aspm_cap_init(struct pci_dev *pdev)
+{
+ struct pci_dev *child_dev;
+ u32 state, tmp;
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ /* upstream component states */
+ pcie_aspm_get_cap_device(pdev, &link_state->support_state,
+ &link_state->l0s_upper_latency,
+ &link_state->l1_upper_latency,
+ &link_state->enabled_state);
+ /* downstream component states, all functions have the same setting */
+ child_dev = list_entry(pdev->subordinate->devices.next, struct pci_dev,
+ bus_list);
+ pcie_aspm_get_cap_device(child_dev, &state,
+ &link_state->l0s_down_latency,
+ &link_state->l1_down_latency,
+ &tmp);
+ link_state->support_state &= state;
+ if (!link_state->support_state)
+ return;
+ link_state->enabled_state &= link_state->support_state;
+ link_state->bios_aspm_state = link_state->enabled_state;
+
+ /* ENDPOINT states*/
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ int pos;
+ u32 reg32;
+ unsigned int latency;
+ struct endpoint_state *ep_state =
+ &link_state->endpoints[PCI_FUNC(child_dev->devfn)];
+
+ if (child_dev->pcie_type != PCI_EXP_TYPE_ENDPOINT &&
+ child_dev->pcie_type != PCI_EXP_TYPE_LEG_END)
+ continue;
+
+ pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+ pci_read_config_dword(child_dev, pos + PCI_EXP_DEVCAP, ®32);
+ latency = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6;
+ latency = calc_L0S_latency(latency, 1);
+ ep_state->l0s_acceptable_latency = latency;
+ if (link_state->support_state & PCIE_LINK_STATE_L1) {
+ latency = (reg32 & PCI_EXP_DEVCAP_L1) >> 9;
+ latency = calc_L1_latency(latency, 1);
+ ep_state->l1_acceptable_latency = latency;
+ }
+ }
+}
+
+static unsigned int __pcie_aspm_check_state_one(struct pci_dev *pdev,
+ unsigned int state)
+{
+ struct pci_dev *parent_dev, *tmp_dev;
+ unsigned int latency, l1_latency = 0;
+ struct pcie_link_state *link_state;
+ struct endpoint_state *ep_state;
+
+ parent_dev = pdev->bus->self;
+ link_state = parent_dev->link_state;
+ state &= link_state->support_state;
+ if (state == 0)
+ return 0;
+ ep_state = &link_state->endpoints[PCI_FUNC(pdev->devfn)];
+
+ /*
+ * Check latency for endpoint device.
+ * TBD: The latency from the endpoint to root complex vary per
+ * switch's upstream link state above the device. Here we just do a
+ * simple check which assumes all links above the device can be in L1
+ * state, that is we just consider the worst case. If switch's upstream
+ * link can't be put into L0S/L1, then our check is too strictly.
+ */
+ tmp_dev = pdev;
+ while (state & (PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1)) {
+ parent_dev = tmp_dev->bus->self;
+ link_state = parent_dev->link_state;
+ if (state & PCIE_LINK_STATE_L0S) {
+ latency = max_t(unsigned int,
+ link_state->l0s_upper_latency,
+ link_state->l0s_down_latency);
+ if (latency > ep_state->l0s_acceptable_latency)
+ state &= ~PCIE_LINK_STATE_L0S;
+ }
+ if (state & PCIE_LINK_STATE_L1) {
+ latency = max_t(unsigned int,
+ link_state->l1_upper_latency,
+ link_state->l1_down_latency);
+ if (latency + l1_latency >
+ ep_state->l1_acceptable_latency)
+ state &= ~PCIE_LINK_STATE_L1;
+ }
+ if (!parent_dev->bus->self) /* parent_dev is a root port */
+ break;
+ else {
+ /*
+ * parent_dev is the downstream port of a switch, make
+ * tmp_dev the upstream port of the switch
+ */
+ tmp_dev = parent_dev->bus->self;
+ /*
+ * every switch on the path to root complex need 1 more
+ * microsecond for L1. Spec doesn't mention L0S.
+ */
+ if (state & PCIE_LINK_STATE_L1)
+ l1_latency += 1000;
+ }
+ }
+ return state;
+}
+
+static unsigned int pcie_aspm_check_state(struct pci_dev *pdev,
+ unsigned int state)
+{
+ struct pci_dev *child_dev;
+
+ /* If no child, disable the link */
+ if (list_empty(&pdev->subordinate->devices))
+ return 0;
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ if (child_dev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE) {
+ /*
+ * If downstream component of a link is pci bridge, we
+ * disable ASPM for now for the link
+ * */
+ state = 0;
+ break;
+ }
+ if ((child_dev->pcie_type != PCI_EXP_TYPE_ENDPOINT &&
+ child_dev->pcie_type != PCI_EXP_TYPE_LEG_END))
+ continue;
+ /* Device not in D0 doesn't need check latency */
+ if (child_dev->current_state == PCI_D1 ||
+ child_dev->current_state == PCI_D2 ||
+ child_dev->current_state == PCI_D3hot ||
+ child_dev->current_state == PCI_D3cold)
+ continue;
+ state = __pcie_aspm_check_state_one(child_dev, state);
+ }
+ return state;
+}
+
+static void __pcie_aspm_config_one_dev(struct pci_dev *pdev, unsigned int state)
+{
+ u16 reg16;
+ int pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
+
+ pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16);
+ reg16 &= ~0x3;
+ reg16 |= state;
+ pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16);
+}
+
+static void __pcie_aspm_config_link(struct pci_dev *pdev, unsigned int state)
+{
+ struct pci_dev *child_dev;
+ int valid = 1;
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ /*
+ * if the downstream component has pci bridge function, don't do ASPM
+ * now
+ */
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ if (child_dev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE) {
+ valid = 0;
+ break;
+ }
+ }
+ if (!valid)
+ return;
+
+ /*
+ * spec 2.0 suggests all functions should be configured the same
+ * setting for ASPM. Enabling ASPM L1 should be done in upstream
+ * component first and then downstream, and vice versa for disabling
+ * ASPM L1. Spec doesn't mention L0S.
+ */
+ if (state & PCIE_LINK_STATE_L1)
+ __pcie_aspm_config_one_dev(pdev, state);
+
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list)
+ __pcie_aspm_config_one_dev(child_dev, state);
+
+ if (!(state & PCIE_LINK_STATE_L1))
+ __pcie_aspm_config_one_dev(pdev, state);
+
+ link_state->enabled_state = state;
+}
+
+static void __pcie_aspm_configure_link_state(struct pci_dev *pdev,
+ unsigned int state)
+{
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ if (link_state->support_state == 0)
+ return;
+ state &= PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1;
+
+ /* state 0 means disabling aspm */
+ state = pcie_aspm_check_state(pdev, state);
+ if (link_state->enabled_state == state)
+ return;
+ __pcie_aspm_config_link(pdev, state);
+}
+
+/*
+ * pcie_aspm_configure_link_state: enable/disable PCI express link state
+ * @pdev: the root port or switch downstream port
+ */
+static void pcie_aspm_configure_link_state(struct pci_dev *pdev,
+ unsigned int state)
+{
+ down_read(&pci_bus_sem);
+ mutex_lock(&aspm_lock);
+ __pcie_aspm_configure_link_state(pdev, state);
+ mutex_unlock(&aspm_lock);
+ up_read(&pci_bus_sem);
+}
+
+static void free_link_state(struct pci_dev *pdev)
+{
+ kfree(pdev->link_state);
+ pdev->link_state = NULL;
+}
+
+/*
+ * pcie_aspm_init_link_state: Initiate PCI express link state.
+ * It is called after the pcie and its children devices are scaned.
+ * @pdev: the root port or switch downstream port
+ */
+void pcie_aspm_init_link_state(struct pci_dev *pdev)
+{
+ unsigned int state;
+ struct pcie_link_state *link_state;
+ int error = 0;
+
+ if (aspm_disabled || !pdev->is_pcie || pdev->link_state)
+ return;
+ if (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
+ pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
+ return;
+ down_read(&pci_bus_sem);
+ if (list_empty(&pdev->subordinate->devices))
+ goto out;
+
+ mutex_lock(&aspm_lock);
+
+ link_state = kzalloc(sizeof(*link_state), GFP_KERNEL);
+ if (!link_state)
+ goto unlock_out;
+ pdev->link_state = link_state;
+
+ pcie_aspm_configure_common_clock(pdev);
+
+ pcie_aspm_cap_init(pdev);
+
+ /* config link state to avoid BIOS error */
+ state = pcie_aspm_check_state(pdev, policy_to_aspm_state(pdev));
+ __pcie_aspm_config_link(pdev, state);
+
+ pcie_check_clock_pm(pdev);
+
+ link_state->pdev = pdev;
+ list_add(&link_state->sibiling, &link_list);
+
+unlock_out:
+ if (error)
+ free_link_state(pdev);
+ mutex_unlock(&aspm_lock);
+out:
+ up_read(&pci_bus_sem);
+}
+
+/* @pdev: the endpoint device */
+void pcie_aspm_exit_link_state(struct pci_dev *pdev)
+{
+ struct pci_dev *parent = pdev->bus->self;
+ struct pcie_link_state *link_state = parent->link_state;
+
+ if (aspm_disabled || !pdev->is_pcie || !parent || !link_state)
+ return;
+ if (parent->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
+ parent->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
+ return;
+ down_read(&pci_bus_sem);
+ mutex_lock(&aspm_lock);
+
+ /*
+ * All PCIe functions are in one slot, remove one function will remove
+ * the the whole slot, so just wait
+ */
+ if (!list_empty(&parent->subordinate->devices))
+ goto out;
+
+ /* All functions are removed, so just disable ASPM for the link */
+ __pcie_aspm_config_one_dev(parent, 0);
+ list_del(&link_state->sibiling);
+ /* Clock PM is for endpoint device */
+
+ free_link_state(parent);
+out:
+ mutex_unlock(&aspm_lock);
+ up_read(&pci_bus_sem);
+}
+
+/* @pdev: the root port or switch downstream port */
+void pcie_aspm_pm_state_change(struct pci_dev *pdev)
+{
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ if (aspm_disabled || !pdev->is_pcie || !pdev->link_state)
+ return;
+ if (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
+ pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
+ return;
+ /*
+ * devices changed PM state, we should recheck if latency meets all
+ * functions' requirement
+ */
+ pcie_aspm_configure_link_state(pdev, link_state->enabled_state);
+}
+
+/*
+ * pci_disable_link_state - disable pci device's link state, so the link will
+ * never enter specific states
+ */
+void pci_disable_link_state(struct pci_dev *pdev, int state)
+{
+ struct pci_dev *parent = pdev->bus->self;
+ struct pcie_link_state *link_state;
+
+ if (aspm_disabled || !pdev->is_pcie)
+ return;
+ if (pdev->pcie_type == PCI_EXP_TYPE_ROOT_PORT ||
+ pdev->pcie_type == PCI_EXP_TYPE_DOWNSTREAM)
+ parent = pdev;
+ if (!parent)
+ return;
+
+ down_read(&pci_bus_sem);
+ mutex_lock(&aspm_lock);
+ link_state = parent->link_state;
+ link_state->support_state &=
+ ~(state & (PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1));
+ if (state & PCIE_LINK_STATE_CLKPM)
+ link_state->clk_pm_capable = 0;
+
+ __pcie_aspm_configure_link_state(parent, link_state->enabled_state);
+ if (!link_state->clk_pm_capable && link_state->clk_pm_enabled)
+ pcie_set_clock_pm(parent, 0);
+ mutex_unlock(&aspm_lock);
+ up_read(&pci_bus_sem);
+}
+EXPORT_SYMBOL(pci_disable_link_state);
+
+static int pcie_aspm_set_policy(const char *val, struct kernel_param *kp)
+{
+ int i;
+ struct pci_dev *pdev;
+ struct pcie_link_state *link_state;
+
+ for (i = 0; i < ARRAY_SIZE(policy_str); i++)
+ if (!strncmp(val, policy_str[i], strlen(policy_str[i])))
+ break;
+ if (i >= ARRAY_SIZE(policy_str))
+ return -EINVAL;
+ if (i == aspm_policy)
+ return 0;
+
+ down_read(&pci_bus_sem);
+ mutex_lock(&aspm_lock);
+ aspm_policy = i;
+ list_for_each_entry(link_state, &link_list, sibiling) {
+ pdev = link_state->pdev;
+ __pcie_aspm_configure_link_state(pdev,
+ policy_to_aspm_state(pdev));
+ if (link_state->clk_pm_capable &&
+ link_state->clk_pm_enabled != policy_to_clkpm_state(pdev))
+ pcie_set_clock_pm(pdev, policy_to_clkpm_state(pdev));
+
+ }
+ mutex_unlock(&aspm_lock);
+ up_read(&pci_bus_sem);
+ return 0;
+}
+
+static int pcie_aspm_get_policy(char *buffer, struct kernel_param *kp)
+{
+ int i, cnt = 0;
+ for (i = 0; i < ARRAY_SIZE(policy_str); i++)
+ if (i == aspm_policy)
+ cnt += sprintf(buffer + cnt, "[%s] ", policy_str[i]);
+ else
+ cnt += sprintf(buffer + cnt, "%s ", policy_str[i]);
+ return cnt;
+}
+
+module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy,
+ NULL, 0644);
+
+#ifdef CONFIG_PCIEASPM_DEBUG
+static ssize_t link_state_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct pci_dev *pci_device = to_pci_dev(dev);
+ struct pcie_link_state *link_state = pci_device->link_state;
+
+ return sprintf(buf, "%d\n", link_state->enabled_state);
+}
+
+static ssize_t link_state_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf,
+ size_t n)
+{
+ struct pci_dev *pci_device = to_pci_dev(dev);
+ int state;
+
+ if (n < 1)
+ return -EINVAL;
+ state = buf[0]-'0';
+ if (state >= 0 && state <= 3) {
+ /* setup link aspm state */
+ pcie_aspm_configure_link_state(pci_device, state);
+ return n;
+ }
+
+ return -EINVAL;
+}
+
+static ssize_t clk_ctl_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct pci_dev *pci_device = to_pci_dev(dev);
+ struct pcie_link_state *link_state = pci_device->link_state;
+
+ return sprintf(buf, "%d\n", link_state->clk_pm_enabled);
+}
+
+static ssize_t clk_ctl_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf,
+ size_t n)
+{
+ struct pci_dev *pci_device = to_pci_dev(dev);
+ int state;
+
+ if (n < 1)
+ return -EINVAL;
+ state = buf[0]-'0';
+
+ down_read(&pci_bus_sem);
+ mutex_lock(&aspm_lock);
+ pcie_set_clock_pm(pci_device, !!state);
+ mutex_unlock(&aspm_lock);
+ up_read(&pci_bus_sem);
+
+ return n;
+}
+
+static DEVICE_ATTR(link_state, 0644, link_state_show, link_state_store);
+static DEVICE_ATTR(clk_ctl, 0644, clk_ctl_show, clk_ctl_store);
+
+static char power_group[] = "power";
+void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev)
+{
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ if (!pdev->is_pcie || (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
+ pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM))
+ return;
+
+ if (link_state->support_state)
+ sysfs_add_file_to_group(&pdev->dev.kobj,
+ &dev_attr_link_state.attr, power_group);
+ if (link_state->clk_pm_capable)
+ sysfs_add_file_to_group(&pdev->dev.kobj,
+ &dev_attr_clk_ctl.attr, power_group);
+}
+
+void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev)
+{
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ if (!pdev->is_pcie || (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
+ pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM))
+ return;
+
+ if (link_state->support_state)
+ sysfs_remove_file_from_group(&pdev->dev.kobj,
+ &dev_attr_link_state.attr, power_group);
+ if (link_state->clk_pm_capable)
+ sysfs_remove_file_from_group(&pdev->dev.kobj,
+ &dev_attr_clk_ctl.attr, power_group);
+}
+#endif
+
+static int __init pcie_aspm_disable(char *str)
+{
+ aspm_disabled = 1;
+ return 1;
+}
+
+__setup("pcie_noaspm", pcie_aspm_disable);
+
+static int __init pcie_aspm_init(void)
+{
+ if (aspm_disabled)
+ return 0;
+ pci_osc_support_set(OSC_ACTIVE_STATE_PWR_SUPPORT|
+ OSC_CLOCK_PWR_CAPABILITY_SUPPORT);
+ return 0;
+}
+
+fs_initcall(pcie_aspm_init);
--- a/drivers/pci/probe.c
+++ b/drivers/pci/probe.c
@@ -9,6 +9,7 @@
#include <linux/slab.h>
#include <linux/module.h>
#include <linux/cpumask.h>
+#include <linux/aspm.h>
#include "pci.h"
#define CARDBUS_LATENCY_TIMER 176 /* secondary latency timer */
@@ -1001,6 +1002,10 @@ int pci_scan_slot(struct pci_bus *bus, i
break;
}
}
+
+ if (bus->self)
+ pcie_aspm_init_link_state(bus->self);
+
return nr;
}
--- a/drivers/pci/remove.c
+++ b/drivers/pci/remove.c
@@ -1,5 +1,6 @@
#include <linux/pci.h>
#include <linux/module.h>
+#include <linux/aspm.h>
#include "pci.h"
static void pci_free_resources(struct pci_dev *dev)
@@ -30,6 +31,9 @@ static void pci_stop_dev(struct pci_dev
dev->global_list.next = dev->global_list.prev = NULL;
up_write(&pci_bus_sem);
}
+
+ if (dev->bus->self)
+ pcie_aspm_exit_link_state(dev);
}
static void pci_destroy_dev(struct pci_dev *dev)
--- /dev/null
+++ b/include/linux/aspm.h
@@ -0,0 +1,44 @@
+/*
+ * aspm.h
+ *
+ * PCI Express ASPM defines and function prototypes
+ *
+ * Copyright (C) 2007 Intel Corp.
+ * Zhang Yanmin ([email protected])
+ * Shaohua Li ([email protected])
+ *
+ * For more information, please consult the following manuals (look at
+ * http://www.pcisig.com/ for how to get them):
+ *
+ * PCI Express Specification
+ */
+
+#ifndef LINUX_ASPM_H
+#define LINUX_ASPM_H
+
+#include <linux/pci.h>
+
+#define PCIE_LINK_STATE_L0S 1
+#define PCIE_LINK_STATE_L1 2
+#define PCIE_LINK_STATE_CLKPM 4
+
+#ifdef CONFIG_PCIEASPM
+extern void pcie_aspm_init_link_state(struct pci_dev *pdev);
+extern void pcie_aspm_exit_link_state(struct pci_dev *pdev);
+extern void pcie_aspm_pm_state_change(struct pci_dev *pdev);
+extern void pci_disable_link_state(struct pci_dev *pdev, int state);
+#else
+#define pcie_aspm_init_link_state(pdev) do {} while (0)
+#define pcie_aspm_exit_link_state(pdev) do {} while (0)
+#define pcie_aspm_pm_state_change(pdev) do {} while (0)
+#define pci_disable_link_state(pdev, state) do {} while (0)
+#endif
+
+#ifdef CONFIG_PCIEASPM_DEBUG /* this depends on CONFIG_PCIEASPM */
+extern void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev);
+extern void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev);
+#else
+#define pcie_aspm_create_sysfs_dev_files(pdev) do {} while (0)
+#define pcie_aspm_remove_sysfs_dev_files(pdev) do {} while (0)
+#endif
+#endif /* LINUX_ASPM_H */
--- a/include/linux/pci.h
+++ b/include/linux/pci.h
@@ -128,6 +128,7 @@ struct pci_cap_saved_state {
u32 data[0];
};
+struct pcie_link_state;
/*
* The pci_dev structure is used to describe PCI devices.
*/
@@ -163,6 +164,10 @@ struct pci_dev {
this is D0-D3, D0 being fully functional,
and D3 being off. */
+#ifdef CONFIG_PCIEASPM
+ struct pcie_link_state *link_state; /* ASPM link state. */
+#endif
+
pci_channel_state_t error_state; /* current connectivity state */
struct device dev; /* Generic device interface */
--- a/include/linux/pci_regs.h
+++ b/include/linux/pci_regs.h
@@ -395,9 +395,17 @@
#define PCI_EXP_DEVSTA_AUXPD 0x10 /* AUX Power Detected */
#define PCI_EXP_DEVSTA_TRPND 0x20 /* Transactions Pending */
#define PCI_EXP_LNKCAP 12 /* Link Capabilities */
+#define PCI_EXP_LNKCAP_ASPMS 0xc00 /* ASPM Support */
+#define PCI_EXP_LNKCAP_L0SEL 0x7000 /* L0s Exit Latency */
+#define PCI_EXP_LNKCAP_L1EL 0x38000 /* L1 Exit Latency */
+#define PCI_EXP_LNKCAP_CLKPM 0x40000 /* L1 Clock Power Management */
#define PCI_EXP_LNKCTL 16 /* Link Control */
+#define PCI_EXP_LNKCTL_RL 0x20 /* Retrain Link */
+#define PCI_EXP_LNKCTL_CCC 0x40 /* Common Clock COnfiguration */
#define PCI_EXP_LNKCTL_CLKREQ_EN 0x100 /* Enable clkreq */
#define PCI_EXP_LNKSTA 18 /* Link Status */
+#define PCI_EXP_LNKSTA_LT 0x800 /* Link Training */
+#define PCI_EXP_LNKSTA_SLC 0x1000 /* Slot Clock Configuration */
#define PCI_EXP_SLTCAP 20 /* Slot Capabilities */
#define PCI_EXP_SLTCTL 24 /* Slot Control */
#define PCI_EXP_SLTSTA 26 /* Slot Status */
Patches currently in gregkh-2.6 which might be from [email protected] are
driver/kobject-change-drivers-cpuidle-sysfs.c-to-use-kobject_init_and_add.patch
pci/pcie-port-driver-correctly-detect-native-pme-feature.patch
pci/pcie-utilize-pcie-transaction-pending-bit.patch
pci/pci-avoid-save-the-same-type-of-cap-multiple-times.patch
pci/pci-correctly-initialize-a-structure-for-pcie_save_pcix_state.patch
pci/pci-fix-typo-in-pci_save_pcix_state.patch
pci/pci-pcie-aspm-support.patch
Pavel Machek wrote:
> Hi!
>
>> v3->v2, fixed the issues Matthew Wilcox raised.
>>
>> PCI Express ASPM defines a protocol for PCI Express components in the D0
>> state to reduce Link power by placing their Links into a low power state
>> and instructing the other end of the Link to do likewise. This
>> capability allows hardware-autonomous, dynamic Link power reduction
>> beyond what is achievable by software-only controlled power management.
>> However, The device should be configured by software appropriately.
>> Enabling ASPM will save power, but will introduce device latency.
>
> How big is the latency? 1msec? 10msec? 100usec?
the latency is different for each device but the timing is negotiated and the
pci-e spec lists possible timings that can be used. The maximum is (I think...)
64usec but can be as low as 1 or 2 usec.
Auke
* [email protected] <[email protected]> wrote:
> This is a note to let you know that I've just added the patch titled
>
> Subject: PCI: PCIE ASPM support
>
> to my gregkh-2.6 tree. Its filename is
>
> pci-pcie-aspm-support.patch
uhm. One week ago this patch was added to your PCI tree. It never
touched -mm AFAICS and today it was merged upstream (commit
6c723d5bd89f03fc3ef627d50f89ade054d2ee3b):
Which is not necessarily a problem in itself, as long as you test it
through and are reasonably sure that it wont break systems en masse. But
this patch very evidently was not tested in any sufficient manner on its
primary platform (x86) because my randconfig testsystems (bog standard
x86 hw) started crashing during bootup almost immediately:
[ 22.716747] initcall 0xc0a7b517 ran for 0 msecs: random32_reseed+0x0/0x24()
[ 22.722413] Calling initcall 0xc0a7bac1: pci_sysfs_init+0x0/0x44()
[ 22.730099] BUG: unable to handle kernel NULL pointer dereference at 0000000c
[ 22.737093] IP: [<c034226b>] pcie_aspm_create_sysfs_dev_files+0x1e/0x58
[ 22.740003] *pde = 00000000
[ 22.742848] Oops: 0000 [#1] PREEMPT DEBUG_PAGEALLOC
[ 22.747699]
[ 22.749174] Pid: 1, comm: swapper Not tainted (2.6.24 #5)
[ 22.750000] EIP: 0060:[<c034226b>] EFLAGS: 00010246 CPU: 0
[ 22.750000] EIP is at pcie_aspm_create_sysfs_dev_files+0x1e/0x58
[ 22.750000] EAX: c32cf504 EBX: c32cf5d0 ECX: 00000000 EDX: 00000000
[ 22.750000] ESI: 00000000 EDI: c32cf5d0 EBP: c327df50 ESP: c327df48
[ 22.750000] DS: 007b ES: 007b FS: 0000 GS: 0000 SS: 0068
[ 22.750000] Process swapper (pid: 1, ti=c327d000 task=c327c000 task.ti=c327d000)
[ 22.750000] Stack: 00000000 00000000 c327df84 c0340c38 c327df74 c0340188 ffffffff c32cf708
[ 22.750000] 00000000 00000000 00000006 c32cf6a8 00000000 c32cf5d0 00000000 c327df94
[ 22.750000] c0a7badb 00000000 00000000 c327dfe0 c0a6c673 dbc866c8 00000008 00000000
[ 22.750000] Call Trace:
[ 22.750000] [<c0340c38>] ? pci_create_sysfs_dev_files+0x24f/0x2f2
[ 22.750000] [<c0340188>] ? pci_get_subsys+0xc2/0xcb
[ 22.750000] [<c0a7badb>] ? pci_sysfs_init+0x1a/0x44
[ 22.750000] [<c0a6c673>] ? kernel_init+0xb2/0x20b
[ 22.750000] [<c0a6c5c1>] ? kernel_init+0x0/0x20b
[ 22.750000] [<c01057a7>] ? kernel_thread_helper+0x7/0x10
[ 22.750000] =======================
[ 22.750000] Code: 84 16 9b c0 e8 2d 6b e4 ff 5b 5e 5d c3 55 89 e5 56 53 89 c3 8b 70 48 f6 80 dd 03 00 00 04 74 41 8a 40 32 3c 04 74 04 3c 06 75 36 <83> 7e 0c 00 74 15 8d 83 38 01 00 00 b9 7c 16 9b c0 ba 68 16 9b
[ 22.750000] EIP: [<c034226b>] pcie_aspm_create_sysfs_dev_files+0x1e/0x58 SS:ESP 0068:c327df48
[ 22.750003] ---[ end trace 204172de9b5128e1 ]---
config attached. The crash is fully repeatable on multiple systems. (can
try any test-patch)
To add insult to injury, the new option is also default-enabled _AND_
experimental:
config PCIEASPM
bool "PCI Express ASPM support(Experimental)"
depends on PCI && EXPERIMENTAL
default y
and it only depends on PCI, so this will crash just about every system
out there that tracks -git.
please ...
i've punched it out of the randconfig space via the hack below, to keep
the tests going.
Ingo
Index: linux/drivers/pci/pcie/Kconfig
===================================================================
--- linux.orig/drivers/pci/pcie/Kconfig
+++ linux/drivers/pci/pcie/Kconfig
@@ -33,6 +33,10 @@ source "drivers/pci/pcie/aer/Kconfig"
config PCIEASPM
bool "PCI Express ASPM support(Experimental)"
depends on PCI && EXPERIMENTAL
+
+ # bootup crashes, Sat Feb 2 11:37:58 CET 2008
+ depends on 0
+
default y
help
This enables PCI Express ASPM (Active State Power Management) and
On Sat, Feb 02, 2008 at 11:55:06AM +0100, Ingo Molnar wrote:
>
> * [email protected] <[email protected]> wrote:
>
> > This is a note to let you know that I've just added the patch titled
> >
> > Subject: PCI: PCIE ASPM support
> >
> > to my gregkh-2.6 tree. Its filename is
> >
> > pci-pcie-aspm-support.patch
>
> uhm. One week ago this patch was added to your PCI tree. It never
> touched -mm AFAICS and today it was merged upstream (commit
> 6c723d5bd89f03fc3ef627d50f89ade054d2ee3b):
>
> Which is not necessarily a problem in itself, as long as you test it
> through and are reasonably sure that it wont break systems en masse. But
> this patch very evidently was not tested in any sufficient manner on its
> primary platform (x86) because my randconfig testsystems (bog standard
> x86 hw) started crashing during bootup almost immediately:
Ugh, this is causing just too many problems, I'm just going to revert
it.
Shaohua, care to look into this crash, fix up the config issues, and
resubmit it when it's working a bit better?
thanks,
greg k-h
* Greg KH <[email protected]> wrote:
> Ugh, this is causing just too many problems, I'm just going to revert
> it.
>
> Shaohua, care to look into this crash, fix up the config issues, and
> resubmit it when it's working a bit better?
i can test any patches, the crash was fully reproducible. It's probably
something rather trivial - i dont have the hardware so it should just
have bailed out.
Ingo
On Sun, 2008-02-03 at 02:51 +0800, Greg KH wrote:
> On Sat, Feb 02, 2008 at 11:55:06AM +0100, Ingo Molnar wrote:
> >
> > * [email protected] <[email protected]> wrote:
> >
> > > This is a note to let you know that I've just added the patch
> titled
> > >
> > > Subject: PCI: PCIE ASPM support
> > >
> > > to my gregkh-2.6 tree. Its filename is
> > >
> > > pci-pcie-aspm-support.patch
> >
> > uhm. One week ago this patch was added to your PCI tree. It never
> > touched -mm AFAICS and today it was merged upstream (commit
> > 6c723d5bd89f03fc3ef627d50f89ade054d2ee3b):
> >
> > Which is not necessarily a problem in itself, as long as you test it
> > through and are reasonably sure that it wont break systems en masse.
> But
> > this patch very evidently was not tested in any sufficient manner on
> its
> > primary platform (x86) because my randconfig testsystems (bog
> standard
> > x86 hw) started crashing during bootup almost immediately:
>
> Ugh, this is causing just too many problems, I'm just going to revert
> it.
>
> Shaohua, care to look into this crash, fix up the config issues, and
> resubmit it when it's working a bit better?
Sorry for the long delay, I just back from vocation. I fixed all the
issues I found in the list (except one hang issue, which should be fixed
in specific driver, I'll reply that thread soon), can you re-add the
patch to your test tree.
PCI Express ASPM defines a protocol for PCI Express components in the D0
state to reduce Link power by placing their Links into a low power state
and instructing the other end of the Link to do likewise. This
capability allows hardware-autonomous, dynamic Link power reduction
beyond what is achievable by software-only controlled power management.
However, The device should be configured by software appropriately.
Enabling ASPM will save power, but will introduce device latency.
This patch adds ASPM support in Linux. It introduces a global policy for
ASPM, a sysfs file /sys/module/pcie_aspm/parameters/policy can control
it. The interface can be used as a boot option too. Currently we have
below setting:
-default, BIOS default setting
-powersave, highest power saving mode, enable all available ASPM
state and clock power management
-performance, highest performance, disable ASPM and clock power
management
By default, the 'default' policy is used currently.
In my test, power difference between powersave mode and performance mode
is about 1.3w in a system with 3 PCIE links.
Note: some devices might not work well with aspm, either because chipset
issue or device issue. The patch provide API (pci_disable_link_state),
driver can disable ASPM for specific device.
Signed-off-by: Shaohua Li <[email protected]>
---
drivers/pci/pci-sysfs.c | 5
drivers/pci/pci.c | 4
drivers/pci/pcie/Kconfig | 20 +
drivers/pci/pcie/Makefile | 3
drivers/pci/pcie/aspm.c | 801 ++++++++++++++++++++++++++++++++++++++++++++++
drivers/pci/probe.c | 5
drivers/pci/remove.c | 4
include/linux/aspm.h | 56 +++
include/linux/pci-acpi.h | 1
include/linux/pci.h | 5
include/linux/pci_regs.h | 8
11 files changed, 912 insertions(+)
Index: linux/drivers/pci/pcie/Makefile
===================================================================
--- linux.orig/drivers/pci/pcie/Makefile 2008-01-24 10:23:09.000000000 +0800
+++ linux/drivers/pci/pcie/Makefile 2008-02-19 10:54:12.000000000 +0800
@@ -2,6 +2,9 @@
# Makefile for PCI-Express PORT Driver
#
+# Build PCI Express ASPM if needed
+obj-$(CONFIG_PCIEASPM) += aspm.o
+
pcieportdrv-y := portdrv_core.o portdrv_pci.o portdrv_bus.o
obj-$(CONFIG_PCIEPORTBUS) += pcieportdrv.o
Index: linux/drivers/pci/pcie/aspm.c
===================================================================
--- /dev/null 1970-01-01 00:00:00.000000000 +0000
+++ linux/drivers/pci/pcie/aspm.c 2008-02-19 13:35:55.000000000 +0800
@@ -0,0 +1,801 @@
+/*
+ * File: drivers/pci/pcie/aspm.c
+ * Enabling PCIE link L0s/L1 state and Clock Power Management
+ *
+ * Copyright (C) 2007 Intel
+ * Copyright (C) Zhang Yanmin ([email protected])
+ * Copyright (C) Shaohua Li ([email protected])
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/pci.h>
+#include <linux/pci_regs.h>
+#include <linux/errno.h>
+#include <linux/pm.h>
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <linux/aspm.h>
+#include <linux/pci-acpi.h>
+#include "../pci.h"
+
+#ifdef MODULE_PARAM_PREFIX
+#undef MODULE_PARAM_PREFIX
+#endif
+#define MODULE_PARAM_PREFIX "pcie_aspm."
+
+struct endpoint_state {
+ unsigned int l0s_acceptable_latency;
+ unsigned int l1_acceptable_latency;
+};
+
+struct pcie_link_state {
+ struct list_head sibiling;
+ struct pci_dev *pdev;
+
+ /* ASPM state */
+ unsigned int support_state;
+ unsigned int enabled_state;
+ unsigned int bios_aspm_state;
+ /* upstream component */
+ unsigned int l0s_upper_latency;
+ unsigned int l1_upper_latency;
+ /* downstream component */
+ unsigned int l0s_down_latency;
+ unsigned int l1_down_latency;
+ /* Clock PM state*/
+ unsigned int clk_pm_capable;
+ unsigned int clk_pm_enabled;
+ unsigned int bios_clk_state;
+
+ /*
+ * A pcie downstream port only has one slot under it, so at most there
+ * are 8 functions
+ */
+ struct endpoint_state endpoints[8];
+};
+
+static int aspm_disabled;
+static DEFINE_MUTEX(aspm_lock);
+static LIST_HEAD(link_list);
+
+#define POLICY_DEFAULT 0 /* BIOS default setting */
+#define POLICY_PERFORMANCE 1 /* high performance */
+#define POLICY_POWERSAVE 2 /* high power saving */
+static int aspm_policy;
+static const char *policy_str[] = {
+ [POLICY_DEFAULT] = "default",
+ [POLICY_PERFORMANCE] = "performance",
+ [POLICY_POWERSAVE] = "powersave"
+};
+
+static int policy_to_aspm_state(struct pci_dev *pdev)
+{
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ switch (aspm_policy) {
+ case POLICY_PERFORMANCE:
+ /* Disable ASPM and Clock PM */
+ return 0;
+ case POLICY_POWERSAVE:
+ /* Enable ASPM L0s/L1 */
+ return PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1;
+ case POLICY_DEFAULT:
+ return link_state->bios_aspm_state;
+ }
+ return 0;
+}
+
+static int policy_to_clkpm_state(struct pci_dev *pdev)
+{
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ switch (aspm_policy) {
+ case POLICY_PERFORMANCE:
+ /* Disable ASPM and Clock PM */
+ return 0;
+ case POLICY_POWERSAVE:
+ /* Disable Clock PM */
+ return 1;
+ case POLICY_DEFAULT:
+ return link_state->bios_clk_state;
+ }
+ return 0;
+}
+
+static void pcie_set_clock_pm(struct pci_dev *pdev, int enable)
+{
+ struct pci_dev *child_dev;
+ int pos;
+ u16 reg16;
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+ if (!pos)
+ return;
+ pci_read_config_word(child_dev, pos + PCI_EXP_LNKCTL, ®16);
+ if (enable)
+ reg16 |= PCI_EXP_LNKCTL_CLKREQ_EN;
+ else
+ reg16 &= ~PCI_EXP_LNKCTL_CLKREQ_EN;
+ pci_write_config_word(child_dev, pos + PCI_EXP_LNKCTL, reg16);
+ }
+ link_state->clk_pm_enabled = !!enable;
+}
+
+static void pcie_check_clock_pm(struct pci_dev *pdev)
+{
+ int pos;
+ u32 reg32;
+ u16 reg16;
+ int capable = 1, enabled = 1;
+ struct pci_dev *child_dev;
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ /* All functions should have the same cap and state, take the worst */
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+ if (!pos)
+ return;
+ pci_read_config_dword(child_dev, pos + PCI_EXP_LNKCAP, ®32);
+ if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) {
+ capable = 0;
+ enabled = 0;
+ break;
+ }
+ pci_read_config_word(child_dev, pos + PCI_EXP_LNKCTL, ®16);
+ if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN))
+ enabled = 0;
+ }
+ link_state->clk_pm_capable = capable;
+ link_state->clk_pm_enabled = enabled;
+ link_state->bios_clk_state = enabled;
+ pcie_set_clock_pm(pdev, policy_to_clkpm_state(pdev));
+}
+
+/*
+ * pcie_aspm_configure_common_clock: check if the 2 ends of a link
+ * could use common clock. If they are, configure them to use the
+ * common clock. That will reduce the ASPM state exit latency.
+ */
+static void pcie_aspm_configure_common_clock(struct pci_dev *pdev)
+{
+ int pos, child_pos;
+ u16 reg16 = 0;
+ struct pci_dev *child_dev;
+ int same_clock = 1;
+
+ /*
+ * all functions of a slot should have the same Slot Clock
+ * Configuration, so just check one function
+ * */
+ child_dev = list_entry(pdev->subordinate->devices.next, struct pci_dev,
+ bus_list);
+ BUG_ON(!child_dev->is_pcie);
+
+ /* Check downstream component if bit Slot Clock Configuration is 1 */
+ child_pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+ pci_read_config_word(child_dev, child_pos + PCI_EXP_LNKSTA, ®16);
+ if (!(reg16 & PCI_EXP_LNKSTA_SLC))
+ same_clock = 0;
+
+ /* Check upstream component if bit Slot Clock Configuration is 1 */
+ pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
+ pci_read_config_word(pdev, pos + PCI_EXP_LNKSTA, ®16);
+ if (!(reg16 & PCI_EXP_LNKSTA_SLC))
+ same_clock = 0;
+
+ /* Configure downstream component, all functions */
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ child_pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+ pci_read_config_word(child_dev, child_pos + PCI_EXP_LNKCTL,
+ ®16);
+ if (same_clock)
+ reg16 |= PCI_EXP_LNKCTL_CCC;
+ else
+ reg16 &= ~PCI_EXP_LNKCTL_CCC;
+ pci_write_config_word(child_dev, child_pos + PCI_EXP_LNKCTL,
+ reg16);
+ }
+
+ /* Configure upstream component */
+ pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16);
+ if (same_clock)
+ reg16 |= PCI_EXP_LNKCTL_CCC;
+ else
+ reg16 &= ~PCI_EXP_LNKCTL_CCC;
+ pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16);
+
+ /* retrain link */
+ reg16 |= PCI_EXP_LNKCTL_RL;
+ pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16);
+
+ /* Wait for link training end */
+ while (1) {
+ pci_read_config_word(pdev, pos + PCI_EXP_LNKSTA, ®16);
+ if (!(reg16 & PCI_EXP_LNKSTA_LT))
+ break;
+ cpu_relax();
+ }
+}
+
+/*
+ * calc_L0S_latency: Convert L0s latency encoding to ns
+ */
+static unsigned int calc_L0S_latency(unsigned int latency_encoding, int ac)
+{
+ unsigned int ns = 64;
+
+ if (latency_encoding == 0x7) {
+ if (ac)
+ ns = -1U;
+ else
+ ns = 5*1000; /* > 4us */
+ } else
+ ns *= (1 << latency_encoding);
+ return ns;
+}
+
+/*
+ * calc_L1_latency: Convert L1 latency encoding to ns
+ */
+static unsigned int calc_L1_latency(unsigned int latency_encoding, int ac)
+{
+ unsigned int ns = 1000;
+
+ if (latency_encoding == 0x7) {
+ if (ac)
+ ns = -1U;
+ else
+ ns = 65*1000; /* > 64us */
+ } else
+ ns *= (1 << latency_encoding);
+ return ns;
+}
+
+static void pcie_aspm_get_cap_device(struct pci_dev *pdev, u32 *state,
+ unsigned int *l0s, unsigned int *l1, unsigned int *enabled)
+{
+ int pos;
+ u16 reg16;
+ u32 reg32;
+ unsigned int latency;
+
+ pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
+ pci_read_config_dword(pdev, pos + PCI_EXP_LNKCAP, ®32);
+ *state = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10;
+ if (*state != PCIE_LINK_STATE_L0S &&
+ *state != (PCIE_LINK_STATE_L1|PCIE_LINK_STATE_L0S))
+ *state = 0;
+ if (*state == 0)
+ return;
+
+ latency = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12;
+ *l0s = calc_L0S_latency(latency, 0);
+ if (*state & PCIE_LINK_STATE_L1) {
+ latency = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15;
+ *l1 = calc_L1_latency(latency, 0);
+ }
+ pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16);
+ *enabled = reg16 & (PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1);
+}
+
+static void pcie_aspm_cap_init(struct pci_dev *pdev)
+{
+ struct pci_dev *child_dev;
+ u32 state, tmp;
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ /* upstream component states */
+ pcie_aspm_get_cap_device(pdev, &link_state->support_state,
+ &link_state->l0s_upper_latency,
+ &link_state->l1_upper_latency,
+ &link_state->enabled_state);
+ /* downstream component states, all functions have the same setting */
+ child_dev = list_entry(pdev->subordinate->devices.next, struct pci_dev,
+ bus_list);
+ pcie_aspm_get_cap_device(child_dev, &state,
+ &link_state->l0s_down_latency,
+ &link_state->l1_down_latency,
+ &tmp);
+ link_state->support_state &= state;
+ if (!link_state->support_state)
+ return;
+ link_state->enabled_state &= link_state->support_state;
+ link_state->bios_aspm_state = link_state->enabled_state;
+
+ /* ENDPOINT states*/
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ int pos;
+ u32 reg32;
+ unsigned int latency;
+ struct endpoint_state *ep_state =
+ &link_state->endpoints[PCI_FUNC(child_dev->devfn)];
+
+ if (child_dev->pcie_type != PCI_EXP_TYPE_ENDPOINT &&
+ child_dev->pcie_type != PCI_EXP_TYPE_LEG_END)
+ continue;
+
+ pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+ pci_read_config_dword(child_dev, pos + PCI_EXP_DEVCAP, ®32);
+ latency = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6;
+ latency = calc_L0S_latency(latency, 1);
+ ep_state->l0s_acceptable_latency = latency;
+ if (link_state->support_state & PCIE_LINK_STATE_L1) {
+ latency = (reg32 & PCI_EXP_DEVCAP_L1) >> 9;
+ latency = calc_L1_latency(latency, 1);
+ ep_state->l1_acceptable_latency = latency;
+ }
+ }
+}
+
+static unsigned int __pcie_aspm_check_state_one(struct pci_dev *pdev,
+ unsigned int state)
+{
+ struct pci_dev *parent_dev, *tmp_dev;
+ unsigned int latency, l1_latency = 0;
+ struct pcie_link_state *link_state;
+ struct endpoint_state *ep_state;
+
+ parent_dev = pdev->bus->self;
+ link_state = parent_dev->link_state;
+ state &= link_state->support_state;
+ if (state == 0)
+ return 0;
+ ep_state = &link_state->endpoints[PCI_FUNC(pdev->devfn)];
+
+ /*
+ * Check latency for endpoint device.
+ * TBD: The latency from the endpoint to root complex vary per
+ * switch's upstream link state above the device. Here we just do a
+ * simple check which assumes all links above the device can be in L1
+ * state, that is we just consider the worst case. If switch's upstream
+ * link can't be put into L0S/L1, then our check is too strictly.
+ */
+ tmp_dev = pdev;
+ while (state & (PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1)) {
+ parent_dev = tmp_dev->bus->self;
+ link_state = parent_dev->link_state;
+ if (state & PCIE_LINK_STATE_L0S) {
+ latency = max_t(unsigned int,
+ link_state->l0s_upper_latency,
+ link_state->l0s_down_latency);
+ if (latency > ep_state->l0s_acceptable_latency)
+ state &= ~PCIE_LINK_STATE_L0S;
+ }
+ if (state & PCIE_LINK_STATE_L1) {
+ latency = max_t(unsigned int,
+ link_state->l1_upper_latency,
+ link_state->l1_down_latency);
+ if (latency + l1_latency >
+ ep_state->l1_acceptable_latency)
+ state &= ~PCIE_LINK_STATE_L1;
+ }
+ if (!parent_dev->bus->self) /* parent_dev is a root port */
+ break;
+ else {
+ /*
+ * parent_dev is the downstream port of a switch, make
+ * tmp_dev the upstream port of the switch
+ */
+ tmp_dev = parent_dev->bus->self;
+ /*
+ * every switch on the path to root complex need 1 more
+ * microsecond for L1. Spec doesn't mention L0S.
+ */
+ if (state & PCIE_LINK_STATE_L1)
+ l1_latency += 1000;
+ }
+ }
+ return state;
+}
+
+static unsigned int pcie_aspm_check_state(struct pci_dev *pdev,
+ unsigned int state)
+{
+ struct pci_dev *child_dev;
+
+ /* If no child, disable the link */
+ if (list_empty(&pdev->subordinate->devices))
+ return 0;
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ if (child_dev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE) {
+ /*
+ * If downstream component of a link is pci bridge, we
+ * disable ASPM for now for the link
+ * */
+ state = 0;
+ break;
+ }
+ if ((child_dev->pcie_type != PCI_EXP_TYPE_ENDPOINT &&
+ child_dev->pcie_type != PCI_EXP_TYPE_LEG_END))
+ continue;
+ /* Device not in D0 doesn't need check latency */
+ if (child_dev->current_state == PCI_D1 ||
+ child_dev->current_state == PCI_D2 ||
+ child_dev->current_state == PCI_D3hot ||
+ child_dev->current_state == PCI_D3cold)
+ continue;
+ state = __pcie_aspm_check_state_one(child_dev, state);
+ }
+ return state;
+}
+
+static void __pcie_aspm_config_one_dev(struct pci_dev *pdev, unsigned int state)
+{
+ u16 reg16;
+ int pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
+
+ pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16);
+ reg16 &= ~0x3;
+ reg16 |= state;
+ pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16);
+}
+
+static void __pcie_aspm_config_link(struct pci_dev *pdev, unsigned int state)
+{
+ struct pci_dev *child_dev;
+ int valid = 1;
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ /*
+ * if the downstream component has pci bridge function, don't do ASPM
+ * now
+ */
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ if (child_dev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE) {
+ valid = 0;
+ break;
+ }
+ }
+ if (!valid)
+ return;
+
+ /*
+ * spec 2.0 suggests all functions should be configured the same
+ * setting for ASPM. Enabling ASPM L1 should be done in upstream
+ * component first and then downstream, and vice versa for disabling
+ * ASPM L1. Spec doesn't mention L0S.
+ */
+ if (state & PCIE_LINK_STATE_L1)
+ __pcie_aspm_config_one_dev(pdev, state);
+
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list)
+ __pcie_aspm_config_one_dev(child_dev, state);
+
+ if (!(state & PCIE_LINK_STATE_L1))
+ __pcie_aspm_config_one_dev(pdev, state);
+
+ link_state->enabled_state = state;
+}
+
+static void __pcie_aspm_configure_link_state(struct pci_dev *pdev,
+ unsigned int state)
+{
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ if (link_state->support_state == 0)
+ return;
+ state &= PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1;
+
+ /* state 0 means disabling aspm */
+ state = pcie_aspm_check_state(pdev, state);
+ if (link_state->enabled_state == state)
+ return;
+ __pcie_aspm_config_link(pdev, state);
+}
+
+/*
+ * pcie_aspm_configure_link_state: enable/disable PCI express link state
+ * @pdev: the root port or switch downstream port
+ */
+static void pcie_aspm_configure_link_state(struct pci_dev *pdev,
+ unsigned int state)
+{
+ down_read(&pci_bus_sem);
+ mutex_lock(&aspm_lock);
+ __pcie_aspm_configure_link_state(pdev, state);
+ mutex_unlock(&aspm_lock);
+ up_read(&pci_bus_sem);
+}
+
+static void free_link_state(struct pci_dev *pdev)
+{
+ kfree(pdev->link_state);
+ pdev->link_state = NULL;
+}
+
+/*
+ * pcie_aspm_init_link_state: Initiate PCI express link state.
+ * It is called after the pcie and its children devices are scaned.
+ * @pdev: the root port or switch downstream port
+ */
+void pcie_aspm_init_link_state(struct pci_dev *pdev)
+{
+ unsigned int state;
+ struct pcie_link_state *link_state;
+ int error = 0;
+
+ if (aspm_disabled || !pdev->is_pcie || pdev->link_state)
+ return;
+ if (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
+ pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
+ return;
+ down_read(&pci_bus_sem);
+ if (list_empty(&pdev->subordinate->devices))
+ goto out;
+
+ mutex_lock(&aspm_lock);
+
+ link_state = kzalloc(sizeof(*link_state), GFP_KERNEL);
+ if (!link_state)
+ goto unlock_out;
+ pdev->link_state = link_state;
+
+ pcie_aspm_configure_common_clock(pdev);
+
+ pcie_aspm_cap_init(pdev);
+
+ /* config link state to avoid BIOS error */
+ state = pcie_aspm_check_state(pdev, policy_to_aspm_state(pdev));
+ __pcie_aspm_config_link(pdev, state);
+
+ pcie_check_clock_pm(pdev);
+
+ link_state->pdev = pdev;
+ list_add(&link_state->sibiling, &link_list);
+
+unlock_out:
+ if (error)
+ free_link_state(pdev);
+ mutex_unlock(&aspm_lock);
+out:
+ up_read(&pci_bus_sem);
+}
+
+/* @pdev: the endpoint device */
+void pcie_aspm_exit_link_state(struct pci_dev *pdev)
+{
+ struct pci_dev *parent = pdev->bus->self;
+ struct pcie_link_state *link_state = parent->link_state;
+
+ if (aspm_disabled || !pdev->is_pcie || !parent || !link_state)
+ return;
+ if (parent->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
+ parent->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
+ return;
+ down_read(&pci_bus_sem);
+ mutex_lock(&aspm_lock);
+
+ /*
+ * All PCIe functions are in one slot, remove one function will remove
+ * the the whole slot, so just wait
+ */
+ if (!list_empty(&parent->subordinate->devices))
+ goto out;
+
+ /* All functions are removed, so just disable ASPM for the link */
+ __pcie_aspm_config_one_dev(parent, 0);
+ list_del(&link_state->sibiling);
+ /* Clock PM is for endpoint device */
+
+ free_link_state(parent);
+out:
+ mutex_unlock(&aspm_lock);
+ up_read(&pci_bus_sem);
+}
+
+/* @pdev: the root port or switch downstream port */
+void pcie_aspm_pm_state_change(struct pci_dev *pdev)
+{
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ if (aspm_disabled || !pdev->is_pcie || !pdev->link_state)
+ return;
+ if (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
+ pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
+ return;
+ /*
+ * devices changed PM state, we should recheck if latency meets all
+ * functions' requirement
+ */
+ pcie_aspm_configure_link_state(pdev, link_state->enabled_state);
+}
+
+/*
+ * pci_disable_link_state - disable pci device's link state, so the link will
+ * never enter specific states
+ */
+void pci_disable_link_state(struct pci_dev *pdev, int state)
+{
+ struct pci_dev *parent = pdev->bus->self;
+ struct pcie_link_state *link_state;
+
+ if (aspm_disabled || !pdev->is_pcie)
+ return;
+ if (pdev->pcie_type == PCI_EXP_TYPE_ROOT_PORT ||
+ pdev->pcie_type == PCI_EXP_TYPE_DOWNSTREAM)
+ parent = pdev;
+ if (!parent || !parent->link_state)
+ return;
+
+ down_read(&pci_bus_sem);
+ mutex_lock(&aspm_lock);
+ link_state = parent->link_state;
+ link_state->support_state &=
+ ~(state & (PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1));
+ if (state & PCIE_LINK_STATE_CLKPM)
+ link_state->clk_pm_capable = 0;
+
+ __pcie_aspm_configure_link_state(parent, link_state->enabled_state);
+ if (!link_state->clk_pm_capable && link_state->clk_pm_enabled)
+ pcie_set_clock_pm(parent, 0);
+ mutex_unlock(&aspm_lock);
+ up_read(&pci_bus_sem);
+}
+EXPORT_SYMBOL(pci_disable_link_state);
+
+static int pcie_aspm_set_policy(const char *val, struct kernel_param *kp)
+{
+ int i;
+ struct pci_dev *pdev;
+ struct pcie_link_state *link_state;
+
+ for (i = 0; i < ARRAY_SIZE(policy_str); i++)
+ if (!strncmp(val, policy_str[i], strlen(policy_str[i])))
+ break;
+ if (i >= ARRAY_SIZE(policy_str))
+ return -EINVAL;
+ if (i == aspm_policy)
+ return 0;
+
+ down_read(&pci_bus_sem);
+ mutex_lock(&aspm_lock);
+ aspm_policy = i;
+ list_for_each_entry(link_state, &link_list, sibiling) {
+ pdev = link_state->pdev;
+ __pcie_aspm_configure_link_state(pdev,
+ policy_to_aspm_state(pdev));
+ if (link_state->clk_pm_capable &&
+ link_state->clk_pm_enabled != policy_to_clkpm_state(pdev))
+ pcie_set_clock_pm(pdev, policy_to_clkpm_state(pdev));
+
+ }
+ mutex_unlock(&aspm_lock);
+ up_read(&pci_bus_sem);
+ return 0;
+}
+
+static int pcie_aspm_get_policy(char *buffer, struct kernel_param *kp)
+{
+ int i, cnt = 0;
+ for (i = 0; i < ARRAY_SIZE(policy_str); i++)
+ if (i == aspm_policy)
+ cnt += sprintf(buffer + cnt, "[%s] ", policy_str[i]);
+ else
+ cnt += sprintf(buffer + cnt, "%s ", policy_str[i]);
+ return cnt;
+}
+
+module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy,
+ NULL, 0644);
+
+#ifdef CONFIG_PCIEASPM_DEBUG
+static ssize_t link_state_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct pci_dev *pci_device = to_pci_dev(dev);
+ struct pcie_link_state *link_state = pci_device->link_state;
+
+ return sprintf(buf, "%d\n", link_state->enabled_state);
+}
+
+static ssize_t link_state_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf,
+ size_t n)
+{
+ struct pci_dev *pci_device = to_pci_dev(dev);
+ int state;
+
+ if (n < 1)
+ return -EINVAL;
+ state = buf[0]-'0';
+ if (state >= 0 && state <= 3) {
+ /* setup link aspm state */
+ pcie_aspm_configure_link_state(pci_device, state);
+ return n;
+ }
+
+ return -EINVAL;
+}
+
+static ssize_t clk_ctl_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct pci_dev *pci_device = to_pci_dev(dev);
+ struct pcie_link_state *link_state = pci_device->link_state;
+
+ return sprintf(buf, "%d\n", link_state->clk_pm_enabled);
+}
+
+static ssize_t clk_ctl_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf,
+ size_t n)
+{
+ struct pci_dev *pci_device = to_pci_dev(dev);
+ int state;
+
+ if (n < 1)
+ return -EINVAL;
+ state = buf[0]-'0';
+
+ down_read(&pci_bus_sem);
+ mutex_lock(&aspm_lock);
+ pcie_set_clock_pm(pci_device, !!state);
+ mutex_unlock(&aspm_lock);
+ up_read(&pci_bus_sem);
+
+ return n;
+}
+
+static DEVICE_ATTR(link_state, 0644, link_state_show, link_state_store);
+static DEVICE_ATTR(clk_ctl, 0644, clk_ctl_show, clk_ctl_store);
+
+static char power_group[] = "power";
+void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev)
+{
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ if (!pdev->is_pcie || (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
+ pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) || !link_state)
+ return;
+
+ if (link_state->support_state)
+ sysfs_add_file_to_group(&pdev->dev.kobj,
+ &dev_attr_link_state.attr, power_group);
+ if (link_state->clk_pm_capable)
+ sysfs_add_file_to_group(&pdev->dev.kobj,
+ &dev_attr_clk_ctl.attr, power_group);
+}
+
+void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev)
+{
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ if (!pdev->is_pcie || (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
+ pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) || !link_state)
+ return;
+
+ if (link_state->support_state)
+ sysfs_remove_file_from_group(&pdev->dev.kobj,
+ &dev_attr_link_state.attr, power_group);
+ if (link_state->clk_pm_capable)
+ sysfs_remove_file_from_group(&pdev->dev.kobj,
+ &dev_attr_clk_ctl.attr, power_group);
+}
+#endif
+
+static int __init pcie_aspm_disable(char *str)
+{
+ aspm_disabled = 1;
+ return 1;
+}
+
+__setup("pcie_noaspm", pcie_aspm_disable);
+
+static int __init pcie_aspm_init(void)
+{
+ if (aspm_disabled)
+ return 0;
+ pci_osc_support_set(OSC_ACTIVE_STATE_PWR_SUPPORT|
+ OSC_CLOCK_PWR_CAPABILITY_SUPPORT);
+ return 0;
+}
+
+fs_initcall(pcie_aspm_init);
Index: linux/include/linux/aspm.h
===================================================================
--- /dev/null 1970-01-01 00:00:00.000000000 +0000
+++ linux/include/linux/aspm.h 2008-02-19 11:37:55.000000000 +0800
@@ -0,0 +1,56 @@
+/*
+ * aspm.h
+ *
+ * PCI Express ASPM defines and function prototypes
+ *
+ * Copyright (C) 2007 Intel Corp.
+ * Zhang Yanmin ([email protected])
+ * Shaohua Li ([email protected])
+ *
+ * For more information, please consult the following manuals (look at
+ * http://www.pcisig.com/ for how to get them):
+ *
+ * PCI Express Specification
+ */
+
+#ifndef LINUX_ASPM_H
+#define LINUX_ASPM_H
+
+#include <linux/pci.h>
+
+#define PCIE_LINK_STATE_L0S 1
+#define PCIE_LINK_STATE_L1 2
+#define PCIE_LINK_STATE_CLKPM 4
+
+#ifdef CONFIG_PCIEASPM
+extern void pcie_aspm_init_link_state(struct pci_dev *pdev);
+extern void pcie_aspm_exit_link_state(struct pci_dev *pdev);
+extern void pcie_aspm_pm_state_change(struct pci_dev *pdev);
+extern void pci_disable_link_state(struct pci_dev *pdev, int state);
+#else
+static inline void pcie_aspm_init_link_state(struct pci_dev *pdev)
+{
+}
+static inline void pcie_aspm_exit_link_state(struct pci_dev *pdev)
+{
+}
+static inline void pcie_aspm_pm_state_change(struct pci_dev *pdev)
+{
+}
+static inline void pci_disable_link_state(struct pci_dev *pdev, int state)
+{
+}
+#endif
+
+#ifdef CONFIG_PCIEASPM_DEBUG /* this depends on CONFIG_PCIEASPM */
+extern void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev);
+extern void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev);
+#else
+static inline void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev)
+{
+}
+static inline void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev)
+{
+}
+#endif
+#endif /* LINUX_ASPM_H */
Index: linux/drivers/pci/pcie/Kconfig
===================================================================
--- linux.orig/drivers/pci/pcie/Kconfig 2008-01-24 10:23:09.000000000 +0800
+++ linux/drivers/pci/pcie/Kconfig 2008-02-19 11:32:10.000000000 +0800
@@ -26,3 +26,23 @@ config HOTPLUG_PCI_PCIE
When in doubt, say N.
source "drivers/pci/pcie/aer/Kconfig"
+
+#
+# PCI Express ASPM
+#
+config PCIEASPM
+ bool "PCI Express ASPM support(Experimental)"
+ depends on PCI && EXPERIMENTAL && PCIEPORTBUS
+ default y
+ help
+ This enables PCI Express ASPM (Active State Power Management) and
+ Clock Power Management. ASPM supports state L0/L0s/L1.
+
+ When in doubt, say N.
+config PCIEASPM_DEBUG
+ bool "Debug PCI Express ASPM"
+ depends on PCIEASPM
+ default n
+ help
+ This enables PCI Express ASPM debug support. It will add per-device
+ interface to control ASPM.
Index: linux/include/linux/pci.h
===================================================================
--- linux.orig/include/linux/pci.h 2008-02-19 10:51:48.000000000 +0800
+++ linux/include/linux/pci.h 2008-02-19 10:54:12.000000000 +0800
@@ -128,6 +128,7 @@ struct pci_cap_saved_state {
u32 data[0];
};
+struct pcie_link_state;
/*
* The pci_dev structure is used to describe PCI devices.
*/
@@ -165,6 +166,10 @@ struct pci_dev {
this is D0-D3, D0 being fully functional,
and D3 being off. */
+#ifdef CONFIG_PCIEASPM
+ struct pcie_link_state *link_state; /* ASPM link state. */
+#endif
+
pci_channel_state_t error_state; /* current connectivity state */
struct device dev; /* Generic device interface */
Index: linux/include/linux/pci_regs.h
===================================================================
--- linux.orig/include/linux/pci_regs.h 2008-01-24 10:23:09.000000000 +0800
+++ linux/include/linux/pci_regs.h 2008-02-19 10:54:12.000000000 +0800
@@ -395,9 +395,17 @@
#define PCI_EXP_DEVSTA_AUXPD 0x10 /* AUX Power Detected */
#define PCI_EXP_DEVSTA_TRPND 0x20 /* Transactions Pending */
#define PCI_EXP_LNKCAP 12 /* Link Capabilities */
+#define PCI_EXP_LNKCAP_ASPMS 0xc00 /* ASPM Support */
+#define PCI_EXP_LNKCAP_L0SEL 0x7000 /* L0s Exit Latency */
+#define PCI_EXP_LNKCAP_L1EL 0x38000 /* L1 Exit Latency */
+#define PCI_EXP_LNKCAP_CLKPM 0x40000 /* L1 Clock Power Management */
#define PCI_EXP_LNKCTL 16 /* Link Control */
+#define PCI_EXP_LNKCTL_RL 0x20 /* Retrain Link */
+#define PCI_EXP_LNKCTL_CCC 0x40 /* Common Clock COnfiguration */
#define PCI_EXP_LNKCTL_CLKREQ_EN 0x100 /* Enable clkreq */
#define PCI_EXP_LNKSTA 18 /* Link Status */
+#define PCI_EXP_LNKSTA_LT 0x800 /* Link Training */
+#define PCI_EXP_LNKSTA_SLC 0x1000 /* Slot Clock Configuration */
#define PCI_EXP_SLTCAP 20 /* Slot Capabilities */
#define PCI_EXP_SLTCTL 24 /* Slot Control */
#define PCI_EXP_SLTSTA 26 /* Slot Status */
Index: linux/drivers/pci/probe.c
===================================================================
--- linux.orig/drivers/pci/probe.c 2008-02-19 10:51:46.000000000 +0800
+++ linux/drivers/pci/probe.c 2008-02-19 10:54:12.000000000 +0800
@@ -9,6 +9,7 @@
#include <linux/slab.h>
#include <linux/module.h>
#include <linux/cpumask.h>
+#include <linux/aspm.h>
#include "pci.h"
#define CARDBUS_LATENCY_TIMER 176 /* secondary latency timer */
@@ -1005,6 +1006,10 @@ int pci_scan_slot(struct pci_bus *bus, i
break;
}
}
+
+ if (bus->self)
+ pcie_aspm_init_link_state(bus->self);
+
return nr;
}
Index: linux/drivers/pci/remove.c
===================================================================
--- linux.orig/drivers/pci/remove.c 2008-02-19 10:51:46.000000000 +0800
+++ linux/drivers/pci/remove.c 2008-02-19 10:54:12.000000000 +0800
@@ -1,5 +1,6 @@
#include <linux/pci.h>
#include <linux/module.h>
+#include <linux/aspm.h>
#include "pci.h"
static void pci_free_resources(struct pci_dev *dev)
@@ -30,6 +31,9 @@ static void pci_stop_dev(struct pci_dev
dev->global_list.next = dev->global_list.prev = NULL;
up_write(&pci_bus_sem);
}
+
+ if (dev->bus->self)
+ pcie_aspm_exit_link_state(dev);
}
static void pci_destroy_dev(struct pci_dev *dev)
Index: linux/drivers/pci/pci.c
===================================================================
--- linux.orig/drivers/pci/pci.c 2008-02-19 10:51:46.000000000 +0800
+++ linux/drivers/pci/pci.c 2008-02-19 10:54:12.000000000 +0800
@@ -18,6 +18,7 @@
#include <linux/spinlock.h>
#include <linux/string.h>
#include <linux/log2.h>
+#include <linux/aspm.h>
#include <asm/dma.h> /* isa_dma_bridge_buggy */
#include "pci.h"
@@ -519,6 +520,9 @@ pci_set_power_state(struct pci_dev *dev,
if (need_restore)
pci_restore_bars(dev);
+ if (dev->bus->self)
+ pcie_aspm_pm_state_change(dev->bus->self);
+
return 0;
}
Index: linux/drivers/pci/pci-sysfs.c
===================================================================
--- linux.orig/drivers/pci/pci-sysfs.c 2008-02-19 10:51:46.000000000 +0800
+++ linux/drivers/pci/pci-sysfs.c 2008-02-19 10:54:12.000000000 +0800
@@ -21,6 +21,7 @@
#include <linux/topology.h>
#include <linux/mm.h>
#include <linux/capability.h>
+#include <linux/aspm.h>
#include "pci.h"
static int sysfs_initialized; /* = 0 */
@@ -650,6 +651,8 @@ int __must_check pci_create_sysfs_dev_fi
if (pcibios_add_platform_entries(pdev))
goto err_rom_file;
+ pcie_aspm_create_sysfs_dev_files(pdev);
+
return 0;
err_rom_file:
@@ -679,6 +682,8 @@ void pci_remove_sysfs_dev_files(struct p
if (!sysfs_initialized)
return;
+ pcie_aspm_remove_sysfs_dev_files(pdev);
+
if (pdev->cfg_size < 4096)
sysfs_remove_bin_file(&pdev->dev.kobj, &pci_config_attr);
else
Index: linux/include/linux/pci-acpi.h
===================================================================
--- linux.orig/include/linux/pci-acpi.h 2008-02-19 11:03:51.000000000 +0800
+++ linux/include/linux/pci-acpi.h 2008-02-19 11:05:34.000000000 +0800
@@ -47,6 +47,7 @@
OSC_PCI_EXPRESS_CAP_STRUCTURE_CONTROL)
#ifdef CONFIG_ACPI
+#include <acpi/acpi_bus.h>
extern acpi_status pci_osc_control_set(acpi_handle handle, u32 flags);
extern acpi_status __pci_osc_support_set(u32 flags, const char *hid);
static inline acpi_status pci_osc_support_set(u32 flags)
On Tue, Feb 19, 2008 at 01:45:45PM +0800, Shaohua Li wrote:
> Sorry for the long delay, I just back from vocation. I fixed all the
> issues I found in the list (except one hang issue, which should be fixed
> in specific driver, I'll reply that thread soon), can you re-add the
> patch to your test tree.
Hm, I get the following build error with this patch:
CC drivers/pci/pcie/aspm.o
distcc[13135] ERROR: compile /home/gregkh/.ccache/aspm.tmp.mini.13131.i on localhost failed
In file included from drivers/pci/pcie/aspm.c:21:
include/linux/pci-acpi.h:66: error: expected ')' before 'handle'
make[2]: *** [drivers/pci/pcie/aspm.o] Error 1
make[1]: *** [drivers/pci/pcie] Error 2
make: *** [_module_drivers/pci] Error 2
Also, can you rename your new header file to have "pci-" in front of it?
thanks,
greg k-h
On Tue, 2008-02-19 at 15:45 -0800, Greg KH wrote:
> On Tue, Feb 19, 2008 at 01:45:45PM +0800, Shaohua Li wrote:
> > Sorry for the long delay, I just back from vocation. I fixed all the
> > issues I found in the list (except one hang issue, which should be fixed
> > in specific driver, I'll reply that thread soon), can you re-add the
> > patch to your test tree.
>
> Hm, I get the following build error with this patch:
> CC drivers/pci/pcie/aspm.o
> distcc[13135] ERROR: compile /home/gregkh/.ccache/aspm.tmp.mini.13131.i on localhost failed
> In file included from drivers/pci/pcie/aspm.c:21:
> include/linux/pci-acpi.h:66: error: expected ')' before 'handle'
> make[2]: *** [drivers/pci/pcie/aspm.o] Error 1
> make[1]: *** [drivers/pci/pcie] Error 2
> make: *** [_module_drivers/pci] Error 2
>
>
> Also, can you rename your new header file to have "pci-" in front of it?
Oops, looks pci-acpi.h isn't self-contained in non-acpi config. I fixed
it (and the head file name).
PCI Express ASPM defines a protocol for PCI Express components in the D0
state to reduce Link power by placing their Links into a low power state
and instructing the other end of the Link to do likewise. This
capability allows hardware-autonomous, dynamic Link power reduction
beyond what is achievable by software-only controlled power management.
However, The device should be configured by software appropriately.
Enabling ASPM will save power, but will introduce device latency.
This patch adds ASPM support in Linux. It introduces a global policy for
ASPM, a sysfs file /sys/module/pcie_aspm/parameters/policy can control
it. The interface can be used as a boot option too. Currently we have
below setting:
-default, BIOS default setting
-powersave, highest power saving mode, enable all available ASPM
state and clock power management
-performance, highest performance, disable ASPM and clock power
management
By default, the 'default' policy is used currently.
In my test, power difference between powersave mode and performance mode
is about 1.3w in a system with 3 PCIE links.
Note: some devices might not work well with aspm, either because chipset
issue or device issue. The patch provide API (pci_disable_link_state),
driver can disable ASPM for specific device.
Signed-off-by: Shaohua Li <[email protected]>
---
drivers/pci/pci-sysfs.c | 5
drivers/pci/pci.c | 4
drivers/pci/pcie/Kconfig | 20 +
drivers/pci/pcie/Makefile | 3
drivers/pci/pcie/aspm.c | 801 ++++++++++++++++++++++++++++++++++++++++++++++
drivers/pci/probe.c | 5
drivers/pci/remove.c | 4
include/linux/pci-acpi.h | 13
include/linux/pci-aspm.h | 56 +++
include/linux/pci.h | 5
include/linux/pci_regs.h | 8
11 files changed, 917 insertions(+), 7 deletions(-)
Index: linux/drivers/pci/pcie/Makefile
===================================================================
--- linux.orig/drivers/pci/pcie/Makefile 2008-01-24 10:23:09.000000000 +0800
+++ linux/drivers/pci/pcie/Makefile 2008-02-19 10:54:12.000000000 +0800
@@ -2,6 +2,9 @@
# Makefile for PCI-Express PORT Driver
#
+# Build PCI Express ASPM if needed
+obj-$(CONFIG_PCIEASPM) += aspm.o
+
pcieportdrv-y := portdrv_core.o portdrv_pci.o portdrv_bus.o
obj-$(CONFIG_PCIEPORTBUS) += pcieportdrv.o
Index: linux/drivers/pci/pcie/aspm.c
===================================================================
--- /dev/null 1970-01-01 00:00:00.000000000 +0000
+++ linux/drivers/pci/pcie/aspm.c 2008-02-20 09:23:45.000000000 +0800
@@ -0,0 +1,801 @@
+/*
+ * File: drivers/pci/pcie/aspm.c
+ * Enabling PCIE link L0s/L1 state and Clock Power Management
+ *
+ * Copyright (C) 2007 Intel
+ * Copyright (C) Zhang Yanmin ([email protected])
+ * Copyright (C) Shaohua Li ([email protected])
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/pci.h>
+#include <linux/pci_regs.h>
+#include <linux/errno.h>
+#include <linux/pm.h>
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <linux/pci-aspm.h>
+#include <linux/pci-acpi.h>
+#include "../pci.h"
+
+#ifdef MODULE_PARAM_PREFIX
+#undef MODULE_PARAM_PREFIX
+#endif
+#define MODULE_PARAM_PREFIX "pcie_aspm."
+
+struct endpoint_state {
+ unsigned int l0s_acceptable_latency;
+ unsigned int l1_acceptable_latency;
+};
+
+struct pcie_link_state {
+ struct list_head sibiling;
+ struct pci_dev *pdev;
+
+ /* ASPM state */
+ unsigned int support_state;
+ unsigned int enabled_state;
+ unsigned int bios_aspm_state;
+ /* upstream component */
+ unsigned int l0s_upper_latency;
+ unsigned int l1_upper_latency;
+ /* downstream component */
+ unsigned int l0s_down_latency;
+ unsigned int l1_down_latency;
+ /* Clock PM state*/
+ unsigned int clk_pm_capable;
+ unsigned int clk_pm_enabled;
+ unsigned int bios_clk_state;
+
+ /*
+ * A pcie downstream port only has one slot under it, so at most there
+ * are 8 functions
+ */
+ struct endpoint_state endpoints[8];
+};
+
+static int aspm_disabled;
+static DEFINE_MUTEX(aspm_lock);
+static LIST_HEAD(link_list);
+
+#define POLICY_DEFAULT 0 /* BIOS default setting */
+#define POLICY_PERFORMANCE 1 /* high performance */
+#define POLICY_POWERSAVE 2 /* high power saving */
+static int aspm_policy;
+static const char *policy_str[] = {
+ [POLICY_DEFAULT] = "default",
+ [POLICY_PERFORMANCE] = "performance",
+ [POLICY_POWERSAVE] = "powersave"
+};
+
+static int policy_to_aspm_state(struct pci_dev *pdev)
+{
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ switch (aspm_policy) {
+ case POLICY_PERFORMANCE:
+ /* Disable ASPM and Clock PM */
+ return 0;
+ case POLICY_POWERSAVE:
+ /* Enable ASPM L0s/L1 */
+ return PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1;
+ case POLICY_DEFAULT:
+ return link_state->bios_aspm_state;
+ }
+ return 0;
+}
+
+static int policy_to_clkpm_state(struct pci_dev *pdev)
+{
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ switch (aspm_policy) {
+ case POLICY_PERFORMANCE:
+ /* Disable ASPM and Clock PM */
+ return 0;
+ case POLICY_POWERSAVE:
+ /* Disable Clock PM */
+ return 1;
+ case POLICY_DEFAULT:
+ return link_state->bios_clk_state;
+ }
+ return 0;
+}
+
+static void pcie_set_clock_pm(struct pci_dev *pdev, int enable)
+{
+ struct pci_dev *child_dev;
+ int pos;
+ u16 reg16;
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+ if (!pos)
+ return;
+ pci_read_config_word(child_dev, pos + PCI_EXP_LNKCTL, ®16);
+ if (enable)
+ reg16 |= PCI_EXP_LNKCTL_CLKREQ_EN;
+ else
+ reg16 &= ~PCI_EXP_LNKCTL_CLKREQ_EN;
+ pci_write_config_word(child_dev, pos + PCI_EXP_LNKCTL, reg16);
+ }
+ link_state->clk_pm_enabled = !!enable;
+}
+
+static void pcie_check_clock_pm(struct pci_dev *pdev)
+{
+ int pos;
+ u32 reg32;
+ u16 reg16;
+ int capable = 1, enabled = 1;
+ struct pci_dev *child_dev;
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ /* All functions should have the same cap and state, take the worst */
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+ if (!pos)
+ return;
+ pci_read_config_dword(child_dev, pos + PCI_EXP_LNKCAP, ®32);
+ if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) {
+ capable = 0;
+ enabled = 0;
+ break;
+ }
+ pci_read_config_word(child_dev, pos + PCI_EXP_LNKCTL, ®16);
+ if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN))
+ enabled = 0;
+ }
+ link_state->clk_pm_capable = capable;
+ link_state->clk_pm_enabled = enabled;
+ link_state->bios_clk_state = enabled;
+ pcie_set_clock_pm(pdev, policy_to_clkpm_state(pdev));
+}
+
+/*
+ * pcie_aspm_configure_common_clock: check if the 2 ends of a link
+ * could use common clock. If they are, configure them to use the
+ * common clock. That will reduce the ASPM state exit latency.
+ */
+static void pcie_aspm_configure_common_clock(struct pci_dev *pdev)
+{
+ int pos, child_pos;
+ u16 reg16 = 0;
+ struct pci_dev *child_dev;
+ int same_clock = 1;
+
+ /*
+ * all functions of a slot should have the same Slot Clock
+ * Configuration, so just check one function
+ * */
+ child_dev = list_entry(pdev->subordinate->devices.next, struct pci_dev,
+ bus_list);
+ BUG_ON(!child_dev->is_pcie);
+
+ /* Check downstream component if bit Slot Clock Configuration is 1 */
+ child_pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+ pci_read_config_word(child_dev, child_pos + PCI_EXP_LNKSTA, ®16);
+ if (!(reg16 & PCI_EXP_LNKSTA_SLC))
+ same_clock = 0;
+
+ /* Check upstream component if bit Slot Clock Configuration is 1 */
+ pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
+ pci_read_config_word(pdev, pos + PCI_EXP_LNKSTA, ®16);
+ if (!(reg16 & PCI_EXP_LNKSTA_SLC))
+ same_clock = 0;
+
+ /* Configure downstream component, all functions */
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ child_pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+ pci_read_config_word(child_dev, child_pos + PCI_EXP_LNKCTL,
+ ®16);
+ if (same_clock)
+ reg16 |= PCI_EXP_LNKCTL_CCC;
+ else
+ reg16 &= ~PCI_EXP_LNKCTL_CCC;
+ pci_write_config_word(child_dev, child_pos + PCI_EXP_LNKCTL,
+ reg16);
+ }
+
+ /* Configure upstream component */
+ pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16);
+ if (same_clock)
+ reg16 |= PCI_EXP_LNKCTL_CCC;
+ else
+ reg16 &= ~PCI_EXP_LNKCTL_CCC;
+ pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16);
+
+ /* retrain link */
+ reg16 |= PCI_EXP_LNKCTL_RL;
+ pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16);
+
+ /* Wait for link training end */
+ while (1) {
+ pci_read_config_word(pdev, pos + PCI_EXP_LNKSTA, ®16);
+ if (!(reg16 & PCI_EXP_LNKSTA_LT))
+ break;
+ cpu_relax();
+ }
+}
+
+/*
+ * calc_L0S_latency: Convert L0s latency encoding to ns
+ */
+static unsigned int calc_L0S_latency(unsigned int latency_encoding, int ac)
+{
+ unsigned int ns = 64;
+
+ if (latency_encoding == 0x7) {
+ if (ac)
+ ns = -1U;
+ else
+ ns = 5*1000; /* > 4us */
+ } else
+ ns *= (1 << latency_encoding);
+ return ns;
+}
+
+/*
+ * calc_L1_latency: Convert L1 latency encoding to ns
+ */
+static unsigned int calc_L1_latency(unsigned int latency_encoding, int ac)
+{
+ unsigned int ns = 1000;
+
+ if (latency_encoding == 0x7) {
+ if (ac)
+ ns = -1U;
+ else
+ ns = 65*1000; /* > 64us */
+ } else
+ ns *= (1 << latency_encoding);
+ return ns;
+}
+
+static void pcie_aspm_get_cap_device(struct pci_dev *pdev, u32 *state,
+ unsigned int *l0s, unsigned int *l1, unsigned int *enabled)
+{
+ int pos;
+ u16 reg16;
+ u32 reg32;
+ unsigned int latency;
+
+ pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
+ pci_read_config_dword(pdev, pos + PCI_EXP_LNKCAP, ®32);
+ *state = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10;
+ if (*state != PCIE_LINK_STATE_L0S &&
+ *state != (PCIE_LINK_STATE_L1|PCIE_LINK_STATE_L0S))
+ *state = 0;
+ if (*state == 0)
+ return;
+
+ latency = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12;
+ *l0s = calc_L0S_latency(latency, 0);
+ if (*state & PCIE_LINK_STATE_L1) {
+ latency = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15;
+ *l1 = calc_L1_latency(latency, 0);
+ }
+ pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16);
+ *enabled = reg16 & (PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1);
+}
+
+static void pcie_aspm_cap_init(struct pci_dev *pdev)
+{
+ struct pci_dev *child_dev;
+ u32 state, tmp;
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ /* upstream component states */
+ pcie_aspm_get_cap_device(pdev, &link_state->support_state,
+ &link_state->l0s_upper_latency,
+ &link_state->l1_upper_latency,
+ &link_state->enabled_state);
+ /* downstream component states, all functions have the same setting */
+ child_dev = list_entry(pdev->subordinate->devices.next, struct pci_dev,
+ bus_list);
+ pcie_aspm_get_cap_device(child_dev, &state,
+ &link_state->l0s_down_latency,
+ &link_state->l1_down_latency,
+ &tmp);
+ link_state->support_state &= state;
+ if (!link_state->support_state)
+ return;
+ link_state->enabled_state &= link_state->support_state;
+ link_state->bios_aspm_state = link_state->enabled_state;
+
+ /* ENDPOINT states*/
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ int pos;
+ u32 reg32;
+ unsigned int latency;
+ struct endpoint_state *ep_state =
+ &link_state->endpoints[PCI_FUNC(child_dev->devfn)];
+
+ if (child_dev->pcie_type != PCI_EXP_TYPE_ENDPOINT &&
+ child_dev->pcie_type != PCI_EXP_TYPE_LEG_END)
+ continue;
+
+ pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+ pci_read_config_dword(child_dev, pos + PCI_EXP_DEVCAP, ®32);
+ latency = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6;
+ latency = calc_L0S_latency(latency, 1);
+ ep_state->l0s_acceptable_latency = latency;
+ if (link_state->support_state & PCIE_LINK_STATE_L1) {
+ latency = (reg32 & PCI_EXP_DEVCAP_L1) >> 9;
+ latency = calc_L1_latency(latency, 1);
+ ep_state->l1_acceptable_latency = latency;
+ }
+ }
+}
+
+static unsigned int __pcie_aspm_check_state_one(struct pci_dev *pdev,
+ unsigned int state)
+{
+ struct pci_dev *parent_dev, *tmp_dev;
+ unsigned int latency, l1_latency = 0;
+ struct pcie_link_state *link_state;
+ struct endpoint_state *ep_state;
+
+ parent_dev = pdev->bus->self;
+ link_state = parent_dev->link_state;
+ state &= link_state->support_state;
+ if (state == 0)
+ return 0;
+ ep_state = &link_state->endpoints[PCI_FUNC(pdev->devfn)];
+
+ /*
+ * Check latency for endpoint device.
+ * TBD: The latency from the endpoint to root complex vary per
+ * switch's upstream link state above the device. Here we just do a
+ * simple check which assumes all links above the device can be in L1
+ * state, that is we just consider the worst case. If switch's upstream
+ * link can't be put into L0S/L1, then our check is too strictly.
+ */
+ tmp_dev = pdev;
+ while (state & (PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1)) {
+ parent_dev = tmp_dev->bus->self;
+ link_state = parent_dev->link_state;
+ if (state & PCIE_LINK_STATE_L0S) {
+ latency = max_t(unsigned int,
+ link_state->l0s_upper_latency,
+ link_state->l0s_down_latency);
+ if (latency > ep_state->l0s_acceptable_latency)
+ state &= ~PCIE_LINK_STATE_L0S;
+ }
+ if (state & PCIE_LINK_STATE_L1) {
+ latency = max_t(unsigned int,
+ link_state->l1_upper_latency,
+ link_state->l1_down_latency);
+ if (latency + l1_latency >
+ ep_state->l1_acceptable_latency)
+ state &= ~PCIE_LINK_STATE_L1;
+ }
+ if (!parent_dev->bus->self) /* parent_dev is a root port */
+ break;
+ else {
+ /*
+ * parent_dev is the downstream port of a switch, make
+ * tmp_dev the upstream port of the switch
+ */
+ tmp_dev = parent_dev->bus->self;
+ /*
+ * every switch on the path to root complex need 1 more
+ * microsecond for L1. Spec doesn't mention L0S.
+ */
+ if (state & PCIE_LINK_STATE_L1)
+ l1_latency += 1000;
+ }
+ }
+ return state;
+}
+
+static unsigned int pcie_aspm_check_state(struct pci_dev *pdev,
+ unsigned int state)
+{
+ struct pci_dev *child_dev;
+
+ /* If no child, disable the link */
+ if (list_empty(&pdev->subordinate->devices))
+ return 0;
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ if (child_dev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE) {
+ /*
+ * If downstream component of a link is pci bridge, we
+ * disable ASPM for now for the link
+ * */
+ state = 0;
+ break;
+ }
+ if ((child_dev->pcie_type != PCI_EXP_TYPE_ENDPOINT &&
+ child_dev->pcie_type != PCI_EXP_TYPE_LEG_END))
+ continue;
+ /* Device not in D0 doesn't need check latency */
+ if (child_dev->current_state == PCI_D1 ||
+ child_dev->current_state == PCI_D2 ||
+ child_dev->current_state == PCI_D3hot ||
+ child_dev->current_state == PCI_D3cold)
+ continue;
+ state = __pcie_aspm_check_state_one(child_dev, state);
+ }
+ return state;
+}
+
+static void __pcie_aspm_config_one_dev(struct pci_dev *pdev, unsigned int state)
+{
+ u16 reg16;
+ int pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
+
+ pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16);
+ reg16 &= ~0x3;
+ reg16 |= state;
+ pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16);
+}
+
+static void __pcie_aspm_config_link(struct pci_dev *pdev, unsigned int state)
+{
+ struct pci_dev *child_dev;
+ int valid = 1;
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ /*
+ * if the downstream component has pci bridge function, don't do ASPM
+ * now
+ */
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ if (child_dev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE) {
+ valid = 0;
+ break;
+ }
+ }
+ if (!valid)
+ return;
+
+ /*
+ * spec 2.0 suggests all functions should be configured the same
+ * setting for ASPM. Enabling ASPM L1 should be done in upstream
+ * component first and then downstream, and vice versa for disabling
+ * ASPM L1. Spec doesn't mention L0S.
+ */
+ if (state & PCIE_LINK_STATE_L1)
+ __pcie_aspm_config_one_dev(pdev, state);
+
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list)
+ __pcie_aspm_config_one_dev(child_dev, state);
+
+ if (!(state & PCIE_LINK_STATE_L1))
+ __pcie_aspm_config_one_dev(pdev, state);
+
+ link_state->enabled_state = state;
+}
+
+static void __pcie_aspm_configure_link_state(struct pci_dev *pdev,
+ unsigned int state)
+{
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ if (link_state->support_state == 0)
+ return;
+ state &= PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1;
+
+ /* state 0 means disabling aspm */
+ state = pcie_aspm_check_state(pdev, state);
+ if (link_state->enabled_state == state)
+ return;
+ __pcie_aspm_config_link(pdev, state);
+}
+
+/*
+ * pcie_aspm_configure_link_state: enable/disable PCI express link state
+ * @pdev: the root port or switch downstream port
+ */
+static void pcie_aspm_configure_link_state(struct pci_dev *pdev,
+ unsigned int state)
+{
+ down_read(&pci_bus_sem);
+ mutex_lock(&aspm_lock);
+ __pcie_aspm_configure_link_state(pdev, state);
+ mutex_unlock(&aspm_lock);
+ up_read(&pci_bus_sem);
+}
+
+static void free_link_state(struct pci_dev *pdev)
+{
+ kfree(pdev->link_state);
+ pdev->link_state = NULL;
+}
+
+/*
+ * pcie_aspm_init_link_state: Initiate PCI express link state.
+ * It is called after the pcie and its children devices are scaned.
+ * @pdev: the root port or switch downstream port
+ */
+void pcie_aspm_init_link_state(struct pci_dev *pdev)
+{
+ unsigned int state;
+ struct pcie_link_state *link_state;
+ int error = 0;
+
+ if (aspm_disabled || !pdev->is_pcie || pdev->link_state)
+ return;
+ if (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
+ pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
+ return;
+ down_read(&pci_bus_sem);
+ if (list_empty(&pdev->subordinate->devices))
+ goto out;
+
+ mutex_lock(&aspm_lock);
+
+ link_state = kzalloc(sizeof(*link_state), GFP_KERNEL);
+ if (!link_state)
+ goto unlock_out;
+ pdev->link_state = link_state;
+
+ pcie_aspm_configure_common_clock(pdev);
+
+ pcie_aspm_cap_init(pdev);
+
+ /* config link state to avoid BIOS error */
+ state = pcie_aspm_check_state(pdev, policy_to_aspm_state(pdev));
+ __pcie_aspm_config_link(pdev, state);
+
+ pcie_check_clock_pm(pdev);
+
+ link_state->pdev = pdev;
+ list_add(&link_state->sibiling, &link_list);
+
+unlock_out:
+ if (error)
+ free_link_state(pdev);
+ mutex_unlock(&aspm_lock);
+out:
+ up_read(&pci_bus_sem);
+}
+
+/* @pdev: the endpoint device */
+void pcie_aspm_exit_link_state(struct pci_dev *pdev)
+{
+ struct pci_dev *parent = pdev->bus->self;
+ struct pcie_link_state *link_state = parent->link_state;
+
+ if (aspm_disabled || !pdev->is_pcie || !parent || !link_state)
+ return;
+ if (parent->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
+ parent->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
+ return;
+ down_read(&pci_bus_sem);
+ mutex_lock(&aspm_lock);
+
+ /*
+ * All PCIe functions are in one slot, remove one function will remove
+ * the the whole slot, so just wait
+ */
+ if (!list_empty(&parent->subordinate->devices))
+ goto out;
+
+ /* All functions are removed, so just disable ASPM for the link */
+ __pcie_aspm_config_one_dev(parent, 0);
+ list_del(&link_state->sibiling);
+ /* Clock PM is for endpoint device */
+
+ free_link_state(parent);
+out:
+ mutex_unlock(&aspm_lock);
+ up_read(&pci_bus_sem);
+}
+
+/* @pdev: the root port or switch downstream port */
+void pcie_aspm_pm_state_change(struct pci_dev *pdev)
+{
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ if (aspm_disabled || !pdev->is_pcie || !pdev->link_state)
+ return;
+ if (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
+ pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
+ return;
+ /*
+ * devices changed PM state, we should recheck if latency meets all
+ * functions' requirement
+ */
+ pcie_aspm_configure_link_state(pdev, link_state->enabled_state);
+}
+
+/*
+ * pci_disable_link_state - disable pci device's link state, so the link will
+ * never enter specific states
+ */
+void pci_disable_link_state(struct pci_dev *pdev, int state)
+{
+ struct pci_dev *parent = pdev->bus->self;
+ struct pcie_link_state *link_state;
+
+ if (aspm_disabled || !pdev->is_pcie)
+ return;
+ if (pdev->pcie_type == PCI_EXP_TYPE_ROOT_PORT ||
+ pdev->pcie_type == PCI_EXP_TYPE_DOWNSTREAM)
+ parent = pdev;
+ if (!parent || !parent->link_state)
+ return;
+
+ down_read(&pci_bus_sem);
+ mutex_lock(&aspm_lock);
+ link_state = parent->link_state;
+ link_state->support_state &=
+ ~(state & (PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1));
+ if (state & PCIE_LINK_STATE_CLKPM)
+ link_state->clk_pm_capable = 0;
+
+ __pcie_aspm_configure_link_state(parent, link_state->enabled_state);
+ if (!link_state->clk_pm_capable && link_state->clk_pm_enabled)
+ pcie_set_clock_pm(parent, 0);
+ mutex_unlock(&aspm_lock);
+ up_read(&pci_bus_sem);
+}
+EXPORT_SYMBOL(pci_disable_link_state);
+
+static int pcie_aspm_set_policy(const char *val, struct kernel_param *kp)
+{
+ int i;
+ struct pci_dev *pdev;
+ struct pcie_link_state *link_state;
+
+ for (i = 0; i < ARRAY_SIZE(policy_str); i++)
+ if (!strncmp(val, policy_str[i], strlen(policy_str[i])))
+ break;
+ if (i >= ARRAY_SIZE(policy_str))
+ return -EINVAL;
+ if (i == aspm_policy)
+ return 0;
+
+ down_read(&pci_bus_sem);
+ mutex_lock(&aspm_lock);
+ aspm_policy = i;
+ list_for_each_entry(link_state, &link_list, sibiling) {
+ pdev = link_state->pdev;
+ __pcie_aspm_configure_link_state(pdev,
+ policy_to_aspm_state(pdev));
+ if (link_state->clk_pm_capable &&
+ link_state->clk_pm_enabled != policy_to_clkpm_state(pdev))
+ pcie_set_clock_pm(pdev, policy_to_clkpm_state(pdev));
+
+ }
+ mutex_unlock(&aspm_lock);
+ up_read(&pci_bus_sem);
+ return 0;
+}
+
+static int pcie_aspm_get_policy(char *buffer, struct kernel_param *kp)
+{
+ int i, cnt = 0;
+ for (i = 0; i < ARRAY_SIZE(policy_str); i++)
+ if (i == aspm_policy)
+ cnt += sprintf(buffer + cnt, "[%s] ", policy_str[i]);
+ else
+ cnt += sprintf(buffer + cnt, "%s ", policy_str[i]);
+ return cnt;
+}
+
+module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy,
+ NULL, 0644);
+
+#ifdef CONFIG_PCIEASPM_DEBUG
+static ssize_t link_state_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct pci_dev *pci_device = to_pci_dev(dev);
+ struct pcie_link_state *link_state = pci_device->link_state;
+
+ return sprintf(buf, "%d\n", link_state->enabled_state);
+}
+
+static ssize_t link_state_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf,
+ size_t n)
+{
+ struct pci_dev *pci_device = to_pci_dev(dev);
+ int state;
+
+ if (n < 1)
+ return -EINVAL;
+ state = buf[0]-'0';
+ if (state >= 0 && state <= 3) {
+ /* setup link aspm state */
+ pcie_aspm_configure_link_state(pci_device, state);
+ return n;
+ }
+
+ return -EINVAL;
+}
+
+static ssize_t clk_ctl_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct pci_dev *pci_device = to_pci_dev(dev);
+ struct pcie_link_state *link_state = pci_device->link_state;
+
+ return sprintf(buf, "%d\n", link_state->clk_pm_enabled);
+}
+
+static ssize_t clk_ctl_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf,
+ size_t n)
+{
+ struct pci_dev *pci_device = to_pci_dev(dev);
+ int state;
+
+ if (n < 1)
+ return -EINVAL;
+ state = buf[0]-'0';
+
+ down_read(&pci_bus_sem);
+ mutex_lock(&aspm_lock);
+ pcie_set_clock_pm(pci_device, !!state);
+ mutex_unlock(&aspm_lock);
+ up_read(&pci_bus_sem);
+
+ return n;
+}
+
+static DEVICE_ATTR(link_state, 0644, link_state_show, link_state_store);
+static DEVICE_ATTR(clk_ctl, 0644, clk_ctl_show, clk_ctl_store);
+
+static char power_group[] = "power";
+void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev)
+{
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ if (!pdev->is_pcie || (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
+ pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) || !link_state)
+ return;
+
+ if (link_state->support_state)
+ sysfs_add_file_to_group(&pdev->dev.kobj,
+ &dev_attr_link_state.attr, power_group);
+ if (link_state->clk_pm_capable)
+ sysfs_add_file_to_group(&pdev->dev.kobj,
+ &dev_attr_clk_ctl.attr, power_group);
+}
+
+void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev)
+{
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ if (!pdev->is_pcie || (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
+ pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) || !link_state)
+ return;
+
+ if (link_state->support_state)
+ sysfs_remove_file_from_group(&pdev->dev.kobj,
+ &dev_attr_link_state.attr, power_group);
+ if (link_state->clk_pm_capable)
+ sysfs_remove_file_from_group(&pdev->dev.kobj,
+ &dev_attr_clk_ctl.attr, power_group);
+}
+#endif
+
+static int __init pcie_aspm_disable(char *str)
+{
+ aspm_disabled = 1;
+ return 1;
+}
+
+__setup("pcie_noaspm", pcie_aspm_disable);
+
+static int __init pcie_aspm_init(void)
+{
+ if (aspm_disabled)
+ return 0;
+ pci_osc_support_set(OSC_ACTIVE_STATE_PWR_SUPPORT|
+ OSC_CLOCK_PWR_CAPABILITY_SUPPORT);
+ return 0;
+}
+
+fs_initcall(pcie_aspm_init);
Index: linux/drivers/pci/pcie/Kconfig
===================================================================
--- linux.orig/drivers/pci/pcie/Kconfig 2008-01-24 10:23:09.000000000 +0800
+++ linux/drivers/pci/pcie/Kconfig 2008-02-19 11:32:10.000000000 +0800
@@ -26,3 +26,23 @@ config HOTPLUG_PCI_PCIE
When in doubt, say N.
source "drivers/pci/pcie/aer/Kconfig"
+
+#
+# PCI Express ASPM
+#
+config PCIEASPM
+ bool "PCI Express ASPM support(Experimental)"
+ depends on PCI && EXPERIMENTAL && PCIEPORTBUS
+ default y
+ help
+ This enables PCI Express ASPM (Active State Power Management) and
+ Clock Power Management. ASPM supports state L0/L0s/L1.
+
+ When in doubt, say N.
+config PCIEASPM_DEBUG
+ bool "Debug PCI Express ASPM"
+ depends on PCIEASPM
+ default n
+ help
+ This enables PCI Express ASPM debug support. It will add per-device
+ interface to control ASPM.
Index: linux/include/linux/pci.h
===================================================================
--- linux.orig/include/linux/pci.h 2008-02-19 10:51:48.000000000 +0800
+++ linux/include/linux/pci.h 2008-02-19 10:54:12.000000000 +0800
@@ -128,6 +128,7 @@ struct pci_cap_saved_state {
u32 data[0];
};
+struct pcie_link_state;
/*
* The pci_dev structure is used to describe PCI devices.
*/
@@ -165,6 +166,10 @@ struct pci_dev {
this is D0-D3, D0 being fully functional,
and D3 being off. */
+#ifdef CONFIG_PCIEASPM
+ struct pcie_link_state *link_state; /* ASPM link state. */
+#endif
+
pci_channel_state_t error_state; /* current connectivity state */
struct device dev; /* Generic device interface */
Index: linux/include/linux/pci_regs.h
===================================================================
--- linux.orig/include/linux/pci_regs.h 2008-01-24 10:23:09.000000000 +0800
+++ linux/include/linux/pci_regs.h 2008-02-19 10:54:12.000000000 +0800
@@ -395,9 +395,17 @@
#define PCI_EXP_DEVSTA_AUXPD 0x10 /* AUX Power Detected */
#define PCI_EXP_DEVSTA_TRPND 0x20 /* Transactions Pending */
#define PCI_EXP_LNKCAP 12 /* Link Capabilities */
+#define PCI_EXP_LNKCAP_ASPMS 0xc00 /* ASPM Support */
+#define PCI_EXP_LNKCAP_L0SEL 0x7000 /* L0s Exit Latency */
+#define PCI_EXP_LNKCAP_L1EL 0x38000 /* L1 Exit Latency */
+#define PCI_EXP_LNKCAP_CLKPM 0x40000 /* L1 Clock Power Management */
#define PCI_EXP_LNKCTL 16 /* Link Control */
+#define PCI_EXP_LNKCTL_RL 0x20 /* Retrain Link */
+#define PCI_EXP_LNKCTL_CCC 0x40 /* Common Clock COnfiguration */
#define PCI_EXP_LNKCTL_CLKREQ_EN 0x100 /* Enable clkreq */
#define PCI_EXP_LNKSTA 18 /* Link Status */
+#define PCI_EXP_LNKSTA_LT 0x800 /* Link Training */
+#define PCI_EXP_LNKSTA_SLC 0x1000 /* Slot Clock Configuration */
#define PCI_EXP_SLTCAP 20 /* Slot Capabilities */
#define PCI_EXP_SLTCTL 24 /* Slot Control */
#define PCI_EXP_SLTSTA 26 /* Slot Status */
Index: linux/drivers/pci/probe.c
===================================================================
--- linux.orig/drivers/pci/probe.c 2008-02-19 10:51:46.000000000 +0800
+++ linux/drivers/pci/probe.c 2008-02-20 09:24:02.000000000 +0800
@@ -9,6 +9,7 @@
#include <linux/slab.h>
#include <linux/module.h>
#include <linux/cpumask.h>
+#include <linux/pci-aspm.h>
#include "pci.h"
#define CARDBUS_LATENCY_TIMER 176 /* secondary latency timer */
@@ -1005,6 +1006,10 @@ int pci_scan_slot(struct pci_bus *bus, i
break;
}
}
+
+ if (bus->self)
+ pcie_aspm_init_link_state(bus->self);
+
return nr;
}
Index: linux/drivers/pci/remove.c
===================================================================
--- linux.orig/drivers/pci/remove.c 2008-02-19 10:51:46.000000000 +0800
+++ linux/drivers/pci/remove.c 2008-02-20 09:24:14.000000000 +0800
@@ -1,5 +1,6 @@
#include <linux/pci.h>
#include <linux/module.h>
+#include <linux/pci-aspm.h>
#include "pci.h"
static void pci_free_resources(struct pci_dev *dev)
@@ -30,6 +31,9 @@ static void pci_stop_dev(struct pci_dev
dev->global_list.next = dev->global_list.prev = NULL;
up_write(&pci_bus_sem);
}
+
+ if (dev->bus->self)
+ pcie_aspm_exit_link_state(dev);
}
static void pci_destroy_dev(struct pci_dev *dev)
Index: linux/drivers/pci/pci.c
===================================================================
--- linux.orig/drivers/pci/pci.c 2008-02-19 10:51:46.000000000 +0800
+++ linux/drivers/pci/pci.c 2008-02-20 09:24:36.000000000 +0800
@@ -18,6 +18,7 @@
#include <linux/spinlock.h>
#include <linux/string.h>
#include <linux/log2.h>
+#include <linux/pci-aspm.h>
#include <asm/dma.h> /* isa_dma_bridge_buggy */
#include "pci.h"
@@ -519,6 +520,9 @@ pci_set_power_state(struct pci_dev *dev,
if (need_restore)
pci_restore_bars(dev);
+ if (dev->bus->self)
+ pcie_aspm_pm_state_change(dev->bus->self);
+
return 0;
}
Index: linux/drivers/pci/pci-sysfs.c
===================================================================
--- linux.orig/drivers/pci/pci-sysfs.c 2008-02-19 10:51:46.000000000 +0800
+++ linux/drivers/pci/pci-sysfs.c 2008-02-20 09:24:51.000000000 +0800
@@ -21,6 +21,7 @@
#include <linux/topology.h>
#include <linux/mm.h>
#include <linux/capability.h>
+#include <linux/pci-aspm.h>
#include "pci.h"
static int sysfs_initialized; /* = 0 */
@@ -650,6 +651,8 @@ int __must_check pci_create_sysfs_dev_fi
if (pcibios_add_platform_entries(pdev))
goto err_rom_file;
+ pcie_aspm_create_sysfs_dev_files(pdev);
+
return 0;
err_rom_file:
@@ -679,6 +682,8 @@ void pci_remove_sysfs_dev_files(struct p
if (!sysfs_initialized)
return;
+ pcie_aspm_remove_sysfs_dev_files(pdev);
+
if (pdev->cfg_size < 4096)
sysfs_remove_bin_file(&pdev->dev.kobj, &pci_config_attr);
else
Index: linux/include/linux/pci-acpi.h
===================================================================
--- linux.orig/include/linux/pci-acpi.h 2008-02-19 11:03:51.000000000 +0800
+++ linux/include/linux/pci-acpi.h 2008-02-20 09:19:15.000000000 +0800
@@ -47,6 +47,7 @@
OSC_PCI_EXPRESS_CAP_STRUCTURE_CONTROL)
#ifdef CONFIG_ACPI
+#include <acpi/acpi_bus.h>
extern acpi_status pci_osc_control_set(acpi_handle handle, u32 flags);
extern acpi_status __pci_osc_support_set(u32 flags, const char *hid);
static inline acpi_status pci_osc_support_set(u32 flags)
@@ -59,13 +60,11 @@ static inline acpi_status pcie_osc_suppo
}
#else
#if !defined(AE_ERROR)
-typedef u32 acpi_status;
-#define AE_ERROR (acpi_status) (0x0001)
-#endif
-static inline acpi_status pci_osc_control_set(acpi_handle handle, u32 flags)
-{return AE_ERROR;}
-static inline acpi_status pci_osc_support_set(u32 flags) {return AE_ERROR;}
-static inline acpi_status pcie_osc_support_set(u32 flags) {return AE_ERROR;}
+#define AE_ERROR (0x0001)
+#endif
+#define pci_osc_control_set(handle, flags) (AE_ERROR)
+#define pci_osc_support_set(flags) (AE_ERROR)
+#define pcie_osc_support_set(flags) (AE_ERROR)
#endif
#endif /* _PCI_ACPI_H_ */
Index: linux/include/linux/pci-aspm.h
===================================================================
--- /dev/null 1970-01-01 00:00:00.000000000 +0000
+++ linux/include/linux/pci-aspm.h 2008-02-19 11:37:55.000000000 +0800
@@ -0,0 +1,56 @@
+/*
+ * aspm.h
+ *
+ * PCI Express ASPM defines and function prototypes
+ *
+ * Copyright (C) 2007 Intel Corp.
+ * Zhang Yanmin ([email protected])
+ * Shaohua Li ([email protected])
+ *
+ * For more information, please consult the following manuals (look at
+ * http://www.pcisig.com/ for how to get them):
+ *
+ * PCI Express Specification
+ */
+
+#ifndef LINUX_ASPM_H
+#define LINUX_ASPM_H
+
+#include <linux/pci.h>
+
+#define PCIE_LINK_STATE_L0S 1
+#define PCIE_LINK_STATE_L1 2
+#define PCIE_LINK_STATE_CLKPM 4
+
+#ifdef CONFIG_PCIEASPM
+extern void pcie_aspm_init_link_state(struct pci_dev *pdev);
+extern void pcie_aspm_exit_link_state(struct pci_dev *pdev);
+extern void pcie_aspm_pm_state_change(struct pci_dev *pdev);
+extern void pci_disable_link_state(struct pci_dev *pdev, int state);
+#else
+static inline void pcie_aspm_init_link_state(struct pci_dev *pdev)
+{
+}
+static inline void pcie_aspm_exit_link_state(struct pci_dev *pdev)
+{
+}
+static inline void pcie_aspm_pm_state_change(struct pci_dev *pdev)
+{
+}
+static inline void pci_disable_link_state(struct pci_dev *pdev, int state)
+{
+}
+#endif
+
+#ifdef CONFIG_PCIEASPM_DEBUG /* this depends on CONFIG_PCIEASPM */
+extern void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev);
+extern void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev);
+#else
+static inline void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev)
+{
+}
+static inline void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev)
+{
+}
+#endif
+#endif /* LINUX_ASPM_H */
On Wed, Feb 20, 2008 at 09:36:07AM +0800, Shaohua Li wrote:
> --- linux.orig/include/linux/pci-acpi.h 2008-02-19 11:03:51.000000000 +0800
> +++ linux/include/linux/pci-acpi.h 2008-02-20 09:19:15.000000000 +0800
> @@ -47,6 +47,7 @@
> OSC_PCI_EXPRESS_CAP_STRUCTURE_CONTROL)
>
> #ifdef CONFIG_ACPI
> +#include <acpi/acpi_bus.h>
> extern acpi_status pci_osc_control_set(acpi_handle handle, u32 flags);
> extern acpi_status __pci_osc_support_set(u32 flags, const char *hid);
> static inline acpi_status pci_osc_support_set(u32 flags)
> @@ -59,13 +60,11 @@ static inline acpi_status pcie_osc_suppo
> }
> #else
> #if !defined(AE_ERROR)
> -typedef u32 acpi_status;
> -#define AE_ERROR (acpi_status) (0x0001)
> -#endif
> -static inline acpi_status pci_osc_control_set(acpi_handle handle, u32 flags)
> -{return AE_ERROR;}
> -static inline acpi_status pci_osc_support_set(u32 flags) {return AE_ERROR;}
> -static inline acpi_status pcie_osc_support_set(u32 flags) {return AE_ERROR;}
> +#define AE_ERROR (0x0001)
> +#endif
> +#define pci_osc_control_set(handle, flags) (AE_ERROR)
> +#define pci_osc_support_set(flags) (AE_ERROR)
> +#define pcie_osc_support_set(flags) (AE_ERROR)
No, please use inline functions, don't change these functions that
should be just fine. Why are you needing to change them?
thanks,
greg k-h
On Tue, 2008-02-19 at 20:14 -0800, Greg KH wrote:
> On Wed, Feb 20, 2008 at 09:36:07AM +0800, Shaohua Li wrote:
> > --- linux.orig/include/linux/pci-acpi.h 2008-02-19 11:03:51.000000000 +0800
> > +++ linux/include/linux/pci-acpi.h 2008-02-20 09:19:15.000000000 +0800
> > @@ -47,6 +47,7 @@
> > OSC_PCI_EXPRESS_CAP_STRUCTURE_CONTROL)
> >
> > #ifdef CONFIG_ACPI
> > +#include <acpi/acpi_bus.h>
> > extern acpi_status pci_osc_control_set(acpi_handle handle, u32 flags);
> > extern acpi_status __pci_osc_support_set(u32 flags, const char *hid);
> > static inline acpi_status pci_osc_support_set(u32 flags)
> > @@ -59,13 +60,11 @@ static inline acpi_status pcie_osc_suppo
> > }
> > #else
> > #if !defined(AE_ERROR)
> > -typedef u32 acpi_status;
> > -#define AE_ERROR (acpi_status) (0x0001)
> > -#endif
> > -static inline acpi_status pci_osc_control_set(acpi_handle handle, u32 flags)
> > -{return AE_ERROR;}
> > -static inline acpi_status pci_osc_support_set(u32 flags) {return AE_ERROR;}
> > -static inline acpi_status pcie_osc_support_set(u32 flags) {return AE_ERROR;}
> > +#define AE_ERROR (0x0001)
> > +#endif
> > +#define pci_osc_control_set(handle, flags) (AE_ERROR)
> > +#define pci_osc_support_set(flags) (AE_ERROR)
> > +#define pcie_osc_support_set(flags) (AE_ERROR)
>
> No, please use inline functions, don't change these functions that
> should be just fine. Why are you needing to change them?
some types aren't defined in non-ACPI, like acpi_handle, acpi_status.
Thanks,
Shaohua
On Wed, Feb 20, 2008 at 12:48:21PM +0800, Shaohua Li wrote:
>
> On Tue, 2008-02-19 at 20:14 -0800, Greg KH wrote:
> > On Wed, Feb 20, 2008 at 09:36:07AM +0800, Shaohua Li wrote:
> > > --- linux.orig/include/linux/pci-acpi.h 2008-02-19 11:03:51.000000000 +0800
> > > +++ linux/include/linux/pci-acpi.h 2008-02-20 09:19:15.000000000 +0800
> > > @@ -47,6 +47,7 @@
> > > OSC_PCI_EXPRESS_CAP_STRUCTURE_CONTROL)
> > >
> > > #ifdef CONFIG_ACPI
> > > +#include <acpi/acpi_bus.h>
> > > extern acpi_status pci_osc_control_set(acpi_handle handle, u32 flags);
> > > extern acpi_status __pci_osc_support_set(u32 flags, const char *hid);
> > > static inline acpi_status pci_osc_support_set(u32 flags)
> > > @@ -59,13 +60,11 @@ static inline acpi_status pcie_osc_suppo
> > > }
> > > #else
> > > #if !defined(AE_ERROR)
> > > -typedef u32 acpi_status;
> > > -#define AE_ERROR (acpi_status) (0x0001)
> > > -#endif
> > > -static inline acpi_status pci_osc_control_set(acpi_handle handle, u32 flags)
> > > -{return AE_ERROR;}
> > > -static inline acpi_status pci_osc_support_set(u32 flags) {return AE_ERROR;}
> > > -static inline acpi_status pcie_osc_support_set(u32 flags) {return AE_ERROR;}
> > > +#define AE_ERROR (0x0001)
> > > +#endif
> > > +#define pci_osc_control_set(handle, flags) (AE_ERROR)
> > > +#define pci_osc_support_set(flags) (AE_ERROR)
> > > +#define pcie_osc_support_set(flags) (AE_ERROR)
> >
> > No, please use inline functions, don't change these functions that
> > should be just fine. Why are you needing to change them?
> some types aren't defined in non-ACPI, like acpi_handle, acpi_status.
Then why include a non-ACPI header file in non-ACPI .c files?
thanks,
greg k-h
On Tue, 2008-02-19 at 21:04 -0800, Greg KH wrote:
> On Wed, Feb 20, 2008 at 12:48:21PM +0800, Shaohua Li wrote:
> >
> > On Tue, 2008-02-19 at 20:14 -0800, Greg KH wrote:
> > > On Wed, Feb 20, 2008 at 09:36:07AM +0800, Shaohua Li wrote:
> > > > --- linux.orig/include/linux/pci-acpi.h 2008-02-19 11:03:51.000000000 +0800
> > > > +++ linux/include/linux/pci-acpi.h 2008-02-20 09:19:15.000000000 +0800
> > > > @@ -47,6 +47,7 @@
> > > > OSC_PCI_EXPRESS_CAP_STRUCTURE_CONTROL)
> > > >
> > > > #ifdef CONFIG_ACPI
> > > > +#include <acpi/acpi_bus.h>
> > > > extern acpi_status pci_osc_control_set(acpi_handle handle, u32 flags);
> > > > extern acpi_status __pci_osc_support_set(u32 flags, const char *hid);
> > > > static inline acpi_status pci_osc_support_set(u32 flags)
> > > > @@ -59,13 +60,11 @@ static inline acpi_status pcie_osc_suppo
> > > > }
> > > > #else
> > > > #if !defined(AE_ERROR)
> > > > -typedef u32 acpi_status;
> > > > -#define AE_ERROR (acpi_status) (0x0001)
> > > > -#endif
> > > > -static inline acpi_status pci_osc_control_set(acpi_handle handle, u32 flags)
> > > > -{return AE_ERROR;}
> > > > -static inline acpi_status pci_osc_support_set(u32 flags) {return AE_ERROR;}
> > > > -static inline acpi_status pcie_osc_support_set(u32 flags) {return AE_ERROR;}
> > > > +#define AE_ERROR (0x0001)
> > > > +#endif
> > > > +#define pci_osc_control_set(handle, flags) (AE_ERROR)
> > > > +#define pci_osc_support_set(flags) (AE_ERROR)
> > > > +#define pcie_osc_support_set(flags) (AE_ERROR)
> > >
> > > No, please use inline functions, don't change these functions that
> > > should be just fine. Why are you needing to change them?
> > some types aren't defined in non-ACPI, like acpi_handle, acpi_status.
>
> Then why include a non-ACPI header file in non-ACPI .c files?
aspm is generic, but in ACPI platform, it needs special handling. I can
add 'ifdef CONFIG_ACPI' in aspm.c to avoid changing pci-acpi.h, but
thought it's better pci-acpi.h is self-contained.
Thanks,
Shaohua
On Wed, Feb 20, 2008 at 01:24:48PM +0800, Shaohua Li wrote:
>
> On Tue, 2008-02-19 at 21:04 -0800, Greg KH wrote:
> > On Wed, Feb 20, 2008 at 12:48:21PM +0800, Shaohua Li wrote:
> > >
> > > On Tue, 2008-02-19 at 20:14 -0800, Greg KH wrote:
> > > > On Wed, Feb 20, 2008 at 09:36:07AM +0800, Shaohua Li wrote:
> > > > > --- linux.orig/include/linux/pci-acpi.h 2008-02-19 11:03:51.000000000 +0800
> > > > > +++ linux/include/linux/pci-acpi.h 2008-02-20 09:19:15.000000000 +0800
> > > > > @@ -47,6 +47,7 @@
> > > > > OSC_PCI_EXPRESS_CAP_STRUCTURE_CONTROL)
> > > > >
> > > > > #ifdef CONFIG_ACPI
> > > > > +#include <acpi/acpi_bus.h>
> > > > > extern acpi_status pci_osc_control_set(acpi_handle handle, u32 flags);
> > > > > extern acpi_status __pci_osc_support_set(u32 flags, const char *hid);
> > > > > static inline acpi_status pci_osc_support_set(u32 flags)
> > > > > @@ -59,13 +60,11 @@ static inline acpi_status pcie_osc_suppo
> > > > > }
> > > > > #else
> > > > > #if !defined(AE_ERROR)
> > > > > -typedef u32 acpi_status;
> > > > > -#define AE_ERROR (acpi_status) (0x0001)
> > > > > -#endif
> > > > > -static inline acpi_status pci_osc_control_set(acpi_handle handle, u32 flags)
> > > > > -{return AE_ERROR;}
> > > > > -static inline acpi_status pci_osc_support_set(u32 flags) {return AE_ERROR;}
> > > > > -static inline acpi_status pcie_osc_support_set(u32 flags) {return AE_ERROR;}
> > > > > +#define AE_ERROR (0x0001)
> > > > > +#endif
> > > > > +#define pci_osc_control_set(handle, flags) (AE_ERROR)
> > > > > +#define pci_osc_support_set(flags) (AE_ERROR)
> > > > > +#define pcie_osc_support_set(flags) (AE_ERROR)
> > > >
> > > > No, please use inline functions, don't change these functions that
> > > > should be just fine. Why are you needing to change them?
> > > some types aren't defined in non-ACPI, like acpi_handle, acpi_status.
> >
> > Then why include a non-ACPI header file in non-ACPI .c files?
> aspm is generic, but in ACPI platform, it needs special handling. I can
> add 'ifdef CONFIG_ACPI' in aspm.c to avoid changing pci-acpi.h, but
> thought it's better pci-acpi.h is self-contained.
Ugh, "generic" stuff needing ACPI, that's an oxymoron...
Will this ever work on non-ACPI systems? If so, then I expect to see
some #ifdefs in the .c file (or split it into two) to handle that. If
not, why not only include it if ACPI is enabled, as that means it really
is a dependancy :(
thanks,
greg k-h
On Tue, 2008-02-19 at 21:42 -0800, Greg KH wrote:
> On Wed, Feb 20, 2008 at 01:24:48PM +0800, Shaohua Li wrote:
> >
> > On Tue, 2008-02-19 at 21:04 -0800, Greg KH wrote:
> > > On Wed, Feb 20, 2008 at 12:48:21PM +0800, Shaohua Li wrote:
> > > >
> > > > On Tue, 2008-02-19 at 20:14 -0800, Greg KH wrote:
> > > > > On Wed, Feb 20, 2008 at 09:36:07AM +0800, Shaohua Li wrote:
> > > > > > --- linux.orig/include/linux/pci-acpi.h 2008-02-19 11:03:51.000000000 +0800
> > > > > > +++ linux/include/linux/pci-acpi.h 2008-02-20 09:19:15.000000000 +0800
> > > > > > @@ -47,6 +47,7 @@
> > > > > > OSC_PCI_EXPRESS_CAP_STRUCTURE_CONTROL)
> > > > > >
> > > > > > #ifdef CONFIG_ACPI
> > > > > > +#include <acpi/acpi_bus.h>
> > > > > > extern acpi_status pci_osc_control_set(acpi_handle handle, u32 flags);
> > > > > > extern acpi_status __pci_osc_support_set(u32 flags, const char *hid);
> > > > > > static inline acpi_status pci_osc_support_set(u32 flags)
> > > > > > @@ -59,13 +60,11 @@ static inline acpi_status pcie_osc_suppo
> > > > > > }
> > > > > > #else
> > > > > > #if !defined(AE_ERROR)
> > > > > > -typedef u32 acpi_status;
> > > > > > -#define AE_ERROR (acpi_status) (0x0001)
> > > > > > -#endif
> > > > > > -static inline acpi_status pci_osc_control_set(acpi_handle handle, u32 flags)
> > > > > > -{return AE_ERROR;}
> > > > > > -static inline acpi_status pci_osc_support_set(u32 flags) {return AE_ERROR;}
> > > > > > -static inline acpi_status pcie_osc_support_set(u32 flags) {return AE_ERROR;}
> > > > > > +#define AE_ERROR (0x0001)
> > > > > > +#endif
> > > > > > +#define pci_osc_control_set(handle, flags) (AE_ERROR)
> > > > > > +#define pci_osc_support_set(flags) (AE_ERROR)
> > > > > > +#define pcie_osc_support_set(flags) (AE_ERROR)
> > > > >
> > > > > No, please use inline functions, don't change these functions that
> > > > > should be just fine. Why are you needing to change them?
> > > > some types aren't defined in non-ACPI, like acpi_handle, acpi_status.
> > >
> > > Then why include a non-ACPI header file in non-ACPI .c files?
> > aspm is generic, but in ACPI platform, it needs special handling. I can
> > add 'ifdef CONFIG_ACPI' in aspm.c to avoid changing pci-acpi.h, but
> > thought it's better pci-acpi.h is self-contained.
>
> Ugh, "generic" stuff needing ACPI, that's an oxymoron...
>
> Will this ever work on non-ACPI systems? If so, then I expect to see
> some #ifdefs in the .c file (or split it into two) to handle that. If
> not, why not only include it if ACPI is enabled, as that means it really
> is a dependancy :(
It should work. Ok, this version doesn't change pci-acpi.h
PCI Express ASPM defines a protocol for PCI Express components in the D0
state to reduce Link power by placing their Links into a low power state
and instructing the other end of the Link to do likewise. This
capability allows hardware-autonomous, dynamic Link power reduction
beyond what is achievable by software-only controlled power management.
However, The device should be configured by software appropriately.
Enabling ASPM will save power, but will introduce device latency.
This patch adds ASPM support in Linux. It introduces a global policy for
ASPM, a sysfs file /sys/module/pcie_aspm/parameters/policy can control
it. The interface can be used as a boot option too. Currently we have
below setting:
-default, BIOS default setting
-powersave, highest power saving mode, enable all available ASPM
state and clock power management
-performance, highest performance, disable ASPM and clock power
management
By default, the 'default' policy is used currently.
In my test, power difference between powersave mode and performance mode
is about 1.3w in a system with 3 PCIE links.
Note: some devices might not work well with aspm, either because chipset
issue or device issue. The patch provide API (pci_disable_link_state),
driver can disable ASPM for specific device.
Signed-off-by: Shaohua Li <[email protected]>
---
drivers/pci/pci-sysfs.c | 5
drivers/pci/pci.c | 4
drivers/pci/pcie/Kconfig | 20 +
drivers/pci/pcie/Makefile | 3
drivers/pci/pcie/aspm.c | 811 ++++++++++++++++++++++++++++++++++++++++++++++
drivers/pci/probe.c | 5
drivers/pci/remove.c | 4
include/linux/pci-aspm.h | 56 +++
include/linux/pci.h | 5
include/linux/pci_regs.h | 8
10 files changed, 921 insertions(+)
Index: linux/drivers/pci/pcie/Makefile
===================================================================
--- linux.orig/drivers/pci/pcie/Makefile 2008-02-20 10:22:16.000000000 +0800
+++ linux/drivers/pci/pcie/Makefile 2008-02-20 13:59:10.000000000 +0800
@@ -2,6 +2,9 @@
# Makefile for PCI-Express PORT Driver
#
+# Build PCI Express ASPM if needed
+obj-$(CONFIG_PCIEASPM) += aspm.o
+
pcieportdrv-y := portdrv_core.o portdrv_pci.o portdrv_bus.o
obj-$(CONFIG_PCIEPORTBUS) += pcieportdrv.o
Index: linux/drivers/pci/pcie/aspm.c
===================================================================
--- /dev/null 1970-01-01 00:00:00.000000000 +0000
+++ linux/drivers/pci/pcie/aspm.c 2008-02-20 14:09:59.000000000 +0800
@@ -0,0 +1,811 @@
+/*
+ * File: drivers/pci/pcie/aspm.c
+ * Enabling PCIE link L0s/L1 state and Clock Power Management
+ *
+ * Copyright (C) 2007 Intel
+ * Copyright (C) Zhang Yanmin ([email protected])
+ * Copyright (C) Shaohua Li ([email protected])
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/pci.h>
+#include <linux/pci_regs.h>
+#include <linux/errno.h>
+#include <linux/pm.h>
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <linux/pci-aspm.h>
+#include "../pci.h"
+
+#ifdef MODULE_PARAM_PREFIX
+#undef MODULE_PARAM_PREFIX
+#endif
+#define MODULE_PARAM_PREFIX "pcie_aspm."
+
+struct endpoint_state {
+ unsigned int l0s_acceptable_latency;
+ unsigned int l1_acceptable_latency;
+};
+
+struct pcie_link_state {
+ struct list_head sibiling;
+ struct pci_dev *pdev;
+
+ /* ASPM state */
+ unsigned int support_state;
+ unsigned int enabled_state;
+ unsigned int bios_aspm_state;
+ /* upstream component */
+ unsigned int l0s_upper_latency;
+ unsigned int l1_upper_latency;
+ /* downstream component */
+ unsigned int l0s_down_latency;
+ unsigned int l1_down_latency;
+ /* Clock PM state*/
+ unsigned int clk_pm_capable;
+ unsigned int clk_pm_enabled;
+ unsigned int bios_clk_state;
+
+ /*
+ * A pcie downstream port only has one slot under it, so at most there
+ * are 8 functions
+ */
+ struct endpoint_state endpoints[8];
+};
+
+static int aspm_disabled;
+static DEFINE_MUTEX(aspm_lock);
+static LIST_HEAD(link_list);
+
+#define POLICY_DEFAULT 0 /* BIOS default setting */
+#define POLICY_PERFORMANCE 1 /* high performance */
+#define POLICY_POWERSAVE 2 /* high power saving */
+static int aspm_policy;
+static const char *policy_str[] = {
+ [POLICY_DEFAULT] = "default",
+ [POLICY_PERFORMANCE] = "performance",
+ [POLICY_POWERSAVE] = "powersave"
+};
+
+static int policy_to_aspm_state(struct pci_dev *pdev)
+{
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ switch (aspm_policy) {
+ case POLICY_PERFORMANCE:
+ /* Disable ASPM and Clock PM */
+ return 0;
+ case POLICY_POWERSAVE:
+ /* Enable ASPM L0s/L1 */
+ return PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1;
+ case POLICY_DEFAULT:
+ return link_state->bios_aspm_state;
+ }
+ return 0;
+}
+
+static int policy_to_clkpm_state(struct pci_dev *pdev)
+{
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ switch (aspm_policy) {
+ case POLICY_PERFORMANCE:
+ /* Disable ASPM and Clock PM */
+ return 0;
+ case POLICY_POWERSAVE:
+ /* Disable Clock PM */
+ return 1;
+ case POLICY_DEFAULT:
+ return link_state->bios_clk_state;
+ }
+ return 0;
+}
+
+static void pcie_set_clock_pm(struct pci_dev *pdev, int enable)
+{
+ struct pci_dev *child_dev;
+ int pos;
+ u16 reg16;
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+ if (!pos)
+ return;
+ pci_read_config_word(child_dev, pos + PCI_EXP_LNKCTL, ®16);
+ if (enable)
+ reg16 |= PCI_EXP_LNKCTL_CLKREQ_EN;
+ else
+ reg16 &= ~PCI_EXP_LNKCTL_CLKREQ_EN;
+ pci_write_config_word(child_dev, pos + PCI_EXP_LNKCTL, reg16);
+ }
+ link_state->clk_pm_enabled = !!enable;
+}
+
+static void pcie_check_clock_pm(struct pci_dev *pdev)
+{
+ int pos;
+ u32 reg32;
+ u16 reg16;
+ int capable = 1, enabled = 1;
+ struct pci_dev *child_dev;
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ /* All functions should have the same cap and state, take the worst */
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+ if (!pos)
+ return;
+ pci_read_config_dword(child_dev, pos + PCI_EXP_LNKCAP, ®32);
+ if (!(reg32 & PCI_EXP_LNKCAP_CLKPM)) {
+ capable = 0;
+ enabled = 0;
+ break;
+ }
+ pci_read_config_word(child_dev, pos + PCI_EXP_LNKCTL, ®16);
+ if (!(reg16 & PCI_EXP_LNKCTL_CLKREQ_EN))
+ enabled = 0;
+ }
+ link_state->clk_pm_capable = capable;
+ link_state->clk_pm_enabled = enabled;
+ link_state->bios_clk_state = enabled;
+ pcie_set_clock_pm(pdev, policy_to_clkpm_state(pdev));
+}
+
+/*
+ * pcie_aspm_configure_common_clock: check if the 2 ends of a link
+ * could use common clock. If they are, configure them to use the
+ * common clock. That will reduce the ASPM state exit latency.
+ */
+static void pcie_aspm_configure_common_clock(struct pci_dev *pdev)
+{
+ int pos, child_pos;
+ u16 reg16 = 0;
+ struct pci_dev *child_dev;
+ int same_clock = 1;
+
+ /*
+ * all functions of a slot should have the same Slot Clock
+ * Configuration, so just check one function
+ * */
+ child_dev = list_entry(pdev->subordinate->devices.next, struct pci_dev,
+ bus_list);
+ BUG_ON(!child_dev->is_pcie);
+
+ /* Check downstream component if bit Slot Clock Configuration is 1 */
+ child_pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+ pci_read_config_word(child_dev, child_pos + PCI_EXP_LNKSTA, ®16);
+ if (!(reg16 & PCI_EXP_LNKSTA_SLC))
+ same_clock = 0;
+
+ /* Check upstream component if bit Slot Clock Configuration is 1 */
+ pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
+ pci_read_config_word(pdev, pos + PCI_EXP_LNKSTA, ®16);
+ if (!(reg16 & PCI_EXP_LNKSTA_SLC))
+ same_clock = 0;
+
+ /* Configure downstream component, all functions */
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ child_pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+ pci_read_config_word(child_dev, child_pos + PCI_EXP_LNKCTL,
+ ®16);
+ if (same_clock)
+ reg16 |= PCI_EXP_LNKCTL_CCC;
+ else
+ reg16 &= ~PCI_EXP_LNKCTL_CCC;
+ pci_write_config_word(child_dev, child_pos + PCI_EXP_LNKCTL,
+ reg16);
+ }
+
+ /* Configure upstream component */
+ pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16);
+ if (same_clock)
+ reg16 |= PCI_EXP_LNKCTL_CCC;
+ else
+ reg16 &= ~PCI_EXP_LNKCTL_CCC;
+ pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16);
+
+ /* retrain link */
+ reg16 |= PCI_EXP_LNKCTL_RL;
+ pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16);
+
+ /* Wait for link training end */
+ while (1) {
+ pci_read_config_word(pdev, pos + PCI_EXP_LNKSTA, ®16);
+ if (!(reg16 & PCI_EXP_LNKSTA_LT))
+ break;
+ cpu_relax();
+ }
+}
+
+/*
+ * calc_L0S_latency: Convert L0s latency encoding to ns
+ */
+static unsigned int calc_L0S_latency(unsigned int latency_encoding, int ac)
+{
+ unsigned int ns = 64;
+
+ if (latency_encoding == 0x7) {
+ if (ac)
+ ns = -1U;
+ else
+ ns = 5*1000; /* > 4us */
+ } else
+ ns *= (1 << latency_encoding);
+ return ns;
+}
+
+/*
+ * calc_L1_latency: Convert L1 latency encoding to ns
+ */
+static unsigned int calc_L1_latency(unsigned int latency_encoding, int ac)
+{
+ unsigned int ns = 1000;
+
+ if (latency_encoding == 0x7) {
+ if (ac)
+ ns = -1U;
+ else
+ ns = 65*1000; /* > 64us */
+ } else
+ ns *= (1 << latency_encoding);
+ return ns;
+}
+
+static void pcie_aspm_get_cap_device(struct pci_dev *pdev, u32 *state,
+ unsigned int *l0s, unsigned int *l1, unsigned int *enabled)
+{
+ int pos;
+ u16 reg16;
+ u32 reg32;
+ unsigned int latency;
+
+ pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
+ pci_read_config_dword(pdev, pos + PCI_EXP_LNKCAP, ®32);
+ *state = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10;
+ if (*state != PCIE_LINK_STATE_L0S &&
+ *state != (PCIE_LINK_STATE_L1|PCIE_LINK_STATE_L0S))
+ *state = 0;
+ if (*state == 0)
+ return;
+
+ latency = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12;
+ *l0s = calc_L0S_latency(latency, 0);
+ if (*state & PCIE_LINK_STATE_L1) {
+ latency = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15;
+ *l1 = calc_L1_latency(latency, 0);
+ }
+ pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16);
+ *enabled = reg16 & (PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1);
+}
+
+static void pcie_aspm_cap_init(struct pci_dev *pdev)
+{
+ struct pci_dev *child_dev;
+ u32 state, tmp;
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ /* upstream component states */
+ pcie_aspm_get_cap_device(pdev, &link_state->support_state,
+ &link_state->l0s_upper_latency,
+ &link_state->l1_upper_latency,
+ &link_state->enabled_state);
+ /* downstream component states, all functions have the same setting */
+ child_dev = list_entry(pdev->subordinate->devices.next, struct pci_dev,
+ bus_list);
+ pcie_aspm_get_cap_device(child_dev, &state,
+ &link_state->l0s_down_latency,
+ &link_state->l1_down_latency,
+ &tmp);
+ link_state->support_state &= state;
+ if (!link_state->support_state)
+ return;
+ link_state->enabled_state &= link_state->support_state;
+ link_state->bios_aspm_state = link_state->enabled_state;
+
+ /* ENDPOINT states*/
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ int pos;
+ u32 reg32;
+ unsigned int latency;
+ struct endpoint_state *ep_state =
+ &link_state->endpoints[PCI_FUNC(child_dev->devfn)];
+
+ if (child_dev->pcie_type != PCI_EXP_TYPE_ENDPOINT &&
+ child_dev->pcie_type != PCI_EXP_TYPE_LEG_END)
+ continue;
+
+ pos = pci_find_capability(child_dev, PCI_CAP_ID_EXP);
+ pci_read_config_dword(child_dev, pos + PCI_EXP_DEVCAP, ®32);
+ latency = (reg32 & PCI_EXP_DEVCAP_L0S) >> 6;
+ latency = calc_L0S_latency(latency, 1);
+ ep_state->l0s_acceptable_latency = latency;
+ if (link_state->support_state & PCIE_LINK_STATE_L1) {
+ latency = (reg32 & PCI_EXP_DEVCAP_L1) >> 9;
+ latency = calc_L1_latency(latency, 1);
+ ep_state->l1_acceptable_latency = latency;
+ }
+ }
+}
+
+static unsigned int __pcie_aspm_check_state_one(struct pci_dev *pdev,
+ unsigned int state)
+{
+ struct pci_dev *parent_dev, *tmp_dev;
+ unsigned int latency, l1_latency = 0;
+ struct pcie_link_state *link_state;
+ struct endpoint_state *ep_state;
+
+ parent_dev = pdev->bus->self;
+ link_state = parent_dev->link_state;
+ state &= link_state->support_state;
+ if (state == 0)
+ return 0;
+ ep_state = &link_state->endpoints[PCI_FUNC(pdev->devfn)];
+
+ /*
+ * Check latency for endpoint device.
+ * TBD: The latency from the endpoint to root complex vary per
+ * switch's upstream link state above the device. Here we just do a
+ * simple check which assumes all links above the device can be in L1
+ * state, that is we just consider the worst case. If switch's upstream
+ * link can't be put into L0S/L1, then our check is too strictly.
+ */
+ tmp_dev = pdev;
+ while (state & (PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1)) {
+ parent_dev = tmp_dev->bus->self;
+ link_state = parent_dev->link_state;
+ if (state & PCIE_LINK_STATE_L0S) {
+ latency = max_t(unsigned int,
+ link_state->l0s_upper_latency,
+ link_state->l0s_down_latency);
+ if (latency > ep_state->l0s_acceptable_latency)
+ state &= ~PCIE_LINK_STATE_L0S;
+ }
+ if (state & PCIE_LINK_STATE_L1) {
+ latency = max_t(unsigned int,
+ link_state->l1_upper_latency,
+ link_state->l1_down_latency);
+ if (latency + l1_latency >
+ ep_state->l1_acceptable_latency)
+ state &= ~PCIE_LINK_STATE_L1;
+ }
+ if (!parent_dev->bus->self) /* parent_dev is a root port */
+ break;
+ else {
+ /*
+ * parent_dev is the downstream port of a switch, make
+ * tmp_dev the upstream port of the switch
+ */
+ tmp_dev = parent_dev->bus->self;
+ /*
+ * every switch on the path to root complex need 1 more
+ * microsecond for L1. Spec doesn't mention L0S.
+ */
+ if (state & PCIE_LINK_STATE_L1)
+ l1_latency += 1000;
+ }
+ }
+ return state;
+}
+
+static unsigned int pcie_aspm_check_state(struct pci_dev *pdev,
+ unsigned int state)
+{
+ struct pci_dev *child_dev;
+
+ /* If no child, disable the link */
+ if (list_empty(&pdev->subordinate->devices))
+ return 0;
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ if (child_dev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE) {
+ /*
+ * If downstream component of a link is pci bridge, we
+ * disable ASPM for now for the link
+ * */
+ state = 0;
+ break;
+ }
+ if ((child_dev->pcie_type != PCI_EXP_TYPE_ENDPOINT &&
+ child_dev->pcie_type != PCI_EXP_TYPE_LEG_END))
+ continue;
+ /* Device not in D0 doesn't need check latency */
+ if (child_dev->current_state == PCI_D1 ||
+ child_dev->current_state == PCI_D2 ||
+ child_dev->current_state == PCI_D3hot ||
+ child_dev->current_state == PCI_D3cold)
+ continue;
+ state = __pcie_aspm_check_state_one(child_dev, state);
+ }
+ return state;
+}
+
+static void __pcie_aspm_config_one_dev(struct pci_dev *pdev, unsigned int state)
+{
+ u16 reg16;
+ int pos = pci_find_capability(pdev, PCI_CAP_ID_EXP);
+
+ pci_read_config_word(pdev, pos + PCI_EXP_LNKCTL, ®16);
+ reg16 &= ~0x3;
+ reg16 |= state;
+ pci_write_config_word(pdev, pos + PCI_EXP_LNKCTL, reg16);
+}
+
+static void __pcie_aspm_config_link(struct pci_dev *pdev, unsigned int state)
+{
+ struct pci_dev *child_dev;
+ int valid = 1;
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ /*
+ * if the downstream component has pci bridge function, don't do ASPM
+ * now
+ */
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list) {
+ if (child_dev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE) {
+ valid = 0;
+ break;
+ }
+ }
+ if (!valid)
+ return;
+
+ /*
+ * spec 2.0 suggests all functions should be configured the same
+ * setting for ASPM. Enabling ASPM L1 should be done in upstream
+ * component first and then downstream, and vice versa for disabling
+ * ASPM L1. Spec doesn't mention L0S.
+ */
+ if (state & PCIE_LINK_STATE_L1)
+ __pcie_aspm_config_one_dev(pdev, state);
+
+ list_for_each_entry(child_dev, &pdev->subordinate->devices, bus_list)
+ __pcie_aspm_config_one_dev(child_dev, state);
+
+ if (!(state & PCIE_LINK_STATE_L1))
+ __pcie_aspm_config_one_dev(pdev, state);
+
+ link_state->enabled_state = state;
+}
+
+static void __pcie_aspm_configure_link_state(struct pci_dev *pdev,
+ unsigned int state)
+{
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ if (link_state->support_state == 0)
+ return;
+ state &= PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1;
+
+ /* state 0 means disabling aspm */
+ state = pcie_aspm_check_state(pdev, state);
+ if (link_state->enabled_state == state)
+ return;
+ __pcie_aspm_config_link(pdev, state);
+}
+
+/*
+ * pcie_aspm_configure_link_state: enable/disable PCI express link state
+ * @pdev: the root port or switch downstream port
+ */
+static void pcie_aspm_configure_link_state(struct pci_dev *pdev,
+ unsigned int state)
+{
+ down_read(&pci_bus_sem);
+ mutex_lock(&aspm_lock);
+ __pcie_aspm_configure_link_state(pdev, state);
+ mutex_unlock(&aspm_lock);
+ up_read(&pci_bus_sem);
+}
+
+static void free_link_state(struct pci_dev *pdev)
+{
+ kfree(pdev->link_state);
+ pdev->link_state = NULL;
+}
+
+/*
+ * pcie_aspm_init_link_state: Initiate PCI express link state.
+ * It is called after the pcie and its children devices are scaned.
+ * @pdev: the root port or switch downstream port
+ */
+void pcie_aspm_init_link_state(struct pci_dev *pdev)
+{
+ unsigned int state;
+ struct pcie_link_state *link_state;
+ int error = 0;
+
+ if (aspm_disabled || !pdev->is_pcie || pdev->link_state)
+ return;
+ if (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
+ pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
+ return;
+ down_read(&pci_bus_sem);
+ if (list_empty(&pdev->subordinate->devices))
+ goto out;
+
+ mutex_lock(&aspm_lock);
+
+ link_state = kzalloc(sizeof(*link_state), GFP_KERNEL);
+ if (!link_state)
+ goto unlock_out;
+ pdev->link_state = link_state;
+
+ pcie_aspm_configure_common_clock(pdev);
+
+ pcie_aspm_cap_init(pdev);
+
+ /* config link state to avoid BIOS error */
+ state = pcie_aspm_check_state(pdev, policy_to_aspm_state(pdev));
+ __pcie_aspm_config_link(pdev, state);
+
+ pcie_check_clock_pm(pdev);
+
+ link_state->pdev = pdev;
+ list_add(&link_state->sibiling, &link_list);
+
+unlock_out:
+ if (error)
+ free_link_state(pdev);
+ mutex_unlock(&aspm_lock);
+out:
+ up_read(&pci_bus_sem);
+}
+
+/* @pdev: the endpoint device */
+void pcie_aspm_exit_link_state(struct pci_dev *pdev)
+{
+ struct pci_dev *parent = pdev->bus->self;
+ struct pcie_link_state *link_state = parent->link_state;
+
+ if (aspm_disabled || !pdev->is_pcie || !parent || !link_state)
+ return;
+ if (parent->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
+ parent->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
+ return;
+ down_read(&pci_bus_sem);
+ mutex_lock(&aspm_lock);
+
+ /*
+ * All PCIe functions are in one slot, remove one function will remove
+ * the the whole slot, so just wait
+ */
+ if (!list_empty(&parent->subordinate->devices))
+ goto out;
+
+ /* All functions are removed, so just disable ASPM for the link */
+ __pcie_aspm_config_one_dev(parent, 0);
+ list_del(&link_state->sibiling);
+ /* Clock PM is for endpoint device */
+
+ free_link_state(parent);
+out:
+ mutex_unlock(&aspm_lock);
+ up_read(&pci_bus_sem);
+}
+
+/* @pdev: the root port or switch downstream port */
+void pcie_aspm_pm_state_change(struct pci_dev *pdev)
+{
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ if (aspm_disabled || !pdev->is_pcie || !pdev->link_state)
+ return;
+ if (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
+ pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM)
+ return;
+ /*
+ * devices changed PM state, we should recheck if latency meets all
+ * functions' requirement
+ */
+ pcie_aspm_configure_link_state(pdev, link_state->enabled_state);
+}
+
+/*
+ * pci_disable_link_state - disable pci device's link state, so the link will
+ * never enter specific states
+ */
+void pci_disable_link_state(struct pci_dev *pdev, int state)
+{
+ struct pci_dev *parent = pdev->bus->self;
+ struct pcie_link_state *link_state;
+
+ if (aspm_disabled || !pdev->is_pcie)
+ return;
+ if (pdev->pcie_type == PCI_EXP_TYPE_ROOT_PORT ||
+ pdev->pcie_type == PCI_EXP_TYPE_DOWNSTREAM)
+ parent = pdev;
+ if (!parent || !parent->link_state)
+ return;
+
+ down_read(&pci_bus_sem);
+ mutex_lock(&aspm_lock);
+ link_state = parent->link_state;
+ link_state->support_state &=
+ ~(state & (PCIE_LINK_STATE_L0S|PCIE_LINK_STATE_L1));
+ if (state & PCIE_LINK_STATE_CLKPM)
+ link_state->clk_pm_capable = 0;
+
+ __pcie_aspm_configure_link_state(parent, link_state->enabled_state);
+ if (!link_state->clk_pm_capable && link_state->clk_pm_enabled)
+ pcie_set_clock_pm(parent, 0);
+ mutex_unlock(&aspm_lock);
+ up_read(&pci_bus_sem);
+}
+EXPORT_SYMBOL(pci_disable_link_state);
+
+static int pcie_aspm_set_policy(const char *val, struct kernel_param *kp)
+{
+ int i;
+ struct pci_dev *pdev;
+ struct pcie_link_state *link_state;
+
+ for (i = 0; i < ARRAY_SIZE(policy_str); i++)
+ if (!strncmp(val, policy_str[i], strlen(policy_str[i])))
+ break;
+ if (i >= ARRAY_SIZE(policy_str))
+ return -EINVAL;
+ if (i == aspm_policy)
+ return 0;
+
+ down_read(&pci_bus_sem);
+ mutex_lock(&aspm_lock);
+ aspm_policy = i;
+ list_for_each_entry(link_state, &link_list, sibiling) {
+ pdev = link_state->pdev;
+ __pcie_aspm_configure_link_state(pdev,
+ policy_to_aspm_state(pdev));
+ if (link_state->clk_pm_capable &&
+ link_state->clk_pm_enabled != policy_to_clkpm_state(pdev))
+ pcie_set_clock_pm(pdev, policy_to_clkpm_state(pdev));
+
+ }
+ mutex_unlock(&aspm_lock);
+ up_read(&pci_bus_sem);
+ return 0;
+}
+
+static int pcie_aspm_get_policy(char *buffer, struct kernel_param *kp)
+{
+ int i, cnt = 0;
+ for (i = 0; i < ARRAY_SIZE(policy_str); i++)
+ if (i == aspm_policy)
+ cnt += sprintf(buffer + cnt, "[%s] ", policy_str[i]);
+ else
+ cnt += sprintf(buffer + cnt, "%s ", policy_str[i]);
+ return cnt;
+}
+
+module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy,
+ NULL, 0644);
+
+#ifdef CONFIG_PCIEASPM_DEBUG
+static ssize_t link_state_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct pci_dev *pci_device = to_pci_dev(dev);
+ struct pcie_link_state *link_state = pci_device->link_state;
+
+ return sprintf(buf, "%d\n", link_state->enabled_state);
+}
+
+static ssize_t link_state_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf,
+ size_t n)
+{
+ struct pci_dev *pci_device = to_pci_dev(dev);
+ int state;
+
+ if (n < 1)
+ return -EINVAL;
+ state = buf[0]-'0';
+ if (state >= 0 && state <= 3) {
+ /* setup link aspm state */
+ pcie_aspm_configure_link_state(pci_device, state);
+ return n;
+ }
+
+ return -EINVAL;
+}
+
+static ssize_t clk_ctl_show(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct pci_dev *pci_device = to_pci_dev(dev);
+ struct pcie_link_state *link_state = pci_device->link_state;
+
+ return sprintf(buf, "%d\n", link_state->clk_pm_enabled);
+}
+
+static ssize_t clk_ctl_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf,
+ size_t n)
+{
+ struct pci_dev *pci_device = to_pci_dev(dev);
+ int state;
+
+ if (n < 1)
+ return -EINVAL;
+ state = buf[0]-'0';
+
+ down_read(&pci_bus_sem);
+ mutex_lock(&aspm_lock);
+ pcie_set_clock_pm(pci_device, !!state);
+ mutex_unlock(&aspm_lock);
+ up_read(&pci_bus_sem);
+
+ return n;
+}
+
+static DEVICE_ATTR(link_state, 0644, link_state_show, link_state_store);
+static DEVICE_ATTR(clk_ctl, 0644, clk_ctl_show, clk_ctl_store);
+
+static char power_group[] = "power";
+void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev)
+{
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ if (!pdev->is_pcie || (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
+ pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) || !link_state)
+ return;
+
+ if (link_state->support_state)
+ sysfs_add_file_to_group(&pdev->dev.kobj,
+ &dev_attr_link_state.attr, power_group);
+ if (link_state->clk_pm_capable)
+ sysfs_add_file_to_group(&pdev->dev.kobj,
+ &dev_attr_clk_ctl.attr, power_group);
+}
+
+void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev)
+{
+ struct pcie_link_state *link_state = pdev->link_state;
+
+ if (!pdev->is_pcie || (pdev->pcie_type != PCI_EXP_TYPE_ROOT_PORT &&
+ pdev->pcie_type != PCI_EXP_TYPE_DOWNSTREAM) || !link_state)
+ return;
+
+ if (link_state->support_state)
+ sysfs_remove_file_from_group(&pdev->dev.kobj,
+ &dev_attr_link_state.attr, power_group);
+ if (link_state->clk_pm_capable)
+ sysfs_remove_file_from_group(&pdev->dev.kobj,
+ &dev_attr_clk_ctl.attr, power_group);
+}
+#endif
+
+static int __init pcie_aspm_disable(char *str)
+{
+ aspm_disabled = 1;
+ return 1;
+}
+
+__setup("pcie_noaspm", pcie_aspm_disable);
+
+#ifdef CONFIG_ACPI
+#include <acpi/acpi_bus.h>
+#include <linux/pci-acpi.h>
+static void pcie_aspm_platform_init(void)
+{
+ pcie_osc_support_set(OSC_ACTIVE_STATE_PWR_SUPPORT|
+ OSC_CLOCK_PWR_CAPABILITY_SUPPORT);
+}
+#else
+static inline void pcie_aspm_platform_init(void) { }
+#endif
+
+static int __init pcie_aspm_init(void)
+{
+ if (aspm_disabled)
+ return 0;
+ pcie_aspm_platform_init();
+ return 0;
+}
+
+fs_initcall(pcie_aspm_init);
Index: linux/drivers/pci/pcie/Kconfig
===================================================================
--- linux.orig/drivers/pci/pcie/Kconfig 2008-02-20 10:22:16.000000000 +0800
+++ linux/drivers/pci/pcie/Kconfig 2008-02-20 13:59:10.000000000 +0800
@@ -26,3 +26,23 @@ config HOTPLUG_PCI_PCIE
When in doubt, say N.
source "drivers/pci/pcie/aer/Kconfig"
+
+#
+# PCI Express ASPM
+#
+config PCIEASPM
+ bool "PCI Express ASPM support(Experimental)"
+ depends on PCI && EXPERIMENTAL && PCIEPORTBUS
+ default y
+ help
+ This enables PCI Express ASPM (Active State Power Management) and
+ Clock Power Management. ASPM supports state L0/L0s/L1.
+
+ When in doubt, say N.
+config PCIEASPM_DEBUG
+ bool "Debug PCI Express ASPM"
+ depends on PCIEASPM
+ default n
+ help
+ This enables PCI Express ASPM debug support. It will add per-device
+ interface to control ASPM.
Index: linux/include/linux/pci.h
===================================================================
--- linux.orig/include/linux/pci.h 2008-02-20 10:22:16.000000000 +0800
+++ linux/include/linux/pci.h 2008-02-20 13:59:10.000000000 +0800
@@ -128,6 +128,7 @@ struct pci_cap_saved_state {
u32 data[0];
};
+struct pcie_link_state;
/*
* The pci_dev structure is used to describe PCI devices.
*/
@@ -165,6 +166,10 @@ struct pci_dev {
this is D0-D3, D0 being fully functional,
and D3 being off. */
+#ifdef CONFIG_PCIEASPM
+ struct pcie_link_state *link_state; /* ASPM link state. */
+#endif
+
pci_channel_state_t error_state; /* current connectivity state */
struct device dev; /* Generic device interface */
Index: linux/include/linux/pci_regs.h
===================================================================
--- linux.orig/include/linux/pci_regs.h 2008-02-20 10:22:16.000000000 +0800
+++ linux/include/linux/pci_regs.h 2008-02-20 13:59:10.000000000 +0800
@@ -395,9 +395,17 @@
#define PCI_EXP_DEVSTA_AUXPD 0x10 /* AUX Power Detected */
#define PCI_EXP_DEVSTA_TRPND 0x20 /* Transactions Pending */
#define PCI_EXP_LNKCAP 12 /* Link Capabilities */
+#define PCI_EXP_LNKCAP_ASPMS 0xc00 /* ASPM Support */
+#define PCI_EXP_LNKCAP_L0SEL 0x7000 /* L0s Exit Latency */
+#define PCI_EXP_LNKCAP_L1EL 0x38000 /* L1 Exit Latency */
+#define PCI_EXP_LNKCAP_CLKPM 0x40000 /* L1 Clock Power Management */
#define PCI_EXP_LNKCTL 16 /* Link Control */
+#define PCI_EXP_LNKCTL_RL 0x20 /* Retrain Link */
+#define PCI_EXP_LNKCTL_CCC 0x40 /* Common Clock COnfiguration */
#define PCI_EXP_LNKCTL_CLKREQ_EN 0x100 /* Enable clkreq */
#define PCI_EXP_LNKSTA 18 /* Link Status */
+#define PCI_EXP_LNKSTA_LT 0x800 /* Link Training */
+#define PCI_EXP_LNKSTA_SLC 0x1000 /* Slot Clock Configuration */
#define PCI_EXP_SLTCAP 20 /* Slot Capabilities */
#define PCI_EXP_SLTCTL 24 /* Slot Control */
#define PCI_EXP_SLTSTA 26 /* Slot Status */
Index: linux/drivers/pci/probe.c
===================================================================
--- linux.orig/drivers/pci/probe.c 2008-02-20 10:22:16.000000000 +0800
+++ linux/drivers/pci/probe.c 2008-02-20 13:59:10.000000000 +0800
@@ -9,6 +9,7 @@
#include <linux/slab.h>
#include <linux/module.h>
#include <linux/cpumask.h>
+#include <linux/pci-aspm.h>
#include "pci.h"
#define CARDBUS_LATENCY_TIMER 176 /* secondary latency timer */
@@ -1005,6 +1006,10 @@ int pci_scan_slot(struct pci_bus *bus, i
break;
}
}
+
+ if (bus->self)
+ pcie_aspm_init_link_state(bus->self);
+
return nr;
}
Index: linux/drivers/pci/remove.c
===================================================================
--- linux.orig/drivers/pci/remove.c 2008-02-20 10:22:16.000000000 +0800
+++ linux/drivers/pci/remove.c 2008-02-20 13:59:10.000000000 +0800
@@ -1,5 +1,6 @@
#include <linux/pci.h>
#include <linux/module.h>
+#include <linux/pci-aspm.h>
#include "pci.h"
static void pci_free_resources(struct pci_dev *dev)
@@ -30,6 +31,9 @@ static void pci_stop_dev(struct pci_dev
dev->global_list.next = dev->global_list.prev = NULL;
up_write(&pci_bus_sem);
}
+
+ if (dev->bus->self)
+ pcie_aspm_exit_link_state(dev);
}
static void pci_destroy_dev(struct pci_dev *dev)
Index: linux/drivers/pci/pci.c
===================================================================
--- linux.orig/drivers/pci/pci.c 2008-02-20 10:22:16.000000000 +0800
+++ linux/drivers/pci/pci.c 2008-02-20 13:59:10.000000000 +0800
@@ -18,6 +18,7 @@
#include <linux/spinlock.h>
#include <linux/string.h>
#include <linux/log2.h>
+#include <linux/pci-aspm.h>
#include <asm/dma.h> /* isa_dma_bridge_buggy */
#include "pci.h"
@@ -519,6 +520,9 @@ pci_set_power_state(struct pci_dev *dev,
if (need_restore)
pci_restore_bars(dev);
+ if (dev->bus->self)
+ pcie_aspm_pm_state_change(dev->bus->self);
+
return 0;
}
Index: linux/drivers/pci/pci-sysfs.c
===================================================================
--- linux.orig/drivers/pci/pci-sysfs.c 2008-02-20 10:22:16.000000000 +0800
+++ linux/drivers/pci/pci-sysfs.c 2008-02-20 13:59:10.000000000 +0800
@@ -21,6 +21,7 @@
#include <linux/topology.h>
#include <linux/mm.h>
#include <linux/capability.h>
+#include <linux/pci-aspm.h>
#include "pci.h"
static int sysfs_initialized; /* = 0 */
@@ -650,6 +651,8 @@ int __must_check pci_create_sysfs_dev_fi
if (pcibios_add_platform_entries(pdev))
goto err_rom_file;
+ pcie_aspm_create_sysfs_dev_files(pdev);
+
return 0;
err_rom_file:
@@ -679,6 +682,8 @@ void pci_remove_sysfs_dev_files(struct p
if (!sysfs_initialized)
return;
+ pcie_aspm_remove_sysfs_dev_files(pdev);
+
if (pdev->cfg_size < 4096)
sysfs_remove_bin_file(&pdev->dev.kobj, &pci_config_attr);
else
Index: linux/include/linux/pci-aspm.h
===================================================================
--- /dev/null 1970-01-01 00:00:00.000000000 +0000
+++ linux/include/linux/pci-aspm.h 2008-02-20 13:59:10.000000000 +0800
@@ -0,0 +1,56 @@
+/*
+ * aspm.h
+ *
+ * PCI Express ASPM defines and function prototypes
+ *
+ * Copyright (C) 2007 Intel Corp.
+ * Zhang Yanmin ([email protected])
+ * Shaohua Li ([email protected])
+ *
+ * For more information, please consult the following manuals (look at
+ * http://www.pcisig.com/ for how to get them):
+ *
+ * PCI Express Specification
+ */
+
+#ifndef LINUX_ASPM_H
+#define LINUX_ASPM_H
+
+#include <linux/pci.h>
+
+#define PCIE_LINK_STATE_L0S 1
+#define PCIE_LINK_STATE_L1 2
+#define PCIE_LINK_STATE_CLKPM 4
+
+#ifdef CONFIG_PCIEASPM
+extern void pcie_aspm_init_link_state(struct pci_dev *pdev);
+extern void pcie_aspm_exit_link_state(struct pci_dev *pdev);
+extern void pcie_aspm_pm_state_change(struct pci_dev *pdev);
+extern void pci_disable_link_state(struct pci_dev *pdev, int state);
+#else
+static inline void pcie_aspm_init_link_state(struct pci_dev *pdev)
+{
+}
+static inline void pcie_aspm_exit_link_state(struct pci_dev *pdev)
+{
+}
+static inline void pcie_aspm_pm_state_change(struct pci_dev *pdev)
+{
+}
+static inline void pci_disable_link_state(struct pci_dev *pdev, int state)
+{
+}
+#endif
+
+#ifdef CONFIG_PCIEASPM_DEBUG /* this depends on CONFIG_PCIEASPM */
+extern void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev);
+extern void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev);
+#else
+static inline void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev)
+{
+}
+static inline void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev)
+{
+}
+#endif
+#endif /* LINUX_ASPM_H */