Similar to regular BAR, drivers can use pci_resize_resource() to resize
an IOV BAR to the desired size provided that is supported by the
hardware, which can be queried using pci_rebar_get_possible_sizes().
This feature is based on the fact that (default VF BAR size) * (supported
VF number) covers all possible resource/address ranges (rounded up to the
power of 2 size). For example, the total size of the resource behind the
BAR is 256GB, the supported maximum VF number is 4, the default VF BAR
size should then be set to 64GB. When the enabled vf_num changes, the VF
BAR size will adjust accordingly as
- For 1 VF, VF BAR size is 256GB
- For 2 VFs, VF BAR size is 128GB
- For 4 VFs, VF BAR size is 64GB
This feature is necessary to accommodate the limited address per PCI port.
Lianjie Shi (1):
PCI: Support VF resizable BAR
drivers/pci/pci.c | 44 +++++++++++++++++++++++++++++++++-
drivers/pci/setup-res.c | 45 +++++++++++++++++++++++++++++------
include/uapi/linux/pci_regs.h | 1 +
3 files changed, 82 insertions(+), 8 deletions(-)
base-commit: cf87f46fd34d6c19283d9625a7822f20d90b64a4
--
2.34.1
Add support for VF resizable BAR PCI extended cap.
Similar to regular BAR, drivers can use pci_resize_resource() to
resize an IOV BAR. For each VF, dev->sriov->barsz of the IOV BAR is
resized, but the total resource size of the IOV resource should not
exceed its original size upon init.
Based on following patch series:
Link: https://lore.kernel.org/lkml/YbqGplTKl5i%2F1%2FkY@rocinante/T/
Signed-off-by: Lianjie Shi <[email protected]>
---
drivers/pci/pci.c | 44 +++++++++++++++++++++++++++++++++-
drivers/pci/setup-res.c | 45 +++++++++++++++++++++++++++++------
include/uapi/linux/pci_regs.h | 1 +
3 files changed, 82 insertions(+), 8 deletions(-)
diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c
index e5f243dd4..bb9a2f322 100644
--- a/drivers/pci/pci.c
+++ b/drivers/pci/pci.c
@@ -1867,6 +1867,39 @@ static void pci_restore_rebar_state(struct pci_dev *pdev)
}
}
+static void pci_restore_vf_rebar_state(struct pci_dev *pdev)
+{
+#ifdef CONFIG_PCI_IOV
+ unsigned int pos, nbars, i;
+ u32 ctrl;
+ u16 total;
+
+ pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_VF_REBAR);
+ if (!pos)
+ return;
+
+ pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
+ nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, ctrl);
+
+ for (i = 0; i < nbars; i++, pos += 8) {
+ struct resource *res;
+ int bar_idx, size;
+
+ pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
+ bar_idx = ctrl & PCI_REBAR_CTRL_BAR_IDX;
+ total = pdev->sriov->total_VFs;
+ if (!total)
+ return;
+
+ res = pdev->resource + bar_idx + PCI_IOV_RESOURCES;
+ size = pci_rebar_bytes_to_size(resource_size(res) / total);
+ ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE;
+ ctrl |= FIELD_PREP(PCI_REBAR_CTRL_BAR_SIZE, size);
+ pci_write_config_dword(pdev, pos + PCI_REBAR_CTRL, ctrl);
+ }
+#endif
+}
+
/**
* pci_restore_state - Restore the saved state of a PCI device
* @dev: PCI device that we're dealing with
@@ -1882,6 +1915,7 @@ void pci_restore_state(struct pci_dev *dev)
pci_restore_ats_state(dev);
pci_restore_vc_state(dev);
pci_restore_rebar_state(dev);
+ pci_restore_vf_rebar_state(dev);
pci_restore_dpc_state(dev);
pci_restore_ptm_state(dev);
@@ -3677,10 +3711,18 @@ void pci_acs_init(struct pci_dev *dev)
*/
static int pci_rebar_find_pos(struct pci_dev *pdev, int bar)
{
+ int cap = PCI_EXT_CAP_ID_REBAR;
unsigned int pos, nbars, i;
u32 ctrl;
- pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_REBAR);
+#ifdef CONFIG_PCI_IOV
+ if (bar >= PCI_IOV_RESOURCES) {
+ cap = PCI_EXT_CAP_ID_VF_REBAR;
+ bar -= PCI_IOV_RESOURCES;
+ }
+#endif
+
+ pos = pci_find_ext_capability(pdev, cap);
if (!pos)
return -ENOTSUPP;
diff --git a/drivers/pci/setup-res.c b/drivers/pci/setup-res.c
index c6d933ddf..d978a2ccf 100644
--- a/drivers/pci/setup-res.c
+++ b/drivers/pci/setup-res.c
@@ -427,13 +427,32 @@ void pci_release_resource(struct pci_dev *dev, int resno)
}
EXPORT_SYMBOL(pci_release_resource);
+static int pci_memory_decoding(struct pci_dev *dev, int resno)
+{
+ u16 cmd;
+
+#ifdef CONFIG_PCI_IOV
+ if (resno >= PCI_IOV_RESOURCES) {
+ pci_read_config_word(dev, dev->sriov->pos + PCI_SRIOV_CTRL, &cmd);
+ if (cmd & PCI_SRIOV_CTRL_MSE)
+ return -EBUSY;
+ else
+ return 0;
+ }
+#endif
+ pci_read_config_word(dev, PCI_COMMAND, &cmd);
+ if (cmd & PCI_COMMAND_MEMORY)
+ return -EBUSY;
+
+ return 0;
+}
+
int pci_resize_resource(struct pci_dev *dev, int resno, int size)
{
struct resource *res = dev->resource + resno;
struct pci_host_bridge *host;
int old, ret;
u32 sizes;
- u16 cmd;
/* Check if we must preserve the firmware's resource assignment */
host = pci_find_host_bridge(dev->bus);
@@ -444,9 +463,9 @@ int pci_resize_resource(struct pci_dev *dev, int resno, int size)
if (!(res->flags & IORESOURCE_UNSET))
return -EBUSY;
- pci_read_config_word(dev, PCI_COMMAND, &cmd);
- if (cmd & PCI_COMMAND_MEMORY)
- return -EBUSY;
+ ret = pci_memory_decoding(dev, resno);
+ if (ret)
+ return ret;
sizes = pci_rebar_get_possible_sizes(dev, resno);
if (!sizes)
@@ -463,19 +482,31 @@ int pci_resize_resource(struct pci_dev *dev, int resno, int size)
if (ret)
return ret;
- res->end = res->start + pci_rebar_size_to_bytes(size) - 1;
+#ifdef CONFIG_PCI_IOV
+ if (resno >= PCI_IOV_RESOURCES)
+ dev->sriov->barsz[resno - PCI_IOV_RESOURCES] = pci_rebar_size_to_bytes(size);
+ else
+#endif
+ res->end = res->start + pci_rebar_size_to_bytes(size) - 1;
/* Check if the new config works by trying to assign everything. */
if (dev->bus->self) {
ret = pci_reassign_bridge_resources(dev->bus->self, res->flags);
- if (ret)
+ if (ret && ret != -ENOENT)
goto error_resize;
}
return 0;
error_resize:
pci_rebar_set_size(dev, resno, old);
- res->end = res->start + pci_rebar_size_to_bytes(old) - 1;
+
+#ifdef CONFIG_PCI_IOV
+ if (resno >= PCI_IOV_RESOURCES)
+ dev->sriov->barsz[resno - PCI_IOV_RESOURCES] = pci_rebar_size_to_bytes(old);
+ else
+#endif
+ res->end = res->start + pci_rebar_size_to_bytes(old) - 1;
+
return ret;
}
EXPORT_SYMBOL(pci_resize_resource);
diff --git a/include/uapi/linux/pci_regs.h b/include/uapi/linux/pci_regs.h
index a39193213..a66b90982 100644
--- a/include/uapi/linux/pci_regs.h
+++ b/include/uapi/linux/pci_regs.h
@@ -738,6 +738,7 @@
#define PCI_EXT_CAP_ID_L1SS 0x1E /* L1 PM Substates */
#define PCI_EXT_CAP_ID_PTM 0x1F /* Precision Time Measurement */
#define PCI_EXT_CAP_ID_DVSEC 0x23 /* Designated Vendor-Specific */
+#define PCI_EXT_CAP_ID_VF_REBAR 0x24 /* VF Resizable BAR */
#define PCI_EXT_CAP_ID_DLF 0x25 /* Data Link Feature */
#define PCI_EXT_CAP_ID_PL_16GT 0x26 /* Physical Layer 16.0 GT/s */
#define PCI_EXT_CAP_ID_PL_32GT 0x2A /* Physical Layer 32.0 GT/s */
--
2.34.1
Hi Lianjie,
kernel test robot noticed the following build errors:
[auto build test ERROR on cf87f46fd34d6c19283d9625a7822f20d90b64a4]
url: https://github.com/intel-lab-lkp/linux/commits/Lianjie-Shi/PCI-Support-VF-resizable-BAR/20240516-173624
base: cf87f46fd34d6c19283d9625a7822f20d90b64a4
patch link: https://lore.kernel.org/r/20240516093334.2266599-2-Lianjie.Shi%40amd.com
patch subject: [PATCH 1/1] PCI: Support VF resizable BAR
config: i386-randconfig-014-20240518 (https://download.01.org/0day-ci/archive/20240518/[email protected]/config)
compiler: gcc-13 (Ubuntu 13.2.0-4ubuntu3) 13.2.0
reproduce (this is a W=1 build): (https://download.01.org/0day-ci/archive/20240518/[email protected]/reproduce)
If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <[email protected]>
| Closes: https://lore.kernel.org/oe-kbuild-all/[email protected]/
All errors (new ones prefixed by >>):
ld: drivers/pci/pci.o: in function `pci_restore_vf_rebar_state':
>> drivers/pci/pci.c:1895:(.text+0x3bff): undefined reference to `__udivdi3'
vim +1895 drivers/pci/pci.c
1869
1870 static void pci_restore_vf_rebar_state(struct pci_dev *pdev)
1871 {
1872 #ifdef CONFIG_PCI_IOV
1873 unsigned int pos, nbars, i;
1874 u32 ctrl;
1875 u16 total;
1876
1877 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_VF_REBAR);
1878 if (!pos)
1879 return;
1880
1881 pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
1882 nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, ctrl);
1883
1884 for (i = 0; i < nbars; i++, pos += 8) {
1885 struct resource *res;
1886 int bar_idx, size;
1887
1888 pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
1889 bar_idx = ctrl & PCI_REBAR_CTRL_BAR_IDX;
1890 total = pdev->sriov->total_VFs;
1891 if (!total)
1892 return;
1893
1894 res = pdev->resource + bar_idx + PCI_IOV_RESOURCES;
> 1895 size = pci_rebar_bytes_to_size(resource_size(res) / total);
1896 ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE;
1897 ctrl |= FIELD_PREP(PCI_REBAR_CTRL_BAR_SIZE, size);
1898 pci_write_config_dword(pdev, pos + PCI_REBAR_CTRL, ctrl);
1899 }
1900 #endif
1901 }
1902
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests/wiki