Similar to regular BAR, drivers can use pci_resize_resource() to resize
an IOV BAR to the desired size provided that is supported by the
hardware, which can be queried using pci_rebar_get_possible_sizes().
This feature is based on the fact that (default VF BAR size) * (supported
VF number) covers all possible resource/address ranges (rounded up to the
power of 2 size). For example, the total size of the resource behind the
BAR is 256GB, the supported maximum VF number is 4, the default VF BAR
size should then be set to 64GB. When the enabled vf_num changes, the VF
BAR size will adjust accordingly as
- For 1 VF, VF BAR size is 256GB
- For 2 VFs, VF BAR size is 128GB
- For 4 VFs, VF BAR size is 64GB
This feature is necessary to accommodate the limited address per PCI port.
Lianjie Shi (1):
PCI: Support VF resizable BAR
drivers/pci/pci.c | 47 ++++++++++++++++++++++++++++++++++-
drivers/pci/setup-res.c | 45 +++++++++++++++++++++++++++------
include/uapi/linux/pci_regs.h | 1 +
3 files changed, 85 insertions(+), 8 deletions(-)
base-commit: cf87f46fd34d6c19283d9625a7822f20d90b64a4
--
2.34.1
Add support for VF resizable BAR PCI extended cap.
Similar to regular BAR, drivers can use pci_resize_resource() to
resize an IOV BAR. For each VF, dev->sriov->barsz of the IOV BAR is
resized, but the total resource size of the IOV resource should not
exceed its original size upon init.
Based on following patch series:
Link: https://lore.kernel.org/lkml/YbqGplTKl5i%2F1%2FkY@rocinante/T/
Signed-off-by: Lianjie Shi <[email protected]>
---
drivers/pci/pci.c | 47 ++++++++++++++++++++++++++++++++++-
drivers/pci/setup-res.c | 45 +++++++++++++++++++++++++++------
include/uapi/linux/pci_regs.h | 1 +
3 files changed, 85 insertions(+), 8 deletions(-)
diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c
index e5f243dd4..12f86e00a 100644
--- a/drivers/pci/pci.c
+++ b/drivers/pci/pci.c
@@ -1867,6 +1867,42 @@ static void pci_restore_rebar_state(struct pci_dev *pdev)
}
}
+static void pci_restore_vf_rebar_state(struct pci_dev *pdev)
+{
+#ifdef CONFIG_PCI_IOV
+ unsigned int pos, nbars, i;
+ u32 ctrl;
+ u16 total;
+
+ pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_VF_REBAR);
+ if (!pos)
+ return;
+
+ pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
+ nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, ctrl);
+
+ for (i = 0; i < nbars; i++, pos += 8) {
+ struct resource *res;
+ int bar_idx, size;
+ u64 tmp;
+
+ pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
+ bar_idx = ctrl & PCI_REBAR_CTRL_BAR_IDX;
+ total = pdev->sriov->total_VFs;
+ if (!total)
+ return;
+
+ res = pdev->resource + bar_idx + PCI_IOV_RESOURCES;
+ tmp = resource_size(res);
+ do_div(tmp, total);
+ size = pci_rebar_bytes_to_size(tmp);
+ ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE;
+ ctrl |= FIELD_PREP(PCI_REBAR_CTRL_BAR_SIZE, size);
+ pci_write_config_dword(pdev, pos + PCI_REBAR_CTRL, ctrl);
+ }
+#endif
+}
+
/**
* pci_restore_state - Restore the saved state of a PCI device
* @dev: PCI device that we're dealing with
@@ -1882,6 +1918,7 @@ void pci_restore_state(struct pci_dev *dev)
pci_restore_ats_state(dev);
pci_restore_vc_state(dev);
pci_restore_rebar_state(dev);
+ pci_restore_vf_rebar_state(dev);
pci_restore_dpc_state(dev);
pci_restore_ptm_state(dev);
@@ -3677,10 +3714,18 @@ void pci_acs_init(struct pci_dev *dev)
*/
static int pci_rebar_find_pos(struct pci_dev *pdev, int bar)
{
+ int cap = PCI_EXT_CAP_ID_REBAR;
unsigned int pos, nbars, i;
u32 ctrl;
- pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_REBAR);
+#ifdef CONFIG_PCI_IOV
+ if (bar >= PCI_IOV_RESOURCES) {
+ cap = PCI_EXT_CAP_ID_VF_REBAR;
+ bar -= PCI_IOV_RESOURCES;
+ }
+#endif
+
+ pos = pci_find_ext_capability(pdev, cap);
if (!pos)
return -ENOTSUPP;
diff --git a/drivers/pci/setup-res.c b/drivers/pci/setup-res.c
index c6d933ddf..d978a2ccf 100644
--- a/drivers/pci/setup-res.c
+++ b/drivers/pci/setup-res.c
@@ -427,13 +427,32 @@ void pci_release_resource(struct pci_dev *dev, int resno)
}
EXPORT_SYMBOL(pci_release_resource);
+static int pci_memory_decoding(struct pci_dev *dev, int resno)
+{
+ u16 cmd;
+
+#ifdef CONFIG_PCI_IOV
+ if (resno >= PCI_IOV_RESOURCES) {
+ pci_read_config_word(dev, dev->sriov->pos + PCI_SRIOV_CTRL, &cmd);
+ if (cmd & PCI_SRIOV_CTRL_MSE)
+ return -EBUSY;
+ else
+ return 0;
+ }
+#endif
+ pci_read_config_word(dev, PCI_COMMAND, &cmd);
+ if (cmd & PCI_COMMAND_MEMORY)
+ return -EBUSY;
+
+ return 0;
+}
+
int pci_resize_resource(struct pci_dev *dev, int resno, int size)
{
struct resource *res = dev->resource + resno;
struct pci_host_bridge *host;
int old, ret;
u32 sizes;
- u16 cmd;
/* Check if we must preserve the firmware's resource assignment */
host = pci_find_host_bridge(dev->bus);
@@ -444,9 +463,9 @@ int pci_resize_resource(struct pci_dev *dev, int resno, int size)
if (!(res->flags & IORESOURCE_UNSET))
return -EBUSY;
- pci_read_config_word(dev, PCI_COMMAND, &cmd);
- if (cmd & PCI_COMMAND_MEMORY)
- return -EBUSY;
+ ret = pci_memory_decoding(dev, resno);
+ if (ret)
+ return ret;
sizes = pci_rebar_get_possible_sizes(dev, resno);
if (!sizes)
@@ -463,19 +482,31 @@ int pci_resize_resource(struct pci_dev *dev, int resno, int size)
if (ret)
return ret;
- res->end = res->start + pci_rebar_size_to_bytes(size) - 1;
+#ifdef CONFIG_PCI_IOV
+ if (resno >= PCI_IOV_RESOURCES)
+ dev->sriov->barsz[resno - PCI_IOV_RESOURCES] = pci_rebar_size_to_bytes(size);
+ else
+#endif
+ res->end = res->start + pci_rebar_size_to_bytes(size) - 1;
/* Check if the new config works by trying to assign everything. */
if (dev->bus->self) {
ret = pci_reassign_bridge_resources(dev->bus->self, res->flags);
- if (ret)
+ if (ret && ret != -ENOENT)
goto error_resize;
}
return 0;
error_resize:
pci_rebar_set_size(dev, resno, old);
- res->end = res->start + pci_rebar_size_to_bytes(old) - 1;
+
+#ifdef CONFIG_PCI_IOV
+ if (resno >= PCI_IOV_RESOURCES)
+ dev->sriov->barsz[resno - PCI_IOV_RESOURCES] = pci_rebar_size_to_bytes(old);
+ else
+#endif
+ res->end = res->start + pci_rebar_size_to_bytes(old) - 1;
+
return ret;
}
EXPORT_SYMBOL(pci_resize_resource);
diff --git a/include/uapi/linux/pci_regs.h b/include/uapi/linux/pci_regs.h
index a39193213..a66b90982 100644
--- a/include/uapi/linux/pci_regs.h
+++ b/include/uapi/linux/pci_regs.h
@@ -738,6 +738,7 @@
#define PCI_EXT_CAP_ID_L1SS 0x1E /* L1 PM Substates */
#define PCI_EXT_CAP_ID_PTM 0x1F /* Precision Time Measurement */
#define PCI_EXT_CAP_ID_DVSEC 0x23 /* Designated Vendor-Specific */
+#define PCI_EXT_CAP_ID_VF_REBAR 0x24 /* VF Resizable BAR */
#define PCI_EXT_CAP_ID_DLF 0x25 /* Data Link Feature */
#define PCI_EXT_CAP_ID_PL_16GT 0x26 /* Physical Layer 16.0 GT/s */
#define PCI_EXT_CAP_ID_PL_32GT 0x2A /* Physical Layer 32.0 GT/s */
--
2.34.1
[AMD Official Use Only - AMD Internal Distribution Only]
Hi, Bjorn and pci driver maintainers
Could you review the VF resizable BAR patch?
Thanks,
HaiJun
-----Original Message-----
From: Lianjie Shi <[email protected]>
Sent: Thursday, May 23, 2024 3:11 PM
To: [email protected]; [email protected]; Bjorn Helgaas <[email protected]>
Cc: Chang, HaiJun <[email protected]>; Jiang, Jerry (SW) <[email protected]>; Zhang, Andy <[email protected]>; Shi, Lianjie <[email protected]>
Subject: [PATCH 1/1][v2] PCI: Support VF resizable BAR
Add support for VF resizable BAR PCI extended cap.
Similar to regular BAR, drivers can use pci_resize_resource() to resize an IOV BAR. For each VF, dev->sriov->barsz of the IOV BAR is resized, but the total resource size of the IOV resource should not exceed its original size upon init.
Based on following patch series:
Link: https://lore.kernel.org/lkml/YbqGplTKl5i%2F1%2FkY@rocinante/T/
Signed-off-by: Lianjie Shi <[email protected]>
---
drivers/pci/pci.c | 47 ++++++++++++++++++++++++++++++++++-
drivers/pci/setup-res.c | 45 +++++++++++++++++++++++++++------
include/uapi/linux/pci_regs.h | 1 +
3 files changed, 85 insertions(+), 8 deletions(-)
diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index e5f243dd4..12f86e00a 100644
--- a/drivers/pci/pci.c
+++ b/drivers/pci/pci.c
@@ -1867,6 +1867,42 @@ static void pci_restore_rebar_state(struct pci_dev *pdev)
}
}
+static void pci_restore_vf_rebar_state(struct pci_dev *pdev) { #ifdef
+CONFIG_PCI_IOV
+ unsigned int pos, nbars, i;
+ u32 ctrl;
+ u16 total;
+
+ pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_VF_REBAR);
+ if (!pos)
+ return;
+
+ pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
+ nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, ctrl);
+
+ for (i = 0; i < nbars; i++, pos += 8) {
+ struct resource *res;
+ int bar_idx, size;
+ u64 tmp;
+
+ pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
+ bar_idx = ctrl & PCI_REBAR_CTRL_BAR_IDX;
+ total = pdev->sriov->total_VFs;
+ if (!total)
+ return;
+
+ res = pdev->resource + bar_idx + PCI_IOV_RESOURCES;
+ tmp = resource_size(res);
+ do_div(tmp, total);
+ size = pci_rebar_bytes_to_size(tmp);
+ ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE;
+ ctrl |= FIELD_PREP(PCI_REBAR_CTRL_BAR_SIZE, size);
+ pci_write_config_dword(pdev, pos + PCI_REBAR_CTRL, ctrl);
+ }
+#endif
+}
+
/**
* pci_restore_state - Restore the saved state of a PCI device
* @dev: PCI device that we're dealing with @@ -1882,6 +1918,7 @@ void pci_restore_state(struct pci_dev *dev)
pci_restore_ats_state(dev);
pci_restore_vc_state(dev);
pci_restore_rebar_state(dev);
+ pci_restore_vf_rebar_state(dev);
pci_restore_dpc_state(dev);
pci_restore_ptm_state(dev);
@@ -3677,10 +3714,18 @@ void pci_acs_init(struct pci_dev *dev)
*/
static int pci_rebar_find_pos(struct pci_dev *pdev, int bar) {
+ int cap = PCI_EXT_CAP_ID_REBAR;
unsigned int pos, nbars, i;
u32 ctrl;
- pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_REBAR);
+#ifdef CONFIG_PCI_IOV
+ if (bar >= PCI_IOV_RESOURCES) {
+ cap = PCI_EXT_CAP_ID_VF_REBAR;
+ bar -= PCI_IOV_RESOURCES;
+ }
+#endif
+
+ pos = pci_find_ext_capability(pdev, cap);
if (!pos)
return -ENOTSUPP;
diff --git a/drivers/pci/setup-res.c b/drivers/pci/setup-res.c index c6d933ddf..d978a2ccf 100644
--- a/drivers/pci/setup-res.c
+++ b/drivers/pci/setup-res.c
@@ -427,13 +427,32 @@ void pci_release_resource(struct pci_dev *dev, int resno) } EXPORT_SYMBOL(pci_release_resource);
+static int pci_memory_decoding(struct pci_dev *dev, int resno) {
+ u16 cmd;
+
+#ifdef CONFIG_PCI_IOV
+ if (resno >= PCI_IOV_RESOURCES) {
+ pci_read_config_word(dev, dev->sriov->pos + PCI_SRIOV_CTRL, &cmd);
+ if (cmd & PCI_SRIOV_CTRL_MSE)
+ return -EBUSY;
+ else
+ return 0;
+ }
+#endif
+ pci_read_config_word(dev, PCI_COMMAND, &cmd);
+ if (cmd & PCI_COMMAND_MEMORY)
+ return -EBUSY;
+
+ return 0;
+}
+
int pci_resize_resource(struct pci_dev *dev, int resno, int size) {
struct resource *res = dev->resource + resno;
struct pci_host_bridge *host;
int old, ret;
u32 sizes;
- u16 cmd;
/* Check if we must preserve the firmware's resource assignment */
host = pci_find_host_bridge(dev->bus); @@ -444,9 +463,9 @@ int pci_resize_resource(struct pci_dev *dev, int resno, int size)
if (!(res->flags & IORESOURCE_UNSET))
return -EBUSY;
- pci_read_config_word(dev, PCI_COMMAND, &cmd);
- if (cmd & PCI_COMMAND_MEMORY)
- return -EBUSY;
+ ret = pci_memory_decoding(dev, resno);
+ if (ret)
+ return ret;
sizes = pci_rebar_get_possible_sizes(dev, resno);
if (!sizes)
@@ -463,19 +482,31 @@ int pci_resize_resource(struct pci_dev *dev, int resno, int size)
if (ret)
return ret;
- res->end = res->start + pci_rebar_size_to_bytes(size) - 1;
+#ifdef CONFIG_PCI_IOV
+ if (resno >= PCI_IOV_RESOURCES)
+ dev->sriov->barsz[resno - PCI_IOV_RESOURCES] = pci_rebar_size_to_bytes(size);
+ else
+#endif
+ res->end = res->start + pci_rebar_size_to_bytes(size) - 1;
/* Check if the new config works by trying to assign everything. */
if (dev->bus->self) {
ret = pci_reassign_bridge_resources(dev->bus->self, res->flags);
- if (ret)
+ if (ret && ret != -ENOENT)
goto error_resize;
}
return 0;
error_resize:
pci_rebar_set_size(dev, resno, old);
- res->end = res->start + pci_rebar_size_to_bytes(old) - 1;
+
+#ifdef CONFIG_PCI_IOV
+ if (resno >= PCI_IOV_RESOURCES)
+ dev->sriov->barsz[resno - PCI_IOV_RESOURCES] = pci_rebar_size_to_bytes(old);
+ else
+#endif
+ res->end = res->start + pci_rebar_size_to_bytes(old) - 1;
+
return ret;
}
EXPORT_SYMBOL(pci_resize_resource);
diff --git a/include/uapi/linux/pci_regs.h b/include/uapi/linux/pci_regs.h index a39193213..a66b90982 100644
--- a/include/uapi/linux/pci_regs.h
+++ b/include/uapi/linux/pci_regs.h
@@ -738,6 +738,7 @@
#define PCI_EXT_CAP_ID_L1SS 0x1E /* L1 PM Substates */
#define PCI_EXT_CAP_ID_PTM 0x1F /* Precision Time Measurement */
#define PCI_EXT_CAP_ID_DVSEC 0x23 /* Designated Vendor-Specific */
+#define PCI_EXT_CAP_ID_VF_REBAR 0x24 /* VF Resizable BAR */
#define PCI_EXT_CAP_ID_DLF 0x25 /* Data Link Feature */
#define PCI_EXT_CAP_ID_PL_16GT 0x26 /* Physical Layer 16.0 GT/s */
#define PCI_EXT_CAP_ID_PL_32GT 0x2A /* Physical Layer 32.0 GT/s */
--
2.34.1
[AMD Official Use Only - AMD Internal Distribution Only]
+ Christian to review the VF resizable BAR patch.
-----Original Message-----
From: Chang, HaiJun
Sent: Friday, May 31, 2024 12:26 PM
To: Lianjie Shi <[email protected]>; [email protected]; [email protected]; Bjorn Helgaas <[email protected]>
Cc: Jiang, Jerry (SW) <[email protected]>; Zhang, Andy <[email protected]>; Shi, Lianjie <[email protected]>
Subject: RE: [PATCH 1/1][v2] PCI: Support VF resizable BAR
Hi, Bjorn and pci driver maintainers
Could you review the VF resizable BAR patch?
Thanks,
HaiJun
-----Original Message-----
From: Lianjie Shi <[email protected]>
Sent: Thursday, May 23, 2024 3:11 PM
To: [email protected]; [email protected]; Bjorn Helgaas <[email protected]>
Cc: Chang, HaiJun <[email protected]>; Jiang, Jerry (SW) <[email protected]>; Zhang, Andy <[email protected]>; Shi, Lianjie <[email protected]>
Subject: [PATCH 1/1][v2] PCI: Support VF resizable BAR
Add support for VF resizable BAR PCI extended cap.
Similar to regular BAR, drivers can use pci_resize_resource() to resize an IOV BAR. For each VF, dev->sriov->barsz of the IOV BAR is resized, but the total resource size of the IOV resource should not exceed its original size upon init.
Based on following patch series:
Link: https://lore.kernel.org/lkml/YbqGplTKl5i%2F1%2FkY@rocinante/T/
Signed-off-by: Lianjie Shi <[email protected]>
---
drivers/pci/pci.c | 47 ++++++++++++++++++++++++++++++++++-
drivers/pci/setup-res.c | 45 +++++++++++++++++++++++++++------
include/uapi/linux/pci_regs.h | 1 +
3 files changed, 85 insertions(+), 8 deletions(-)
diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index e5f243dd4..12f86e00a 100644
--- a/drivers/pci/pci.c
+++ b/drivers/pci/pci.c
@@ -1867,6 +1867,42 @@ static void pci_restore_rebar_state(struct pci_dev *pdev)
}
}
+static void pci_restore_vf_rebar_state(struct pci_dev *pdev) { #ifdef
+CONFIG_PCI_IOV
+ unsigned int pos, nbars, i;
+ u32 ctrl;
+ u16 total;
+
+ pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_VF_REBAR);
+ if (!pos)
+ return;
+
+ pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
+ nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, ctrl);
+
+ for (i = 0; i < nbars; i++, pos += 8) {
+ struct resource *res;
+ int bar_idx, size;
+ u64 tmp;
+
+ pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
+ bar_idx = ctrl & PCI_REBAR_CTRL_BAR_IDX;
+ total = pdev->sriov->total_VFs;
+ if (!total)
+ return;
+
+ res = pdev->resource + bar_idx + PCI_IOV_RESOURCES;
+ tmp = resource_size(res);
+ do_div(tmp, total);
+ size = pci_rebar_bytes_to_size(tmp);
+ ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE;
+ ctrl |= FIELD_PREP(PCI_REBAR_CTRL_BAR_SIZE, size);
+ pci_write_config_dword(pdev, pos + PCI_REBAR_CTRL, ctrl);
+ }
+#endif
+}
+
/**
* pci_restore_state - Restore the saved state of a PCI device
* @dev: PCI device that we're dealing with @@ -1882,6 +1918,7 @@ void pci_restore_state(struct pci_dev *dev)
pci_restore_ats_state(dev);
pci_restore_vc_state(dev);
pci_restore_rebar_state(dev);
+ pci_restore_vf_rebar_state(dev);
pci_restore_dpc_state(dev);
pci_restore_ptm_state(dev);
@@ -3677,10 +3714,18 @@ void pci_acs_init(struct pci_dev *dev)
*/
static int pci_rebar_find_pos(struct pci_dev *pdev, int bar) {
+ int cap = PCI_EXT_CAP_ID_REBAR;
unsigned int pos, nbars, i;
u32 ctrl;
- pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_REBAR);
+#ifdef CONFIG_PCI_IOV
+ if (bar >= PCI_IOV_RESOURCES) {
+ cap = PCI_EXT_CAP_ID_VF_REBAR;
+ bar -= PCI_IOV_RESOURCES;
+ }
+#endif
+
+ pos = pci_find_ext_capability(pdev, cap);
if (!pos)
return -ENOTSUPP;
diff --git a/drivers/pci/setup-res.c b/drivers/pci/setup-res.c index c6d933ddf..d978a2ccf 100644
--- a/drivers/pci/setup-res.c
+++ b/drivers/pci/setup-res.c
@@ -427,13 +427,32 @@ void pci_release_resource(struct pci_dev *dev, int resno) } EXPORT_SYMBOL(pci_release_resource);
+static int pci_memory_decoding(struct pci_dev *dev, int resno) {
+ u16 cmd;
+
+#ifdef CONFIG_PCI_IOV
+ if (resno >= PCI_IOV_RESOURCES) {
+ pci_read_config_word(dev, dev->sriov->pos + PCI_SRIOV_CTRL, &cmd);
+ if (cmd & PCI_SRIOV_CTRL_MSE)
+ return -EBUSY;
+ else
+ return 0;
+ }
+#endif
+ pci_read_config_word(dev, PCI_COMMAND, &cmd);
+ if (cmd & PCI_COMMAND_MEMORY)
+ return -EBUSY;
+
+ return 0;
+}
+
int pci_resize_resource(struct pci_dev *dev, int resno, int size) {
struct resource *res = dev->resource + resno;
struct pci_host_bridge *host;
int old, ret;
u32 sizes;
- u16 cmd;
/* Check if we must preserve the firmware's resource assignment */
host = pci_find_host_bridge(dev->bus); @@ -444,9 +463,9 @@ int pci_resize_resource(struct pci_dev *dev, int resno, int size)
if (!(res->flags & IORESOURCE_UNSET))
return -EBUSY;
- pci_read_config_word(dev, PCI_COMMAND, &cmd);
- if (cmd & PCI_COMMAND_MEMORY)
- return -EBUSY;
+ ret = pci_memory_decoding(dev, resno);
+ if (ret)
+ return ret;
sizes = pci_rebar_get_possible_sizes(dev, resno);
if (!sizes)
@@ -463,19 +482,31 @@ int pci_resize_resource(struct pci_dev *dev, int resno, int size)
if (ret)
return ret;
- res->end = res->start + pci_rebar_size_to_bytes(size) - 1;
+#ifdef CONFIG_PCI_IOV
+ if (resno >= PCI_IOV_RESOURCES)
+ dev->sriov->barsz[resno - PCI_IOV_RESOURCES] = pci_rebar_size_to_bytes(size);
+ else
+#endif
+ res->end = res->start + pci_rebar_size_to_bytes(size) - 1;
/* Check if the new config works by trying to assign everything. */
if (dev->bus->self) {
ret = pci_reassign_bridge_resources(dev->bus->self, res->flags);
- if (ret)
+ if (ret && ret != -ENOENT)
goto error_resize;
}
return 0;
error_resize:
pci_rebar_set_size(dev, resno, old);
- res->end = res->start + pci_rebar_size_to_bytes(old) - 1;
+
+#ifdef CONFIG_PCI_IOV
+ if (resno >= PCI_IOV_RESOURCES)
+ dev->sriov->barsz[resno - PCI_IOV_RESOURCES] = pci_rebar_size_to_bytes(old);
+ else
+#endif
+ res->end = res->start + pci_rebar_size_to_bytes(old) - 1;
+
return ret;
}
EXPORT_SYMBOL(pci_resize_resource);
diff --git a/include/uapi/linux/pci_regs.h b/include/uapi/linux/pci_regs.h index a39193213..a66b90982 100644
--- a/include/uapi/linux/pci_regs.h
+++ b/include/uapi/linux/pci_regs.h
@@ -738,6 +738,7 @@
#define PCI_EXT_CAP_ID_L1SS 0x1E /* L1 PM Substates */
#define PCI_EXT_CAP_ID_PTM 0x1F /* Precision Time Measurement */
#define PCI_EXT_CAP_ID_DVSEC 0x23 /* Designated Vendor-Specific */
+#define PCI_EXT_CAP_ID_VF_REBAR 0x24 /* VF Resizable BAR */
#define PCI_EXT_CAP_ID_DLF 0x25 /* Data Link Feature */
#define PCI_EXT_CAP_ID_PL_16GT 0x26 /* Physical Layer 16.0 GT/s */
#define PCI_EXT_CAP_ID_PL_32GT 0x2A /* Physical Layer 32.0 GT/s */
--
2.34.1
I need the original set of patches, e.g. directly use git send-email
with my mail address.
Forwarded and mangled multiple times the patches are not applicable to
any tree any more.
Regards,
Christian.
Am 07.06.24 um 10:42 schrieb Chang, HaiJun:
> [AMD Official Use Only - AMD Internal Distribution Only]
>
> + Christian to review the VF resizable BAR patch.
>
> -----Original Message-----
> From: Chang, HaiJun
> Sent: Friday, May 31, 2024 12:26 PM
> To: Lianjie Shi <[email protected]>; [email protected]; [email protected]; Bjorn Helgaas <[email protected]>
> Cc: Jiang, Jerry (SW) <[email protected]>; Zhang, Andy <[email protected]>; Shi, Lianjie <[email protected]>
> Subject: RE: [PATCH 1/1][v2] PCI: Support VF resizable BAR
>
> Hi, Bjorn and pci driver maintainers
>
> Could you review the VF resizable BAR patch?
>
> Thanks,
> HaiJun
>
> -----Original Message-----
> From: Lianjie Shi <[email protected]>
> Sent: Thursday, May 23, 2024 3:11 PM
> To: [email protected]; [email protected]; Bjorn Helgaas <[email protected]>
> Cc: Chang, HaiJun <[email protected]>; Jiang, Jerry (SW) <[email protected]>; Zhang, Andy <[email protected]>; Shi, Lianjie <[email protected]>
> Subject: [PATCH 1/1][v2] PCI: Support VF resizable BAR
>
> Add support for VF resizable BAR PCI extended cap.
> Similar to regular BAR, drivers can use pci_resize_resource() to resize an IOV BAR. For each VF, dev->sriov->barsz of the IOV BAR is resized, but the total resource size of the IOV resource should not exceed its original size upon init.
>
> Based on following patch series:
> Link: https://lore.kernel.org/lkml/YbqGplTKl5i%2F1%2FkY@rocinante/T/
>
> Signed-off-by: Lianjie Shi <[email protected]>
> ---
> drivers/pci/pci.c | 47 ++++++++++++++++++++++++++++++++++-
> drivers/pci/setup-res.c | 45 +++++++++++++++++++++++++++------
> include/uapi/linux/pci_regs.h | 1 +
> 3 files changed, 85 insertions(+), 8 deletions(-)
>
> diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index e5f243dd4..12f86e00a 100644
> --- a/drivers/pci/pci.c
> +++ b/drivers/pci/pci.c
> @@ -1867,6 +1867,42 @@ static void pci_restore_rebar_state(struct pci_dev *pdev)
> }
> }
>
> +static void pci_restore_vf_rebar_state(struct pci_dev *pdev) { #ifdef
> +CONFIG_PCI_IOV
> + unsigned int pos, nbars, i;
> + u32 ctrl;
> + u16 total;
> +
> + pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_VF_REBAR);
> + if (!pos)
> + return;
> +
> + pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
> + nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, ctrl);
> +
> + for (i = 0; i < nbars; i++, pos += 8) {
> + struct resource *res;
> + int bar_idx, size;
> + u64 tmp;
> +
> + pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
> + bar_idx = ctrl & PCI_REBAR_CTRL_BAR_IDX;
> + total = pdev->sriov->total_VFs;
> + if (!total)
> + return;
> +
> + res = pdev->resource + bar_idx + PCI_IOV_RESOURCES;
> + tmp = resource_size(res);
> + do_div(tmp, total);
> + size = pci_rebar_bytes_to_size(tmp);
> + ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE;
> + ctrl |= FIELD_PREP(PCI_REBAR_CTRL_BAR_SIZE, size);
> + pci_write_config_dword(pdev, pos + PCI_REBAR_CTRL, ctrl);
> + }
> +#endif
> +}
> +
> /**
> * pci_restore_state - Restore the saved state of a PCI device
> * @dev: PCI device that we're dealing with @@ -1882,6 +1918,7 @@ void pci_restore_state(struct pci_dev *dev)
> pci_restore_ats_state(dev);
> pci_restore_vc_state(dev);
> pci_restore_rebar_state(dev);
> + pci_restore_vf_rebar_state(dev);
> pci_restore_dpc_state(dev);
> pci_restore_ptm_state(dev);
>
> @@ -3677,10 +3714,18 @@ void pci_acs_init(struct pci_dev *dev)
> */
> static int pci_rebar_find_pos(struct pci_dev *pdev, int bar) {
> + int cap = PCI_EXT_CAP_ID_REBAR;
> unsigned int pos, nbars, i;
> u32 ctrl;
>
> - pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_REBAR);
> +#ifdef CONFIG_PCI_IOV
> + if (bar >= PCI_IOV_RESOURCES) {
> + cap = PCI_EXT_CAP_ID_VF_REBAR;
> + bar -= PCI_IOV_RESOURCES;
> + }
> +#endif
> +
> + pos = pci_find_ext_capability(pdev, cap);
> if (!pos)
> return -ENOTSUPP;
>
> diff --git a/drivers/pci/setup-res.c b/drivers/pci/setup-res.c index c6d933ddf..d978a2ccf 100644
> --- a/drivers/pci/setup-res.c
> +++ b/drivers/pci/setup-res.c
> @@ -427,13 +427,32 @@ void pci_release_resource(struct pci_dev *dev, int resno) } EXPORT_SYMBOL(pci_release_resource);
>
> +static int pci_memory_decoding(struct pci_dev *dev, int resno) {
> + u16 cmd;
> +
> +#ifdef CONFIG_PCI_IOV
> + if (resno >= PCI_IOV_RESOURCES) {
> + pci_read_config_word(dev, dev->sriov->pos + PCI_SRIOV_CTRL, &cmd);
> + if (cmd & PCI_SRIOV_CTRL_MSE)
> + return -EBUSY;
> + else
> + return 0;
> + }
> +#endif
> + pci_read_config_word(dev, PCI_COMMAND, &cmd);
> + if (cmd & PCI_COMMAND_MEMORY)
> + return -EBUSY;
> +
> + return 0;
> +}
> +
> int pci_resize_resource(struct pci_dev *dev, int resno, int size) {
> struct resource *res = dev->resource + resno;
> struct pci_host_bridge *host;
> int old, ret;
> u32 sizes;
> - u16 cmd;
>
> /* Check if we must preserve the firmware's resource assignment */
> host = pci_find_host_bridge(dev->bus); @@ -444,9 +463,9 @@ int pci_resize_resource(struct pci_dev *dev, int resno, int size)
> if (!(res->flags & IORESOURCE_UNSET))
> return -EBUSY;
>
> - pci_read_config_word(dev, PCI_COMMAND, &cmd);
> - if (cmd & PCI_COMMAND_MEMORY)
> - return -EBUSY;
> + ret = pci_memory_decoding(dev, resno);
> + if (ret)
> + return ret;
>
> sizes = pci_rebar_get_possible_sizes(dev, resno);
> if (!sizes)
> @@ -463,19 +482,31 @@ int pci_resize_resource(struct pci_dev *dev, int resno, int size)
> if (ret)
> return ret;
>
> - res->end = res->start + pci_rebar_size_to_bytes(size) - 1;
> +#ifdef CONFIG_PCI_IOV
> + if (resno >= PCI_IOV_RESOURCES)
> + dev->sriov->barsz[resno - PCI_IOV_RESOURCES] = pci_rebar_size_to_bytes(size);
> + else
> +#endif
> + res->end = res->start + pci_rebar_size_to_bytes(size) - 1;
>
> /* Check if the new config works by trying to assign everything. */
> if (dev->bus->self) {
> ret = pci_reassign_bridge_resources(dev->bus->self, res->flags);
> - if (ret)
> + if (ret && ret != -ENOENT)
> goto error_resize;
> }
> return 0;
>
> error_resize:
> pci_rebar_set_size(dev, resno, old);
> - res->end = res->start + pci_rebar_size_to_bytes(old) - 1;
> +
> +#ifdef CONFIG_PCI_IOV
> + if (resno >= PCI_IOV_RESOURCES)
> + dev->sriov->barsz[resno - PCI_IOV_RESOURCES] = pci_rebar_size_to_bytes(old);
> + else
> +#endif
> + res->end = res->start + pci_rebar_size_to_bytes(old) - 1;
> +
> return ret;
> }
> EXPORT_SYMBOL(pci_resize_resource);
> diff --git a/include/uapi/linux/pci_regs.h b/include/uapi/linux/pci_regs.h index a39193213..a66b90982 100644
> --- a/include/uapi/linux/pci_regs.h
> +++ b/include/uapi/linux/pci_regs.h
> @@ -738,6 +738,7 @@
> #define PCI_EXT_CAP_ID_L1SS 0x1E /* L1 PM Substates */
> #define PCI_EXT_CAP_ID_PTM 0x1F /* Precision Time Measurement */
> #define PCI_EXT_CAP_ID_DVSEC 0x23 /* Designated Vendor-Specific */
> +#define PCI_EXT_CAP_ID_VF_REBAR 0x24 /* VF Resizable BAR */
> #define PCI_EXT_CAP_ID_DLF 0x25 /* Data Link Feature */
> #define PCI_EXT_CAP_ID_PL_16GT 0x26 /* Physical Layer 16.0 GT/s */
> #define PCI_EXT_CAP_ID_PL_32GT 0x2A /* Physical Layer 32.0 GT/s */
> --
> 2.34.1
>
On Fri, Jun 07, 2024 at 01:37:21PM +0200, Christian König wrote:
> I need the original set of patches, e.g. directly use git send-email with my
> mail address.
This worked for me:
$ git checkout -b wip/2406-lianjie-vf-rebar-v2 v6.10-rc1
$ b4 am https://lore.kernel.org/r/[email protected]
$ git am ./v2_20240523_lianjie_shi_pci_vf_resizable_bar.mbx
> Am 07.06.24 um 10:42 schrieb Chang, HaiJun:
> > [AMD Official Use Only - AMD Internal Distribution Only]
> >
> > + Christian to review the VF resizable BAR patch.
> >
> > -----Original Message-----
> > From: Chang, HaiJun
> > Sent: Friday, May 31, 2024 12:26 PM
> > To: Lianjie Shi <[email protected]>; [email protected]; [email protected]; Bjorn Helgaas <[email protected]>
> > Cc: Jiang, Jerry (SW) <[email protected]>; Zhang, Andy <[email protected]>; Shi, Lianjie <[email protected]>
> > Subject: RE: [PATCH 1/1][v2] PCI: Support VF resizable BAR
> >
> > Hi, Bjorn and pci driver maintainers
> >
> > Could you review the VF resizable BAR patch?
> >
> > Thanks,
> > HaiJun
> >
> > -----Original Message-----
> > From: Lianjie Shi <[email protected]>
> > Sent: Thursday, May 23, 2024 3:11 PM
> > To: [email protected]; [email protected]; Bjorn Helgaas <[email protected]>
> > Cc: Chang, HaiJun <[email protected]>; Jiang, Jerry (SW) <[email protected]>; Zhang, Andy <[email protected]>; Shi, Lianjie <[email protected]>
> > Subject: [PATCH 1/1][v2] PCI: Support VF resizable BAR
> >
> > Add support for VF resizable BAR PCI extended cap.
> > Similar to regular BAR, drivers can use pci_resize_resource() to resize an IOV BAR. For each VF, dev->sriov->barsz of the IOV BAR is resized, but the total resource size of the IOV resource should not exceed its original size upon init.
> >
> > Based on following patch series:
> > Link: https://lore.kernel.org/lkml/YbqGplTKl5i%2F1%2FkY@rocinante/T/
> >
> > Signed-off-by: Lianjie Shi <[email protected]>
> > ---
> > drivers/pci/pci.c | 47 ++++++++++++++++++++++++++++++++++-
> > drivers/pci/setup-res.c | 45 +++++++++++++++++++++++++++------
> > include/uapi/linux/pci_regs.h | 1 +
> > 3 files changed, 85 insertions(+), 8 deletions(-)
> >
> > diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index e5f243dd4..12f86e00a 100644
> > --- a/drivers/pci/pci.c
> > +++ b/drivers/pci/pci.c
> > @@ -1867,6 +1867,42 @@ static void pci_restore_rebar_state(struct pci_dev *pdev)
> > }
> > }
> >
> > +static void pci_restore_vf_rebar_state(struct pci_dev *pdev) { #ifdef
> > +CONFIG_PCI_IOV
> > + unsigned int pos, nbars, i;
> > + u32 ctrl;
> > + u16 total;
> > +
> > + pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_VF_REBAR);
> > + if (!pos)
> > + return;
> > +
> > + pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
> > + nbars = FIELD_GET(PCI_REBAR_CTRL_NBAR_MASK, ctrl);
> > +
> > + for (i = 0; i < nbars; i++, pos += 8) {
> > + struct resource *res;
> > + int bar_idx, size;
> > + u64 tmp;
> > +
> > + pci_read_config_dword(pdev, pos + PCI_REBAR_CTRL, &ctrl);
> > + bar_idx = ctrl & PCI_REBAR_CTRL_BAR_IDX;
> > + total = pdev->sriov->total_VFs;
> > + if (!total)
> > + return;
> > +
> > + res = pdev->resource + bar_idx + PCI_IOV_RESOURCES;
> > + tmp = resource_size(res);
> > + do_div(tmp, total);
> > + size = pci_rebar_bytes_to_size(tmp);
> > + ctrl &= ~PCI_REBAR_CTRL_BAR_SIZE;
> > + ctrl |= FIELD_PREP(PCI_REBAR_CTRL_BAR_SIZE, size);
> > + pci_write_config_dword(pdev, pos + PCI_REBAR_CTRL, ctrl);
> > + }
> > +#endif
> > +}
> > +
> > /**
> > * pci_restore_state - Restore the saved state of a PCI device
> > * @dev: PCI device that we're dealing with @@ -1882,6 +1918,7 @@ void pci_restore_state(struct pci_dev *dev)
> > pci_restore_ats_state(dev);
> > pci_restore_vc_state(dev);
> > pci_restore_rebar_state(dev);
> > + pci_restore_vf_rebar_state(dev);
> > pci_restore_dpc_state(dev);
> > pci_restore_ptm_state(dev);
> >
> > @@ -3677,10 +3714,18 @@ void pci_acs_init(struct pci_dev *dev)
> > */
> > static int pci_rebar_find_pos(struct pci_dev *pdev, int bar) {
> > + int cap = PCI_EXT_CAP_ID_REBAR;
> > unsigned int pos, nbars, i;
> > u32 ctrl;
> >
> > - pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_REBAR);
> > +#ifdef CONFIG_PCI_IOV
> > + if (bar >= PCI_IOV_RESOURCES) {
> > + cap = PCI_EXT_CAP_ID_VF_REBAR;
> > + bar -= PCI_IOV_RESOURCES;
> > + }
> > +#endif
> > +
> > + pos = pci_find_ext_capability(pdev, cap);
> > if (!pos)
> > return -ENOTSUPP;
> >
> > diff --git a/drivers/pci/setup-res.c b/drivers/pci/setup-res.c index c6d933ddf..d978a2ccf 100644
> > --- a/drivers/pci/setup-res.c
> > +++ b/drivers/pci/setup-res.c
> > @@ -427,13 +427,32 @@ void pci_release_resource(struct pci_dev *dev, int resno) } EXPORT_SYMBOL(pci_release_resource);
> >
> > +static int pci_memory_decoding(struct pci_dev *dev, int resno) {
> > + u16 cmd;
> > +
> > +#ifdef CONFIG_PCI_IOV
> > + if (resno >= PCI_IOV_RESOURCES) {
> > + pci_read_config_word(dev, dev->sriov->pos + PCI_SRIOV_CTRL, &cmd);
> > + if (cmd & PCI_SRIOV_CTRL_MSE)
> > + return -EBUSY;
> > + else
> > + return 0;
> > + }
> > +#endif
> > + pci_read_config_word(dev, PCI_COMMAND, &cmd);
> > + if (cmd & PCI_COMMAND_MEMORY)
> > + return -EBUSY;
> > +
> > + return 0;
> > +}
> > +
> > int pci_resize_resource(struct pci_dev *dev, int resno, int size) {
> > struct resource *res = dev->resource + resno;
> > struct pci_host_bridge *host;
> > int old, ret;
> > u32 sizes;
> > - u16 cmd;
> >
> > /* Check if we must preserve the firmware's resource assignment */
> > host = pci_find_host_bridge(dev->bus); @@ -444,9 +463,9 @@ int pci_resize_resource(struct pci_dev *dev, int resno, int size)
> > if (!(res->flags & IORESOURCE_UNSET))
> > return -EBUSY;
> >
> > - pci_read_config_word(dev, PCI_COMMAND, &cmd);
> > - if (cmd & PCI_COMMAND_MEMORY)
> > - return -EBUSY;
> > + ret = pci_memory_decoding(dev, resno);
> > + if (ret)
> > + return ret;
> >
> > sizes = pci_rebar_get_possible_sizes(dev, resno);
> > if (!sizes)
> > @@ -463,19 +482,31 @@ int pci_resize_resource(struct pci_dev *dev, int resno, int size)
> > if (ret)
> > return ret;
> >
> > - res->end = res->start + pci_rebar_size_to_bytes(size) - 1;
> > +#ifdef CONFIG_PCI_IOV
> > + if (resno >= PCI_IOV_RESOURCES)
> > + dev->sriov->barsz[resno - PCI_IOV_RESOURCES] = pci_rebar_size_to_bytes(size);
> > + else
> > +#endif
> > + res->end = res->start + pci_rebar_size_to_bytes(size) - 1;
> >
> > /* Check if the new config works by trying to assign everything. */
> > if (dev->bus->self) {
> > ret = pci_reassign_bridge_resources(dev->bus->self, res->flags);
> > - if (ret)
> > + if (ret && ret != -ENOENT)
> > goto error_resize;
> > }
> > return 0;
> >
> > error_resize:
> > pci_rebar_set_size(dev, resno, old);
> > - res->end = res->start + pci_rebar_size_to_bytes(old) - 1;
> > +
> > +#ifdef CONFIG_PCI_IOV
> > + if (resno >= PCI_IOV_RESOURCES)
> > + dev->sriov->barsz[resno - PCI_IOV_RESOURCES] = pci_rebar_size_to_bytes(old);
> > + else
> > +#endif
> > + res->end = res->start + pci_rebar_size_to_bytes(old) - 1;
> > +
> > return ret;
> > }
> > EXPORT_SYMBOL(pci_resize_resource);
> > diff --git a/include/uapi/linux/pci_regs.h b/include/uapi/linux/pci_regs.h index a39193213..a66b90982 100644
> > --- a/include/uapi/linux/pci_regs.h
> > +++ b/include/uapi/linux/pci_regs.h
> > @@ -738,6 +738,7 @@
> > #define PCI_EXT_CAP_ID_L1SS 0x1E /* L1 PM Substates */
> > #define PCI_EXT_CAP_ID_PTM 0x1F /* Precision Time Measurement */
> > #define PCI_EXT_CAP_ID_DVSEC 0x23 /* Designated Vendor-Specific */
> > +#define PCI_EXT_CAP_ID_VF_REBAR 0x24 /* VF Resizable BAR */
> > #define PCI_EXT_CAP_ID_DLF 0x25 /* Data Link Feature */
> > #define PCI_EXT_CAP_ID_PL_16GT 0x26 /* Physical Layer 16.0 GT/s */
> > #define PCI_EXT_CAP_ID_PL_32GT 0x2A /* Physical Layer 32.0 GT/s */
> > --
> > 2.34.1
> >
>