Patch series
*) Adds support to add virtual functions to enable endpoint controller
which supports SR-IOV capability
*) Add support in Cadence endpoint driver to configure virtual functions
*) Enable pci_endpoint_test driver to create pci_device for virtual
functions
v1 of the patch series can be found at [1]
v2 of the patch series can be found at [2]
v3 of the patch series can be found at [3]
v4 of the patch series can be found at [4]
v5 of the patch series can be found at [5]
v6 of the patch series can be found at [6]
v7 of the patch series can be found at [7]
Here both physical functions and virtual functions use the same
pci_endpoint_test driver and existing pcitest utility can be used
to test virtual functions as well.
Changes from v7:
1) Added conditional operator to consicely write code to configure BAR
(Added a new patch to first simplify configuring BAR for physical
function)
2) Return error if virtual function number is > 1 for configuring config
space header
Changes from v6:
*) Rebased to 5.14-rc4
Changes from v5:
*) Rebased to 5.13-rc1
Changes from v4:
*) Added a fix in Cadence driver which was overwriting BAR configuration
of physical function.
*) Didn't include Tom's Acked-by since Cadence driver is modified in
this revision.
Changes from v3:
*) Fixed Rob's comment and added his Reviewed-by as suggested by him.
Changes from v2:
*) Fixed DT binding documentation comment by Rob
*) Fixed the error check in pci-epc-core.c
Changes from v1:
*) Re-based and Re-worked to latest kernel 5.10.0-rc2+ (now has generic
binding for EP)
[1] -> http://lore.kernel.org/r/[email protected]
[2] -> http://lore.kernel.org/r/[email protected]
[3] -> https://lore.kernel.org/r/[email protected]
[4] -> http://lore.kernel.org/r/[email protected]
[5] -> https://lore.kernel.org/r/[email protected]
[6] -> http://lore.kernel.org/r/[email protected]
[7] -> https://lore.kernel.org/r/[email protected]
Kishon Vijay Abraham I (8):
dt-bindings: PCI: pci-ep: Add binding to specify virtual function
PCI: endpoint: Add support to add virtual function in endpoint core
PCI: endpoint: Add support to link a physical function to a virtual
function
PCI: endpoint: Add virtual function number in pci_epc ops
PCI: cadence: Simplify code to get register base address for
configuring BAR
PCI: cadence: Add support to configure virtual functions
misc: pci_endpoint_test: Populate sriov_configure ops to configure
SR-IOV device
Documentation: PCI: endpoint/pci-endpoint-cfs: Guide to use SR-IOV
.../PCI/endpoint/pci-endpoint-cfs.rst | 12 +-
.../devicetree/bindings/pci/pci-ep.yaml | 7 +
drivers/misc/pci_endpoint_test.c | 1 +
.../pci/controller/cadence/pcie-cadence-ep.c | 194 +++++++++++++-----
drivers/pci/controller/cadence/pcie-cadence.h | 11 +
.../pci/controller/dwc/pcie-designware-ep.c | 36 ++--
drivers/pci/controller/pcie-rcar-ep.c | 19 +-
drivers/pci/controller/pcie-rockchip-ep.c | 18 +-
drivers/pci/endpoint/functions/pci-epf-ntb.c | 89 ++++----
drivers/pci/endpoint/functions/pci-epf-test.c | 74 ++++---
drivers/pci/endpoint/pci-ep-cfs.c | 24 +++
drivers/pci/endpoint/pci-epc-core.c | 134 ++++++++----
drivers/pci/endpoint/pci-epf-core.c | 144 ++++++++++++-
include/linux/pci-epc.h | 57 ++---
include/linux/pci-epf.h | 16 +-
15 files changed, 611 insertions(+), 225 deletions(-)
--
2.17.1
Populate sriov_configure ops with pci_sriov_configure_simple to
configure SR-IOV device.
Signed-off-by: Kishon Vijay Abraham I <[email protected]>
---
drivers/misc/pci_endpoint_test.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/drivers/misc/pci_endpoint_test.c b/drivers/misc/pci_endpoint_test.c
index 1b2868ca4f2a..c7ee34013485 100644
--- a/drivers/misc/pci_endpoint_test.c
+++ b/drivers/misc/pci_endpoint_test.c
@@ -978,6 +978,7 @@ static struct pci_driver pci_endpoint_test_driver = {
.id_table = pci_endpoint_test_tbl,
.probe = pci_endpoint_test_probe,
.remove = pci_endpoint_test_remove,
+ .sriov_configure = pci_sriov_configure_simple,
};
module_pci_driver(pci_endpoint_test_driver);
--
2.17.1
No functional change. Simplify code to get register base address for
configuring PCI BAR.
Signed-off-by: Kishon Vijay Abraham I <[email protected]>
---
.../pci/controller/cadence/pcie-cadence-ep.c | 18 ++++--------------
drivers/pci/controller/cadence/pcie-cadence.h | 2 ++
2 files changed, 6 insertions(+), 14 deletions(-)
diff --git a/drivers/pci/controller/cadence/pcie-cadence-ep.c b/drivers/pci/controller/cadence/pcie-cadence-ep.c
index 912a15be8bfd..f337f0842400 100644
--- a/drivers/pci/controller/cadence/pcie-cadence-ep.c
+++ b/drivers/pci/controller/cadence/pcie-cadence-ep.c
@@ -97,13 +97,8 @@ static int cdns_pcie_ep_set_bar(struct pci_epc *epc, u8 fn, u8 vfn,
cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn, bar),
addr1);
- if (bar < BAR_4) {
- reg = CDNS_PCIE_LM_EP_FUNC_BAR_CFG0(fn);
- b = bar;
- } else {
- reg = CDNS_PCIE_LM_EP_FUNC_BAR_CFG1(fn);
- b = bar - BAR_4;
- }
+ reg = CDNS_PCIE_LM_EP_FUNC_BAR_CFG(bar, fn);
+ b = (bar < BAR_4) ? bar : bar - BAR_4;
cfg = cdns_pcie_readl(pcie, reg);
cfg &= ~(CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) |
@@ -126,13 +121,8 @@ static void cdns_pcie_ep_clear_bar(struct pci_epc *epc, u8 fn, u8 vfn,
enum pci_barno bar = epf_bar->barno;
u32 reg, cfg, b, ctrl;
- if (bar < BAR_4) {
- reg = CDNS_PCIE_LM_EP_FUNC_BAR_CFG0(fn);
- b = bar;
- } else {
- reg = CDNS_PCIE_LM_EP_FUNC_BAR_CFG1(fn);
- b = bar - BAR_4;
- }
+ reg = CDNS_PCIE_LM_EP_FUNC_BAR_CFG(bar, fn);
+ b = (bar < BAR_4) ? bar : bar - BAR_4;
ctrl = CDNS_PCIE_LM_BAR_CFG_CTRL_DISABLED;
cfg = cdns_pcie_readl(pcie, reg);
diff --git a/drivers/pci/controller/cadence/pcie-cadence.h b/drivers/pci/controller/cadence/pcie-cadence.h
index 30db2d68c17a..d5b1fcf2c39d 100644
--- a/drivers/pci/controller/cadence/pcie-cadence.h
+++ b/drivers/pci/controller/cadence/pcie-cadence.h
@@ -46,6 +46,8 @@
#define CDNS_PCIE_LM_EP_ID_BUS_SHIFT 8
/* Endpoint Function f BAR b Configuration Registers */
+#define CDNS_PCIE_LM_EP_FUNC_BAR_CFG(bar, fn) \
+ (((bar) < 4) ? CDNS_PCIE_LM_EP_FUNC_BAR_CFG0(fn) : CDNS_PCIE_LM_EP_FUNC_BAR_CFG1(fn))
#define CDNS_PCIE_LM_EP_FUNC_BAR_CFG0(fn) \
(CDNS_PCIE_LM_BASE + 0x0240 + (fn) * 0x0008)
#define CDNS_PCIE_LM_EP_FUNC_BAR_CFG1(fn) \
--
2.17.1
Add Documentation to help users use PCI endpoint to create virtual
functions using configfs. An endpoint function is designated as a
virtual endpoint function device when it is linked to a physical
endpoint function device (instead of a endpoint controller).
Signed-off-by: Kishon Vijay Abraham I <[email protected]>
---
Documentation/PCI/endpoint/pci-endpoint-cfs.rst | 12 +++++++++++-
1 file changed, 11 insertions(+), 1 deletion(-)
diff --git a/Documentation/PCI/endpoint/pci-endpoint-cfs.rst b/Documentation/PCI/endpoint/pci-endpoint-cfs.rst
index db609b97ad58..fb73345cfb8a 100644
--- a/Documentation/PCI/endpoint/pci-endpoint-cfs.rst
+++ b/Documentation/PCI/endpoint/pci-endpoint-cfs.rst
@@ -43,6 +43,7 @@ entries corresponding to EPF driver will be created by the EPF core.
.. <EPF Driver1>/
... <EPF Device 11>/
... <EPF Device 21>/
+ ... <EPF Device 31>/
.. <EPF Driver2>/
... <EPF Device 12>/
... <EPF Device 22>/
@@ -68,6 +69,7 @@ created)
... subsys_vendor_id
... subsys_id
... interrupt_pin
+ ... <Symlink EPF Device 31>/
... primary/
... <Symlink EPC Device1>/
... secondary/
@@ -79,6 +81,13 @@ interface should be added in 'primary' directory and symlink of endpoint
controller connected to secondary interface should be added in 'secondary'
directory.
+The <EPF Device> directory can have a list of symbolic links
+(<Symlink EPF Device 31>) to other <EPF Device>. These symbolic links should
+be created by the user to represent the virtual functions that are bound to
+the physical function. In the above directory structure <EPF Device 11> is a
+physical function and <EPF Device 31> is a virtual function. An EPF device once
+it's linked to another EPF device, cannot be linked to a EPC device.
+
EPC Device
==========
@@ -98,7 +107,8 @@ entries corresponding to EPC device will be created by the EPC core.
The <EPC Device> directory will have a list of symbolic links to
<EPF Device>. These symbolic links should be created by the user to
-represent the functions present in the endpoint device.
+represent the functions present in the endpoint device. Only <EPF Device>
+that represents a physical function can be linked to a EPC device.
The <EPC Device> directory will also have a *start* field. Once
"1" is written to this field, the endpoint device will be ready to
--
2.17.1
Now that support for SR-IOV is added in PCIe endpoint core, add support
to configure virtual functions in the Cadence PCIe EP driver.
Signed-off-by: Kishon Vijay Abraham I <[email protected]>
---
.../pci/controller/cadence/pcie-cadence-ep.c | 136 +++++++++++++++---
drivers/pci/controller/cadence/pcie-cadence.h | 9 ++
2 files changed, 125 insertions(+), 20 deletions(-)
diff --git a/drivers/pci/controller/cadence/pcie-cadence-ep.c b/drivers/pci/controller/cadence/pcie-cadence-ep.c
index f337f0842400..443dd06dfcfc 100644
--- a/drivers/pci/controller/cadence/pcie-cadence-ep.c
+++ b/drivers/pci/controller/cadence/pcie-cadence-ep.c
@@ -16,11 +16,37 @@
#define CDNS_PCIE_EP_IRQ_PCI_ADDR_NONE 0x1
#define CDNS_PCIE_EP_IRQ_PCI_ADDR_LEGACY 0x3
+static u8 cdns_pcie_get_fn_from_vfn(struct cdns_pcie *pcie, u8 fn, u8 vfn)
+{
+ u32 cap = CDNS_PCIE_EP_FUNC_SRIOV_CAP_OFFSET;
+ u32 first_vf_offset, stride;
+
+ if (vfn == 0)
+ return fn;
+
+ first_vf_offset = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_SRIOV_VF_OFFSET);
+ stride = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_SRIOV_VF_STRIDE);
+ fn = fn + first_vf_offset + ((vfn - 1) * stride);
+
+ return fn;
+}
+
static int cdns_pcie_ep_write_header(struct pci_epc *epc, u8 fn, u8 vfn,
struct pci_epf_header *hdr)
{
struct cdns_pcie_ep *ep = epc_get_drvdata(epc);
+ u32 cap = CDNS_PCIE_EP_FUNC_SRIOV_CAP_OFFSET;
struct cdns_pcie *pcie = &ep->pcie;
+ u32 reg;
+
+ if (vfn > 1) {
+ dev_err(&epc->dev, "Only Virtual Function #1 has deviceID\n");
+ return -EINVAL;
+ } else if (vfn == 1) {
+ reg = cap + PCI_SRIOV_VF_DID;
+ cdns_pcie_ep_fn_writew(pcie, fn, reg, hdr->deviceid);
+ return 0;
+ }
cdns_pcie_ep_fn_writew(pcie, fn, PCI_DEVICE_ID, hdr->deviceid);
cdns_pcie_ep_fn_writeb(pcie, fn, PCI_REVISION_ID, hdr->revid);
@@ -92,21 +118,29 @@ static int cdns_pcie_ep_set_bar(struct pci_epc *epc, u8 fn, u8 vfn,
addr0 = lower_32_bits(bar_phys);
addr1 = upper_32_bits(bar_phys);
- cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR0(fn, bar),
- addr0);
- cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn, bar),
- addr1);
reg = CDNS_PCIE_LM_EP_FUNC_BAR_CFG(bar, fn);
+ if (vfn == 1)
+ reg = CDNS_PCIE_LM_EP_VFUNC_BAR_CFG(bar, fn);
b = (bar < BAR_4) ? bar : bar - BAR_4;
- cfg = cdns_pcie_readl(pcie, reg);
- cfg &= ~(CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) |
- CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b));
- cfg |= (CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE(b, aperture) |
- CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL(b, ctrl));
- cdns_pcie_writel(pcie, reg, cfg);
+ if (vfn == 0 || vfn == 1) {
+ cfg = cdns_pcie_readl(pcie, reg);
+ cfg &= ~(CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) |
+ CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b));
+ cfg |= (CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE(b, aperture) |
+ CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL(b, ctrl));
+ cdns_pcie_writel(pcie, reg, cfg);
+ }
+ fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn);
+ cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR0(fn, bar),
+ addr0);
+ cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn, bar),
+ addr1);
+
+ if (vfn > 0)
+ epf = &epf->epf[vfn - 1];
epf->epf_bar[bar] = epf_bar;
return 0;
@@ -122,18 +156,25 @@ static void cdns_pcie_ep_clear_bar(struct pci_epc *epc, u8 fn, u8 vfn,
u32 reg, cfg, b, ctrl;
reg = CDNS_PCIE_LM_EP_FUNC_BAR_CFG(bar, fn);
+ if (vfn == 1)
+ reg = CDNS_PCIE_LM_EP_VFUNC_BAR_CFG(bar, fn);
b = (bar < BAR_4) ? bar : bar - BAR_4;
- ctrl = CDNS_PCIE_LM_BAR_CFG_CTRL_DISABLED;
- cfg = cdns_pcie_readl(pcie, reg);
- cfg &= ~(CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) |
- CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b));
- cfg |= CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL(b, ctrl);
- cdns_pcie_writel(pcie, reg, cfg);
+ if (vfn == 0 || vfn == 1) {
+ ctrl = CDNS_PCIE_LM_BAR_CFG_CTRL_DISABLED;
+ cfg = cdns_pcie_readl(pcie, reg);
+ cfg &= ~(CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) |
+ CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b));
+ cfg |= CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL(b, ctrl);
+ cdns_pcie_writel(pcie, reg, cfg);
+ }
+ fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn);
cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR0(fn, bar), 0);
cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn, bar), 0);
+ if (vfn > 0)
+ epf = &epf->epf[vfn - 1];
epf->epf_bar[bar] = NULL;
}
@@ -151,6 +192,7 @@ static int cdns_pcie_ep_map_addr(struct pci_epc *epc, u8 fn, u8 vfn,
return -EINVAL;
}
+ fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn);
cdns_pcie_set_outbound_region(pcie, 0, fn, r, false, addr, pci_addr, size);
set_bit(r, &ep->ob_region_map);
@@ -186,6 +228,8 @@ static int cdns_pcie_ep_set_msi(struct pci_epc *epc, u8 fn, u8 vfn, u8 mmc)
u32 cap = CDNS_PCIE_EP_FUNC_MSI_CAP_OFFSET;
u16 flags;
+ fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn);
+
/*
* Set the Multiple Message Capable bitfield into the Message Control
* register.
@@ -206,6 +250,8 @@ static int cdns_pcie_ep_get_msi(struct pci_epc *epc, u8 fn, u8 vfn)
u32 cap = CDNS_PCIE_EP_FUNC_MSI_CAP_OFFSET;
u16 flags, mme;
+ fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn);
+
/* Validate that the MSI feature is actually enabled. */
flags = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_MSI_FLAGS);
if (!(flags & PCI_MSI_FLAGS_ENABLE))
@@ -227,6 +273,8 @@ static int cdns_pcie_ep_get_msix(struct pci_epc *epc, u8 func_no, u8 vfunc_no)
u32 cap = CDNS_PCIE_EP_FUNC_MSIX_CAP_OFFSET;
u32 val, reg;
+ func_no = cdns_pcie_get_fn_from_vfn(pcie, func_no, vfunc_no);
+
reg = cap + PCI_MSIX_FLAGS;
val = cdns_pcie_ep_fn_readw(pcie, func_no, reg);
if (!(val & PCI_MSIX_FLAGS_ENABLE))
@@ -246,6 +294,8 @@ static int cdns_pcie_ep_set_msix(struct pci_epc *epc, u8 fn, u8 vfn,
u32 cap = CDNS_PCIE_EP_FUNC_MSIX_CAP_OFFSET;
u32 val, reg;
+ fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn);
+
reg = cap + PCI_MSIX_FLAGS;
val = cdns_pcie_ep_fn_readw(pcie, fn, reg);
val &= ~PCI_MSIX_FLAGS_QSIZE;
@@ -265,8 +315,8 @@ static int cdns_pcie_ep_set_msix(struct pci_epc *epc, u8 fn, u8 vfn,
return 0;
}
-static void cdns_pcie_ep_assert_intx(struct cdns_pcie_ep *ep, u8 fn,
- u8 intx, bool is_asserted)
+static void cdns_pcie_ep_assert_intx(struct cdns_pcie_ep *ep, u8 fn, u8 intx,
+ bool is_asserted)
{
struct cdns_pcie *pcie = &ep->pcie;
unsigned long flags;
@@ -335,6 +385,8 @@ static int cdns_pcie_ep_send_msi_irq(struct cdns_pcie_ep *ep, u8 fn, u8 vfn,
u8 msi_count;
u64 pci_addr, pci_addr_mask = 0xff;
+ fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn);
+
/* Check whether the MSI feature has been enabled by the PCI host. */
flags = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_MSI_FLAGS);
if (!(flags & PCI_MSI_FLAGS_ENABLE))
@@ -388,6 +440,8 @@ static int cdns_pcie_ep_map_msi_irq(struct pci_epc *epc, u8 fn, u8 vfn,
int ret;
int i;
+ fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn);
+
/* Check whether the MSI feature has been enabled by the PCI host. */
flags = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_MSI_FLAGS);
if (!(flags & PCI_MSI_FLAGS_ENABLE))
@@ -438,6 +492,12 @@ static int cdns_pcie_ep_send_msix_irq(struct cdns_pcie_ep *ep, u8 fn, u8 vfn,
u16 flags;
u8 bir;
+ epf = &ep->epf[fn];
+ if (vfn > 0)
+ epf = &epf->epf[vfn - 1];
+
+ fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn);
+
/* Check whether the MSI-X feature has been enabled by the PCI host. */
flags = cdns_pcie_ep_fn_readw(pcie, fn, cap + PCI_MSIX_FLAGS);
if (!(flags & PCI_MSIX_FLAGS_ENABLE))
@@ -448,7 +508,6 @@ static int cdns_pcie_ep_send_msix_irq(struct cdns_pcie_ep *ep, u8 fn, u8 vfn,
bir = tbl_offset & PCI_MSIX_TABLE_BIR;
tbl_offset &= PCI_MSIX_TABLE_OFFSET;
- epf = &ep->epf[fn];
msix_tbl = epf->epf_bar[bir]->addr + tbl_offset;
msg_addr = msix_tbl[(interrupt_num - 1)].msg_addr;
msg_data = msix_tbl[(interrupt_num - 1)].msg_data;
@@ -475,9 +534,15 @@ static int cdns_pcie_ep_raise_irq(struct pci_epc *epc, u8 fn, u8 vfn,
u16 interrupt_num)
{
struct cdns_pcie_ep *ep = epc_get_drvdata(epc);
+ struct cdns_pcie *pcie = &ep->pcie;
+ struct device *dev = pcie->dev;
switch (type) {
case PCI_EPC_IRQ_LEGACY:
+ if (vfn > 0) {
+ dev_err(dev, "Cannot raise legacy interrupts for VF\n");
+ return -EINVAL;
+ }
return cdns_pcie_ep_send_legacy_irq(ep, fn, vfn, 0);
case PCI_EPC_IRQ_MSI:
@@ -515,6 +580,13 @@ static int cdns_pcie_ep_start(struct pci_epc *epc)
return 0;
}
+static const struct pci_epc_features cdns_pcie_epc_vf_features = {
+ .linkup_notifier = false,
+ .msi_capable = true,
+ .msix_capable = true,
+ .align = 65536,
+};
+
static const struct pci_epc_features cdns_pcie_epc_features = {
.linkup_notifier = false,
.msi_capable = true,
@@ -525,7 +597,10 @@ static const struct pci_epc_features cdns_pcie_epc_features = {
static const struct pci_epc_features*
cdns_pcie_ep_get_features(struct pci_epc *epc, u8 func_no, u8 vfunc_no)
{
- return &cdns_pcie_epc_features;
+ if (!vfunc_no)
+ return &cdns_pcie_epc_features;
+
+ return &cdns_pcie_epc_vf_features;
}
static const struct pci_epc_ops cdns_pcie_epc_ops = {
@@ -551,9 +626,11 @@ int cdns_pcie_ep_setup(struct cdns_pcie_ep *ep)
struct platform_device *pdev = to_platform_device(dev);
struct device_node *np = dev->of_node;
struct cdns_pcie *pcie = &ep->pcie;
+ struct cdns_pcie_epf *epf;
struct resource *res;
struct pci_epc *epc;
int ret;
+ int i;
pcie->is_rc = false;
@@ -598,6 +675,25 @@ int cdns_pcie_ep_setup(struct cdns_pcie_ep *ep)
if (!ep->epf)
return -ENOMEM;
+ epc->max_vfs = devm_kcalloc(dev, epc->max_functions,
+ sizeof(*epc->max_vfs), GFP_KERNEL);
+ if (!epc->max_vfs)
+ return -ENOMEM;
+
+ ret = of_property_read_u8_array(np, "max-virtual-functions",
+ epc->max_vfs, epc->max_functions);
+ if (ret == 0) {
+ for (i = 0; i < epc->max_functions; i++) {
+ epf = &ep->epf[i];
+ if (epc->max_vfs[i] == 0)
+ continue;
+ epf->epf = devm_kcalloc(dev, epc->max_vfs[i],
+ sizeof(*ep->epf), GFP_KERNEL);
+ if (!epf->epf)
+ return -ENOMEM;
+ }
+ }
+
ret = pci_epc_mem_init(epc, pcie->mem_res->start,
resource_size(pcie->mem_res), PAGE_SIZE);
if (ret < 0) {
diff --git a/drivers/pci/controller/cadence/pcie-cadence.h b/drivers/pci/controller/cadence/pcie-cadence.h
index d5b1fcf2c39d..b72de578b996 100644
--- a/drivers/pci/controller/cadence/pcie-cadence.h
+++ b/drivers/pci/controller/cadence/pcie-cadence.h
@@ -52,6 +52,12 @@
(CDNS_PCIE_LM_BASE + 0x0240 + (fn) * 0x0008)
#define CDNS_PCIE_LM_EP_FUNC_BAR_CFG1(fn) \
(CDNS_PCIE_LM_BASE + 0x0244 + (fn) * 0x0008)
+#define CDNS_PCIE_LM_EP_VFUNC_BAR_CFG(bar, fn) \
+ (((bar) < 4) ? CDNS_PCIE_LM_EP_VFUNC_BAR_CFG0(fn) : CDNS_PCIE_LM_EP_VFUNC_BAR_CFG1(fn))
+#define CDNS_PCIE_LM_EP_VFUNC_BAR_CFG0(fn) \
+ (CDNS_PCIE_LM_BASE + 0x0280 + (fn) * 0x0008)
+#define CDNS_PCIE_LM_EP_VFUNC_BAR_CFG1(fn) \
+ (CDNS_PCIE_LM_BASE + 0x0284 + (fn) * 0x0008)
#define CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) \
(GENMASK(4, 0) << ((b) * 8))
#define CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE(b, a) \
@@ -116,6 +122,7 @@
#define CDNS_PCIE_EP_FUNC_MSI_CAP_OFFSET 0x90
#define CDNS_PCIE_EP_FUNC_MSIX_CAP_OFFSET 0xb0
+#define CDNS_PCIE_EP_FUNC_SRIOV_CAP_OFFSET 0x200
/*
* Root Port Registers (PCI configuration space for the root port function)
@@ -310,9 +317,11 @@ struct cdns_pcie_rc {
/**
* struct cdns_pcie_epf - Structure to hold info about endpoint function
+ * @epf: Info about virtual functions attached to the physical function
* @epf_bar: reference to the pci_epf_bar for the six Base Address Registers
*/
struct cdns_pcie_epf {
+ struct cdns_pcie_epf *epf;
struct pci_epf_bar *epf_bar[PCI_STD_NUM_BARS];
};
--
2.17.1
On Wed, 11 Aug 2021 12:16:48 +0530, Kishon Vijay Abraham I wrote:
> Patch series
> *) Adds support to add virtual functions to enable endpoint controller
> which supports SR-IOV capability
> *) Add support in Cadence endpoint driver to configure virtual functions
> *) Enable pci_endpoint_test driver to create pci_device for virtual
> functions
>
> [...]
Applied to pci/endpoint, thanks!
[1/8] dt-bindings: PCI: pci-ep: Add binding to specify virtual function
https://git.kernel.org/lpieralisi/pci/c/ea96be4a7b
[2/8] PCI: endpoint: Add support to add virtual function in endpoint core
https://git.kernel.org/lpieralisi/pci/c/b64215ff2b
[3/8] PCI: endpoint: Add support to link a physical function to a virtual function
https://git.kernel.org/lpieralisi/pci/c/555d7c5f83
[4/8] PCI: endpoint: Add virtual function number in pci_epc ops
https://git.kernel.org/lpieralisi/pci/c/22a8013ade
[5/8] PCI: cadence: Simplify code to get register base address for configuring BAR
https://git.kernel.org/lpieralisi/pci/c/9d26fd710f
[6/8] PCI: cadence: Add support to configure virtual functions
https://git.kernel.org/lpieralisi/pci/c/f52e1cf18d
[7/8] misc: pci_endpoint_test: Populate sriov_configure ops to configure SR-IOV device
https://git.kernel.org/lpieralisi/pci/c/681883d1d1
[8/8] Documentation: PCI: endpoint/pci-endpoint-cfs: Guide to use SR-IOV
https://git.kernel.org/lpieralisi/pci/c/805ff686a1
Thanks,
Lorenzo
On Wed, Aug 11, 2021 at 12:16:53PM +0530, Kishon Vijay Abraham I wrote:
> No functional change. Simplify code to get register base address for
> configuring PCI BAR.
>
> Signed-off-by: Kishon Vijay Abraham I <[email protected]>
> ---
> .../pci/controller/cadence/pcie-cadence-ep.c | 18 ++++--------------
> drivers/pci/controller/cadence/pcie-cadence.h | 2 ++
> 2 files changed, 6 insertions(+), 14 deletions(-)
>
> diff --git a/drivers/pci/controller/cadence/pcie-cadence-ep.c b/drivers/pci/controller/cadence/pcie-cadence-ep.c
> index 912a15be8bfd..f337f0842400 100644
> --- a/drivers/pci/controller/cadence/pcie-cadence-ep.c
> +++ b/drivers/pci/controller/cadence/pcie-cadence-ep.c
> @@ -97,13 +97,8 @@ static int cdns_pcie_ep_set_bar(struct pci_epc *epc, u8 fn, u8 vfn,
> cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn, bar),
> addr1);
>
> - if (bar < BAR_4) {
> - reg = CDNS_PCIE_LM_EP_FUNC_BAR_CFG0(fn);
> - b = bar;
> - } else {
> - reg = CDNS_PCIE_LM_EP_FUNC_BAR_CFG1(fn);
> - b = bar - BAR_4;
> - }
> + reg = CDNS_PCIE_LM_EP_FUNC_BAR_CFG(bar, fn);
> + b = (bar < BAR_4) ? bar : bar - BAR_4;
>
> cfg = cdns_pcie_readl(pcie, reg);
> cfg &= ~(CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) |
> @@ -126,13 +121,8 @@ static void cdns_pcie_ep_clear_bar(struct pci_epc *epc, u8 fn, u8 vfn,
> enum pci_barno bar = epf_bar->barno;
> u32 reg, cfg, b, ctrl;
>
> - if (bar < BAR_4) {
> - reg = CDNS_PCIE_LM_EP_FUNC_BAR_CFG0(fn);
> - b = bar;
> - } else {
> - reg = CDNS_PCIE_LM_EP_FUNC_BAR_CFG1(fn);
> - b = bar - BAR_4;
> - }
> + reg = CDNS_PCIE_LM_EP_FUNC_BAR_CFG(bar, fn);
> + b = (bar < BAR_4) ? bar : bar - BAR_4;
>
> ctrl = CDNS_PCIE_LM_BAR_CFG_CTRL_DISABLED;
> cfg = cdns_pcie_readl(pcie, reg);
> diff --git a/drivers/pci/controller/cadence/pcie-cadence.h b/drivers/pci/controller/cadence/pcie-cadence.h
> index 30db2d68c17a..d5b1fcf2c39d 100644
> --- a/drivers/pci/controller/cadence/pcie-cadence.h
> +++ b/drivers/pci/controller/cadence/pcie-cadence.h
> @@ -46,6 +46,8 @@
> #define CDNS_PCIE_LM_EP_ID_BUS_SHIFT 8
>
> /* Endpoint Function f BAR b Configuration Registers */
> +#define CDNS_PCIE_LM_EP_FUNC_BAR_CFG(bar, fn) \
> + (((bar) < 4) ? CDNS_PCIE_LM_EP_FUNC_BAR_CFG0(fn) : CDNS_PCIE_LM_EP_FUNC_BAR_CFG1(fn))
Why do we use "BAR_4" above and "4" here? Shouldn't they look the
same?
> #define CDNS_PCIE_LM_EP_FUNC_BAR_CFG0(fn) \
> (CDNS_PCIE_LM_BASE + 0x0240 + (fn) * 0x0008)
> #define CDNS_PCIE_LM_EP_FUNC_BAR_CFG1(fn) \
> --
> 2.17.1
>
On Wed, Aug 11, 2021 at 12:16:54PM +0530, Kishon Vijay Abraham I wrote:
> Now that support for SR-IOV is added in PCIe endpoint core, add support
> to configure virtual functions in the Cadence PCIe EP driver.
>
> Signed-off-by: Kishon Vijay Abraham I <[email protected]>
> ---
> .../pci/controller/cadence/pcie-cadence-ep.c | 136 +++++++++++++++---
> drivers/pci/controller/cadence/pcie-cadence.h | 9 ++
> 2 files changed, 125 insertions(+), 20 deletions(-)
> @@ -92,21 +118,29 @@ static int cdns_pcie_ep_set_bar(struct pci_epc *epc, u8 fn, u8 vfn,
>
> addr0 = lower_32_bits(bar_phys);
> addr1 = upper_32_bits(bar_phys);
> - cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR0(fn, bar),
> - addr0);
> - cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn, bar),
> - addr1);
>
> reg = CDNS_PCIE_LM_EP_FUNC_BAR_CFG(bar, fn);
> + if (vfn == 1)
> + reg = CDNS_PCIE_LM_EP_VFUNC_BAR_CFG(bar, fn);
Seems sort of weird to compute "reg", then sometimes overwrite it, as
opposed to:
if (vfn == 1)
reg = CDNS_PCIE_LM_EP_VFUNC_BAR_CFG(bar, fn);
else
reg = CDNS_PCIE_LM_EP_FUNC_BAR_CFG(bar, fn);
Also slightly weird that "vfn" is basically used as a boolean, but
it's actually a u8 virtual function number. I guess VF 1 is special
and not like the other VFs?
> b = (bar < BAR_4) ? bar : bar - BAR_4;
>
> - cfg = cdns_pcie_readl(pcie, reg);
> - cfg &= ~(CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) |
> - CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b));
> - cfg |= (CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE(b, aperture) |
> - CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL(b, ctrl));
> - cdns_pcie_writel(pcie, reg, cfg);
> + if (vfn == 0 || vfn == 1) {
> + cfg = cdns_pcie_readl(pcie, reg);
> + cfg &= ~(CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) |
> + CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b));
> + cfg |= (CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE(b, aperture) |
> + CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL(b, ctrl));
> + cdns_pcie_writel(pcie, reg, cfg);
> + }
>
> + fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn);
> + cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR0(fn, bar),
> + addr0);
> + cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn, bar),
> + addr1);
> +
> + if (vfn > 0)
> + epf = &epf->epf[vfn - 1];
> epf->epf_bar[bar] = epf_bar;
>
> return 0;
> @@ -122,18 +156,25 @@ static void cdns_pcie_ep_clear_bar(struct pci_epc *epc, u8 fn, u8 vfn,
> u32 reg, cfg, b, ctrl;
>
> reg = CDNS_PCIE_LM_EP_FUNC_BAR_CFG(bar, fn);
> + if (vfn == 1)
> + reg = CDNS_PCIE_LM_EP_VFUNC_BAR_CFG(bar, fn);
Similar recomputation of "reg".
> b = (bar < BAR_4) ? bar : bar - BAR_4;
>
> - ctrl = CDNS_PCIE_LM_BAR_CFG_CTRL_DISABLED;
> - cfg = cdns_pcie_readl(pcie, reg);
> - cfg &= ~(CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) |
> - CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b));
> - cfg |= CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL(b, ctrl);
> - cdns_pcie_writel(pcie, reg, cfg);
> + if (vfn == 0 || vfn == 1) {
> + ctrl = CDNS_PCIE_LM_BAR_CFG_CTRL_DISABLED;
> + cfg = cdns_pcie_readl(pcie, reg);
> + cfg &= ~(CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) |
> + CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b));
> + cfg |= CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL(b, ctrl);
> + cdns_pcie_writel(pcie, reg, cfg);
> + }
Hi Bjorn,
On 17/08/21 9:08 pm, Bjorn Helgaas wrote:
> On Wed, Aug 11, 2021 at 12:16:54PM +0530, Kishon Vijay Abraham I wrote:
>> Now that support for SR-IOV is added in PCIe endpoint core, add support
>> to configure virtual functions in the Cadence PCIe EP driver.
>>
>> Signed-off-by: Kishon Vijay Abraham I <[email protected]>
>> ---
>> .../pci/controller/cadence/pcie-cadence-ep.c | 136 +++++++++++++++---
>> drivers/pci/controller/cadence/pcie-cadence.h | 9 ++
>> 2 files changed, 125 insertions(+), 20 deletions(-)
>
>> @@ -92,21 +118,29 @@ static int cdns_pcie_ep_set_bar(struct pci_epc *epc, u8 fn, u8 vfn,
>>
>> addr0 = lower_32_bits(bar_phys);
>> addr1 = upper_32_bits(bar_phys);
>> - cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR0(fn, bar),
>> - addr0);
>> - cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn, bar),
>> - addr1);
>>
>> reg = CDNS_PCIE_LM_EP_FUNC_BAR_CFG(bar, fn);
>> + if (vfn == 1)
>> + reg = CDNS_PCIE_LM_EP_VFUNC_BAR_CFG(bar, fn);
>
> Seems sort of weird to compute "reg", then sometimes overwrite it, as
> opposed to:
>
> if (vfn == 1)
> reg = CDNS_PCIE_LM_EP_VFUNC_BAR_CFG(bar, fn);
> else
> reg = CDNS_PCIE_LM_EP_FUNC_BAR_CFG(bar, fn);
I tried to write it without "else". But I can change it back.
>
> Also slightly weird that "vfn" is basically used as a boolean, but
> it's actually a u8 virtual function number. I guess VF 1 is special
> and not like the other VFs?
VF1 is special in that it's enough for configuring the SR-IOV capability
but below the "vfn" is used for configuring inbound window.
Thanks
Kishon
>
>> b = (bar < BAR_4) ? bar : bar - BAR_4;
>>
>> - cfg = cdns_pcie_readl(pcie, reg);
>> - cfg &= ~(CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) |
>> - CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b));
>> - cfg |= (CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE(b, aperture) |
>> - CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL(b, ctrl));
>> - cdns_pcie_writel(pcie, reg, cfg);
>> + if (vfn == 0 || vfn == 1) {
>> + cfg = cdns_pcie_readl(pcie, reg);
>> + cfg &= ~(CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) |
>> + CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b));
>> + cfg |= (CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE(b, aperture) |
>> + CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL(b, ctrl));
>> + cdns_pcie_writel(pcie, reg, cfg);
>> + }
>>
>> + fn = cdns_pcie_get_fn_from_vfn(pcie, fn, vfn);
>> + cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR0(fn, bar),
>> + addr0);
>> + cdns_pcie_writel(pcie, CDNS_PCIE_AT_IB_EP_FUNC_BAR_ADDR1(fn, bar),
>> + addr1);
>> +
>> + if (vfn > 0)
>> + epf = &epf->epf[vfn - 1];
>> epf->epf_bar[bar] = epf_bar;
>>
>> return 0;
>> @@ -122,18 +156,25 @@ static void cdns_pcie_ep_clear_bar(struct pci_epc *epc, u8 fn, u8 vfn,
>> u32 reg, cfg, b, ctrl;
>>
>> reg = CDNS_PCIE_LM_EP_FUNC_BAR_CFG(bar, fn);
>> + if (vfn == 1)
>> + reg = CDNS_PCIE_LM_EP_VFUNC_BAR_CFG(bar, fn);
>
> Similar recomputation of "reg".
>
>> b = (bar < BAR_4) ? bar : bar - BAR_4;
>>
>> - ctrl = CDNS_PCIE_LM_BAR_CFG_CTRL_DISABLED;
>> - cfg = cdns_pcie_readl(pcie, reg);
>> - cfg &= ~(CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) |
>> - CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b));
>> - cfg |= CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL(b, ctrl);
>> - cdns_pcie_writel(pcie, reg, cfg);
>> + if (vfn == 0 || vfn == 1) {
>> + ctrl = CDNS_PCIE_LM_BAR_CFG_CTRL_DISABLED;
>> + cfg = cdns_pcie_readl(pcie, reg);
>> + cfg &= ~(CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_APERTURE_MASK(b) |
>> + CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL_MASK(b));
>> + cfg |= CDNS_PCIE_LM_EP_FUNC_BAR_CFG_BAR_CTRL(b, ctrl);
>> + cdns_pcie_writel(pcie, reg, cfg);
>> + }