by Leo Li

[permalink] [raw]

Subject: Re: [PATCH 06/13] soc/fsl/qbman_portals: add APIs to retrieve the probing status

On Fri, Mar 29, 2019 at 9:03 AM <[email protected]> wrote:
>
> From: Laurentiu Tudor <[email protected]>
>
> Add a couple of new APIs to check the probing status of the required
> cpu bound qman and bman portals:
> 'int bman_portals_probed()' and 'int qman_portals_probed()'.
> They return the following values.
> * 1 if qman/bman portals were all probed correctly
> * 0 if qman/bman portals were not yet probed
> * -1 if probing of qman/bman portals failed
> Portals are considered successful probed if no error occurred during
> the probing of any of the portals and if enough portals were probed
> to have one available for each cpu.
> The error handling paths were slightly rearranged in order to fit this
> new functionality without being too intrusive.
> Drivers that use qman/bman portal driver services are required to use
> these APIs before calling any functions exported by these drivers or
> otherwise they will crash the kernel.
> First user will be the dpaa1 ethernet driver, coming in a subsequent
> patch.
>
> Signed-off-by: Laurentiu Tudor <[email protected]>
> ---
> drivers/soc/fsl/qbman/bman_portal.c | 22 ++++++++++++++++++----
> drivers/soc/fsl/qbman/qman_portal.c | 23 +++++++++++++++++++----
> include/soc/fsl/bman.h | 8 ++++++++
> include/soc/fsl/qman.h | 9 +++++++++
> 4 files changed, 54 insertions(+), 8 deletions(-)
>
> diff --git a/drivers/soc/fsl/qbman/bman_portal.c b/drivers/soc/fsl/qbman/bman_portal.c
> index 2c95cf59f3e7..7819bc29936d 100644
> --- a/drivers/soc/fsl/qbman/bman_portal.c
> +++ b/drivers/soc/fsl/qbman/bman_portal.c
> @@ -32,6 +32,7 @@
>
> static struct bman_portal *affine_bportals[NR_CPUS];
> static struct cpumask portal_cpus;
> +static int __bman_portals_probed;
> /* protect bman global registers and global data shared among portals */
> static DEFINE_SPINLOCK(bman_lock);
>
> @@ -87,6 +88,12 @@ static int bman_online_cpu(unsigned int cpu)
> return 0;
> }
>
> +int bman_portals_probed(void)
> +{
> + return __bman_portals_probed;
> +}
> +EXPORT_SYMBOL_GPL(bman_portals_probed);
> +
> static int bman_portal_probe(struct platform_device *pdev)
> {
> struct device *dev = &pdev->dev;
> @@ -104,8 +111,10 @@ static int bman_portal_probe(struct platform_device *pdev)
> }
>
> pcfg = devm_kmalloc(dev, sizeof(*pcfg), GFP_KERNEL);
> - if (!pcfg)
> + if (!pcfg) {
> + __bman_portals_probed = -1;
> return -ENOMEM;
> + }
>
> pcfg->dev = dev;
>
> @@ -113,14 +122,14 @@ static int bman_portal_probe(struct platform_device *pdev)
> DPAA_PORTAL_CE);
> if (!addr_phys[0]) {
> dev_err(dev, "Can't get %pOF property 'reg::CE'\n", node);
> - return -ENXIO;
> + goto err_ioremap1;
> }
>
> addr_phys[1] = platform_get_resource(pdev, IORESOURCE_MEM,
> DPAA_PORTAL_CI);
> if (!addr_phys[1]) {
> dev_err(dev, "Can't get %pOF property 'reg::CI'\n", node);
> - return -ENXIO;
> + goto err_ioremap1;
> }
>
> pcfg->cpu = -1;
> @@ -128,7 +137,7 @@ static int bman_portal_probe(struct platform_device *pdev)
> irq = platform_get_irq(pdev, 0);
> if (irq <= 0) {
> dev_err(dev, "Can't get %pOF IRQ'\n", node);
> - return -ENXIO;
> + goto err_ioremap1;
> }
> pcfg->irq = irq;
>
> @@ -156,6 +165,9 @@ static int bman_portal_probe(struct platform_device *pdev)
> }
>
> cpumask_set_cpu(cpu, &portal_cpus);
> + if (!__bman_portals_probed &&
> + cpumask_weight(&portal_cpus) == num_online_cpus())
> + __bman_portals_probed = 1;

Given the fact that the portal_cpus bit will get set even for offline
cpus, this is not correct either. Probably the previous code for
checking cpu >= nr_cpu_ids is actually the right way to do it.

> spin_unlock(&bman_lock);
> pcfg->cpu = cpu;
>
> @@ -175,6 +187,8 @@ static int bman_portal_probe(struct platform_device *pdev)
> err_ioremap2:
> memunmap(pcfg->addr_virt_ce);
> err_ioremap1:
> + __bman_portals_probed = -1;
> +
> return -ENXIO;
> }
>
> diff --git a/drivers/soc/fsl/qbman/qman_portal.c b/drivers/soc/fsl/qbman/qman_portal.c
> index bce56da2b01f..11ba6c77c0d6 100644
> --- a/drivers/soc/fsl/qbman/qman_portal.c
> +++ b/drivers/soc/fsl/qbman/qman_portal.c
> @@ -39,6 +39,7 @@ EXPORT_SYMBOL(qman_dma_portal);
> #define CONFIG_FSL_DPA_PIRQ_FAST 1
>
> static struct cpumask portal_cpus;
> +static int __qman_portals_probed;
> /* protect qman global registers and global data shared among portals */
> static DEFINE_SPINLOCK(qman_lock);
>
> @@ -221,6 +222,12 @@ static int qman_online_cpu(unsigned int cpu)
> return 0;
> }
>
> +int qman_portals_probed(void)
> +{
> + return __qman_portals_probed;
> +}
> +EXPORT_SYMBOL_GPL(qman_portals_probed);
> +
> static int qman_portal_probe(struct platform_device *pdev)
> {
> struct device *dev = &pdev->dev;
> @@ -240,8 +247,10 @@ static int qman_portal_probe(struct platform_device *pdev)
> }
>
> pcfg = devm_kmalloc(dev, sizeof(*pcfg), GFP_KERNEL);
> - if (!pcfg)
> + if (!pcfg) {
> + __qman_portals_probed = -1;
> return -ENOMEM;
> + }
>
> pcfg->dev = dev;
>
> @@ -249,19 +258,20 @@ static int qman_portal_probe(struct platform_device *pdev)
> DPAA_PORTAL_CE);
> if (!addr_phys[0]) {
> dev_err(dev, "Can't get %pOF property 'reg::CE'\n", node);
> - return -ENXIO;
> + goto err_ioremap1;
> }
>
> addr_phys[1] = platform_get_resource(pdev, IORESOURCE_MEM,
> DPAA_PORTAL_CI);
> if (!addr_phys[1]) {
> dev_err(dev, "Can't get %pOF property 'reg::CI'\n", node);
> - return -ENXIO;
> + goto err_ioremap1;
> }
>
> err = of_property_read_u32(node, "cell-index", &val);
> if (err) {
> dev_err(dev, "Can't get %pOF property 'cell-index'\n", node);
> + __qman_portals_probed = -1;
> return err;
> }
> pcfg->channel = val;
> @@ -269,7 +279,7 @@ static int qman_portal_probe(struct platform_device *pdev)
> irq = platform_get_irq(pdev, 0);
> if (irq <= 0) {
> dev_err(dev, "Can't get %pOF IRQ\n", node);
> - return -ENXIO;
> + goto err_ioremap1;
> }
> pcfg->irq = irq;
>
> @@ -314,6 +324,9 @@ static int qman_portal_probe(struct platform_device *pdev)
> }
>
> cpumask_set_cpu(cpu, &portal_cpus);
> + if (!__qman_portals_probed &&
> + cpumask_weight(&portal_cpus) == num_online_cpus())
> + __qman_portals_probed = 1;
> spin_unlock(&qman_lock);
> pcfg->cpu = cpu;
>
> @@ -338,6 +351,8 @@ static int qman_portal_probe(struct platform_device *pdev)
> err_ioremap2:
> memunmap(pcfg->addr_virt_ce);
> err_ioremap1:
> + __qman_portals_probed = -1;
> +
> return -ENXIO;
> }
>
> diff --git a/include/soc/fsl/bman.h b/include/soc/fsl/bman.h
> index 5b99cb2ea5ef..173e4049d963 100644
> --- a/include/soc/fsl/bman.h
> +++ b/include/soc/fsl/bman.h
> @@ -133,5 +133,13 @@ int bman_acquire(struct bman_pool *pool, struct bm_buffer *bufs, u8 num);
> * failed to probe or 0 if the bman driver did not probed yet.
> */
> int bman_is_probed(void);
> +/**
> + * bman_portals_probed - Check if all cpu bound bman portals are probed
> + *
> + * Returns 1 if all the required cpu bound bman portals successfully probed,
> + * -1 if probe errors appeared or 0 if the bman portals did not yet finished
> + * probing.
> + */
> +int bman_portals_probed(void);
>
> #endif /* __FSL_BMAN_H */
> diff --git a/include/soc/fsl/qman.h b/include/soc/fsl/qman.h
> index 5cc7af06c1ba..aa31c05a103a 100644
> --- a/include/soc/fsl/qman.h
> +++ b/include/soc/fsl/qman.h
> @@ -1194,6 +1194,15 @@ int qman_release_cgrid(u32 id);
> */
> int qman_is_probed(void);
>
> +/**
> + * qman_portals_probed - Check if all cpu bound qman portals are probed
> + *
> + * Returns 1 if all the required cpu bound qman portals successfully probed,
> + * -1 if probe errors appeared or 0 if the qman portals did not yet finished
> + * probing.
> + */
> +int qman_portals_probed(void);
> +
> /**
> * qman_dqrr_get_ithresh - Get coalesce interrupt threshold
> * @portal: portal to get the value for
> --
> 2.17.1
>

2019-04-19 18:22:13

by Laurentiu Tudor

[permalink] [raw]

Subject: RE: [PATCH 02/13] soc/fsl/bman: map FBPR area in the iommu

Hi Robin,

> -----Original Message-----
> From: Robin Murphy <[email protected]>
> Sent: Friday, March 29, 2019 4:51 PM
>
> On 29/03/2019 14:00, [email protected] wrote:
> > From: Laurentiu Tudor <[email protected]>
> >
> > Add a one-to-one iommu mapping for bman private data memory (FBPR).
> > This is required for BMAN to work without faults behind an iommu.
> >
> > Signed-off-by: Laurentiu Tudor <[email protected]>
> > ---
> > drivers/soc/fsl/qbman/bman_ccsr.c | 11 +++++++++++
> > 1 file changed, 11 insertions(+)
> >
> > diff --git a/drivers/soc/fsl/qbman/bman_ccsr.c
> b/drivers/soc/fsl/qbman/bman_ccsr.c
> > index 7c3cc968053c..b209c79511bb 100644
> > --- a/drivers/soc/fsl/qbman/bman_ccsr.c
> > +++ b/drivers/soc/fsl/qbman/bman_ccsr.c
> > @@ -29,6 +29,7 @@
> > */
> >
> > #include "bman_priv.h"
> > +#include <linux/iommu.h>
> >
> > u16 bman_ip_rev;
> > EXPORT_SYMBOL(bman_ip_rev);
> > @@ -178,6 +179,7 @@ static int fsl_bman_probe(struct platform_device
> *pdev)
> > int ret, err_irq;
> > struct device *dev = &pdev->dev;
> > struct device_node *node = dev->of_node;
> > + struct iommu_domain *domain;
> > struct resource *res;
> > u16 id, bm_pool_cnt;
> > u8 major, minor;
> > @@ -225,6 +227,15 @@ static int fsl_bman_probe(struct platform_device
> *pdev)
> >
> > dev_dbg(dev, "Allocated FBPR 0x%llx 0x%zx\n", fbpr_a, fbpr_sz);
> >
> > + /* Create an 1-to-1 iommu mapping for FBPR area */
> > + domain = iommu_get_domain_for_dev(dev);
>
> If that's expected to be the default domain that you're grabbing, then
> this is *incredibly* fragile. There's nothing to stop the IOVA that you
> forcibly map from being automatically allocated later and causing some
> other DMA mapping to fail noisily and unexpectedly. Furthermore, have
> you tried this with "iommu.passthrough=1"?
>
> That said, I really don't understand what's going on here anyway :/
>
> As far as I can tell from qbman_init_private_mem(), fbpr_a comes from
> dma_alloc_coherent() and thus would already be a mapped IOVA - isn't
> this the stuff that Roy converted to nicely use shared-dma-pool regions
> a while ago?
>

Finally found some time to look into this, sorry for the delay. It seems that on the code path taken in our case (dma_alloc_coherent() -> dma_alloc_attrs() -> dma_alloc_from_dev_coherent() -> __dma_alloc_from_coherent()) there's no call into the iommu layer, thus no mapping in the smmu. I plan to come up with a RFC patch early next week so we have something concrete to discuss on.

---
Best Regards, Laurentiu