On systems where DMA addresses and physical addresses are not 1:1
(such as Xen PV guests), the generic dma_get_required_mask() will not
return the correct mask (since it uses max_pfn).
Some device drivers (such as mptsas, mpt2sas) use
dma_get_required_mask() to set the device's DMA mask to allow them to use
only 32-bit DMA addresses in hardware structures. This results in
unnecessary use of the SWIOTLB if DMA addresses are more than 32-bits,
impacting performance significantly.
This series allows Xen PV guests to override the default
dma_get_required_mask() with one that calculates the DMA mask from the
maximum MFN (and not the PFN).
Changes in v3:
- fix off-by-one in xen_dma_get_required_mask()
- split ia64 changes into separate patch.
Changes in v2:
- split x86 and xen changes into separate patches
David
A generic dma_get_required_mask() is useful even for architectures (such
as ia64) that define ARCH_HAS_GET_REQUIRED_MASK.
Signed-off-by: David Vrabel <[email protected]>
Reviewed-by: Stefano Stabellini <[email protected]>
---
drivers/base/platform.c | 10 ++++++++--
include/linux/dma-mapping.h | 1 +
2 files changed, 9 insertions(+), 2 deletions(-)
diff --git a/drivers/base/platform.c b/drivers/base/platform.c
index b2afc29..f9f3930 100644
--- a/drivers/base/platform.c
+++ b/drivers/base/platform.c
@@ -1009,8 +1009,7 @@ int __init platform_bus_init(void)
return error;
}
-#ifndef ARCH_HAS_DMA_GET_REQUIRED_MASK
-u64 dma_get_required_mask(struct device *dev)
+u64 dma_get_required_mask_from_max_pfn(struct device *dev)
{
u32 low_totalram = ((max_pfn - 1) << PAGE_SHIFT);
u32 high_totalram = ((max_pfn - 1) >> (32 - PAGE_SHIFT));
@@ -1028,6 +1027,13 @@ u64 dma_get_required_mask(struct device *dev)
}
return mask;
}
+EXPORT_SYMBOL_GPL(dma_get_required_mask_from_max_pfn);
+
+#ifndef ARCH_HAS_DMA_GET_REQUIRED_MASK
+u64 dma_get_required_mask(struct device *dev)
+{
+ return dma_get_required_mask_from_max_pfn(dev);
+}
EXPORT_SYMBOL_GPL(dma_get_required_mask);
#endif
diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h
index d5d3881..6e2fdfc 100644
--- a/include/linux/dma-mapping.h
+++ b/include/linux/dma-mapping.h
@@ -127,6 +127,7 @@ static inline int dma_coerce_mask_and_coherent(struct device *dev, u64 mask)
return dma_set_mask_and_coherent(dev, mask);
}
+extern u64 dma_get_required_mask_from_max_pfn(struct device *dev);
extern u64 dma_get_required_mask(struct device *dev);
#ifndef set_arch_dma_coherent_ops
--
1.7.10.4
Signed-off-by: David Vrabel <[email protected]>
Cc: Tony Luck <[email protected]>
Cc: Fenghua Yu <[email protected]>
Cc: [email protected]
---
arch/ia64/include/asm/machvec.h | 2 +-
arch/ia64/include/asm/machvec_init.h | 1 -
arch/ia64/pci/pci.c | 20 --------------------
3 files changed, 1 insertion(+), 22 deletions(-)
diff --git a/arch/ia64/include/asm/machvec.h b/arch/ia64/include/asm/machvec.h
index 9c39bdf..beaa47d 100644
--- a/arch/ia64/include/asm/machvec.h
+++ b/arch/ia64/include/asm/machvec.h
@@ -287,7 +287,7 @@ extern struct dma_map_ops *dma_get_ops(struct device *);
# define platform_dma_get_ops dma_get_ops
#endif
#ifndef platform_dma_get_required_mask
-# define platform_dma_get_required_mask ia64_dma_get_required_mask
+# define platform_dma_get_required_mask dma_get_required_mask_from_max_pfn
#endif
#ifndef platform_irq_to_vector
# define platform_irq_to_vector __ia64_irq_to_vector
diff --git a/arch/ia64/include/asm/machvec_init.h b/arch/ia64/include/asm/machvec_init.h
index 37a4698..ef964b2 100644
--- a/arch/ia64/include/asm/machvec_init.h
+++ b/arch/ia64/include/asm/machvec_init.h
@@ -3,7 +3,6 @@
extern ia64_mv_send_ipi_t ia64_send_ipi;
extern ia64_mv_global_tlb_purge_t ia64_global_tlb_purge;
-extern ia64_mv_dma_get_required_mask ia64_dma_get_required_mask;
extern ia64_mv_irq_to_vector __ia64_irq_to_vector;
extern ia64_mv_local_vector_to_irq __ia64_local_vector_to_irq;
extern ia64_mv_pci_get_legacy_mem_t ia64_pci_get_legacy_mem;
diff --git a/arch/ia64/pci/pci.c b/arch/ia64/pci/pci.c
index 291a582..79da21b 100644
--- a/arch/ia64/pci/pci.c
+++ b/arch/ia64/pci/pci.c
@@ -791,26 +791,6 @@ static void __init set_pci_dfl_cacheline_size(void)
pci_dfl_cache_line_size = (1 << cci.pcci_line_size) / 4;
}
-u64 ia64_dma_get_required_mask(struct device *dev)
-{
- u32 low_totalram = ((max_pfn - 1) << PAGE_SHIFT);
- u32 high_totalram = ((max_pfn - 1) >> (32 - PAGE_SHIFT));
- u64 mask;
-
- if (!high_totalram) {
- /* convert to mask just covering totalram */
- low_totalram = (1 << (fls(low_totalram) - 1));
- low_totalram += low_totalram - 1;
- mask = low_totalram;
- } else {
- high_totalram = (1 << (fls(high_totalram) - 1));
- high_totalram += high_totalram - 1;
- mask = (((u64)high_totalram) << 32) + 0xffffffff;
- }
- return mask;
-}
-EXPORT_SYMBOL_GPL(ia64_dma_get_required_mask);
-
u64 dma_get_required_mask(struct device *dev)
{
return platform_dma_get_required_mask(dev);
--
1.7.10.4
On a Xen PV guest the DMA addresses and physical addresses are not 1:1
(such as Xen PV guests) and the generic dma_get_required_mask() does
not return the correct mask (since it uses max_pfn).
Some device drivers (such as mptsas, mpt2sas) use
dma_get_required_mask() to set the device's DMA mask to allow them to
use only 32-bit DMA addresses in hardware structures. This results in
unnecessary use of the SWIOTLB if DMA addresses are more than 32-bits,
impacting performance significantly.
Provide a get_required_mask op that uses the maximum MFN to calculate
the DMA mask.
Signed-off-by: David Vrabel <[email protected]>
---
arch/x86/xen/pci-swiotlb-xen.c | 1 +
drivers/xen/swiotlb-xen.c | 13 +++++++++++++
include/xen/swiotlb-xen.h | 4 ++++
3 files changed, 18 insertions(+)
diff --git a/arch/x86/xen/pci-swiotlb-xen.c b/arch/x86/xen/pci-swiotlb-xen.c
index 0e98e5d..a5d180a 100644
--- a/arch/x86/xen/pci-swiotlb-xen.c
+++ b/arch/x86/xen/pci-swiotlb-xen.c
@@ -31,6 +31,7 @@ static struct dma_map_ops xen_swiotlb_dma_ops = {
.map_page = xen_swiotlb_map_page,
.unmap_page = xen_swiotlb_unmap_page,
.dma_supported = xen_swiotlb_dma_supported,
+ .get_required_mask = xen_swiotlb_get_required_mask,
};
/*
diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c
index ebd8f21..654587d 100644
--- a/drivers/xen/swiotlb-xen.c
+++ b/drivers/xen/swiotlb-xen.c
@@ -42,9 +42,11 @@
#include <xen/page.h>
#include <xen/xen-ops.h>
#include <xen/hvc-console.h>
+#include <xen/interface/memory.h>
#include <asm/dma-mapping.h>
#include <asm/xen/page-coherent.h>
+#include <asm/xen/hypercall.h>
#include <trace/events/swiotlb.h>
/*
@@ -683,3 +685,14 @@ xen_swiotlb_set_dma_mask(struct device *dev, u64 dma_mask)
return 0;
}
EXPORT_SYMBOL_GPL(xen_swiotlb_set_dma_mask);
+
+u64
+xen_swiotlb_get_required_mask(struct device *dev)
+{
+ unsigned long max_mfn;
+
+ max_mfn = HYPERVISOR_memory_op(XENMEM_maximum_ram_page, NULL);
+
+ return DMA_BIT_MASK(fls_long(max_mfn - 1) + PAGE_SHIFT);
+}
+EXPORT_SYMBOL_GPL(xen_swiotlb_get_required_mask);
diff --git a/include/xen/swiotlb-xen.h b/include/xen/swiotlb-xen.h
index 8b2eb93..6408888 100644
--- a/include/xen/swiotlb-xen.h
+++ b/include/xen/swiotlb-xen.h
@@ -58,4 +58,8 @@ xen_swiotlb_dma_supported(struct device *hwdev, u64 mask);
extern int
xen_swiotlb_set_dma_mask(struct device *dev, u64 dma_mask);
+
+extern u64
+xen_swiotlb_get_required_mask(struct device *dev);
+
#endif /* __LINUX_SWIOTLB_XEN_H */
--
1.7.10.4
Use dma_ops->get_required_mask() if provided, defaulting to
dma_get_requried_mask_from_max_pfn().
This is needed on systems (such as Xen PV guests) where the DMA
address and the physical address are not equal.
ARCH_HAS_DMA_GET_REQUIRED_MASK is defined in asm/device.h instead of
asm/dma-mapping.h because linux/dma-mapping.h uses the define before
including asm/dma-mapping.h
Signed-off-by: David Vrabel <[email protected]>
Reviewed-by: Stefano Stabellini <[email protected]>
---
arch/x86/include/asm/device.h | 2 ++
arch/x86/kernel/pci-dma.c | 8 ++++++++
2 files changed, 10 insertions(+)
diff --git a/arch/x86/include/asm/device.h b/arch/x86/include/asm/device.h
index 03dd729..10bc628 100644
--- a/arch/x86/include/asm/device.h
+++ b/arch/x86/include/asm/device.h
@@ -13,4 +13,6 @@ struct dev_archdata {
struct pdev_archdata {
};
+#define ARCH_HAS_DMA_GET_REQUIRED_MASK
+
#endif /* _ASM_X86_DEVICE_H */
diff --git a/arch/x86/kernel/pci-dma.c b/arch/x86/kernel/pci-dma.c
index a25e202..5154400 100644
--- a/arch/x86/kernel/pci-dma.c
+++ b/arch/x86/kernel/pci-dma.c
@@ -140,6 +140,14 @@ void dma_generic_free_coherent(struct device *dev, size_t size, void *vaddr,
free_pages((unsigned long)vaddr, get_order(size));
}
+u64 dma_get_required_mask(struct device *dev)
+{
+ if (dma_ops->get_required_mask)
+ return dma_ops->get_required_mask(dev);
+ return dma_get_required_mask_from_max_pfn(dev);
+}
+EXPORT_SYMBOL_GPL(dma_get_required_mask);
+
/*
* See <Documentation/x86/x86_64/boot-options.txt> for the iommu kernel
* parameter documentation.
--
1.7.10.4
On Wed, 19 Nov 2014, David Vrabel wrote:
> Signed-off-by: David Vrabel <[email protected]>
> Cc: Tony Luck <[email protected]>
> Cc: Fenghua Yu <[email protected]>
> Cc: [email protected]
Reviewed-by: Stefano Stabellini <[email protected]>
> arch/ia64/include/asm/machvec.h | 2 +-
> arch/ia64/include/asm/machvec_init.h | 1 -
> arch/ia64/pci/pci.c | 20 --------------------
> 3 files changed, 1 insertion(+), 22 deletions(-)
>
> diff --git a/arch/ia64/include/asm/machvec.h b/arch/ia64/include/asm/machvec.h
> index 9c39bdf..beaa47d 100644
> --- a/arch/ia64/include/asm/machvec.h
> +++ b/arch/ia64/include/asm/machvec.h
> @@ -287,7 +287,7 @@ extern struct dma_map_ops *dma_get_ops(struct device *);
> # define platform_dma_get_ops dma_get_ops
> #endif
> #ifndef platform_dma_get_required_mask
> -# define platform_dma_get_required_mask ia64_dma_get_required_mask
> +# define platform_dma_get_required_mask dma_get_required_mask_from_max_pfn
> #endif
> #ifndef platform_irq_to_vector
> # define platform_irq_to_vector __ia64_irq_to_vector
> diff --git a/arch/ia64/include/asm/machvec_init.h b/arch/ia64/include/asm/machvec_init.h
> index 37a4698..ef964b2 100644
> --- a/arch/ia64/include/asm/machvec_init.h
> +++ b/arch/ia64/include/asm/machvec_init.h
> @@ -3,7 +3,6 @@
>
> extern ia64_mv_send_ipi_t ia64_send_ipi;
> extern ia64_mv_global_tlb_purge_t ia64_global_tlb_purge;
> -extern ia64_mv_dma_get_required_mask ia64_dma_get_required_mask;
> extern ia64_mv_irq_to_vector __ia64_irq_to_vector;
> extern ia64_mv_local_vector_to_irq __ia64_local_vector_to_irq;
> extern ia64_mv_pci_get_legacy_mem_t ia64_pci_get_legacy_mem;
> diff --git a/arch/ia64/pci/pci.c b/arch/ia64/pci/pci.c
> index 291a582..79da21b 100644
> --- a/arch/ia64/pci/pci.c
> +++ b/arch/ia64/pci/pci.c
> @@ -791,26 +791,6 @@ static void __init set_pci_dfl_cacheline_size(void)
> pci_dfl_cache_line_size = (1 << cci.pcci_line_size) / 4;
> }
>
> -u64 ia64_dma_get_required_mask(struct device *dev)
> -{
> - u32 low_totalram = ((max_pfn - 1) << PAGE_SHIFT);
> - u32 high_totalram = ((max_pfn - 1) >> (32 - PAGE_SHIFT));
> - u64 mask;
> -
> - if (!high_totalram) {
> - /* convert to mask just covering totalram */
> - low_totalram = (1 << (fls(low_totalram) - 1));
> - low_totalram += low_totalram - 1;
> - mask = low_totalram;
> - } else {
> - high_totalram = (1 << (fls(high_totalram) - 1));
> - high_totalram += high_totalram - 1;
> - mask = (((u64)high_totalram) << 32) + 0xffffffff;
> - }
> - return mask;
> -}
> -EXPORT_SYMBOL_GPL(ia64_dma_get_required_mask);
> -
> u64 dma_get_required_mask(struct device *dev)
> {
> return platform_dma_get_required_mask(dev);
> --
> 1.7.10.4
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to [email protected]
> More majordomo info at http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at http://secure-web.cisco.com/1uzdEOuzPh9ddYCppJ7edARD7taQXur82_EMioIJqXcGS1lEgfETQB2j546iHGLqo8mraFv4u9YxUpICa6DurqoTbYGXFrH14KuGQfFFzn4DHYx5HIksjcOqO2hiw74xfemY9frjnyDwhuBoBc3quJ5I8zLhf8kRz1AJGBKOKY_o/http%3A%2F%2Fwww.tux.org%2Flkml%2F
>
On Wed, 19 Nov 2014, David Vrabel wrote:
> On a Xen PV guest the DMA addresses and physical addresses are not 1:1
> (such as Xen PV guests) and the generic dma_get_required_mask() does
> not return the correct mask (since it uses max_pfn).
>
> Some device drivers (such as mptsas, mpt2sas) use
> dma_get_required_mask() to set the device's DMA mask to allow them to
> use only 32-bit DMA addresses in hardware structures. This results in
> unnecessary use of the SWIOTLB if DMA addresses are more than 32-bits,
> impacting performance significantly.
>
> Provide a get_required_mask op that uses the maximum MFN to calculate
> the DMA mask.
>
> Signed-off-by: David Vrabel <[email protected]>
> ---
> arch/x86/xen/pci-swiotlb-xen.c | 1 +
> drivers/xen/swiotlb-xen.c | 13 +++++++++++++
> include/xen/swiotlb-xen.h | 4 ++++
> 3 files changed, 18 insertions(+)
>
> diff --git a/arch/x86/xen/pci-swiotlb-xen.c b/arch/x86/xen/pci-swiotlb-xen.c
> index 0e98e5d..a5d180a 100644
> --- a/arch/x86/xen/pci-swiotlb-xen.c
> +++ b/arch/x86/xen/pci-swiotlb-xen.c
> @@ -31,6 +31,7 @@ static struct dma_map_ops xen_swiotlb_dma_ops = {
> .map_page = xen_swiotlb_map_page,
> .unmap_page = xen_swiotlb_unmap_page,
> .dma_supported = xen_swiotlb_dma_supported,
> + .get_required_mask = xen_swiotlb_get_required_mask,
> };
>
> /*
> diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c
> index ebd8f21..654587d 100644
> --- a/drivers/xen/swiotlb-xen.c
> +++ b/drivers/xen/swiotlb-xen.c
> @@ -42,9 +42,11 @@
> #include <xen/page.h>
> #include <xen/xen-ops.h>
> #include <xen/hvc-console.h>
> +#include <xen/interface/memory.h>
>
> #include <asm/dma-mapping.h>
> #include <asm/xen/page-coherent.h>
> +#include <asm/xen/hypercall.h>
>
> #include <trace/events/swiotlb.h>
> /*
> @@ -683,3 +685,14 @@ xen_swiotlb_set_dma_mask(struct device *dev, u64 dma_mask)
> return 0;
> }
> EXPORT_SYMBOL_GPL(xen_swiotlb_set_dma_mask);
> +
> +u64
> +xen_swiotlb_get_required_mask(struct device *dev)
> +{
> + unsigned long max_mfn;
> +
> + max_mfn = HYPERVISOR_memory_op(XENMEM_maximum_ram_page, NULL);
As Jan pointed out, I think you need to change the prototype of
HYPERVISOR_memory_op to return long. Please do consistently across all
relevant archs.
> + return DMA_BIT_MASK(fls_long(max_mfn - 1) + PAGE_SHIFT);
> +}
> +EXPORT_SYMBOL_GPL(xen_swiotlb_get_required_mask);
> diff --git a/include/xen/swiotlb-xen.h b/include/xen/swiotlb-xen.h
> index 8b2eb93..6408888 100644
> --- a/include/xen/swiotlb-xen.h
> +++ b/include/xen/swiotlb-xen.h
> @@ -58,4 +58,8 @@ xen_swiotlb_dma_supported(struct device *hwdev, u64 mask);
>
> extern int
> xen_swiotlb_set_dma_mask(struct device *dev, u64 dma_mask);
> +
> +extern u64
> +xen_swiotlb_get_required_mask(struct device *dev);
> +
> #endif /* __LINUX_SWIOTLB_XEN_H */
> --
> 1.7.10.4
>
>
> _______________________________________________
> Xen-devel mailing list
> [email protected]
> http://lists.xen.org/xen-devel
>
>>> On 19.11.14 at 17:02, <[email protected]> wrote:
> On a Xen PV guest the DMA addresses and physical addresses are not 1:1
> (such as Xen PV guests) and the generic dma_get_required_mask() does
> not return the correct mask (since it uses max_pfn).
>
> Some device drivers (such as mptsas, mpt2sas) use
> dma_get_required_mask() to set the device's DMA mask to allow them to
> use only 32-bit DMA addresses in hardware structures. This results in
> unnecessary use of the SWIOTLB if DMA addresses are more than 32-bits,
> impacting performance significantly.
>
> Provide a get_required_mask op that uses the maximum MFN to calculate
> the DMA mask.
>
> Signed-off-by: David Vrabel <[email protected]>
Reviewed-by: Jan Beulich <[email protected]>
On 19/11/14 17:51, Stefano Stabellini wrote:
> On Wed, 19 Nov 2014, David Vrabel wrote:
>>
>> +u64
>> +xen_swiotlb_get_required_mask(struct device *dev)
>> +{
>> + unsigned long max_mfn;
>> +
>> + max_mfn = HYPERVISOR_memory_op(XENMEM_maximum_ram_page, NULL);
>
> As Jan pointed out, I think you need to change the prototype of
> HYPERVISOR_memory_op to return long. Please do consistently across all
> relevant archs.
This doesn't help since 32-bit guests will still truncate. A new
hypercall op that returns the result in a uint64_t parameter is required.
There is another reason why max_mfn isn't suitable -- IOMMU usage so I
think we should assume a 64-bit DMA mask is required (this is actually
the change I put into XenServer's kernel).
David