A lot of architectures have essentially identical dma_map_ops
implementations to use swiotlb. This series adds new generic
swiotlb_alloc/free helpers that take the attrs argument exposed
in dma_map_ops, and which do an enhanced direct allocation
modelled after x86 and reused from the dma-direct code, and
then switches most architectures over to it. The only exceptions
are mips, which requires additional cache flushing which will
need a new abstraction, and x86 itself which will be handled in
a later series with other x86 dma mapping changes.
To support the generic code a few architectures that currently
use ZONE_DMA/GFP_DMA for <= 32-bit allocations are switched to
implement ZONE_DMA32 instead.
This series is based on the previously sent series to consolidate
the direct dma mapping implementation. A git tree with this
series as well as the prerequisites is available here:
git://git.infradead.org/users/hch/misc.git swiotlb
Gitweb:
http://git.infradead.org/users/hch/misc.git/shortlog/refs/heads/swiotlb
From: Christian König <[email protected]>
TTM tries to allocate coherent memory in chunks of 2MB first to improve
TLB efficiency and falls back to allocating 4K pages if that fails.
Suppress the warning when the 2MB allocations fails since there is a
valid fall back path.
Signed-off-by: Christian König <[email protected]>
Reported-by: Mike Galbraith <[email protected]>
Acked-by: Konrad Rzeszutek Wilk <[email protected]>
Bug: https://bugs.freedesktop.org/show_bug.cgi?id=104082
CC: [email protected]
Signed-off-by: Christoph Hellwig <[email protected]>
---
lib/swiotlb.c | 15 +++++++++------
1 file changed, 9 insertions(+), 6 deletions(-)
diff --git a/lib/swiotlb.c b/lib/swiotlb.c
index 6583f3512386..125c1062119f 100644
--- a/lib/swiotlb.c
+++ b/lib/swiotlb.c
@@ -586,7 +586,7 @@ phys_addr_t swiotlb_tbl_map_single(struct device *hwdev,
not_found:
spin_unlock_irqrestore(&io_tlb_lock, flags);
- if (printk_ratelimit())
+ if (!(attrs & DMA_ATTR_NO_WARN) && printk_ratelimit())
dev_warn(hwdev, "swiotlb buffer is full (sz: %zd bytes)\n", size);
return SWIOTLB_MAP_ERROR;
found:
@@ -713,6 +713,7 @@ void *
swiotlb_alloc_coherent(struct device *hwdev, size_t size,
dma_addr_t *dma_handle, gfp_t flags)
{
+ bool warn = !(flags & __GFP_NOWARN);
dma_addr_t dev_addr;
void *ret;
int order = get_order(size);
@@ -738,8 +739,8 @@ swiotlb_alloc_coherent(struct device *hwdev, size_t size,
* GFP_DMA memory; fall back on map_single(), which
* will grab memory from the lowest available address range.
*/
- phys_addr_t paddr = map_single(hwdev, 0, size,
- DMA_FROM_DEVICE, 0);
+ phys_addr_t paddr = map_single(hwdev, 0, size, DMA_FROM_DEVICE,
+ warn ? 0 : DMA_ATTR_NO_WARN);
if (paddr == SWIOTLB_MAP_ERROR)
goto err_warn;
@@ -769,9 +770,11 @@ swiotlb_alloc_coherent(struct device *hwdev, size_t size,
return ret;
err_warn:
- pr_warn("swiotlb: coherent allocation failed for device %s size=%zu\n",
- dev_name(hwdev), size);
- dump_stack();
+ if (warn && printk_ratelimit()) {
+ pr_warn("swiotlb: coherent allocation failed for device %s size=%zu\n",
+ dev_name(hwdev), size);
+ dump_stack();
+ }
return NULL;
}
--
2.14.2
We'll need that name for a generic implementation soon.
Signed-off-by: Christoph Hellwig <[email protected]>
---
arch/arm64/mm/dma-mapping.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c
index f3a637b98487..6840426bbe77 100644
--- a/arch/arm64/mm/dma-mapping.c
+++ b/arch/arm64/mm/dma-mapping.c
@@ -368,7 +368,7 @@ static int __swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t addr)
return 0;
}
-static const struct dma_map_ops swiotlb_dma_ops = {
+static const struct dma_map_ops arm64_swiotlb_dma_ops = {
.alloc = __dma_alloc,
.free = __dma_free,
.mmap = __swiotlb_mmap,
@@ -923,7 +923,7 @@ void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
const struct iommu_ops *iommu, bool coherent)
{
if (!dev->dma_ops)
- dev->dma_ops = &swiotlb_dma_ops;
+ dev->dma_ops = &arm64_swiotlb_dma_ops;
dev->archdata.dma_coherent = coherent;
__iommu_setup_dma_ops(dev, dma_base, size, iommu);
--
2.14.2
We'll need that name for a generic implementation soon.
Signed-off-by: Christoph Hellwig <[email protected]>
---
arch/powerpc/include/asm/swiotlb.h | 2 +-
arch/powerpc/kernel/dma-swiotlb.c | 4 ++--
arch/powerpc/kernel/dma.c | 2 +-
arch/powerpc/sysdev/fsl_pci.c | 2 +-
4 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/arch/powerpc/include/asm/swiotlb.h b/arch/powerpc/include/asm/swiotlb.h
index 9341ee804d19..f65ecf57b66c 100644
--- a/arch/powerpc/include/asm/swiotlb.h
+++ b/arch/powerpc/include/asm/swiotlb.h
@@ -13,7 +13,7 @@
#include <linux/swiotlb.h>
-extern const struct dma_map_ops swiotlb_dma_ops;
+extern const struct dma_map_ops powerpc_swiotlb_dma_ops;
extern unsigned int ppc_swiotlb_enable;
int __init swiotlb_setup_bus_notifier(void);
diff --git a/arch/powerpc/kernel/dma-swiotlb.c b/arch/powerpc/kernel/dma-swiotlb.c
index f1e99b9cee97..506ac4fafac5 100644
--- a/arch/powerpc/kernel/dma-swiotlb.c
+++ b/arch/powerpc/kernel/dma-swiotlb.c
@@ -46,7 +46,7 @@ static u64 swiotlb_powerpc_get_required(struct device *dev)
* map_page, and unmap_page on highmem, use normal dma_ops
* for everything else.
*/
-const struct dma_map_ops swiotlb_dma_ops = {
+const struct dma_map_ops powerpc_swiotlb_dma_ops = {
.alloc = __dma_nommu_alloc_coherent,
.free = __dma_nommu_free_coherent,
.mmap = dma_nommu_mmap_coherent,
@@ -89,7 +89,7 @@ static int ppc_swiotlb_bus_notify(struct notifier_block *nb,
/* May need to bounce if the device can't address all of DRAM */
if ((dma_get_mask(dev) + 1) < memblock_end_of_DRAM())
- set_dma_ops(dev, &swiotlb_dma_ops);
+ set_dma_ops(dev, &powerpc_swiotlb_dma_ops);
return NOTIFY_DONE;
}
diff --git a/arch/powerpc/kernel/dma.c b/arch/powerpc/kernel/dma.c
index 76079841d3d0..da20569de9d4 100644
--- a/arch/powerpc/kernel/dma.c
+++ b/arch/powerpc/kernel/dma.c
@@ -33,7 +33,7 @@ static u64 __maybe_unused get_pfn_limit(struct device *dev)
struct dev_archdata __maybe_unused *sd = &dev->archdata;
#ifdef CONFIG_SWIOTLB
- if (sd->max_direct_dma_addr && dev->dma_ops == &swiotlb_dma_ops)
+ if (sd->max_direct_dma_addr && dev->dma_ops == &powerpc_swiotlb_dma_ops)
pfn = min_t(u64, pfn, sd->max_direct_dma_addr >> PAGE_SHIFT);
#endif
diff --git a/arch/powerpc/sysdev/fsl_pci.c b/arch/powerpc/sysdev/fsl_pci.c
index e4d0133bbeeb..61e07c78d64f 100644
--- a/arch/powerpc/sysdev/fsl_pci.c
+++ b/arch/powerpc/sysdev/fsl_pci.c
@@ -118,7 +118,7 @@ static void setup_swiotlb_ops(struct pci_controller *hose)
{
if (ppc_swiotlb_enable) {
hose->controller_ops.dma_dev_setup = pci_dma_dev_setup_swiotlb;
- set_pci_dma_ops(&swiotlb_dma_ops);
+ set_pci_dma_ops(&powerpc_swiotlb_dma_ops);
}
}
#else
--
2.14.2
Currently all architectures that want to use swiotlb have to implement
their own dma_map_ops instances. Provide a generic one based on the
x86 implementation which first calls into dma_direct to try a full blown
direct mapping implementation (including e.g. CMA) before falling back
allocating from the swiotlb buffer.
Signed-off-by: Christoph Hellwig <[email protected]>
---
include/linux/swiotlb.h | 8 ++++++++
lib/swiotlb.c | 43 +++++++++++++++++++++++++++++++++++++++++++
2 files changed, 51 insertions(+)
diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h
index 606375e35d87..5b1f2a00491c 100644
--- a/include/linux/swiotlb.h
+++ b/include/linux/swiotlb.h
@@ -66,6 +66,12 @@ extern void swiotlb_tbl_sync_single(struct device *hwdev,
enum dma_sync_target target);
/* Accessory functions. */
+
+void *swiotlb_alloc(struct device *hwdev, size_t size, dma_addr_t *dma_handle,
+ gfp_t flags, unsigned long attrs);
+void swiotlb_free(struct device *dev, size_t size, void *vaddr,
+ dma_addr_t dma_addr, unsigned long attrs);
+
extern void
*swiotlb_alloc_coherent(struct device *hwdev, size_t size,
dma_addr_t *dma_handle, gfp_t flags);
@@ -126,4 +132,6 @@ extern void swiotlb_print_info(void);
extern int is_swiotlb_buffer(phys_addr_t paddr);
extern void swiotlb_set_max_segment(unsigned int);
+extern const struct dma_map_ops swiotlb_dma_ops;
+
#endif /* __LINUX_SWIOTLB_H */
diff --git a/lib/swiotlb.c b/lib/swiotlb.c
index cf5311908fa9..0fae2f45c3c0 100644
--- a/lib/swiotlb.c
+++ b/lib/swiotlb.c
@@ -1087,3 +1087,46 @@ swiotlb_dma_supported(struct device *hwdev, u64 mask)
return swiotlb_phys_to_dma(hwdev, io_tlb_end - 1) <= mask;
}
EXPORT_SYMBOL(swiotlb_dma_supported);
+
+#ifdef CONFIG_DMA_DIRECT_OPS
+void *swiotlb_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
+ gfp_t gfp, unsigned long attrs)
+{
+ void *vaddr;
+
+ /*
+ * Don't print a warning when the first allocation attempt fails.
+ * swiotlb_alloc_coherent() will print a warning when the DMA memory
+ * allocation ultimately failed.
+ */
+ gfp |= __GFP_NOWARN;
+
+ vaddr = dma_direct_alloc(dev, size, dma_handle, gfp, attrs);
+ if (!vaddr)
+ vaddr = swiotlb_alloc_coherent(dev, size, dma_handle, gfp);
+ return vaddr;
+}
+
+void swiotlb_free(struct device *dev, size_t size, void *vaddr,
+ dma_addr_t dma_addr, unsigned long attrs)
+{
+ if (is_swiotlb_buffer(dma_to_phys(dev, dma_addr)))
+ swiotlb_free_coherent(dev, size, vaddr, dma_addr);
+ else
+ dma_direct_free(dev, size, vaddr, dma_addr, attrs);
+}
+
+const struct dma_map_ops swiotlb_dma_ops = {
+ .mapping_error = swiotlb_dma_mapping_error,
+ .alloc = swiotlb_alloc,
+ .free = swiotlb_free,
+ .sync_single_for_cpu = swiotlb_sync_single_for_cpu,
+ .sync_single_for_device = swiotlb_sync_single_for_device,
+ .sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
+ .sync_sg_for_device = swiotlb_sync_sg_for_device,
+ .map_sg = swiotlb_map_sg_attrs,
+ .unmap_sg = swiotlb_unmap_sg_attrs,
+ .map_page = swiotlb_map_page,
+ .unmap_page = swiotlb_unmap_page,
+};
+#endif /* CONFIG_DMA_DIRECT_OPS */
--
2.14.2
Factor out a new swiotlb_free_buffer helper that checks if an address
is allocated from the swiotlb bounce buffer, and if yes frees it.
This allows to simplify the swiotlb_free implemenation that uses
dma_direct_free to free the non-bounce buffer allocations.
Signed-off-by: Christoph Hellwig <[email protected]>
---
lib/swiotlb.c | 35 +++++++++++++++++++++--------------
1 file changed, 21 insertions(+), 14 deletions(-)
diff --git a/lib/swiotlb.c b/lib/swiotlb.c
index 539fd1099ba9..1a147f1354a1 100644
--- a/lib/swiotlb.c
+++ b/lib/swiotlb.c
@@ -780,22 +780,31 @@ swiotlb_alloc_coherent(struct device *hwdev, size_t size,
}
EXPORT_SYMBOL(swiotlb_alloc_coherent);
+static bool swiotlb_free_buffer(struct device *dev, size_t size,
+ dma_addr_t dma_addr)
+{
+ phys_addr_t phys_addr = dma_to_phys(dev, dma_addr);
+
+ WARN_ON_ONCE(irqs_disabled());
+
+ if (!is_swiotlb_buffer(phys_addr))
+ return false;
+
+ /*
+ * DMA_TO_DEVICE to avoid memcpy in swiotlb_tbl_unmap_single.
+ * DMA_ATTR_SKIP_CPU_SYNC is optional.
+ */
+ swiotlb_tbl_unmap_single(dev, phys_addr, size, DMA_TO_DEVICE,
+ DMA_ATTR_SKIP_CPU_SYNC);
+ return true;
+}
+
void
swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr,
dma_addr_t dev_addr)
{
- phys_addr_t paddr = dma_to_phys(hwdev, dev_addr);
-
- WARN_ON(irqs_disabled());
- if (!is_swiotlb_buffer(paddr))
+ if (!swiotlb_free_buffer(hwdev, size, dev_addr))
free_pages((unsigned long)vaddr, get_order(size));
- else
- /*
- * DMA_TO_DEVICE to avoid memcpy in swiotlb_tbl_unmap_single.
- * DMA_ATTR_SKIP_CPU_SYNC is optional.
- */
- swiotlb_tbl_unmap_single(hwdev, paddr, size, DMA_TO_DEVICE,
- DMA_ATTR_SKIP_CPU_SYNC);
}
EXPORT_SYMBOL(swiotlb_free_coherent);
@@ -1110,9 +1119,7 @@ void *swiotlb_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
void swiotlb_free(struct device *dev, size_t size, void *vaddr,
dma_addr_t dma_addr, unsigned long attrs)
{
- if (is_swiotlb_buffer(dma_to_phys(dev, dma_addr)))
- swiotlb_free_coherent(dev, size, vaddr, dma_addr);
- else
+ if (!swiotlb_free_buffer(dev, size, dma_addr))
dma_direct_free(dev, size, vaddr, dma_addr, attrs);
}
--
2.14.2
All these symbols are only used by arch dma_ops implementations or
xen-swiotlb. None of which can be modular.
Signed-off-by: Christoph Hellwig <[email protected]>
---
lib/swiotlb.c | 13 -------------
1 file changed, 13 deletions(-)
diff --git a/lib/swiotlb.c b/lib/swiotlb.c
index bf2d19ee91c1..1eac51ff77a4 100644
--- a/lib/swiotlb.c
+++ b/lib/swiotlb.c
@@ -605,7 +605,6 @@ phys_addr_t swiotlb_tbl_map_single(struct device *hwdev,
return tlb_addr;
}
-EXPORT_SYMBOL_GPL(swiotlb_tbl_map_single);
/*
* Allocates bounce buffer and returns its kernel virtual address.
@@ -675,7 +674,6 @@ void swiotlb_tbl_unmap_single(struct device *hwdev, phys_addr_t tlb_addr,
}
spin_unlock_irqrestore(&io_tlb_lock, flags);
}
-EXPORT_SYMBOL_GPL(swiotlb_tbl_unmap_single);
void swiotlb_tbl_sync_single(struct device *hwdev, phys_addr_t tlb_addr,
size_t size, enum dma_data_direction dir,
@@ -707,7 +705,6 @@ void swiotlb_tbl_sync_single(struct device *hwdev, phys_addr_t tlb_addr,
BUG();
}
}
-EXPORT_SYMBOL_GPL(swiotlb_tbl_sync_single);
static inline bool dma_coherent_ok(struct device *dev, dma_addr_t addr,
size_t size)
@@ -884,7 +881,6 @@ dma_addr_t swiotlb_map_page(struct device *dev, struct page *page,
return swiotlb_phys_to_dma(dev, io_tlb_overflow_buffer);
}
-EXPORT_SYMBOL_GPL(swiotlb_map_page);
/*
* Unmap a single streaming mode DMA translation. The dma_addr and size must
@@ -925,7 +921,6 @@ void swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr,
{
unmap_single(hwdev, dev_addr, size, dir, attrs);
}
-EXPORT_SYMBOL_GPL(swiotlb_unmap_page);
/*
* Make physical memory consistent for a single streaming mode DMA translation
@@ -963,7 +958,6 @@ swiotlb_sync_single_for_cpu(struct device *hwdev, dma_addr_t dev_addr,
{
swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_CPU);
}
-EXPORT_SYMBOL(swiotlb_sync_single_for_cpu);
void
swiotlb_sync_single_for_device(struct device *hwdev, dma_addr_t dev_addr,
@@ -971,7 +965,6 @@ swiotlb_sync_single_for_device(struct device *hwdev, dma_addr_t dev_addr,
{
swiotlb_sync_single(hwdev, dev_addr, size, dir, SYNC_FOR_DEVICE);
}
-EXPORT_SYMBOL(swiotlb_sync_single_for_device);
/*
* Map a set of buffers described by scatterlist in streaming mode for DMA.
@@ -1023,7 +1016,6 @@ swiotlb_map_sg_attrs(struct device *hwdev, struct scatterlist *sgl, int nelems,
}
return nelems;
}
-EXPORT_SYMBOL(swiotlb_map_sg_attrs);
/*
* Unmap a set of streaming mode DMA translations. Again, cpu read rules
@@ -1043,7 +1035,6 @@ swiotlb_unmap_sg_attrs(struct device *hwdev, struct scatterlist *sgl,
unmap_single(hwdev, sg->dma_address, sg_dma_len(sg), dir,
attrs);
}
-EXPORT_SYMBOL(swiotlb_unmap_sg_attrs);
/*
* Make physical memory consistent for a set of streaming mode DMA translations
@@ -1071,7 +1062,6 @@ swiotlb_sync_sg_for_cpu(struct device *hwdev, struct scatterlist *sg,
{
swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_CPU);
}
-EXPORT_SYMBOL(swiotlb_sync_sg_for_cpu);
void
swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg,
@@ -1079,14 +1069,12 @@ swiotlb_sync_sg_for_device(struct device *hwdev, struct scatterlist *sg,
{
swiotlb_sync_sg(hwdev, sg, nelems, dir, SYNC_FOR_DEVICE);
}
-EXPORT_SYMBOL(swiotlb_sync_sg_for_device);
int
swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t dma_addr)
{
return (dma_addr == swiotlb_phys_to_dma(hwdev, io_tlb_overflow_buffer));
}
-EXPORT_SYMBOL(swiotlb_dma_mapping_error);
/*
* Return whether the given device DMA address mask can be supported
@@ -1099,7 +1087,6 @@ swiotlb_dma_supported(struct device *hwdev, u64 mask)
{
return swiotlb_phys_to_dma(hwdev, io_tlb_end - 1) <= mask;
}
-EXPORT_SYMBOL(swiotlb_dma_supported);
#ifdef CONFIG_DMA_DIRECT_OPS
void *swiotlb_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
--
2.14.2
These are identical to the ia64 ops, and would also support CMA
if enabled on ia64.
Signed-off-by: Christoph Hellwig <[email protected]>
---
arch/ia64/Kconfig | 5 +++++
arch/ia64/hp/common/hwsw_iommu.c | 4 ++--
arch/ia64/hp/common/sba_iommu.c | 6 +++---
arch/ia64/kernel/pci-swiotlb.c | 38 +++-----------------------------------
4 files changed, 13 insertions(+), 40 deletions(-)
diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig
index 888acdb163cb..29148fe4bf5a 100644
--- a/arch/ia64/Kconfig
+++ b/arch/ia64/Kconfig
@@ -146,6 +146,7 @@ config IA64_GENERIC
bool "generic"
select NUMA
select ACPI_NUMA
+ select DMA_DIRECT_OPS
select SWIOTLB
select PCI_MSI
help
@@ -166,6 +167,7 @@ config IA64_GENERIC
config IA64_DIG
bool "DIG-compliant"
+ select DMA_DIRECT_OPS
select SWIOTLB
config IA64_DIG_VTD
@@ -181,6 +183,7 @@ config IA64_HP_ZX1
config IA64_HP_ZX1_SWIOTLB
bool "HP-zx1/sx1000 with software I/O TLB"
+ select DMA_DIRECT_OPS
select SWIOTLB
help
Build a kernel that runs on HP zx1 and sx1000 systems even when they
@@ -204,6 +207,7 @@ config IA64_SGI_UV
bool "SGI-UV"
select NUMA
select ACPI_NUMA
+ select DMA_DIRECT_OPS
select SWIOTLB
help
Selecting this option will optimize the kernel for use on UV based
@@ -214,6 +218,7 @@ config IA64_SGI_UV
config IA64_HP_SIM
bool "Ski-simulator"
+ select DMA_DIRECT_OPS
select SWIOTLB
depends on !PM
diff --git a/arch/ia64/hp/common/hwsw_iommu.c b/arch/ia64/hp/common/hwsw_iommu.c
index 41279f0442bd..58969039bed2 100644
--- a/arch/ia64/hp/common/hwsw_iommu.c
+++ b/arch/ia64/hp/common/hwsw_iommu.c
@@ -19,7 +19,7 @@
#include <linux/export.h>
#include <asm/machvec.h>
-extern const struct dma_map_ops sba_dma_ops, ia64_swiotlb_dma_ops;
+extern const struct dma_map_ops sba_dma_ops;
/* swiotlb declarations & definitions: */
extern int swiotlb_late_init_with_default_size (size_t size);
@@ -38,7 +38,7 @@ static inline int use_swiotlb(struct device *dev)
const struct dma_map_ops *hwsw_dma_get_ops(struct device *dev)
{
if (use_swiotlb(dev))
- return &ia64_swiotlb_dma_ops;
+ return &swiotlb_dma_ops;
return &sba_dma_ops;
}
EXPORT_SYMBOL(hwsw_dma_get_ops);
diff --git a/arch/ia64/hp/common/sba_iommu.c b/arch/ia64/hp/common/sba_iommu.c
index 8c0a9ae6afec..aec4a3354abe 100644
--- a/arch/ia64/hp/common/sba_iommu.c
+++ b/arch/ia64/hp/common/sba_iommu.c
@@ -2096,7 +2096,7 @@ static int __init acpi_sba_ioc_init_acpi(void)
/* This has to run before acpi_scan_init(). */
arch_initcall(acpi_sba_ioc_init_acpi);
-extern const struct dma_map_ops ia64_swiotlb_dma_ops;
+extern const struct dma_map_ops swiotlb_dma_ops;
static int __init
sba_init(void)
@@ -2111,7 +2111,7 @@ sba_init(void)
* a successful kdump kernel boot is to use the swiotlb.
*/
if (is_kdump_kernel()) {
- dma_ops = &ia64_swiotlb_dma_ops;
+ dma_ops = &swiotlb_dma_ops;
if (swiotlb_late_init_with_default_size(64 * (1<<20)) != 0)
panic("Unable to initialize software I/O TLB:"
" Try machvec=dig boot option");
@@ -2133,7 +2133,7 @@ sba_init(void)
* If we didn't find something sba_iommu can claim, we
* need to setup the swiotlb and switch to the dig machvec.
*/
- dma_ops = &ia64_swiotlb_dma_ops;
+ dma_ops = &swiotlb_dma_ops;
if (swiotlb_late_init_with_default_size(64 * (1<<20)) != 0)
panic("Unable to find SBA IOMMU or initialize "
"software I/O TLB: Try machvec=dig boot option");
diff --git a/arch/ia64/kernel/pci-swiotlb.c b/arch/ia64/kernel/pci-swiotlb.c
index 4a9a6e58ad6a..0f8d5fbd86bd 100644
--- a/arch/ia64/kernel/pci-swiotlb.c
+++ b/arch/ia64/kernel/pci-swiotlb.c
@@ -6,8 +6,7 @@
#include <linux/cache.h>
#include <linux/module.h>
#include <linux/dma-mapping.h>
-
-#include <asm/swiotlb.h>
+#include <linux/swiotlb.h>
#include <asm/dma.h>
#include <asm/iommu.h>
#include <asm/machvec.h>
@@ -15,40 +14,9 @@
int swiotlb __read_mostly;
EXPORT_SYMBOL(swiotlb);
-static void *ia64_swiotlb_alloc_coherent(struct device *dev, size_t size,
- dma_addr_t *dma_handle, gfp_t gfp,
- unsigned long attrs)
-{
- if (dev->coherent_dma_mask != DMA_BIT_MASK(64))
- gfp |= GFP_DMA32;
- return swiotlb_alloc_coherent(dev, size, dma_handle, gfp);
-}
-
-static void ia64_swiotlb_free_coherent(struct device *dev, size_t size,
- void *vaddr, dma_addr_t dma_addr,
- unsigned long attrs)
-{
- swiotlb_free_coherent(dev, size, vaddr, dma_addr);
-}
-
-const struct dma_map_ops ia64_swiotlb_dma_ops = {
- .alloc = ia64_swiotlb_alloc_coherent,
- .free = ia64_swiotlb_free_coherent,
- .map_page = swiotlb_map_page,
- .unmap_page = swiotlb_unmap_page,
- .map_sg = swiotlb_map_sg_attrs,
- .unmap_sg = swiotlb_unmap_sg_attrs,
- .sync_single_for_cpu = swiotlb_sync_single_for_cpu,
- .sync_single_for_device = swiotlb_sync_single_for_device,
- .sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
- .sync_sg_for_device = swiotlb_sync_sg_for_device,
- .dma_supported = swiotlb_dma_supported,
- .mapping_error = swiotlb_dma_mapping_error,
-};
-
void __init swiotlb_dma_init(void)
{
- dma_ops = &ia64_swiotlb_dma_ops;
+ dma_ops = &swiotlb_dma_ops;
swiotlb_init(1);
}
@@ -60,7 +28,7 @@ void __init pci_swiotlb_init(void)
printk(KERN_INFO "PCI-DMA: Re-initialize machine vector.\n");
machvec_init("dig");
swiotlb_init(1);
- dma_ops = &ia64_swiotlb_dma_ops;
+ dma_ops = &swiotlb_dma_ops;
#else
panic("Unable to find Intel IOMMU");
#endif
--
2.14.2
These are identical to the unicore32 ops, and would also support CMA
if enabled on unicore32.
Signed-off-by: Christoph Hellwig <[email protected]>
---
arch/unicore32/include/asm/dma-mapping.h | 9 +-----
arch/unicore32/mm/Kconfig | 1 +
arch/unicore32/mm/Makefile | 2 --
arch/unicore32/mm/dma-swiotlb.c | 48 --------------------------------
4 files changed, 2 insertions(+), 58 deletions(-)
delete mode 100644 arch/unicore32/mm/dma-swiotlb.c
diff --git a/arch/unicore32/include/asm/dma-mapping.h b/arch/unicore32/include/asm/dma-mapping.h
index f2bfec273aa7..790bc2ef4af2 100644
--- a/arch/unicore32/include/asm/dma-mapping.h
+++ b/arch/unicore32/include/asm/dma-mapping.h
@@ -12,18 +12,11 @@
#ifndef __UNICORE_DMA_MAPPING_H__
#define __UNICORE_DMA_MAPPING_H__
-#ifdef __KERNEL__
-
-#include <linux/mm_types.h>
-#include <linux/scatterlist.h>
#include <linux/swiotlb.h>
-extern const struct dma_map_ops swiotlb_dma_map_ops;
-
static inline const struct dma_map_ops *get_arch_dma_ops(struct bus_type *bus)
{
- return &swiotlb_dma_map_ops;
+ return &swiotlb_dma_ops;
}
-#endif /* __KERNEL__ */
#endif
diff --git a/arch/unicore32/mm/Kconfig b/arch/unicore32/mm/Kconfig
index c256460cd363..e9154a59d561 100644
--- a/arch/unicore32/mm/Kconfig
+++ b/arch/unicore32/mm/Kconfig
@@ -42,6 +42,7 @@ config CPU_TLB_SINGLE_ENTRY_DISABLE
config SWIOTLB
def_bool y
+ select DMA_DIRECT_OPS
config IOMMU_HELPER
def_bool SWIOTLB
diff --git a/arch/unicore32/mm/Makefile b/arch/unicore32/mm/Makefile
index 681c0ef5ec9e..8106260583ab 100644
--- a/arch/unicore32/mm/Makefile
+++ b/arch/unicore32/mm/Makefile
@@ -6,8 +6,6 @@
obj-y := extable.o fault.o init.o pgd.o mmu.o
obj-y += flush.o ioremap.o
-obj-$(CONFIG_SWIOTLB) += dma-swiotlb.o
-
obj-$(CONFIG_MODULES) += proc-syms.o
obj-$(CONFIG_ALIGNMENT_TRAP) += alignment.o
diff --git a/arch/unicore32/mm/dma-swiotlb.c b/arch/unicore32/mm/dma-swiotlb.c
deleted file mode 100644
index 525413d6690e..000000000000
--- a/arch/unicore32/mm/dma-swiotlb.c
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Contains routines needed to support swiotlb for UniCore32.
- *
- * Copyright (C) 2010 Guan Xuetao
- *
- * This program is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License as published by the
- * Free Software Foundation; either version 2 of the License, or (at your
- * option) any later version.
- */
-#include <linux/pci.h>
-#include <linux/cache.h>
-#include <linux/module.h>
-#include <linux/dma-mapping.h>
-#include <linux/swiotlb.h>
-#include <linux/bootmem.h>
-
-#include <asm/dma.h>
-
-static void *unicore_swiotlb_alloc_coherent(struct device *dev, size_t size,
- dma_addr_t *dma_handle, gfp_t flags,
- unsigned long attrs)
-{
- return swiotlb_alloc_coherent(dev, size, dma_handle, flags);
-}
-
-static void unicore_swiotlb_free_coherent(struct device *dev, size_t size,
- void *vaddr, dma_addr_t dma_addr,
- unsigned long attrs)
-{
- swiotlb_free_coherent(dev, size, vaddr, dma_addr);
-}
-
-const struct dma_map_ops swiotlb_dma_map_ops = {
- .alloc = unicore_swiotlb_alloc_coherent,
- .free = unicore_swiotlb_free_coherent,
- .map_sg = swiotlb_map_sg_attrs,
- .unmap_sg = swiotlb_unmap_sg_attrs,
- .dma_supported = swiotlb_dma_supported,
- .map_page = swiotlb_map_page,
- .unmap_page = swiotlb_unmap_page,
- .sync_single_for_cpu = swiotlb_sync_single_for_cpu,
- .sync_single_for_device = swiotlb_sync_single_for_device,
- .sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
- .sync_sg_for_device = swiotlb_sync_sg_for_device,
- .mapping_error = swiotlb_dma_mapping_error,
-};
-EXPORT_SYMBOL(swiotlb_dma_map_ops);
--
2.14.2
These are identical to the tile ops, and would also support CMA
if enabled on tile.
Signed-off-by: Christoph Hellwig <[email protected]>
---
arch/tile/Kconfig | 1 +
arch/tile/kernel/pci-dma.c | 36 +++---------------------------------
2 files changed, 4 insertions(+), 33 deletions(-)
diff --git a/arch/tile/Kconfig b/arch/tile/Kconfig
index 30c586686f29..ef9d403cbbe4 100644
--- a/arch/tile/Kconfig
+++ b/arch/tile/Kconfig
@@ -261,6 +261,7 @@ config NEED_SG_DMA_LENGTH
config SWIOTLB
bool
default TILEGX
+ select DMA_DIRECT_OPS
select IOMMU_HELPER
select NEED_SG_DMA_LENGTH
select ARCH_HAS_DMA_SET_COHERENT_MASK
diff --git a/arch/tile/kernel/pci-dma.c b/arch/tile/kernel/pci-dma.c
index a267fa740048..6a1efe5543fa 100644
--- a/arch/tile/kernel/pci-dma.c
+++ b/arch/tile/kernel/pci-dma.c
@@ -509,39 +509,9 @@ EXPORT_SYMBOL(gx_pci_dma_map_ops);
/* PCI DMA mapping functions for legacy PCI devices */
#ifdef CONFIG_SWIOTLB
-static void *tile_swiotlb_alloc_coherent(struct device *dev, size_t size,
- dma_addr_t *dma_handle, gfp_t gfp,
- unsigned long attrs)
-{
- gfp |= GFP_DMA32;
- return swiotlb_alloc_coherent(dev, size, dma_handle, gfp);
-}
-
-static void tile_swiotlb_free_coherent(struct device *dev, size_t size,
- void *vaddr, dma_addr_t dma_addr,
- unsigned long attrs)
-{
- swiotlb_free_coherent(dev, size, vaddr, dma_addr);
-}
-
-static const struct dma_map_ops pci_swiotlb_dma_ops = {
- .alloc = tile_swiotlb_alloc_coherent,
- .free = tile_swiotlb_free_coherent,
- .map_page = swiotlb_map_page,
- .unmap_page = swiotlb_unmap_page,
- .map_sg = swiotlb_map_sg_attrs,
- .unmap_sg = swiotlb_unmap_sg_attrs,
- .sync_single_for_cpu = swiotlb_sync_single_for_cpu,
- .sync_single_for_device = swiotlb_sync_single_for_device,
- .sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
- .sync_sg_for_device = swiotlb_sync_sg_for_device,
- .dma_supported = swiotlb_dma_supported,
- .mapping_error = swiotlb_dma_mapping_error,
-};
-
static const struct dma_map_ops pci_hybrid_dma_ops = {
- .alloc = tile_swiotlb_alloc_coherent,
- .free = tile_swiotlb_free_coherent,
+ .alloc = swiotlb_alloc,
+ .free = swiotlb_free,
.map_page = tile_pci_dma_map_page,
.unmap_page = tile_pci_dma_unmap_page,
.map_sg = tile_pci_dma_map_sg,
@@ -552,7 +522,7 @@ static const struct dma_map_ops pci_hybrid_dma_ops = {
.sync_sg_for_device = tile_pci_dma_sync_sg_for_device,
};
-const struct dma_map_ops *gx_legacy_pci_dma_map_ops = &pci_swiotlb_dma_ops;
+const struct dma_map_ops *gx_legacy_pci_dma_map_ops = &swiotlb_dma_ops;
const struct dma_map_ops *gx_hybrid_pci_dma_map_ops = &pci_hybrid_dma_ops;
#else
const struct dma_map_ops *gx_legacy_pci_dma_map_ops;
--
2.14.2
nlm_swiotlb_dma_ops is unused code, so the whole swiotlb support is dead.
If it gets resurrected at some point it should use the generic
swiotlb_dma_ops instead.
Signed-off-by: Christoph Hellwig <[email protected]>
---
arch/mips/include/asm/netlogic/common.h | 3 --
arch/mips/netlogic/Kconfig | 5 --
arch/mips/netlogic/common/Makefile | 1 -
arch/mips/netlogic/common/nlm-dma.c | 94 ---------------------------------
4 files changed, 103 deletions(-)
delete mode 100644 arch/mips/netlogic/common/nlm-dma.c
diff --git a/arch/mips/include/asm/netlogic/common.h b/arch/mips/include/asm/netlogic/common.h
index a6e6cbebe046..57616649b4f3 100644
--- a/arch/mips/include/asm/netlogic/common.h
+++ b/arch/mips/include/asm/netlogic/common.h
@@ -87,9 +87,6 @@ unsigned int nlm_get_cpu_frequency(void);
extern const struct plat_smp_ops nlm_smp_ops;
extern char nlm_reset_entry[], nlm_reset_entry_end[];
-/* SWIOTLB */
-extern const struct dma_map_ops nlm_swiotlb_dma_ops;
-
extern unsigned int nlm_threads_per_core;
extern cpumask_t nlm_cpumask;
diff --git a/arch/mips/netlogic/Kconfig b/arch/mips/netlogic/Kconfig
index 8296b13affd2..7fcfc7fe9f14 100644
--- a/arch/mips/netlogic/Kconfig
+++ b/arch/mips/netlogic/Kconfig
@@ -89,9 +89,4 @@ config IOMMU_HELPER
config NEED_SG_DMA_LENGTH
bool
-config SWIOTLB
- def_bool y
- select NEED_SG_DMA_LENGTH
- select IOMMU_HELPER
-
endif
diff --git a/arch/mips/netlogic/common/Makefile b/arch/mips/netlogic/common/Makefile
index 60d00b5d748e..89f6e3f39fed 100644
--- a/arch/mips/netlogic/common/Makefile
+++ b/arch/mips/netlogic/common/Makefile
@@ -1,6 +1,5 @@
# SPDX-License-Identifier: GPL-2.0
obj-y += irq.o time.o
-obj-y += nlm-dma.o
obj-y += reset.o
obj-$(CONFIG_SMP) += smp.o smpboot.o
obj-$(CONFIG_EARLY_PRINTK) += earlycons.o
diff --git a/arch/mips/netlogic/common/nlm-dma.c b/arch/mips/netlogic/common/nlm-dma.c
deleted file mode 100644
index 49c975b6aa28..000000000000
--- a/arch/mips/netlogic/common/nlm-dma.c
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
-* Copyright (C) 2003-2013 Broadcom Corporation
-* All Rights Reserved
- *
- * This software is available to you under a choice of one of two
- * licenses. You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the Broadcom
- * license below:
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY BROADCOM ``AS IS'' AND ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
- * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED. IN NO EVENT SHALL BROADCOM OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
- * OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
- * IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-#include <linux/dma-mapping.h>
-#include <linux/scatterlist.h>
-#include <linux/bootmem.h>
-#include <linux/export.h>
-#include <linux/swiotlb.h>
-#include <linux/types.h>
-#include <linux/init.h>
-#include <linux/mm.h>
-
-#include <asm/bootinfo.h>
-
-static char *nlm_swiotlb;
-
-static void *nlm_dma_alloc_coherent(struct device *dev, size_t size,
- dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
-{
-#ifdef CONFIG_ZONE_DMA32
- if (dev->coherent_dma_mask <= DMA_BIT_MASK(32))
- gfp |= __GFP_DMA32;
-#endif
-
- /* Don't invoke OOM killer */
- gfp |= __GFP_NORETRY;
-
- return swiotlb_alloc_coherent(dev, size, dma_handle, gfp);
-}
-
-static void nlm_dma_free_coherent(struct device *dev, size_t size,
- void *vaddr, dma_addr_t dma_handle, unsigned long attrs)
-{
- swiotlb_free_coherent(dev, size, vaddr, dma_handle);
-}
-
-const struct dma_map_ops nlm_swiotlb_dma_ops = {
- .alloc = nlm_dma_alloc_coherent,
- .free = nlm_dma_free_coherent,
- .map_page = swiotlb_map_page,
- .unmap_page = swiotlb_unmap_page,
- .map_sg = swiotlb_map_sg_attrs,
- .unmap_sg = swiotlb_unmap_sg_attrs,
- .sync_single_for_cpu = swiotlb_sync_single_for_cpu,
- .sync_single_for_device = swiotlb_sync_single_for_device,
- .sync_sg_for_cpu = swiotlb_sync_sg_for_cpu,
- .sync_sg_for_device = swiotlb_sync_sg_for_device,
- .mapping_error = swiotlb_dma_mapping_error,
- .dma_supported = swiotlb_dma_supported
-};
-
-void __init plat_swiotlb_setup(void)
-{
- size_t swiotlbsize;
- unsigned long swiotlb_nslabs;
-
- swiotlbsize = 1 << 20; /* 1 MB for now */
- swiotlb_nslabs = swiotlbsize >> IO_TLB_SHIFT;
- swiotlb_nslabs = ALIGN(swiotlb_nslabs, IO_TLB_SEGSIZE);
- swiotlbsize = swiotlb_nslabs << IO_TLB_SHIFT;
-
- nlm_swiotlb = alloc_bootmem_low_pages(swiotlbsize);
- swiotlb_init_with_tbl(nlm_swiotlb, swiotlb_nslabs, 1);
-}
--
2.14.2
arm64 uses ZONE_DMA for allocations below 32-bits. These days we
name the zone for that ZONE_DMA32, which will allow to use the
dma-direct and generic swiotlb code as-is, so rename it.
Signed-off-by: Christoph Hellwig <[email protected]>
---
arch/arm64/Kconfig | 2 +-
arch/arm64/mm/dma-mapping.c | 6 +++---
arch/arm64/mm/init.c | 16 ++++++++--------
3 files changed, 12 insertions(+), 12 deletions(-)
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index c9a7e9e1414f..6b6985f15d02 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -227,7 +227,7 @@ config GENERIC_CSUM
config GENERIC_CALIBRATE_DELAY
def_bool y
-config ZONE_DMA
+config ZONE_DMA32
def_bool y
config HAVE_GENERIC_GUP
diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c
index 6840426bbe77..0d641875b20e 100644
--- a/arch/arm64/mm/dma-mapping.c
+++ b/arch/arm64/mm/dma-mapping.c
@@ -95,9 +95,9 @@ static void *__dma_alloc_coherent(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t flags,
unsigned long attrs)
{
- if (IS_ENABLED(CONFIG_ZONE_DMA) &&
+ if (IS_ENABLED(CONFIG_ZONE_DMA32) &&
dev->coherent_dma_mask <= DMA_BIT_MASK(32))
- flags |= GFP_DMA;
+ flags |= GFP_DMA32;
if (dev_get_cma_area(dev) && gfpflags_allow_blocking(flags)) {
struct page *page;
void *addr;
@@ -397,7 +397,7 @@ static int __init atomic_pool_init(void)
page = dma_alloc_from_contiguous(NULL, nr_pages,
pool_size_order, GFP_KERNEL);
else
- page = alloc_pages(GFP_DMA, pool_size_order);
+ page = alloc_pages(GFP_DMA32, pool_size_order);
if (page) {
int ret;
diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index 00e7b900ca41..8f03276443c9 100644
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -217,7 +217,7 @@ static void __init reserve_elfcorehdr(void)
}
#endif /* CONFIG_CRASH_DUMP */
/*
- * Return the maximum physical address for ZONE_DMA (DMA_BIT_MASK(32)). It
+ * Return the maximum physical address for ZONE_DMA32 (DMA_BIT_MASK(32)). It
* currently assumes that for memory starting above 4G, 32-bit devices will
* use a DMA offset.
*/
@@ -233,8 +233,8 @@ static void __init zone_sizes_init(unsigned long min, unsigned long max)
{
unsigned long max_zone_pfns[MAX_NR_ZONES] = {0};
- if (IS_ENABLED(CONFIG_ZONE_DMA))
- max_zone_pfns[ZONE_DMA] = PFN_DOWN(max_zone_dma_phys());
+ if (IS_ENABLED(CONFIG_ZONE_DMA32))
+ max_zone_pfns[ZONE_DMA32] = PFN_DOWN(max_zone_dma_phys());
max_zone_pfns[ZONE_NORMAL] = max;
free_area_init_nodes(max_zone_pfns);
@@ -251,9 +251,9 @@ static void __init zone_sizes_init(unsigned long min, unsigned long max)
memset(zone_size, 0, sizeof(zone_size));
/* 4GB maximum for 32-bit only capable devices */
-#ifdef CONFIG_ZONE_DMA
+#ifdef CONFIG_ZONE_DMA32
max_dma = PFN_DOWN(arm64_dma_phys_limit);
- zone_size[ZONE_DMA] = max_dma - min;
+ zone_size[ZONE_DMA32] = max_dma - min;
#endif
zone_size[ZONE_NORMAL] = max - max_dma;
@@ -266,10 +266,10 @@ static void __init zone_sizes_init(unsigned long min, unsigned long max)
if (start >= max)
continue;
-#ifdef CONFIG_ZONE_DMA
+#ifdef CONFIG_ZONE_DMA32
if (start < max_dma) {
unsigned long dma_end = min(end, max_dma);
- zhole_size[ZONE_DMA] -= dma_end - start;
+ zhole_size[ZONE_DMA32] -= dma_end - start;
}
#endif
if (end > max_dma) {
@@ -467,7 +467,7 @@ void __init arm64_memblock_init(void)
early_init_fdt_scan_reserved_mem();
/* 4GB maximum for 32-bit only capable devices */
- if (IS_ENABLED(CONFIG_ZONE_DMA))
+ if (IS_ENABLED(CONFIG_ZONE_DMA32))
arm64_dma_phys_limit = max_zone_dma_phys();
else
arm64_dma_phys_limit = PHYS_MASK + 1;
--
2.14.2
The generic swiotlb_alloc and swiotlb_free routines already take care
of CMA allocations and adding GFP_DMA32 where needed, so use them
instead of the arm specific helpers.
Signed-off-by: Christoph Hellwig <[email protected]>
---
arch/arm64/Kconfig | 1 +
arch/arm64/mm/dma-mapping.c | 46 +++------------------------------------------
2 files changed, 4 insertions(+), 43 deletions(-)
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 6b6985f15d02..53205c02b18a 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -59,6 +59,7 @@ config ARM64
select COMMON_CLK
select CPU_PM if (SUSPEND || CPU_IDLE)
select DCACHE_WORD_ACCESS
+ select DMA_DIRECT_OPS
select EDAC_SUPPORT
select FRAME_POINTER
select GENERIC_ALLOCATOR
diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c
index 0d641875b20e..a96ec0181818 100644
--- a/arch/arm64/mm/dma-mapping.c
+++ b/arch/arm64/mm/dma-mapping.c
@@ -91,46 +91,6 @@ static int __free_from_pool(void *start, size_t size)
return 1;
}
-static void *__dma_alloc_coherent(struct device *dev, size_t size,
- dma_addr_t *dma_handle, gfp_t flags,
- unsigned long attrs)
-{
- if (IS_ENABLED(CONFIG_ZONE_DMA32) &&
- dev->coherent_dma_mask <= DMA_BIT_MASK(32))
- flags |= GFP_DMA32;
- if (dev_get_cma_area(dev) && gfpflags_allow_blocking(flags)) {
- struct page *page;
- void *addr;
-
- page = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT,
- get_order(size), flags);
- if (!page)
- return NULL;
-
- *dma_handle = phys_to_dma(dev, page_to_phys(page));
- addr = page_address(page);
- memset(addr, 0, size);
- return addr;
- } else {
- return swiotlb_alloc_coherent(dev, size, dma_handle, flags);
- }
-}
-
-static void __dma_free_coherent(struct device *dev, size_t size,
- void *vaddr, dma_addr_t dma_handle,
- unsigned long attrs)
-{
- bool freed;
- phys_addr_t paddr = dma_to_phys(dev, dma_handle);
-
-
- freed = dma_release_from_contiguous(dev,
- phys_to_page(paddr),
- size >> PAGE_SHIFT);
- if (!freed)
- swiotlb_free_coherent(dev, size, vaddr, dma_handle);
-}
-
static void *__dma_alloc(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t flags,
unsigned long attrs)
@@ -152,7 +112,7 @@ static void *__dma_alloc(struct device *dev, size_t size,
return addr;
}
- ptr = __dma_alloc_coherent(dev, size, dma_handle, flags, attrs);
+ ptr = swiotlb_alloc(dev, size, dma_handle, flags, attrs);
if (!ptr)
goto no_mem;
@@ -173,7 +133,7 @@ static void *__dma_alloc(struct device *dev, size_t size,
return coherent_ptr;
no_map:
- __dma_free_coherent(dev, size, ptr, *dma_handle, attrs);
+ swiotlb_free(dev, size, ptr, *dma_handle, attrs);
no_mem:
return NULL;
}
@@ -191,7 +151,7 @@ static void __dma_free(struct device *dev, size_t size,
return;
vunmap(vaddr);
}
- __dma_free_coherent(dev, size, swiotlb_addr, dma_handle, attrs);
+ swiotlb_free(dev, size, swiotlb_addr, dma_handle, attrs);
}
static dma_addr_t __swiotlb_map_page(struct device *dev, struct page *page,
--
2.14.2
These already include the GFP_DMA/GFP_DMA32 usage, and will use CMA
memory if enabled, thus avoiding the GFP_NORETRY hack.
Signed-off-by: Christoph Hellwig <[email protected]>
---
arch/mips/cavium-octeon/Kconfig | 1 +
arch/mips/cavium-octeon/dma-octeon.c | 26 +++-----------------------
arch/mips/loongson64/Kconfig | 1 +
arch/mips/loongson64/common/dma-swiotlb.c | 21 ++-------------------
4 files changed, 7 insertions(+), 42 deletions(-)
diff --git a/arch/mips/cavium-octeon/Kconfig b/arch/mips/cavium-octeon/Kconfig
index 204a1670fd9b..b5eee1a57d6c 100644
--- a/arch/mips/cavium-octeon/Kconfig
+++ b/arch/mips/cavium-octeon/Kconfig
@@ -75,6 +75,7 @@ config NEED_SG_DMA_LENGTH
config SWIOTLB
def_bool y
+ select DMA_DIRECT_OPS
select IOMMU_HELPER
select NEED_SG_DMA_LENGTH
diff --git a/arch/mips/cavium-octeon/dma-octeon.c b/arch/mips/cavium-octeon/dma-octeon.c
index 5baf79fce643..c7bb8a407041 100644
--- a/arch/mips/cavium-octeon/dma-octeon.c
+++ b/arch/mips/cavium-octeon/dma-octeon.c
@@ -159,33 +159,13 @@ static void octeon_dma_sync_sg_for_device(struct device *dev,
static void *octeon_dma_alloc_coherent(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
{
- void *ret;
-
- if (IS_ENABLED(CONFIG_ZONE_DMA) && dev == NULL)
- gfp |= __GFP_DMA;
- else if (IS_ENABLED(CONFIG_ZONE_DMA) &&
- dev->coherent_dma_mask <= DMA_BIT_MASK(24))
- gfp |= __GFP_DMA;
- else if (IS_ENABLED(CONFIG_ZONE_DMA32) &&
- dev->coherent_dma_mask <= DMA_BIT_MASK(32))
- gfp |= __GFP_DMA32;
-
- /* Don't invoke OOM killer */
- gfp |= __GFP_NORETRY;
-
- ret = swiotlb_alloc_coherent(dev, size, dma_handle, gfp);
+ void *ret = swiotlb_alloc(dev, size, dma_handle, gfp, attrs);
mb();
return ret;
}
-static void octeon_dma_free_coherent(struct device *dev, size_t size,
- void *vaddr, dma_addr_t dma_handle, unsigned long attrs)
-{
- swiotlb_free_coherent(dev, size, vaddr, dma_handle);
-}
-
static dma_addr_t octeon_unity_phys_to_dma(struct device *dev, phys_addr_t paddr)
{
return paddr;
@@ -225,7 +205,7 @@ EXPORT_SYMBOL(dma_to_phys);
static struct octeon_dma_map_ops octeon_linear_dma_map_ops = {
.dma_map_ops = {
.alloc = octeon_dma_alloc_coherent,
- .free = octeon_dma_free_coherent,
+ .free = swiotlb_free,
.map_page = octeon_dma_map_page,
.unmap_page = swiotlb_unmap_page,
.map_sg = octeon_dma_map_sg,
@@ -311,7 +291,7 @@ void __init plat_swiotlb_setup(void)
static struct octeon_dma_map_ops _octeon_pci_dma_map_ops = {
.dma_map_ops = {
.alloc = octeon_dma_alloc_coherent,
- .free = octeon_dma_free_coherent,
+ .free = swiotlb_free,
.map_page = octeon_dma_map_page,
.unmap_page = swiotlb_unmap_page,
.map_sg = octeon_dma_map_sg,
diff --git a/arch/mips/loongson64/Kconfig b/arch/mips/loongson64/Kconfig
index 0d249fc3cfe9..6f109bb54cdb 100644
--- a/arch/mips/loongson64/Kconfig
+++ b/arch/mips/loongson64/Kconfig
@@ -136,6 +136,7 @@ config SWIOTLB
bool "Soft IOMMU Support for All-Memory DMA"
default y
depends on CPU_LOONGSON3
+ select DMA_DIRECT_OPS
select IOMMU_HELPER
select NEED_SG_DMA_LENGTH
select NEED_DMA_MAP_STATE
diff --git a/arch/mips/loongson64/common/dma-swiotlb.c b/arch/mips/loongson64/common/dma-swiotlb.c
index 15388c24a504..7bbcf89475f3 100644
--- a/arch/mips/loongson64/common/dma-swiotlb.c
+++ b/arch/mips/loongson64/common/dma-swiotlb.c
@@ -13,29 +13,12 @@
static void *loongson_dma_alloc_coherent(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
{
- void *ret;
+ void *ret = swiotlb_alloc(dev, size, dma_handle, gfp, attrs);
- if ((IS_ENABLED(CONFIG_ISA) && dev == NULL) ||
- (IS_ENABLED(CONFIG_ZONE_DMA) &&
- dev->coherent_dma_mask < DMA_BIT_MASK(32)))
- gfp |= __GFP_DMA;
- else if (IS_ENABLED(CONFIG_ZONE_DMA32) &&
- dev->coherent_dma_mask < DMA_BIT_MASK(40))
- gfp |= __GFP_DMA32;
-
- gfp |= __GFP_NORETRY;
-
- ret = swiotlb_alloc_coherent(dev, size, dma_handle, gfp);
mb();
return ret;
}
-static void loongson_dma_free_coherent(struct device *dev, size_t size,
- void *vaddr, dma_addr_t dma_handle, unsigned long attrs)
-{
- swiotlb_free_coherent(dev, size, vaddr, dma_handle);
-}
-
static dma_addr_t loongson_dma_map_page(struct device *dev, struct page *page,
unsigned long offset, size_t size,
enum dma_data_direction dir,
@@ -106,7 +89,7 @@ phys_addr_t dma_to_phys(struct device *dev, dma_addr_t daddr)
static const struct dma_map_ops loongson_dma_map_ops = {
.alloc = loongson_dma_alloc_coherent,
- .free = loongson_dma_free_coherent,
+ .free = swiotlb_free,
.map_page = loongson_dma_map_page,
.unmap_page = swiotlb_unmap_page,
.map_sg = loongson_dma_map_sg,
--
2.14.2
The file is only compiled if CONFIG_INTEL_IOMMU is set to start with.
Signed-off-by: Christoph Hellwig <[email protected]>
---
arch/ia64/kernel/pci-dma.c | 7 -------
1 file changed, 7 deletions(-)
diff --git a/arch/ia64/kernel/pci-dma.c b/arch/ia64/kernel/pci-dma.c
index 35e0cad33b7d..b5df084c0af4 100644
--- a/arch/ia64/kernel/pci-dma.c
+++ b/arch/ia64/kernel/pci-dma.c
@@ -12,12 +12,7 @@
#include <asm/iommu.h>
#include <asm/machvec.h>
#include <linux/dma-mapping.h>
-
-
-#ifdef CONFIG_INTEL_IOMMU
-
#include <linux/kernel.h>
-
#include <asm/page.h>
dma_addr_t bad_dma_address __read_mostly;
@@ -115,5 +110,3 @@ void __init pci_iommu_alloc(void)
}
#endif /* CONFIG_SWIOTLB */
}
-
-#endif
--
2.14.2
tile uses ZONE_DMA for allocations below 32-bits. These days we
name the zone for that ZONE_DMA32, which will allow to use the
dma-direct and generic swiotlb code as-is, so rename it.
Signed-off-by: Christoph Hellwig <[email protected]>
---
arch/tile/Kconfig | 2 +-
arch/tile/kernel/pci-dma.c | 4 ++--
arch/tile/kernel/setup.c | 8 ++++----
3 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/arch/tile/Kconfig b/arch/tile/Kconfig
index 02f269cfa538..30c586686f29 100644
--- a/arch/tile/Kconfig
+++ b/arch/tile/Kconfig
@@ -249,7 +249,7 @@ config HIGHMEM
If unsure, say "true".
-config ZONE_DMA
+config ZONE_DMA32
def_bool y
config IOMMU_HELPER
diff --git a/arch/tile/kernel/pci-dma.c b/arch/tile/kernel/pci-dma.c
index f2abedc8a080..a267fa740048 100644
--- a/arch/tile/kernel/pci-dma.c
+++ b/arch/tile/kernel/pci-dma.c
@@ -54,7 +54,7 @@ static void *tile_dma_alloc_coherent(struct device *dev, size_t size,
* which case we will return NULL. But such devices are uncommon.
*/
if (dma_mask <= DMA_BIT_MASK(32)) {
- gfp |= GFP_DMA;
+ gfp |= GFP_DMA32;
node = 0;
}
@@ -513,7 +513,7 @@ static void *tile_swiotlb_alloc_coherent(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t gfp,
unsigned long attrs)
{
- gfp |= GFP_DMA;
+ gfp |= GFP_DMA32;
return swiotlb_alloc_coherent(dev, size, dma_handle, gfp);
}
diff --git a/arch/tile/kernel/setup.c b/arch/tile/kernel/setup.c
index ad83c1e66dbd..eb4e198f6f93 100644
--- a/arch/tile/kernel/setup.c
+++ b/arch/tile/kernel/setup.c
@@ -814,11 +814,11 @@ static void __init zone_sizes_init(void)
#endif
if (start < dma_end) {
- zones_size[ZONE_DMA] = min(zones_size[ZONE_NORMAL],
+ zones_size[ZONE_DMA32] = min(zones_size[ZONE_NORMAL],
dma_end - start);
- zones_size[ZONE_NORMAL] -= zones_size[ZONE_DMA];
+ zones_size[ZONE_NORMAL] -= zones_size[ZONE_DMA32];
} else {
- zones_size[ZONE_DMA] = 0;
+ zones_size[ZONE_DMA32] = 0;
}
/* Take zone metadata from controller 0 if we're isolnode. */
@@ -830,7 +830,7 @@ static void __init zone_sizes_init(void)
PFN_UP(node_percpu[i]));
/* Track the type of memory on each node */
- if (zones_size[ZONE_NORMAL] || zones_size[ZONE_DMA])
+ if (zones_size[ZONE_NORMAL] || zones_size[ZONE_DMA32])
node_set_state(i, N_NORMAL_MEMORY);
#ifdef CONFIG_HIGHMEM
if (end != start)
--
2.14.2
Move the few remaining bits of swiotlb glue towards their callers,
and remove the pointless on ia64 swiotlb variable.
Signed-off-by: Christoph Hellwig <[email protected]>
---
arch/ia64/include/asm/dma-mapping.h | 1 -
arch/ia64/include/asm/swiotlb.h | 18 ------------------
arch/ia64/kernel/dma-mapping.c | 9 +++++++++
arch/ia64/kernel/pci-dma.c | 12 ++++++++++--
arch/ia64/kernel/pci-swiotlb.c | 36 ------------------------------------
drivers/iommu/intel-iommu.c | 2 +-
6 files changed, 20 insertions(+), 58 deletions(-)
delete mode 100644 arch/ia64/include/asm/swiotlb.h
delete mode 100644 arch/ia64/kernel/pci-swiotlb.c
diff --git a/arch/ia64/include/asm/dma-mapping.h b/arch/ia64/include/asm/dma-mapping.h
index eabee56d995c..76e4d6632d68 100644
--- a/arch/ia64/include/asm/dma-mapping.h
+++ b/arch/ia64/include/asm/dma-mapping.h
@@ -8,7 +8,6 @@
*/
#include <asm/machvec.h>
#include <linux/scatterlist.h>
-#include <asm/swiotlb.h>
#include <linux/dma-debug.h>
#define ARCH_HAS_DMA_GET_REQUIRED_MASK
diff --git a/arch/ia64/include/asm/swiotlb.h b/arch/ia64/include/asm/swiotlb.h
deleted file mode 100644
index 841e2c7d0b21..000000000000
--- a/arch/ia64/include/asm/swiotlb.h
+++ /dev/null
@@ -1,18 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0 */
-#ifndef ASM_IA64__SWIOTLB_H
-#define ASM_IA64__SWIOTLB_H
-
-#include <linux/dma-mapping.h>
-#include <linux/swiotlb.h>
-
-#ifdef CONFIG_SWIOTLB
-extern int swiotlb;
-extern void pci_swiotlb_init(void);
-#else
-#define swiotlb 0
-static inline void pci_swiotlb_init(void)
-{
-}
-#endif
-
-#endif /* ASM_IA64__SWIOTLB_H */
diff --git a/arch/ia64/kernel/dma-mapping.c b/arch/ia64/kernel/dma-mapping.c
index 7a82c9259609..f2d57e66fd86 100644
--- a/arch/ia64/kernel/dma-mapping.c
+++ b/arch/ia64/kernel/dma-mapping.c
@@ -1,5 +1,6 @@
// SPDX-License-Identifier: GPL-2.0
#include <linux/dma-mapping.h>
+#include <linux/swiotlb.h>
#include <linux/export.h>
/* Set this to 1 if there is a HW IOMMU in the system */
@@ -23,3 +24,11 @@ const struct dma_map_ops *dma_get_ops(struct device *dev)
return dma_ops;
}
EXPORT_SYMBOL(dma_get_ops);
+
+#ifdef CONFIG_SWIOTLB
+void __init swiotlb_dma_init(void)
+{
+ dma_ops = &swiotlb_dma_ops;
+ swiotlb_init(1);
+}
+#endif
diff --git a/arch/ia64/kernel/pci-dma.c b/arch/ia64/kernel/pci-dma.c
index 3ba87c22dfbc..35e0cad33b7d 100644
--- a/arch/ia64/kernel/pci-dma.c
+++ b/arch/ia64/kernel/pci-dma.c
@@ -104,8 +104,16 @@ void __init pci_iommu_alloc(void)
detect_intel_iommu();
#ifdef CONFIG_SWIOTLB
- pci_swiotlb_init();
-#endif
+ if (!iommu_detected) {
+#ifdef CONFIG_IA64_GENERIC
+ printk(KERN_INFO "PCI-DMA: Re-initialize machine vector.\n");
+ machvec_init("dig");
+ swiotlb_dma_init();
+#else
+ panic("Unable to find Intel IOMMU");
+#endif /* CONFIG_IA64_GENERIC */
+ }
+#endif /* CONFIG_SWIOTLB */
}
#endif
diff --git a/arch/ia64/kernel/pci-swiotlb.c b/arch/ia64/kernel/pci-swiotlb.c
deleted file mode 100644
index 0f8d5fbd86bd..000000000000
--- a/arch/ia64/kernel/pci-swiotlb.c
+++ /dev/null
@@ -1,36 +0,0 @@
-// SPDX-License-Identifier: GPL-2.0
-/* Glue code to lib/swiotlb.c */
-
-#include <linux/pci.h>
-#include <linux/gfp.h>
-#include <linux/cache.h>
-#include <linux/module.h>
-#include <linux/dma-mapping.h>
-#include <linux/swiotlb.h>
-#include <asm/dma.h>
-#include <asm/iommu.h>
-#include <asm/machvec.h>
-
-int swiotlb __read_mostly;
-EXPORT_SYMBOL(swiotlb);
-
-void __init swiotlb_dma_init(void)
-{
- dma_ops = &swiotlb_dma_ops;
- swiotlb_init(1);
-}
-
-void __init pci_swiotlb_init(void)
-{
- if (!iommu_detected) {
-#ifdef CONFIG_IA64_GENERIC
- swiotlb = 1;
- printk(KERN_INFO "PCI-DMA: Re-initialize machine vector.\n");
- machvec_init("dig");
- swiotlb_init(1);
- dma_ops = &swiotlb_dma_ops;
-#else
- panic("Unable to find Intel IOMMU");
-#endif
- }
-}
diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c
index 4a2de34895ec..a1373cf34326 100644
--- a/drivers/iommu/intel-iommu.c
+++ b/drivers/iommu/intel-iommu.c
@@ -4808,7 +4808,7 @@ int __init intel_iommu_init(void)
up_write(&dmar_global_lock);
pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
-#ifdef CONFIG_SWIOTLB
+#if defined(CONFIG_X86) && defined(CONFIG_SWIOTLB)
swiotlb = 0;
#endif
dma_ops = &intel_dma_ops;
--
2.14.2
ia64 uses ZONE_DMA for allocations below 32-bits. These days we
name the zone for that ZONE_DMA32, which will allow to use the
dma-direct and generic swiotlb code as-is, so rename it.
Signed-off-by: Christoph Hellwig <[email protected]>
---
arch/ia64/Kconfig | 2 +-
arch/ia64/kernel/pci-swiotlb.c | 2 +-
arch/ia64/mm/contig.c | 4 ++--
arch/ia64/mm/discontig.c | 8 ++++----
4 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/arch/ia64/Kconfig b/arch/ia64/Kconfig
index 4d18fca885ee..888acdb163cb 100644
--- a/arch/ia64/Kconfig
+++ b/arch/ia64/Kconfig
@@ -66,7 +66,7 @@ config 64BIT
select ATA_NONSTANDARD if ATA
default y
-config ZONE_DMA
+config ZONE_DMA32
def_bool y
depends on !IA64_SGI_SN2
diff --git a/arch/ia64/kernel/pci-swiotlb.c b/arch/ia64/kernel/pci-swiotlb.c
index f1ae873a8c35..4a9a6e58ad6a 100644
--- a/arch/ia64/kernel/pci-swiotlb.c
+++ b/arch/ia64/kernel/pci-swiotlb.c
@@ -20,7 +20,7 @@ static void *ia64_swiotlb_alloc_coherent(struct device *dev, size_t size,
unsigned long attrs)
{
if (dev->coherent_dma_mask != DMA_BIT_MASK(64))
- gfp |= GFP_DMA;
+ gfp |= GFP_DMA32;
return swiotlb_alloc_coherent(dev, size, dma_handle, gfp);
}
diff --git a/arch/ia64/mm/contig.c b/arch/ia64/mm/contig.c
index 52715a71aede..7d64b30913d1 100644
--- a/arch/ia64/mm/contig.c
+++ b/arch/ia64/mm/contig.c
@@ -237,9 +237,9 @@ paging_init (void)
unsigned long max_zone_pfns[MAX_NR_ZONES];
memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
-#ifdef CONFIG_ZONE_DMA
+#ifdef CONFIG_ZONE_DMA32
max_dma = virt_to_phys((void *) MAX_DMA_ADDRESS) >> PAGE_SHIFT;
- max_zone_pfns[ZONE_DMA] = max_dma;
+ max_zone_pfns[ZONE_DMA32] = max_dma;
#endif
max_zone_pfns[ZONE_NORMAL] = max_low_pfn;
diff --git a/arch/ia64/mm/discontig.c b/arch/ia64/mm/discontig.c
index 9b2d994cddf6..ac46f0d60b66 100644
--- a/arch/ia64/mm/discontig.c
+++ b/arch/ia64/mm/discontig.c
@@ -38,7 +38,7 @@ struct early_node_data {
struct ia64_node_data *node_data;
unsigned long pernode_addr;
unsigned long pernode_size;
-#ifdef CONFIG_ZONE_DMA
+#ifdef CONFIG_ZONE_DMA32
unsigned long num_dma_physpages;
#endif
unsigned long min_pfn;
@@ -669,7 +669,7 @@ static __init int count_node_pages(unsigned long start, unsigned long len, int n
{
unsigned long end = start + len;
-#ifdef CONFIG_ZONE_DMA
+#ifdef CONFIG_ZONE_DMA32
if (start <= __pa(MAX_DMA_ADDRESS))
mem_data[node].num_dma_physpages +=
(min(end, __pa(MAX_DMA_ADDRESS)) - start) >>PAGE_SHIFT;
@@ -724,8 +724,8 @@ void __init paging_init(void)
}
memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
-#ifdef CONFIG_ZONE_DMA
- max_zone_pfns[ZONE_DMA] = max_dma;
+#ifdef CONFIG_ZONE_DMA32
+ max_zone_pfns[ZONE_DMA32] = max_dma;
#endif
max_zone_pfns[ZONE_NORMAL] = max_pfn;
free_area_init_nodes(max_zone_pfns);
--
2.14.2
Factor out a new swiotlb_alloc_buffer helper that allocates DMA coherent
memory from the swiotlb bounce buffer.
This allows to simplify the swiotlb_alloc implemenation that uses
dma_direct_alloc to try to allocate a reachable buffer first.
Signed-off-by: Christoph Hellwig <[email protected]>
---
lib/swiotlb.c | 122 +++++++++++++++++++++++++++++++---------------------------
1 file changed, 65 insertions(+), 57 deletions(-)
diff --git a/lib/swiotlb.c b/lib/swiotlb.c
index 1a147f1354a1..bf2d19ee91c1 100644
--- a/lib/swiotlb.c
+++ b/lib/swiotlb.c
@@ -709,75 +709,79 @@ void swiotlb_tbl_sync_single(struct device *hwdev, phys_addr_t tlb_addr,
}
EXPORT_SYMBOL_GPL(swiotlb_tbl_sync_single);
-void *
-swiotlb_alloc_coherent(struct device *hwdev, size_t size,
- dma_addr_t *dma_handle, gfp_t flags)
+static inline bool dma_coherent_ok(struct device *dev, dma_addr_t addr,
+ size_t size)
{
- bool warn = !(flags & __GFP_NOWARN);
- dma_addr_t dev_addr;
- void *ret;
- int order = get_order(size);
- u64 dma_mask = DMA_BIT_MASK(32);
+ u64 mask = DMA_BIT_MASK(32);
- if (hwdev && hwdev->coherent_dma_mask)
- dma_mask = hwdev->coherent_dma_mask;
+ if (dev && dev->coherent_dma_mask)
+ mask = dev->coherent_dma_mask;
+ return addr + size - 1 <= mask;
+}
- ret = (void *)__get_free_pages(flags, order);
- if (ret) {
- dev_addr = swiotlb_virt_to_bus(hwdev, ret);
- if (dev_addr + size - 1 > dma_mask) {
- /*
- * The allocated memory isn't reachable by the device.
- */
- free_pages((unsigned long) ret, order);
- ret = NULL;
- }
- }
- if (!ret) {
- /*
- * We are either out of memory or the device can't DMA to
- * GFP_DMA memory; fall back on map_single(), which
- * will grab memory from the lowest available address range.
- */
- phys_addr_t paddr = map_single(hwdev, 0, size, DMA_FROM_DEVICE,
- warn ? 0 : DMA_ATTR_NO_WARN);
- if (paddr == SWIOTLB_MAP_ERROR)
- goto err_warn;
+static void *
+swiotlb_alloc_buffer(struct device *dev, size_t size, dma_addr_t *dma_handle,
+ unsigned long attrs)
+{
+ phys_addr_t phys_addr;
+
+ if (swiotlb_force == SWIOTLB_NO_FORCE)
+ goto out_warn;
- ret = phys_to_virt(paddr);
- dev_addr = swiotlb_phys_to_dma(hwdev, paddr);
+ phys_addr = swiotlb_tbl_map_single(dev,
+ swiotlb_phys_to_dma(dev, io_tlb_start),
+ 0, size, DMA_FROM_DEVICE, 0);
+ if (phys_addr == SWIOTLB_MAP_ERROR)
+ goto out_warn;
- /* Confirm address can be DMA'd by device */
- if (dev_addr + size - 1 > dma_mask) {
- printk("hwdev DMA mask = 0x%016Lx, dev_addr = 0x%016Lx\n",
- (unsigned long long)dma_mask,
- (unsigned long long)dev_addr);
+ *dma_handle = swiotlb_phys_to_dma(dev, phys_addr);
- /*
- * DMA_TO_DEVICE to avoid memcpy in unmap_single.
- * The DMA_ATTR_SKIP_CPU_SYNC is optional.
- */
- swiotlb_tbl_unmap_single(hwdev, paddr,
- size, DMA_TO_DEVICE,
- DMA_ATTR_SKIP_CPU_SYNC);
- goto err_warn;
- }
- }
+ if (dma_coherent_ok(dev, *dma_handle, size))
+ goto out_unmap;
- *dma_handle = dev_addr;
- memset(ret, 0, size);
+ memset(phys_to_virt(phys_addr), 0, size);
+ return phys_to_virt(phys_addr);
- return ret;
+out_unmap:
+ dev_warn(dev, "hwdev DMA mask = 0x%016Lx, dev_addr = 0x%016Lx\n",
+ (unsigned long long)(dev ? dev->coherent_dma_mask : 0),
+ (unsigned long long)*dma_handle);
-err_warn:
- if (warn && printk_ratelimit()) {
- pr_warn("swiotlb: coherent allocation failed for device %s size=%zu\n",
- dev_name(hwdev), size);
+ /*
+ * DMA_TO_DEVICE to avoid memcpy in unmap_single.
+ * DMA_ATTR_SKIP_CPU_SYNC is optional.
+ */
+ swiotlb_tbl_unmap_single(dev, phys_addr, size, DMA_TO_DEVICE,
+ DMA_ATTR_SKIP_CPU_SYNC);
+out_warn:
+ if ((attrs & DMA_ATTR_NO_WARN) && printk_ratelimit()) {
+ dev_warn(dev,
+ "swiotlb: coherent allocation failed, size=%zu\n",
+ size);
dump_stack();
}
-
return NULL;
}
+
+void *
+swiotlb_alloc_coherent(struct device *hwdev, size_t size,
+ dma_addr_t *dma_handle, gfp_t flags)
+{
+ int order = get_order(size);
+ unsigned long attrs = (flags & __GFP_NOWARN) ? DMA_ATTR_NO_WARN : 0;
+ void *ret;
+
+ ret = (void *)__get_free_pages(flags, order);
+ if (ret) {
+ *dma_handle = swiotlb_virt_to_bus(hwdev, ret);
+ if (dma_coherent_ok(hwdev, *dma_handle, size)) {
+ memset(ret, 0, size);
+ return ret;
+ }
+ }
+
+ return swiotlb_alloc_buffer(hwdev, size, dma_handle, attrs);
+}
EXPORT_SYMBOL(swiotlb_alloc_coherent);
static bool swiotlb_free_buffer(struct device *dev, size_t size,
@@ -1103,6 +1107,10 @@ void *swiotlb_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
{
void *vaddr;
+ /* temporary workaround: */
+ if (gfp & __GFP_NOWARN)
+ attrs |= DMA_ATTR_NO_WARN;
+
/*
* Don't print a warning when the first allocation attempt fails.
* swiotlb_alloc_coherent() will print a warning when the DMA memory
@@ -1112,7 +1120,7 @@ void *swiotlb_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
vaddr = dma_direct_alloc(dev, size, dma_handle, gfp, attrs);
if (!vaddr)
- vaddr = swiotlb_alloc_coherent(dev, size, dma_handle, gfp);
+ vaddr = swiotlb_alloc_buffer(dev, size, dma_handle, attrs);
return vaddr;
}
--
2.14.2
To properly reject too small DMA masks based on the addressability of the
bounce buffer.
Signed-off-by: Christoph Hellwig <[email protected]>
---
lib/swiotlb.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/lib/swiotlb.c b/lib/swiotlb.c
index 0fae2f45c3c0..539fd1099ba9 100644
--- a/lib/swiotlb.c
+++ b/lib/swiotlb.c
@@ -1128,5 +1128,6 @@ const struct dma_map_ops swiotlb_dma_ops = {
.unmap_sg = swiotlb_unmap_sg_attrs,
.map_page = swiotlb_map_page,
.unmap_page = swiotlb_unmap_page,
+ .dma_supported = swiotlb_dma_supported,
};
#endif /* CONFIG_DMA_DIRECT_OPS */
--
2.14.2
Signed-off-by: Christoph Hellwig <[email protected]>
---
arch/powerpc/kernel/dma-swiotlb.c | 2 +-
arch/x86/kernel/pci-swiotlb.c | 2 +-
include/linux/swiotlb.h | 4 ++--
lib/swiotlb.c | 2 +-
4 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/arch/powerpc/kernel/dma-swiotlb.c b/arch/powerpc/kernel/dma-swiotlb.c
index 506ac4fafac5..88f3963ca30f 100644
--- a/arch/powerpc/kernel/dma-swiotlb.c
+++ b/arch/powerpc/kernel/dma-swiotlb.c
@@ -121,7 +121,7 @@ static int __init check_swiotlb_enabled(void)
if (ppc_swiotlb_enable)
swiotlb_print_info();
else
- swiotlb_free();
+ swiotlb_exit();
return 0;
}
diff --git a/arch/x86/kernel/pci-swiotlb.c b/arch/x86/kernel/pci-swiotlb.c
index 0d77603c2f50..0ee0f8f34251 100644
--- a/arch/x86/kernel/pci-swiotlb.c
+++ b/arch/x86/kernel/pci-swiotlb.c
@@ -120,7 +120,7 @@ void __init pci_swiotlb_late_init(void)
{
/* An IOMMU turned us off. */
if (!swiotlb)
- swiotlb_free();
+ swiotlb_exit();
else {
printk(KERN_INFO "PCI-DMA: "
"Using software bounce buffering for IO (SWIOTLB)\n");
diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h
index 24ed817082ee..606375e35d87 100644
--- a/include/linux/swiotlb.h
+++ b/include/linux/swiotlb.h
@@ -115,10 +115,10 @@ extern int
swiotlb_dma_supported(struct device *hwdev, u64 mask);
#ifdef CONFIG_SWIOTLB
-extern void __init swiotlb_free(void);
+extern void __init swiotlb_exit(void);
unsigned int swiotlb_max_segment(void);
#else
-static inline void swiotlb_free(void) { }
+static inline void swiotlb_exit(void) { }
static inline unsigned int swiotlb_max_segment(void) { return 0; }
#endif
diff --git a/lib/swiotlb.c b/lib/swiotlb.c
index 125c1062119f..cf5311908fa9 100644
--- a/lib/swiotlb.c
+++ b/lib/swiotlb.c
@@ -417,7 +417,7 @@ swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs)
return -ENOMEM;
}
-void __init swiotlb_free(void)
+void __init swiotlb_exit(void)
{
if (!io_tlb_orig_addr)
return;
--
2.14.2
We'll need that name for a generic implementation soon.
Signed-off-by: Christoph Hellwig <[email protected]>
---
arch/x86/kernel/pci-swiotlb.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/arch/x86/kernel/pci-swiotlb.c b/arch/x86/kernel/pci-swiotlb.c
index 9d3e35c33d94..0d77603c2f50 100644
--- a/arch/x86/kernel/pci-swiotlb.c
+++ b/arch/x86/kernel/pci-swiotlb.c
@@ -48,7 +48,7 @@ void x86_swiotlb_free_coherent(struct device *dev, size_t size,
dma_generic_free_coherent(dev, size, vaddr, dma_addr, attrs);
}
-static const struct dma_map_ops swiotlb_dma_ops = {
+static const struct dma_map_ops x86_swiotlb_dma_ops = {
.mapping_error = swiotlb_dma_mapping_error,
.alloc = x86_swiotlb_alloc_coherent,
.free = x86_swiotlb_free_coherent,
@@ -112,7 +112,7 @@ void __init pci_swiotlb_init(void)
{
if (swiotlb) {
swiotlb_init(0);
- dma_ops = &swiotlb_dma_ops;
+ dma_ops = &x86_swiotlb_dma_ops;
}
}
--
2.14.2
We'll need that name for a generic implementation soon.
Signed-off-by: Christoph Hellwig <[email protected]>
---
arch/ia64/hp/common/hwsw_iommu.c | 4 ++--
arch/ia64/hp/common/sba_iommu.c | 6 +++---
arch/ia64/kernel/pci-swiotlb.c | 6 +++---
3 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/arch/ia64/hp/common/hwsw_iommu.c b/arch/ia64/hp/common/hwsw_iommu.c
index 63d8e1d2477f..41279f0442bd 100644
--- a/arch/ia64/hp/common/hwsw_iommu.c
+++ b/arch/ia64/hp/common/hwsw_iommu.c
@@ -19,7 +19,7 @@
#include <linux/export.h>
#include <asm/machvec.h>
-extern const struct dma_map_ops sba_dma_ops, swiotlb_dma_ops;
+extern const struct dma_map_ops sba_dma_ops, ia64_swiotlb_dma_ops;
/* swiotlb declarations & definitions: */
extern int swiotlb_late_init_with_default_size (size_t size);
@@ -38,7 +38,7 @@ static inline int use_swiotlb(struct device *dev)
const struct dma_map_ops *hwsw_dma_get_ops(struct device *dev)
{
if (use_swiotlb(dev))
- return &swiotlb_dma_ops;
+ return &ia64_swiotlb_dma_ops;
return &sba_dma_ops;
}
EXPORT_SYMBOL(hwsw_dma_get_ops);
diff --git a/arch/ia64/hp/common/sba_iommu.c b/arch/ia64/hp/common/sba_iommu.c
index aec4a3354abe..8c0a9ae6afec 100644
--- a/arch/ia64/hp/common/sba_iommu.c
+++ b/arch/ia64/hp/common/sba_iommu.c
@@ -2096,7 +2096,7 @@ static int __init acpi_sba_ioc_init_acpi(void)
/* This has to run before acpi_scan_init(). */
arch_initcall(acpi_sba_ioc_init_acpi);
-extern const struct dma_map_ops swiotlb_dma_ops;
+extern const struct dma_map_ops ia64_swiotlb_dma_ops;
static int __init
sba_init(void)
@@ -2111,7 +2111,7 @@ sba_init(void)
* a successful kdump kernel boot is to use the swiotlb.
*/
if (is_kdump_kernel()) {
- dma_ops = &swiotlb_dma_ops;
+ dma_ops = &ia64_swiotlb_dma_ops;
if (swiotlb_late_init_with_default_size(64 * (1<<20)) != 0)
panic("Unable to initialize software I/O TLB:"
" Try machvec=dig boot option");
@@ -2133,7 +2133,7 @@ sba_init(void)
* If we didn't find something sba_iommu can claim, we
* need to setup the swiotlb and switch to the dig machvec.
*/
- dma_ops = &swiotlb_dma_ops;
+ dma_ops = &ia64_swiotlb_dma_ops;
if (swiotlb_late_init_with_default_size(64 * (1<<20)) != 0)
panic("Unable to find SBA IOMMU or initialize "
"software I/O TLB: Try machvec=dig boot option");
diff --git a/arch/ia64/kernel/pci-swiotlb.c b/arch/ia64/kernel/pci-swiotlb.c
index 5e50939aa03e..f1ae873a8c35 100644
--- a/arch/ia64/kernel/pci-swiotlb.c
+++ b/arch/ia64/kernel/pci-swiotlb.c
@@ -31,7 +31,7 @@ static void ia64_swiotlb_free_coherent(struct device *dev, size_t size,
swiotlb_free_coherent(dev, size, vaddr, dma_addr);
}
-const struct dma_map_ops swiotlb_dma_ops = {
+const struct dma_map_ops ia64_swiotlb_dma_ops = {
.alloc = ia64_swiotlb_alloc_coherent,
.free = ia64_swiotlb_free_coherent,
.map_page = swiotlb_map_page,
@@ -48,7 +48,7 @@ const struct dma_map_ops swiotlb_dma_ops = {
void __init swiotlb_dma_init(void)
{
- dma_ops = &swiotlb_dma_ops;
+ dma_ops = &ia64_swiotlb_dma_ops;
swiotlb_init(1);
}
@@ -60,7 +60,7 @@ void __init pci_swiotlb_init(void)
printk(KERN_INFO "PCI-DMA: Re-initialize machine vector.\n");
machvec_init("dig");
swiotlb_init(1);
- dma_ops = &swiotlb_dma_ops;
+ dma_ops = &ia64_swiotlb_dma_ops;
#else
panic("Unable to find Intel IOMMU");
#endif
--
2.14.2
Acked-by: Christian König <[email protected]> for the whole series.
Regards,
Christian.
Am 10.01.2018 um 09:09 schrieb Christoph Hellwig:
> A lot of architectures have essentially identical dma_map_ops
> implementations to use swiotlb. This series adds new generic
> swiotlb_alloc/free helpers that take the attrs argument exposed
> in dma_map_ops, and which do an enhanced direct allocation
> modelled after x86 and reused from the dma-direct code, and
> then switches most architectures over to it. The only exceptions
> are mips, which requires additional cache flushing which will
> need a new abstraction, and x86 itself which will be handled in
> a later series with other x86 dma mapping changes.
>
> To support the generic code a few architectures that currently
> use ZONE_DMA/GFP_DMA for <= 32-bit allocations are switched to
> implement ZONE_DMA32 instead.
>
> This series is based on the previously sent series to consolidate
> the direct dma mapping implementation. A git tree with this
> series as well as the prerequisites is available here:
>
> git://git.infradead.org/users/hch/misc.git swiotlb
>
> Gitweb:
>
> http://git.infradead.org/users/hch/misc.git/shortlog/refs/heads/swiotlb
On 10/01/18 08:09, Christoph Hellwig wrote:
> We'll need that name for a generic implementation soon.
Reviewed-by: Robin Murphy <[email protected]>
> Signed-off-by: Christoph Hellwig <[email protected]>
> ---
> arch/arm64/mm/dma-mapping.c | 4 ++--
> 1 file changed, 2 insertions(+), 2 deletions(-)
>
> diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c
> index f3a637b98487..6840426bbe77 100644
> --- a/arch/arm64/mm/dma-mapping.c
> +++ b/arch/arm64/mm/dma-mapping.c
> @@ -368,7 +368,7 @@ static int __swiotlb_dma_mapping_error(struct device *hwdev, dma_addr_t addr)
> return 0;
> }
>
> -static const struct dma_map_ops swiotlb_dma_ops = {
> +static const struct dma_map_ops arm64_swiotlb_dma_ops = {
> .alloc = __dma_alloc,
> .free = __dma_free,
> .mmap = __swiotlb_mmap,
> @@ -923,7 +923,7 @@ void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
> const struct iommu_ops *iommu, bool coherent)
> {
> if (!dev->dma_ops)
> - dev->dma_ops = &swiotlb_dma_ops;
> + dev->dma_ops = &arm64_swiotlb_dma_ops;
>
> dev->archdata.dma_coherent = coherent;
> __iommu_setup_dma_ops(dev, dma_base, size, iommu);
>
On 10/01/18 08:09, Christoph Hellwig wrote:
> To properly reject too small DMA masks based on the addressability of the
> bounce buffer.
I reckon this is self-evident enough that it should simply be squashed
into the previous patch.
Robin.
> Signed-off-by: Christoph Hellwig <[email protected]>
> ---
> lib/swiotlb.c | 1 +
> 1 file changed, 1 insertion(+)
>
> diff --git a/lib/swiotlb.c b/lib/swiotlb.c
> index 0fae2f45c3c0..539fd1099ba9 100644
> --- a/lib/swiotlb.c
> +++ b/lib/swiotlb.c
> @@ -1128,5 +1128,6 @@ const struct dma_map_ops swiotlb_dma_ops = {
> .unmap_sg = swiotlb_unmap_sg_attrs,
> .map_page = swiotlb_map_page,
> .unmap_page = swiotlb_unmap_page,
> + .dma_supported = swiotlb_dma_supported,
> };
> #endif /* CONFIG_DMA_DIRECT_OPS */
>
On 10/01/18 08:09, Christoph Hellwig wrote:
> Factor out a new swiotlb_alloc_buffer helper that allocates DMA coherent
> memory from the swiotlb bounce buffer.
>
> This allows to simplify the swiotlb_alloc implemenation that uses
> dma_direct_alloc to try to allocate a reachable buffer first.
>
> Signed-off-by: Christoph Hellwig <[email protected]>
> ---
> lib/swiotlb.c | 122 +++++++++++++++++++++++++++++++---------------------------
> 1 file changed, 65 insertions(+), 57 deletions(-)
>
> diff --git a/lib/swiotlb.c b/lib/swiotlb.c
> index 1a147f1354a1..bf2d19ee91c1 100644
> --- a/lib/swiotlb.c
> +++ b/lib/swiotlb.c
> @@ -709,75 +709,79 @@ void swiotlb_tbl_sync_single(struct device *hwdev, phys_addr_t tlb_addr,
> }
> EXPORT_SYMBOL_GPL(swiotlb_tbl_sync_single);
>
> -void *
> -swiotlb_alloc_coherent(struct device *hwdev, size_t size,
> - dma_addr_t *dma_handle, gfp_t flags)
> +static inline bool dma_coherent_ok(struct device *dev, dma_addr_t addr,
> + size_t size)
> {
> - bool warn = !(flags & __GFP_NOWARN);
> - dma_addr_t dev_addr;
> - void *ret;
> - int order = get_order(size);
> - u64 dma_mask = DMA_BIT_MASK(32);
> + u64 mask = DMA_BIT_MASK(32);
>
> - if (hwdev && hwdev->coherent_dma_mask)
> - dma_mask = hwdev->coherent_dma_mask;
> + if (dev && dev->coherent_dma_mask)
> + mask = dev->coherent_dma_mask;
> + return addr + size - 1 <= mask;
> +}
>
> - ret = (void *)__get_free_pages(flags, order);
> - if (ret) {
> - dev_addr = swiotlb_virt_to_bus(hwdev, ret);
> - if (dev_addr + size - 1 > dma_mask) {
> - /*
> - * The allocated memory isn't reachable by the device.
> - */
> - free_pages((unsigned long) ret, order);
> - ret = NULL;
> - }
> - }
> - if (!ret) {
> - /*
> - * We are either out of memory or the device can't DMA to
> - * GFP_DMA memory; fall back on map_single(), which
> - * will grab memory from the lowest available address range.
> - */
> - phys_addr_t paddr = map_single(hwdev, 0, size, DMA_FROM_DEVICE,
> - warn ? 0 : DMA_ATTR_NO_WARN);
> - if (paddr == SWIOTLB_MAP_ERROR)
> - goto err_warn;
> +static void *
> +swiotlb_alloc_buffer(struct device *dev, size_t size, dma_addr_t *dma_handle,
> + unsigned long attrs)
> +{
> + phys_addr_t phys_addr;
> +
> + if (swiotlb_force == SWIOTLB_NO_FORCE)
> + goto out_warn;
>
> - ret = phys_to_virt(paddr);
> - dev_addr = swiotlb_phys_to_dma(hwdev, paddr);
> + phys_addr = swiotlb_tbl_map_single(dev,
> + swiotlb_phys_to_dma(dev, io_tlb_start),
> + 0, size, DMA_FROM_DEVICE, 0);
> + if (phys_addr == SWIOTLB_MAP_ERROR)
> + goto out_warn;
>
> - /* Confirm address can be DMA'd by device */
> - if (dev_addr + size - 1 > dma_mask) {
> - printk("hwdev DMA mask = 0x%016Lx, dev_addr = 0x%016Lx\n",
> - (unsigned long long)dma_mask,
> - (unsigned long long)dev_addr);
> + *dma_handle = swiotlb_phys_to_dma(dev, phys_addr);
nit: this should probably go after the dma_coherent_ok() check (as with
the original logic).
>
> - /*
> - * DMA_TO_DEVICE to avoid memcpy in unmap_single.
> - * The DMA_ATTR_SKIP_CPU_SYNC is optional.
> - */
> - swiotlb_tbl_unmap_single(hwdev, paddr,
> - size, DMA_TO_DEVICE,
> - DMA_ATTR_SKIP_CPU_SYNC);
> - goto err_warn;
> - }
> - }
> + if (dma_coherent_ok(dev, *dma_handle, size))
> + goto out_unmap;
>
> - *dma_handle = dev_addr;
> - memset(ret, 0, size);
> + memset(phys_to_virt(phys_addr), 0, size);
> + return phys_to_virt(phys_addr);
>
> - return ret;
> +out_unmap:
> + dev_warn(dev, "hwdev DMA mask = 0x%016Lx, dev_addr = 0x%016Lx\n",
> + (unsigned long long)(dev ? dev->coherent_dma_mask : 0),
> + (unsigned long long)*dma_handle);
>
> -err_warn:
> - if (warn && printk_ratelimit()) {
> - pr_warn("swiotlb: coherent allocation failed for device %s size=%zu\n",
> - dev_name(hwdev), size);
> + /*
> + * DMA_TO_DEVICE to avoid memcpy in unmap_single.
> + * DMA_ATTR_SKIP_CPU_SYNC is optional.
> + */
> + swiotlb_tbl_unmap_single(dev, phys_addr, size, DMA_TO_DEVICE,
> + DMA_ATTR_SKIP_CPU_SYNC);
> +out_warn:
> + if ((attrs & DMA_ATTR_NO_WARN) && printk_ratelimit()) {
> + dev_warn(dev,
> + "swiotlb: coherent allocation failed, size=%zu\n",
> + size);
> dump_stack();
> }
> -
> return NULL;
> }
> +
> +void *
> +swiotlb_alloc_coherent(struct device *hwdev, size_t size,
> + dma_addr_t *dma_handle, gfp_t flags)
> +{
> + int order = get_order(size);
> + unsigned long attrs = (flags & __GFP_NOWARN) ? DMA_ATTR_NO_WARN : 0;
> + void *ret;
> +
> + ret = (void *)__get_free_pages(flags, order);
> + if (ret) {
> + *dma_handle = swiotlb_virt_to_bus(hwdev, ret);
> + if (dma_coherent_ok(hwdev, *dma_handle, size)) {
> + memset(ret, 0, size);
> + return ret;
> + }
Aren't we leaking the pages here?
Robin.
> + }
> +
> + return swiotlb_alloc_buffer(hwdev, size, dma_handle, attrs);
> +}
> EXPORT_SYMBOL(swiotlb_alloc_coherent);
>
> static bool swiotlb_free_buffer(struct device *dev, size_t size,
> @@ -1103,6 +1107,10 @@ void *swiotlb_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
> {
> void *vaddr;
>
> + /* temporary workaround: */
> + if (gfp & __GFP_NOWARN)
> + attrs |= DMA_ATTR_NO_WARN;
> +
> /*
> * Don't print a warning when the first allocation attempt fails.
> * swiotlb_alloc_coherent() will print a warning when the DMA memory
> @@ -1112,7 +1120,7 @@ void *swiotlb_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle,
>
> vaddr = dma_direct_alloc(dev, size, dma_handle, gfp, attrs);
> if (!vaddr)
> - vaddr = swiotlb_alloc_coherent(dev, size, dma_handle, gfp);
> + vaddr = swiotlb_alloc_buffer(dev, size, dma_handle, attrs);
> return vaddr;
> }
>
>
On 10/01/18 08:09, Christoph Hellwig wrote:
> arm64 uses ZONE_DMA for allocations below 32-bits. These days we
> name the zone for that ZONE_DMA32, which will allow to use the
> dma-direct and generic swiotlb code as-is, so rename it.
I do wonder if we could also "upgrade" GFP_DMA to GFP_DMA32 somehow when
!ZONE_DMA - there are almost certainly arm64 drivers out there using a
combination of GFP_DMA and streaming mappings which will no longer get
the guaranteed 32-bit addresses they expect after this. I'm not sure
quite how feasible that is, though :/
That said, I do agree that this is an appropriate change (the legacy of
GFP_DMA is obviously horrible), so, provided we get plenty of time to
find and fix the fallout when it lands:
Reviewed-by: Robin Murphy <[email protected]>
Robin.
> Signed-off-by: Christoph Hellwig <[email protected]>
> ---
> arch/arm64/Kconfig | 2 +-
> arch/arm64/mm/dma-mapping.c | 6 +++---
> arch/arm64/mm/init.c | 16 ++++++++--------
> 3 files changed, 12 insertions(+), 12 deletions(-)
>
> diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
> index c9a7e9e1414f..6b6985f15d02 100644
> --- a/arch/arm64/Kconfig
> +++ b/arch/arm64/Kconfig
> @@ -227,7 +227,7 @@ config GENERIC_CSUM
> config GENERIC_CALIBRATE_DELAY
> def_bool y
>
> -config ZONE_DMA
> +config ZONE_DMA32
> def_bool y
>
> config HAVE_GENERIC_GUP
> diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c
> index 6840426bbe77..0d641875b20e 100644
> --- a/arch/arm64/mm/dma-mapping.c
> +++ b/arch/arm64/mm/dma-mapping.c
> @@ -95,9 +95,9 @@ static void *__dma_alloc_coherent(struct device *dev, size_t size,
> dma_addr_t *dma_handle, gfp_t flags,
> unsigned long attrs)
> {
> - if (IS_ENABLED(CONFIG_ZONE_DMA) &&
> + if (IS_ENABLED(CONFIG_ZONE_DMA32) &&
> dev->coherent_dma_mask <= DMA_BIT_MASK(32))
> - flags |= GFP_DMA;
> + flags |= GFP_DMA32;
> if (dev_get_cma_area(dev) && gfpflags_allow_blocking(flags)) {
> struct page *page;
> void *addr;
> @@ -397,7 +397,7 @@ static int __init atomic_pool_init(void)
> page = dma_alloc_from_contiguous(NULL, nr_pages,
> pool_size_order, GFP_KERNEL);
> else
> - page = alloc_pages(GFP_DMA, pool_size_order);
> + page = alloc_pages(GFP_DMA32, pool_size_order);
>
> if (page) {
> int ret;
> diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
> index 00e7b900ca41..8f03276443c9 100644
> --- a/arch/arm64/mm/init.c
> +++ b/arch/arm64/mm/init.c
> @@ -217,7 +217,7 @@ static void __init reserve_elfcorehdr(void)
> }
> #endif /* CONFIG_CRASH_DUMP */
> /*
> - * Return the maximum physical address for ZONE_DMA (DMA_BIT_MASK(32)). It
> + * Return the maximum physical address for ZONE_DMA32 (DMA_BIT_MASK(32)). It
> * currently assumes that for memory starting above 4G, 32-bit devices will
> * use a DMA offset.
> */
> @@ -233,8 +233,8 @@ static void __init zone_sizes_init(unsigned long min, unsigned long max)
> {
> unsigned long max_zone_pfns[MAX_NR_ZONES] = {0};
>
> - if (IS_ENABLED(CONFIG_ZONE_DMA))
> - max_zone_pfns[ZONE_DMA] = PFN_DOWN(max_zone_dma_phys());
> + if (IS_ENABLED(CONFIG_ZONE_DMA32))
> + max_zone_pfns[ZONE_DMA32] = PFN_DOWN(max_zone_dma_phys());
> max_zone_pfns[ZONE_NORMAL] = max;
>
> free_area_init_nodes(max_zone_pfns);
> @@ -251,9 +251,9 @@ static void __init zone_sizes_init(unsigned long min, unsigned long max)
> memset(zone_size, 0, sizeof(zone_size));
>
> /* 4GB maximum for 32-bit only capable devices */
> -#ifdef CONFIG_ZONE_DMA
> +#ifdef CONFIG_ZONE_DMA32
> max_dma = PFN_DOWN(arm64_dma_phys_limit);
> - zone_size[ZONE_DMA] = max_dma - min;
> + zone_size[ZONE_DMA32] = max_dma - min;
> #endif
> zone_size[ZONE_NORMAL] = max - max_dma;
>
> @@ -266,10 +266,10 @@ static void __init zone_sizes_init(unsigned long min, unsigned long max)
> if (start >= max)
> continue;
>
> -#ifdef CONFIG_ZONE_DMA
> +#ifdef CONFIG_ZONE_DMA32
> if (start < max_dma) {
> unsigned long dma_end = min(end, max_dma);
> - zhole_size[ZONE_DMA] -= dma_end - start;
> + zhole_size[ZONE_DMA32] -= dma_end - start;
> }
> #endif
> if (end > max_dma) {
> @@ -467,7 +467,7 @@ void __init arm64_memblock_init(void)
> early_init_fdt_scan_reserved_mem();
>
> /* 4GB maximum for 32-bit only capable devices */
> - if (IS_ENABLED(CONFIG_ZONE_DMA))
> + if (IS_ENABLED(CONFIG_ZONE_DMA32))
> arm64_dma_phys_limit = max_zone_dma_phys();
> else
> arm64_dma_phys_limit = PHYS_MASK + 1;
>
On 10/01/18 08:09, Christoph Hellwig wrote:
> The generic swiotlb_alloc and swiotlb_free routines already take care
> of CMA allocations and adding GFP_DMA32 where needed, so use them
> instead of the arm specific helpers.
It took a while to satisfy myself that the GFP_DMA(32) handling ends up
equivalent to the current behaviour, but I think it checks out. This
will certainly help with the long-overdue cleanup of this file that I've
had sat around half-finished for ages.
Reviewed-by: Robin Murphy <[email protected]>
> Signed-off-by: Christoph Hellwig <[email protected]>
> ---
> arch/arm64/Kconfig | 1 +
> arch/arm64/mm/dma-mapping.c | 46 +++------------------------------------------
> 2 files changed, 4 insertions(+), 43 deletions(-)
>
> diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
> index 6b6985f15d02..53205c02b18a 100644
> --- a/arch/arm64/Kconfig
> +++ b/arch/arm64/Kconfig
> @@ -59,6 +59,7 @@ config ARM64
> select COMMON_CLK
> select CPU_PM if (SUSPEND || CPU_IDLE)
> select DCACHE_WORD_ACCESS
> + select DMA_DIRECT_OPS
> select EDAC_SUPPORT
> select FRAME_POINTER
> select GENERIC_ALLOCATOR
> diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c
> index 0d641875b20e..a96ec0181818 100644
> --- a/arch/arm64/mm/dma-mapping.c
> +++ b/arch/arm64/mm/dma-mapping.c
> @@ -91,46 +91,6 @@ static int __free_from_pool(void *start, size_t size)
> return 1;
> }
>
> -static void *__dma_alloc_coherent(struct device *dev, size_t size,
> - dma_addr_t *dma_handle, gfp_t flags,
> - unsigned long attrs)
> -{
> - if (IS_ENABLED(CONFIG_ZONE_DMA32) &&
> - dev->coherent_dma_mask <= DMA_BIT_MASK(32))
> - flags |= GFP_DMA32;
> - if (dev_get_cma_area(dev) && gfpflags_allow_blocking(flags)) {
> - struct page *page;
> - void *addr;
> -
> - page = dma_alloc_from_contiguous(dev, size >> PAGE_SHIFT,
> - get_order(size), flags);
> - if (!page)
> - return NULL;
> -
> - *dma_handle = phys_to_dma(dev, page_to_phys(page));
> - addr = page_address(page);
> - memset(addr, 0, size);
> - return addr;
> - } else {
> - return swiotlb_alloc_coherent(dev, size, dma_handle, flags);
> - }
> -}
> -
> -static void __dma_free_coherent(struct device *dev, size_t size,
> - void *vaddr, dma_addr_t dma_handle,
> - unsigned long attrs)
> -{
> - bool freed;
> - phys_addr_t paddr = dma_to_phys(dev, dma_handle);
> -
> -
> - freed = dma_release_from_contiguous(dev,
> - phys_to_page(paddr),
> - size >> PAGE_SHIFT);
> - if (!freed)
> - swiotlb_free_coherent(dev, size, vaddr, dma_handle);
> -}
> -
> static void *__dma_alloc(struct device *dev, size_t size,
> dma_addr_t *dma_handle, gfp_t flags,
> unsigned long attrs)
> @@ -152,7 +112,7 @@ static void *__dma_alloc(struct device *dev, size_t size,
> return addr;
> }
>
> - ptr = __dma_alloc_coherent(dev, size, dma_handle, flags, attrs);
> + ptr = swiotlb_alloc(dev, size, dma_handle, flags, attrs);
> if (!ptr)
> goto no_mem;
>
> @@ -173,7 +133,7 @@ static void *__dma_alloc(struct device *dev, size_t size,
> return coherent_ptr;
>
> no_map:
> - __dma_free_coherent(dev, size, ptr, *dma_handle, attrs);
> + swiotlb_free(dev, size, ptr, *dma_handle, attrs);
> no_mem:
> return NULL;
> }
> @@ -191,7 +151,7 @@ static void __dma_free(struct device *dev, size_t size,
> return;
> vunmap(vaddr);
> }
> - __dma_free_coherent(dev, size, swiotlb_addr, dma_handle, attrs);
> + swiotlb_free(dev, size, swiotlb_addr, dma_handle, attrs);
> }
>
> static dma_addr_t __swiotlb_map_page(struct device *dev, struct page *page,
>
On Wed, Jan 10, 2018 at 12:16:15PM +0000, Robin Murphy wrote:
> On 10/01/18 08:09, Christoph Hellwig wrote:
>> To properly reject too small DMA masks based on the addressability of the
>> bounce buffer.
>
> I reckon this is self-evident enough that it should simply be squashed into
> the previous patch.
x86 didn't wire it up before, so I want a clear blaimpoint for this
change instead of mixing it up.
On Wed, Jan 10, 2018 at 12:22:18PM +0000, Robin Murphy wrote:
>> + if (phys_addr == SWIOTLB_MAP_ERROR)
>> + goto out_warn;
>> - /* Confirm address can be DMA'd by device */
>> - if (dev_addr + size - 1 > dma_mask) {
>> - printk("hwdev DMA mask = 0x%016Lx, dev_addr = 0x%016Lx\n",
>> - (unsigned long long)dma_mask,
>> - (unsigned long long)dev_addr);
>> + *dma_handle = swiotlb_phys_to_dma(dev, phys_addr);
>
> nit: this should probably go after the dma_coherent_ok() check (as with the
> original logic).
But the originall logic also needs the dma_addr_t for the
dma_coherent_ok check:
dev_addr = swiotlb_phys_to_dma(hwdev, paddr);
/* Confirm address can be DMA'd by device */
if (dev_addr + size - 1 > dma_mask) {
...
goto err_warn;
}
or do you mean assining to *dma_handle? The dma_handle is not
valid for a failure return, so I don't think this should matter.
>> + if (ret) {
>> + *dma_handle = swiotlb_virt_to_bus(hwdev, ret);
>> + if (dma_coherent_ok(hwdev, *dma_handle, size)) {
>> + memset(ret, 0, size);
>> + return ret;
>> + }
>
> Aren't we leaking the pages here?
Yes, that free_pages got lost somewhere in the rebases, I've added
it back.
On Wed, Jan 10, 2018 at 12:58:14PM +0000, Robin Murphy wrote:
> On 10/01/18 08:09, Christoph Hellwig wrote:
>> arm64 uses ZONE_DMA for allocations below 32-bits. These days we
>> name the zone for that ZONE_DMA32, which will allow to use the
>> dma-direct and generic swiotlb code as-is, so rename it.
>
> I do wonder if we could also "upgrade" GFP_DMA to GFP_DMA32 somehow when
> !ZONE_DMA - there are almost certainly arm64 drivers out there using a
> combination of GFP_DMA and streaming mappings which will no longer get the
> guaranteed 32-bit addresses they expect after this. I'm not sure quite how
> feasible that is, though :/
I can't find anything obvious in the tree. The alternative would be
to keep ZONE_DMA and set ARCH_ZONE_DMA_BITS.
> That said, I do agree that this is an appropriate change (the legacy of
> GFP_DMA is obviously horrible), so, provided we get plenty of time to find
> and fix the fallout when it lands:
>
> Reviewed-by: Robin Murphy <[email protected]>
I was hoping to get this into 4.15. What would be proper time to
fix the fallout?
On Wed, Jan 10, 2018 at 04:55:17PM +0100, Christoph Hellwig wrote:
> On Wed, Jan 10, 2018 at 12:58:14PM +0000, Robin Murphy wrote:
> > On 10/01/18 08:09, Christoph Hellwig wrote:
> >> arm64 uses ZONE_DMA for allocations below 32-bits. These days we
> >> name the zone for that ZONE_DMA32, which will allow to use the
> >> dma-direct and generic swiotlb code as-is, so rename it.
> >
> > I do wonder if we could also "upgrade" GFP_DMA to GFP_DMA32 somehow when
> > !ZONE_DMA - there are almost certainly arm64 drivers out there using a
> > combination of GFP_DMA and streaming mappings which will no longer get the
> > guaranteed 32-bit addresses they expect after this. I'm not sure quite how
> > feasible that is, though :/
>
> I can't find anything obvious in the tree. The alternative would be
> to keep ZONE_DMA and set ARCH_ZONE_DMA_BITS.
>
> > That said, I do agree that this is an appropriate change (the legacy of
> > GFP_DMA is obviously horrible), so, provided we get plenty of time to find
> > and fix the fallout when it lands:
> >
> > Reviewed-by: Robin Murphy <[email protected]>
>
> I was hoping to get this into 4.15. What would be proper time to
> fix the fallout?
Err, 4.16 of course.
On 10/01/18 15:46, Christoph Hellwig wrote:
> On Wed, Jan 10, 2018 at 12:22:18PM +0000, Robin Murphy wrote:
>>> + if (phys_addr == SWIOTLB_MAP_ERROR)
>>> + goto out_warn;
>>> - /* Confirm address can be DMA'd by device */
>>> - if (dev_addr + size - 1 > dma_mask) {
>>> - printk("hwdev DMA mask = 0x%016Lx, dev_addr = 0x%016Lx\n",
>>> - (unsigned long long)dma_mask,
>>> - (unsigned long long)dev_addr);
>>> + *dma_handle = swiotlb_phys_to_dma(dev, phys_addr);
>>
>> nit: this should probably go after the dma_coherent_ok() check (as with the
>> original logic).
>
> But the originall logic also needs the dma_addr_t for the
> dma_coherent_ok check:
>
> dev_addr = swiotlb_phys_to_dma(hwdev, paddr);
> /* Confirm address can be DMA'd by device */
> if (dev_addr + size - 1 > dma_mask) {
> ...
> goto err_warn;
> }
>
> or do you mean assining to *dma_handle? The dma_handle is not
> valid for a failure return, so I don't think this should matter.
Yeah, only the assignment - as I said, it's just a stylistic nit; no big
deal either way.
>>> + if (ret) {
>>> + *dma_handle = swiotlb_virt_to_bus(hwdev, ret);
>>> + if (dma_coherent_ok(hwdev, *dma_handle, size)) {
>>> + memset(ret, 0, size);
>>> + return ret;
>>> + }
>>
>> Aren't we leaking the pages here?
>
> Yes, that free_pages got lost somewhere in the rebases, I've added
> it back.
Cool.
Robin.
On 10/01/18 15:55, Christoph Hellwig wrote:
> On Wed, Jan 10, 2018 at 04:55:17PM +0100, Christoph Hellwig wrote:
>> On Wed, Jan 10, 2018 at 12:58:14PM +0000, Robin Murphy wrote:
>>> On 10/01/18 08:09, Christoph Hellwig wrote:
>>>> arm64 uses ZONE_DMA for allocations below 32-bits. These days we
>>>> name the zone for that ZONE_DMA32, which will allow to use the
>>>> dma-direct and generic swiotlb code as-is, so rename it.
>>>
>>> I do wonder if we could also "upgrade" GFP_DMA to GFP_DMA32 somehow when
>>> !ZONE_DMA - there are almost certainly arm64 drivers out there using a
>>> combination of GFP_DMA and streaming mappings which will no longer get the
>>> guaranteed 32-bit addresses they expect after this. I'm not sure quite how
>>> feasible that is, though :/
>>
>> I can't find anything obvious in the tree. The alternative would be
>> to keep ZONE_DMA and set ARCH_ZONE_DMA_BITS.
>>
>>> That said, I do agree that this is an appropriate change (the legacy of
>>> GFP_DMA is obviously horrible), so, provided we get plenty of time to find
>>> and fix the fallout when it lands:
>>>
>>> Reviewed-by: Robin Murphy <[email protected]>
>>
>> I was hoping to get this into 4.15. What would be proper time to
>> fix the fallout?
>
> Err, 4.16 of course.
Hee hee - cramming it into 4.15 is exactly what I wouldn't want to do,
even if Linus would accept it :)
Landing it this merge window for 4.16-rc1 sounds good if we can manage that.
Robin.
On 10/01/18 15:35, Christoph Hellwig wrote:
> On Wed, Jan 10, 2018 at 12:16:15PM +0000, Robin Murphy wrote:
>> On 10/01/18 08:09, Christoph Hellwig wrote:
>>> To properly reject too small DMA masks based on the addressability of the
>>> bounce buffer.
>>
>> I reckon this is self-evident enough that it should simply be squashed into
>> the previous patch.
>
> x86 didn't wire it up before, so I want a clear blaimpoint for this
> change instead of mixing it up.
That almost makes sense, if x86 were using this generic swiotlb_dma_ops
already. AFAICS it's only ia64, unicore and tile who end up using it,
and they all had swiotlb_dma_supported hooked up to begin with. Am I
missing something?
If regressions are going to happen, they'll surely point at whichever
commit pulls the ops into the relevant arch code - there doesn't seem to
be a great deal of value in having a piecemeal history of said ops
*before* that point.
Robin.
On Wed, Jan 10, 2018 at 09:09:13AM +0100, Christoph Hellwig wrote:
> We'll need that name for a generic implementation soon.
>
> Signed-off-by: Christoph Hellwig <[email protected]>
Reviewed-by: Konrad Rzeszutek Wilk <[email protected]>
> ---
> arch/ia64/hp/common/hwsw_iommu.c | 4 ++--
> arch/ia64/hp/common/sba_iommu.c | 6 +++---
> arch/ia64/kernel/pci-swiotlb.c | 6 +++---
> 3 files changed, 8 insertions(+), 8 deletions(-)
>
> diff --git a/arch/ia64/hp/common/hwsw_iommu.c b/arch/ia64/hp/common/hwsw_iommu.c
> index 63d8e1d2477f..41279f0442bd 100644
> --- a/arch/ia64/hp/common/hwsw_iommu.c
> +++ b/arch/ia64/hp/common/hwsw_iommu.c
> @@ -19,7 +19,7 @@
> #include <linux/export.h>
> #include <asm/machvec.h>
>
> -extern const struct dma_map_ops sba_dma_ops, swiotlb_dma_ops;
> +extern const struct dma_map_ops sba_dma_ops, ia64_swiotlb_dma_ops;
>
> /* swiotlb declarations & definitions: */
> extern int swiotlb_late_init_with_default_size (size_t size);
> @@ -38,7 +38,7 @@ static inline int use_swiotlb(struct device *dev)
> const struct dma_map_ops *hwsw_dma_get_ops(struct device *dev)
> {
> if (use_swiotlb(dev))
> - return &swiotlb_dma_ops;
> + return &ia64_swiotlb_dma_ops;
> return &sba_dma_ops;
> }
> EXPORT_SYMBOL(hwsw_dma_get_ops);
> diff --git a/arch/ia64/hp/common/sba_iommu.c b/arch/ia64/hp/common/sba_iommu.c
> index aec4a3354abe..8c0a9ae6afec 100644
> --- a/arch/ia64/hp/common/sba_iommu.c
> +++ b/arch/ia64/hp/common/sba_iommu.c
> @@ -2096,7 +2096,7 @@ static int __init acpi_sba_ioc_init_acpi(void)
> /* This has to run before acpi_scan_init(). */
> arch_initcall(acpi_sba_ioc_init_acpi);
>
> -extern const struct dma_map_ops swiotlb_dma_ops;
> +extern const struct dma_map_ops ia64_swiotlb_dma_ops;
>
> static int __init
> sba_init(void)
> @@ -2111,7 +2111,7 @@ sba_init(void)
> * a successful kdump kernel boot is to use the swiotlb.
> */
> if (is_kdump_kernel()) {
> - dma_ops = &swiotlb_dma_ops;
> + dma_ops = &ia64_swiotlb_dma_ops;
> if (swiotlb_late_init_with_default_size(64 * (1<<20)) != 0)
> panic("Unable to initialize software I/O TLB:"
> " Try machvec=dig boot option");
> @@ -2133,7 +2133,7 @@ sba_init(void)
> * If we didn't find something sba_iommu can claim, we
> * need to setup the swiotlb and switch to the dig machvec.
> */
> - dma_ops = &swiotlb_dma_ops;
> + dma_ops = &ia64_swiotlb_dma_ops;
> if (swiotlb_late_init_with_default_size(64 * (1<<20)) != 0)
> panic("Unable to find SBA IOMMU or initialize "
> "software I/O TLB: Try machvec=dig boot option");
> diff --git a/arch/ia64/kernel/pci-swiotlb.c b/arch/ia64/kernel/pci-swiotlb.c
> index 5e50939aa03e..f1ae873a8c35 100644
> --- a/arch/ia64/kernel/pci-swiotlb.c
> +++ b/arch/ia64/kernel/pci-swiotlb.c
> @@ -31,7 +31,7 @@ static void ia64_swiotlb_free_coherent(struct device *dev, size_t size,
> swiotlb_free_coherent(dev, size, vaddr, dma_addr);
> }
>
> -const struct dma_map_ops swiotlb_dma_ops = {
> +const struct dma_map_ops ia64_swiotlb_dma_ops = {
> .alloc = ia64_swiotlb_alloc_coherent,
> .free = ia64_swiotlb_free_coherent,
> .map_page = swiotlb_map_page,
> @@ -48,7 +48,7 @@ const struct dma_map_ops swiotlb_dma_ops = {
>
> void __init swiotlb_dma_init(void)
> {
> - dma_ops = &swiotlb_dma_ops;
> + dma_ops = &ia64_swiotlb_dma_ops;
> swiotlb_init(1);
> }
>
> @@ -60,7 +60,7 @@ void __init pci_swiotlb_init(void)
> printk(KERN_INFO "PCI-DMA: Re-initialize machine vector.\n");
> machvec_init("dig");
> swiotlb_init(1);
> - dma_ops = &swiotlb_dma_ops;
> + dma_ops = &ia64_swiotlb_dma_ops;
> #else
> panic("Unable to find Intel IOMMU");
> #endif
> --
> 2.14.2
>
On Wed, Jan 10, 2018 at 09:09:14AM +0100, Christoph Hellwig wrote:
> We'll need that name for a generic implementation soon.
>
Reviewed-by: Konrad Rzeszutek Wilk <[email protected]>
> Signed-off-by: Christoph Hellwig <[email protected]>
> ---
> arch/powerpc/include/asm/swiotlb.h | 2 +-
> arch/powerpc/kernel/dma-swiotlb.c | 4 ++--
> arch/powerpc/kernel/dma.c | 2 +-
> arch/powerpc/sysdev/fsl_pci.c | 2 +-
> 4 files changed, 5 insertions(+), 5 deletions(-)
>
> diff --git a/arch/powerpc/include/asm/swiotlb.h b/arch/powerpc/include/asm/swiotlb.h
> index 9341ee804d19..f65ecf57b66c 100644
> --- a/arch/powerpc/include/asm/swiotlb.h
> +++ b/arch/powerpc/include/asm/swiotlb.h
> @@ -13,7 +13,7 @@
>
> #include <linux/swiotlb.h>
>
> -extern const struct dma_map_ops swiotlb_dma_ops;
> +extern const struct dma_map_ops powerpc_swiotlb_dma_ops;
>
> extern unsigned int ppc_swiotlb_enable;
> int __init swiotlb_setup_bus_notifier(void);
> diff --git a/arch/powerpc/kernel/dma-swiotlb.c b/arch/powerpc/kernel/dma-swiotlb.c
> index f1e99b9cee97..506ac4fafac5 100644
> --- a/arch/powerpc/kernel/dma-swiotlb.c
> +++ b/arch/powerpc/kernel/dma-swiotlb.c
> @@ -46,7 +46,7 @@ static u64 swiotlb_powerpc_get_required(struct device *dev)
> * map_page, and unmap_page on highmem, use normal dma_ops
> * for everything else.
> */
> -const struct dma_map_ops swiotlb_dma_ops = {
> +const struct dma_map_ops powerpc_swiotlb_dma_ops = {
> .alloc = __dma_nommu_alloc_coherent,
> .free = __dma_nommu_free_coherent,
> .mmap = dma_nommu_mmap_coherent,
> @@ -89,7 +89,7 @@ static int ppc_swiotlb_bus_notify(struct notifier_block *nb,
>
> /* May need to bounce if the device can't address all of DRAM */
> if ((dma_get_mask(dev) + 1) < memblock_end_of_DRAM())
> - set_dma_ops(dev, &swiotlb_dma_ops);
> + set_dma_ops(dev, &powerpc_swiotlb_dma_ops);
>
> return NOTIFY_DONE;
> }
> diff --git a/arch/powerpc/kernel/dma.c b/arch/powerpc/kernel/dma.c
> index 76079841d3d0..da20569de9d4 100644
> --- a/arch/powerpc/kernel/dma.c
> +++ b/arch/powerpc/kernel/dma.c
> @@ -33,7 +33,7 @@ static u64 __maybe_unused get_pfn_limit(struct device *dev)
> struct dev_archdata __maybe_unused *sd = &dev->archdata;
>
> #ifdef CONFIG_SWIOTLB
> - if (sd->max_direct_dma_addr && dev->dma_ops == &swiotlb_dma_ops)
> + if (sd->max_direct_dma_addr && dev->dma_ops == &powerpc_swiotlb_dma_ops)
> pfn = min_t(u64, pfn, sd->max_direct_dma_addr >> PAGE_SHIFT);
> #endif
>
> diff --git a/arch/powerpc/sysdev/fsl_pci.c b/arch/powerpc/sysdev/fsl_pci.c
> index e4d0133bbeeb..61e07c78d64f 100644
> --- a/arch/powerpc/sysdev/fsl_pci.c
> +++ b/arch/powerpc/sysdev/fsl_pci.c
> @@ -118,7 +118,7 @@ static void setup_swiotlb_ops(struct pci_controller *hose)
> {
> if (ppc_swiotlb_enable) {
> hose->controller_ops.dma_dev_setup = pci_dma_dev_setup_swiotlb;
> - set_pci_dma_ops(&swiotlb_dma_ops);
> + set_pci_dma_ops(&powerpc_swiotlb_dma_ops);
> }
> }
> #else
> --
> 2.14.2
>
On Wed, Jan 10, 2018 at 09:09:15AM +0100, Christoph Hellwig wrote:
> We'll need that name for a generic implementation soon.
>
> Signed-off-by: Christoph Hellwig <[email protected]>
Reviewed-by: Konrad Rzeszutek Wilk <[email protected]>
> ---
> arch/x86/kernel/pci-swiotlb.c | 4 ++--
> 1 file changed, 2 insertions(+), 2 deletions(-)
>
> diff --git a/arch/x86/kernel/pci-swiotlb.c b/arch/x86/kernel/pci-swiotlb.c
> index 9d3e35c33d94..0d77603c2f50 100644
> --- a/arch/x86/kernel/pci-swiotlb.c
> +++ b/arch/x86/kernel/pci-swiotlb.c
> @@ -48,7 +48,7 @@ void x86_swiotlb_free_coherent(struct device *dev, size_t size,
> dma_generic_free_coherent(dev, size, vaddr, dma_addr, attrs);
> }
>
> -static const struct dma_map_ops swiotlb_dma_ops = {
> +static const struct dma_map_ops x86_swiotlb_dma_ops = {
> .mapping_error = swiotlb_dma_mapping_error,
> .alloc = x86_swiotlb_alloc_coherent,
> .free = x86_swiotlb_free_coherent,
> @@ -112,7 +112,7 @@ void __init pci_swiotlb_init(void)
> {
> if (swiotlb) {
> swiotlb_init(0);
> - dma_ops = &swiotlb_dma_ops;
> + dma_ops = &x86_swiotlb_dma_ops;
> }
> }
>
> --
> 2.14.2
>
On Wed, Jan 10, 2018 at 09:09:16AM +0100, Christoph Hellwig wrote:
OK?
Reviewed-by: Konrad Rzeszutek Wilk <[email protected]>
> Signed-off-by: Christoph Hellwig <[email protected]>
> ---
> arch/powerpc/kernel/dma-swiotlb.c | 2 +-
> arch/x86/kernel/pci-swiotlb.c | 2 +-
> include/linux/swiotlb.h | 4 ++--
> lib/swiotlb.c | 2 +-
> 4 files changed, 5 insertions(+), 5 deletions(-)
>
> diff --git a/arch/powerpc/kernel/dma-swiotlb.c b/arch/powerpc/kernel/dma-swiotlb.c
> index 506ac4fafac5..88f3963ca30f 100644
> --- a/arch/powerpc/kernel/dma-swiotlb.c
> +++ b/arch/powerpc/kernel/dma-swiotlb.c
> @@ -121,7 +121,7 @@ static int __init check_swiotlb_enabled(void)
> if (ppc_swiotlb_enable)
> swiotlb_print_info();
> else
> - swiotlb_free();
> + swiotlb_exit();
>
> return 0;
> }
> diff --git a/arch/x86/kernel/pci-swiotlb.c b/arch/x86/kernel/pci-swiotlb.c
> index 0d77603c2f50..0ee0f8f34251 100644
> --- a/arch/x86/kernel/pci-swiotlb.c
> +++ b/arch/x86/kernel/pci-swiotlb.c
> @@ -120,7 +120,7 @@ void __init pci_swiotlb_late_init(void)
> {
> /* An IOMMU turned us off. */
> if (!swiotlb)
> - swiotlb_free();
> + swiotlb_exit();
> else {
> printk(KERN_INFO "PCI-DMA: "
> "Using software bounce buffering for IO (SWIOTLB)\n");
> diff --git a/include/linux/swiotlb.h b/include/linux/swiotlb.h
> index 24ed817082ee..606375e35d87 100644
> --- a/include/linux/swiotlb.h
> +++ b/include/linux/swiotlb.h
> @@ -115,10 +115,10 @@ extern int
> swiotlb_dma_supported(struct device *hwdev, u64 mask);
>
> #ifdef CONFIG_SWIOTLB
> -extern void __init swiotlb_free(void);
> +extern void __init swiotlb_exit(void);
> unsigned int swiotlb_max_segment(void);
> #else
> -static inline void swiotlb_free(void) { }
> +static inline void swiotlb_exit(void) { }
> static inline unsigned int swiotlb_max_segment(void) { return 0; }
> #endif
>
> diff --git a/lib/swiotlb.c b/lib/swiotlb.c
> index 125c1062119f..cf5311908fa9 100644
> --- a/lib/swiotlb.c
> +++ b/lib/swiotlb.c
> @@ -417,7 +417,7 @@ swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs)
> return -ENOMEM;
> }
>
> -void __init swiotlb_free(void)
> +void __init swiotlb_exit(void)
> {
> if (!io_tlb_orig_addr)
> return;
> --
> 2.14.2
>
On Wed, Jan 10, 2018 at 05:02:30PM +0000, Robin Murphy wrote:
>>>
>>> Aren't we leaking the pages here?
>>
>> Yes, that free_pages got lost somewhere in the rebases, I've added
>> it back.
>
> Cool.
FYI, here is the fixed version, I don't want to re-send the whole
series for this fix:
http://git.infradead.org/users/hch/misc.git/commitdiff/0176adb004065d6815a8e67946752df4cd947c5b
I've pulled this into the dma-mapping for-next tree, including the
missing free_pages noted. I'd be fine to rebase another day or two
for additional reviews or important fixes.
Hi Konrad,
can you send the first patch to Linus for inclusion in 4.15 if you
haven't already done so?
I'm still getting reports from people complaining about the error message.
Thanks,
Christian.
Am 16.01.2018 um 08:53 schrieb Christoph Hellwig:
> I've pulled this into the dma-mapping for-next tree, including the
> missing free_pages noted. I'd be fine to rebase another day or two
> for additional reviews or important fixes.
On Tue, Jan 16, 2018 at 09:22:52AM +0100, Christian K?nig wrote:
> Hi Konrad,
>
> can you send the first patch to Linus for inclusion in 4.15 if you haven't
> already done so?
It's in the 4.16 queue with a cc to stable. I guess we're ok with
a duplicate commit if we have to.
Am 16.01.2018 um 09:28 schrieb Christoph Hellwig:
> On Tue, Jan 16, 2018 at 09:22:52AM +0100, Christian König wrote:
>> Hi Konrad,
>>
>> can you send the first patch to Linus for inclusion in 4.15 if you haven't
>> already done so?
> It's in the 4.16 queue with a cc to stable. I guess we're ok with
> a duplicate commit if we have to.
Yeah, while it's only a false positive warning it would be really nice
to have in 4.15.
It affects all drivers using TTM in the system and not just the two I'm
the maintainer of.
Regards,
Christian.