When copying from/to an iomem mapped memory, the current sg_copy & sg_pcopy can't
be used and lead to local variants in drivers like in [1] & [2].
This introduces an I/O variant to be used instead of the local variants.
[1] mv_cesa_sg_copy in drivers/crypto/marvell/cesa/tdma.c
[2] meson_mmc_copy_buffer in drivers/mmc/host/meson-gx-mmc.c
Cc: Robin Murphy <[email protected]>
Signed-off-by: Neil Armstrong <[email protected]>
---
include/linux/scatterlist.h | 14 +++++
lib/scatterlist.c | 119 ++++++++++++++++++++++++++++++++++++
2 files changed, 133 insertions(+)
diff --git a/include/linux/scatterlist.h b/include/linux/scatterlist.h
index 6f70572b2938..6ef339ba5290 100644
--- a/include/linux/scatterlist.h
+++ b/include/linux/scatterlist.h
@@ -308,15 +308,29 @@ void sgl_free(struct scatterlist *sgl);
size_t sg_copy_buffer(struct scatterlist *sgl, unsigned int nents, void *buf,
size_t buflen, off_t skip, bool to_buffer);
+size_t sg_copy_io(struct scatterlist *sgl, unsigned int nents, void __iomem *buf,
+ size_t buflen, off_t skip, bool to_buffer);
+
size_t sg_copy_from_buffer(struct scatterlist *sgl, unsigned int nents,
const void *buf, size_t buflen);
size_t sg_copy_to_buffer(struct scatterlist *sgl, unsigned int nents,
void *buf, size_t buflen);
+size_t sg_copy_from_io(struct scatterlist *sgl, unsigned int nents,
+ const void __iomem *buf, size_t buflen);
+size_t sg_copy_to_io(struct scatterlist *sgl, unsigned int nents,
+ void __iomem *buf, size_t buflen);
+
size_t sg_pcopy_from_buffer(struct scatterlist *sgl, unsigned int nents,
const void *buf, size_t buflen, off_t skip);
size_t sg_pcopy_to_buffer(struct scatterlist *sgl, unsigned int nents,
void *buf, size_t buflen, off_t skip);
+
+size_t sg_pcopy_from_io(struct scatterlist *sgl, unsigned int nents,
+ const void __iomem *buf, size_t buflen, off_t skip);
+size_t sg_pcopy_to_io(struct scatterlist *sgl, unsigned int nents,
+ void __iomem *buf, size_t buflen, off_t skip);
+
size_t sg_zero_buffer(struct scatterlist *sgl, unsigned int nents,
size_t buflen, off_t skip);
diff --git a/lib/scatterlist.c b/lib/scatterlist.c
index a59778946404..e52f37b181fa 100644
--- a/lib/scatterlist.c
+++ b/lib/scatterlist.c
@@ -954,6 +954,55 @@ size_t sg_copy_buffer(struct scatterlist *sgl, unsigned int nents, void *buf,
}
EXPORT_SYMBOL(sg_copy_buffer);
+/**
+ * sg_copy_io - Copy data between an I/O mapped buffer and an SG list
+ * @sgl: The SG list
+ * @nents: Number of SG entries
+ * @buf: Where to copy from
+ * @buflen: The number of bytes to copy
+ * @skip: Number of bytes to skip before copying
+ * @to_buffer: transfer direction (true == from an sg list to a
+ * buffer, false == from a buffer to an sg list)
+ *
+ * Returns the number of copied bytes.
+ *
+ **/
+size_t sg_copy_io(struct scatterlist *sgl, unsigned int nents, void __iomem *buf,
+ size_t buflen, off_t skip, bool to_buffer)
+{
+ unsigned int offset = 0;
+ struct sg_mapping_iter miter;
+ unsigned int sg_flags = SG_MITER_ATOMIC;
+
+ if (to_buffer)
+ sg_flags |= SG_MITER_FROM_SG;
+ else
+ sg_flags |= SG_MITER_TO_SG;
+
+ sg_miter_start(&miter, sgl, nents, sg_flags);
+
+ if (!sg_miter_skip(&miter, skip))
+ return 0;
+
+ while ((offset < buflen) && sg_miter_next(&miter)) {
+ unsigned int len;
+
+ len = min(miter.length, buflen - offset);
+
+ if (to_buffer)
+ memcpy_toio(buf + offset, miter.addr, len);
+ else
+ memcpy_fromio(miter.addr, buf + offset, len);
+
+ offset += len;
+ }
+
+ sg_miter_stop(&miter);
+
+ return offset;
+}
+EXPORT_SYMBOL(sg_copy_io);
+
/**
* sg_copy_from_buffer - Copy from a linear buffer to an SG list
* @sgl: The SG list
@@ -988,6 +1037,40 @@ size_t sg_copy_to_buffer(struct scatterlist *sgl, unsigned int nents,
}
EXPORT_SYMBOL(sg_copy_to_buffer);
+/**
+ * sg_copy_from_io - Copy from an I/O mapped buffer to an SG list
+ * @sgl: The SG list
+ * @nents: Number of SG entries
+ * @buf: Where to copy from
+ * @buflen: The number of bytes to copy
+ *
+ * Returns the number of copied bytes.
+ *
+ **/
+size_t sg_copy_from_io(struct scatterlist *sgl, unsigned int nents,
+ const void *buf, size_t buflen)
+{
+ return sg_copy_io(sgl, nents, (void __iomem *)buf, buflen, 0, false);
+}
+EXPORT_SYMBOL(sg_copy_from_io);
+
+/**
+ * sg_copy_to_io - Copy from an SG list to an I/O mapped buffer
+ * @sgl: The SG list
+ * @nents: Number of SG entries
+ * @buf: Where to copy to
+ * @buflen: The number of bytes to copy
+ *
+ * Returns the number of copied bytes.
+ *
+ **/
+size_t sg_copy_to_io(struct scatterlist *sgl, unsigned int nents,
+ void __iomem *buf, size_t buflen)
+{
+ return sg_copy_io(sgl, nents, buf, buflen, 0, true);
+}
+EXPORT_SYMBOL(sg_copy_to_io);
+
/**
* sg_pcopy_from_buffer - Copy from a linear buffer to an SG list
* @sgl: The SG list
@@ -1024,6 +1107,42 @@ size_t sg_pcopy_to_buffer(struct scatterlist *sgl, unsigned int nents,
}
EXPORT_SYMBOL(sg_pcopy_to_buffer);
+/**
+ * sg_pcopy_from_io - Copy from an I/O mapped buffer to an SG list
+ * @sgl: The SG list
+ * @nents: Number of SG entries
+ * @buf: Where to copy from
+ * @buflen: The number of bytes to copy
+ * @skip: Number of bytes to skip before copying
+ *
+ * Returns the number of copied bytes.
+ *
+ **/
+size_t sg_pcopy_from_io(struct scatterlist *sgl, unsigned int nents,
+ const void __iomem *buf, size_t buflen, off_t skip)
+{
+ return sg_copy_io(sgl, nents, (void __iomem *)buf, buflen, skip, false);
+}
+EXPORT_SYMBOL(sg_pcopy_from_io);
+
+/**
+ * sg_pcopy_to_io - Copy from an SG list to an I/O mapped buffer
+ * @sgl: The SG list
+ * @nents: Number of SG entries
+ * @buf: Where to copy to
+ * @buflen: The number of bytes to copy
+ * @skip: Number of bytes to skip before copying
+ *
+ * Returns the number of copied bytes.
+ *
+ **/
+size_t sg_pcopy_to_io(struct scatterlist *sgl, unsigned int nents,
+ void __iomem *buf, size_t buflen, off_t skip)
+{
+ return sg_copy_io(sgl, nents, buf, buflen, skip, true);
+}
+EXPORT_SYMBOL(sg_pcopy_to_io);
+
/**
* sg_zero_buffer - Zero-out a part of a SG list
* @sgl: The SG list
--
2.25.1
On 2021-06-28 13:34, Neil Armstrong wrote:
> When copying from/to an iomem mapped memory, the current sg_copy & sg_pcopy can't
> be used and lead to local variants in drivers like in [1] & [2].
>
> This introduces an I/O variant to be used instead of the local variants.
>
> [1] mv_cesa_sg_copy in drivers/crypto/marvell/cesa/tdma.c
> [2] meson_mmc_copy_buffer in drivers/mmc/host/meson-gx-mmc.c
Other than one apparent typo below, this looks like what I imagined,
thanks for putting it together!
> Cc: Robin Murphy <[email protected]>
> Signed-off-by: Neil Armstrong <[email protected]>
> ---
> include/linux/scatterlist.h | 14 +++++
> lib/scatterlist.c | 119 ++++++++++++++++++++++++++++++++++++
> 2 files changed, 133 insertions(+)
>
> diff --git a/include/linux/scatterlist.h b/include/linux/scatterlist.h
> index 6f70572b2938..6ef339ba5290 100644
> --- a/include/linux/scatterlist.h
> +++ b/include/linux/scatterlist.h
> @@ -308,15 +308,29 @@ void sgl_free(struct scatterlist *sgl);
> size_t sg_copy_buffer(struct scatterlist *sgl, unsigned int nents, void *buf,
> size_t buflen, off_t skip, bool to_buffer);
>
> +size_t sg_copy_io(struct scatterlist *sgl, unsigned int nents, void __iomem *buf,
> + size_t buflen, off_t skip, bool to_buffer);
> +
> size_t sg_copy_from_buffer(struct scatterlist *sgl, unsigned int nents,
> const void *buf, size_t buflen);
> size_t sg_copy_to_buffer(struct scatterlist *sgl, unsigned int nents,
> void *buf, size_t buflen);
>
> +size_t sg_copy_from_io(struct scatterlist *sgl, unsigned int nents,
> + const void __iomem *buf, size_t buflen);
> +size_t sg_copy_to_io(struct scatterlist *sgl, unsigned int nents,
> + void __iomem *buf, size_t buflen);
> +
> size_t sg_pcopy_from_buffer(struct scatterlist *sgl, unsigned int nents,
> const void *buf, size_t buflen, off_t skip);
> size_t sg_pcopy_to_buffer(struct scatterlist *sgl, unsigned int nents,
> void *buf, size_t buflen, off_t skip);
> +
> +size_t sg_pcopy_from_io(struct scatterlist *sgl, unsigned int nents,
> + const void __iomem *buf, size_t buflen, off_t skip);
> +size_t sg_pcopy_to_io(struct scatterlist *sgl, unsigned int nents,
> + void __iomem *buf, size_t buflen, off_t skip);
> +
> size_t sg_zero_buffer(struct scatterlist *sgl, unsigned int nents,
> size_t buflen, off_t skip);
>
> diff --git a/lib/scatterlist.c b/lib/scatterlist.c
> index a59778946404..e52f37b181fa 100644
> --- a/lib/scatterlist.c
> +++ b/lib/scatterlist.c
> @@ -954,6 +954,55 @@ size_t sg_copy_buffer(struct scatterlist *sgl, unsigned int nents, void *buf,
> }
> EXPORT_SYMBOL(sg_copy_buffer);
>
> +/**
> + * sg_copy_io - Copy data between an I/O mapped buffer and an SG list
> + * @sgl: The SG list
> + * @nents: Number of SG entries
> + * @buf: Where to copy from
> + * @buflen: The number of bytes to copy
> + * @skip: Number of bytes to skip before copying
> + * @to_buffer: transfer direction (true == from an sg list to a
> + * buffer, false == from a buffer to an sg list)
> + *
> + * Returns the number of copied bytes.
> + *
> + **/
> +size_t sg_copy_io(struct scatterlist *sgl, unsigned int nents, void __iomem *buf,
> + size_t buflen, off_t skip, bool to_buffer)
> +{
> + unsigned int offset = 0;
> + struct sg_mapping_iter miter;
> + unsigned int sg_flags = SG_MITER_ATOMIC;
> +
> + if (to_buffer)
> + sg_flags |= SG_MITER_FROM_SG;
> + else
> + sg_flags |= SG_MITER_TO_SG;
> +
> + sg_miter_start(&miter, sgl, nents, sg_flags);
> +
> + if (!sg_miter_skip(&miter, skip))
> + return 0;
> +
> + while ((offset < buflen) && sg_miter_next(&miter)) {
> + unsigned int len;
> +
> + len = min(miter.length, buflen - offset);
> +
> + if (to_buffer)
> + memcpy_toio(buf + offset, miter.addr, len);
> + else
> + memcpy_fromio(miter.addr, buf + offset, len);
> +
> + offset += len;
> + }
> +
> + sg_miter_stop(&miter);
> +
> + return offset;
> +}
> +EXPORT_SYMBOL(sg_copy_io);
> +
> /**
> * sg_copy_from_buffer - Copy from a linear buffer to an SG list
> * @sgl: The SG list
> @@ -988,6 +1037,40 @@ size_t sg_copy_to_buffer(struct scatterlist *sgl, unsigned int nents,
> }
> EXPORT_SYMBOL(sg_copy_to_buffer);
>
> +/**
> + * sg_copy_from_io - Copy from an I/O mapped buffer to an SG list
> + * @sgl: The SG list
> + * @nents: Number of SG entries
> + * @buf: Where to copy from
> + * @buflen: The number of bytes to copy
> + *
> + * Returns the number of copied bytes.
> + *
> + **/
> +size_t sg_copy_from_io(struct scatterlist *sgl, unsigned int nents,
> + const void *buf, size_t buflen)
The __iomem annotation wants to be on buf here raher than cast in below,
to match the prototype (and everything else).
Cheers,
Robin.
> +{
> + return sg_copy_io(sgl, nents, (void __iomem *)buf, buflen, 0, false);
> +}
> +EXPORT_SYMBOL(sg_copy_from_io);
> +
> +/**
> + * sg_copy_to_io - Copy from an SG list to an I/O mapped buffer
> + * @sgl: The SG list
> + * @nents: Number of SG entries
> + * @buf: Where to copy to
> + * @buflen: The number of bytes to copy
> + *
> + * Returns the number of copied bytes.
> + *
> + **/
> +size_t sg_copy_to_io(struct scatterlist *sgl, unsigned int nents,
> + void __iomem *buf, size_t buflen)
> +{
> + return sg_copy_io(sgl, nents, buf, buflen, 0, true);
> +}
> +EXPORT_SYMBOL(sg_copy_to_io);
> +
> /**
> * sg_pcopy_from_buffer - Copy from a linear buffer to an SG list
> * @sgl: The SG list
> @@ -1024,6 +1107,42 @@ size_t sg_pcopy_to_buffer(struct scatterlist *sgl, unsigned int nents,
> }
> EXPORT_SYMBOL(sg_pcopy_to_buffer);
>
> +/**
> + * sg_pcopy_from_io - Copy from an I/O mapped buffer to an SG list
> + * @sgl: The SG list
> + * @nents: Number of SG entries
> + * @buf: Where to copy from
> + * @buflen: The number of bytes to copy
> + * @skip: Number of bytes to skip before copying
> + *
> + * Returns the number of copied bytes.
> + *
> + **/
> +size_t sg_pcopy_from_io(struct scatterlist *sgl, unsigned int nents,
> + const void __iomem *buf, size_t buflen, off_t skip)
> +{
> + return sg_copy_io(sgl, nents, (void __iomem *)buf, buflen, skip, false);
> +}
> +EXPORT_SYMBOL(sg_pcopy_from_io);
> +
> +/**
> + * sg_pcopy_to_io - Copy from an SG list to an I/O mapped buffer
> + * @sgl: The SG list
> + * @nents: Number of SG entries
> + * @buf: Where to copy to
> + * @buflen: The number of bytes to copy
> + * @skip: Number of bytes to skip before copying
> + *
> + * Returns the number of copied bytes.
> + *
> + **/
> +size_t sg_pcopy_to_io(struct scatterlist *sgl, unsigned int nents,
> + void __iomem *buf, size_t buflen, off_t skip)
> +{
> + return sg_copy_io(sgl, nents, buf, buflen, skip, true);
> +}
> +EXPORT_SYMBOL(sg_pcopy_to_io);
> +
> /**
> * sg_zero_buffer - Zero-out a part of a SG list
> * @sgl: The SG list
>