These patches are in preparation for more substantial changes to how
Read chunks are managed by svcrdma. There are a lot of patches in
this series, but each change is narrow and no external behavior
changes are expected.
The main benefit of this series is that svc_rdma_read_info is no
longer dynamically allocated.
---
Chuck Lever (21):
svcrdma: Reduce size of struct svc_rdma_rw_ctxt
svcrdma: Acquire the svcxprt_rdma pointer from the CQ context
svcrdma: Explicitly pass the transport into Write chunk I/O paths
svcrdma: Explicitly pass the transport into Read chunk I/O paths
svcrdma: Explicitly pass the transport to svc_rdma_post_chunk_ctxt()
svcrdma: Pass a pointer to the transport to svc_rdma_cc_release()
svcrdma: Remove the svc_rdma_chunk_ctxt::cc_rdma field
svcrdma: Move struct svc_rdma_chunk_ctxt to svc_rdma.h
svcrdma: Start moving fields out of struct svc_rdma_read_info
svcrdma: Move svc_rdma_read_info::ri_pageno to struct svc_rdma_recv_ctxt
svcrdma: Move read_info::ri_pageoff into struct svc_rdma_recv_ctxt
svcrdma: Update synopsis of svc_rdma_build_read_segment()
svcrdma: Update synopsis of svc_rdma_build_read_chunk()
svcrdma: Update synopsis of svc_rdma_read_chunk_range()
svcrdma: Update the synopsis of svc_rdma_read_data_item()
svcrdma: Update synopsis of svc_rdma_copy_inline_range()
svcrdma: Update synopsis of svc_rdma_read_multiple_chunks()
svcrdma: Update the synopsis of svc_rdma_read_call_chunk()
svcrdma: Update the synopsis of svc_rdma_read_special()
svcrdma: Remove struct svc_rdma_read_info
svcrdma: Move the svc_rdma_cc_init() call
include/linux/sunrpc/svc_rdma.h | 30 +++
net/sunrpc/xprtrdma/svc_rdma_recvfrom.c | 1 +
net/sunrpc/xprtrdma/svc_rdma_rw.c | 310 +++++++++++-------------
3 files changed, 169 insertions(+), 172 deletions(-)
--
Chuck Lever
From: Chuck Lever <[email protected]>
SG_CHUNK_SIZE is 128, making struct svc_rdma_rw_ctxt + the first
SGL array more than 4200 bytes in length, pushing the memory
allocation well into order 1.
Even so, the RDMA rw core doesn't seem to use more than max_send_sge
entries in that array (typically 32 or less), so that is all wasted
space.
Signed-off-by: Chuck Lever <[email protected]>
---
net/sunrpc/xprtrdma/svc_rdma_rw.c | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c
index c06676714417..69010ab7f0c3 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_rw.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c
@@ -39,6 +39,7 @@ struct svc_rdma_rw_ctxt {
struct list_head rw_list;
struct rdma_rw_ctx rw_ctx;
unsigned int rw_nents;
+ unsigned int rw_first_sgl_nents;
struct sg_table rw_sg_table;
struct scatterlist rw_first_sgl[];
};
@@ -53,6 +54,8 @@ svc_rdma_next_ctxt(struct list_head *list)
static struct svc_rdma_rw_ctxt *
svc_rdma_get_rw_ctxt(struct svcxprt_rdma *rdma, unsigned int sges)
{
+ struct ib_device *dev = rdma->sc_cm_id->device;
+ unsigned int first_sgl_nents = dev->attrs.max_send_sge;
struct svc_rdma_rw_ctxt *ctxt;
struct llist_node *node;
@@ -62,18 +65,19 @@ svc_rdma_get_rw_ctxt(struct svcxprt_rdma *rdma, unsigned int sges)
if (node) {
ctxt = llist_entry(node, struct svc_rdma_rw_ctxt, rw_node);
} else {
- ctxt = kmalloc_node(struct_size(ctxt, rw_first_sgl, SG_CHUNK_SIZE),
- GFP_KERNEL, ibdev_to_node(rdma->sc_cm_id->device));
+ ctxt = kmalloc_node(struct_size(ctxt, rw_first_sgl, first_sgl_nents),
+ GFP_KERNEL, ibdev_to_node(dev));
if (!ctxt)
goto out_noctx;
INIT_LIST_HEAD(&ctxt->rw_list);
+ ctxt->rw_first_sgl_nents = first_sgl_nents;
}
ctxt->rw_sg_table.sgl = ctxt->rw_first_sgl;
if (sg_alloc_table_chained(&ctxt->rw_sg_table, sges,
ctxt->rw_sg_table.sgl,
- SG_CHUNK_SIZE))
+ first_sgl_nents))
goto out_free;
return ctxt;
@@ -87,7 +91,7 @@ svc_rdma_get_rw_ctxt(struct svcxprt_rdma *rdma, unsigned int sges)
static void __svc_rdma_put_rw_ctxt(struct svc_rdma_rw_ctxt *ctxt,
struct llist_head *list)
{
- sg_free_table_chained(&ctxt->rw_sg_table, SG_CHUNK_SIZE);
+ sg_free_table_chained(&ctxt->rw_sg_table, ctxt->rw_first_sgl_nents);
llist_add(&ctxt->rw_node, list);
}
From: Chuck Lever <[email protected]>
Enable the removal of the svc_rdma_chunk_ctxt::cc_rdma field in a
subsequent patch.
Signed-off-by: Chuck Lever <[email protected]>
---
net/sunrpc/xprtrdma/svc_rdma_rw.c | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c
index 69010ab7f0c3..6fa818dc5b11 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_rw.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c
@@ -278,10 +278,10 @@ static void svc_rdma_write_info_free(struct svc_rdma_write_info *info)
*/
static void svc_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc)
{
+ struct svcxprt_rdma *rdma = cq->cq_context;
struct ib_cqe *cqe = wc->wr_cqe;
struct svc_rdma_chunk_ctxt *cc =
container_of(cqe, struct svc_rdma_chunk_ctxt, cc_cqe);
- struct svcxprt_rdma *rdma = cc->cc_rdma;
struct svc_rdma_write_info *info =
container_of(cc, struct svc_rdma_write_info, wi_cc);
@@ -345,6 +345,7 @@ static void svc_rdma_read_info_free(struct svc_rdma_read_info *info)
*/
static void svc_rdma_wc_read_done(struct ib_cq *cq, struct ib_wc *wc)
{
+ struct svcxprt_rdma *rdma = cq->cq_context;
struct ib_cqe *cqe = wc->wr_cqe;
struct svc_rdma_chunk_ctxt *cc =
container_of(cqe, struct svc_rdma_chunk_ctxt, cc_cqe);
@@ -363,7 +364,7 @@ static void svc_rdma_wc_read_done(struct ib_cq *cq, struct ib_wc *wc)
trace_svcrdma_wc_read_err(wc, &cc->cc_cid);
}
- svc_rdma_wake_send_waiters(cc->cc_rdma, cc->cc_sqecount);
+ svc_rdma_wake_send_waiters(rdma, cc->cc_sqecount);
cc->cc_status = wc->status;
complete(&cc->cc_done);
return;
From: Chuck Lever <[email protected]>
Enable the eventual removal of the svc_rdma_chunk_ctxt::cc_rdma
field.
Signed-off-by: Chuck Lever <[email protected]>
---
net/sunrpc/xprtrdma/svc_rdma_rw.c | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c
index 6fa818dc5b11..ef2579141c33 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_rw.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c
@@ -220,6 +220,8 @@ static void svc_rdma_cc_release(struct svc_rdma_chunk_ctxt *cc,
* - Stores arguments for the SGL constructor functions
*/
struct svc_rdma_write_info {
+ struct svcxprt_rdma *wi_rdma;
+
const struct svc_rdma_chunk *wi_chunk;
/* write state of this chunk */
@@ -246,6 +248,7 @@ svc_rdma_write_info_alloc(struct svcxprt_rdma *rdma,
if (!info)
return info;
+ info->wi_rdma = rdma;
info->wi_chunk = chunk;
info->wi_seg_off = 0;
info->wi_seg_no = 0;
@@ -489,7 +492,7 @@ svc_rdma_build_writes(struct svc_rdma_write_info *info,
unsigned int remaining)
{
struct svc_rdma_chunk_ctxt *cc = &info->wi_cc;
- struct svcxprt_rdma *rdma = cc->cc_rdma;
+ struct svcxprt_rdma *rdma = info->wi_rdma;
const struct svc_rdma_segment *seg;
struct svc_rdma_rw_ctxt *ctxt;
int ret;
From: Chuck Lever <[email protected]>
Enable the eventual removal of the svc_rdma_chunk_ctxt::cc_rdma
field.
Signed-off-by: Chuck Lever <[email protected]>
---
net/sunrpc/xprtrdma/svc_rdma_rw.c | 58 +++++++++++++++++++++++--------------
1 file changed, 36 insertions(+), 22 deletions(-)
diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c
index ef2579141c33..cda57a5f8ba0 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_rw.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c
@@ -712,6 +712,7 @@ int svc_rdma_send_reply_chunk(struct svcxprt_rdma *rdma,
/**
* svc_rdma_build_read_segment - Build RDMA Read WQEs to pull one RDMA segment
+ * @rdma: controlling transport
* @info: context for ongoing I/O
* @segment: co-ordinates of remote memory to be read
*
@@ -721,7 +722,8 @@ int svc_rdma_send_reply_chunk(struct svcxprt_rdma *rdma,
* %-ENOMEM: allocating a local resources failed
* %-EIO: a DMA mapping error occurred
*/
-static int svc_rdma_build_read_segment(struct svc_rdma_read_info *info,
+static int svc_rdma_build_read_segment(struct svcxprt_rdma *rdma,
+ struct svc_rdma_read_info *info,
const struct svc_rdma_segment *segment)
{
struct svc_rdma_recv_ctxt *head = info->ri_readctxt;
@@ -734,7 +736,7 @@ static int svc_rdma_build_read_segment(struct svc_rdma_read_info *info,
len = segment->rs_length;
sge_no = PAGE_ALIGN(info->ri_pageoff + len) >> PAGE_SHIFT;
- ctxt = svc_rdma_get_rw_ctxt(cc->cc_rdma, sge_no);
+ ctxt = svc_rdma_get_rw_ctxt(rdma, sge_no);
if (!ctxt)
return -ENOMEM;
ctxt->rw_nents = sge_no;
@@ -764,7 +766,7 @@ static int svc_rdma_build_read_segment(struct svc_rdma_read_info *info,
goto out_overrun;
}
- ret = svc_rdma_rw_ctx_init(cc->cc_rdma, ctxt, segment->rs_offset,
+ ret = svc_rdma_rw_ctx_init(rdma, ctxt, segment->rs_offset,
segment->rs_handle, DMA_FROM_DEVICE);
if (ret < 0)
return -EIO;
@@ -781,6 +783,7 @@ static int svc_rdma_build_read_segment(struct svc_rdma_read_info *info,
/**
* svc_rdma_build_read_chunk - Build RDMA Read WQEs to pull one RDMA chunk
+ * @rdma: controlling transport
* @info: context for ongoing I/O
* @chunk: Read chunk to pull
*
@@ -790,7 +793,8 @@ static int svc_rdma_build_read_segment(struct svc_rdma_read_info *info,
* %-ENOMEM: allocating a local resources failed
* %-EIO: a DMA mapping error occurred
*/
-static int svc_rdma_build_read_chunk(struct svc_rdma_read_info *info,
+static int svc_rdma_build_read_chunk(struct svcxprt_rdma *rdma,
+ struct svc_rdma_read_info *info,
const struct svc_rdma_chunk *chunk)
{
const struct svc_rdma_segment *segment;
@@ -798,7 +802,7 @@ static int svc_rdma_build_read_chunk(struct svc_rdma_read_info *info,
ret = -EINVAL;
pcl_for_each_segment(segment, chunk) {
- ret = svc_rdma_build_read_segment(info, segment);
+ ret = svc_rdma_build_read_segment(rdma, info, segment);
if (ret < 0)
break;
info->ri_totalbytes += segment->rs_length;
@@ -858,6 +862,7 @@ static int svc_rdma_copy_inline_range(struct svc_rdma_read_info *info,
/**
* svc_rdma_read_multiple_chunks - Construct RDMA Reads to pull data item Read chunks
+ * @rdma: controlling transport
* @info: context for RDMA Reads
*
* The chunk data lands in rqstp->rq_arg as a series of contiguous pages,
@@ -870,7 +875,8 @@ static int svc_rdma_copy_inline_range(struct svc_rdma_read_info *info,
* %-ENOTCONN: posting failed (connection is lost),
* %-EIO: rdma_rw initialization failed (DMA mapping, etc).
*/
-static noinline int svc_rdma_read_multiple_chunks(struct svc_rdma_read_info *info)
+static noinline int svc_rdma_read_multiple_chunks(struct svcxprt_rdma *rdma,
+ struct svc_rdma_read_info *info)
{
struct svc_rdma_recv_ctxt *head = info->ri_readctxt;
const struct svc_rdma_pcl *pcl = &head->rc_read_pcl;
@@ -887,7 +893,7 @@ static noinline int svc_rdma_read_multiple_chunks(struct svc_rdma_read_info *inf
return ret;
pcl_for_each_chunk(chunk, pcl) {
- ret = svc_rdma_build_read_chunk(info, chunk);
+ ret = svc_rdma_build_read_chunk(rdma, info, chunk);
if (ret < 0)
return ret;
@@ -920,6 +926,7 @@ static noinline int svc_rdma_read_multiple_chunks(struct svc_rdma_read_info *inf
/**
* svc_rdma_read_data_item - Construct RDMA Reads to pull data item Read chunks
+ * @rdma: controlling transport
* @info: context for RDMA Reads
*
* The chunk data lands in the page list of rqstp->rq_arg.pages.
@@ -935,7 +942,8 @@ static noinline int svc_rdma_read_multiple_chunks(struct svc_rdma_read_info *inf
* %-ENOTCONN: posting failed (connection is lost),
* %-EIO: rdma_rw initialization failed (DMA mapping, etc).
*/
-static int svc_rdma_read_data_item(struct svc_rdma_read_info *info)
+static int svc_rdma_read_data_item(struct svcxprt_rdma *rdma,
+ struct svc_rdma_read_info *info)
{
struct svc_rdma_recv_ctxt *head = info->ri_readctxt;
struct xdr_buf *buf = &info->ri_rqst->rq_arg;
@@ -944,7 +952,7 @@ static int svc_rdma_read_data_item(struct svc_rdma_read_info *info)
int ret;
chunk = pcl_first_chunk(&head->rc_read_pcl);
- ret = svc_rdma_build_read_chunk(info, chunk);
+ ret = svc_rdma_build_read_chunk(rdma, info, chunk);
if (ret < 0)
goto out;
@@ -978,6 +986,7 @@ static int svc_rdma_read_data_item(struct svc_rdma_read_info *info)
/**
* svc_rdma_read_chunk_range - Build RDMA Read WQEs for portion of a chunk
+ * @rdma: controlling transport
* @info: context for RDMA Reads
* @chunk: parsed Call chunk to pull
* @offset: offset of region to pull
@@ -990,7 +999,8 @@ static int svc_rdma_read_data_item(struct svc_rdma_read_info *info)
* %-ENOTCONN: posting failed (connection is lost),
* %-EIO: rdma_rw initialization failed (DMA mapping, etc).
*/
-static int svc_rdma_read_chunk_range(struct svc_rdma_read_info *info,
+static int svc_rdma_read_chunk_range(struct svcxprt_rdma *rdma,
+ struct svc_rdma_read_info *info,
const struct svc_rdma_chunk *chunk,
unsigned int offset, unsigned int length)
{
@@ -1010,7 +1020,7 @@ static int svc_rdma_read_chunk_range(struct svc_rdma_read_info *info,
dummy.rs_length = min_t(u32, length, segment->rs_length) - offset;
dummy.rs_offset = segment->rs_offset + offset;
- ret = svc_rdma_build_read_segment(info, &dummy);
+ ret = svc_rdma_build_read_segment(rdma, info, &dummy);
if (ret < 0)
break;
@@ -1023,6 +1033,7 @@ static int svc_rdma_read_chunk_range(struct svc_rdma_read_info *info,
/**
* svc_rdma_read_call_chunk - Build RDMA Read WQEs to pull a Long Message
+ * @rdma: controlling transport
* @info: context for RDMA Reads
*
* Return values:
@@ -1032,7 +1043,8 @@ static int svc_rdma_read_chunk_range(struct svc_rdma_read_info *info,
* %-ENOTCONN: posting failed (connection is lost),
* %-EIO: rdma_rw initialization failed (DMA mapping, etc).
*/
-static int svc_rdma_read_call_chunk(struct svc_rdma_read_info *info)
+static int svc_rdma_read_call_chunk(struct svcxprt_rdma *rdma,
+ struct svc_rdma_read_info *info)
{
struct svc_rdma_recv_ctxt *head = info->ri_readctxt;
const struct svc_rdma_chunk *call_chunk =
@@ -1043,17 +1055,17 @@ static int svc_rdma_read_call_chunk(struct svc_rdma_read_info *info)
int ret;
if (pcl_is_empty(pcl))
- return svc_rdma_build_read_chunk(info, call_chunk);
+ return svc_rdma_build_read_chunk(rdma, info, call_chunk);
start = 0;
chunk = pcl_first_chunk(pcl);
length = chunk->ch_position;
- ret = svc_rdma_read_chunk_range(info, call_chunk, start, length);
+ ret = svc_rdma_read_chunk_range(rdma, info, call_chunk, start, length);
if (ret < 0)
return ret;
pcl_for_each_chunk(chunk, pcl) {
- ret = svc_rdma_build_read_chunk(info, chunk);
+ ret = svc_rdma_build_read_chunk(rdma, info, chunk);
if (ret < 0)
return ret;
@@ -1063,7 +1075,7 @@ static int svc_rdma_read_call_chunk(struct svc_rdma_read_info *info)
start += length;
length = next->ch_position - info->ri_totalbytes;
- ret = svc_rdma_read_chunk_range(info, call_chunk,
+ ret = svc_rdma_read_chunk_range(rdma, info, call_chunk,
start, length);
if (ret < 0)
return ret;
@@ -1071,11 +1083,12 @@ static int svc_rdma_read_call_chunk(struct svc_rdma_read_info *info)
start += length;
length = call_chunk->ch_length - start;
- return svc_rdma_read_chunk_range(info, call_chunk, start, length);
+ return svc_rdma_read_chunk_range(rdma, info, call_chunk, start, length);
}
/**
* svc_rdma_read_special - Build RDMA Read WQEs to pull a Long Message
+ * @rdma: controlling transport
* @info: context for RDMA Reads
*
* The start of the data lands in the first page just after the
@@ -1092,12 +1105,13 @@ static int svc_rdma_read_call_chunk(struct svc_rdma_read_info *info)
* %-ENOTCONN: posting failed (connection is lost),
* %-EIO: rdma_rw initialization failed (DMA mapping, etc).
*/
-static noinline int svc_rdma_read_special(struct svc_rdma_read_info *info)
+static noinline int svc_rdma_read_special(struct svcxprt_rdma *rdma,
+ struct svc_rdma_read_info *info)
{
struct xdr_buf *buf = &info->ri_rqst->rq_arg;
int ret;
- ret = svc_rdma_read_call_chunk(info);
+ ret = svc_rdma_read_call_chunk(rdma, info);
if (ret < 0)
goto out;
@@ -1156,11 +1170,11 @@ int svc_rdma_process_read_list(struct svcxprt_rdma *rdma,
if (pcl_is_empty(&head->rc_call_pcl)) {
if (head->rc_read_pcl.cl_count == 1)
- ret = svc_rdma_read_data_item(info);
+ ret = svc_rdma_read_data_item(rdma, info);
else
- ret = svc_rdma_read_multiple_chunks(info);
+ ret = svc_rdma_read_multiple_chunks(rdma, info);
} else
- ret = svc_rdma_read_special(info);
+ ret = svc_rdma_read_special(rdma, info);
if (ret < 0)
goto out_err;
From: Chuck Lever <[email protected]>
Enable the eventual removal of the svc_rdma_chunk_ctxt::cc_rdma
field.
Signed-off-by: Chuck Lever <[email protected]>
---
net/sunrpc/xprtrdma/svc_rdma_rw.c | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c
index cda57a5f8ba0..c0b64a79197e 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_rw.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c
@@ -379,9 +379,9 @@ static void svc_rdma_wc_read_done(struct ib_cq *cq, struct ib_wc *wc)
* even if one or more WRs are flushed. This is true when posting
* an rdma_rw_ctx or when posting a single signaled WR.
*/
-static int svc_rdma_post_chunk_ctxt(struct svc_rdma_chunk_ctxt *cc)
+static int svc_rdma_post_chunk_ctxt(struct svcxprt_rdma *rdma,
+ struct svc_rdma_chunk_ctxt *cc)
{
- struct svcxprt_rdma *rdma = cc->cc_rdma;
struct ib_send_wr *first_wr;
const struct ib_send_wr *bad_wr;
struct list_head *tmp;
@@ -652,7 +652,7 @@ int svc_rdma_send_write_chunk(struct svcxprt_rdma *rdma,
goto out_err;
trace_svcrdma_post_write_chunk(&cc->cc_cid, cc->cc_sqecount);
- ret = svc_rdma_post_chunk_ctxt(cc);
+ ret = svc_rdma_post_chunk_ctxt(rdma, cc);
if (ret < 0)
goto out_err;
return xdr->len;
@@ -699,7 +699,7 @@ int svc_rdma_send_reply_chunk(struct svcxprt_rdma *rdma,
goto out_err;
trace_svcrdma_post_reply_chunk(&cc->cc_cid, cc->cc_sqecount);
- ret = svc_rdma_post_chunk_ctxt(cc);
+ ret = svc_rdma_post_chunk_ctxt(rdma, cc);
if (ret < 0)
goto out_err;
@@ -1180,7 +1180,7 @@ int svc_rdma_process_read_list(struct svcxprt_rdma *rdma,
trace_svcrdma_post_read_chunk(&cc->cc_cid, cc->cc_sqecount);
init_completion(&cc->cc_done);
- ret = svc_rdma_post_chunk_ctxt(cc);
+ ret = svc_rdma_post_chunk_ctxt(rdma, cc);
if (ret < 0)
goto out_err;
From: Chuck Lever <[email protected]>
Enable the eventual removal of the svc_rdma_chunk_ctxt::cc_rdma
field.
Signed-off-by: Chuck Lever <[email protected]>
---
net/sunrpc/xprtrdma/svc_rdma_rw.c | 13 +++++++------
1 file changed, 7 insertions(+), 6 deletions(-)
diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c
index c0b64a79197e..7676b9df024b 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_rw.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c
@@ -187,10 +187,10 @@ static void svc_rdma_cc_init(struct svcxprt_rdma *rdma,
* that only one atomic llist operation is needed to put them all
* back on the free list.
*/
-static void svc_rdma_cc_release(struct svc_rdma_chunk_ctxt *cc,
+static void svc_rdma_cc_release(struct svcxprt_rdma *rdma,
+ struct svc_rdma_chunk_ctxt *cc,
enum dma_data_direction dir)
{
- struct svcxprt_rdma *rdma = cc->cc_rdma;
struct llist_node *first, *last;
struct svc_rdma_rw_ctxt *ctxt;
LLIST_HEAD(free);
@@ -262,7 +262,7 @@ static void svc_rdma_write_info_free_async(struct work_struct *work)
struct svc_rdma_write_info *info;
info = container_of(work, struct svc_rdma_write_info, wi_work);
- svc_rdma_cc_release(&info->wi_cc, DMA_TO_DEVICE);
+ svc_rdma_cc_release(info->wi_rdma, &info->wi_cc, DMA_TO_DEVICE);
kfree(info);
}
@@ -334,9 +334,10 @@ svc_rdma_read_info_alloc(struct svcxprt_rdma *rdma)
return info;
}
-static void svc_rdma_read_info_free(struct svc_rdma_read_info *info)
+static void svc_rdma_read_info_free(struct svcxprt_rdma *rdma,
+ struct svc_rdma_read_info *info)
{
- svc_rdma_cc_release(&info->ri_cc, DMA_FROM_DEVICE);
+ svc_rdma_cc_release(rdma, &info->ri_cc, DMA_FROM_DEVICE);
kfree(info);
}
@@ -1197,6 +1198,6 @@ int svc_rdma_process_read_list(struct svcxprt_rdma *rdma,
head->rc_page_count = 0;
out_err:
- svc_rdma_read_info_free(info);
+ svc_rdma_read_info_free(rdma, info);
return ret;
}
From: Chuck Lever <[email protected]>
In every instance, the pointer address in that field is now
available by other means.
Signed-off-by: Chuck Lever <[email protected]>
---
net/sunrpc/xprtrdma/svc_rdma_rw.c | 2 --
1 file changed, 2 deletions(-)
diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c
index 7676b9df024b..cfa5973c9277 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_rw.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c
@@ -157,7 +157,6 @@ static int svc_rdma_rw_ctx_init(struct svcxprt_rdma *rdma,
struct svc_rdma_chunk_ctxt {
struct rpc_rdma_cid cc_cid;
struct ib_cqe cc_cqe;
- struct svcxprt_rdma *cc_rdma;
struct list_head cc_rwctxts;
ktime_t cc_posttime;
int cc_sqecount;
@@ -176,7 +175,6 @@ static void svc_rdma_cc_init(struct svcxprt_rdma *rdma,
struct svc_rdma_chunk_ctxt *cc)
{
svc_rdma_cc_cid_init(rdma, &cc->cc_cid);
- cc->cc_rdma = rdma;
INIT_LIST_HEAD(&cc->cc_rwctxts);
cc->cc_sqecount = 0;
From: Chuck Lever <[email protected]>
Prepare for nestling these into the send and recv ctxts so they
no longer have to be allocated dynamically.
Signed-off-by: Chuck Lever <[email protected]>
---
include/linux/sunrpc/svc_rdma.h | 15 +++++++++++++++
net/sunrpc/xprtrdma/svc_rdma_rw.c | 18 ------------------
2 files changed, 15 insertions(+), 18 deletions(-)
diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h
index ab250017b99f..50c4f18a9b7f 100644
--- a/include/linux/sunrpc/svc_rdma.h
+++ b/include/linux/sunrpc/svc_rdma.h
@@ -127,6 +127,21 @@ enum {
#define RPCSVC_MAXPAYLOAD_RDMA RPCSVC_MAXPAYLOAD
+/*
+ * A chunk context tracks all I/O for moving one Read or Write
+ * chunk. This is a set of rdma_rw's that handle data movement
+ * for all segments of one chunk.
+ */
+struct svc_rdma_chunk_ctxt {
+ struct rpc_rdma_cid cc_cid;
+ struct ib_cqe cc_cqe;
+ struct list_head cc_rwctxts;
+ ktime_t cc_posttime;
+ int cc_sqecount;
+ enum ib_wc_status cc_status;
+ struct completion cc_done;
+};
+
struct svc_rdma_recv_ctxt {
struct llist_node rc_node;
struct list_head rc_list;
diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c
index cfa5973c9277..1de56e9fea91 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_rw.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c
@@ -146,24 +146,6 @@ static int svc_rdma_rw_ctx_init(struct svcxprt_rdma *rdma,
return ret;
}
-/* A chunk context tracks all I/O for moving one Read or Write
- * chunk. This is a set of rdma_rw's that handle data movement
- * for all segments of one chunk.
- *
- * These are small, acquired with a single allocator call, and
- * no more than one is needed per chunk. They are allocated on
- * demand, and not cached.
- */
-struct svc_rdma_chunk_ctxt {
- struct rpc_rdma_cid cc_cid;
- struct ib_cqe cc_cqe;
- struct list_head cc_rwctxts;
- ktime_t cc_posttime;
- int cc_sqecount;
- enum ib_wc_status cc_status;
- struct completion cc_done;
-};
-
static void svc_rdma_cc_cid_init(struct svcxprt_rdma *rdma,
struct rpc_rdma_cid *cid)
{
From: Chuck Lever <[email protected]>
Since the request's svc_rdma_recv_ctxt will stay around for the
duration of the RDMA Read operation, the contents of struct
svc_rdma_read_info can reside in the request's svc_rdma_recv_ctxt
rather than being allocated separately. This will eventually save a
call to kmalloc() in a hot path.
Start this clean-up by moving the Read chunk's svc_rdma_chunk_ctxt.
Signed-off-by: Chuck Lever <[email protected]>
---
include/linux/sunrpc/svc_rdma.h | 4 +++
net/sunrpc/xprtrdma/svc_rdma_rw.c | 57 +++++++++++++++++--------------------
2 files changed, 30 insertions(+), 31 deletions(-)
diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h
index 50c4f18a9b7f..6c7501ae4e29 100644
--- a/include/linux/sunrpc/svc_rdma.h
+++ b/include/linux/sunrpc/svc_rdma.h
@@ -156,6 +156,10 @@ struct svc_rdma_recv_ctxt {
u32 rc_inv_rkey;
__be32 rc_msgtype;
+ /* State for pulling a Read chunk */
+ unsigned int rc_readbytes;
+ struct svc_rdma_chunk_ctxt rc_cc;
+
struct svc_rdma_pcl rc_call_pcl;
struct svc_rdma_pcl rc_read_pcl;
diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c
index 1de56e9fea91..a27b8f338ae5 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_rw.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c
@@ -294,9 +294,6 @@ struct svc_rdma_read_info {
struct svc_rdma_recv_ctxt *ri_readctxt;
unsigned int ri_pageno;
unsigned int ri_pageoff;
- unsigned int ri_totalbytes;
-
- struct svc_rdma_chunk_ctxt ri_cc;
};
static struct svc_rdma_read_info *
@@ -304,20 +301,13 @@ svc_rdma_read_info_alloc(struct svcxprt_rdma *rdma)
{
struct svc_rdma_read_info *info;
- info = kmalloc_node(sizeof(*info), GFP_KERNEL,
+ return kmalloc_node(sizeof(*info), GFP_KERNEL,
ibdev_to_node(rdma->sc_cm_id->device));
- if (!info)
- return info;
-
- svc_rdma_cc_init(rdma, &info->ri_cc);
- info->ri_cc.cc_cqe.done = svc_rdma_wc_read_done;
- return info;
}
static void svc_rdma_read_info_free(struct svcxprt_rdma *rdma,
struct svc_rdma_read_info *info)
{
- svc_rdma_cc_release(rdma, &info->ri_cc, DMA_FROM_DEVICE);
kfree(info);
}
@@ -333,12 +323,12 @@ static void svc_rdma_wc_read_done(struct ib_cq *cq, struct ib_wc *wc)
struct ib_cqe *cqe = wc->wr_cqe;
struct svc_rdma_chunk_ctxt *cc =
container_of(cqe, struct svc_rdma_chunk_ctxt, cc_cqe);
- struct svc_rdma_read_info *info;
+ struct svc_rdma_recv_ctxt *ctxt;
switch (wc->status) {
case IB_WC_SUCCESS:
- info = container_of(cc, struct svc_rdma_read_info, ri_cc);
- trace_svcrdma_wc_read(wc, &cc->cc_cid, info->ri_totalbytes,
+ ctxt = container_of(cc, struct svc_rdma_recv_ctxt, rc_cc);
+ trace_svcrdma_wc_read(wc, &cc->cc_cid, ctxt->rc_readbytes,
cc->cc_posttime);
break;
case IB_WC_WR_FLUSH_ERR:
@@ -708,7 +698,7 @@ static int svc_rdma_build_read_segment(struct svcxprt_rdma *rdma,
const struct svc_rdma_segment *segment)
{
struct svc_rdma_recv_ctxt *head = info->ri_readctxt;
- struct svc_rdma_chunk_ctxt *cc = &info->ri_cc;
+ struct svc_rdma_chunk_ctxt *cc = &head->rc_cc;
struct svc_rqst *rqstp = info->ri_rqst;
unsigned int sge_no, seg_len, len;
struct svc_rdma_rw_ctxt *ctxt;
@@ -778,6 +768,7 @@ static int svc_rdma_build_read_chunk(struct svcxprt_rdma *rdma,
struct svc_rdma_read_info *info,
const struct svc_rdma_chunk *chunk)
{
+ struct svc_rdma_recv_ctxt *head = info->ri_readctxt;
const struct svc_rdma_segment *segment;
int ret;
@@ -786,7 +777,7 @@ static int svc_rdma_build_read_chunk(struct svcxprt_rdma *rdma,
ret = svc_rdma_build_read_segment(rdma, info, segment);
if (ret < 0)
break;
- info->ri_totalbytes += segment->rs_length;
+ head->rc_readbytes += segment->rs_length;
}
return ret;
}
@@ -828,7 +819,7 @@ static int svc_rdma_copy_inline_range(struct svc_rdma_read_info *info,
dst = page_address(rqstp->rq_pages[info->ri_pageno]);
memcpy(dst + info->ri_pageno, src + offset, page_len);
- info->ri_totalbytes += page_len;
+ head->rc_readbytes += page_len;
info->ri_pageoff += page_len;
if (info->ri_pageoff == PAGE_SIZE) {
info->ri_pageno++;
@@ -883,7 +874,7 @@ static noinline int svc_rdma_read_multiple_chunks(struct svcxprt_rdma *rdma,
break;
start += length;
- length = next->ch_position - info->ri_totalbytes;
+ length = next->ch_position - head->rc_readbytes;
ret = svc_rdma_copy_inline_range(info, start, length);
if (ret < 0)
return ret;
@@ -895,13 +886,13 @@ static noinline int svc_rdma_read_multiple_chunks(struct svcxprt_rdma *rdma,
if (ret < 0)
return ret;
- buf->len += info->ri_totalbytes;
- buf->buflen += info->ri_totalbytes;
+ buf->len += head->rc_readbytes;
+ buf->buflen += head->rc_readbytes;
buf->head[0].iov_base = page_address(info->ri_rqst->rq_pages[0]);
- buf->head[0].iov_len = min_t(size_t, PAGE_SIZE, info->ri_totalbytes);
+ buf->head[0].iov_len = min_t(size_t, PAGE_SIZE, head->rc_readbytes);
buf->pages = &info->ri_rqst->rq_pages[1];
- buf->page_len = info->ri_totalbytes - buf->head[0].iov_len;
+ buf->page_len = head->rc_readbytes - buf->head[0].iov_len;
return 0;
}
@@ -985,6 +976,7 @@ static int svc_rdma_read_chunk_range(struct svcxprt_rdma *rdma,
const struct svc_rdma_chunk *chunk,
unsigned int offset, unsigned int length)
{
+ struct svc_rdma_recv_ctxt *head = info->ri_readctxt;
const struct svc_rdma_segment *segment;
int ret;
@@ -1005,7 +997,7 @@ static int svc_rdma_read_chunk_range(struct svcxprt_rdma *rdma,
if (ret < 0)
break;
- info->ri_totalbytes += dummy.rs_length;
+ head->rc_readbytes += dummy.rs_length;
length -= dummy.rs_length;
offset = 0;
}
@@ -1055,7 +1047,7 @@ static int svc_rdma_read_call_chunk(struct svcxprt_rdma *rdma,
break;
start += length;
- length = next->ch_position - info->ri_totalbytes;
+ length = next->ch_position - head->rc_readbytes;
ret = svc_rdma_read_chunk_range(rdma, info, call_chunk,
start, length);
if (ret < 0)
@@ -1089,6 +1081,7 @@ static int svc_rdma_read_call_chunk(struct svcxprt_rdma *rdma,
static noinline int svc_rdma_read_special(struct svcxprt_rdma *rdma,
struct svc_rdma_read_info *info)
{
+ struct svc_rdma_recv_ctxt *head = info->ri_readctxt;
struct xdr_buf *buf = &info->ri_rqst->rq_arg;
int ret;
@@ -1096,13 +1089,13 @@ static noinline int svc_rdma_read_special(struct svcxprt_rdma *rdma,
if (ret < 0)
goto out;
- buf->len += info->ri_totalbytes;
- buf->buflen += info->ri_totalbytes;
+ buf->len += head->rc_readbytes;
+ buf->buflen += head->rc_readbytes;
buf->head[0].iov_base = page_address(info->ri_rqst->rq_pages[0]);
- buf->head[0].iov_len = min_t(size_t, PAGE_SIZE, info->ri_totalbytes);
+ buf->head[0].iov_len = min_t(size_t, PAGE_SIZE, head->rc_readbytes);
buf->pages = &info->ri_rqst->rq_pages[1];
- buf->page_len = info->ri_totalbytes - buf->head[0].iov_len;
+ buf->page_len = head->rc_readbytes - buf->head[0].iov_len;
out:
return ret;
@@ -1135,19 +1128,20 @@ int svc_rdma_process_read_list(struct svcxprt_rdma *rdma,
struct svc_rqst *rqstp,
struct svc_rdma_recv_ctxt *head)
{
+ struct svc_rdma_chunk_ctxt *cc = &head->rc_cc;
struct svc_rdma_read_info *info;
- struct svc_rdma_chunk_ctxt *cc;
int ret;
info = svc_rdma_read_info_alloc(rdma);
if (!info)
return -ENOMEM;
- cc = &info->ri_cc;
info->ri_rqst = rqstp;
info->ri_readctxt = head;
info->ri_pageno = 0;
info->ri_pageoff = 0;
- info->ri_totalbytes = 0;
+ svc_rdma_cc_init(rdma, cc);
+ cc->cc_cqe.done = svc_rdma_wc_read_done;
+ head->rc_readbytes = 0;
if (pcl_is_empty(&head->rc_call_pcl)) {
if (head->rc_read_pcl.cl_count == 1)
@@ -1178,6 +1172,7 @@ int svc_rdma_process_read_list(struct svcxprt_rdma *rdma,
head->rc_page_count = 0;
out_err:
+ svc_rdma_cc_release(rdma, cc, DMA_FROM_DEVICE);
svc_rdma_read_info_free(rdma, info);
return ret;
}
From: Chuck Lever <[email protected]>
Further clean up: move the page index field into svc_rdma_recv_ctxt.
Signed-off-by: Chuck Lever <[email protected]>
---
include/linux/sunrpc/svc_rdma.h | 1 +
net/sunrpc/xprtrdma/svc_rdma_rw.c | 21 +++++++++------------
2 files changed, 10 insertions(+), 12 deletions(-)
diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h
index 6c7501ae4e29..0ea66f73bec2 100644
--- a/include/linux/sunrpc/svc_rdma.h
+++ b/include/linux/sunrpc/svc_rdma.h
@@ -157,6 +157,7 @@ struct svc_rdma_recv_ctxt {
__be32 rc_msgtype;
/* State for pulling a Read chunk */
+ unsigned int rc_curpage;
unsigned int rc_readbytes;
struct svc_rdma_chunk_ctxt rc_cc;
diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c
index a27b8f338ae5..487acb192558 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_rw.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c
@@ -292,7 +292,6 @@ static void svc_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc)
struct svc_rdma_read_info {
struct svc_rqst *ri_rqst;
struct svc_rdma_recv_ctxt *ri_readctxt;
- unsigned int ri_pageno;
unsigned int ri_pageoff;
};
@@ -720,20 +719,18 @@ static int svc_rdma_build_read_segment(struct svcxprt_rdma *rdma,
if (!info->ri_pageoff)
head->rc_page_count++;
- sg_set_page(sg, rqstp->rq_pages[info->ri_pageno],
+ sg_set_page(sg, rqstp->rq_pages[head->rc_curpage],
seg_len, info->ri_pageoff);
sg = sg_next(sg);
info->ri_pageoff += seg_len;
if (info->ri_pageoff == PAGE_SIZE) {
- info->ri_pageno++;
+ head->rc_curpage++;
info->ri_pageoff = 0;
}
len -= seg_len;
- /* Safety check */
- if (len &&
- &rqstp->rq_pages[info->ri_pageno + 1] > rqstp->rq_page_end)
+ if (len && ((head->rc_curpage + 1) > ARRAY_SIZE(rqstp->rq_pages)))
goto out_overrun;
}
@@ -748,7 +745,7 @@ static int svc_rdma_build_read_segment(struct svcxprt_rdma *rdma,
return 0;
out_overrun:
- trace_svcrdma_page_overrun_err(&cc->cc_cid, info->ri_pageno);
+ trace_svcrdma_page_overrun_err(&cc->cc_cid, head->rc_curpage);
return -EINVAL;
}
@@ -790,7 +787,7 @@ static int svc_rdma_build_read_chunk(struct svcxprt_rdma *rdma,
*
* Take a page at a time from rqstp->rq_pages and copy the inline
* content from the Receive buffer into that page. Update
- * info->ri_pageno and info->ri_pageoff so that the next RDMA Read
+ * head->rc_curpage and info->ri_pageoff so that the next RDMA Read
* result will land contiguously with the copied content.
*
* Return values:
@@ -816,13 +813,13 @@ static int svc_rdma_copy_inline_range(struct svc_rdma_read_info *info,
if (!info->ri_pageoff)
head->rc_page_count++;
- dst = page_address(rqstp->rq_pages[info->ri_pageno]);
- memcpy(dst + info->ri_pageno, src + offset, page_len);
+ dst = page_address(rqstp->rq_pages[head->rc_curpage]);
+ memcpy(dst + head->rc_curpage, src + offset, page_len);
head->rc_readbytes += page_len;
info->ri_pageoff += page_len;
if (info->ri_pageoff == PAGE_SIZE) {
- info->ri_pageno++;
+ head->rc_curpage++;
info->ri_pageoff = 0;
}
remaining -= page_len;
@@ -1137,10 +1134,10 @@ int svc_rdma_process_read_list(struct svcxprt_rdma *rdma,
return -ENOMEM;
info->ri_rqst = rqstp;
info->ri_readctxt = head;
- info->ri_pageno = 0;
info->ri_pageoff = 0;
svc_rdma_cc_init(rdma, cc);
cc->cc_cqe.done = svc_rdma_wc_read_done;
+ head->rc_curpage = 0;
head->rc_readbytes = 0;
if (pcl_is_empty(&head->rc_call_pcl)) {
From: Chuck Lever <[email protected]>
Further clean up: move the starting byte offset field into
svc_rdma_recv_ctxt.
Signed-off-by: Chuck Lever <[email protected]>
---
include/linux/sunrpc/svc_rdma.h | 1 +
net/sunrpc/xprtrdma/svc_rdma_rw.c | 31 +++++++++++++++----------------
2 files changed, 16 insertions(+), 16 deletions(-)
diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h
index 0ea66f73bec2..44a14eaf8c40 100644
--- a/include/linux/sunrpc/svc_rdma.h
+++ b/include/linux/sunrpc/svc_rdma.h
@@ -157,6 +157,7 @@ struct svc_rdma_recv_ctxt {
__be32 rc_msgtype;
/* State for pulling a Read chunk */
+ unsigned int rc_pageoff;
unsigned int rc_curpage;
unsigned int rc_readbytes;
struct svc_rdma_chunk_ctxt rc_cc;
diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c
index 487acb192558..dbced8970779 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_rw.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c
@@ -292,7 +292,6 @@ static void svc_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc)
struct svc_rdma_read_info {
struct svc_rqst *ri_rqst;
struct svc_rdma_recv_ctxt *ri_readctxt;
- unsigned int ri_pageoff;
};
static struct svc_rdma_read_info *
@@ -705,7 +704,7 @@ static int svc_rdma_build_read_segment(struct svcxprt_rdma *rdma,
int ret;
len = segment->rs_length;
- sge_no = PAGE_ALIGN(info->ri_pageoff + len) >> PAGE_SHIFT;
+ sge_no = PAGE_ALIGN(head->rc_pageoff + len) >> PAGE_SHIFT;
ctxt = svc_rdma_get_rw_ctxt(rdma, sge_no);
if (!ctxt)
return -ENOMEM;
@@ -714,19 +713,19 @@ static int svc_rdma_build_read_segment(struct svcxprt_rdma *rdma,
sg = ctxt->rw_sg_table.sgl;
for (sge_no = 0; sge_no < ctxt->rw_nents; sge_no++) {
seg_len = min_t(unsigned int, len,
- PAGE_SIZE - info->ri_pageoff);
+ PAGE_SIZE - head->rc_pageoff);
- if (!info->ri_pageoff)
+ if (!head->rc_pageoff)
head->rc_page_count++;
sg_set_page(sg, rqstp->rq_pages[head->rc_curpage],
- seg_len, info->ri_pageoff);
+ seg_len, head->rc_pageoff);
sg = sg_next(sg);
- info->ri_pageoff += seg_len;
- if (info->ri_pageoff == PAGE_SIZE) {
+ head->rc_pageoff += seg_len;
+ if (head->rc_pageoff == PAGE_SIZE) {
head->rc_curpage++;
- info->ri_pageoff = 0;
+ head->rc_pageoff = 0;
}
len -= seg_len;
@@ -787,7 +786,7 @@ static int svc_rdma_build_read_chunk(struct svcxprt_rdma *rdma,
*
* Take a page at a time from rqstp->rq_pages and copy the inline
* content from the Receive buffer into that page. Update
- * head->rc_curpage and info->ri_pageoff so that the next RDMA Read
+ * head->rc_curpage and head->rc_pageoff so that the next RDMA Read
* result will land contiguously with the copied content.
*
* Return values:
@@ -803,24 +802,24 @@ static int svc_rdma_copy_inline_range(struct svc_rdma_read_info *info,
struct svc_rqst *rqstp = info->ri_rqst;
unsigned int page_no, numpages;
- numpages = PAGE_ALIGN(info->ri_pageoff + remaining) >> PAGE_SHIFT;
+ numpages = PAGE_ALIGN(head->rc_pageoff + remaining) >> PAGE_SHIFT;
for (page_no = 0; page_no < numpages; page_no++) {
unsigned int page_len;
page_len = min_t(unsigned int, remaining,
- PAGE_SIZE - info->ri_pageoff);
+ PAGE_SIZE - head->rc_pageoff);
- if (!info->ri_pageoff)
+ if (!head->rc_pageoff)
head->rc_page_count++;
dst = page_address(rqstp->rq_pages[head->rc_curpage]);
memcpy(dst + head->rc_curpage, src + offset, page_len);
head->rc_readbytes += page_len;
- info->ri_pageoff += page_len;
- if (info->ri_pageoff == PAGE_SIZE) {
+ head->rc_pageoff += page_len;
+ if (head->rc_pageoff == PAGE_SIZE) {
head->rc_curpage++;
- info->ri_pageoff = 0;
+ head->rc_pageoff = 0;
}
remaining -= page_len;
offset += page_len;
@@ -1134,9 +1133,9 @@ int svc_rdma_process_read_list(struct svcxprt_rdma *rdma,
return -ENOMEM;
info->ri_rqst = rqstp;
info->ri_readctxt = head;
- info->ri_pageoff = 0;
svc_rdma_cc_init(rdma, cc);
cc->cc_cqe.done = svc_rdma_wc_read_done;
+ head->rc_pageoff = 0;
head->rc_curpage = 0;
head->rc_readbytes = 0;
From: Chuck Lever <[email protected]>
Since the RDMA Read I/O state is now contained in the recv_ctxt,
svc_rdma_build_read_segment() can use the recv_ctxt to derive that
information rather than the other way around. This removes one usage
of the ri_readctxt field, enabling its removal in a subsequent
patch.
At the same time, the use of ri_rqst can similarly be replaced with
a passed-in function parameter.
Start with build_read_segment() because it is a common utility
function at the bottom of the Read chunk path.
Signed-off-by: Chuck Lever <[email protected]>
---
include/linux/sunrpc/svc_rdma.h | 7 +++++++
net/sunrpc/xprtrdma/svc_rdma_rw.c | 17 +++++++++--------
2 files changed, 16 insertions(+), 8 deletions(-)
diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h
index 44a14eaf8c40..f03f9909fb97 100644
--- a/include/linux/sunrpc/svc_rdma.h
+++ b/include/linux/sunrpc/svc_rdma.h
@@ -116,6 +116,13 @@ struct svcxprt_rdma {
/* sc_flags */
#define RDMAXPRT_CONN_PENDING 3
+static inline struct svcxprt_rdma *svc_rdma_rqst_rdma(struct svc_rqst *rqstp)
+{
+ struct svc_xprt *xprt = rqstp->rq_xprt;
+
+ return container_of(xprt, struct svcxprt_rdma, sc_xprt);
+}
+
/*
* Default connection parameters
*/
diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c
index dbced8970779..c2d0e4bb454e 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_rw.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c
@@ -681,8 +681,8 @@ int svc_rdma_send_reply_chunk(struct svcxprt_rdma *rdma,
/**
* svc_rdma_build_read_segment - Build RDMA Read WQEs to pull one RDMA segment
- * @rdma: controlling transport
- * @info: context for ongoing I/O
+ * @rqstp: RPC transaction context
+ * @head: context for ongoing I/O
* @segment: co-ordinates of remote memory to be read
*
* Returns:
@@ -691,13 +691,12 @@ int svc_rdma_send_reply_chunk(struct svcxprt_rdma *rdma,
* %-ENOMEM: allocating a local resources failed
* %-EIO: a DMA mapping error occurred
*/
-static int svc_rdma_build_read_segment(struct svcxprt_rdma *rdma,
- struct svc_rdma_read_info *info,
+static int svc_rdma_build_read_segment(struct svc_rqst *rqstp,
+ struct svc_rdma_recv_ctxt *head,
const struct svc_rdma_segment *segment)
{
- struct svc_rdma_recv_ctxt *head = info->ri_readctxt;
+ struct svcxprt_rdma *rdma = svc_rdma_rqst_rdma(rqstp);
struct svc_rdma_chunk_ctxt *cc = &head->rc_cc;
- struct svc_rqst *rqstp = info->ri_rqst;
unsigned int sge_no, seg_len, len;
struct svc_rdma_rw_ctxt *ctxt;
struct scatterlist *sg;
@@ -770,7 +769,8 @@ static int svc_rdma_build_read_chunk(struct svcxprt_rdma *rdma,
ret = -EINVAL;
pcl_for_each_segment(segment, chunk) {
- ret = svc_rdma_build_read_segment(rdma, info, segment);
+ ret = svc_rdma_build_read_segment(info->ri_rqst,
+ info->ri_readctxt, segment);
if (ret < 0)
break;
head->rc_readbytes += segment->rs_length;
@@ -989,7 +989,8 @@ static int svc_rdma_read_chunk_range(struct svcxprt_rdma *rdma,
dummy.rs_length = min_t(u32, length, segment->rs_length) - offset;
dummy.rs_offset = segment->rs_offset + offset;
- ret = svc_rdma_build_read_segment(rdma, info, &dummy);
+ ret = svc_rdma_build_read_segment(info->ri_rqst,
+ info->ri_readctxt, &dummy);
if (ret < 0)
break;
From: Chuck Lever <[email protected]>
Since the RDMA Read I/O state is now contained in the recv_ctxt,
svc_rdma_build_read_chunk() can use that recv_ctxt to derive that
information rather than the other way around. This removes another
usage of the ri_readctxt field, enabling its removal in a
subsequent patch.
Signed-off-by: Chuck Lever <[email protected]>
---
net/sunrpc/xprtrdma/svc_rdma_rw.c | 21 ++++++++++-----------
1 file changed, 10 insertions(+), 11 deletions(-)
diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c
index c2d0e4bb454e..b10341cd1df2 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_rw.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c
@@ -749,8 +749,8 @@ static int svc_rdma_build_read_segment(struct svc_rqst *rqstp,
/**
* svc_rdma_build_read_chunk - Build RDMA Read WQEs to pull one RDMA chunk
- * @rdma: controlling transport
- * @info: context for ongoing I/O
+ * @rqstp: RPC transaction context
+ * @head: context for ongoing I/O
* @chunk: Read chunk to pull
*
* Return values:
@@ -759,18 +759,16 @@ static int svc_rdma_build_read_segment(struct svc_rqst *rqstp,
* %-ENOMEM: allocating a local resources failed
* %-EIO: a DMA mapping error occurred
*/
-static int svc_rdma_build_read_chunk(struct svcxprt_rdma *rdma,
- struct svc_rdma_read_info *info,
+static int svc_rdma_build_read_chunk(struct svc_rqst *rqstp,
+ struct svc_rdma_recv_ctxt *head,
const struct svc_rdma_chunk *chunk)
{
- struct svc_rdma_recv_ctxt *head = info->ri_readctxt;
const struct svc_rdma_segment *segment;
int ret;
ret = -EINVAL;
pcl_for_each_segment(segment, chunk) {
- ret = svc_rdma_build_read_segment(info->ri_rqst,
- info->ri_readctxt, segment);
+ ret = svc_rdma_build_read_segment(rqstp, head, segment);
if (ret < 0)
break;
head->rc_readbytes += segment->rs_length;
@@ -861,7 +859,7 @@ static noinline int svc_rdma_read_multiple_chunks(struct svcxprt_rdma *rdma,
return ret;
pcl_for_each_chunk(chunk, pcl) {
- ret = svc_rdma_build_read_chunk(rdma, info, chunk);
+ ret = svc_rdma_build_read_chunk(info->ri_rqst, head, chunk);
if (ret < 0)
return ret;
@@ -920,7 +918,7 @@ static int svc_rdma_read_data_item(struct svcxprt_rdma *rdma,
int ret;
chunk = pcl_first_chunk(&head->rc_read_pcl);
- ret = svc_rdma_build_read_chunk(rdma, info, chunk);
+ ret = svc_rdma_build_read_chunk(info->ri_rqst, head, chunk);
if (ret < 0)
goto out;
@@ -1025,7 +1023,8 @@ static int svc_rdma_read_call_chunk(struct svcxprt_rdma *rdma,
int ret;
if (pcl_is_empty(pcl))
- return svc_rdma_build_read_chunk(rdma, info, call_chunk);
+ return svc_rdma_build_read_chunk(info->ri_rqst, head,
+ call_chunk);
start = 0;
chunk = pcl_first_chunk(pcl);
@@ -1035,7 +1034,7 @@ static int svc_rdma_read_call_chunk(struct svcxprt_rdma *rdma,
return ret;
pcl_for_each_chunk(chunk, pcl) {
- ret = svc_rdma_build_read_chunk(rdma, info, chunk);
+ ret = svc_rdma_build_read_chunk(info->ri_rqst, head, chunk);
if (ret < 0)
return ret;
From: Chuck Lever <[email protected]>
Since the RDMA Read I/O state is now contained in the recv_ctxt,
svc_rdma_build_read_chunk_range() can use that recv_ctxt to derive
that information rather than the other way around. This removes
another usage of the ri_readctxt field, enabling its removal in a
subsequent patch.
Signed-off-by: Chuck Lever <[email protected]>
---
net/sunrpc/xprtrdma/svc_rdma_rw.c | 24 ++++++++++++------------
1 file changed, 12 insertions(+), 12 deletions(-)
diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c
index b10341cd1df2..63546e495cb3 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_rw.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c
@@ -951,9 +951,9 @@ static int svc_rdma_read_data_item(struct svcxprt_rdma *rdma,
}
/**
- * svc_rdma_read_chunk_range - Build RDMA Read WQEs for portion of a chunk
- * @rdma: controlling transport
- * @info: context for RDMA Reads
+ * svc_rdma_read_chunk_range - Build RDMA Read WRs for portion of a chunk
+ * @rqstp: RPC transaction context
+ * @head: context for ongoing I/O
* @chunk: parsed Call chunk to pull
* @offset: offset of region to pull
* @length: length of region to pull
@@ -965,12 +965,11 @@ static int svc_rdma_read_data_item(struct svcxprt_rdma *rdma,
* %-ENOTCONN: posting failed (connection is lost),
* %-EIO: rdma_rw initialization failed (DMA mapping, etc).
*/
-static int svc_rdma_read_chunk_range(struct svcxprt_rdma *rdma,
- struct svc_rdma_read_info *info,
+static int svc_rdma_read_chunk_range(struct svc_rqst *rqstp,
+ struct svc_rdma_recv_ctxt *head,
const struct svc_rdma_chunk *chunk,
unsigned int offset, unsigned int length)
{
- struct svc_rdma_recv_ctxt *head = info->ri_readctxt;
const struct svc_rdma_segment *segment;
int ret;
@@ -987,8 +986,7 @@ static int svc_rdma_read_chunk_range(struct svcxprt_rdma *rdma,
dummy.rs_length = min_t(u32, length, segment->rs_length) - offset;
dummy.rs_offset = segment->rs_offset + offset;
- ret = svc_rdma_build_read_segment(info->ri_rqst,
- info->ri_readctxt, &dummy);
+ ret = svc_rdma_build_read_segment(rqstp, head, &dummy);
if (ret < 0)
break;
@@ -1029,7 +1027,8 @@ static int svc_rdma_read_call_chunk(struct svcxprt_rdma *rdma,
start = 0;
chunk = pcl_first_chunk(pcl);
length = chunk->ch_position;
- ret = svc_rdma_read_chunk_range(rdma, info, call_chunk, start, length);
+ ret = svc_rdma_read_chunk_range(info->ri_rqst, head, call_chunk,
+ start, length);
if (ret < 0)
return ret;
@@ -1044,15 +1043,16 @@ static int svc_rdma_read_call_chunk(struct svcxprt_rdma *rdma,
start += length;
length = next->ch_position - head->rc_readbytes;
- ret = svc_rdma_read_chunk_range(rdma, info, call_chunk,
- start, length);
+ ret = svc_rdma_read_chunk_range(info->ri_rqst, head,
+ call_chunk, start, length);
if (ret < 0)
return ret;
}
start += length;
length = call_chunk->ch_length - start;
- return svc_rdma_read_chunk_range(rdma, info, call_chunk, start, length);
+ return svc_rdma_read_chunk_range(info->ri_rqst, head, call_chunk,
+ start, length);
}
/**
From: Chuck Lever <[email protected]>
Since the RDMA Read I/O state is now contained in the recv_ctxt,
svc_rdma_build_read_data_item() can use that recv_ctxt to derive
that information rather than the other way around. This removes
another usage of the ri_readctxt field, enabling its removal in a
subsequent patch.
Signed-off-by: Chuck Lever <[email protected]>
---
net/sunrpc/xprtrdma/svc_rdma_rw.c | 17 ++++++++---------
1 file changed, 8 insertions(+), 9 deletions(-)
diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c
index 63546e495cb3..1953f3983695 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_rw.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c
@@ -892,8 +892,8 @@ static noinline int svc_rdma_read_multiple_chunks(struct svcxprt_rdma *rdma,
/**
* svc_rdma_read_data_item - Construct RDMA Reads to pull data item Read chunks
- * @rdma: controlling transport
- * @info: context for RDMA Reads
+ * @rqstp: RPC transaction context
+ * @head: context for ongoing I/O
*
* The chunk data lands in the page list of rqstp->rq_arg.pages.
*
@@ -908,17 +908,16 @@ static noinline int svc_rdma_read_multiple_chunks(struct svcxprt_rdma *rdma,
* %-ENOTCONN: posting failed (connection is lost),
* %-EIO: rdma_rw initialization failed (DMA mapping, etc).
*/
-static int svc_rdma_read_data_item(struct svcxprt_rdma *rdma,
- struct svc_rdma_read_info *info)
+static int svc_rdma_read_data_item(struct svc_rqst *rqstp,
+ struct svc_rdma_recv_ctxt *head)
{
- struct svc_rdma_recv_ctxt *head = info->ri_readctxt;
- struct xdr_buf *buf = &info->ri_rqst->rq_arg;
+ struct xdr_buf *buf = &rqstp->rq_arg;
struct svc_rdma_chunk *chunk;
unsigned int length;
int ret;
chunk = pcl_first_chunk(&head->rc_read_pcl);
- ret = svc_rdma_build_read_chunk(info->ri_rqst, head, chunk);
+ ret = svc_rdma_build_read_chunk(rqstp, head, chunk);
if (ret < 0)
goto out;
@@ -940,7 +939,7 @@ static int svc_rdma_read_data_item(struct svcxprt_rdma *rdma,
* Currently these chunks always start at page offset 0,
* thus the rounded-up length never crosses a page boundary.
*/
- buf->pages = &info->ri_rqst->rq_pages[0];
+ buf->pages = &rqstp->rq_pages[0];
length = xdr_align_size(chunk->ch_length);
buf->page_len = length;
buf->len += length;
@@ -1141,7 +1140,7 @@ int svc_rdma_process_read_list(struct svcxprt_rdma *rdma,
if (pcl_is_empty(&head->rc_call_pcl)) {
if (head->rc_read_pcl.cl_count == 1)
- ret = svc_rdma_read_data_item(rdma, info);
+ ret = svc_rdma_read_data_item(rqstp, head);
else
ret = svc_rdma_read_multiple_chunks(rdma, info);
} else
From: Chuck Lever <[email protected]>
Since the RDMA Read I/O state is now contained in the recv_ctxt,
svc_rdma_copy_inline_range() can use that recv_ctxt to derive the
read_info rather than the other way around. This removes another
usage of the ri_readctxt field, enabling its removal in a
subsequent patch.
Signed-off-by: Chuck Lever <[email protected]>
---
net/sunrpc/xprtrdma/svc_rdma_rw.c | 17 +++++++++--------
1 file changed, 9 insertions(+), 8 deletions(-)
diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c
index 1953f3983695..ec546fe094e8 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_rw.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c
@@ -778,7 +778,8 @@ static int svc_rdma_build_read_chunk(struct svc_rqst *rqstp,
/**
* svc_rdma_copy_inline_range - Copy part of the inline content into pages
- * @info: context for RDMA Reads
+ * @rqstp: RPC transaction context
+ * @head: context for ongoing I/O
* @offset: offset into the Receive buffer of region to copy
* @remaining: length of region to copy
*
@@ -791,13 +792,12 @@ static int svc_rdma_build_read_chunk(struct svc_rqst *rqstp,
* %0: Inline content was successfully copied
* %-EINVAL: offset or length was incorrect
*/
-static int svc_rdma_copy_inline_range(struct svc_rdma_read_info *info,
+static int svc_rdma_copy_inline_range(struct svc_rqst *rqstp,
+ struct svc_rdma_recv_ctxt *head,
unsigned int offset,
unsigned int remaining)
{
- struct svc_rdma_recv_ctxt *head = info->ri_readctxt;
unsigned char *dst, *src = head->rc_recv_buf;
- struct svc_rqst *rqstp = info->ri_rqst;
unsigned int page_no, numpages;
numpages = PAGE_ALIGN(head->rc_pageoff + remaining) >> PAGE_SHIFT;
@@ -846,7 +846,8 @@ static noinline int svc_rdma_read_multiple_chunks(struct svcxprt_rdma *rdma,
{
struct svc_rdma_recv_ctxt *head = info->ri_readctxt;
const struct svc_rdma_pcl *pcl = &head->rc_read_pcl;
- struct xdr_buf *buf = &info->ri_rqst->rq_arg;
+ struct svc_rqst *rqstp = info->ri_rqst;
+ struct xdr_buf *buf = &rqstp->rq_arg;
struct svc_rdma_chunk *chunk, *next;
unsigned int start, length;
int ret;
@@ -854,7 +855,7 @@ static noinline int svc_rdma_read_multiple_chunks(struct svcxprt_rdma *rdma,
start = 0;
chunk = pcl_first_chunk(pcl);
length = chunk->ch_position;
- ret = svc_rdma_copy_inline_range(info, start, length);
+ ret = svc_rdma_copy_inline_range(rqstp, head, start, length);
if (ret < 0)
return ret;
@@ -869,14 +870,14 @@ static noinline int svc_rdma_read_multiple_chunks(struct svcxprt_rdma *rdma,
start += length;
length = next->ch_position - head->rc_readbytes;
- ret = svc_rdma_copy_inline_range(info, start, length);
+ ret = svc_rdma_copy_inline_range(rqstp, head, start, length);
if (ret < 0)
return ret;
}
start += length;
length = head->rc_byte_len - start;
- ret = svc_rdma_copy_inline_range(info, start, length);
+ ret = svc_rdma_copy_inline_range(rqstp, head, start, length);
if (ret < 0)
return ret;
From: Chuck Lever <[email protected]>
Since the RDMA Read I/O state is now contained in the recv_ctxt,
svc_rdma_read_multiple_chunks() can use that recv_ctxt to derive the
read_info rather than the other way around. This removes another
usage of the ri_readctxt field, enabling its removal in a
subsequent patch.
Signed-off-by: Chuck Lever <[email protected]>
---
net/sunrpc/xprtrdma/svc_rdma_rw.c | 19 +++++++++----------
1 file changed, 9 insertions(+), 10 deletions(-)
diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c
index ec546fe094e8..56a8e602706a 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_rw.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c
@@ -828,8 +828,8 @@ static int svc_rdma_copy_inline_range(struct svc_rqst *rqstp,
/**
* svc_rdma_read_multiple_chunks - Construct RDMA Reads to pull data item Read chunks
- * @rdma: controlling transport
- * @info: context for RDMA Reads
+ * @rqstp: RPC transaction context
+ * @head: context for ongoing I/O
*
* The chunk data lands in rqstp->rq_arg as a series of contiguous pages,
* like an incoming TCP call.
@@ -841,12 +841,11 @@ static int svc_rdma_copy_inline_range(struct svc_rqst *rqstp,
* %-ENOTCONN: posting failed (connection is lost),
* %-EIO: rdma_rw initialization failed (DMA mapping, etc).
*/
-static noinline int svc_rdma_read_multiple_chunks(struct svcxprt_rdma *rdma,
- struct svc_rdma_read_info *info)
+static noinline int
+svc_rdma_read_multiple_chunks(struct svc_rqst *rqstp,
+ struct svc_rdma_recv_ctxt *head)
{
- struct svc_rdma_recv_ctxt *head = info->ri_readctxt;
const struct svc_rdma_pcl *pcl = &head->rc_read_pcl;
- struct svc_rqst *rqstp = info->ri_rqst;
struct xdr_buf *buf = &rqstp->rq_arg;
struct svc_rdma_chunk *chunk, *next;
unsigned int start, length;
@@ -860,7 +859,7 @@ static noinline int svc_rdma_read_multiple_chunks(struct svcxprt_rdma *rdma,
return ret;
pcl_for_each_chunk(chunk, pcl) {
- ret = svc_rdma_build_read_chunk(info->ri_rqst, head, chunk);
+ ret = svc_rdma_build_read_chunk(rqstp, head, chunk);
if (ret < 0)
return ret;
@@ -884,9 +883,9 @@ static noinline int svc_rdma_read_multiple_chunks(struct svcxprt_rdma *rdma,
buf->len += head->rc_readbytes;
buf->buflen += head->rc_readbytes;
- buf->head[0].iov_base = page_address(info->ri_rqst->rq_pages[0]);
+ buf->head[0].iov_base = page_address(rqstp->rq_pages[0]);
buf->head[0].iov_len = min_t(size_t, PAGE_SIZE, head->rc_readbytes);
- buf->pages = &info->ri_rqst->rq_pages[1];
+ buf->pages = &rqstp->rq_pages[1];
buf->page_len = head->rc_readbytes - buf->head[0].iov_len;
return 0;
}
@@ -1143,7 +1142,7 @@ int svc_rdma_process_read_list(struct svcxprt_rdma *rdma,
if (head->rc_read_pcl.cl_count == 1)
ret = svc_rdma_read_data_item(rqstp, head);
else
- ret = svc_rdma_read_multiple_chunks(rdma, info);
+ ret = svc_rdma_read_multiple_chunks(rqstp, head);
} else
ret = svc_rdma_read_special(rdma, info);
if (ret < 0)
From: Chuck Lever <[email protected]>
Since the RDMA Read I/O state is now contained in the recv_ctxt,
svc_rdma_read_call_chunk() can use that recv_ctxt to derive the
read_info rather than the other way around. This removes another
usage of the ri_readctxt field, enabling its removal in a
subsequent patch.
Signed-off-by: Chuck Lever <[email protected]>
---
net/sunrpc/xprtrdma/svc_rdma_rw.c | 24 +++++++++++-------------
1 file changed, 11 insertions(+), 13 deletions(-)
diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c
index 56a8e602706a..f9d1b0463282 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_rw.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c
@@ -998,8 +998,8 @@ static int svc_rdma_read_chunk_range(struct svc_rqst *rqstp,
/**
* svc_rdma_read_call_chunk - Build RDMA Read WQEs to pull a Long Message
- * @rdma: controlling transport
- * @info: context for RDMA Reads
+ * @rqstp: RPC transaction context
+ * @head: context for ongoing I/O
*
* Return values:
* %0: RDMA Read WQEs were successfully built
@@ -1008,10 +1008,9 @@ static int svc_rdma_read_chunk_range(struct svc_rqst *rqstp,
* %-ENOTCONN: posting failed (connection is lost),
* %-EIO: rdma_rw initialization failed (DMA mapping, etc).
*/
-static int svc_rdma_read_call_chunk(struct svcxprt_rdma *rdma,
- struct svc_rdma_read_info *info)
+static int svc_rdma_read_call_chunk(struct svc_rqst *rqstp,
+ struct svc_rdma_recv_ctxt *head)
{
- struct svc_rdma_recv_ctxt *head = info->ri_readctxt;
const struct svc_rdma_chunk *call_chunk =
pcl_first_chunk(&head->rc_call_pcl);
const struct svc_rdma_pcl *pcl = &head->rc_read_pcl;
@@ -1020,19 +1019,18 @@ static int svc_rdma_read_call_chunk(struct svcxprt_rdma *rdma,
int ret;
if (pcl_is_empty(pcl))
- return svc_rdma_build_read_chunk(info->ri_rqst, head,
- call_chunk);
+ return svc_rdma_build_read_chunk(rqstp, head, call_chunk);
start = 0;
chunk = pcl_first_chunk(pcl);
length = chunk->ch_position;
- ret = svc_rdma_read_chunk_range(info->ri_rqst, head, call_chunk,
+ ret = svc_rdma_read_chunk_range(rqstp, head, call_chunk,
start, length);
if (ret < 0)
return ret;
pcl_for_each_chunk(chunk, pcl) {
- ret = svc_rdma_build_read_chunk(info->ri_rqst, head, chunk);
+ ret = svc_rdma_build_read_chunk(rqstp, head, chunk);
if (ret < 0)
return ret;
@@ -1042,15 +1040,15 @@ static int svc_rdma_read_call_chunk(struct svcxprt_rdma *rdma,
start += length;
length = next->ch_position - head->rc_readbytes;
- ret = svc_rdma_read_chunk_range(info->ri_rqst, head,
- call_chunk, start, length);
+ ret = svc_rdma_read_chunk_range(rqstp, head, call_chunk,
+ start, length);
if (ret < 0)
return ret;
}
start += length;
length = call_chunk->ch_length - start;
- return svc_rdma_read_chunk_range(info->ri_rqst, head, call_chunk,
+ return svc_rdma_read_chunk_range(rqstp, head, call_chunk,
start, length);
}
@@ -1080,7 +1078,7 @@ static noinline int svc_rdma_read_special(struct svcxprt_rdma *rdma,
struct xdr_buf *buf = &info->ri_rqst->rq_arg;
int ret;
- ret = svc_rdma_read_call_chunk(rdma, info);
+ ret = svc_rdma_read_call_chunk(info->ri_rqst, info->ri_readctxt);
if (ret < 0)
goto out;
From: Chuck Lever <[email protected]>
Since the RDMA Read I/O state is now contained in the recv_ctxt,
svc_rdma_read_special() can use that recv_ctxt to derive the
read_info rather than the other way around. This removes another
usage of the ri_readctxt field, enabling its removal in a
subsequent patch.
Signed-off-by: Chuck Lever <[email protected]>
---
net/sunrpc/xprtrdma/svc_rdma_rw.c | 19 +++++++++----------
1 file changed, 9 insertions(+), 10 deletions(-)
diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c
index f9d1b0463282..a3003c2dc0a2 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_rw.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c
@@ -1054,8 +1054,8 @@ static int svc_rdma_read_call_chunk(struct svc_rqst *rqstp,
/**
* svc_rdma_read_special - Build RDMA Read WQEs to pull a Long Message
- * @rdma: controlling transport
- * @info: context for RDMA Reads
+ * @rqstp: RPC transaction context
+ * @head: context for ongoing I/O
*
* The start of the data lands in the first page just after the
* Transport header, and the rest lands in rqstp->rq_arg.pages.
@@ -1071,23 +1071,22 @@ static int svc_rdma_read_call_chunk(struct svc_rqst *rqstp,
* %-ENOTCONN: posting failed (connection is lost),
* %-EIO: rdma_rw initialization failed (DMA mapping, etc).
*/
-static noinline int svc_rdma_read_special(struct svcxprt_rdma *rdma,
- struct svc_rdma_read_info *info)
+static noinline int svc_rdma_read_special(struct svc_rqst *rqstp,
+ struct svc_rdma_recv_ctxt *head)
{
- struct svc_rdma_recv_ctxt *head = info->ri_readctxt;
- struct xdr_buf *buf = &info->ri_rqst->rq_arg;
+ struct xdr_buf *buf = &rqstp->rq_arg;
int ret;
- ret = svc_rdma_read_call_chunk(info->ri_rqst, info->ri_readctxt);
+ ret = svc_rdma_read_call_chunk(rqstp, head);
if (ret < 0)
goto out;
buf->len += head->rc_readbytes;
buf->buflen += head->rc_readbytes;
- buf->head[0].iov_base = page_address(info->ri_rqst->rq_pages[0]);
+ buf->head[0].iov_base = page_address(rqstp->rq_pages[0]);
buf->head[0].iov_len = min_t(size_t, PAGE_SIZE, head->rc_readbytes);
- buf->pages = &info->ri_rqst->rq_pages[1];
+ buf->pages = &rqstp->rq_pages[1];
buf->page_len = head->rc_readbytes - buf->head[0].iov_len;
out:
@@ -1142,7 +1141,7 @@ int svc_rdma_process_read_list(struct svcxprt_rdma *rdma,
else
ret = svc_rdma_read_multiple_chunks(rqstp, head);
} else
- ret = svc_rdma_read_special(rdma, info);
+ ret = svc_rdma_read_special(rqstp, head);
if (ret < 0)
goto out_err;
From: Chuck Lever <[email protected]>
The remaining fields of struct svc_rdma_read_info are no longer
referenced.
Signed-off-by: Chuck Lever <[email protected]>
---
net/sunrpc/xprtrdma/svc_rdma_rw.c | 29 -----------------------------
1 file changed, 29 deletions(-)
diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c
index a3003c2dc0a2..0ccb21f1089e 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_rw.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c
@@ -287,28 +287,6 @@ static void svc_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc)
svc_rdma_write_info_free(info);
}
-/* State for pulling a Read chunk.
- */
-struct svc_rdma_read_info {
- struct svc_rqst *ri_rqst;
- struct svc_rdma_recv_ctxt *ri_readctxt;
-};
-
-static struct svc_rdma_read_info *
-svc_rdma_read_info_alloc(struct svcxprt_rdma *rdma)
-{
- struct svc_rdma_read_info *info;
-
- return kmalloc_node(sizeof(*info), GFP_KERNEL,
- ibdev_to_node(rdma->sc_cm_id->device));
-}
-
-static void svc_rdma_read_info_free(struct svcxprt_rdma *rdma,
- struct svc_rdma_read_info *info)
-{
- kfree(info);
-}
-
/**
* svc_rdma_wc_read_done - Handle completion of an RDMA Read ctx
* @cq: controlling Completion Queue
@@ -1121,14 +1099,8 @@ int svc_rdma_process_read_list(struct svcxprt_rdma *rdma,
struct svc_rdma_recv_ctxt *head)
{
struct svc_rdma_chunk_ctxt *cc = &head->rc_cc;
- struct svc_rdma_read_info *info;
int ret;
- info = svc_rdma_read_info_alloc(rdma);
- if (!info)
- return -ENOMEM;
- info->ri_rqst = rqstp;
- info->ri_readctxt = head;
svc_rdma_cc_init(rdma, cc);
cc->cc_cqe.done = svc_rdma_wc_read_done;
head->rc_pageoff = 0;
@@ -1165,6 +1137,5 @@ int svc_rdma_process_read_list(struct svcxprt_rdma *rdma,
out_err:
svc_rdma_cc_release(rdma, cc, DMA_FROM_DEVICE);
- svc_rdma_read_info_free(rdma, info);
return ret;
}
From: Chuck Lever <[email protected]>
Now that the chunk_ctxt for Reads is no longer dynamically allocated
it can be initialized once for the life of the object that contains
it (struct svc_rdma_recv_ctxt).
Signed-off-by: Chuck Lever <[email protected]>
---
include/linux/sunrpc/svc_rdma.h | 2 ++
net/sunrpc/xprtrdma/svc_rdma_recvfrom.c | 1 +
net/sunrpc/xprtrdma/svc_rdma_rw.c | 11 ++++++++---
3 files changed, 11 insertions(+), 3 deletions(-)
diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h
index f03f9909fb97..051fefde8d51 100644
--- a/include/linux/sunrpc/svc_rdma.h
+++ b/include/linux/sunrpc/svc_rdma.h
@@ -211,6 +211,8 @@ extern int svc_rdma_recvfrom(struct svc_rqst *);
/* svc_rdma_rw.c */
extern void svc_rdma_destroy_rw_ctxts(struct svcxprt_rdma *rdma);
+extern void svc_rdma_cc_init(struct svcxprt_rdma *rdma,
+ struct svc_rdma_chunk_ctxt *cc);
extern int svc_rdma_send_write_chunk(struct svcxprt_rdma *rdma,
const struct svc_rdma_chunk *chunk,
const struct xdr_buf *xdr);
diff --git a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
index 72374033bb2b..392a91dc8a99 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
@@ -156,6 +156,7 @@ svc_rdma_recv_ctxt_alloc(struct svcxprt_rdma *rdma)
ctxt->rc_recv_sge.length = rdma->sc_max_req_size;
ctxt->rc_recv_sge.lkey = rdma->sc_pd->local_dma_lkey;
ctxt->rc_recv_buf = buffer;
+ svc_rdma_cc_init(rdma, &ctxt->rc_cc);
return ctxt;
fail2:
diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c
index 0ccb21f1089e..4d2db06ccfd2 100644
--- a/net/sunrpc/xprtrdma/svc_rdma_rw.c
+++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c
@@ -153,8 +153,13 @@ static void svc_rdma_cc_cid_init(struct svcxprt_rdma *rdma,
cid->ci_completion_id = atomic_inc_return(&rdma->sc_completion_ids);
}
-static void svc_rdma_cc_init(struct svcxprt_rdma *rdma,
- struct svc_rdma_chunk_ctxt *cc)
+/**
+ * svc_rdma_cc_init - Initialize an svc_rdma_chunk_ctxt
+ * @rdma: controlling transport instance
+ * @cc: svc_rdma_chunk_ctxt to be initialized
+ */
+void svc_rdma_cc_init(struct svcxprt_rdma *rdma,
+ struct svc_rdma_chunk_ctxt *cc)
{
svc_rdma_cc_cid_init(rdma, &cc->cc_cid);
@@ -1101,8 +1106,8 @@ int svc_rdma_process_read_list(struct svcxprt_rdma *rdma,
struct svc_rdma_chunk_ctxt *cc = &head->rc_cc;
int ret;
- svc_rdma_cc_init(rdma, cc);
cc->cc_cqe.done = svc_rdma_wc_read_done;
+ cc->cc_sqecount = 0;
head->rc_pageoff = 0;
head->rc_curpage = 0;
head->rc_readbytes = 0;
On Mon, Dec 04, 2023 at 09:56:18AM -0500, Chuck Lever wrote:
> These patches are in preparation for more substantial changes to how
> Read chunks are managed by svcrdma. There are a lot of patches in
> this series, but each change is narrow and no external behavior
> changes are expected.
>
> The main benefit of this series is that svc_rdma_read_info is no
> longer dynamically allocated.
Note that the full series is available in the svcrdma-next branch
in this repo:
https://git.kernel.org/pub/scm/linux/kernel/git/cel/linux.git
> ---
>
> Chuck Lever (21):
> svcrdma: Reduce size of struct svc_rdma_rw_ctxt
> svcrdma: Acquire the svcxprt_rdma pointer from the CQ context
> svcrdma: Explicitly pass the transport into Write chunk I/O paths
> svcrdma: Explicitly pass the transport into Read chunk I/O paths
> svcrdma: Explicitly pass the transport to svc_rdma_post_chunk_ctxt()
> svcrdma: Pass a pointer to the transport to svc_rdma_cc_release()
> svcrdma: Remove the svc_rdma_chunk_ctxt::cc_rdma field
> svcrdma: Move struct svc_rdma_chunk_ctxt to svc_rdma.h
> svcrdma: Start moving fields out of struct svc_rdma_read_info
> svcrdma: Move svc_rdma_read_info::ri_pageno to struct svc_rdma_recv_ctxt
> svcrdma: Move read_info::ri_pageoff into struct svc_rdma_recv_ctxt
> svcrdma: Update synopsis of svc_rdma_build_read_segment()
> svcrdma: Update synopsis of svc_rdma_build_read_chunk()
> svcrdma: Update synopsis of svc_rdma_read_chunk_range()
> svcrdma: Update the synopsis of svc_rdma_read_data_item()
> svcrdma: Update synopsis of svc_rdma_copy_inline_range()
> svcrdma: Update synopsis of svc_rdma_read_multiple_chunks()
> svcrdma: Update the synopsis of svc_rdma_read_call_chunk()
> svcrdma: Update the synopsis of svc_rdma_read_special()
> svcrdma: Remove struct svc_rdma_read_info
> svcrdma: Move the svc_rdma_cc_init() call
>
>
> include/linux/sunrpc/svc_rdma.h | 30 +++
> net/sunrpc/xprtrdma/svc_rdma_recvfrom.c | 1 +
> net/sunrpc/xprtrdma/svc_rdma_rw.c | 310 +++++++++++-------------
> 3 files changed, 169 insertions(+), 172 deletions(-)
>
> --
> Chuck Lever
>
>
--
Chuck Lever