Received-SPF: pass (google.com: domain of linux-nfs-owner@vger.kernel.org designates 23.128.96.18 as permitted sender) client-ip=23.128.96.18;
From:   Chuck Lever III <chuck.lever@oracle.com>
To:     Bruce Fields <bfields@fieldses.org>
CC:     Linux NFS Mailing List <linux-nfs@vger.kernel.org>,
        "linux-rdma@vger.kernel.org" <linux-rdma@vger.kernel.org>
Subject: Re: [PATCH v1 4/6] svcrdma: Add a batch Receive posting mechanism
Thread-Topic: [PATCH v1 4/6] svcrdma: Add a batch Receive posting mechanism
Thread-Index: AQHXHMy5/M1iAGH6sEyybGTKPV1KDKqQRIYAgAAAXwA=
Date:   Mon, 22 Mar 2021 17:17:35 +0000
Message-ID: <756ACA71-4AFC-441C-BD5C-4D95EF713C9C@oracle.com>
References: <161616413550.173092.13403865110684484953.stgit@klimt.1015granger.net>
 <161616429593.173092.14052996014785354959.stgit@klimt.1015granger.net>
 <20210322171614.GB24580@fieldses.org>
In-Reply-To: <20210322171614.GB24580@fieldses.org>
Accept-Language: en-US
Content-Language: en-US
Content-Type: text/plain; charset="us-ascii"
Content-ID: <1FFE3899AF91A04C9F3A6F802B7AC939@namprd10.prod.outlook.com>
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0
X-MS-Exchange-CrossTenant-Network-Message-Id: d8b3e642-ffc4-4957-b6dd-08d8ed5662ef
X-MS-Exchange-CrossTenant-originalarrivaltime: 22 Mar 2021 17:17:35.4577
 (UTC)
X-MS-Exchange-CrossTenant-fromentityheader: Hosted
X-MS-Exchange-CrossTenant-id: 4e2c6054-71cb-48f1-bd6c-3a9705aca71b
X-MS-Exchange-CrossTenant-mailboxtype: HOSTED
X-MS-Exchange-CrossTenant-userprincipalname: OaLQ28DZGPpHcr05Up/HlN9F5NdYHlkBafK25xykuUasQlVXaEF7Pe6JaF/SQGMRMH1gextyALibsyMxNVTb5Q==
X-MS-Exchange-Transport-CrossTenantHeadersStamped: BY5PR10MB3826
X-Proofpoint-Virus-Version: vendor=nai engine=6200 definitions=9931 signatures=668683
X-Proofpoint-Spam-Details: rule=notspam policy=default score=0 adultscore=0 bulkscore=0 phishscore=0
 mlxlogscore=999 suspectscore=0 spamscore=0 malwarescore=0 mlxscore=0
 classifier=spam adjust=0 reason=mlx scancount=1 engine=8.12.0-2009150000
 definitions=main-2103220124
X-Proofpoint-Virus-Version: vendor=nai engine=6200 definitions=9931 signatures=668683
X-Proofpoint-Spam-Details: rule=notspam policy=default score=0 bulkscore=0 phishscore=0
 lowpriorityscore=0 suspectscore=0 clxscore=1015 priorityscore=1501
 spamscore=0 adultscore=0 impostorscore=0 mlxlogscore=999 mlxscore=0
 malwarescore=0 classifier=spam adjust=0 reason=mlx scancount=1
 engine=8.12.0-2009150000 definitions=main-2103220124
Precedence: bulk
List-ID: <linux-nfs.vger.kernel.org>
X-Mailing-List: linux-nfs@vger.kernel.org


> On Mar 22, 2021, at 1:16 PM, J. Bruce Fields <bfields@fieldses.org> wrote=
:
>=20
> On Fri, Mar 19, 2021 at 10:31:35AM -0400, Chuck Lever wrote:
>> Introduce a server-side mechanism similar to commit e340c2d6ef2a
>> ("xprtrdma: Reduce the doorbell rate (Receive)") to post Receive
>> WRs in batch. It's first consumer is svc_rdma_post_recvs().
>=20
> s/It's/Its'/.

D'oh!


> Patches look OK to me.--b.

Thanks for the review!


>> Signed-off-by: Chuck Lever <chuck.lever@oracle.com>
>> ---
>> net/sunrpc/xprtrdma/svc_rdma_recvfrom.c |   56 +++++++++++++++++++++++--=
------
>> 1 file changed, 42 insertions(+), 14 deletions(-)
>>=20
>> diff --git a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c b/net/sunrpc/xprtrd=
ma/svc_rdma_recvfrom.c
>> index 04148a656b2a..0c6aa8693f20 100644
>> --- a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
>> +++ b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c
>> @@ -264,6 +264,47 @@ void svc_rdma_release_rqst(struct svc_rqst *rqstp)
>> 		svc_rdma_recv_ctxt_put(rdma, ctxt);
>> }
>>=20
>> +static bool svc_rdma_refresh_recvs(struct svcxprt_rdma *rdma,
>> +				   unsigned int wanted, bool temp)
>> +{
>> +	const struct ib_recv_wr *bad_wr =3D NULL;
>> +	struct svc_rdma_recv_ctxt *ctxt;
>> +	struct ib_recv_wr *recv_chain;
>> +	int ret;
>> +
>> +	if (test_bit(XPT_CLOSE, &rdma->sc_xprt.xpt_flags))
>> +		return false;
>> +
>> +	recv_chain =3D NULL;
>> +	while (wanted--) {
>> +		ctxt =3D svc_rdma_recv_ctxt_get(rdma);
>> +		if (!ctxt)
>> +			break;
>> +
>> +		trace_svcrdma_post_recv(ctxt);
>> +		ctxt->rc_temp =3D temp;
>> +		ctxt->rc_recv_wr.next =3D recv_chain;
>> +		recv_chain =3D &ctxt->rc_recv_wr;
>> +	}
>> +	if (!recv_chain)
>> +		return false;
>> +
>> +	ret =3D ib_post_recv(rdma->sc_qp, recv_chain, &bad_wr);
>> +	if (ret)
>> +		goto err_free;
>> +	return true;
>> +
>> +err_free:
>> +	trace_svcrdma_rq_post_err(rdma, ret);
>> +	while (bad_wr) {
>> +		ctxt =3D container_of(bad_wr, struct svc_rdma_recv_ctxt,
>> +				    rc_recv_wr);
>> +		bad_wr =3D bad_wr->next;
>> +		svc_rdma_recv_ctxt_put(rdma, ctxt);
>> +	}
>> +	return false;
>> +}
>> +
>> static int __svc_rdma_post_recv(struct svcxprt_rdma *rdma,
>> 				struct svc_rdma_recv_ctxt *ctxt)
>> {
>> @@ -301,20 +342,7 @@ static int svc_rdma_post_recv(struct svcxprt_rdma *=
rdma)
>>  */
>> bool svc_rdma_post_recvs(struct svcxprt_rdma *rdma)
>> {
>> -	struct svc_rdma_recv_ctxt *ctxt;
>> -	unsigned int i;
>> -	int ret;
>> -
>> -	for (i =3D 0; i < rdma->sc_max_requests; i++) {
>> -		ctxt =3D svc_rdma_recv_ctxt_get(rdma);
>> -		if (!ctxt)
>> -			return false;
>> -		ctxt->rc_temp =3D true;
>> -		ret =3D __svc_rdma_post_recv(rdma, ctxt);
>> -		if (ret)
>> -			return false;
>> -	}
>> -	return true;
>> +	return svc_rdma_refresh_recvs(rdma, rdma->sc_max_requests, true);
>> }
>>=20
>> /**
>>=20

--
Chuck Lever