Add one more condition for sending credit update during dequeue from
stream socket: when number of bytes in the rx queue is smaller than
SO_RCVLOWAT value of the socket. This is actual for non-default value
of SO_RCVLOWAT (e.g. not 1) - idea is to "kick" peer to continue data
transmission, because we need at least SO_RCVLOWAT bytes in our rx
queue to wake up user for reading data (in corner case it is also
possible to stuck both tx and rx sides, this is why 'Fixes' is used).
Fixes: b89d882dc9fc ("vsock/virtio: reduce credit update messages")
Signed-off-by: Arseniy Krasnov <[email protected]>
---
Changelog:
v6 -> v7:
* Handle wrap of 'fwd_cnt'.
* Do to send credit update when 'fwd_cnt' == 'last_fwd_cnt'.
v7 -> v8:
* Remove unneeded/wrong handling of wrap for 'fwd_cnt'.
net/vmw_vsock/virtio_transport_common.c | 13 ++++++++++---
1 file changed, 10 insertions(+), 3 deletions(-)
diff --git a/net/vmw_vsock/virtio_transport_common.c b/net/vmw_vsock/virtio_transport_common.c
index e137d740804e..8572f94bba88 100644
--- a/net/vmw_vsock/virtio_transport_common.c
+++ b/net/vmw_vsock/virtio_transport_common.c
@@ -558,6 +558,8 @@ virtio_transport_stream_do_dequeue(struct vsock_sock *vsk,
struct virtio_vsock_sock *vvs = vsk->trans;
size_t bytes, total = 0;
struct sk_buff *skb;
+ u32 fwd_cnt_delta;
+ bool low_rx_bytes;
int err = -EFAULT;
u32 free_space;
@@ -601,7 +603,10 @@ virtio_transport_stream_do_dequeue(struct vsock_sock *vsk,
}
}
- free_space = vvs->buf_alloc - (vvs->fwd_cnt - vvs->last_fwd_cnt);
+ fwd_cnt_delta = vvs->fwd_cnt - vvs->last_fwd_cnt;
+ free_space = vvs->buf_alloc - fwd_cnt_delta;
+ low_rx_bytes = (vvs->rx_bytes <
+ sock_rcvlowat(sk_vsock(vsk), 0, INT_MAX));
spin_unlock_bh(&vvs->rx_lock);
@@ -611,9 +616,11 @@ virtio_transport_stream_do_dequeue(struct vsock_sock *vsk,
* too high causes extra messages. Too low causes transmitter
* stalls. As stalls are in theory more expensive than extra
* messages, we set the limit to a high value. TODO: experiment
- * with different values.
+ * with different values. Also send credit update message when
+ * number of bytes in rx queue is not enough to wake up reader.
*/
- if (free_space < VIRTIO_VSOCK_MAX_PKT_BUF_SIZE)
+ if (fwd_cnt_delta &&
+ (free_space < VIRTIO_VSOCK_MAX_PKT_BUF_SIZE || low_rx_bytes))
virtio_transport_send_credit_update(vsk);
return total;
--
2.25.1
On Tue, Dec 12, 2023 at 12:16:57AM +0300, Arseniy Krasnov wrote:
>Add one more condition for sending credit update during dequeue from
>stream socket: when number of bytes in the rx queue is smaller than
>SO_RCVLOWAT value of the socket. This is actual for non-default value
>of SO_RCVLOWAT (e.g. not 1) - idea is to "kick" peer to continue data
>transmission, because we need at least SO_RCVLOWAT bytes in our rx
>queue to wake up user for reading data (in corner case it is also
>possible to stuck both tx and rx sides, this is why 'Fixes' is used).
>
>Fixes: b89d882dc9fc ("vsock/virtio: reduce credit update messages")
>Signed-off-by: Arseniy Krasnov <[email protected]>
>---
> Changelog:
> v6 -> v7:
> * Handle wrap of 'fwd_cnt'.
> * Do to send credit update when 'fwd_cnt' == 'last_fwd_cnt'.
> v7 -> v8:
> * Remove unneeded/wrong handling of wrap for 'fwd_cnt'.
>
> net/vmw_vsock/virtio_transport_common.c | 13 ++++++++++---
> 1 file changed, 10 insertions(+), 3 deletions(-)
Reviewed-by: Stefano Garzarella <[email protected]>
Thanks!
Stefano
>
>diff --git a/net/vmw_vsock/virtio_transport_common.c b/net/vmw_vsock/virtio_transport_common.c
>index e137d740804e..8572f94bba88 100644
>--- a/net/vmw_vsock/virtio_transport_common.c
>+++ b/net/vmw_vsock/virtio_transport_common.c
>@@ -558,6 +558,8 @@ virtio_transport_stream_do_dequeue(struct vsock_sock *vsk,
> struct virtio_vsock_sock *vvs = vsk->trans;
> size_t bytes, total = 0;
> struct sk_buff *skb;
>+ u32 fwd_cnt_delta;
>+ bool low_rx_bytes;
> int err = -EFAULT;
> u32 free_space;
>
>@@ -601,7 +603,10 @@ virtio_transport_stream_do_dequeue(struct vsock_sock *vsk,
> }
> }
>
>- free_space = vvs->buf_alloc - (vvs->fwd_cnt - vvs->last_fwd_cnt);
>+ fwd_cnt_delta = vvs->fwd_cnt - vvs->last_fwd_cnt;
>+ free_space = vvs->buf_alloc - fwd_cnt_delta;
>+ low_rx_bytes = (vvs->rx_bytes <
>+ sock_rcvlowat(sk_vsock(vsk), 0, INT_MAX));
>
> spin_unlock_bh(&vvs->rx_lock);
>
>@@ -611,9 +616,11 @@ virtio_transport_stream_do_dequeue(struct vsock_sock *vsk,
> * too high causes extra messages. Too low causes transmitter
> * stalls. As stalls are in theory more expensive than extra
> * messages, we set the limit to a high value. TODO: experiment
>- * with different values.
>+ * with different values. Also send credit update message when
>+ * number of bytes in rx queue is not enough to wake up reader.
> */
>- if (free_space < VIRTIO_VSOCK_MAX_PKT_BUF_SIZE)
>+ if (fwd_cnt_delta &&
>+ (free_space < VIRTIO_VSOCK_MAX_PKT_BUF_SIZE || low_rx_bytes))
> virtio_transport_send_credit_update(vsk);
>
> return total;
>--
>2.25.1
>
On Tue, Dec 12, 2023 at 12:16:57AM +0300, Arseniy Krasnov wrote:
> Add one more condition for sending credit update during dequeue from
> stream socket: when number of bytes in the rx queue is smaller than
> SO_RCVLOWAT value of the socket. This is actual for non-default value
> of SO_RCVLOWAT (e.g. not 1) - idea is to "kick" peer to continue data
> transmission, because we need at least SO_RCVLOWAT bytes in our rx
> queue to wake up user for reading data (in corner case it is also
> possible to stuck both tx and rx sides, this is why 'Fixes' is used).
I don't get what does "to stuck both tx and rx sides" mean.
Besides being agrammatical, is there a way to do this without
playing with SO_RCVLOWAT?
>
> Fixes: b89d882dc9fc ("vsock/virtio: reduce credit update messages")
> Signed-off-by: Arseniy Krasnov <[email protected]>
> ---
> Changelog:
> v6 -> v7:
> * Handle wrap of 'fwd_cnt'.
> * Do to send credit update when 'fwd_cnt' == 'last_fwd_cnt'.
> v7 -> v8:
> * Remove unneeded/wrong handling of wrap for 'fwd_cnt'.
>
> net/vmw_vsock/virtio_transport_common.c | 13 ++++++++++---
> 1 file changed, 10 insertions(+), 3 deletions(-)
>
> diff --git a/net/vmw_vsock/virtio_transport_common.c b/net/vmw_vsock/virtio_transport_common.c
> index e137d740804e..8572f94bba88 100644
> --- a/net/vmw_vsock/virtio_transport_common.c
> +++ b/net/vmw_vsock/virtio_transport_common.c
> @@ -558,6 +558,8 @@ virtio_transport_stream_do_dequeue(struct vsock_sock *vsk,
> struct virtio_vsock_sock *vvs = vsk->trans;
> size_t bytes, total = 0;
> struct sk_buff *skb;
> + u32 fwd_cnt_delta;
> + bool low_rx_bytes;
> int err = -EFAULT;
> u32 free_space;
>
> @@ -601,7 +603,10 @@ virtio_transport_stream_do_dequeue(struct vsock_sock *vsk,
> }
> }
>
> - free_space = vvs->buf_alloc - (vvs->fwd_cnt - vvs->last_fwd_cnt);
> + fwd_cnt_delta = vvs->fwd_cnt - vvs->last_fwd_cnt;
> + free_space = vvs->buf_alloc - fwd_cnt_delta;
> + low_rx_bytes = (vvs->rx_bytes <
> + sock_rcvlowat(sk_vsock(vsk), 0, INT_MAX));
>
> spin_unlock_bh(&vvs->rx_lock);
>
> @@ -611,9 +616,11 @@ virtio_transport_stream_do_dequeue(struct vsock_sock *vsk,
> * too high causes extra messages. Too low causes transmitter
> * stalls. As stalls are in theory more expensive than extra
> * messages, we set the limit to a high value. TODO: experiment
> - * with different values.
> + * with different values. Also send credit update message when
> + * number of bytes in rx queue is not enough to wake up reader.
> */
> - if (free_space < VIRTIO_VSOCK_MAX_PKT_BUF_SIZE)
> + if (fwd_cnt_delta &&
> + (free_space < VIRTIO_VSOCK_MAX_PKT_BUF_SIZE || low_rx_bytes))
> virtio_transport_send_credit_update(vsk);
>
> return total;
> --
> 2.25.1
On 12.12.2023 18:54, Michael S. Tsirkin wrote:
> On Tue, Dec 12, 2023 at 12:16:57AM +0300, Arseniy Krasnov wrote:
>> Add one more condition for sending credit update during dequeue from
>> stream socket: when number of bytes in the rx queue is smaller than
>> SO_RCVLOWAT value of the socket. This is actual for non-default value
>> of SO_RCVLOWAT (e.g. not 1) - idea is to "kick" peer to continue data
>> transmission, because we need at least SO_RCVLOWAT bytes in our rx
>> queue to wake up user for reading data (in corner case it is also
>> possible to stuck both tx and rx sides, this is why 'Fixes' is used).
>
> I don't get what does "to stuck both tx and rx sides" mean.
I meant situation when tx waits for the free space, while rx doesn't send
credit update, just waiting for more data. Sorry for my English :)
> Besides being agrammatical, is there a way to do this without
> playing with SO_RCVLOWAT?
No, this may happen only with non-default SO_RCVLOWAT values (e.g. != 1)
Thanks, Arseniy
>
>>
>> Fixes: b89d882dc9fc ("vsock/virtio: reduce credit update messages")
>> Signed-off-by: Arseniy Krasnov <[email protected]>
>> ---
>> Changelog:
>> v6 -> v7:
>> * Handle wrap of 'fwd_cnt'.
>> * Do to send credit update when 'fwd_cnt' == 'last_fwd_cnt'.
>> v7 -> v8:
>> * Remove unneeded/wrong handling of wrap for 'fwd_cnt'.
>>
>> net/vmw_vsock/virtio_transport_common.c | 13 ++++++++++---
>> 1 file changed, 10 insertions(+), 3 deletions(-)
>>
>> diff --git a/net/vmw_vsock/virtio_transport_common.c b/net/vmw_vsock/virtio_transport_common.c
>> index e137d740804e..8572f94bba88 100644
>> --- a/net/vmw_vsock/virtio_transport_common.c
>> +++ b/net/vmw_vsock/virtio_transport_common.c
>> @@ -558,6 +558,8 @@ virtio_transport_stream_do_dequeue(struct vsock_sock *vsk,
>> struct virtio_vsock_sock *vvs = vsk->trans;
>> size_t bytes, total = 0;
>> struct sk_buff *skb;
>> + u32 fwd_cnt_delta;
>> + bool low_rx_bytes;
>> int err = -EFAULT;
>> u32 free_space;
>>
>> @@ -601,7 +603,10 @@ virtio_transport_stream_do_dequeue(struct vsock_sock *vsk,
>> }
>> }
>>
>> - free_space = vvs->buf_alloc - (vvs->fwd_cnt - vvs->last_fwd_cnt);
>> + fwd_cnt_delta = vvs->fwd_cnt - vvs->last_fwd_cnt;
>> + free_space = vvs->buf_alloc - fwd_cnt_delta;
>> + low_rx_bytes = (vvs->rx_bytes <
>> + sock_rcvlowat(sk_vsock(vsk), 0, INT_MAX));
>>
>> spin_unlock_bh(&vvs->rx_lock);
>>
>> @@ -611,9 +616,11 @@ virtio_transport_stream_do_dequeue(struct vsock_sock *vsk,
>> * too high causes extra messages. Too low causes transmitter
>> * stalls. As stalls are in theory more expensive than extra
>> * messages, we set the limit to a high value. TODO: experiment
>> - * with different values.
>> + * with different values. Also send credit update message when
>> + * number of bytes in rx queue is not enough to wake up reader.
>> */
>> - if (free_space < VIRTIO_VSOCK_MAX_PKT_BUF_SIZE)
>> + if (fwd_cnt_delta &&
>> + (free_space < VIRTIO_VSOCK_MAX_PKT_BUF_SIZE || low_rx_bytes))
>> virtio_transport_send_credit_update(vsk);
>>
>> return total;
>> --
>> 2.25.1
>
On Tue, Dec 12, 2023 at 06:50:39PM +0300, Arseniy Krasnov wrote:
>
>
> On 12.12.2023 18:54, Michael S. Tsirkin wrote:
> > On Tue, Dec 12, 2023 at 12:16:57AM +0300, Arseniy Krasnov wrote:
> >> Add one more condition for sending credit update during dequeue from
> >> stream socket: when number of bytes in the rx queue is smaller than
> >> SO_RCVLOWAT value of the socket. This is actual for non-default value
> >> of SO_RCVLOWAT (e.g. not 1) - idea is to "kick" peer to continue data
> >> transmission, because we need at least SO_RCVLOWAT bytes in our rx
> >> queue to wake up user for reading data (in corner case it is also
> >> possible to stuck both tx and rx sides, this is why 'Fixes' is used).
> >
> > I don't get what does "to stuck both tx and rx sides" mean.
>
> I meant situation when tx waits for the free space, while rx doesn't send
> credit update, just waiting for more data. Sorry for my English :)
>
> > Besides being agrammatical, is there a way to do this without
> > playing with SO_RCVLOWAT?
>
> No, this may happen only with non-default SO_RCVLOWAT values (e.g. != 1)
>
> Thanks, Arseniy
I am split on whether we need the Fixes tag. I guess if the other side
is vhost with SO_RCVLOWAT then it might be stuck and it might apply
without SO_RCVLOWAT on the local kernel?
> >
> >>
> >> Fixes: b89d882dc9fc ("vsock/virtio: reduce credit update messages")
> >> Signed-off-by: Arseniy Krasnov <[email protected]>
> >> ---
> >> Changelog:
> >> v6 -> v7:
> >> * Handle wrap of 'fwd_cnt'.
> >> * Do to send credit update when 'fwd_cnt' == 'last_fwd_cnt'.
> >> v7 -> v8:
> >> * Remove unneeded/wrong handling of wrap for 'fwd_cnt'.
> >>
> >> net/vmw_vsock/virtio_transport_common.c | 13 ++++++++++---
> >> 1 file changed, 10 insertions(+), 3 deletions(-)
> >>
> >> diff --git a/net/vmw_vsock/virtio_transport_common.c b/net/vmw_vsock/virtio_transport_common.c
> >> index e137d740804e..8572f94bba88 100644
> >> --- a/net/vmw_vsock/virtio_transport_common.c
> >> +++ b/net/vmw_vsock/virtio_transport_common.c
> >> @@ -558,6 +558,8 @@ virtio_transport_stream_do_dequeue(struct vsock_sock *vsk,
> >> struct virtio_vsock_sock *vvs = vsk->trans;
> >> size_t bytes, total = 0;
> >> struct sk_buff *skb;
> >> + u32 fwd_cnt_delta;
> >> + bool low_rx_bytes;
> >> int err = -EFAULT;
> >> u32 free_space;
> >>
> >> @@ -601,7 +603,10 @@ virtio_transport_stream_do_dequeue(struct vsock_sock *vsk,
> >> }
> >> }
> >>
> >> - free_space = vvs->buf_alloc - (vvs->fwd_cnt - vvs->last_fwd_cnt);
> >> + fwd_cnt_delta = vvs->fwd_cnt - vvs->last_fwd_cnt;
> >> + free_space = vvs->buf_alloc - fwd_cnt_delta;
> >> + low_rx_bytes = (vvs->rx_bytes <
> >> + sock_rcvlowat(sk_vsock(vsk), 0, INT_MAX));
> >>
> >> spin_unlock_bh(&vvs->rx_lock);
> >>
> >> @@ -611,9 +616,11 @@ virtio_transport_stream_do_dequeue(struct vsock_sock *vsk,
> >> * too high causes extra messages. Too low causes transmitter
> >> * stalls. As stalls are in theory more expensive than extra
> >> * messages, we set the limit to a high value. TODO: experiment
> >> - * with different values.
> >> + * with different values. Also send credit update message when
> >> + * number of bytes in rx queue is not enough to wake up reader.
> >> */
> >> - if (free_space < VIRTIO_VSOCK_MAX_PKT_BUF_SIZE)
> >> + if (fwd_cnt_delta &&
> >> + (free_space < VIRTIO_VSOCK_MAX_PKT_BUF_SIZE || low_rx_bytes))
> >> virtio_transport_send_credit_update(vsk);
> >>
> >> return total;
> >> --
> >> 2.25.1
> >
On 12.12.2023 19:11, Michael S. Tsirkin wrote:
> On Tue, Dec 12, 2023 at 06:50:39PM +0300, Arseniy Krasnov wrote:
>>
>>
>> On 12.12.2023 18:54, Michael S. Tsirkin wrote:
>>> On Tue, Dec 12, 2023 at 12:16:57AM +0300, Arseniy Krasnov wrote:
>>>> Add one more condition for sending credit update during dequeue from
>>>> stream socket: when number of bytes in the rx queue is smaller than
>>>> SO_RCVLOWAT value of the socket. This is actual for non-default value
>>>> of SO_RCVLOWAT (e.g. not 1) - idea is to "kick" peer to continue data
>>>> transmission, because we need at least SO_RCVLOWAT bytes in our rx
>>>> queue to wake up user for reading data (in corner case it is also
>>>> possible to stuck both tx and rx sides, this is why 'Fixes' is used).
>>>
>>> I don't get what does "to stuck both tx and rx sides" mean.
>>
>> I meant situation when tx waits for the free space, while rx doesn't send
>> credit update, just waiting for more data. Sorry for my English :)
>>
>>> Besides being agrammatical, is there a way to do this without
>>> playing with SO_RCVLOWAT?
>>
>> No, this may happen only with non-default SO_RCVLOWAT values (e.g. != 1)
>>
>> Thanks, Arseniy
>
> I am split on whether we need the Fixes tag. I guess if the other side
> is vhost with SO_RCVLOWAT then it might be stuck and it might apply
> without SO_RCVLOWAT on the local kernel?
IIUC your question, then this problem is actual for any transports: g2h, h2g and
loopback.
Thanks, Arseniy
>
>
>>>
>>>>
>>>> Fixes: b89d882dc9fc ("vsock/virtio: reduce credit update messages")
>>>> Signed-off-by: Arseniy Krasnov <[email protected]>
>>>> ---
>>>> Changelog:
>>>> v6 -> v7:
>>>> * Handle wrap of 'fwd_cnt'.
>>>> * Do to send credit update when 'fwd_cnt' == 'last_fwd_cnt'.
>>>> v7 -> v8:
>>>> * Remove unneeded/wrong handling of wrap for 'fwd_cnt'.
>>>>
>>>> net/vmw_vsock/virtio_transport_common.c | 13 ++++++++++---
>>>> 1 file changed, 10 insertions(+), 3 deletions(-)
>>>>
>>>> diff --git a/net/vmw_vsock/virtio_transport_common.c b/net/vmw_vsock/virtio_transport_common.c
>>>> index e137d740804e..8572f94bba88 100644
>>>> --- a/net/vmw_vsock/virtio_transport_common.c
>>>> +++ b/net/vmw_vsock/virtio_transport_common.c
>>>> @@ -558,6 +558,8 @@ virtio_transport_stream_do_dequeue(struct vsock_sock *vsk,
>>>> struct virtio_vsock_sock *vvs = vsk->trans;
>>>> size_t bytes, total = 0;
>>>> struct sk_buff *skb;
>>>> + u32 fwd_cnt_delta;
>>>> + bool low_rx_bytes;
>>>> int err = -EFAULT;
>>>> u32 free_space;
>>>>
>>>> @@ -601,7 +603,10 @@ virtio_transport_stream_do_dequeue(struct vsock_sock *vsk,
>>>> }
>>>> }
>>>>
>>>> - free_space = vvs->buf_alloc - (vvs->fwd_cnt - vvs->last_fwd_cnt);
>>>> + fwd_cnt_delta = vvs->fwd_cnt - vvs->last_fwd_cnt;
>>>> + free_space = vvs->buf_alloc - fwd_cnt_delta;
>>>> + low_rx_bytes = (vvs->rx_bytes <
>>>> + sock_rcvlowat(sk_vsock(vsk), 0, INT_MAX));
>>>>
>>>> spin_unlock_bh(&vvs->rx_lock);
>>>>
>>>> @@ -611,9 +616,11 @@ virtio_transport_stream_do_dequeue(struct vsock_sock *vsk,
>>>> * too high causes extra messages. Too low causes transmitter
>>>> * stalls. As stalls are in theory more expensive than extra
>>>> * messages, we set the limit to a high value. TODO: experiment
>>>> - * with different values.
>>>> + * with different values. Also send credit update message when
>>>> + * number of bytes in rx queue is not enough to wake up reader.
>>>> */
>>>> - if (free_space < VIRTIO_VSOCK_MAX_PKT_BUF_SIZE)
>>>> + if (fwd_cnt_delta &&
>>>> + (free_space < VIRTIO_VSOCK_MAX_PKT_BUF_SIZE || low_rx_bytes))
>>>> virtio_transport_send_credit_update(vsk);
>>>>
>>>> return total;
>>>> --
>>>> 2.25.1
>>>
>