Hello,
DESCRIPTION
This patchset fixes old problem with hungup of both rx/tx sides and adds
test for it. This happens due to non-default SO_RCVLOWAT value and
deferred credit update in virtio/vsock. Link to previous old patchset:
https://lore.kernel.org/netdev/[email protected]/
Here is what happens step by step:
TEST
INITIAL CONDITIONS
1) Vsock buffer size is 128KB.
2) Maximum packet size is also 64KB as defined in header (yes it is
hardcoded, just to remind about that value).
3) SO_RCVLOWAT is default, e.g. 1 byte.
STEPS
SENDER RECEIVER
1) sends 128KB + 1 byte in a
single buffer. 128KB will
be sent, but for 1 byte
sender will wait for free
space at peer. Sender goes
to sleep.
2) reads 64KB, credit update not sent
3) sets SO_RCVLOWAT to 64KB + 1
4) poll() -> wait forever, there is
only 64KB available to read.
So in step 4) receiver also goes to sleep, waiting for enough data or
connection shutdown message from the sender. Idea to fix it is that rx
kicks tx side to continue transmission (and may be close connection)
when rx changes number of bytes to be woken up (e.g. SO_RCVLOWAT) and
this value is bigger than number of available bytes to read.
I've added small test for this, but not sure as it uses hardcoded value
for maximum packet length, this value is defined in kernel header and
used to control deferred credit update. And as this is not available to
userspace, I can't control test parameters correctly (if one day this
define will be changed - test may become useless).
Head for this patchset is:
https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next.git/commit/?id=18de1e517ed37ebaf33e771e46faf052e966e163
Link to v1:
https://lore.kernel.org/netdev/[email protected]/
Link to v2:
https://lore.kernel.org/netdev/[email protected]/
Changelog:
v1 -> v2:
* Patchset rebased and tested on new HEAD of net-next (see hash above).
* New patch is added as 0001 - it removes return from SO_RCVLOWAT set
callback in 'af_vsock.c' when transport callback is set - with that
we can set 'sk_rcvlowat' only once in 'af_vsock.c' and in future do
not copy-paste it to every transport. It was discussed in v1.
* See per-patch changelog after ---.
v2 -> v3:
* See changelog after --- in 0003 only (0001 and 0002 still same).
Arseniy Krasnov (3):
vsock: update SO_RCVLOWAT setting callback
virtio/vsock: send credit update during setting SO_RCVLOWAT
vsock/test: SO_RCVLOWAT + deferred credit update test
drivers/vhost/vsock.c | 2 +
include/linux/virtio_vsock.h | 1 +
net/vmw_vsock/af_vsock.c | 9 +-
net/vmw_vsock/virtio_transport.c | 2 +
net/vmw_vsock/virtio_transport_common.c | 28 +++++
net/vmw_vsock/vsock_loopback.c | 2 +
tools/testing/vsock/vsock_test.c | 142 ++++++++++++++++++++++++
7 files changed, 184 insertions(+), 2 deletions(-)
--
2.25.1
Send credit update message when SO_RCVLOWAT is updated and it is bigger
than number of bytes in rx queue. It is needed, because 'poll()' will
wait until number of bytes in rx queue will be not smaller than
SO_RCVLOWAT, so kick sender to send more data. Otherwise mutual hungup
for tx/rx is possible: sender waits for free space and receiver is
waiting data in 'poll()'.
Signed-off-by: Arseniy Krasnov <[email protected]>
---
Changelog:
v1 -> v2:
* Update commit message by removing 'This patch adds XXX' manner.
* Do not initialize 'send_update' variable - set it directly during
first usage.
drivers/vhost/vsock.c | 2 ++
include/linux/virtio_vsock.h | 1 +
net/vmw_vsock/virtio_transport.c | 2 ++
net/vmw_vsock/virtio_transport_common.c | 28 +++++++++++++++++++++++++
net/vmw_vsock/vsock_loopback.c | 2 ++
5 files changed, 35 insertions(+)
diff --git a/drivers/vhost/vsock.c b/drivers/vhost/vsock.c
index f75731396b7e..ecfa5c11f5ee 100644
--- a/drivers/vhost/vsock.c
+++ b/drivers/vhost/vsock.c
@@ -451,6 +451,8 @@ static struct virtio_transport vhost_transport = {
.notify_buffer_size = virtio_transport_notify_buffer_size,
.read_skb = virtio_transport_read_skb,
+
+ .set_rcvlowat = virtio_transport_set_rcvlowat
},
.send_pkt = vhost_transport_send_pkt,
diff --git a/include/linux/virtio_vsock.h b/include/linux/virtio_vsock.h
index ebb3ce63d64d..97dc1bebc69c 100644
--- a/include/linux/virtio_vsock.h
+++ b/include/linux/virtio_vsock.h
@@ -256,4 +256,5 @@ void virtio_transport_put_credit(struct virtio_vsock_sock *vvs, u32 credit);
void virtio_transport_deliver_tap_pkt(struct sk_buff *skb);
int virtio_transport_purge_skbs(void *vsk, struct sk_buff_head *list);
int virtio_transport_read_skb(struct vsock_sock *vsk, skb_read_actor_t read_actor);
+int virtio_transport_set_rcvlowat(struct vsock_sock *vsk, int val);
#endif /* _LINUX_VIRTIO_VSOCK_H */
diff --git a/net/vmw_vsock/virtio_transport.c b/net/vmw_vsock/virtio_transport.c
index af5bab1acee1..cf3431189d0c 100644
--- a/net/vmw_vsock/virtio_transport.c
+++ b/net/vmw_vsock/virtio_transport.c
@@ -539,6 +539,8 @@ static struct virtio_transport virtio_transport = {
.notify_buffer_size = virtio_transport_notify_buffer_size,
.read_skb = virtio_transport_read_skb,
+
+ .set_rcvlowat = virtio_transport_set_rcvlowat
},
.send_pkt = virtio_transport_send_pkt,
diff --git a/net/vmw_vsock/virtio_transport_common.c b/net/vmw_vsock/virtio_transport_common.c
index f6dc896bf44c..4acee21b4350 100644
--- a/net/vmw_vsock/virtio_transport_common.c
+++ b/net/vmw_vsock/virtio_transport_common.c
@@ -1684,6 +1684,34 @@ int virtio_transport_read_skb(struct vsock_sock *vsk, skb_read_actor_t recv_acto
}
EXPORT_SYMBOL_GPL(virtio_transport_read_skb);
+int virtio_transport_set_rcvlowat(struct vsock_sock *vsk, int val)
+{
+ struct virtio_vsock_sock *vvs = vsk->trans;
+ bool send_update;
+
+ spin_lock_bh(&vvs->rx_lock);
+
+ /* If number of available bytes is less than new
+ * SO_RCVLOWAT value, kick sender to send more
+ * data, because sender may sleep in its 'send()'
+ * syscall waiting for enough space at our side.
+ */
+ send_update = vvs->rx_bytes < val;
+
+ spin_unlock_bh(&vvs->rx_lock);
+
+ if (send_update) {
+ int err;
+
+ err = virtio_transport_send_credit_update(vsk);
+ if (err < 0)
+ return err;
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(virtio_transport_set_rcvlowat);
+
MODULE_LICENSE("GPL v2");
MODULE_AUTHOR("Asias He");
MODULE_DESCRIPTION("common code for virtio vsock");
diff --git a/net/vmw_vsock/vsock_loopback.c b/net/vmw_vsock/vsock_loopback.c
index 048640167411..388c157f6633 100644
--- a/net/vmw_vsock/vsock_loopback.c
+++ b/net/vmw_vsock/vsock_loopback.c
@@ -98,6 +98,8 @@ static struct virtio_transport loopback_transport = {
.notify_buffer_size = virtio_transport_notify_buffer_size,
.read_skb = virtio_transport_read_skb,
+
+ .set_rcvlowat = virtio_transport_set_rcvlowat
},
.send_pkt = vsock_loopback_send_pkt,
--
2.25.1
On Wed, Nov 22, 2023 at 09:05:09PM +0300, Arseniy Krasnov wrote:
>Send credit update message when SO_RCVLOWAT is updated and it is bigger
>than number of bytes in rx queue. It is needed, because 'poll()' will
>wait until number of bytes in rx queue will be not smaller than
>SO_RCVLOWAT, so kick sender to send more data. Otherwise mutual hungup
>for tx/rx is possible: sender waits for free space and receiver is
>waiting data in 'poll()'.
>
>Signed-off-by: Arseniy Krasnov <[email protected]>
>---
> Changelog:
> v1 -> v2:
> * Update commit message by removing 'This patch adds XXX' manner.
> * Do not initialize 'send_update' variable - set it directly during
> first usage.
>
> drivers/vhost/vsock.c | 2 ++
> include/linux/virtio_vsock.h | 1 +
> net/vmw_vsock/virtio_transport.c | 2 ++
> net/vmw_vsock/virtio_transport_common.c | 28 +++++++++++++++++++++++++
> net/vmw_vsock/vsock_loopback.c | 2 ++
> 5 files changed, 35 insertions(+)
>
>diff --git a/drivers/vhost/vsock.c b/drivers/vhost/vsock.c
>index f75731396b7e..ecfa5c11f5ee 100644
>--- a/drivers/vhost/vsock.c
>+++ b/drivers/vhost/vsock.c
>@@ -451,6 +451,8 @@ static struct virtio_transport vhost_transport = {
> .notify_buffer_size = virtio_transport_notify_buffer_size,
>
> .read_skb = virtio_transport_read_skb,
>+
>+ .set_rcvlowat = virtio_transport_set_rcvlowat
Since now we don't set it anymore in the callback, what about following
the notify_* callbacks and rename it in `notify_set_rcvlowat`?
Eventually I think we can rename it in the previous patch.
> },
>
> .send_pkt = vhost_transport_send_pkt,
>diff --git a/include/linux/virtio_vsock.h b/include/linux/virtio_vsock.h
>index ebb3ce63d64d..97dc1bebc69c 100644
>--- a/include/linux/virtio_vsock.h
>+++ b/include/linux/virtio_vsock.h
>@@ -256,4 +256,5 @@ void virtio_transport_put_credit(struct virtio_vsock_sock *vvs, u32 credit);
> void virtio_transport_deliver_tap_pkt(struct sk_buff *skb);
> int virtio_transport_purge_skbs(void *vsk, struct sk_buff_head *list);
> int virtio_transport_read_skb(struct vsock_sock *vsk, skb_read_actor_t read_actor);
>+int virtio_transport_set_rcvlowat(struct vsock_sock *vsk, int val);
> #endif /* _LINUX_VIRTIO_VSOCK_H */
>diff --git a/net/vmw_vsock/virtio_transport.c b/net/vmw_vsock/virtio_transport.c
>index af5bab1acee1..cf3431189d0c 100644
>--- a/net/vmw_vsock/virtio_transport.c
>+++ b/net/vmw_vsock/virtio_transport.c
>@@ -539,6 +539,8 @@ static struct virtio_transport virtio_transport = {
> .notify_buffer_size = virtio_transport_notify_buffer_size,
>
> .read_skb = virtio_transport_read_skb,
>+
>+ .set_rcvlowat = virtio_transport_set_rcvlowat
> },
>
> .send_pkt = virtio_transport_send_pkt,
>diff --git a/net/vmw_vsock/virtio_transport_common.c b/net/vmw_vsock/virtio_transport_common.c
>index f6dc896bf44c..4acee21b4350 100644
>--- a/net/vmw_vsock/virtio_transport_common.c
>+++ b/net/vmw_vsock/virtio_transport_common.c
>@@ -1684,6 +1684,34 @@ int virtio_transport_read_skb(struct vsock_sock *vsk, skb_read_actor_t recv_acto
> }
> EXPORT_SYMBOL_GPL(virtio_transport_read_skb);
>
>+int virtio_transport_set_rcvlowat(struct vsock_sock *vsk, int val)
>+{
>+ struct virtio_vsock_sock *vvs = vsk->trans;
>+ bool send_update;
>+
>+ spin_lock_bh(&vvs->rx_lock);
>+
>+ /* If number of available bytes is less than new
>+ * SO_RCVLOWAT value, kick sender to send more
>+ * data, because sender may sleep in its 'send()'
>+ * syscall waiting for enough space at our side.
>+ */
Let's try to use at least the full 80 characters so we can reduce the
lines in this comment block.
>+ send_update = vvs->rx_bytes < val;
>+
>+ spin_unlock_bh(&vvs->rx_lock);
>+
>+ if (send_update) {
>+ int err;
>+
>+ err = virtio_transport_send_credit_update(vsk);
>+ if (err < 0)
>+ return err;
>+ }
>+
>+ return 0;
>+}
>+EXPORT_SYMBOL_GPL(virtio_transport_set_rcvlowat);
>+
> MODULE_LICENSE("GPL v2");
> MODULE_AUTHOR("Asias He");
> MODULE_DESCRIPTION("common code for virtio vsock");
>diff --git a/net/vmw_vsock/vsock_loopback.c b/net/vmw_vsock/vsock_loopback.c
>index 048640167411..388c157f6633 100644
>--- a/net/vmw_vsock/vsock_loopback.c
>+++ b/net/vmw_vsock/vsock_loopback.c
>@@ -98,6 +98,8 @@ static struct virtio_transport loopback_transport = {
> .notify_buffer_size = virtio_transport_notify_buffer_size,
>
> .read_skb = virtio_transport_read_skb,
>+
>+ .set_rcvlowat = virtio_transport_set_rcvlowat
> },
>
> .send_pkt = vsock_loopback_send_pkt,
>--
>2.25.1
>