Hello,
I submitted an RFC patchset to add sendto(2) and recvfrom(2) support
for io_uring. This RFC patchset adds the support for the liburing.
There are 5 patches in this series. 4 from me. 1 from Nugra.
For PATCH 1/5, it is just a .gitignore clean up.
## Changes Summary
- Update io_uring.h header (sync with the kernel).
- Add `io_uring_prep_{sendto,sendto}` functions.
- Add test program for `IORING_OP_SENDTO` and `IORING_OP_RECVFROM`.
- Add documentation for `io_uring_prep_{sendto,sendto}` functions.
## How to test
This patchset is based on branch "xattr-getdents64" commit:
18d71076f6c97e1b25aa0e3b0e12a913ec4717fa ("src/include/liburing.h: style cleanups")
Signed-off-by: Nugra <[email protected]>
Signed-off-by: Ammar Faizi <[email protected]>
---
Ammar Faizi (4):
.gitignore: Add `/test/xattr` and `/test/getdents`
io_uring.h: Add `IORING_OP_SENDTO` and `IORING_OP_RECVFROM`
liburing.h: Add `io_uring_prep_{sendto,sendto}` helper
test: Add sendto_recvfrom test program
Nugra (1):
man: Add `io_uring_prep_{sendto,recvfrom}` docs
.gitignore | 3 +
man/io_uring_prep_recvfrom.3 | 33 +++
man/io_uring_prep_sendto.3 | 34 +++
src/include/liburing.h | 22 ++
src/include/liburing/io_uring.h | 2 +
test/Makefile | 2 +
test/sendto_recvfrom.c | 384 ++++++++++++++++++++++++++++++++
7 files changed, 480 insertions(+)
create mode 100644 man/io_uring_prep_recvfrom.3
create mode 100644 man/io_uring_prep_sendto.3
create mode 100644 test/sendto_recvfrom.c
base-commit: 18d71076f6c97e1b25aa0e3b0e12a913ec4717fa
--
2.32.0
When Stefan added these two tests, he forgot to add them to
the .gitignore. Add them.
Signed-off-by: Ammar Faizi <[email protected]>
---
.gitignore | 2 ++
1 file changed, 2 insertions(+)
diff --git a/.gitignore b/.gitignore
index 0a72f03..4e70f20 100644
--- a/.gitignore
+++ b/.gitignore
@@ -60,6 +60,7 @@
/test/files-exit-hang-timeout
/test/fixed-link
/test/fsync
+/test/getdents
/test/hardlink
/test/io-cancel
/test/io_uring_enter
@@ -133,6 +134,7 @@
/test/submit-link-fail
/test/exec-target
/test/skip-cqe
+/test/xattr
/test/*.dmesg
/test/output/
--
2.32.0
Sync with the kernel to support for `sendto(2)` and `recvfrom(2)`,
this adds those two new opcodes.
Signed-off-by: Ammar Faizi <[email protected]>
---
src/include/liburing/io_uring.h | 2 ++
1 file changed, 2 insertions(+)
diff --git a/src/include/liburing/io_uring.h b/src/include/liburing/io_uring.h
index 9d8c7f9..47038f6 100644
--- a/src/include/liburing/io_uring.h
+++ b/src/include/liburing/io_uring.h
@@ -154,6 +154,8 @@ enum {
IORING_OP_SETXATTR,
IORING_OP_FGETXATTR,
IORING_OP_GETXATTR,
+ IORING_OP_SENDTO,
+ IORING_OP_RECVFROM,
/* this goes last, obviously */
IORING_OP_LAST,
--
2.32.0
This adds IORING_OP_SENDTO and IORING_OP_RECVFROM prep helper.
Signed-off-by: Ammar Faizi <[email protected]>
---
src/include/liburing.h | 22 ++++++++++++++++++++++
1 file changed, 22 insertions(+)
diff --git a/src/include/liburing.h b/src/include/liburing.h
index a36d3f6..c22e69c 100644
--- a/src/include/liburing.h
+++ b/src/include/liburing.h
@@ -750,6 +750,28 @@ static inline void io_uring_prep_fsetxattr(struct io_uring_sqe *sqe,
sqe->xattr_flags = flags;
}
+static inline void io_uring_prep_sendto(struct io_uring_sqe *sqe, int sockfd,
+ const void *buf, size_t len, int flags,
+ const struct sockaddr *dest_addr,
+ socklen_t addrlen)
+{
+ io_uring_prep_rw(IORING_OP_SENDTO, sqe, sockfd, buf, (__u32) len, 0);
+ sqe->msg_flags = (__u32) flags;
+ sqe->addr2 = (__u64) (uintptr_t) dest_addr;
+ sqe->addr3 = (__u64) addrlen;
+}
+
+static inline void io_uring_prep_recvfrom(struct io_uring_sqe *sqe, int sockfd,
+ void *buf, size_t len, int flags,
+ struct sockaddr *src_addr,
+ socklen_t *addrlen)
+{
+ io_uring_prep_rw(IORING_OP_RECVFROM, sqe, sockfd, buf, (__u32) len, 0);
+ sqe->msg_flags = (__u32) flags;
+ sqe->addr2 = (__u64) (uintptr_t) src_addr;
+ sqe->addr3 = (__u64) (uintptr_t) addrlen;
+}
+
/*
* Returns number of unconsumed (if SQPOLL) or unsubmitted entries exist in
* the SQ ring
--
2.32.0
Add a test program for `IORING_OP_SENDTO` and `IORING_OP_RECVFROM`.
This test is based on `send_recv` test. Additional thing that needs
extra attention here is that `sendto` and `recvfrom` can specify
explicit destination and source.
Apart from that, it is exactly the same with `send` and `recv`, but
with 5-th and 6-th argument be zero.
IOW:
`recv(sockfd, buf, len, flags)` call
is equivalent to:
`recvfrom(sockfd, buf, len, flags, NULL, NULL)` call.
And
`send(sockfd, buf, len, flags)` call
is equivalent to:
`sendto(sockfd, buf, len, flags, NULL, 0)` call.
Tested-by: Nugra <[email protected]>
Signed-off-by: Ammar Faizi <[email protected]>
---
.gitignore | 1 +
test/Makefile | 2 +
test/sendto_recvfrom.c | 384 +++++++++++++++++++++++++++++++++++++++++
3 files changed, 387 insertions(+)
create mode 100644 test/sendto_recvfrom.c
diff --git a/.gitignore b/.gitignore
index 4e70f20..6fe0df2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -98,6 +98,7 @@
/test/send_recv
/test/send_recvmsg
/test/sendmsg_fs_cve
+/test/sendto_recvfrom
/test/shared-wq
/test/short-read
/test/shutdown
diff --git a/test/Makefile b/test/Makefile
index 357c9f7..48c8182 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -118,6 +118,7 @@ test_srcs := \
rw_merge_test.c \
self.c \
sendmsg_fs_cve.c \
+ sendto_recvfrom.c \
send_recv.c \
send_recvmsg.c \
shared-wq.c \
@@ -219,6 +220,7 @@ thread-exit: override LDFLAGS += -lpthread
ring-leak2: override LDFLAGS += -lpthread
poll-mshot-update: override LDFLAGS += -lpthread
exit-no-cleanup: override LDFLAGS += -lpthread
+sendto_recvfrom: override LDFLAGS += -lpthread
install: $(test_targets) runtests.sh runtests-loop.sh
$(INSTALL) -D -d -m 755 $(datadir)/liburing-test/
diff --git a/test/sendto_recvfrom.c b/test/sendto_recvfrom.c
new file mode 100644
index 0000000..720f52f
--- /dev/null
+++ b/test/sendto_recvfrom.c
@@ -0,0 +1,384 @@
+/* SPDX-License-Identifier: MIT */
+/*
+ * Simple test case showing using sendto and recvfrom through io_uring
+ */
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <arpa/inet.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <pthread.h>
+
+#include "liburing.h"
+#include "helpers.h"
+
+static char str[] = "This is a test of sendto and recvfrom over io_uring!";
+
+#define MAX_MSG 128
+
+#define PORT2 10201
+#define PORT 10200
+#define HOST "127.0.0.1"
+
+struct recvfrom_data {
+ pthread_mutex_t mutex;
+ int use_sqthread;
+ int registerfiles;
+ int explicit_dst_src;
+ struct sockaddr_in recvfrom_src;
+};
+
+static int recvfrom_prep(struct io_uring *ring, struct iovec *iov, int *sock,
+ struct recvfrom_data *rd)
+{
+ struct sockaddr_in saddr;
+ struct sockaddr *saddr_p;
+ socklen_t *saddr_len_p;
+ socklen_t saddr_len;
+ struct io_uring_sqe *sqe;
+ int sockfd, ret, val, use_fd;
+
+ memset(&saddr, 0, sizeof(saddr));
+ saddr.sin_family = AF_INET;
+ saddr.sin_addr.s_addr = htonl(INADDR_ANY);
+ saddr.sin_port = htons(PORT);
+
+ sockfd = socket(AF_INET, SOCK_DGRAM, 0);
+ if (sockfd < 0) {
+ perror("socket");
+ return 1;
+ }
+
+ val = 1;
+ setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val));
+
+ ret = bind(sockfd, (struct sockaddr *)&saddr, sizeof(saddr));
+ if (ret < 0) {
+ perror("bind");
+ goto err;
+ }
+
+ if (rd->explicit_dst_src) {
+ memset(&rd->recvfrom_src, 0, sizeof(rd->recvfrom_src));
+ saddr_len = sizeof(rd->recvfrom_src);
+ saddr_p = (struct sockaddr *) &rd->recvfrom_src;
+ saddr_len_p = &saddr_len;
+ } else {
+ saddr_p = NULL;
+ saddr_len_p = NULL;
+ }
+
+ if (rd->registerfiles) {
+ ret = io_uring_register_files(ring, &sockfd, 1);
+ if (ret) {
+ fprintf(stderr, "file reg failed\n");
+ goto err;
+ }
+ use_fd = 0;
+ } else {
+ use_fd = sockfd;
+ }
+
+ sqe = io_uring_get_sqe(ring);
+ io_uring_prep_recvfrom(sqe, use_fd, iov->iov_base, iov->iov_len, 0,
+ saddr_p, saddr_len_p);
+
+ if (rd->registerfiles)
+ sqe->flags |= IOSQE_FIXED_FILE;
+ sqe->user_data = 2;
+
+ ret = io_uring_submit(ring);
+ if (ret <= 0) {
+ fprintf(stderr, "submit failed: %d\n", ret);
+ goto err;
+ }
+
+ *sock = sockfd;
+ return 0;
+err:
+ close(sockfd);
+ return 1;
+}
+
+static int do_recv(struct io_uring *ring, struct iovec *iov,
+ struct recvfrom_data *rd)
+{
+ struct io_uring_cqe *cqe;
+ int ret;
+
+ ret = io_uring_wait_cqe(ring, &cqe);
+ if (ret) {
+ fprintf(stdout, "wait_cqe: %d\n", ret);
+ goto err;
+ }
+ if (cqe->res == -EINVAL) {
+ fprintf(stdout, "recvfrom not supported, skipping\n");
+ return 0;
+ }
+ if (cqe->res < 0) {
+ fprintf(stderr, "failed cqe: %d\n", cqe->res);
+ goto err;
+ }
+
+ if (cqe->res -1 != strlen(str)) {
+ fprintf(stderr, "got wrong length: %d/%d\n", cqe->res,
+ (int) strlen(str) + 1);
+ goto err;
+ }
+
+ if (strcmp(str, iov->iov_base)) {
+ fprintf(stderr, "string mismatch\n");
+ goto err;
+ }
+
+ if (rd->explicit_dst_src) {
+ if (rd->recvfrom_src.sin_family != AF_INET) {
+ fprintf(stderr, "wrong saddr2.sin_family\n");
+ goto err;
+ }
+
+ if (rd->recvfrom_src.sin_addr.s_addr != inet_addr(HOST)) {
+ fprintf(stderr, "wrong saddr2.s_addr\n");
+ goto err;
+ }
+
+ if (rd->recvfrom_src.sin_port != htons(PORT2)) {
+ fprintf(stderr, "wrong saddr2.sin_port\n");
+ goto err;
+ }
+ }
+
+ return 0;
+err:
+ return 1;
+}
+
+static void *recvfrom_fn(void *data)
+{
+ struct recvfrom_data *rd = data;
+ char buf[MAX_MSG + 1];
+ struct iovec iov = {
+ .iov_base = buf,
+ .iov_len = sizeof(buf) - 1,
+ };
+ struct io_uring_params p = { };
+ struct io_uring ring;
+ int ret, sock;
+
+ if (rd->use_sqthread)
+ p.flags = IORING_SETUP_SQPOLL;
+ ret = t_create_ring_params(1, &ring, &p);
+ if (ret == T_SETUP_SKIP) {
+ pthread_mutex_unlock(&rd->mutex);
+ ret = 0;
+ goto err;
+ } else if (ret < 0) {
+ pthread_mutex_unlock(&rd->mutex);
+ goto err;
+ }
+
+ if (rd->use_sqthread && !rd->registerfiles) {
+ if (!(p.features & IORING_FEAT_SQPOLL_NONFIXED)) {
+ fprintf(stdout, "Non-registered SQPOLL not available, skipping\n");
+ pthread_mutex_unlock(&rd->mutex);
+ goto err;
+ }
+ }
+
+ ret = recvfrom_prep(&ring, &iov, &sock, rd);
+ if (ret) {
+ fprintf(stderr, "recvfrom_prep failed: %d\n", ret);
+ goto err;
+ }
+ pthread_mutex_unlock(&rd->mutex);
+ ret = do_recv(&ring, &iov, rd);
+
+ close(sock);
+ io_uring_queue_exit(&ring);
+err:
+ return (void *)(intptr_t)ret;
+}
+
+static int bind_socket_for_sendto(int sockfd)
+{
+ struct sockaddr_in saddr;
+ int ret, val;
+
+ memset(&saddr, 0, sizeof(saddr));
+ saddr.sin_family = AF_INET;
+ saddr.sin_addr.s_addr = inet_addr(HOST);
+ saddr.sin_port = htons(PORT2);
+
+ val = 1;
+ setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val));
+
+ ret = bind(sockfd, (struct sockaddr *)&saddr, sizeof(saddr));
+ if (ret < 0) {
+ perror("bind in bind_socket_for_sendto");
+ return 1;
+ }
+
+
+ return 0;
+}
+
+static int do_sendto(struct recvfrom_data *rd)
+{
+ struct sockaddr_in saddr;
+ struct iovec iov = {
+ .iov_base = str,
+ .iov_len = sizeof(str),
+ };
+ struct io_uring ring;
+ struct io_uring_cqe *cqe;
+ struct io_uring_sqe *sqe;
+ struct sockaddr *saddr_p;
+ socklen_t saddr_len;
+ int sockfd, ret;
+
+ ret = io_uring_queue_init(1, &ring, 0);
+ if (ret) {
+ fprintf(stderr, "queue init failed: %d\n", ret);
+ return 1;
+ }
+
+ memset(&saddr, 0, sizeof(saddr));
+ saddr.sin_family = AF_INET;
+ saddr.sin_port = htons(PORT);
+ inet_pton(AF_INET, HOST, &saddr.sin_addr);
+
+ sockfd = socket(AF_INET, SOCK_DGRAM, 0);
+ if (sockfd < 0) {
+ perror("socket");
+ return 1;
+ }
+
+ if (rd->explicit_dst_src) {
+ saddr_p = (struct sockaddr *)&saddr;
+ saddr_len = sizeof(saddr);
+
+ /*
+ * We need to bind() here because the recvfrom() side
+ * will use an explicit source (addr and port).
+ */
+ bind_socket_for_sendto(sockfd);
+ } else {
+ saddr_p = NULL;
+ saddr_len = 0;
+ /*
+ * Only connect() when sendto() is done without explicit
+ * destination (addr and port).
+ */
+ ret = connect(sockfd, (struct sockaddr *)&saddr, sizeof(saddr));
+ if (ret < 0) {
+ perror("connect");
+ return 1;
+ }
+ }
+
+ sqe = io_uring_get_sqe(&ring);
+ io_uring_prep_sendto(sqe, sockfd, iov.iov_base, iov.iov_len, 0,
+ saddr_p, saddr_len);
+ sqe->user_data = 1;
+
+ ret = io_uring_submit(&ring);
+ if (ret <= 0) {
+ fprintf(stderr, "submit failed: %d\n", ret);
+ goto err;
+ }
+
+ ret = io_uring_wait_cqe(&ring, &cqe);
+ if (cqe->res == -EINVAL) {
+ fprintf(stdout, "sendto not supported, skipping\n");
+ close(sockfd);
+ return 0;
+ }
+ if (cqe->res != iov.iov_len) {
+ fprintf(stderr, "failed cqe: %d\n", cqe->res);
+ goto err;
+ }
+
+ close(sockfd);
+ return 0;
+err:
+ close(sockfd);
+ return 1;
+}
+
+static int test(int use_sqthread, int regfiles, int explicit_dst_src)
+{
+ pthread_mutexattr_t attr;
+ pthread_t recvfrom_thread;
+ struct recvfrom_data rd;
+ int ret;
+ void *retval;
+
+ pthread_mutexattr_init(&attr);
+ pthread_mutexattr_setpshared(&attr, 1);
+ pthread_mutex_init(&rd.mutex, &attr);
+ pthread_mutex_lock(&rd.mutex);
+ rd.use_sqthread = use_sqthread;
+ rd.registerfiles = regfiles;
+ rd.explicit_dst_src = explicit_dst_src;
+
+ ret = pthread_create(&recvfrom_thread, NULL, recvfrom_fn, &rd);
+ if (ret) {
+ fprintf(stderr, "Thread create failed: %d\n", ret);
+ pthread_mutex_unlock(&rd.mutex);
+ return 1;
+ }
+
+ pthread_mutex_lock(&rd.mutex);
+ do_sendto(&rd);
+ pthread_join(recvfrom_thread, &retval);
+ return (int)(intptr_t)retval;
+}
+
+int main(int argc, char *argv[])
+{
+ int ret;
+
+ if (argc > 1)
+ return 0;
+
+ ret = test(0, 0, 0);
+ if (ret) {
+ fprintf(stderr, "test sqthread=0 failed\n");
+ return ret;
+ }
+
+ ret = test(1, 1, 0);
+ if (ret) {
+ fprintf(stderr, "test sqthread=1 reg=1 failed\n");
+ return ret;
+ }
+
+ ret = test(1, 0, 0);
+ if (ret) {
+ fprintf(stderr, "test sqthread=1 reg=0 failed\n");
+ return ret;
+ }
+
+ ret = test(0, 0, 1);
+ if (ret) {
+ fprintf(stderr, "test sqthread=0 explicit_dst_src=1 failed\n");
+ return ret;
+ }
+
+ ret = test(1, 1, 1);
+ if (ret) {
+ fprintf(stderr, "test sqthread=1 reg=1 explicit_dst_src=1 failed\n");
+ return ret;
+ }
+
+ ret = test(1, 0, 1);
+ if (ret) {
+ fprintf(stderr, "test sqthread=1 reg=0 explicit_dst_src=1 failed\n");
+ return ret;
+ }
+
+ return 0;
+}
--
2.32.0
From: Nugra <[email protected]>
Cc: Ammar Faizi <[email protected]>
Signed-off-by: Nugra <[email protected]>
Signed-off-by: Ammar Faizi <[email protected]>
---
man/io_uring_prep_recvfrom.3 | 33 +++++++++++++++++++++++++++++++++
man/io_uring_prep_sendto.3 | 34 ++++++++++++++++++++++++++++++++++
2 files changed, 67 insertions(+)
create mode 100644 man/io_uring_prep_recvfrom.3
create mode 100644 man/io_uring_prep_sendto.3
diff --git a/man/io_uring_prep_recvfrom.3 b/man/io_uring_prep_recvfrom.3
new file mode 100644
index 0000000..b6cfea7
--- /dev/null
+++ b/man/io_uring_prep_recvfrom.3
@@ -0,0 +1,33 @@
+.\" Copyright (C) 2021 Nugra <[email protected]>
+.\"
+.\" SPDX-License-Identifier: LGPL-2.0-or-later
+.\"
+.TH io_uring_prep_recvfrom 3 "December 30, 2021" "liburing-2.1" "liburing Manual"
+.SH NAME
+io_uring_prep_recvfrom - prepare I/O recvfrom request
+
+.SH SYNOPSIS
+.nf
+.BR "#include <liburing.h>"
+.PP
+.BI "void io_uring_prep_recvfrom(struct io_uring_sqe *sqe, int sockfd,"
+.BI " void *buf, size_t len, int flags,
+.BI " struct sockaddr *src_addr,"
+.BI " socklen_t *addrlen)"
+.SH DESCRIPTION
+The io_uring_prep_recvfrom() prepares receive messages from a socket. The submission queue entry
+.I sqe
+is setup to use the file descriptor
+.I fd
+transmit the request.
+
+After the submission queue entry
+.I sqe
+has been prepared as
+.I recvfrom
+op, it can be submitted with one of the submit functions.
+
+.SH RETURN VALUE
+None
+.SH SEE ALSO
+.BR io_uring_get_sqe (3), io_uring_submit (3)
diff --git a/man/io_uring_prep_sendto.3 b/man/io_uring_prep_sendto.3
new file mode 100644
index 0000000..2ed8263
--- /dev/null
+++ b/man/io_uring_prep_sendto.3
@@ -0,0 +1,34 @@
+.\" Copyright (C) 2021 Nugra <[email protected]>
+.\"
+.\" SPDX-License-Identifier: LGPL-2.0-or-later
+.\"
+.TH io_uring_prep_sendto 3 "December 30, 2021" "liburing-2.1" "liburing Manual"
+.SH NAME
+io_uring_prep_sendto - prepare I/O sendto request
+
+.SH SYNOPSIS
+.nf
+.BR "#include <liburing.h>"
+.PP
+.BI "void io_uring_prep_sendto(struct io_uring_sqe *sqe, int sockfd,"
+.BI " const void *buf, size_t len, int flags,"
+.BI " const struct sockaddr *dest_addr,"
+.BI " socklen_t addrlen)"
+.PP
+.SH DESCRIPTION
+The io_uring_prep_sendto() prepares transmit request to another socket. The submission queue entry
+.I sqe
+is setup to use the file descriptor
+.I fd
+transmit the request.
+
+After the submission queue entry
+.I sqe
+has been prepared as
+.I sendto
+op, it can be submitted with one of the submit functions.
+
+.SH RETURN VALUE
+None
+.SH SEE ALSO
+.BR io_uring_get_sqe (3), io_uring_submit (3)
--
2.32.0