Received-SPF: pass (google.com: domain of linux-nfs+bounces-3776-linux.lists.archive=gmail.com@vger.kernel.org designates 147.75.80.249 as permitted sender) client-ip=147.75.80.249;
From: Jeff Layton <jlayton@kernel.org>
Date: Thu, 13 Jun 2024 12:44:09 -0400
Subject: [PATCH nfs-utils v5 4/4] nfsdctl: add support for pool-mode
Precedence: bulk
MIME-Version: 1.0
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: 7bit
Message-Id: <20240613-nfsdctl-v5-4-0b7a39102bac@kernel.org>
References: <20240613-nfsdctl-v5-0-0b7a39102bac@kernel.org>
In-Reply-To: <20240613-nfsdctl-v5-0-0b7a39102bac@kernel.org>
To: Steve Dickson <steved@redhat.com>
Cc: linux-nfs@vger.kernel.org, Lorenzo Bianconi <lorenzo@kernel.org>, 
 Chuck Lever <chuck.lever@oracle.com>, Neil Brown <neilb@suse.de>, 
 Olga Kornievskaia <kolga@netapp.com>, Dai Ngo <Dai.Ngo@oracle.com>, 
 Tom Talpey <tom@talpey.com>, Jeff Layton <jlayton@kernel.org>

Allow nfsdctl to use the new POOL_MODE_GET/SET operations to configure
the pool-mode on the host. This setting mirrors the behavior of the
sunrpc module parameter. The patch also adds support for a new pool-mode
option in /etc/nfs.conf. With this change, you can now configure a
multipool NFS server by just properly setting up /etc/nfs.conf.

Signed-off-by: Jeff Layton <jlayton@kernel.org>
---
 configure.ac                 |   2 +-
 utils/nfsdctl/nfsd_netlink.h |  10 +++
 utils/nfsdctl/nfsdctl.8      |  23 ++++++-
 utils/nfsdctl/nfsdctl.adoc   |  12 ++++
 utils/nfsdctl/nfsdctl.c      | 154 ++++++++++++++++++++++++++++++++++++++-----
 5 files changed, 183 insertions(+), 18 deletions(-)

diff --git a/configure.ac b/configure.ac
index 8cb42ebc5846..1aebc3a767a7 100644
--- a/configure.ac
+++ b/configure.ac
@@ -265,7 +265,7 @@ AC_ARG_ENABLE(nfsdctl,
 
 		# ensure we have the pool-mode commands
 		AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include <linux/nfsd_netlink.h>]],
-				                   [[int foo = NFSD_CMD_LISTENER_GET;]])],
+				                   [[int foo = NFSD_CMD_POOL_MODE_GET;]])],
 				   [AC_DEFINE([USE_SYSTEM_NFSD_NETLINK_H], 1,
 					      ["Use system's linux/nfsd_netlink.h"])])
 	fi
diff --git a/utils/nfsdctl/nfsd_netlink.h b/utils/nfsdctl/nfsd_netlink.h
index 24c86dbc7ed5..887cbd12b695 100644
--- a/utils/nfsdctl/nfsd_netlink.h
+++ b/utils/nfsdctl/nfsd_netlink.h
@@ -70,6 +70,14 @@ enum {
 	NFSD_A_SERVER_SOCK_MAX = (__NFSD_A_SERVER_SOCK_MAX - 1)
 };
 
+enum {
+	NFSD_A_POOL_MODE_MODE = 1,
+	NFSD_A_POOL_MODE_NPOOLS,
+
+	__NFSD_A_POOL_MODE_MAX,
+	NFSD_A_POOL_MODE_MAX = (__NFSD_A_POOL_MODE_MAX - 1)
+};
+
 enum {
 	NFSD_CMD_RPC_STATUS_GET = 1,
 	NFSD_CMD_THREADS_SET,
@@ -78,6 +86,8 @@ enum {
 	NFSD_CMD_VERSION_GET,
 	NFSD_CMD_LISTENER_SET,
 	NFSD_CMD_LISTENER_GET,
+	NFSD_CMD_POOL_MODE_SET,
+	NFSD_CMD_POOL_MODE_GET,
 
 	__NFSD_CMD_MAX,
 	NFSD_CMD_MAX = (__NFSD_CMD_MAX - 1)
diff --git a/utils/nfsdctl/nfsdctl.8 b/utils/nfsdctl/nfsdctl.8
index 7f5d981042a5..d55862cc1fc1 100644
--- a/utils/nfsdctl/nfsdctl.8
+++ b/utils/nfsdctl/nfsdctl.8
@@ -2,12 +2,12 @@
 .\"     Title: nfsdctl
 .\"    Author: Jeff Layton
 .\" Generator: Asciidoctor 2.0.20
-.\"      Date: 2024-04-23
+.\"      Date: 2024-06-13
 .\"    Manual: \ \&
 .\"    Source: \ \&
 .\"  Language: English
 .\"
-.TH "NFSDCTL" "8" "2024-04-23" "\ \&" "\ \&"
+.TH "NFSDCTL" "8" "2024-06-13" "\ \&" "\ \&"
 .ie \n(.g .ds Aq \(aq
 .el       .ds Aq '
 .ss \n[.ss] 0
@@ -176,6 +176,25 @@ all minorversions for that major version.
 .fi
 .if n .RE
 .RE
+.sp
+\fBpool\-mode\fP
+.RS 4
+Get/set the host\(cqs pool mode. This will cause the server to start threads
+that are pinned to either the CPU or the NUMA node. This can only be set
+when there are no nfsd threads running.
+.sp
+.if n .RS 4
+.nf
+.fam C
+The available options are:
+  global:  single large pool
+  percpu:  pool per CPU
+  pernode: pool per NUMA node
+  auto:    choose a mode based on host configuration
+.fam
+.fi
+.if n .RE
+.RE
 .SH "EXAMPLES"
 .sp
 Start the server with the settings in nfs.conf:
diff --git a/utils/nfsdctl/nfsdctl.adoc b/utils/nfsdctl/nfsdctl.adoc
index b2f00c11829e..03c5c281d92e 100644
--- a/utils/nfsdctl/nfsdctl.adoc
+++ b/utils/nfsdctl/nfsdctl.adoc
@@ -91,6 +91,18 @@ Each subcommand can also accept its own set of options and arguments. The
   The minorversion field is optional. If not given, it will disable or enable
   all minorversions for that major version.
 
+*pool-mode*::
+
+  Get/set the host's pool mode. This will cause the server to start threads
+  that are pinned to either the CPU or the NUMA node. This can only be set
+  when there are no nfsd threads running.
+
+  The available options are:
+    global:  single large pool
+    percpu:  pool per CPU
+    pernode: pool per NUMA node
+    auto:    choose a mode based on host configuration
+
 == EXAMPLES
 
 Start the server with the settings in nfs.conf:
diff --git a/utils/nfsdctl/nfsdctl.c b/utils/nfsdctl/nfsdctl.c
index 986c937b620d..55bf5cd21783 100644
--- a/utils/nfsdctl/nfsdctl.c
+++ b/utils/nfsdctl/nfsdctl.c
@@ -320,20 +320,20 @@ static void parse_threads_get(struct genlmsghdr *gnlh)
 		struct nlattr *a;
 
 		switch (nla_type(attr)) {
-			case NFSD_A_SERVER_GRACETIME:
-				printf("gracetime: %u\n", nla_get_u32(attr));
-				break;
-			case NFSD_A_SERVER_LEASETIME:
-				printf("leasetime: %u\n", nla_get_u32(attr));
-				break;
-			case NFSD_A_SERVER_SCOPE:
-				printf("scope: %s\n", nla_data(attr));
-				break;
-			case NFSD_A_SERVER_THREADS:
-				pool_threads[i++] = nla_get_u32(attr);
-				break;
-			default:
-				break;
+		case NFSD_A_SERVER_GRACETIME:
+			printf("gracetime: %u\n", nla_get_u32(attr));
+			break;
+		case NFSD_A_SERVER_LEASETIME:
+			printf("leasetime: %u\n", nla_get_u32(attr));
+			break;
+		case NFSD_A_SERVER_SCOPE:
+			printf("scope: %s\n", nla_data(attr));
+			break;
+		case NFSD_A_SERVER_THREADS:
+			pool_threads[i++] = nla_get_u32(attr);
+			break;
+		default:
+			break;
 		}
 	}
 
@@ -343,6 +343,26 @@ static void parse_threads_get(struct genlmsghdr *gnlh)
 	putchar('\n');
 }
 
+static void parse_pool_mode_get(struct genlmsghdr *gnlh)
+{
+	struct nlattr *attr;
+	int rem;
+
+	nla_for_each_attr(attr, genlmsg_attrdata(gnlh, 0),
+			  genlmsg_attrlen(gnlh, 0), rem) {
+		switch (nla_type(attr)) {
+		case NFSD_A_POOL_MODE_MODE:
+			printf("pool-mode: %s\n", nla_data(attr));
+			break;
+		case NFSD_A_POOL_MODE_NPOOLS:
+			printf("npools: %u\n", nla_get_u32(attr));
+			break;
+		default:
+			break;
+		}
+	}
+}
+
 static int recv_handler(struct nl_msg *msg, void *arg)
 {
 	struct genlmsghdr *gnlh = nlmsg_data(nlmsg_hdr(msg));
@@ -361,6 +381,9 @@ static int recv_handler(struct nl_msg *msg, void *arg)
 	case NFSD_CMD_LISTENER_GET:
 		parse_listener_get(gnlh);
 		break;
+	case NFSD_CMD_POOL_MODE_GET:
+		parse_pool_mode_get(gnlh);
+		break;
 	default:
 		break;
 	}
@@ -1158,6 +1181,95 @@ static int listener_func(struct nl_sock *sock, int argc, char ** argv)
 	return 0;
 }
 
+static int pool_mode_doit(struct nl_sock *sock, int cmd, const char *pool_mode)
+{
+	struct genlmsghdr *ghdr;
+	struct nlmsghdr *nlh;
+	struct nl_msg *msg;
+	struct nl_cb *cb;
+	int ret;
+
+	msg = netlink_msg_alloc(sock);
+	if (!msg)
+		return 1;
+
+	nlh = nlmsg_hdr(msg);
+	if (cmd == NFSD_CMD_POOL_MODE_SET) {
+		if (pool_mode)
+			nla_put_string(msg, NFSD_A_POOL_MODE_MODE, pool_mode);
+	}
+	ghdr = nlmsg_data(nlh);
+	ghdr->cmd = cmd;
+
+	cb = nl_cb_alloc(NL_CB_CUSTOM);
+	if (!cb) {
+		fprintf(stderr, "failed to allocate netlink callbacks\n");
+		ret = 1;
+		goto out;
+	}
+
+	ret = nl_send_auto(sock, msg);
+	if (ret < 0) {
+		fprintf(stderr, "send failed (%d)!\n", ret);
+		goto out_cb;
+	}
+
+	ret = 1;
+	nl_cb_err(cb, NL_CB_CUSTOM, error_handler, &ret);
+	nl_cb_set(cb, NL_CB_FINISH, NL_CB_CUSTOM, finish_handler, &ret);
+	nl_cb_set(cb, NL_CB_ACK, NL_CB_CUSTOM, ack_handler, &ret);
+	nl_cb_set(cb, NL_CB_VALID, NL_CB_CUSTOM, recv_handler, NULL);
+
+	while (ret > 0)
+		nl_recvmsgs(sock, cb);
+	if (ret < 0) {
+		fprintf(stderr, "Error: %s\n", strerror(-ret));
+		ret = 1;
+	}
+out_cb:
+	nl_cb_put(cb);
+out:
+	nlmsg_free(msg);
+	return ret;
+}
+
+static void pool_mode_usage(void)
+{
+	printf("Usage: %s pool-mode { global | auto | percpu | pernode } ...\n", taskname);
+}
+
+static int pool_mode_func(struct nl_sock *sock, int argc, char **argv)
+{
+	uint8_t cmd = NFSD_CMD_POOL_MODE_GET;
+	const char *pool_mode = NULL;
+	int opt;
+
+	optind = 1;
+	while ((opt = getopt_long(argc, argv, "h", help_only_options, NULL)) != -1) {
+		switch (opt) {
+		case 'h':
+			pool_mode_usage();
+			return 0;
+		}
+	}
+
+	if (optind < argc) {
+		char **targv = &argv[optind];
+		int i;
+
+		cmd = NFSD_CMD_POOL_MODE_SET;
+
+		/* empty string? */
+		if (targv[i][0] == '\0') {
+			fprintf(stderr, "Invalid threads value %s.\n", targv[i]);
+			return 1;
+		}
+
+		pool_mode = targv[0];
+	}
+	return pool_mode_doit(sock, cmd, pool_mode);
+}
+
 #define MAX_LISTENER_LEN (64 * 2 + 16)
 
 static int
@@ -1249,7 +1361,7 @@ static int autostart_func(struct nl_sock *sock, int argc, char ** argv)
 	int *threads, grace, lease, idx, ret, opt;
 	struct conf_list *thread_str;
 	struct conf_list_node *n;
-	char *scope;
+	char *scope, *pool_mode;
 
 	optind = 1;
 	while ((opt = getopt_long(argc, argv, "h", help_only_options, NULL)) != -1) {
@@ -1262,6 +1374,13 @@ static int autostart_func(struct nl_sock *sock, int argc, char ** argv)
 
 	read_nfsd_conf();
 
+	pool_mode = conf_get_str("nfsd", "pool-mode");
+	if (pool_mode) {
+		ret = pool_mode_doit(sock, NFSD_CMD_POOL_MODE_SET, pool_mode);
+		if (ret)
+			return ret;
+	}
+
 	ret = fetch_nfsd_versions(sock);
 	if (ret)
 		return ret;
@@ -1312,6 +1431,7 @@ enum nfsdctl_commands {
 	NFSDCTL_VERSION,
 	NFSDCTL_LISTENER,
 	NFSDCTL_AUTOSTART,
+	NFSDCTL_POOL_MODE,
 };
 
 static int parse_command(char *str)
@@ -1326,6 +1446,8 @@ static int parse_command(char *str)
 		return NFSDCTL_LISTENER;
 	if (!strcmp(str, "autostart"))
 		return NFSDCTL_AUTOSTART;
+	if (!strcmp(str, "pool-mode"))
+		return NFSDCTL_POOL_MODE;
 	return -1;
 }
 
@@ -1337,6 +1459,7 @@ static nfsdctl_func func[] = {
 	[NFSDCTL_VERSION] = version_func,
 	[NFSDCTL_LISTENER] = listener_func,
 	[NFSDCTL_AUTOSTART] = autostart_func,
+	[NFSDCTL_POOL_MODE] = pool_mode_func,
 };
 
 static void usage(void)
@@ -1348,6 +1471,7 @@ static void usage(void)
 	printf("    -d | --debug=NUM     enable debugging\n");
 	printf("    -V | --version       print version info\n");
 	printf("  commands:\n");
+	printf("    pool-mode            get/set host pool mode setting\n");
 	printf("    listener             get/set listener info\n");
 	printf("    version              get/set supported NFS versions\n");
 	printf("    threads              get/set nfsd thread settings\n");

-- 
2.45.2