Received-SPF: pass (google.com: domain of linux-nfs+bounces-3603-linux.lists.archive=gmail.com@vger.kernel.org designates 2604:1380:40f1:3f00::1 as permitted sender) client-ip=2604:1380:40f1:3f00::1;
From: Mike Snitzer <snitzer@kernel.org>
To: linux-nfs@vger.kernel.org
Cc: Jeff Layton <jlayton@kernel.org>,
	Chuck Lever <chuck.lever@oracle.com>,
	Trond Myklebust <trondmy@hammerspace.com>,
	snitzer@hammerspace.com
Subject: [for-6.11 PATCH 18/29] NFS: localio writes need to use a normal workqueue
Date: Fri,  7 Jun 2024 10:26:35 -0400
Message-ID: <20240607142646.20924-19-snitzer@kernel.org>
In-Reply-To: <20240607142646.20924-1-snitzer@kernel.org>
References: <20240607142646.20924-1-snitzer@kernel.org>
Precedence: bulk
MIME-Version: 1.0
Content-Transfer-Encoding: 8bit

From: Trond Myklebust <trond.myklebust@hammerspace.com>

When we start getting low on space, XFS goes and calls flush_work() on a
non-memreclaim work queue, which causes a priority inversion problem.

Signed-off-by: Trond Myklebust <trond.myklebust@hammerspace.com>
Signed-off-by: Mike Snitzer <snitzer@kernel.org>
---
 fs/nfs/inode.c    | 24 +++++++++++++++++++-----
 fs/nfs/internal.h |  1 +
 fs/nfs/localio.c  |  4 ++--
 3 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c
index 4f88b860494f..b80469bce8df 100644
--- a/fs/nfs/inode.c
+++ b/fs/nfs/inode.c
@@ -2394,6 +2394,7 @@ static void nfs_destroy_inodecache(void)
 	kmem_cache_destroy(nfs_inode_cachep);
 }
 
+struct workqueue_struct *nfssync_workqueue;
 struct workqueue_struct *nfsiod_workqueue;
 EXPORT_SYMBOL_GPL(nfsiod_workqueue);
 
@@ -2404,9 +2405,17 @@ static int nfsiod_start(void)
 {
 	struct workqueue_struct *wq;
 	dprintk("RPC:       creating workqueue nfsiod\n");
+	wq = alloc_workqueue("nfs-sync", WQ_UNBOUND, 0);
+	if (wq == NULL)
+		return -ENOMEM;
+	nfssync_workqueue = wq;
 	wq = alloc_workqueue("nfsiod", WQ_MEM_RECLAIM | WQ_UNBOUND, 0);
-	if (wq == NULL)
+	if (wq == NULL) {
+		wq = nfssync_workqueue;
+		nfsiod_workqueue = NULL;
+		destroy_workqueue(wq);
 		return -ENOMEM;
+	}
 	nfsiod_workqueue = wq;
 	return 0;
 }
@@ -2419,10 +2428,15 @@ static void nfsiod_stop(void)
 	struct workqueue_struct *wq;
 
 	wq = nfsiod_workqueue;
-	if (wq == NULL)
-		return;
-	nfsiod_workqueue = NULL;
-	destroy_workqueue(wq);
+	if (wq != NULL) {
+		nfsiod_workqueue = NULL;
+		destroy_workqueue(wq);
+	}
+	wq = nfssync_workqueue;
+	if (wq != NULL) {
+		nfssync_workqueue = NULL;
+		destroy_workqueue(wq);
+	}
 }
 
 unsigned int nfs_net_id;
diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h
index 67b348447a40..0927a1704bbb 100644
--- a/fs/nfs/internal.h
+++ b/fs/nfs/internal.h
@@ -446,6 +446,7 @@ int nfs_check_flags(int);
 
 /* inode.c */
 extern struct workqueue_struct *nfsiod_workqueue;
+extern struct workqueue_struct *nfssync_workqueue;
 extern struct inode *nfs_alloc_inode(struct super_block *sb);
 extern void nfs_free_inode(struct inode *);
 extern int nfs_write_inode(struct inode *, struct writeback_control *);
diff --git a/fs/nfs/localio.c b/fs/nfs/localio.c
index d7918e26aeb6..d724f8d4dd65 100644
--- a/fs/nfs/localio.c
+++ b/fs/nfs/localio.c
@@ -511,7 +511,7 @@ static int nfs_do_local_read(struct nfs_pgio_header *hdr, struct file *filp,
 	args.done = &done;
 	INIT_WORK_ONSTACK(&args.work, nfs_local_call_read);
 
-	queue_work(nfsiod_workqueue, &args.work);
+	queue_work(nfssync_workqueue, &args.work);
 	wait_for_completion(&done);
 	destroy_work_on_stack(&args.work);
 	return 0;
@@ -682,7 +682,7 @@ static int nfs_do_local_write(struct nfs_pgio_header *hdr, struct file *filp,
 	args.done = &done;
 	INIT_WORK_ONSTACK(&args.work, nfs_local_call_write);
 
-	queue_work(nfsiod_workqueue, &args.work);
+	queue_work(nfssync_workqueue, &args.work);
 	wait_for_completion(&done);
 	destroy_work_on_stack(&args.work);
 	return 0;
-- 
2.44.0