From:   Alistair Popple <apopple@nvidia.com>
To:     linux-mm@kvack.org, cgroups@vger.kernel.org
Cc:     linux-kernel@vger.kernel.org, jgg@nvidia.com, jhubbard@nvidia.com,
        tjmercier@google.com, hannes@cmpxchg.org, surenb@google.com,
        mkoutny@suse.com, daniel@ffwll.ch,
        "Daniel P . Berrange" <berrange@redhat.com>,
        Alex Williamson <alex.williamson@redhat.com>,
        Alistair Popple <apopple@nvidia.com>,
        Jens Axboe <axboe@kernel.dk>,
        Pavel Begunkov <asml.silence@gmail.com>,
        io-uring@vger.kernel.org
Subject: [PATCH 09/19] io_uring: convert to use vm_account
Date:   Mon,  6 Feb 2023 18:47:46 +1100
Message-Id: <44e6ead48bc53789191b22b0e140aeb82459e75f.1675669136.git-series.apopple@nvidia.com>
In-Reply-To: <cover.c238416f0e82377b449846dbb2459ae9d7030c8e.1675669136.git-series.apopple@nvidia.com>
References: <cover.c238416f0e82377b449846dbb2459ae9d7030c8e.1675669136.git-series.apopple@nvidia.com>
Content-Transfer-Encoding: 8bit
Content-Type: text/plain
MIME-Version: 1.0
X-MS-Exchange-AntiSpam-MessageData-ChunkCount: 1
X-MS-Exchange-AntiSpam-MessageData-0: =?us-ascii?Q?sJB/lu8SsYSUOQ3U33wUhkBHtA5BcV+QZ1uLtPiwnJiVJHTZQVsIF6mIvext?=
 =?us-ascii?Q?THE/AK9JGcVBH01l9E0igxUfiaJ1F9CooEZA/oxVGED5x7I2uI+w3gvHdaE0?=
 =?us-ascii?Q?J5CPvKJ49e5AmIXuMJucfkCzoemYHXKJ7YtogPikDfMAVySwYs+4Iuwp8Eun?=
 =?us-ascii?Q?OGxhScOVZgz98aToCyeANKipNjLexBg8RoAOjIs6IU93uLI8r3G1lkIx+SS1?=
 =?us-ascii?Q?A6wUOS5pC+1quwX0EtcS0lE2CoeRV1RWNAz2+Higf8Co3Y904daEpX9B9oaV?=
 =?us-ascii?Q?ua0T0HBASCZzamhLEmCuTR+wRbk080IjRkeaAsGUU//Rmf5LMZTOW+lBsVYu?=
 =?us-ascii?Q?RAd5KEtYxmdSDfftbu7zwId76wzC3wszaXdBDUjqvWJys46gJGIl23LWR150?=
 =?us-ascii?Q?HD+WLtpd4iStikgsKl3sUBIlkgONau7dvNbiCbyUwDYLSrv+aOSy9HTSMVbh?=
 =?us-ascii?Q?QyKnbW0fA45JwuBZB+G8wiVtFlwJ4K6mIk06mvAfq3HEHYKhQXdu66gBEkYC?=
 =?us-ascii?Q?UueVVcwv/daULbcZOSYE3di9UCclnhpf3az3qXX1bYD/tblPtltnz1NIZSza?=
 =?us-ascii?Q?9qStOUhEKyb88Mb0iqFT0+cROULMRDc1X8B5Lt7LqTs12CPYWGNHk4gmuAn+?=
 =?us-ascii?Q?DdQdKS+IwZIC5ZhoTM/NDmpTmi/0+ghuIcUqjkFB4v4o2c8tKExlbnce3caW?=
 =?us-ascii?Q?Ilwxmz0MmPKs2p9Bkucyx0tYJPUmjjeJRMdixdYybMrtITDR8amSkmpAWLq+?=
 =?us-ascii?Q?8+VG82Rn4Xxd6LzZ++Zq8vtiaF1ZHxbuFuHfJD9QbEOsQhiWQNwkdoMai3g5?=
 =?us-ascii?Q?uxcEQl/pyYmzSR6/05IwgIZWbL7RP+H/sWzqFftzBefkZKgM9znPhU/rlp9n?=
 =?us-ascii?Q?J+Jjk2zwgqrFBJRYPB7VQnZGELtveM4YZQGe365Gtd9nLLszNWXudTzPKkjM?=
 =?us-ascii?Q?FzVt6p01+YafFolfyynU1Y3z4oAy9eQv+T7cHb/izOxsG3B6ygKHxooP0OGl?=
 =?us-ascii?Q?GPQssUGVIrlPZ7434g7rpOn6/RubZoQdbP61QRc6P1mmMNMgEvRTj72nidTC?=
 =?us-ascii?Q?yES9CyHkEmhd3m3p5h+5MHHZOBNIdTnynbBpY0upXHOf36MLhbdiAe2k9k/3?=
 =?us-ascii?Q?J+NPSm+UfEaRWERCFs3fSkivY2l4YfjAMSYzexKbpGlNMv7aPhJOWtZe6uFN?=
 =?us-ascii?Q?Y+X0ktTB5RPrx+h40MjrF+S++7HzmkNoY76YxITq2J+yI5HOkx5Pg7gRjCSY?=
 =?us-ascii?Q?X0Vq6UOt+1FTHwg2A7XDBR70/zplgj7nS5aPZHxBoXsZM2soI4plGyNgJox4?=
 =?us-ascii?Q?Yd34JGVDmsfZrc/YLVl8RthC+o/kkLPGl0TPCK4WfGKY64Dd06GewbV/UGwZ?=
 =?us-ascii?Q?KHN+hopKXw07i7KXiNyI7xDy4OhkvU/02Ga2kPahSq9pBIo5lCIlDMD24I3D?=
 =?us-ascii?Q?Hg+ahuAdDGC+B80zYArW1qUR87+23ccG8j7yR/Rl1qBzr92LBFjYneJMdBYJ?=
 =?us-ascii?Q?fpKf9wu2UJYBkHLgLp7/Tgbf7tE8QHf1ExoR3vPzs8QC5QhhK+sHiycjFIpk?=
 =?us-ascii?Q?ctXOxQaJ/SoK4GIRqPrkoA+G1VXwuXoQC509DCGh?=
X-OriginatorOrg: Nvidia.com
X-MS-Exchange-CrossTenant-Network-Message-Id: e6c72ef4-683f-4fab-e4ad-08db0816a3f1
X-MS-Exchange-CrossTenant-AuthSource: BYAPR12MB3176.namprd12.prod.outlook.com
X-MS-Exchange-CrossTenant-AuthAs: Internal
X-MS-Exchange-CrossTenant-OriginalArrivalTime: 06 Feb 2023 07:49:13.7304
 (UTC)
X-MS-Exchange-CrossTenant-FromEntityHeader: Hosted
X-MS-Exchange-CrossTenant-Id: 43083d15-7273-40c1-b7db-39efd9ccc17a
X-MS-Exchange-CrossTenant-MailboxType: HOSTED
X-MS-Exchange-CrossTenant-UserPrincipalName: nCGlYGM5jkLy5MtbClu9G5nINFV9C9n+5uWcbYP52urB5fWT5ZaxjYfStseduUlIEppg6KcFL2431p/aXlfS9A==
X-MS-Exchange-Transport-CrossTenantHeadersStamped: DM4PR12MB5118
Precedence: bulk
List-ID: <linux-kernel.vger.kernel.org>
X-Mailing-List: linux-kernel@vger.kernel.org

Convert io_uring to use vm_account instead of directly charging pages
against the user/mm. Rather than charge pages to both user->locked_vm
and mm->pinned_vm this will only charge pages to user->locked_vm.

Signed-off-by: Alistair Popple <apopple@nvidia.com>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: Pavel Begunkov <asml.silence@gmail.com>
Cc: io-uring@vger.kernel.org
Cc: linux-kernel@vger.kernel.org
---
 include/linux/io_uring_types.h |  4 ++--
 io_uring/io_uring.c            | 20 +++---------------
 io_uring/notif.c               |  4 ++--
 io_uring/notif.h               | 10 +++------
 io_uring/rsrc.c                | 38 +++--------------------------------
 io_uring/rsrc.h                |  9 +--------
 6 files changed, 17 insertions(+), 68 deletions(-)

diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h
index 128a67a..45ac75d 100644
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
@@ -5,6 +5,7 @@
 #include <linux/task_work.h>
 #include <linux/bitmap.h>
 #include <linux/llist.h>
+#include <linux/vm_account.h>
 #include <uapi/linux/io_uring.h>
 
 struct io_wq_work_node {
@@ -343,8 +344,7 @@ struct io_ring_ctx {
 	struct io_wq_hash		*hash_map;
 
 	/* Only used for accounting purposes */
-	struct user_struct		*user;
-	struct mm_struct		*mm_account;
+	struct vm_account               vm_account;
 
 	/* ctx exit and cancelation */
 	struct llist_head		fallback_llist;
diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c
index 0a4efad..912da4f 100644
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
@@ -2744,15 +2744,11 @@ static __cold void io_ring_ctx_free(struct io_ring_ctx *ctx)
 #endif
 	WARN_ON_ONCE(!list_empty(&ctx->ltimeout_list));
 
-	if (ctx->mm_account) {
-		mmdrop(ctx->mm_account);
-		ctx->mm_account = NULL;
-	}
+	vm_account_release(&ctx->vm_account);
 	io_mem_free(ctx->rings);
 	io_mem_free(ctx->sq_sqes);
 
 	percpu_ref_exit(&ctx->refs);
-	free_uid(ctx->user);
 	io_req_caches_free(ctx);
 	if (ctx->hash_map)
 		io_wq_put_hash(ctx->hash_map);
@@ -3585,8 +3581,9 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p,
 		ctx->syscall_iopoll = 1;
 
 	ctx->compat = in_compat_syscall();
-	if (!capable(CAP_IPC_LOCK))
-		ctx->user = get_uid(current_user());
+	vm_account_init(&ctx->vm_account, current, current_user(),
+			VM_ACCOUNT_USER |
+			(capable(CAP_IPC_LOCK) ? VM_ACCOUNT_BYPASS : 0));
 
 	/*
 	 * For SQPOLL, we just need a wakeup, always. For !SQPOLL, if
@@ -3619,15 +3616,6 @@ static __cold int io_uring_create(unsigned entries, struct io_uring_params *p,
 		goto err;
 	}
 
-	/*
-	 * This is just grabbed for accounting purposes. When a process exits,
-	 * the mm is exited and dropped before the files, hence we need to hang
-	 * on to this mm purely for the purposes of being able to unaccount
-	 * memory (locked/pinned vm). It's not used for anything else.
-	 */
-	mmgrab(current->mm);
-	ctx->mm_account = current->mm;
-
 	ret = io_allocate_scq_urings(ctx, p);
 	if (ret)
 		goto err;
diff --git a/io_uring/notif.c b/io_uring/notif.c
index c4bb793..0f589fa 100644
--- a/io_uring/notif.c
+++ b/io_uring/notif.c
@@ -17,8 +17,8 @@ static void io_notif_complete_tw_ext(struct io_kiocb *notif, bool *locked)
 	if (nd->zc_report && (nd->zc_copied || !nd->zc_used))
 		notif->cqe.res |= IORING_NOTIF_USAGE_ZC_COPIED;
 
-	if (nd->account_pages && ctx->user) {
-		__io_unaccount_mem(ctx->user, nd->account_pages);
+	if (nd->account_pages) {
+		vm_unaccount_pinned(&ctx->vm_account, nd->account_pages);
 		nd->account_pages = 0;
 	}
 	io_req_task_complete(notif, locked);
diff --git a/io_uring/notif.h b/io_uring/notif.h
index c88c800..e2cb44a 100644
--- a/io_uring/notif.h
+++ b/io_uring/notif.h
@@ -43,11 +43,9 @@ static inline int io_notif_account_mem(struct io_kiocb *notif, unsigned len)
 	unsigned nr_pages = (len >> PAGE_SHIFT) + 2;
 	int ret;
 
-	if (ctx->user) {
-		ret = __io_account_mem(ctx->user, nr_pages);
-		if (ret)
-			return ret;
-		nd->account_pages += nr_pages;
-	}
+	ret = __io_account_mem(&ctx->vm_account, nr_pages);
+	if (ret)
+		return ret;
+	nd->account_pages += nr_pages;
 	return 0;
 }
diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c
index 18de10c..aa44528 100644
--- a/io_uring/rsrc.c
+++ b/io_uring/rsrc.c
@@ -42,49 +42,19 @@ void io_rsrc_refs_drop(struct io_ring_ctx *ctx)
 	}
 }
 
-int __io_account_mem(struct user_struct *user, unsigned long nr_pages)
+int __io_account_mem(struct vm_account *vm_account, unsigned long nr_pages)
 {
-	unsigned long page_limit, cur_pages, new_pages;
-
-	if (!nr_pages)
-		return 0;
-
-	/* Don't allow more pages than we can safely lock */
-	page_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
-
-	cur_pages = atomic_long_read(&user->locked_vm);
-	do {
-		new_pages = cur_pages + nr_pages;
-		if (new_pages > page_limit)
-			return -ENOMEM;
-	} while (!atomic_long_try_cmpxchg(&user->locked_vm,
-					  &cur_pages, new_pages));
-	return 0;
+	return vm_account_pinned(vm_account, nr_pages);
 }
 
 static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
 {
-	if (ctx->user)
-		__io_unaccount_mem(ctx->user, nr_pages);
-
-	if (ctx->mm_account)
-		atomic64_sub(nr_pages, &ctx->mm_account->pinned_vm);
+	vm_unaccount_pinned(&ctx->vm_account, nr_pages);
 }
 
 static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
 {
-	int ret;
-
-	if (ctx->user) {
-		ret = __io_account_mem(ctx->user, nr_pages);
-		if (ret)
-			return ret;
-	}
-
-	if (ctx->mm_account)
-		atomic64_add(nr_pages, &ctx->mm_account->pinned_vm);
-
-	return 0;
+	return vm_account_pinned(&ctx->vm_account, nr_pages);
 }
 
 static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst,
diff --git a/io_uring/rsrc.h b/io_uring/rsrc.h
index 2b87436..d8833d0 100644
--- a/io_uring/rsrc.h
+++ b/io_uring/rsrc.h
@@ -167,12 +167,5 @@ static inline u64 *io_get_tag_slot(struct io_rsrc_data *data, unsigned int idx)
 int io_files_update(struct io_kiocb *req, unsigned int issue_flags);
 int io_files_update_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
 
-int __io_account_mem(struct user_struct *user, unsigned long nr_pages);
-
-static inline void __io_unaccount_mem(struct user_struct *user,
-				      unsigned long nr_pages)
-{
-	atomic_long_sub(nr_pages, &user->locked_vm);
-}
-
+int __io_account_mem(struct vm_account *vm_account, unsigned long nr_pages);
 #endif
-- 
git-series 0.9.1