Received-SPF: pass (google.com: domain of linux-kernel-owner@vger.kernel.org designates 2620:137:e000::1:20 as permitted sender) client-ip=2620:137:e000::1:20;
From:   Nikolay Borisov <nborisov@suse.com>
To:     viro@zeniv.linux.org.uk
Cc:     linux-kernel@vger.kernel.org, agruenba@redhat.com,
        Nikolay Borisov <nborisov@suse.com>
Subject: [PATCH] iov_iter: Microoptimize fault_in_iov_iter_(writable|readable)
Date:   Fri, 22 Jul 2022 16:19:32 +0300
Message-Id: <20220722131932.3453128-1-nborisov@suse.com>
MIME-Version: 1.0
Content-Transfer-Encoding: 8bit
Precedence: bulk

Inverting the condition inside the 2 functions results in reduced nesting
as well as saving 20 bytes of generated codeu as well as eliminating a mandatory
branch when a proper userspace iov iter is passed:

add/remove: 0/0 grow/shrink: 0/2 up/down: 0/-20 (-20)
Function                                     old     new   delta
fault_in_iov_iter_writeable                  131     121     -10
fault_in_iov_iter_readable                   131     121     -10

That's because in the original version the compiler duplicates the
function epilogue right after the initial branch, resulting in the
following code:

0x000000000000081c <+12>:    je     0x828 <fault_in_iov_iter_readable+24>
0x000000000000081e <+14>:    pop    %rbx
0x000000000000081f <+15>:    mov    %r12,%rax
0x0000000000000822 <+18>:    pop    %rbp
0x0000000000000823 <+19>:    pop    %r12
0x0000000000000825 <+21>:    pop    %r13
0x0000000000000827 <+23>:    retq
0x0000000000000828 <+24>:    cmp    %rsi,0x10(%rdi) ;loop begins

There is always a branch penalty before actual executioni begins,
whilst with this patch the generated code is:

0x00000000000004ac <+12>:    jne    0x507 <fault_in_iov_iter_readable+103>
0x00000000000004ae <+14>:    cmp    %rsi,0x10(%rdi)

The branch is taken only when the passed in iter is not an iovec

Signed-off-by: Nikolay Borisov <nborisov@suse.com>
---
 lib/iov_iter.c | 76 +++++++++++++++++++++++++-------------------------
 1 file changed, 38 insertions(+), 38 deletions(-)

diff --git a/lib/iov_iter.c b/lib/iov_iter.c
index 0b64695ab632..897148d9cd12 100644
--- a/lib/iov_iter.c
+++ b/lib/iov_iter.c
@@ -443,26 +443,26 @@ static size_t copy_page_to_iter_pipe(struct page *page, size_t offset, size_t by
  */
 size_t fault_in_iov_iter_readable(const struct iov_iter *i, size_t size)
 {
-	if (iter_is_iovec(i)) {
-		size_t count = min(size, iov_iter_count(i));
-		const struct iovec *p;
-		size_t skip;
-
-		size -= count;
-		for (p = i->iov, skip = i->iov_offset; count; p++, skip = 0) {
-			size_t len = min(count, p->iov_len - skip);
-			size_t ret;
-
-			if (unlikely(!len))
-				continue;
-			ret = fault_in_readable(p->iov_base + skip, len);
-			count -= len - ret;
-			if (ret)
-				break;
-		}
-		return count + size;
+	size_t count = min(size, iov_iter_count(i));
+	const struct iovec *p;
+	size_t skip;
+
+	if (!iter_is_iovec(i))
+		return 0;
+
+	size -= count;
+	for (p = i->iov, skip = i->iov_offset; count; p++, skip = 0) {
+		size_t len = min(count, p->iov_len - skip);
+		size_t ret;
+
+		if (unlikely(!len))
+			continue;
+		ret = fault_in_readable(p->iov_base + skip, len);
+		count -= len - ret; // reduce count by the number faulted in
+		if (ret)
+			break;
 	}
-	return 0;
+	return count + size;
 }
 EXPORT_SYMBOL(fault_in_iov_iter_readable);

@@ -482,26 +482,26 @@ EXPORT_SYMBOL(fault_in_iov_iter_readable);
  */
 size_t fault_in_iov_iter_writeable(const struct iov_iter *i, size_t size)
 {
-	if (iter_is_iovec(i)) {
-		size_t count = min(size, iov_iter_count(i));
-		const struct iovec *p;
-		size_t skip;
-
-		size -= count;
-		for (p = i->iov, skip = i->iov_offset; count; p++, skip = 0) {
-			size_t len = min(count, p->iov_len - skip);
-			size_t ret;
-
-			if (unlikely(!len))
-				continue;
-			ret = fault_in_safe_writeable(p->iov_base + skip, len);
-			count -= len - ret;
-			if (ret)
-				break;
-		}
-		return count + size;
+	size_t count = min(size, iov_iter_count(i));
+	const struct iovec *p;
+	size_t skip;
+
+	if (!iter_is_iovec(i))
+		return 0;
+
+	size -= count;
+	for (p = i->iov, skip = i->iov_offset; count; p++, skip = 0) {
+		size_t len = min(count, p->iov_len - skip);
+		size_t ret;
+
+		if (unlikely(!len))
+			continue;
+		ret = fault_in_safe_writeable(p->iov_base + skip, len);
+		count -= len - ret;
+		if (ret)
+			break;
 	}
-	return 0;
+	return count + size;
 }
 EXPORT_SYMBOL(fault_in_iov_iter_writeable);

--
2.25.1