Return-Path: Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1751455AbdFAG7H (ORCPT ); Thu, 1 Jun 2017 02:59:07 -0400 Received: from mail.fireflyinternet.com ([109.228.58.192]:64175 "EHLO fireflyinternet.com" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1751283AbdFAG7F (ORCPT ); Thu, 1 Jun 2017 02:59:05 -0400 X-Default-Received-SPF: pass (skip=forwardok (res=PASS)) x-ip-name=78.156.65.138; From: Chris Wilson To: linux-kernel@vger.kernel.org Cc: x86@kernel.org, intel-gfx@lists.freedesktop.org, Chris Wilson , Thomas Gleixner , Ingo Molnar , "H. Peter Anvin" Subject: [PATCH 3/3] x86-64: Inline 6/12 byte copy_user Date: Thu, 1 Jun 2017 07:58:43 +0100 Message-Id: <20170601065843.2392-4-chris@chris-wilson.co.uk> X-Mailer: git-send-email 2.11.0 In-Reply-To: <20170601065843.2392-1-chris@chris-wilson.co.uk> References: <20170601065843.2392-1-chris@chris-wilson.co.uk> X-Originating-IP: 78.156.65.138 X-Country: code=GB country="United Kingdom" ip=78.156.65.138 Sender: linux-kernel-owner@vger.kernel.org List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Length: 2962 Lines: 89 Extend the list of replacements for compile-time known sizes to include 6/12 byte copies. These expand to two movs (along with their exception table) and are cheaper to inline than the function call (similar to the 10 byte copy already handled). Signed-off-by: Chris Wilson Cc: Thomas Gleixner Cc: Ingo Molnar Cc: "H. Peter Anvin" --- arch/x86/include/asm/uaccess_64.h | 42 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h index c5504b9a472e..ff2d65baa988 100644 --- a/arch/x86/include/asm/uaccess_64.h +++ b/arch/x86/include/asm/uaccess_64.h @@ -71,6 +71,16 @@ raw_copy_from_user(void *dst, const void __user *src, unsigned long size) ret, "l", "k", "=r", 4); __uaccess_end(); return ret; + case 6: + __uaccess_begin(); + __get_user_asm_nozero(*(u32 *)dst, (u32 __user *)src, + ret, "l", "k", "=r", 6); + if (likely(!ret)) + __get_user_asm_nozero(*(u16 *)(4 + (char *)dst), + (u16 __user *)(4 + (char __user *)src), + ret, "w", "w", "=r", 2); + __uaccess_end(); + return ret; case 8: __uaccess_begin(); __get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src, @@ -87,6 +97,16 @@ raw_copy_from_user(void *dst, const void __user *src, unsigned long size) ret, "w", "w", "=r", 2); __uaccess_end(); return ret; + case 12: + __uaccess_begin(); + __get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src, + ret, "q", "", "=r", 10); + if (likely(!ret)) + __get_user_asm_nozero(*(u32 *)(8 + (char *)dst), + (u32 __user *)(8 + (char __user *)src), + ret, "l", "k", "=r", 4); + __uaccess_end(); + return ret; case 16: __uaccess_begin(); __get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src, @@ -128,6 +148,17 @@ raw_copy_to_user(void __user *dst, const void *src, unsigned long size) ret, "l", "k", "ir", 4); __uaccess_end(); return ret; + case 6: + __uaccess_begin(); + __put_user_asm(*(u32 *)src, (u32 __user *)dst, + ret, "l", "k", "ir", 6); + if (likely(!ret)) { + asm("":::"memory"); + __put_user_asm(2[(u16 *)src], 2 + (u16 __user *)dst, + ret, "w", "w", "ir", 2); + } + __uaccess_end(); + return ret; case 8: __uaccess_begin(); __put_user_asm(*(u64 *)src, (u64 __user *)dst, @@ -145,6 +176,17 @@ raw_copy_to_user(void __user *dst, const void *src, unsigned long size) } __uaccess_end(); return ret; + case 12: + __uaccess_begin(); + __put_user_asm(*(u64 *)src, (u64 __user *)dst, + ret, "q", "", "er", 12); + if (likely(!ret)) { + asm("":::"memory"); + __put_user_asm(2[(u32 *)src], 2 + (u32 __user *)dst, + ret, "l", "k", "ir", 4); + } + __uaccess_end(); + return ret; case 16: __uaccess_begin(); __put_user_asm(*(u64 *)src, (u64 __user *)dst, -- 2.11.0