Received: by 10.213.65.68 with SMTP id h4csp1742172imn; Sun, 1 Apr 2018 13:44:57 -0700 (PDT) X-Google-Smtp-Source: AIpwx48Fy3XE9KToWvogP9L9MsJlIo+6CSsKulzW2iK54ipWFkl9J1DDneBFB8MLp+O0WjhfYHpe X-Received: by 2002:a17:902:8d97:: with SMTP id v23-v6mr7178916plo.285.1522615497208; Sun, 01 Apr 2018 13:44:57 -0700 (PDT) ARC-Seal: i=1; a=rsa-sha256; t=1522615497; cv=none; d=google.com; s=arc-20160816; b=XnIfTSJl7s429kInUHcB5cllbYsx0oYa2zW14wvaUxdEyCiLhCtGVka7XpYpnDVOBV xzF0eLhgJwxUhw0it2oXEorf6SyUVqKRQqJFaw9im3sAwLkkejFcBpQR6zssLHVR+S/A 71+iE9OF082qsVC5pUoV+8YuFFyLb3UDoMkeLzMLysm2DkULYgQyDepmvKZylwds7eHq ai6qGxaNgdi1vfcK6ZNCRlsRknKH4u34H+UIEO7IKBX1ywxR6kUccVyHbD88Kk/crI6T dTYgVcZxhWABeE73VTSMzfsIw8HVLYXB/pIR/MpMC1apdG9pZnmXp2VNm4KVk+0vnBfq 3w2Q== ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816; h=list-id:precedence:sender:content-transfer-encoding:mime-version :user-agent:references:in-reply-to:message-id:date:to:from:subject :organization:arc-authentication-results; bh=0aPnzt8iueSDjLbGszBefDrvQnPiY/XdmgVBTNoQNRM=; b=i+cTnNmC8V4idp4NIhD6OqvqFG8EDnSE3Wk+Z8Ue5d9F9muPlabPHO7mSxHD784gxo KCzyBvd5/qpYCeOj11fwJAAxTMKjWaqqfxJ1F4vKnRleHCGdtdsmOTpq42ZE8QXool6e Ogl4UF0SG/8vpwt4N1tucN6GJh/TVKDAlbQyxJCh3fz+gSi70vTiGMw9/SLihy76FIar ugormSt9dOWuKo7CkukChmptB1rZzaZuhZvewCOKWYcYhO2M1RHEIpPpHyq/Aqs6ejRX 02E4f0sqyaQ4EXaTSvRtQyxafpJuqrtV2uG4Zbe3XOi0UjDeWTxyrR28FGUTkocmCjfq c6Nw== ARC-Authentication-Results: i=1; mx.google.com; spf=pass (google.com: best guess record for domain of linux-kernel-owner@vger.kernel.org designates 209.132.180.67 as permitted sender) smtp.mailfrom=linux-kernel-owner@vger.kernel.org; dmarc=fail (p=NONE sp=NONE dis=NONE) header.from=redhat.com Return-Path: Received: from vger.kernel.org (vger.kernel.org. [209.132.180.67]) by mx.google.com with ESMTP id w69si4492502pfd.332.2018.04.01.13.44.43; Sun, 01 Apr 2018 13:44:57 -0700 (PDT) Received-SPF: pass (google.com: best guess record for domain of linux-kernel-owner@vger.kernel.org designates 209.132.180.67 as permitted sender) client-ip=209.132.180.67; Authentication-Results: mx.google.com; spf=pass (google.com: best guess record for domain of linux-kernel-owner@vger.kernel.org designates 209.132.180.67 as permitted sender) smtp.mailfrom=linux-kernel-owner@vger.kernel.org; dmarc=fail (p=NONE sp=NONE dis=NONE) header.from=redhat.com Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1754241AbeDAUln (ORCPT + 99 others); Sun, 1 Apr 2018 16:41:43 -0400 Received: from mx3-rdu2.redhat.com ([66.187.233.73]:42138 "EHLO mx1.redhat.com" rhost-flags-OK-OK-OK-FAIL) by vger.kernel.org with ESMTP id S1753951AbeDAUll (ORCPT ); Sun, 1 Apr 2018 16:41:41 -0400 Received: from smtp.corp.redhat.com (int-mx03.intmail.prod.int.rdu2.redhat.com [10.11.54.3]) (using TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)) (No client certificate requested) by mx1.redhat.com (Postfix) with ESMTPS id 93E604023141 for ; Sun, 1 Apr 2018 20:41:40 +0000 (UTC) Received: from warthog.procyon.org.uk (ovpn-120-59.rdu2.redhat.com [10.10.120.59]) by smtp.corp.redhat.com (Postfix) with ESMTP id 10AA5111DCF0; Sun, 1 Apr 2018 20:41:39 +0000 (UTC) Organization: Red Hat UK Ltd. Registered Address: Red Hat UK Ltd, Amberley Place, 107-111 Peascod Street, Windsor, Berkshire, SI4 1TE, United Kingdom. Registered in England and Wales under Company Registration No. 3798903 Subject: [PATCH 14/45] C++: x86: Turn put_user(), get_user() & co. into inline template functions From: David Howells To: linux-kernel@vger.kernel.org Date: Sun, 01 Apr 2018 21:41:39 +0100 Message-ID: <152261529959.30503.12922752484575508631.stgit@warthog.procyon.org.uk> In-Reply-To: <152261521484.30503.16131389653845029164.stgit@warthog.procyon.org.uk> References: <152261521484.30503.16131389653845029164.stgit@warthog.procyon.org.uk> User-Agent: StGit/0.17.1-dirty MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: 7bit X-Scanned-By: MIMEDefang 2.78 on 10.11.54.3 X-Greylist: Sender IP whitelisted, not delayed by milter-greylist-4.5.16 (mx1.redhat.com [10.11.55.6]); Sun, 01 Apr 2018 20:41:40 +0000 (UTC) X-Greylist: inspected by milter-greylist-4.5.16 (mx1.redhat.com [10.11.55.6]); Sun, 01 Apr 2018 20:41:40 +0000 (UTC) for IP:'10.11.54.3' DOMAIN:'int-mx03.intmail.prod.int.rdu2.redhat.com' HELO:'smtp.corp.redhat.com' FROM:'dhowells@redhat.com' RCPT:'' Sender: linux-kernel-owner@vger.kernel.org Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Turn put_user(), get_user() and similar functions into C++ inline template functions as the source for this is simpler and more obvious and not heavily macroised. Note that it is unnecessary to stick an extension on the load or store instruction that does the protected access to memory as the assembler can work that out from the size of the source/destination register. Signed-off-by: David Howells --- arch/x86/include/asm/uaccess.h | 630 ++++++++++++++++++++----------------- arch/x86/include/asm/uaccess_64.h | 66 +--- 2 files changed, 361 insertions(+), 335 deletions(-) diff --git a/arch/x86/include/asm/uaccess.h b/arch/x86/include/asm/uaccess.h index aae77eb8491c..64bd782c87c6 100644 --- a/arch/x86/include/asm/uaccess.h +++ b/arch/x86/include/asm/uaccess.h @@ -12,6 +12,10 @@ #include #include +/* FIXME: this hack is definitely wrong -AK */ +struct __large_struct { unsigned long buf[100]; }; +#define __m(x) (*(struct __large_struct __user *)(x)) + /* * The fs value determines whether argument validity checking should be * performed or not. If get_fs() == USER_DS, checking is performed, with @@ -101,6 +105,12 @@ static inline bool __chk_range_not_ok(unsigned long addr, unsigned long size, un likely(!__range_not_ok(addr, size, user_addr_max())); \ }) +extern void __put_user_bad(void); +extern void __get_user_unsupported_size(void) + __compiletime_error("Bad argument size for get_user"); +extern void __put_user_unsupported_size(void) + __compiletime_error("Bad argument size for put_user"); + /* * These are the main single-value transfer routines. They automatically * use the right size if we just have the right pointer type. @@ -130,13 +140,6 @@ extern int __get_user_bad(void); barrier_nospec(); \ }) -/* - * This is a type: either unsigned long, if the argument fits into - * that type, or otherwise unsigned long long. - */ -#define __inttype(x) \ -__typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL)) - /** * get_user: - Get a simple variable from user space. * @x: Variable to store result. @@ -168,62 +171,21 @@ __typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL)) * Clang/LLVM cares about the size of the register, but still wants * the base register for something that ends up being a pair. */ -#define get_user(x, ptr) \ -({ \ - int __ret_gu; \ - register __inttype(*(ptr)) __val_gu asm("%"_ASM_DX); \ - __chk_user_ptr(ptr); \ - might_fault(); \ - asm volatile("call __get_user_%P4" \ - : "=a" (__ret_gu), "=r" (__val_gu), \ - ASM_CALL_CONSTRAINT \ - : "0" (ptr), "i" (sizeof(*(ptr)))); \ - (x) = (__force __typeof__(*(ptr))) __val_gu; \ - __builtin_expect(__ret_gu, 0); \ -}) - -#define __put_user_x(size, x, ptr, __ret_pu) \ - asm volatile("call __put_user_" #size : "=a" (__ret_pu) \ - : "0" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx") - - - -#ifdef CONFIG_X86_32 -#define __put_user_asm_u64(x, addr, err, errret) \ - asm volatile("\n" \ - "1: movl %%eax,0(%2)\n" \ - "2: movl %%edx,4(%2)\n" \ - "3:" \ - ".section .fixup,\"ax\"\n" \ - "4: movl %3,%0\n" \ - " jmp 3b\n" \ - ".previous\n" \ - _ASM_EXTABLE(1b, 4b) \ - _ASM_EXTABLE(2b, 4b) \ - : "=r" (err) \ - : "A" (x), "r" (addr), "i" (errret), "0" (err)) - -#define __put_user_asm_ex_u64(x, addr) \ - asm volatile("\n" \ - "1: movl %%eax,0(%1)\n" \ - "2: movl %%edx,4(%1)\n" \ - "3:" \ - _ASM_EXTABLE_EX(1b, 2b) \ - _ASM_EXTABLE_EX(2b, 3b) \ - : : "A" (x), "r" (addr)) - -#define __put_user_x8(x, ptr, __ret_pu) \ - asm volatile("call __put_user_8" : "=a" (__ret_pu) \ - : "A" ((typeof(*(ptr)))(x)), "c" (ptr) : "ebx") -#else -#define __put_user_asm_u64(x, ptr, retval, errret) \ - __put_user_asm(x, ptr, retval, "q", "", "er", errret) -#define __put_user_asm_ex_u64(x, addr) \ - __put_user_asm_ex(x, addr, "q", "", "er") -#define __put_user_x8(x, ptr, __ret_pu) __put_user_x(8, x, ptr, __ret_pu) -#endif - -extern void __put_user_bad(void); +template +static inline int get_user(VAL &x, TYPE *ptr) +{ + register TYPE val asm("%" _ASM_DX); + int ret; + + __chk_user_ptr(ptr); + might_fault(); + asm volatile("call __get_user_%P4" + : "=a" (ret), "=r" (val), + ASM_CALL_CONSTRAINT + : "0" (ptr), "i" (sizeof(*(ptr)))); + x = val; + return __builtin_expect(ret, 0); +} /* * Strange magic calling convention: pointer in %ecx, @@ -251,157 +213,47 @@ extern void __put_user_8(void); * * Returns zero on success, or -EFAULT on error. */ -#define put_user(x, ptr) \ -({ \ - int __ret_pu; \ - __typeof__(*(ptr)) __pu_val; \ - __chk_user_ptr(ptr); \ - might_fault(); \ - __pu_val = x; \ - switch (sizeof(*(ptr))) { \ - case 1: \ - __put_user_x(1, __pu_val, ptr, __ret_pu); \ - break; \ - case 2: \ - __put_user_x(2, __pu_val, ptr, __ret_pu); \ - break; \ - case 4: \ - __put_user_x(4, __pu_val, ptr, __ret_pu); \ - break; \ - case 8: \ - __put_user_x8(__pu_val, ptr, __ret_pu); \ - break; \ - default: \ - __put_user_x(X, __pu_val, ptr, __ret_pu); \ - break; \ - } \ - __builtin_expect(__ret_pu, 0); \ -}) - -#define __put_user_size(x, ptr, size, retval, errret) \ -do { \ - retval = 0; \ - __chk_user_ptr(ptr); \ - switch (size) { \ - case 1: \ - __put_user_asm(x, ptr, retval, "b", "b", "iq", errret); \ - break; \ - case 2: \ - __put_user_asm(x, ptr, retval, "w", "w", "ir", errret); \ - break; \ - case 4: \ - __put_user_asm(x, ptr, retval, "l", "k", "ir", errret); \ - break; \ - case 8: \ - __put_user_asm_u64((__typeof__(*ptr))(x), ptr, retval, \ - errret); \ - break; \ - default: \ - __put_user_bad(); \ - } \ -} while (0) - -/* - * This doesn't do __uaccess_begin/end - the exception handling - * around it must do that. - */ -#define __put_user_size_ex(x, ptr, size) \ -do { \ - __chk_user_ptr(ptr); \ - switch (size) { \ - case 1: \ - __put_user_asm_ex(x, ptr, "b", "b", "iq"); \ - break; \ - case 2: \ - __put_user_asm_ex(x, ptr, "w", "w", "ir"); \ - break; \ - case 4: \ - __put_user_asm_ex(x, ptr, "l", "k", "ir"); \ - break; \ - case 8: \ - __put_user_asm_ex_u64((__typeof__(*ptr))(x), ptr); \ - break; \ - default: \ - __put_user_bad(); \ - } \ -} while (0) +template +static inline int put_user(VAL x, TYPE *ptr) +{ + register TYPE val asm("%" _ASM_DX); + int ret; + __chk_user_ptr(ptr); + might_fault(); + switch (sizeof(*(ptr))) { + case 8: #ifdef CONFIG_X86_32 -#define __get_user_asm_u64(x, ptr, retval, errret) \ -({ \ - __typeof__(ptr) __ptr = (ptr); \ - asm volatile("\n" \ - "1: movl %2,%%eax\n" \ - "2: movl %3,%%edx\n" \ - "3:\n" \ - ".section .fixup,\"ax\"\n" \ - "4: mov %4,%0\n" \ - " xorl %%eax,%%eax\n" \ - " xorl %%edx,%%edx\n" \ - " jmp 3b\n" \ - ".previous\n" \ - _ASM_EXTABLE(1b, 4b) \ - _ASM_EXTABLE(2b, 4b) \ - : "=r" (retval), "=&A"(x) \ - : "m" (__m(__ptr)), "m" __m(((u32 __user *)(__ptr)) + 1), \ - "i" (errret), "0" (retval)); \ -}) + asm volatile("call __put_user_%8" + : "=a" (ret), "=r" (val), + ASM_CALL_CONSTRAINT + : "A" (val), "c" (ptr) + : "ebx"); + break; +#endif + /* Fall through */ + case 1: + case 2: + case 4: + asm volatile("call __put_user_%P4" + : "=a" (ret), "=r" (val), + ASM_CALL_CONSTRAINT + : "0" (val), "c" (ptr) + : "ebx"); + break; + default: + __put_user_unsupported_size(); + } + return __builtin_expect(ret, 0); +} +#ifdef CONFIG_X86_32 #define __get_user_asm_ex_u64(x, ptr) (x) = __get_user_bad() #else -#define __get_user_asm_u64(x, ptr, retval, errret) \ - __get_user_asm(x, ptr, retval, "q", "", "=r", errret) #define __get_user_asm_ex_u64(x, ptr) \ __get_user_asm_ex(x, ptr, "q", "", "=r") #endif -#define __get_user_size(x, ptr, size, retval, errret) \ -do { \ - retval = 0; \ - __chk_user_ptr(ptr); \ - switch (size) { \ - case 1: \ - __get_user_asm(x, ptr, retval, "b", "b", "=q", errret); \ - break; \ - case 2: \ - __get_user_asm(x, ptr, retval, "w", "w", "=r", errret); \ - break; \ - case 4: \ - __get_user_asm(x, ptr, retval, "l", "k", "=r", errret); \ - break; \ - case 8: \ - __get_user_asm_u64(x, ptr, retval, errret); \ - break; \ - default: \ - (x) = __get_user_bad(); \ - } \ -} while (0) - -#define __get_user_asm(x, addr, err, itype, rtype, ltype, errret) \ - asm volatile("\n" \ - "1: mov"itype" %2,%"rtype"1\n" \ - "2:\n" \ - ".section .fixup,\"ax\"\n" \ - "3: mov %3,%0\n" \ - " xor"itype" %"rtype"1,%"rtype"1\n" \ - " jmp 2b\n" \ - ".previous\n" \ - _ASM_EXTABLE(1b, 3b) \ - : "=r" (err), ltype(x) \ - : "m" (__m(addr)), "i" (errret), "0" (err)) - -#define __get_user_asm_nozero(x, addr, err, itype, rtype, ltype, errret) \ - asm volatile("\n" \ - "1: mov"itype" %2,%"rtype"1\n" \ - "2:\n" \ - ".section .fixup,\"ax\"\n" \ - "3: mov %3,%0\n" \ - " jmp 2b\n" \ - ".previous\n" \ - _ASM_EXTABLE(1b, 3b) \ - : "=r" (err), ltype(x) \ - : "m" (__m(addr)), "i" (errret), "0" (err)) - /* * This doesn't do __uaccess_begin/end - the exception handling * around it must do that. @@ -437,53 +289,6 @@ do { \ _ASM_EXTABLE_EX(1b, 3b) \ : ltype(x) : "m" (__m(addr))) -#define __put_user_nocheck(x, ptr, size) \ -({ \ - int __pu_err; \ - __uaccess_begin(); \ - __put_user_size((x), (ptr), (size), __pu_err, -EFAULT); \ - __uaccess_end(); \ - __builtin_expect(__pu_err, 0); \ -}) - -#define __get_user_nocheck(x, ptr, size) \ -({ \ - int __gu_err; \ - __inttype(*(ptr)) __gu_val; \ - __uaccess_begin_nospec(); \ - __get_user_size(__gu_val, (ptr), (size), __gu_err, -EFAULT); \ - __uaccess_end(); \ - (x) = (__force __typeof__(*(ptr)))__gu_val; \ - __builtin_expect(__gu_err, 0); \ -}) - -/* FIXME: this hack is definitely wrong -AK */ -struct __large_struct { unsigned long buf[100]; }; -#define __m(x) (*(struct __large_struct __user *)(x)) - -/* - * Tell gcc we read from memory instead of writing: this is because - * we do not write to any memory gcc knows about, so there are no - * aliasing issues. - */ -#define __put_user_asm(x, addr, err, itype, rtype, ltype, errret) \ - asm volatile("\n" \ - "1: mov"itype" %"rtype"1,%2\n" \ - "2:\n" \ - ".section .fixup,\"ax\"\n" \ - "3: mov %3,%0\n" \ - " jmp 2b\n" \ - ".previous\n" \ - _ASM_EXTABLE(1b, 3b) \ - : "=r"(err) \ - : ltype(x), "m" (__m(addr)), "i" (errret), "0" (err)) - -#define __put_user_asm_ex(x, addr, itype, rtype, ltype) \ - asm volatile("1: mov"itype" %"rtype"0,%1\n" \ - "2:\n" \ - _ASM_EXTABLE_EX(1b, 2b) \ - : : ltype(x), "m" (__m(addr))) - /* * uaccess_try and catch */ @@ -501,6 +306,111 @@ struct __large_struct { unsigned long buf[100]; }; (err) |= (current->thread.uaccess_err ? -EFAULT : 0); \ } while (0) +/** + * __get_user_unsafe_nozero - Read a value from userspace with no uaccess check + */ +template +static inline int __get_user_unsafe_nozero(VAL &x, const TYPE *ptr, int err=-EFAULT) +{ + TYPE val; + int ret = 0; + + __chk_user_ptr(ptr); + switch (sizeof(TYPE)) { + case 8: +#if __LONG_WIDTH__ == 32 + asm volatile("\n" + "1: mov %2,%%eax\n" + "2: mov %3,%%edx\n" + "3:\n" + ".section .fixup,\"ax\"\n" + "4: mov %4,%0\n" + " jmp 3b\n" + ".previous\n" + _ASM_EXTABLE(1b, 4b) + _ASM_EXTABLE(2b, 4b) + : "=r" (ret), "=&A"(val) + : "m" (__m(ptr)), + "m" (__m((const char *)ptr + 4)), + "i" (err), "0" (ret)); + break; +#endif + case 1: + case 2: + case 4: + asm volatile("\n" + "1: mov %2,%1\n" + "2:\n" + ".section .fixup,\"ax\"\n" + "3: mov %3,%0\n" + " jmp 2b\n" + ".previous\n" + _ASM_EXTABLE(1b, 3b) + : "=r" (ret), "=r"(val) + : "m" (__m(ptr)), "i" (err), "0" (ret)); + break; + default: + __get_user_unsupported_size(); + } + + x = val; + return __builtin_expect(ret, 0); +} + +/** + * __get_user_unsafe - Read a value from userspace with no uaccess check + */ +template +static inline int __get_user_unsafe(VAL &x, const TYPE *ptr, int err=-EFAULT) +{ + TYPE val; + int ret = 0; + + __chk_user_ptr(ptr); + switch (sizeof(TYPE)) { + case 8: +#if __LONG_WIDTH__ == 32 + asm volatile("\n" + "1: mov %2,%%eax\n" + "2: mov %3,%%edx\n" + "3:\n" + ".section .fixup,\"ax\"\n" + "4: mov %4,%0\n" + " xor %%eax,%%eax\n" + " xor %%edx,%%edx\n" + " jmp 3b\n" + ".previous\n" + _ASM_EXTABLE(1b, 4b) + _ASM_EXTABLE(2b, 4b) + : "=r" (ret), "=&A"(val) + : "m" (__m(ptr)), + "m" (__m((const char *)ptr + 4)), + "i" (err), "0" (ret)); + break; +#endif + case 1: + case 2: + case 4: + asm volatile("\n" + "1: mov %2,%1\n" + "2:\n" + ".section .fixup,\"ax\"\n" + "3: mov %3,%0\n" + " xor %1,%1\n" + " jmp 2b\n" + ".previous\n" + _ASM_EXTABLE(1b, 3b) + : "=r" (ret), "=r"(val) + : "m" (__m(ptr)), "i" (err), "0" (ret)); + break; + default: + __get_user_unsupported_size(); + } + + x = val; + return __builtin_expect(ret, 0); +} + /** * __get_user: - Get a simple variable from user space, with less checking. * @x: Variable to store result. @@ -509,9 +419,9 @@ struct __large_struct { unsigned long buf[100]; }; * Context: User context only. This function may sleep if pagefaults are * enabled. * - * This macro copies a single simple variable from user space to kernel - * space. It supports simple types like char and int, but not larger - * data types like structures or arrays. + * This function copies a single simple variable from user space to kernel + * space. It supports simple types like char and int, but not larger data + * types like structures or arrays. * * @ptr must have pointer-to-simple-variable type, and the result of * dereferencing @ptr must be assignable to @x without a cast. @@ -522,9 +432,68 @@ struct __large_struct { unsigned long buf[100]; }; * Returns zero on success, or -EFAULT on error. * On error, the variable @x is set to zero. */ +template +static inline int __get_user(VAL &x, const TYPE *ptr) +{ + int ret; -#define __get_user(x, ptr) \ - __get_user_nocheck((x), (ptr), sizeof(*(ptr))) + __uaccess_begin_nospec(); + ret = __get_user_unsafe(x, ptr); + __uaccess_end(); + return ret; +} + +/** + * __put_user_unsafe - Copy a value to userspace with no uaccess check + */ +template +static inline int __put_user_unsafe(VAL x, TYPE *ptr, int err=-EFAULT) +{ + TYPE val; + int ret = 0; + + __chk_user_ptr(ptr); + switch (sizeof(TYPE)) { + case 8: +#if __LONG_WIDTH__ == 32 + asm volatile("\n" + "1: mov %%eax,%2\n" + "2: mov %%edx,%3\n" + "3:\n" + ".section .fixup,\"ax\"\n" + "4: mov %4,%0\n" + " jmp 3b\n" + ".previous\n" + _ASM_EXTABLE(1b, 4b) + _ASM_EXTABLE(2b, 4b) + : "=r" (ret) + : "A"(val), + "m" (__m(ptr)), + "m" (__m((const char *)ptr + 4)), + "i" (err), "0" (ret)); + break; +#endif + case 1: + case 2: + case 4: + asm volatile("\n" + "1: mov %1,%2\n" + "2:\n" + ".section .fixup,\"ax\"\n" + "3: mov %3,%0\n" + " jmp 2b\n" + ".previous\n" + _ASM_EXTABLE(1b, 3b) + : "=r" (ret), "=r"(val) + : "m" (__m(ptr)), "i" (err), "0" (ret)); + break; + default: + __put_user_unsupported_size(); + } + + x = val; + return __builtin_expect(ret, 0); +} /** * __put_user: - Write a simple value into user space, with less checking. @@ -546,32 +515,117 @@ struct __large_struct { unsigned long buf[100]; }; * * Returns zero on success, or -EFAULT on error. */ +template +static inline int __put_user(VAL x, TYPE *ptr, int err=-EFAULT) +{ + int ret; -#define __put_user(x, ptr) \ - __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) + __uaccess_begin(); + ret = __put_user_unsafe(x, ptr, err); + __uaccess_end(); + return ret; +} -/* - * {get|put}_user_try and catch +/** + * get_user_ex - Read a value from userspace with indirect error * - * get_user_try { - * get_user_ex(...); - * } get_user_catch(err) + * Use as: + * get_user_try { + * get_user_ex(...); + * } get_user_catch(err) */ +template +static inline void get_user_ex(VAL &x, const TYPE *ptr) +{ + TYPE val; + + __chk_user_ptr(ptr); + switch (sizeof(TYPE)) { + case 8: +#if __LONG_WIDTH__ == 32 + asm volatile("\n" + "1: mov %1,%%eax\n" + "2: mov %2,%%edx\n" + "3:\n" + _ASM_EXTABLE(1b, 3b) + _ASM_EXTABLE(2b, 3b) + : "=&A"(val) + : "m" (__m(ptr)), + "m" (__m((const char *)ptr + 4))); + break; +#endif + case 1: + case 2: + case 4: + asm volatile("\n" + "1: mov %1,%0\n" + "2:\n" + _ASM_EXTABLE(1b, 2b) + : "=r"(val) + : "m" (__m(ptr))); + break; + default: + __get_user_unsupported_size(); + } + + x = val; +} #define get_user_try uaccess_try_nospec #define get_user_catch(err) uaccess_catch(err) -#define get_user_ex(x, ptr) do { \ - unsigned long __gue_val; \ - __get_user_size_ex((__gue_val), (ptr), (sizeof(*(ptr)))); \ - (x) = (__force __typeof__(*(ptr)))__gue_val; \ -} while (0) +/** + * put_user_ex - Copy a value to userspace with indirect error + * + * Use as: + * put_user_try { + * put_user_ex(...); + * } put_user_catch(err) + */ +template +static inline void put_user_ex(VAL x, TYPE *ptr) +{ + TYPE val; + + + /* We tell gcc that we're reading from memory instead of writing: this + * is because we do not write to any memory gcc knows about, so there + * are no aliasing issues. + */ + __chk_user_ptr(ptr); + switch (sizeof(TYPE)) { + case 8: +#if __LONG_WIDTH__ == 32 + asm volatile("\n" + "1: mov %%eax,%1\n" + "2: mov %%edx,%2\n" + "3:\n" + _ASM_EXTABLE(1b, 3b) + _ASM_EXTABLE(2b, 3b) + : + : "A"(val), + "m" (__m(ptr)), + "m" (__m((const char *)ptr + 4))); + break; +#endif + case 1: + case 2: + case 4: + asm volatile("\n" + "1: mov %0,%1\n" + "2:\n" + ".previous\n" + _ASM_EXTABLE(1b, 2b) + : + : "=r"(val), "m" (__m(ptr))); + break; + default: + __put_user_unsupported_size(); + } +} #define put_user_try uaccess_try #define put_user_catch(err) uaccess_catch(err) -#define put_user_ex(x, ptr) \ - __put_user_size_ex((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr))) - extern unsigned long copy_from_user_nmi(void *to, const void __user *from, unsigned long n); extern __must_check long @@ -711,22 +765,20 @@ extern struct movsl_mask { #define user_access_begin() __uaccess_begin() #define user_access_end() __uaccess_end() -#define unsafe_put_user(x, ptr, err_label) \ -do { \ - int __pu_err; \ - __typeof__(*(ptr)) __pu_val = (x); \ - __put_user_size(__pu_val, (ptr), sizeof(*(ptr)), __pu_err, -EFAULT); \ - if (unlikely(__pu_err)) goto err_label; \ -} while (0) - -#define unsafe_get_user(x, ptr, err_label) \ -do { \ - int __gu_err; \ - __inttype(*(ptr)) __gu_val; \ - __get_user_size(__gu_val, (ptr), sizeof(*(ptr)), __gu_err, -EFAULT); \ - (x) = (__force __typeof__(*(ptr)))__gu_val; \ - if (unlikely(__gu_err)) goto err_label; \ -} while (0) +#define unsafe_put_user(x, ptr, err_label) \ + do { \ + int __pu_err; \ + __pu_err = __put_user_unsafe(x, ptr); \ + if (unlikely(__gu_err)) \ + goto err_label; \ + } while (0) + +#define unsafe_get_user(x, ptr, err_label) \ + do { \ + int __gu_err; \ + __gu_err = __get_user_unsafe(x, ptr); \ + if (unlikely(__gu_err)) \ + goto err_label; \ + } while (0) #endif /* _ASM_X86_UACCESS_H */ - diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h index 62546b3a398e..24bf7d0a21b6 100644 --- a/arch/x86/include/asm/uaccess_64.h +++ b/arch/x86/include/asm/uaccess_64.h @@ -56,46 +56,40 @@ raw_copy_from_user(void *dst, const void __user *src, unsigned long size) switch (size) { case 1: __uaccess_begin_nospec(); - __get_user_asm_nozero(*(u8 *)dst, (u8 __user *)src, - ret, "b", "b", "=q", 1); + ret = __get_user_unsafe_nozero(*(u8 *)dst, (u8 __user *)src, 1); __uaccess_end(); return ret; case 2: __uaccess_begin_nospec(); - __get_user_asm_nozero(*(u16 *)dst, (u16 __user *)src, - ret, "w", "w", "=r", 2); + ret = __get_user_unsafe_nozero(*(u16 *)dst, (u16 __user *)src, 2); __uaccess_end(); return ret; case 4: __uaccess_begin_nospec(); - __get_user_asm_nozero(*(u32 *)dst, (u32 __user *)src, - ret, "l", "k", "=r", 4); + ret = __get_user_unsafe_nozero(*(u32 *)dst, (u32 __user *)src, 4); __uaccess_end(); return ret; case 8: __uaccess_begin_nospec(); - __get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src, - ret, "q", "", "=r", 8); + ret = __get_user_unsafe_nozero(*(u64 *)dst, (u64 __user *)src, 8); __uaccess_end(); return ret; case 10: __uaccess_begin_nospec(); - __get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src, - ret, "q", "", "=r", 10); + ret = __get_user_unsafe_nozero(*(u64 *)dst, (u64 __user *)src, 10); if (likely(!ret)) - __get_user_asm_nozero(*(u16 *)(8 + (char *)dst), - (u16 __user *)(8 + (char __user *)src), - ret, "w", "w", "=r", 2); + ret = __get_user_unsafe_nozero( + *(u16 *)(8 + (char *)dst), + (u16 __user *)(8 + (char __user *)src), 2); __uaccess_end(); return ret; case 16: __uaccess_begin_nospec(); - __get_user_asm_nozero(*(u64 *)dst, (u64 __user *)src, - ret, "q", "", "=r", 16); + ret = __get_user_unsafe_nozero(*(u64 *)dst, (u64 __user *)src, 16); if (likely(!ret)) - __get_user_asm_nozero(*(u64 *)(8 + (char *)dst), - (u64 __user *)(8 + (char __user *)src), - ret, "q", "", "=r", 8); + ret = __get_user_unsafe_nozero( + *(u64 *)(8 + (char *)dst), + (u64 __user *)(8 + (char __user *)src), 8); __uaccess_end(); return ret; default: @@ -112,48 +106,28 @@ raw_copy_to_user(void __user *dst, const void *src, unsigned long size) return copy_user_generic((__force void *)dst, src, size); switch (size) { case 1: - __uaccess_begin(); - __put_user_asm(*(u8 *)src, (u8 __user *)dst, - ret, "b", "b", "iq", 1); - __uaccess_end(); - return ret; + return __put_user(*(u8 *)src, (u8 __user *)dst, 1); case 2: - __uaccess_begin(); - __put_user_asm(*(u16 *)src, (u16 __user *)dst, - ret, "w", "w", "ir", 2); - __uaccess_end(); - return ret; + return __put_user(*(u16 *)src, (u16 __user *)dst, 2); case 4: - __uaccess_begin(); - __put_user_asm(*(u32 *)src, (u32 __user *)dst, - ret, "l", "k", "ir", 4); - __uaccess_end(); - return ret; + return __put_user(*(u32 *)src, (u32 __user *)dst, 4); case 8: - __uaccess_begin(); - __put_user_asm(*(u64 *)src, (u64 __user *)dst, - ret, "q", "", "er", 8); - __uaccess_end(); - return ret; + return __put_user(*(u64 *)src, (u64 __user *)dst, 8); case 10: __uaccess_begin(); - __put_user_asm(*(u64 *)src, (u64 __user *)dst, - ret, "q", "", "er", 10); + ret = __put_user_unsafe(*(u64 *)src, (u64 __user *)dst, 10); if (likely(!ret)) { asm("":::"memory"); - __put_user_asm(4[(u16 *)src], 4 + (u16 __user *)dst, - ret, "w", "w", "ir", 2); + ret = __put_user_unsafe(4[(u16 *)src], 4 + (u16 __user *)dst, 2); } __uaccess_end(); return ret; case 16: __uaccess_begin(); - __put_user_asm(*(u64 *)src, (u64 __user *)dst, - ret, "q", "", "er", 16); + ret =__put_user_unsafe(*(u64 *)src, (u64 __user *)dst, 16); if (likely(!ret)) { asm("":::"memory"); - __put_user_asm(1[(u64 *)src], 1 + (u64 __user *)dst, - ret, "q", "", "er", 8); + ret = __put_user_unsafe(1[(u64 *)src], 1 + (u64 __user *)dst, 8); } __uaccess_end(); return ret;