Received-SPF: softfail (google.com: domain of transitioning linux-kernel-owner@vger.kernel.org does not designate 23.128.96.19 as permitted sender) client-ip=23.128.96.19;
From:   Ankur Arora <ankur.a.arora@oracle.com>
To:     linux-kernel@vger.kernel.org, linux-mm@kvack.org, x86@kernel.org
Cc:     torvalds@linux-foundation.org, akpm@linux-foundation.org,
        mike.kravetz@oracle.com, mingo@kernel.org, luto@kernel.org,
        tglx@linutronix.de, bp@alien8.de, peterz@infradead.org,
        ak@linux.intel.com, arnd@arndb.de, jgg@nvidia.com,
        jon.grimm@amd.com, boris.ostrovsky@oracle.com,
        konrad.wilk@oracle.com, joao.m.martins@oracle.com,
        ankur.a.arora@oracle.com
Subject: [PATCH v3 07/21] x86/asm: add memset_movnti()
Date:   Mon,  6 Jun 2022 20:37:11 +0000
Message-Id: <20220606203725.1313715-3-ankur.a.arora@oracle.com>
In-Reply-To: <20220606202109.1306034-1-ankur.a.arora@oracle.com>
References: <20220606202109.1306034-1-ankur.a.arora@oracle.com>
Content-Transfer-Encoding: 8bit
Content-Type: text/plain
MIME-Version: 1.0
X-MS-Exchange-AntiSpam-MessageData-ChunkCount: 1
X-MS-Exchange-AntiSpam-MessageData-0: =?us-ascii?Q?Gz2kUE45r+zp3R/qYBjUfVE9yWALT1+phKSOq8XLMHCf5bcwbSwBTrCl8346?=
 =?us-ascii?Q?z5Cop3sALdh8S/sB1Hs2oiDjP4FpnZTPbqA3zKvV8EMhEjTJuXM8GWPd7Wrw?=
 =?us-ascii?Q?KZXI1HDJoFmCK44WmyY+gJ8oqjq71KWbEwXt7PqNO9D1XaZ0Uuo6VtgB88Dw?=
 =?us-ascii?Q?73P/K6+fKBvPG+PQZ3drAukZOGu1DmMCODnHV0kpwfsv1Pjyk+48xTsK7nhu?=
 =?us-ascii?Q?Yp4RlIgEZKkC2AFDJHh0OyGKJQ1KyBeFxBhDZdbpiQ4LVP0PP3qADgu9NhOS?=
 =?us-ascii?Q?6UURIiDYANVQKm07H+qGV6om/nbND+ZVybTfDdG+W42GaFSOXLLUHsJS2Rdx?=
 =?us-ascii?Q?7PoOrlMV9mJs44qIABf7igiBPmj+XIdRFa4NSbbSFxRKowBG1/zj5RMLiAh9?=
 =?us-ascii?Q?qQSq22ZT8ycDvjwUzUNmWDpp9tFbcixkNERsNXbdIirBk+QjaVuBN91Q2EPM?=
 =?us-ascii?Q?MqjcPhNbQp7n24J24hsk2Zb+/CnyTbsPdK23NfLtcV+X54uXfKOpyxa9jZ/j?=
 =?us-ascii?Q?PxLUZT8E97bSs2soDXBBCEgdnefxLU/fZ2ZDtB027FckCnofHQXLnnJ5V9a+?=
 =?us-ascii?Q?Z0FgsQAdfQP5iIXjZVdfvWH5YU0+c5NqcHDTLG+1opfkaWliGRPIwF3G45QX?=
 =?us-ascii?Q?hP5WsllxGGVlzTzGWzhxqa4sEVEkzTmy8E1O5V2xXAkjwHZT/5yBEWgdetSl?=
 =?us-ascii?Q?KvjgMKBXmp2806p7CJh1kFOnGdoQSs34/E0n0Ir7tC7xEW/0QyoAE/1/Oe4w?=
 =?us-ascii?Q?pPjD5D2CtgxAsPWFZFtoENlMh7ulk/XNkFe26nWIbslMbABo/hY56h0rNlr1?=
 =?us-ascii?Q?etjkCKfWYSKFcT4wAYLHdHmBzhqn90y5vkMo/5bpq6dTBRovvnm/jxh9R2C+?=
 =?us-ascii?Q?cQuE90fkrTbf3Ju21z597bHsAagiLfHTvVH19Ye6kGMHsHOhsZctoULP5kMy?=
 =?us-ascii?Q?ymX9FMICHXZj2ed9LaOGuYjVTO8dnMgw01tF+wzB339Dp9o6Dn8+kuORvDWA?=
 =?us-ascii?Q?fS9s8B0xGII6o3Ba+rKyM6paaTxeytJa6EzvnaBGLVZHYYVn9sDd24L9q1U0?=
 =?us-ascii?Q?413lZnUPIdShA1pi5W2UuBB7j4N5ZENb8B7UInKk4Gxl17z6iN3SsZ3s1Y9Z?=
 =?us-ascii?Q?7LwwiriAvpeOG4ADHKicKCisrQWZyuoeMRzyhmjRlWG9/0ulBiiGMeGQwR2x?=
 =?us-ascii?Q?xDTvV3z9munjMkqHYzd+ravgPmyVQT0wwomx2K1+rFTE0jBCAzZh0ePg/sE+?=
 =?us-ascii?Q?tNTBUoBmKrFUt/WXFnBI5PawXmCEcWXHymomEAYeYVUSXkDVCAphCTxcxv/f?=
 =?us-ascii?Q?nYuV03rHOTL6V0Wf8h2KzaP3qTXbKJJ4ZHyuKt2yy+r4lqWNduuX36naFWXW?=
 =?us-ascii?Q?rlgK5qplGWuvl4eu/vtssqhmu081WUWJegkDWpU2N5vXpp5QR3hGAuhH7m6I?=
 =?us-ascii?Q?PAgJnhpn1t4j2P/zhIRSeDtsjwHeZylPifMWGmJbK/1VLbtWsHrXXsvolkns?=
 =?us-ascii?Q?yEh0RXrDTqNj8JQwv7850qlL8xRIGzyiyephnak2IQkGLSVS9d+hzT5d2/Sz?=
 =?us-ascii?Q?R4+UKhWJ16Z9LcozJSZ6DDBIYVk1oAK58TBBopmXtrmhJl9FEgD5itRfcJ72?=
 =?us-ascii?Q?gLKKh1XRLcGWLcMOSwtPUyUnM3BCeZQVWwRQW2PiDAcJNnis8mKMHAjmAcAA?=
 =?us-ascii?Q?oukMnulvyx7vq425u6J0DicDgR1IcDhVyyfhd+TlqCzYzn6ZpTHIYbqqdxeN?=
 =?us-ascii?Q?NetOszvxSJ5DHWppb/K93AtXQW/jDlQ=3D?=
X-OriginatorOrg: oracle.com
X-MS-Exchange-CrossTenant-Network-Message-Id: 49827a37-a8eb-4a11-7508-08da47fc6895
X-MS-Exchange-CrossTenant-AuthSource: DM8PR10MB5416.namprd10.prod.outlook.com
X-MS-Exchange-CrossTenant-AuthAs: Internal
X-MS-Exchange-CrossTenant-OriginalArrivalTime: 06 Jun 2022 20:37:43.9864
 (UTC)
X-MS-Exchange-CrossTenant-FromEntityHeader: Hosted
X-MS-Exchange-CrossTenant-Id: 4e2c6054-71cb-48f1-bd6c-3a9705aca71b
X-MS-Exchange-CrossTenant-MailboxType: HOSTED
X-MS-Exchange-CrossTenant-UserPrincipalName: 0uA6Z3qwHbRCum6xEv2e8SqJ3t7vgIBuCCheXqGf38C40c0QPsluwm3FJLD8T9ZvW0r5PyMJBdUONPt+oZPH7oeSJbr/0MC5xPEYG+7z5rc=
X-MS-Exchange-Transport-CrossTenantHeadersStamped: BN7PR10MB2596
X-Proofpoint-Virus-Version: vendor=fsecure engine=2.50.10434:6.0.517,18.0.874
 definitions=2022-06-06_06:2022-06-02,2022-06-06 signatures=0
X-Proofpoint-Spam-Details: rule=notspam policy=default score=0 malwarescore=0 bulkscore=0 spamscore=0
 phishscore=0 mlxscore=0 adultscore=0 suspectscore=0 mlxlogscore=999
 classifier=spam adjust=0 reason=mlx scancount=1 engine=8.12.0-2204290000
 definitions=main-2206060081
X-Proofpoint-ORIG-GUID: cX51BN8OkZp2Ovz9-xFCzkdwZQLc8vYE
X-Proofpoint-GUID: cX51BN8OkZp2Ovz9-xFCzkdwZQLc8vYE
X-Spam-Status: No, score=-2.0 required=5.0 tests=BAYES_00,DKIM_SIGNED,
	DKIM_VALID,DKIM_VALID_AU,HEADER_FROM_DIFFERENT_DOMAINS,
	MAILING_LIST_MULTI,RDNS_NONE,SPF_HELO_NONE,T_SCC_BODY_TEXT_LINE
	autolearn=no autolearn_force=no version=3.4.6
X-Spam-Checker-Version: SpamAssassin 3.4.6 (2021-04-09) on
	lindbergh.monkeyblade.net
Precedence: bulk
List-ID: <linux-kernel.vger.kernel.org>
X-Mailing-List: linux-kernel@vger.kernel.org

Add a MOVNTI based non-caching implementation of memset().

memset_movnti() only needs to differ from memset_orig() in the opcode
used in the inner loop, so move the memset_orig() logic into a macro,
and use that to generate memset_orig() (now memset_movq()) and
memset_movnti().

Signed-off-by: Ankur Arora <ankur.a.arora@oracle.com>
---
 arch/x86/lib/memset_64.S | 68 ++++++++++++++++++++++------------------
 1 file changed, 38 insertions(+), 30 deletions(-)

diff --git a/arch/x86/lib/memset_64.S b/arch/x86/lib/memset_64.S
index fc9ffd3ff3b2..307b753ca03a 100644
--- a/arch/x86/lib/memset_64.S
+++ b/arch/x86/lib/memset_64.S
@@ -24,7 +24,7 @@ SYM_FUNC_START(__memset)
 	 *
 	 * Otherwise, use original memset function.
 	 */
-	ALTERNATIVE_2 "jmp memset_orig", "", X86_FEATURE_REP_GOOD, \
+	ALTERNATIVE_2 "jmp memset_movq", "", X86_FEATURE_REP_GOOD, \
 		      "jmp memset_erms", X86_FEATURE_ERMS
 
 	movq %rdi,%r9
@@ -66,7 +66,8 @@ SYM_FUNC_START_LOCAL(memset_erms)
 	RET
 SYM_FUNC_END(memset_erms)
 
-SYM_FUNC_START_LOCAL(memset_orig)
+.macro MEMSET_MOV OP fence
+SYM_FUNC_START_LOCAL(memset_\OP)
 	movq %rdi,%r10
 
 	/* expand byte value  */
@@ -77,64 +78,71 @@ SYM_FUNC_START_LOCAL(memset_orig)
 	/* align dst */
 	movl  %edi,%r9d
 	andl  $7,%r9d
-	jnz  .Lbad_alignment
-.Lafter_bad_alignment:
+	jnz  .Lbad_alignment_\@
+.Lafter_bad_alignment_\@:
 
 	movq  %rdx,%rcx
 	shrq  $6,%rcx
-	jz	 .Lhandle_tail
+	jz	 .Lhandle_tail_\@
 
 	.p2align 4
-.Lloop_64:
+.Lloop_64_\@:
 	decq  %rcx
-	movq  %rax,(%rdi)
-	movq  %rax,8(%rdi)
-	movq  %rax,16(%rdi)
-	movq  %rax,24(%rdi)
-	movq  %rax,32(%rdi)
-	movq  %rax,40(%rdi)
-	movq  %rax,48(%rdi)
-	movq  %rax,56(%rdi)
+	\OP  %rax,(%rdi)
+	\OP  %rax,8(%rdi)
+	\OP  %rax,16(%rdi)
+	\OP  %rax,24(%rdi)
+	\OP  %rax,32(%rdi)
+	\OP  %rax,40(%rdi)
+	\OP  %rax,48(%rdi)
+	\OP  %rax,56(%rdi)
 	leaq  64(%rdi),%rdi
-	jnz    .Lloop_64
+	jnz    .Lloop_64_\@
 
 	/* Handle tail in loops. The loops should be faster than hard
 	   to predict jump tables. */
 	.p2align 4
-.Lhandle_tail:
+.Lhandle_tail_\@:
 	movl	%edx,%ecx
 	andl    $63&(~7),%ecx
-	jz 		.Lhandle_7
+	jz 	.Lhandle_7_\@
 	shrl	$3,%ecx
 	.p2align 4
-.Lloop_8:
+.Lloop_8_\@:
 	decl   %ecx
-	movq  %rax,(%rdi)
+	\OP  %rax,(%rdi)
 	leaq  8(%rdi),%rdi
-	jnz    .Lloop_8
+	jnz    .Lloop_8_\@
 
-.Lhandle_7:
+.Lhandle_7_\@:
 	andl	$7,%edx
-	jz      .Lende
+	jz      .Lende_\@
 	.p2align 4
-.Lloop_1:
+.Lloop_1_\@:
 	decl    %edx
 	movb 	%al,(%rdi)
 	leaq	1(%rdi),%rdi
-	jnz     .Lloop_1
+	jnz     .Lloop_1_\@
 
-.Lende:
+.Lende_\@:
+	.if \fence
+	sfence
+	.endif
 	movq	%r10,%rax
 	RET
 
-.Lbad_alignment:
+.Lbad_alignment_\@:
 	cmpq $7,%rdx
-	jbe	.Lhandle_7
+	jbe	.Lhandle_7_\@
 	movq %rax,(%rdi)	/* unaligned store */
 	movq $8,%r8
 	subq %r9,%r8
 	addq %r8,%rdi
 	subq %r8,%rdx
-	jmp .Lafter_bad_alignment
-.Lfinal:
-SYM_FUNC_END(memset_orig)
+	jmp .Lafter_bad_alignment_\@
+.Lfinal_\@:
+SYM_FUNC_END(memset_\OP)
+.endm
+
+MEMSET_MOV OP=movq fence=0
+MEMSET_MOV OP=movnti fence=1
-- 
2.31.1