Return-Path: Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1753256AbYGGJSS (ORCPT ); Mon, 7 Jul 2008 05:18:18 -0400 Received: (majordomo@vger.kernel.org) by vger.kernel.org id S1750827AbYGGJSE (ORCPT ); Mon, 7 Jul 2008 05:18:04 -0400 Received: from mga09.intel.com ([134.134.136.24]:16549 "EHLO mga09.intel.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1750807AbYGGJSB (ORCPT ); Mon, 7 Jul 2008 05:18:01 -0400 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="4.30,315,1212390000"; d="scan'208,223";a="303594203" Subject: [Fwd: [PATCH]Using Intel CRC32 instruction to implement hardware accelerated CRC32c algorithm.] From: austin zhang To: herbert@gondor.apana.org.au, davem@davemloft.net Cc: chaapala@cisco.com, linux-crypto@vger.kernel.org, linux-kernel@vger.kernel.org, arjan@linux.intel.com Content-Type: text/plain Date: Mon, 07 Jul 2008 05:17:38 -0400 Message-Id: <1215422258.19059.26.camel@localhost.localdomain> Mime-Version: 1.0 X-Mailer: Evolution 2.22.1 (2.22.1-2.fc9) Content-Transfer-Encoding: 7bit Sender: linux-kernel-owner@vger.kernel.org List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Length: 5914 Lines: 178 From NHM processor onward, Intel processors can support hardware accelerated CRC32c algorithm with the new CRC32 instruction in SSE 4.2 instruction set. The patch detects the availability of the feature, and chooses the most proper way to calculate CRC32c checksum. Byte code instructions are used for compiler compatibility. No MMX / XMM registers is involved in the implementation. After basic testing with iSCSI and confirmed that the iSCSI head digest routines can be speeded up by 4x - 10x. This patch is created against mainline 2.6.26-rc8 Signed-off-by: Austin Zhang Signed-off-by: Kent Liu --- include/asm-x86/cpufeature.h | 2 + include/asm-x86/crc32c-hw.h | 63 +++++++++++++++++++++++++++++++++++++++++++ include/asm-x86/processor.h | 3 ++ lib/libcrc32c.c | 20 ++++++++++++- 4 files changed, 86 insertions(+), 2 deletions(-) diff -Naurp linux-2.6-rc8/include/asm-x86/cpufeature.h linux-2.6-rc8-patch/include/asm-x86/cpufeature.h --- linux-2.6-rc8/include/asm-x86/cpufeature.h 2008-05-19 03:33:37.000000000 -0400 +++ linux-2.6-rc8-patch/include/asm-x86/cpufeature.h 2008-07-07 17:49:22.000000000 -0400 @@ -90,6 +90,7 @@ #define X86_FEATURE_CX16 (4*32+13) /* CMPXCHG16B */ #define X86_FEATURE_XTPR (4*32+14) /* Send Task Priority Messages */ #define X86_FEATURE_DCA (4*32+18) /* Direct Cache Access */ +#define X86_FEATURE_XMM4_2 (4*32+20) /* Streaming SIMD Extensions-4.2 */ /* VIA/Cyrix/Centaur-defined CPU features, CPUID level 0xC0000001, word 5 */ #define X86_FEATURE_XSTORE (5*32+ 2) /* on-CPU RNG present (xstore insn) */ @@ -187,6 +188,7 @@ extern const char * const x86_power_flag #define cpu_has_gbpages boot_cpu_has(X86_FEATURE_GBPAGES) #define cpu_has_arch_perfmon boot_cpu_has(X86_FEATURE_ARCH_PERFMON) #define cpu_has_pat boot_cpu_has(X86_FEATURE_PAT) +#define cpu_has_xmm4_2 boot_cpu_has(X86_FEATURE_XMM4_2) #if defined(CONFIG_X86_INVLPG) || defined(CONFIG_X86_64) # define cpu_has_invlpg 1 diff -Naurp linux-2.6-rc8/include/asm-x86/crc32c-hw.h linux-2.6-rc8-patch/include/asm-x86/crc32c-hw.h --- linux-2.6-rc8/include/asm-x86/crc32c-hw.h 1969-12-31 19:00:00.000000000 -0500 +++ linux-2.6-rc8-patch/include/asm-x86/crc32c-hw.h 2008-07-07 18:05:39.000000000 -0400 @@ -0,0 +1,63 @@ +/* + * Using hardware provided CRC32 instruction to accelerate the CRC32 disposal. + * CRC32C polynomial:0x1EDC6F41(BE)/0x82F63B78(LE) + * CRC32 is a new instruction in Intel SSE4.2, the reference can be found at: + * http://www.intel.com/products/processor/manuals/ + * Intel(R) 64 and IA-32 Architectures Software Developer's Manual + * Volume 2A: Instruction Set Reference, A-M + */ +#ifndef __ASM_X86_CRC32C_HW_H +#define __ASM_X86_CRC32C_HW_H + +#include +#include + +#ifdef CONFIG_X86_64 +#define REX_PRE "0x48, " +#define SCALE_F 8 +#else +#define REX_PRE +#define SCALE_F 4 +#endif + +u32 crc32c_le_hw_byte(u32 crc, unsigned char const *data, size_t length) +{ + while (length--) { + __asm__ __volatile__( + ".byte 0xf2, 0xf, 0x38, 0xf0, 0xf1" + :"=S"(crc) + :"0"(crc), "c"(*data) + ); + data++; + } + + return crc; +} + +u32 __pure crc32c_le_hw(u32 crc, unsigned char const *p, size_t len) +{ + unsigned int iquotient = len / SCALE_F; + unsigned int iremainder = len % SCALE_F; +#ifdef CONFIG_X86_64 + u64 *ptmp = (u64 *)p; +#else + u32 *ptmp = (u32 *)p; +#endif + + while (iquotient--) { + __asm__ __volatile__( + ".byte 0xf2, " REX_PRE "0xf, 0x38, 0xf1, 0xf1;" + :"=S"(crc) + :"0"(crc), "c"(*ptmp) + ); + ptmp++; + } + + if (iremainder) + crc = crc32c_le_hw_byte(crc, (unsigned char *)ptmp, iremainder); + + return crc; +} + +#endif /* __ASM_X86_CRC32C_HW_H */ + diff -Naurp linux-2.6-rc8/include/asm-x86/processor.h linux-2.6-rc8-patch/include/asm-x86/processor.h --- linux-2.6-rc8/include/asm-x86/processor.h 2008-05-19 03:33:37.000000000 -0400 +++ linux-2.6-rc8-patch/include/asm-x86/processor.h 2008-07-07 18:02:40.000000000 -0400 @@ -777,6 +777,9 @@ extern char ignore_fpu_irq; # define BASE_PREFETCH "prefetcht0 (%1)" #endif +/* Hardware provides CRC32C accelerated instruction */ +#define ARCH_HAS_CRC32C_HW 1 + /* * Prefetch instructions for Pentium III (+) and AMD Athlon (+) * diff -Naurp linux-2.6-rc8/lib/libcrc32c.c linux-2.6-rc8-patch/lib/libcrc32c.c --- linux-2.6-rc8/lib/libcrc32c.c 2008-05-19 03:33:38.000000000 -0400 +++ linux-2.6-rc8-patch/lib/libcrc32c.c 2008-07-07 18:30:42.000000000 -0400 @@ -34,6 +34,10 @@ #include #include +#ifdef ARCH_HAS_CRC32C_HW +#include +#endif + MODULE_AUTHOR("Clay Haapala "); MODULE_DESCRIPTION("CRC32c (Castagnoli) calculations"); MODULE_LICENSE("GPL"); @@ -66,7 +70,7 @@ EXPORT_SYMBOL(crc32c_le); * of space and maintainability in keeping the two modules separate. */ u32 __pure -crc32c_le(u32 crc, unsigned char const *p, size_t len) +crc32c_le_sw(u32 crc, unsigned char const *p, size_t len) { int i; while (len--) { @@ -160,7 +164,7 @@ static const u32 crc32c_table[256] = { */ u32 __pure -crc32c_le(u32 crc, unsigned char const *data, size_t length) +crc32c_le_sw(u32 crc, unsigned char const *data, size_t length) { while (length--) crc = @@ -171,6 +175,18 @@ crc32c_le(u32 crc, unsigned char const * #endif /* CRC_LE_BITS == 8 */ +u32 __pure +crc32c_le(u32 crc, unsigned char const *p, size_t len) +{ +#ifdef ARCH_HAS_CRC32C_HW + if (cpu_has_xmm4_2) + return crc32c_le_hw(crc, p, len); +#endif + + return crc32c_le_sw(crc, p, len); +} + + EXPORT_SYMBOL(crc32c_be); #if CRC_BE_BITS == 1 -- To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/