Add hardware accelerated version of POLYVAL for x86-64 CPUs with
PCLMULQDQ support.
This implementation is accelerated using PCLMULQDQ instructions to
perform the finite field computations. For added efficiency, 8 blocks
of the message are processed simultaneously by precomputing the first
8 powers of the key.
Schoolbook multiplication is used instead of Karatsuba multiplication
because it was found to be slightly faster on x86-64 machines.
Montgomery reduction must be used instead of Barrett reduction due to
the difference in modulus between POLYVAL's field and other finite
fields.
More information on POLYVAL can be found in the HCTR2 paper:
Length-preserving encryption with HCTR2:
https://eprint.iacr.org/2021/1441.pdf
Signed-off-by: Nathan Huckleberry <[email protected]>
---
Changes since v1:
* Rename C, D, EF to LO, MI, HI
* Add comments explaining POLYVAL
* Fix bug in handling of non-block-multiple messages
* Wrap shash in ahash (as done in ghash)
arch/x86/crypto/Makefile | 3 +
arch/x86/crypto/polyval-clmulni_asm.S | 414 +++++++++++++++++++++++++
arch/x86/crypto/polyval-clmulni_glue.c | 365 ++++++++++++++++++++++
crypto/Kconfig | 10 +
4 files changed, 792 insertions(+)
create mode 100644 arch/x86/crypto/polyval-clmulni_asm.S
create mode 100644 arch/x86/crypto/polyval-clmulni_glue.c
diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile
index ee2df489b0d9..c0f13801afba 100644
--- a/arch/x86/crypto/Makefile
+++ b/arch/x86/crypto/Makefile
@@ -69,6 +69,9 @@ libblake2s-x86_64-y := blake2s-core.o blake2s-glue.o
obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o
ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o
+obj-$(CONFIG_CRYPTO_POLYVAL_CLMUL_NI) += polyval-clmulni.o
+polyval-clmulni-y := polyval-clmulni_asm.o polyval-clmulni_glue.o
+
obj-$(CONFIG_CRYPTO_CRC32C_INTEL) += crc32c-intel.o
crc32c-intel-y := crc32c-intel_glue.o
crc32c-intel-$(CONFIG_64BIT) += crc32c-pcl-intel-asm_64.o
diff --git a/arch/x86/crypto/polyval-clmulni_asm.S b/arch/x86/crypto/polyval-clmulni_asm.S
new file mode 100644
index 000000000000..bec1a2046b18
--- /dev/null
+++ b/arch/x86/crypto/polyval-clmulni_asm.S
@@ -0,0 +1,414 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright 2021 Google LLC
+ *
+ * Use of this source code is governed by an MIT-style
+ * license that can be found in the LICENSE file or at
+ * https://opensource.org/licenses/MIT.
+ */
+/*
+ * This is an efficient implementation of POLYVAL using intel PCLMULQDQ-NI
+ * instructions. It works on 8 blocks at a time, by precomputing the first 8
+ * keys powers h^8, ..., h^1 in the POLYVAL finite field. This precomputation
+ * allows us to split finite field multiplication into two steps.
+ *
+ * In the first step, we consider h^i, m_i as normal polynomials of degree less
+ * than 128. We then compute p(x) = h^8m_0 + ... + h^1m_7 where multiplication
+ * is simply polynomial multiplication.
+ *
+ * In the second step, we compute the reduction of p(x) modulo the finite field
+ * modulus g(x) = x^128 + x^127 + x^126 + x^121 + 1.
+ *
+ * This two step process is equivalent to computing h^8m_0 + ... + h^1m_7 where
+ * multiplication is finite field multiplication. The advantage is that the
+ * two-step process only requires 1 finite field reduction for every 8
+ * polynomial multiplications. Further parallelism is gained by interleaving the
+ * multiplications and polynomial reductions.
+ */
+
+#include <linux/linkage.h>
+#include <asm/frame.h>
+
+#define NUM_PRECOMPUTE_POWERS 8
+
+#define GSTAR %xmm7
+#define PL %xmm8
+#define PH %xmm9
+#define T %xmm10
+#define V %xmm11
+#define LO %xmm12
+#define HI %xmm13
+#define MI %xmm14
+#define SUM %xmm15
+
+#define BLOCKS_LEFT %rdx
+#define OP1 %rdi
+#define OP2 %r10
+#define IDX %r11
+#define TMP %rax
+
+.section .rodata.cst16.gstar, "aM", @progbits, 16
+.align 16
+
+Lgstar:
+ .quad 0xc200000000000000, 0xc200000000000000
+
+.text
+
+/*
+ * Performs schoolbook1_iteration on two lists of 128-bit polynomials of length
+ * b pointed to by OP1 and OP2.
+ */
+.macro schoolbook1 b
+ .set by, \b
+ .set i, 0
+ .rept (by)
+ schoolbook1_iteration i 0
+ .set i, (i +1)
+ .endr
+.endm
+
+/*
+ * Computes the product of two 128-bit polynomials at the memory locations
+ * specified by (OP1 + 16*i) and (OP2 + 16*i) and XORs the components of the
+ * 256-bit product into LO, MI, HI.
+ *
+ * The multiplication produces four parts:
+ * LOW: The polynomial given by performing carryless multiplication of the
+ * bottom 64-bits of each polynomial
+ * MID1: The polynomial given by performing carryless multiplication of the
+ * bottom 64-bits of the first polynomial and the top 64-bits of the second
+ * MID2: The polynomial given by performing carryless multiplication of the
+ * bottom 64-bits of the second polynomial and the top 64-bits of the first
+ * HIGH: The polynomial given by performing carryless multiplication of the
+ * top 64-bits of each polynomial
+ *
+ * We compute:
+ * LO ^= LOW
+ * Mi ^= MID1 ^ MID2
+ * Hi ^= HIGH
+ *
+ * Later, the 256-bit result can be extracted as:
+ * [HI_H : HI_L ^ MI_H : LO_H ^ MI_L : LO_L]
+ * This step is done when computing the polynomial reduction for efficiency
+ * reasons.
+ *
+ * If xor_sum == 1 then XOR the value of SUM into m_0.
+ * This avoids an extra multication of SUM and h^N.
+ */
+.macro schoolbook1_iteration i xor_sum
+ .set i, \i
+ .set xor_sum, \xor_sum
+ movups (16*i)(OP1), %xmm0
+ .if(i == 0 && xor_sum == 1)
+ pxor SUM, %xmm0
+ .endif
+ vpclmulqdq $0x01, (16*i)(OP2), %xmm0, %xmm1
+ vpxor %xmm1, MI, MI
+ vpclmulqdq $0x00, (16*i)(OP2), %xmm0, %xmm2
+ vpxor %xmm2, LO, LO
+ vpclmulqdq $0x11, (16*i)(OP2), %xmm0, %xmm3
+ vpxor %xmm3, HI, HI
+ vpclmulqdq $0x10, (16*i)(OP2), %xmm0, %xmm4
+ vpxor %xmm4, MI, MI
+.endm
+
+/*
+ * Performs the same computation as schoolbook1_iteration, except we expect the
+ * arguments to already be loaded into xmm0 and xmm1.
+ */
+.macro schoolbook1_noload
+ vpclmulqdq $0x01, %xmm0, %xmm1, %xmm2
+ vpxor %xmm2, MI, MI
+ vpclmulqdq $0x00, %xmm0, %xmm1, %xmm3
+ vpxor %xmm3, LO, LO
+ vpclmulqdq $0x11, %xmm0, %xmm1, %xmm4
+ vpxor %xmm4, HI, HI
+ vpclmulqdq $0x10, %xmm0, %xmm1, %xmm5
+ vpxor %xmm5, MI, MI
+.endm
+
+/*
+ * Computes the 256-bit polynomial represented by LO, HI, MI. Stores
+ * the result in PL, PH.
+ * [PH :: PL] = [HI_H : HI_L ^ MI_H :: LO_H ^ MI_L : LO_L]
+ */
+.macro schoolbook2
+ vpslldq $8, MI, PL
+ vpsrldq $8, MI, PH
+ pxor LO, PL
+ pxor HI, PH
+.endm
+
+/*
+ * Computes the 128-bit reduction of PL, PH. Stores the result in PH.
+ *
+ * This macro computes p(x) mod g(x) where p(x) is in montgomery form and g(x) =
+ * x^128 + x^127 + x^126 + x^121 + 1.
+ *
+ * The montgomery form of a polynomial p(x) is p(x)x^{128}. Montgomery reduction
+ * works by simultaneously dividing by x^{128} and computing the modular
+ * reduction.
+ *
+ * Suppose we wish to reduce the montgomery form of p(x) = [P_3 : P_2 : P_1 :
+ * P_0] where P_i is a polynomial of degree at most 64 represented as 64-bits.
+ * Thus we would like to compute:
+ * p(x) / x^{128} mod g(x)
+ * = (P_3*x^{192} + P_2*x^{128} + P_1*x^{64} + P_0) / x^{128} mod g(x)
+ *
+ * We would like to divide by x^{128} efficiently. Since P_3*x^{128},
+ * P_2*x^{128} are multiples of x^{128}, we can simply bitshift right by 128.
+ *
+ * We now focus on dividing P_1*x^{64} + P_0 by x^{128}. We do this by making
+ * P_1*x^{64} + P_0 divisble by x^{128} then bitshifting. To add divisibility,
+ * we consider the polynomials mod x^{128}.
+ *
+ * Let c(x) = P_1*x^{64} + P_0.
+ *
+ * Now let m(x) = c(x) mod x^{128}
+ * and
+ * Let z(x) = [c(x) + m(x)g(x)] / x^{128}
+ *
+ * First notice that:
+ * c(x) + m(x)g(x) = c(x) mod g(x).
+ * Furthermore, g(x) mod x^{128} = 1, so we have
+ * c(x) + m(x)g(x) = c(x) + c(x) = 0 (mod x^{128}).
+ *
+ * Thus c(x) + m(x)g(x) is divisible by x^{128} and is equivalent to c(x) mod
+ * g(x).
+ *
+ * In practice we use a slight modification of this idea, by using g*(x) =
+ * x^{63} + x^{62} + x^{57}. This is because we can only multiply 64-bit
+ * polynomials. Notice that g(x) = x^128 + g*(x)*x^{64} + 1
+ *
+ * We do this by substituting g(x) = x^{128} + g*(x)x^{64} + 1
+ * z(x) = [c(x) + c(x)*(x^{128} + g*(x)*x^{64} + 1)] / x^{128}
+ * = [P_1*x^{192} + P_0*x^{128} + P_1*g*(x)x^{128} + P_0*g*(x)*x^{64} z(x)]
+ * / x^{128}
+ * = P_1*x^{64} + P_0 + P_1*g*(x) + P_0*g*(x)*x^{-64}
+ *
+ * The only difficulty left in this expression is P_0*g*(x)x^{-64}.
+ * Let t(x) = P_0*g*(x) = [T_1 : T_0]
+ * Notice that we can repeat the above process:
+ * g(x) mod x^{64} = 1
+ * m'(x) = t(x) mod x^{64}
+ * z'(x) = [t(x) + m'(x)g(x)] / x^64
+ * Thus we get
+ * z'(x) = [t(x) + (x^{128} + g*(x)x^{64} + 1)T_0] / x^64
+ * = T_1 + T_0*x^{64} + g*(x)*T_0
+ *
+ * Recall that this is only the reduction for [P_1*x^{64} + P_0] / x^{64}. The
+ * full computation we need to make is:
+ * p(x) / x^{128} = P_3*x^{64} + P_2 + P_1*x^{64} + P_0 + P_1*g*(x) +
+ * T_1 + T_0*x^{64} + g*(x)*T_0
+ *
+ * Thus we have:
+ * t(x) = g*(x) * P_0 = [T_1 : T_0]
+ * v(x) = g*(x) * (T_0 ^ P_1) = [V_1 : V_0]
+ * p(x) / x^{128} mod g(x) = [P_3 ^ P_1 ^ V_1 ^ T_0 : P_2 ^ P_0 ^ V_0 ^ T_1]
+ */
+.macro montgomery_reduction
+ movdqa PL, T
+ pclmulqdq $0x00, GSTAR, T # T = [P_0 * g*(x)]
+ pshufd $0b01001110, T, V # V = [T_0 : T_1]
+ pxor V, PL # PL = [P_1 ^ T_0 : P_0 ^ T_1]
+ pxor PL, PH # PH = [P_1 ^ T_0 ^ P_3 : P_0 ^ T_1 ^ P_2]
+ pclmulqdq $0x11, GSTAR, PL # PL = [(P_1 ^ T_0) * g*(x)]
+ pxor PL, PH
+.endm
+
+/*
+ * Compute schoolbook multiplication for 8 blocks
+ * M_0h^8 + ... + M_7h^1 (no constant term)
+ *
+ * If reduce is set, computes the montgomery reduction of the
+ * previous full_stride call and XORs with the first message block.
+ * (M_0 + REDUCE(PL, PH))h^8 + ... + M_7h^1 (no constant term)
+ *
+ * Sets PL, PH
+ * Clobbers LO, HI, MI
+ *
+ */
+.macro full_stride reduce
+ .set reduce, \reduce
+ mov %rsi, OP2
+ pxor LO, LO
+ pxor HI, HI
+ pxor MI, MI
+
+ schoolbook1_iteration 7 0
+ .if(reduce)
+ movdqa PL, T
+ .endif
+
+ schoolbook1_iteration 6 0
+ .if(reduce)
+ pclmulqdq $0x00, GSTAR, T # T = [X0 * g*(x)]
+ .endif
+
+ schoolbook1_iteration 5 0
+ .if(reduce)
+ pshufd $0b01001110, T, V # V = [T0 : T1]
+ .endif
+
+ schoolbook1_iteration 4 0
+ .if(reduce)
+ pxor V, PL # PL = [X1 ^ T0 : X0 ^ T1]
+ .endif
+
+ schoolbook1_iteration 3 0
+ .if(reduce)
+ pxor PL, PH # PH = [X1 ^ T0 ^ X3 : X0 ^ T1 ^ X2]
+ .endif
+
+ schoolbook1_iteration 2 0
+ .if(reduce)
+ pclmulqdq $0x11, GSTAR, PL # PL = [X1 ^ T0 * g*(x)]
+ .endif
+
+ schoolbook1_iteration 1 0
+ .if(reduce)
+ pxor PL, PH
+ movdqa PH, SUM
+ .endif
+
+ schoolbook1_iteration 0 1
+
+ addq $(8*16), OP1
+ addq $(8*16), OP2
+ schoolbook2
+.endm
+
+/*
+ * Compute poly on window size of %rdx blocks
+ * 0 < %rdx < NUM_PRECOMPUTE_POWERS
+ */
+.macro partial_stride
+ pxor LO, LO
+ pxor HI, HI
+ pxor MI, MI
+ mov BLOCKS_LEFT, TMP
+ shlq $4, TMP
+ mov %rsi, OP2
+ addq $(16*NUM_PRECOMPUTE_POWERS), OP2
+ subq TMP, OP2
+ # Multiply sum by h^N
+ movups (OP2), %xmm0
+ movdqa SUM, %xmm1
+ schoolbook1_noload
+ schoolbook2
+ montgomery_reduction
+ movdqa PH, SUM
+ pxor LO, LO
+ pxor HI, HI
+ pxor MI, MI
+ xor IDX, IDX
+.LloopPartial:
+ cmpq BLOCKS_LEFT, IDX # IDX < rdx
+ jae .LloopExitPartial
+
+ movq BLOCKS_LEFT, TMP
+ subq IDX, TMP # TMP = rdx - IDX
+
+ cmp $4, TMP # TMP < 4 ?
+ jl .Llt4Partial
+ schoolbook1 4
+ addq $4, IDX
+ addq $(4*16), OP1
+ addq $(4*16), OP2
+ jmp .LoutPartial
+.Llt4Partial:
+ cmp $3, TMP # TMP < 3 ?
+ jl .Llt3Partial
+ schoolbook1 3
+ addq $3, IDX
+ addq $(3*16), OP1
+ addq $(3*16), OP2
+ jmp .LoutPartial
+.Llt3Partial:
+ cmp $2, TMP # TMP < 2 ?
+ jl .Llt2Partial
+ schoolbook1 2
+ addq $2, IDX
+ addq $(2*16), OP1
+ addq $(2*16), OP2
+ jmp .LoutPartial
+.Llt2Partial:
+ schoolbook1 1 # TMP < 1 ?
+ addq $1, IDX
+ addq $(1*16), OP1
+ addq $(1*16), OP2
+.LoutPartial:
+ jmp .LloopPartial
+.LloopExitPartial:
+ schoolbook2
+ montgomery_reduction
+ pxor PH, SUM
+.endm
+
+/*
+ * Perform montgomery multiplication in GF(2^128) and store result in op1.
+ *
+ * Computes op1*op2*x^{-128} mod x^128 + x^127 + x^126 + x^121 + 1
+ * If op1, op2 are in montgomery form, this computes the montgomery
+ * form of op1*op2.
+ *
+ * void clmul_polyval_mul(u8 *op1, const u8 *op2);
+ */
+SYM_FUNC_START(clmul_polyval_mul)
+ FRAME_BEGIN
+ vmovdqa Lgstar(%rip), GSTAR
+ pxor LO, LO
+ pxor HI, HI
+ pxor MI, MI
+ movups (%rdi), %xmm0
+ movups (%rsi), %xmm1
+ schoolbook1_noload
+ schoolbook2
+ montgomery_reduction
+ movups PH, (%rdi)
+ FRAME_END
+ ret
+SYM_FUNC_END(clmul_polyval_mul)
+
+/*
+ * Perform polynomial evaluation as specified by POLYVAL. If nblocks = k, this
+ * routine multiplies the value stored at accumulator by h^k and XORs the
+ * evaluated polynomial into it.
+ *
+ * Computes h^k * accumulator + h^kM_0 + ... + h^1M_{k-1} (No constant term)
+ *
+ * rdi (OP1) - pointer to message blocks
+ * rsi - pointer to precomputed key struct
+ * rdx - number of blocks to hash
+ * rcx - location to XOR with evaluated polynomial
+ *
+ * void clmul_polyval_update(const u8 *in, const struct polyhash_ctx* ctx,
+ * size_t nblocks, u8* accumulator);
+ */
+SYM_FUNC_START(clmul_polyval_update)
+ FRAME_BEGIN
+ vmovdqa Lgstar(%rip), GSTAR
+ movups (%rcx), SUM
+ cmpq $NUM_PRECOMPUTE_POWERS, BLOCKS_LEFT
+ jb .LstrideLoopExit
+ full_stride 0
+ subq $NUM_PRECOMPUTE_POWERS, BLOCKS_LEFT
+.LstrideLoop:
+ cmpq $NUM_PRECOMPUTE_POWERS, BLOCKS_LEFT
+ jb .LstrideLoopExitReduce
+ full_stride 1
+ subq $NUM_PRECOMPUTE_POWERS, BLOCKS_LEFT
+ jmp .LstrideLoop
+.LstrideLoopExitReduce:
+ montgomery_reduction
+ movdqa PH, SUM
+.LstrideLoopExit:
+ test BLOCKS_LEFT, BLOCKS_LEFT
+ je .LskipPartial
+ partial_stride
+.LskipPartial:
+ movups SUM, (%rcx)
+ FRAME_END
+ ret
+SYM_FUNC_END(clmul_polyval_update)
diff --git a/arch/x86/crypto/polyval-clmulni_glue.c b/arch/x86/crypto/polyval-clmulni_glue.c
new file mode 100644
index 000000000000..78cbb19658ac
--- /dev/null
+++ b/arch/x86/crypto/polyval-clmulni_glue.c
@@ -0,0 +1,365 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Accelerated POLYVAL implementation with Intel PCLMULQDQ-NI
+ * instructions. This file contains glue code.
+ *
+ * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <[email protected]>
+ * Copyright (c) 2009 Intel Corp.
+ * Author: Huang Ying <[email protected]>
+ * Copyright 2021 Google LLC
+ */
+/*
+ * Glue code based on ghash-clmulni-intel_glue.c.
+ *
+ * This implementation of POLYVAL uses montgomery multiplication
+ * accelerated by PCLMULQDQ-NI to implement the finite field
+ * operations.
+ *
+ */
+
+#include <crypto/algapi.h>
+#include <crypto/cryptd.h>
+#include <crypto/gf128mul.h>
+#include <crypto/internal/hash.h>
+#include <crypto/internal/simd.h>
+#include <linux/crypto.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <asm/cpu_device_id.h>
+#include <asm/simd.h>
+
+#define POLYVAL_BLOCK_SIZE 16
+#define POLYVAL_DIGEST_SIZE 16
+#define NUM_PRECOMPUTE_POWERS 8
+
+struct polyval_async_ctx {
+ struct cryptd_ahash *cryptd_tfm;
+};
+
+struct polyval_ctx {
+ /*
+ * These powers must be in the order h^8, ..., h^1.
+ */
+ u128 key_powers[NUM_PRECOMPUTE_POWERS];
+};
+
+struct polyval_desc_ctx {
+ u8 buffer[POLYVAL_BLOCK_SIZE];
+ u32 bytes;
+};
+
+asmlinkage void clmul_polyval_update(const u8 *in, struct polyval_ctx *keys,
+ size_t nblocks, u8 *accumulator);
+asmlinkage void clmul_polyval_mul(u8 *op1, const u8 *op2);
+
+static int polyval_init(struct shash_desc *desc)
+{
+ struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
+
+ memset(dctx, 0, sizeof(*dctx));
+
+ return 0;
+}
+
+static int polyval_setkey(struct crypto_shash *tfm,
+ const u8 *key, unsigned int keylen)
+{
+ struct polyval_ctx *ctx = crypto_shash_ctx(tfm);
+ int i;
+
+ if (keylen != POLYVAL_BLOCK_SIZE)
+ return -EINVAL;
+
+ BUILD_BUG_ON(sizeof(u128) != POLYVAL_BLOCK_SIZE);
+
+ memcpy(&ctx->key_powers[NUM_PRECOMPUTE_POWERS-1], key, sizeof(u128));
+
+ kernel_fpu_begin();
+ for (i = NUM_PRECOMPUTE_POWERS-2; i >= 0; i--) {
+ memcpy(&ctx->key_powers[i], key, sizeof(u128));
+ clmul_polyval_mul((u8 *)&ctx->key_powers[i],
+ (u8 *)&ctx->key_powers[i+1]);
+ }
+ kernel_fpu_end();
+
+ return 0;
+}
+
+static int polyval_update(struct shash_desc *desc,
+ const u8 *src, unsigned int srclen)
+{
+ struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
+ struct polyval_ctx *ctx = crypto_shash_ctx(desc->tfm);
+ u8 *dst = dctx->buffer;
+ u8 *pos;
+ unsigned int nblocks;
+ int n;
+
+ kernel_fpu_begin();
+ if (dctx->bytes) {
+ n = min(srclen, dctx->bytes);
+ pos = dst + POLYVAL_BLOCK_SIZE - dctx->bytes;
+
+ dctx->bytes -= n;
+ srclen -= n;
+
+ while (n--)
+ *pos++ ^= *src++;
+
+ if (!dctx->bytes)
+ clmul_polyval_mul(dst,
+ (u8 *)&ctx->key_powers[NUM_PRECOMPUTE_POWERS-1]);
+ }
+
+ nblocks = srclen/POLYVAL_BLOCK_SIZE;
+ clmul_polyval_update(src, ctx, nblocks, dst);
+ srclen -= nblocks*POLYVAL_BLOCK_SIZE;
+ kernel_fpu_end();
+
+ if (srclen) {
+ dctx->bytes = POLYVAL_BLOCK_SIZE - srclen;
+ src += nblocks*POLYVAL_BLOCK_SIZE;
+ pos = dst;
+ while (srclen--)
+ *pos++ ^= *src++;
+ }
+
+ return 0;
+}
+
+static int polyval_final(struct shash_desc *desc, u8 *dst)
+{
+ struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
+ struct polyval_ctx *ctx = crypto_shash_ctx(desc->tfm);
+ u8 *buf = dctx->buffer;
+
+ if (dctx->bytes) {
+ kernel_fpu_begin();
+ clmul_polyval_mul((u8 *)buf,
+ (u8 *)&ctx->key_powers[NUM_PRECOMPUTE_POWERS-1]);
+ kernel_fpu_end();
+ }
+
+ dctx->bytes = 0;
+ memcpy(dst, buf, POLYVAL_BLOCK_SIZE);
+
+ return 0;
+}
+
+static struct shash_alg polyval_alg = {
+ .digestsize = POLYVAL_DIGEST_SIZE,
+ .init = polyval_init,
+ .update = polyval_update,
+ .final = polyval_final,
+ .setkey = polyval_setkey,
+ .descsize = sizeof(struct polyval_desc_ctx),
+ .base = {
+ .cra_name = "__polyval",
+ .cra_driver_name = "__polyval-pclmulqdqni",
+ .cra_priority = 0,
+ .cra_flags = CRYPTO_ALG_INTERNAL,
+ .cra_blocksize = POLYVAL_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct polyval_ctx),
+ .cra_module = THIS_MODULE,
+ },
+};
+
+static int polyval_async_init(struct ahash_request *req)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct polyval_async_ctx *ctx = crypto_ahash_ctx(tfm);
+ struct ahash_request *cryptd_req = ahash_request_ctx(req);
+ struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
+ struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
+ struct crypto_shash *child = cryptd_ahash_child(cryptd_tfm);
+
+ desc->tfm = child;
+ return crypto_shash_init(desc);
+}
+
+static int polyval_async_update(struct ahash_request *req)
+{
+ struct ahash_request *cryptd_req = ahash_request_ctx(req);
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct polyval_async_ctx *ctx = crypto_ahash_ctx(tfm);
+ struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
+ struct shash_desc *desc;
+
+ if (!crypto_simd_usable() ||
+ (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
+ memcpy(cryptd_req, req, sizeof(*req));
+ ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
+ return crypto_ahash_update(cryptd_req);
+ }
+ desc = cryptd_shash_desc(cryptd_req);
+
+ return shash_ahash_update(req, desc);
+}
+
+static int polyval_async_final(struct ahash_request *req)
+{
+ struct ahash_request *cryptd_req = ahash_request_ctx(req);
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct polyval_async_ctx *ctx = crypto_ahash_ctx(tfm);
+ struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
+ struct shash_desc *desc;
+
+ if (!crypto_simd_usable() ||
+ (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
+ memcpy(cryptd_req, req, sizeof(*req));
+ ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
+ return crypto_ahash_final(cryptd_req);
+ }
+ desc = cryptd_shash_desc(cryptd_req);
+
+ return crypto_shash_final(desc, req->result);
+}
+
+static int polyval_async_import(struct ahash_request *req, const void *in)
+{
+ struct ahash_request *cryptd_req = ahash_request_ctx(req);
+ struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
+ struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
+
+ polyval_async_init(req);
+ memcpy(dctx, in, sizeof(*dctx));
+ return 0;
+
+}
+
+static int polyval_async_export(struct ahash_request *req, void *out)
+{
+ struct ahash_request *cryptd_req = ahash_request_ctx(req);
+ struct shash_desc *desc = cryptd_shash_desc(cryptd_req);
+ struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
+
+ memcpy(out, dctx, sizeof(*dctx));
+ return 0;
+
+}
+
+static int polyval_async_digest(struct ahash_request *req)
+{
+ struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
+ struct polyval_async_ctx *ctx = crypto_ahash_ctx(tfm);
+ struct ahash_request *cryptd_req = ahash_request_ctx(req);
+ struct cryptd_ahash *cryptd_tfm = ctx->cryptd_tfm;
+ struct shash_desc *desc;
+ struct crypto_shash *child;
+
+ if (!crypto_simd_usable() ||
+ (in_atomic() && cryptd_ahash_queued(cryptd_tfm))) {
+ memcpy(cryptd_req, req, sizeof(*req));
+ ahash_request_set_tfm(cryptd_req, &cryptd_tfm->base);
+ return crypto_ahash_digest(cryptd_req);
+ }
+ desc = cryptd_shash_desc(cryptd_req);
+ child = cryptd_ahash_child(cryptd_tfm);
+
+ desc->tfm = child;
+ return shash_ahash_digest(req, desc);
+}
+
+static int polyval_async_setkey(struct crypto_ahash *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct polyval_async_ctx *ctx = crypto_ahash_ctx(tfm);
+ struct crypto_ahash *child = &ctx->cryptd_tfm->base;
+
+ crypto_ahash_clear_flags(child, CRYPTO_TFM_REQ_MASK);
+ crypto_ahash_set_flags(child, crypto_ahash_get_flags(tfm)
+ & CRYPTO_TFM_REQ_MASK);
+ return crypto_ahash_setkey(child, key, keylen);
+}
+
+static int polyval_async_init_tfm(struct crypto_tfm *tfm)
+{
+ struct cryptd_ahash *cryptd_tfm;
+ struct polyval_async_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ cryptd_tfm = cryptd_alloc_ahash("__polyval-pclmulqdqni",
+ CRYPTO_ALG_INTERNAL,
+ CRYPTO_ALG_INTERNAL);
+ if (IS_ERR(cryptd_tfm))
+ return PTR_ERR(cryptd_tfm);
+ ctx->cryptd_tfm = cryptd_tfm;
+ crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
+ sizeof(struct ahash_request) +
+ crypto_ahash_reqsize(&cryptd_tfm->base));
+
+ return 0;
+}
+
+static void polyval_async_exit_tfm(struct crypto_tfm *tfm)
+{
+ struct polyval_async_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ cryptd_free_ahash(ctx->cryptd_tfm);
+}
+
+static struct ahash_alg polyval_async_alg = {
+ .init = polyval_async_init,
+ .update = polyval_async_update,
+ .final = polyval_async_final,
+ .setkey = polyval_async_setkey,
+ .digest = polyval_async_digest,
+ .export = polyval_async_export,
+ .import = polyval_async_import,
+ .halg = {
+ .digestsize = POLYVAL_DIGEST_SIZE,
+ .statesize = sizeof(struct polyval_desc_ctx),
+ .base = {
+ .cra_name = "polyval",
+ .cra_driver_name = "polyval-clmulni",
+ .cra_priority = 200,
+ .cra_ctxsize = sizeof(struct polyval_async_ctx),
+ .cra_flags = CRYPTO_ALG_ASYNC,
+ .cra_blocksize = POLYVAL_BLOCK_SIZE,
+ .cra_module = THIS_MODULE,
+ .cra_init = polyval_async_init_tfm,
+ .cra_exit = polyval_async_exit_tfm,
+ },
+ },
+};
+
+static const struct x86_cpu_id pcmul_cpu_id[] = {
+ X86_MATCH_FEATURE(X86_FEATURE_PCLMULQDQ, NULL), /* Pickle-Mickle-Duck */
+ {}
+};
+MODULE_DEVICE_TABLE(x86cpu, pcmul_cpu_id);
+
+static int __init polyval_pclmulqdqni_mod_init(void)
+{
+ int err;
+
+ if (!x86_match_cpu(pcmul_cpu_id))
+ return -ENODEV;
+
+ err = crypto_register_shash(&polyval_alg);
+ if (err)
+ goto err_out;
+ err = crypto_register_ahash(&polyval_async_alg);
+ if (err)
+ goto err_shash;
+
+ return 0;
+
+err_shash:
+ crypto_unregister_shash(&polyval_alg);
+err_out:
+ return err;
+}
+
+static void __exit polyval_pclmulqdqni_mod_exit(void)
+{
+ crypto_unregister_ahash(&polyval_async_alg);
+ crypto_unregister_shash(&polyval_alg);
+}
+
+module_init(polyval_pclmulqdqni_mod_init);
+module_exit(polyval_pclmulqdqni_mod_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("POLYVAL hash function accelerated by PCLMULQDQ-NI");
+MODULE_ALIAS_CRYPTO("polyval");
diff --git a/crypto/Kconfig b/crypto/Kconfig
index 2a9029f51caf..b539a5bdc45e 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -773,12 +773,22 @@ config CRYPTO_GHASH
config CRYPTO_POLYVAL
tristate
+ select CRYPTO_CRYPTD
select CRYPTO_GF128MUL
select CRYPTO_HASH
help
POLYVAL is the hash function used in HCTR2. It is not a general-purpose
cryptographic hash function.
+config CRYPTO_POLYVAL_CLMUL_NI
+ tristate "POLYVAL hash function (CLMUL-NI accelerated)"
+ depends on X86 && 64BIT
+ select CRYPTO_POLYVAL
+ help
+ This is the x86_64 CLMUL-NI accelerated implementation of POLYVAL. It is
+ used to efficiently implement HCTR2 on x86-64 processors that support
+ carry-less multiplication instructions.
+
config CRYPTO_POLY1305
tristate "Poly1305 authenticator algorithm"
select CRYPTO_HASH
--
2.35.1.265.g69c8d7142f-goog
Hi Huck, sorry for the slow reviews. Some comments on this:
On Thu, Feb 10, 2022 at 11:28:11PM +0000, Nathan Huckleberry wrote:
> diff --git a/arch/x86/crypto/polyval-clmulni_asm.S b/arch/x86/crypto/polyval-clmulni_asm.S
> new file mode 100644
> index 000000000000..bec1a2046b18
> --- /dev/null
> +++ b/arch/x86/crypto/polyval-clmulni_asm.S
> @@ -0,0 +1,414 @@
> +/* SPDX-License-Identifier: GPL-2.0 */
> +/*
> + * Copyright 2021 Google LLC
> + *
> + * Use of this source code is governed by an MIT-style
> + * license that can be found in the LICENSE file or at
> + * https://opensource.org/licenses/MIT.
> + */
You're intending this file to be GPL-2.0 only, right? (That's what I'd
recommend.) In that case, the mention of the MIT license needs to be removed.
> +Lgstar:
> + .quad 0xc200000000000000, 0xc200000000000000
Local labels need to start with ".L", not just "L" (note the period).
> +/*
> + * Performs schoolbook1_iteration on two lists of 128-bit polynomials of length
> + * b pointed to by OP1 and OP2.
> + */
How about renaming OP1 and OP2 to MSG and KEY_POWERS, respectively? (Or KEYS
instead of KEY_POWERS if you want something less verbose.) The naming and
comments make it sound like OP1 and OP2 are interchangable, but they actually
aren't. schoolbook1_iteration assumes that OP1 may be unaligned while OP2 is
aligned, which implies that OP1 is the message and OP2 is the key powers.
> +.macro schoolbook1 b
> + .set by, \b
> + .set i, 0
> + .rept (by)
> + schoolbook1_iteration i 0
> + .set i, (i +1)
> + .endr
> +.endm
Rename 'b' to 'count' to make it clear what it is?
Also, the line '.set by, \b' is unnecessary, since you could just use the macro
argument directly instead. (I mentioned unnecessary .set statements elsewhere
on v1. Make sure to go through and address the same issue everywhere.)
I.e., this could be:
.macro schoolbook1 count
.set i, 0
.rept \count
schoolbook1_iteration i 0
.set i, i + 1
.endr
.endm
> +/*
> + * Computes the product of two 128-bit polynomials at the memory locations
> + * specified by (OP1 + 16*i) and (OP2 + 16*i) and XORs the components of the
> + * 256-bit product into LO, MI, HI.
> + *
> + * The multiplication produces four parts:
> + * LOW: The polynomial given by performing carryless multiplication of the
> + * bottom 64-bits of each polynomial
> + * MID1: The polynomial given by performing carryless multiplication of the
> + * bottom 64-bits of the first polynomial and the top 64-bits of the second
> + * MID2: The polynomial given by performing carryless multiplication of the
> + * bottom 64-bits of the second polynomial and the top 64-bits of the first
> + * HIGH: The polynomial given by performing carryless multiplication of the
> + * top 64-bits of each polynomial
> + *
> + * We compute:
> + * LO ^= LOW
> + * Mi ^= MID1 ^ MID2
> + * Hi ^= HIGH
Mi => MI, and Hi => HI?
> + *
> + * Later, the 256-bit result can be extracted as:
> + * [HI_H : HI_L ^ MI_H : LO_H ^ MI_L : LO_L]
> + * This step is done when computing the polynomial reduction for efficiency
> + * reasons.
> + *
> + * If xor_sum == 1 then XOR the value of SUM into m_0.
In comments, please make it clear whether something is an additional behavior or
an alternative behavior. In this case it's the former, so:
If xor_sum == 1, then also XOR ...
> + * This avoids an extra multication of SUM and h^N.
multication => multiplication
> +.macro schoolbook1_iteration i xor_sum
> + .set i, \i
> + .set xor_sum, \xor_sum
> + movups (16*i)(OP1), %xmm0
> + .if(i == 0 && xor_sum == 1)
> + pxor SUM, %xmm0
> + .endif
> + vpclmulqdq $0x01, (16*i)(OP2), %xmm0, %xmm1
> + vpxor %xmm1, MI, MI
> + vpclmulqdq $0x00, (16*i)(OP2), %xmm0, %xmm2
> + vpxor %xmm2, LO, LO
> + vpclmulqdq $0x11, (16*i)(OP2), %xmm0, %xmm3
> + vpxor %xmm3, HI, HI
> + vpclmulqdq $0x10, (16*i)(OP2), %xmm0, %xmm4
> + vpxor %xmm4, MI, MI
Perhaps the above multiplications and XORs should be reordered slightly so that
each XOR doesn't depend on the previous instruction? A good ordering might be:
vpclmulqdq $0x01, (16*\i)(OP2), %xmm0, %xmm1
vpclmulqdq $0x10, (16*\i)(OP2), %xmm0, %xmm2
vpclmulqdq $0x00, (16*\i)(OP2), %xmm0, %xmm3
vpclmulqdq $0x11, (16*\i)(OP2), %xmm0, %xmm4
vpxor %xmm1, MI, MI
vpxor %xmm3, LO, LO
vpxor %xmm4, HI, HI
vpxor %xmm2, MI, MI
With that, no instruction would depend on either of the previous two
instructions.
This might be more important in the ARM64 version than the x86_64 version, as
x86_64 CPUs are pretty aggressive about internally reordering instructions. But
it's something to consider in both versions.
Likewise in schoolbook1_noload.
> +/*
> + * Computes the 128-bit reduction of PL, PH. Stores the result in PH.
> + *
> + * This macro computes p(x) mod g(x) where p(x) is in montgomery form and g(x) =
> + * x^128 + x^127 + x^126 + x^121 + 1.
> + *
> + * The montgomery form of a polynomial p(x) is p(x)x^{128}. Montgomery reduction
> + * works by simultaneously dividing by x^{128} and computing the modular
> + * reduction.
> + *
> + * Suppose we wish to reduce the montgomery form of p(x) = [P_3 : P_2 : P_1 :
> + * P_0] where P_i is a polynomial of degree at most 64 represented as 64-bits.
> + * Thus we would like to compute:
> + * p(x) / x^{128} mod g(x)
> + * = (P_3*x^{192} + P_2*x^{128} + P_1*x^{64} + P_0) / x^{128} mod g(x)
This doesn't explain why there is an additional factor of x^128 that must be
removed. That's one of the key parts to understanding this.
> + * We now focus on dividing P_1*x^{64} + P_0 by x^{128}. We do this by making
> + * P_1*x^{64} + P_0 divisble by x^{128} then bitshifting. To add divisibility,
> + * we consider the polynomials mod x^{128}.
> + *
> + * Let c(x) = P_1*x^{64} + P_0.
> + *
> + * Now let m(x) = c(x) mod x^{128}
> + * and
> + * Let z(x) = [c(x) + m(x)g(x)] / x^{128}
> + *
> + * First notice that:
> + * c(x) + m(x)g(x) = c(x) mod g(x).
> + * Furthermore, g(x) mod x^{128} = 1, so we have
> + * c(x) + m(x)g(x) = c(x) + c(x) = 0 (mod x^{128}).
I don't understand this explanation. m(x) = c(x), so m(x) seems to have no
purpose. Also, g(x) mod x^{128} is not 1.
I stopped reading here due to the above issues. I think this explanation is too
long and misses the mark somewhat. I think that something shorter that explains
the intuition behind the algorithm used would be much more helpful. E.g.:
"We have a 256-bit polynomial P_3 : P_2 : P_1 : P_0 that is the product of
two 128-bit polynomials in Montgomery form. We need to reduce it mod g(x).
Also, since polynomials in Montgomery form have an "extra" factor of x^128,
this product has two extra factors of x^128. To get it back into Montgomery
form, we need to remove one of these factors by dividing by x^128.
To accomplish both of these goals, we add multiples of g(x) that cancel out
the low 128 bits P_1 : P_0, leaving just the high 128 bits, which is the
answer.
Since the only nonzero term in the low 64 bits of g(x) is the constant term,
the multiple of g(x) needed to cancel out P_0 is P_0 itself. The CPU can
only do 64x64 bit multiplications, so split P_0 * g(x) into x^128 * P_0 +
x^64 * g*(x) * P_0 + P_0, where g*(x) is bits 64-127 of g(x). Adding this
to the original polynomial gives P_3 : P_2 + P_0 + T_1 : P_1 + T_0 : 0,
where T = T_1 : T_0 = g*(x) * P0. Thus, bits 0-63 got "folded" into bits
64-191.
Repeating this same process on the next 64 bits "folds" bits 64-127 into
bits 128-255, giving the answer in bits 128-255."
I think that covers everything except the optimization you've implemented where
you avoid a second pshufd, which makes the second "fold" step in your code
appear a bit different from the first. But do I understand correctly that the
math is still the same in both steps in your version, and the difference between
the steps is just where in the registers each part is stored?
> +/*
> + * Compute schoolbook multiplication for 8 blocks
> + * M_0h^8 + ... + M_7h^1 (no constant term)
Some places say M while other say m. It's probably best to use 'm', since the
key is 'h' (both lower-case).
> + *
> + * If reduce is set, computes the montgomery reduction of the
> + * previous full_stride call and XORs with the first message block.
computes => also computes
> + * (M_0 + REDUCE(PL, PH))h^8 + ... + M_7h^1 (no constant term)
"I.e., the first multiplication uses m_0 + REDUCE(PL, PH) instead of m_0."
> +/*
> + * Perform polynomial evaluation as specified by POLYVAL. If nblocks = k, this
> + * routine multiplies the value stored at accumulator by h^k and XORs the
> + * evaluated polynomial into it.
> + *
> + * Computes h^k * accumulator + h^kM_0 + ... + h^1M_{k-1} (No constant term)
The above explanation is confusing, especially the second sentence. How about:
* Perform polynomial evaluation as specified by POLYVAL. This computes:
*
* h^k * accumulator + h^k * M_0 + ... + h^1 * m_{k-1}
*
* ... where k=nblocks, h is the hash key, and m_i are the message blocks.
Also, perhaps use n instead of k, given that k could be confused with the key?
> + * rsi - pointer to precomputed key struct
pointer to the precomputed key powers h^8 ... h^1
> + * rcx - location to XOR with evaluated polynomial
pointer to the current accumulator
> + *
> + * void clmul_polyval_update(const u8 *in, const struct polyhash_ctx* ctx,
polyval_ctx, not polyhash_ctx
> +#define POLYVAL_BLOCK_SIZE 16
> +#define POLYVAL_DIGEST_SIZE 16
Why not get the above definitions by including <crypto/polyval.h>?
Or to put it another way, what is the purpose of having that header file if all
the POLYVAL implementations aren't going to include it?
> +static int polyval_final(struct shash_desc *desc, u8 *dst)
> +{
> + struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
> + struct polyval_ctx *ctx = crypto_shash_ctx(desc->tfm);
> + u8 *buf = dctx->buffer;
> +
> + if (dctx->bytes) {
> + kernel_fpu_begin();
> + clmul_polyval_mul((u8 *)buf,
> + (u8 *)&ctx->key_powers[NUM_PRECOMPUTE_POWERS-1]);
The cast of buf to 'u8 *' is unnecessary.
> + .cra_driver_name = "__polyval-pclmulqdqni",
__polyval-clmulni
> +static int __init polyval_pclmulqdqni_mod_init(void)
polyval_clmulni_mod_init
> +{
> + int err;
> +
> + if (!x86_match_cpu(pcmul_cpu_id))
> + return -ENODEV;
> +
> + err = crypto_register_shash(&polyval_alg);
> + if (err)
> + goto err_out;
> + err = crypto_register_ahash(&polyval_async_alg);
> + if (err)
> + goto err_shash;
> +
> + return 0;
> +
> +err_shash:
> + crypto_unregister_shash(&polyval_alg);
> +err_out:
> + return err;
> +}
> +
> +static void __exit polyval_pclmulqdqni_mod_exit(void)
polyval_clmulni_mod_exit
> +MODULE_ALIAS_CRYPTO("polyval");
Add:
MODULE_ALIAS_CRYPTO("polyval-clmulni");
Otherwise this module won't be loaded if someone requests this by implementation
name.
> config CRYPTO_POLYVAL
> tristate
> + select CRYPTO_CRYPTD
> select CRYPTO_GF128MUL
> select CRYPTO_HASH
> help
> POLYVAL is the hash function used in HCTR2. It is not a general-purpose
> cryptographic hash function.
>
> +config CRYPTO_POLYVAL_CLMUL_NI
> + tristate "POLYVAL hash function (CLMUL-NI accelerated)"
> + depends on X86 && 64BIT
> + select CRYPTO_POLYVAL
> + help
> + This is the x86_64 CLMUL-NI accelerated implementation of POLYVAL. It is
> + used to efficiently implement HCTR2 on x86-64 processors that support
> + carry-less multiplication instructions.
Shouldn't the 'select CRYPTO_CRYPTD' be under CRYPTO_POLYVAL_CLMUL_NI, not
CRYPTO_POLYVAL?
- Eric
On Fri, Feb 18, 2022 at 04:34:28PM -0800, Eric Biggers wrote:
> > +.macro schoolbook1_iteration i xor_sum
> > + .set i, \i
> > + .set xor_sum, \xor_sum
> > + movups (16*i)(OP1), %xmm0
> > + .if(i == 0 && xor_sum == 1)
> > + pxor SUM, %xmm0
> > + .endif
> > + vpclmulqdq $0x01, (16*i)(OP2), %xmm0, %xmm1
> > + vpxor %xmm1, MI, MI
> > + vpclmulqdq $0x00, (16*i)(OP2), %xmm0, %xmm2
> > + vpxor %xmm2, LO, LO
> > + vpclmulqdq $0x11, (16*i)(OP2), %xmm0, %xmm3
> > + vpxor %xmm3, HI, HI
> > + vpclmulqdq $0x10, (16*i)(OP2), %xmm0, %xmm4
> > + vpxor %xmm4, MI, MI
>
> Perhaps the above multiplications and XORs should be reordered slightly so that
> each XOR doesn't depend on the previous instruction? A good ordering might be:
>
> vpclmulqdq $0x01, (16*\i)(OP2), %xmm0, %xmm1
> vpclmulqdq $0x10, (16*\i)(OP2), %xmm0, %xmm2
> vpclmulqdq $0x00, (16*\i)(OP2), %xmm0, %xmm3
> vpclmulqdq $0x11, (16*\i)(OP2), %xmm0, %xmm4
> vpxor %xmm1, MI, MI
> vpxor %xmm3, LO, LO
> vpxor %xmm4, HI, HI
> vpxor %xmm2, MI, MI
>
> With that, no instruction would depend on either of the previous two
> instructions.
>
> This might be more important in the ARM64 version than the x86_64 version, as
> x86_64 CPUs are pretty aggressive about internally reordering instructions. But
> it's something to consider in both versions.
>
> Likewise in schoolbook1_noload.
Or slightly better:
vpclmulqdq $0x01, (16*\i)(OP2), %xmm0, %xmm2
vpclmulqdq $0x00, (16*\i)(OP2), %xmm0, %xmm1
vpclmulqdq $0x10, (16*\i)(OP2), %xmm0, %xmm3
vpclmulqdq $0x11, (16*\i)(OP2), %xmm0, %xmm4
vpxor %xmm2, MI, MI
vpxor %xmm1, LO, LO
vpxor %xmm4, HI, HI
vpxor %xmm3, MI, MI
- Eric