This patchset contains the following gf128mul-related changes:
1. The gf128mul_x_* functions are moved to gf128mul.h for performance reasons.
2. The gf128mul_x_ble function is fixed to use the correct block type.
3. The le128_gf128mul_x_ble function from glue_helper.h is removed and its
usages replaced with gf128mul_x_ble calls.
4. The now obsolete dependency of CRYPTO_XTS on CRYPTO_GF128MUL is removed.
v4 -> v5: added the other three patches
v3 -> v4: a faster version of gf128mul_x_lle
v2 -> v3: constant-time implementation
v1 -> v2: move all _x_ functions to the header, not just gf128mul_x_ble
Ondrej Mosnacek (4):
crypto: gf128mul - define gf128mul_x_* in gf128mul.h
crypto: gf128mul - switch gf128mul_x_ble to le128
crypto: glue_helper - remove the le128_gf128mul_x_ble function
crypto: xts - drop gf128mul dependency
arch/x86/crypto/camellia_glue.c | 4 +--
arch/x86/crypto/glue_helper.c | 3 +-
arch/x86/crypto/serpent_sse2_glue.c | 4 +--
arch/x86/crypto/twofish_glue_3way.c | 4 +--
arch/x86/include/asm/crypto/glue_helper.h | 10 ------
crypto/Kconfig | 1 -
crypto/gf128mul.c | 33 +------------------
crypto/xts.c | 38 ++++++++++-----------
include/crypto/gf128mul.h | 55 +++++++++++++++++++++++++++++--
include/crypto/xts.h | 2 +-
10 files changed, 82 insertions(+), 72 deletions(-)
--
2.9.3
Since the gf128mul_x_ble function used by xts.c is now defined inline
in the header file, the XTS module no longer depends on gf128mul.
Therefore, the 'select CRYPTO_GF128MUL' line can be safely removed.
Signed-off-by: Ondrej Mosnacek <[email protected]>
---
crypto/Kconfig | 1 -
1 file changed, 1 deletion(-)
diff --git a/crypto/Kconfig b/crypto/Kconfig
index 6854c1f..aac4bc9 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -374,7 +374,6 @@ config CRYPTO_XTS
tristate "XTS support"
select CRYPTO_BLKCIPHER
select CRYPTO_MANAGER
- select CRYPTO_GF128MUL
select CRYPTO_ECB
help
XTS: IEEE1619/D16 narrow block cipher use with aes-xts-plain,
--
2.9.3
The gf128mul_x_ble function is currently defined in gf128mul.c, because
it depends on the gf128mul_table_be multiplication table.
However, since the function is very small and only uses two values from
the table, it is better for it to be defined as inline function in
gf128mul.h. That way, the function can be inlined by the compiler for
better performance.
For consistency, the other gf128mul_x_* functions are also moved to the
header file. In addition, the code is rewritten to be constant-time.
After this change, the speed of the generic 'xts(aes)' implementation
increased from ~225 MiB/s to ~235 MiB/s (measured using 'cryptsetup
benchmark -c aes-xts-plain64' on an Intel system with CRYPTO_AES_X86_64
and CRYPTO_AES_NI_INTEL disabled).
Signed-off-by: Ondrej Mosnacek <[email protected]>
Cc: Eric Biggers <[email protected]>
---
crypto/gf128mul.c | 33 +---------------------------
include/crypto/gf128mul.h | 55 +++++++++++++++++++++++++++++++++++++++++++++--
2 files changed, 54 insertions(+), 34 deletions(-)
diff --git a/crypto/gf128mul.c b/crypto/gf128mul.c
index 04facc0..dc01212 100644
--- a/crypto/gf128mul.c
+++ b/crypto/gf128mul.c
@@ -130,43 +130,12 @@ static const u16 gf128mul_table_le[256] = gf128mul_dat(xda_le);
static const u16 gf128mul_table_be[256] = gf128mul_dat(xda_be);
/*
- * The following functions multiply a field element by x or by x^8 in
+ * The following functions multiply a field element by x^8 in
* the polynomial field representation. They use 64-bit word operations
* to gain speed but compensate for machine endianness and hence work
* correctly on both styles of machine.
*/
-static void gf128mul_x_lle(be128 *r, const be128 *x)
-{
- u64 a = be64_to_cpu(x->a);
- u64 b = be64_to_cpu(x->b);
- u64 _tt = gf128mul_table_le[(b << 7) & 0xff];
-
- r->b = cpu_to_be64((b >> 1) | (a << 63));
- r->a = cpu_to_be64((a >> 1) ^ (_tt << 48));
-}
-
-static void gf128mul_x_bbe(be128 *r, const be128 *x)
-{
- u64 a = be64_to_cpu(x->a);
- u64 b = be64_to_cpu(x->b);
- u64 _tt = gf128mul_table_be[a >> 63];
-
- r->a = cpu_to_be64((a << 1) | (b >> 63));
- r->b = cpu_to_be64((b << 1) ^ _tt);
-}
-
-void gf128mul_x_ble(be128 *r, const be128 *x)
-{
- u64 a = le64_to_cpu(x->a);
- u64 b = le64_to_cpu(x->b);
- u64 _tt = gf128mul_table_be[b >> 63];
-
- r->a = cpu_to_le64((a << 1) ^ _tt);
- r->b = cpu_to_le64((b << 1) | (a >> 63));
-}
-EXPORT_SYMBOL(gf128mul_x_ble);
-
static void gf128mul_x8_lle(be128 *x)
{
u64 a = be64_to_cpu(x->a);
diff --git a/include/crypto/gf128mul.h b/include/crypto/gf128mul.h
index 0bc9b5f..35ced9d 100644
--- a/include/crypto/gf128mul.h
+++ b/include/crypto/gf128mul.h
@@ -49,6 +49,7 @@
#ifndef _CRYPTO_GF128MUL_H
#define _CRYPTO_GF128MUL_H
+#include <asm/byteorder.h>
#include <crypto/b128ops.h>
#include <linux/slab.h>
@@ -163,8 +164,58 @@ void gf128mul_lle(be128 *a, const be128 *b);
void gf128mul_bbe(be128 *a, const be128 *b);
-/* multiply by x in ble format, needed by XTS */
-void gf128mul_x_ble(be128 *a, const be128 *b);
+/*
+ * The following functions multiply a field element by x in
+ * the polynomial field representation. They use 64-bit word operations
+ * to gain speed but compensate for machine endianness and hence work
+ * correctly on both styles of machine.
+ *
+ * They are defined here for performance.
+ */
+
+static inline u64 gf128mul_mask_from_bit(u64 x, int which)
+{
+ /* a constant-time version of 'x & ((u64)1 << which) ? (u64)-1 : 0' */
+ return ((s64)(x << (63 - which)) >> 63);
+}
+
+static inline void gf128mul_x_lle(be128 *r, const be128 *x)
+{
+ u64 a = be64_to_cpu(x->a);
+ u64 b = be64_to_cpu(x->b);
+
+ /* equivalent to gf128mul_table_le[(b << 7) & 0xff] << 48
+ * (see crypto/gf128mul.c): */
+ u64 _tt = gf128mul_mask_from_bit(b, 0) & ((u64)0xe1 << 56);
+
+ r->b = cpu_to_be64((b >> 1) | (a << 63));
+ r->a = cpu_to_be64((a >> 1) ^ _tt);
+}
+
+static inline void gf128mul_x_bbe(be128 *r, const be128 *x)
+{
+ u64 a = be64_to_cpu(x->a);
+ u64 b = be64_to_cpu(x->b);
+
+ /* equivalent to gf128mul_table_be[a >> 63] (see crypto/gf128mul.c): */
+ u64 _tt = gf128mul_mask_from_bit(a, 63) & 0x87;
+
+ r->a = cpu_to_be64((a << 1) | (b >> 63));
+ r->b = cpu_to_be64((b << 1) ^ _tt);
+}
+
+/* needed by XTS */
+static inline void gf128mul_x_ble(be128 *r, const be128 *x)
+{
+ u64 a = le64_to_cpu(x->a);
+ u64 b = le64_to_cpu(x->b);
+
+ /* equivalent to gf128mul_table_be[b >> 63] (see crypto/gf128mul.c): */
+ u64 _tt = gf128mul_mask_from_bit(b, 63) & 0x87;
+
+ r->a = cpu_to_le64((a << 1) ^ _tt);
+ r->b = cpu_to_le64((b << 1) | (a >> 63));
+}
/* 4k table optimization */
--
2.9.3
Currently, gf128mul_x_ble works with pointers to be128, even though it
actually interprets the words as little-endian. Consequently, it uses
cpu_to_le64/le64_to_cpu on fields of type __be64, which is incorrect.
This patch fixes that by changing the function to accept pointers to
le128 and updating all users accordingly.
Signed-off-by: Ondrej Mosnacek <[email protected]>
---
arch/x86/crypto/camellia_glue.c | 4 ++--
arch/x86/crypto/serpent_sse2_glue.c | 4 ++--
arch/x86/crypto/twofish_glue_3way.c | 4 ++--
crypto/xts.c | 38 ++++++++++++++++++-------------------
include/crypto/gf128mul.h | 8 ++++----
include/crypto/xts.h | 2 +-
6 files changed, 30 insertions(+), 30 deletions(-)
diff --git a/arch/x86/crypto/camellia_glue.c b/arch/x86/crypto/camellia_glue.c
index aa76cad..af4840a 100644
--- a/arch/x86/crypto/camellia_glue.c
+++ b/arch/x86/crypto/camellia_glue.c
@@ -1522,7 +1522,7 @@ static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct camellia_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
- be128 buf[2 * 4];
+ le128 buf[2 * 4];
struct xts_crypt_req req = {
.tbuf = buf,
.tbuflen = sizeof(buf),
@@ -1540,7 +1540,7 @@ static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct camellia_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
- be128 buf[2 * 4];
+ le128 buf[2 * 4];
struct xts_crypt_req req = {
.tbuf = buf,
.tbuflen = sizeof(buf),
diff --git a/arch/x86/crypto/serpent_sse2_glue.c b/arch/x86/crypto/serpent_sse2_glue.c
index 644f97a..ac0e831 100644
--- a/arch/x86/crypto/serpent_sse2_glue.c
+++ b/arch/x86/crypto/serpent_sse2_glue.c
@@ -328,7 +328,7 @@ static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
- be128 buf[SERPENT_PARALLEL_BLOCKS];
+ le128 buf[SERPENT_PARALLEL_BLOCKS];
struct crypt_priv crypt_ctx = {
.ctx = &ctx->crypt_ctx,
.fpu_enabled = false,
@@ -355,7 +355,7 @@ static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
- be128 buf[SERPENT_PARALLEL_BLOCKS];
+ le128 buf[SERPENT_PARALLEL_BLOCKS];
struct crypt_priv crypt_ctx = {
.ctx = &ctx->crypt_ctx,
.fpu_enabled = false,
diff --git a/arch/x86/crypto/twofish_glue_3way.c b/arch/x86/crypto/twofish_glue_3way.c
index 2ebb5e9..243e90a 100644
--- a/arch/x86/crypto/twofish_glue_3way.c
+++ b/arch/x86/crypto/twofish_glue_3way.c
@@ -296,7 +296,7 @@ static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
- be128 buf[3];
+ le128 buf[3];
struct xts_crypt_req req = {
.tbuf = buf,
.tbuflen = sizeof(buf),
@@ -314,7 +314,7 @@ static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
- be128 buf[3];
+ le128 buf[3];
struct xts_crypt_req req = {
.tbuf = buf,
.tbuflen = sizeof(buf),
diff --git a/crypto/xts.c b/crypto/xts.c
index baeb34d..bd5065c 100644
--- a/crypto/xts.c
+++ b/crypto/xts.c
@@ -39,11 +39,11 @@ struct xts_instance_ctx {
};
struct rctx {
- be128 buf[XTS_BUFFER_SIZE / sizeof(be128)];
+ le128 buf[XTS_BUFFER_SIZE / sizeof(le128)];
- be128 t;
+ le128 t;
- be128 *ext;
+ le128 *ext;
struct scatterlist srcbuf[2];
struct scatterlist dstbuf[2];
@@ -99,7 +99,7 @@ static int setkey(struct crypto_skcipher *parent, const u8 *key,
static int post_crypt(struct skcipher_request *req)
{
struct rctx *rctx = skcipher_request_ctx(req);
- be128 *buf = rctx->ext ?: rctx->buf;
+ le128 *buf = rctx->ext ?: rctx->buf;
struct skcipher_request *subreq;
const int bs = XTS_BLOCK_SIZE;
struct skcipher_walk w;
@@ -112,12 +112,12 @@ static int post_crypt(struct skcipher_request *req)
while (w.nbytes) {
unsigned int avail = w.nbytes;
- be128 *wdst;
+ le128 *wdst;
wdst = w.dst.virt.addr;
do {
- be128_xor(wdst, buf++, wdst);
+ le128_xor(wdst, buf++, wdst);
wdst++;
} while ((avail -= bs) >= bs);
@@ -150,7 +150,7 @@ static int post_crypt(struct skcipher_request *req)
static int pre_crypt(struct skcipher_request *req)
{
struct rctx *rctx = skcipher_request_ctx(req);
- be128 *buf = rctx->ext ?: rctx->buf;
+ le128 *buf = rctx->ext ?: rctx->buf;
struct skcipher_request *subreq;
const int bs = XTS_BLOCK_SIZE;
struct skcipher_walk w;
@@ -174,15 +174,15 @@ static int pre_crypt(struct skcipher_request *req)
while (w.nbytes) {
unsigned int avail = w.nbytes;
- be128 *wsrc;
- be128 *wdst;
+ le128 *wsrc;
+ le128 *wdst;
wsrc = w.src.virt.addr;
wdst = w.dst.virt.addr;
do {
*buf++ = rctx->t;
- be128_xor(wdst++, &rctx->t, wsrc++);
+ le128_xor(wdst++, &rctx->t, wsrc++);
gf128mul_x_ble(&rctx->t, &rctx->t);
} while ((avail -= bs) >= bs);
@@ -350,8 +350,8 @@ int xts_crypt(struct blkcipher_desc *desc, struct scatterlist *sdst,
const unsigned int max_blks = req->tbuflen / bsize;
struct blkcipher_walk walk;
unsigned int nblocks;
- be128 *src, *dst, *t;
- be128 *t_buf = req->tbuf;
+ le128 *src, *dst, *t;
+ le128 *t_buf = req->tbuf;
int err, i;
BUG_ON(max_blks < 1);
@@ -364,8 +364,8 @@ int xts_crypt(struct blkcipher_desc *desc, struct scatterlist *sdst,
return err;
nblocks = min(nbytes / bsize, max_blks);
- src = (be128 *)walk.src.virt.addr;
- dst = (be128 *)walk.dst.virt.addr;
+ src = (le128 *)walk.src.virt.addr;
+ dst = (le128 *)walk.dst.virt.addr;
/* calculate first value of T */
req->tweak_fn(req->tweak_ctx, (u8 *)&t_buf[0], walk.iv);
@@ -381,7 +381,7 @@ int xts_crypt(struct blkcipher_desc *desc, struct scatterlist *sdst,
t = &t_buf[i];
/* PP <- T xor P */
- be128_xor(dst + i, t, src + i);
+ le128_xor(dst + i, t, src + i);
}
/* CC <- E(Key2,PP) */
@@ -390,7 +390,7 @@ int xts_crypt(struct blkcipher_desc *desc, struct scatterlist *sdst,
/* C <- T xor CC */
for (i = 0; i < nblocks; i++)
- be128_xor(dst + i, dst + i, &t_buf[i]);
+ le128_xor(dst + i, dst + i, &t_buf[i]);
src += nblocks;
dst += nblocks;
@@ -398,7 +398,7 @@ int xts_crypt(struct blkcipher_desc *desc, struct scatterlist *sdst,
nblocks = min(nbytes / bsize, max_blks);
} while (nblocks > 0);
- *(be128 *)walk.iv = *t;
+ *(le128 *)walk.iv = *t;
err = blkcipher_walk_done(desc, &walk, nbytes);
nbytes = walk.nbytes;
@@ -406,8 +406,8 @@ int xts_crypt(struct blkcipher_desc *desc, struct scatterlist *sdst,
break;
nblocks = min(nbytes / bsize, max_blks);
- src = (be128 *)walk.src.virt.addr;
- dst = (be128 *)walk.dst.virt.addr;
+ src = (le128 *)walk.src.virt.addr;
+ dst = (le128 *)walk.dst.virt.addr;
}
return err;
diff --git a/include/crypto/gf128mul.h b/include/crypto/gf128mul.h
index 35ced9d..0977fb1 100644
--- a/include/crypto/gf128mul.h
+++ b/include/crypto/gf128mul.h
@@ -205,16 +205,16 @@ static inline void gf128mul_x_bbe(be128 *r, const be128 *x)
}
/* needed by XTS */
-static inline void gf128mul_x_ble(be128 *r, const be128 *x)
+static inline void gf128mul_x_ble(le128 *r, const le128 *x)
{
u64 a = le64_to_cpu(x->a);
u64 b = le64_to_cpu(x->b);
/* equivalent to gf128mul_table_be[b >> 63] (see crypto/gf128mul.c): */
- u64 _tt = gf128mul_mask_from_bit(b, 63) & 0x87;
+ u64 _tt = gf128mul_mask_from_bit(a, 63) & 0x87;
- r->a = cpu_to_le64((a << 1) ^ _tt);
- r->b = cpu_to_le64((b << 1) | (a >> 63));
+ r->a = cpu_to_le64((a << 1) | (b >> 63));
+ r->b = cpu_to_le64((b << 1) ^ _tt);
}
/* 4k table optimization */
diff --git a/include/crypto/xts.h b/include/crypto/xts.h
index 77b6306..c0bde30 100644
--- a/include/crypto/xts.h
+++ b/include/crypto/xts.h
@@ -11,7 +11,7 @@ struct blkcipher_desc;
#define XTS_BLOCK_SIZE 16
struct xts_crypt_req {
- be128 *tbuf;
+ le128 *tbuf;
unsigned int tbuflen;
void *tweak_ctx;
--
2.9.3
The le128_gf128mul_x_ble function in glue_helper.h is now obsolete and
can be replaced with the gf128mul_x_ble function from gf128mul.h.
Signed-off-by: Ondrej Mosnacek <[email protected]>
---
arch/x86/crypto/glue_helper.c | 3 ++-
arch/x86/include/asm/crypto/glue_helper.h | 10 ----------
2 files changed, 2 insertions(+), 11 deletions(-)
diff --git a/arch/x86/crypto/glue_helper.c b/arch/x86/crypto/glue_helper.c
index 260a060..24ac9fa 100644
--- a/arch/x86/crypto/glue_helper.c
+++ b/arch/x86/crypto/glue_helper.c
@@ -27,6 +27,7 @@
#include <linux/module.h>
#include <crypto/b128ops.h>
+#include <crypto/gf128mul.h>
#include <crypto/internal/skcipher.h>
#include <crypto/lrw.h>
#include <crypto/xts.h>
@@ -457,7 +458,7 @@ void glue_xts_crypt_128bit_one(void *ctx, u128 *dst, const u128 *src, le128 *iv,
le128 ivblk = *iv;
/* generate next IV */
- le128_gf128mul_x_ble(iv, &ivblk);
+ gf128mul_x_ble(iv, &ivblk);
/* CC <- T xor C */
u128_xor(dst, src, (u128 *)&ivblk);
diff --git a/arch/x86/include/asm/crypto/glue_helper.h b/arch/x86/include/asm/crypto/glue_helper.h
index 29e53ea..ed8b66d 100644
--- a/arch/x86/include/asm/crypto/glue_helper.h
+++ b/arch/x86/include/asm/crypto/glue_helper.h
@@ -125,16 +125,6 @@ static inline void le128_inc(le128 *i)
i->b = cpu_to_le64(b);
}
-static inline void le128_gf128mul_x_ble(le128 *dst, const le128 *src)
-{
- u64 a = le64_to_cpu(src->a);
- u64 b = le64_to_cpu(src->b);
- u64 _tt = ((s64)a >> 63) & 0x87;
-
- dst->a = cpu_to_le64((a << 1) ^ (b >> 63));
- dst->b = cpu_to_le64((b << 1) ^ _tt);
-}
-
extern int glue_ecb_crypt_128bit(const struct common_glue_ctx *gctx,
struct blkcipher_desc *desc,
struct scatterlist *dst,
--
2.9.3
Hi Ondrej,
On Sun, Apr 02, 2017 at 09:19:12PM +0200, Ondrej Mosnacek wrote:
> This patchset contains the following gf128mul-related changes:
> 1. The gf128mul_x_* functions are moved to gf128mul.h for performance reasons.
> 2. The gf128mul_x_ble function is fixed to use the correct block type.
> 3. The le128_gf128mul_x_ble function from glue_helper.h is removed and its
> usages replaced with gf128mul_x_ble calls.
> 4. The now obsolete dependency of CRYPTO_XTS on CRYPTO_GF128MUL is removed.
>
> v4 -> v5: added the other three patches
> v3 -> v4: a faster version of gf128mul_x_lle
> v2 -> v3: constant-time implementation
> v1 -> v2: move all _x_ functions to the header, not just gf128mul_x_ble
>
> Ondrej Mosnacek (4):
> crypto: gf128mul - define gf128mul_x_* in gf128mul.h
> crypto: gf128mul - switch gf128mul_x_ble to le128
> crypto: glue_helper - remove the le128_gf128mul_x_ble function
> crypto: xts - drop gf128mul dependency
These all look good to me, and you can add
Reviewed-by: Eric Biggers <[email protected]>
to the patches.
I think the change to le128 is an improvement, but what I've actually wanted to
do is have the GF(2^128) elements strongly typed, with ble128, bbe128, and
lle128 types, so that the types would reflect both the bit order and the byte
order. But that's harder to do and I ran into some issues when I tried it last.
- Eric
On Sun, Apr 02, 2017 at 09:19:12PM +0200, Ondrej Mosnacek wrote:
> This patchset contains the following gf128mul-related changes:
> 1. The gf128mul_x_* functions are moved to gf128mul.h for performance reasons.
> 2. The gf128mul_x_ble function is fixed to use the correct block type.
> 3. The le128_gf128mul_x_ble function from glue_helper.h is removed and its
> usages replaced with gf128mul_x_ble calls.
> 4. The now obsolete dependency of CRYPTO_XTS on CRYPTO_GF128MUL is removed.
>
> v4 -> v5: added the other three patches
> v3 -> v4: a faster version of gf128mul_x_lle
> v2 -> v3: constant-time implementation
> v1 -> v2: move all _x_ functions to the header, not just gf128mul_x_ble
All applied. Thanks.
--
Email: Herbert Xu <[email protected]>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt