From: Chuck Ebbert <[email protected]>
crypto: padlock-aes: work around Nano CPU errata in ECB mode
The VIA Nano processor has a bug that makes it prefetch extra data
during encryption operations, causing spurious page faults. Extend
existing workarounds for ECB mode to copy the data to an temporary
buffer to avoid the problem.
Acked-by: HaraldWelte <[email protected]>
Signed-off-by: Chuck Ebbert <[email protected]>
---
Against cryptodev.git as requested.
--- work-2.6.29.4.orig/drivers/crypto/padlock-aes.c
+++ work-2.6.29.4/drivers/crypto/padlock-aes.c
@@ -18,9 +18,17 @@
#include <linux/percpu.h>
#include <linux/smp.h>
#include <asm/byteorder.h>
+#include <asm/processor.h>
#include <asm/i387.h>
#include "padlock.h"
+/* number of data blocks actually fetched for each xcrypt insn */
+static unsigned int ecb_fetch_blocks = 2;
+static unsigned int cbc_fetch_blocks = 1;
+
+#define ecb_fetch_bytes (ecb_fetch_blocks * AES_BLOCK_SIZE)
+#define cbc_fetch_bytes (cbc_fetch_blocks * AES_BLOCK_SIZE)
+
/* Control word. */
struct cword {
unsigned int __attribute__ ((__packed__))
@@ -173,63 +181,59 @@ static inline void padlock_store_cword(s
*/
static inline void padlock_xcrypt(const u8 *input, u8 *output, void *key,
- struct cword *control_word)
+ struct cword *control_word, int count)
{
asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
: "+S"(input), "+D"(output)
- : "d"(control_word), "b"(key), "c"(1));
+ : "d"(control_word), "b"(key), "c"(count));
}
-static void aes_crypt_copy(const u8 *in, u8 *out, u32 *key, struct cword *cword)
+static void aes_crypt_copy(const u8 *in, u8 *out, u32 *key,
+ struct cword *cword, int count)
{
- u8 buf[AES_BLOCK_SIZE * 2 + PADLOCK_ALIGNMENT - 1];
+ /*
+ * Padlock prefetches extra data so we must provide mapped input buffers.
+ * Assume there are at least 16 bytes of stack already in use.
+ */
+ u8 buf[AES_BLOCK_SIZE * 7 + PADLOCK_ALIGNMENT - 1];
u8 *tmp = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);
- memcpy(tmp, in, AES_BLOCK_SIZE);
- padlock_xcrypt(tmp, out, key, cword);
+ memcpy(tmp, in, count * AES_BLOCK_SIZE);
+ padlock_xcrypt(tmp, out, key, cword, count);
}
static inline void aes_crypt(const u8 *in, u8 *out, u32 *key,
- struct cword *cword)
+ struct cword *cword, int count)
{
- /* padlock_xcrypt requires at least two blocks of data. */
- if (unlikely(!(((unsigned long)in ^ (PAGE_SIZE - AES_BLOCK_SIZE)) &
- (PAGE_SIZE - 1)))) {
- aes_crypt_copy(in, out, key, cword);
+ /* Padlock in ECB mode fetches at least ecb_fetch_bytes of data.
+ * We could avoid some copying here but it's probably not worth it.
+ */
+ if (unlikely(((unsigned long)in & PAGE_SIZE) + ecb_fetch_bytes > PAGE_SIZE)) {
+ aes_crypt_copy(in, out, key, cword, count);
return;
}
- padlock_xcrypt(in, out, key, cword);
+ padlock_xcrypt(in, out, key, cword, count);
}
static inline void padlock_xcrypt_ecb(const u8 *input, u8 *output, void *key,
void *control_word, u32 count)
{
- if (count == 1) {
- aes_crypt(input, output, key, control_word);
+ u32 initial = count & (ecb_fetch_blocks - 1);
+
+ if (count < ecb_fetch_blocks) {
+ aes_crypt(input, output, key, control_word, count);
return;
}
- asm volatile ("test $1, %%cl;"
- "je 1f;"
-#ifndef CONFIG_X86_64
- "lea -1(%%ecx), %%eax;"
- "mov $1, %%ecx;"
-#else
- "lea -1(%%rcx), %%rax;"
- "mov $1, %%rcx;"
-#endif
- ".byte 0xf3,0x0f,0xa7,0xc8;" /* rep xcryptecb */
-#ifndef CONFIG_X86_64
- "mov %%eax, %%ecx;"
-#else
- "mov %%rax, %%rcx;"
-#endif
- "1:"
- ".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
+ if (initial)
+ asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
+ : "+S"(input), "+D"(output)
+ : "d"(control_word), "b"(key), "c"(initial));
+
+ asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
: "+S"(input), "+D"(output)
- : "d"(control_word), "b"(key), "c"(count)
- : "ax");
+ : "d"(control_word), "b"(key), "c"(count - initial));
}
static inline u8 *padlock_xcrypt_cbc(const u8 *input, u8 *output, void *key,
@@ -249,7 +253,7 @@ static void aes_encrypt(struct crypto_tf
padlock_reset_key(&ctx->cword.encrypt);
ts_state = irq_ts_save();
- aes_crypt(in, out, ctx->E, &ctx->cword.encrypt);
+ aes_crypt(in, out, ctx->E, &ctx->cword.encrypt, 1);
irq_ts_restore(ts_state);
padlock_store_cword(&ctx->cword.encrypt);
}
@@ -261,7 +265,7 @@ static void aes_decrypt(struct crypto_tf
padlock_reset_key(&ctx->cword.encrypt);
ts_state = irq_ts_save();
- aes_crypt(in, out, ctx->D, &ctx->cword.decrypt);
+ aes_crypt(in, out, ctx->D, &ctx->cword.decrypt, 1);
irq_ts_restore(ts_state);
padlock_store_cword(&ctx->cword.encrypt);
}
@@ -454,6 +458,7 @@ static struct crypto_alg cbc_aes_alg = {
static int __init padlock_init(void)
{
int ret;
+ struct cpuinfo_x86 *c = &cpu_data(0);
if (!cpu_has_xcrypt) {
printk(KERN_NOTICE PFX "VIA PadLock not detected.\n");
@@ -476,6 +481,12 @@ static int __init padlock_init(void)
printk(KERN_NOTICE PFX "Using VIA PadLock ACE for AES algorithm.\n");
+ if (c->x86 == 6 && c->x86_model == 15 && c->x86_mask == 2) {
+ ecb_fetch_blocks = 8;
+ cbc_fetch_blocks = 4; /* NOTE: notused */
+ printk(KERN_NOTICE PFX "VIA Nano stepping 2 detected: enabling workaround.\n");
+ }
+
out:
return ret;
From: Chuck Ebbert <[email protected]>
crypto: padlock-aes: work around Nano CPU errata in CBC mode
Extend previous workarounds for the prefetch bug to cover CBC mode,
clean up the code a bit.
Acked-by: HaraldWelte <[email protected]>
Signed-off-by: Chuck Ebbert <[email protected]>
---
Against cryptodev.git
--- work-2.6.29.4.orig/drivers/crypto/padlock-aes.c
+++ work-2.6.29.4/drivers/crypto/padlock-aes.c
@@ -22,11 +22,16 @@
#include <asm/i387.h>
#include "padlock.h"
-/* number of data blocks actually fetched for each xcrypt insn */
+/*
+ * Number of data blocks actually fetched for each xcrypt insn.
+ * Processors with prefetch errata will fetch extra blocks.
+ */
static unsigned int ecb_fetch_blocks = 2;
-static unsigned int cbc_fetch_blocks = 1;
-
+#define MAX_ECB_FETCH_BLOCKS (8)
#define ecb_fetch_bytes (ecb_fetch_blocks * AES_BLOCK_SIZE)
+
+static unsigned int cbc_fetch_blocks = 1;
+#define MAX_CBC_FETCH_BLOCKS (4)
#define cbc_fetch_bytes (cbc_fetch_blocks * AES_BLOCK_SIZE)
/* Control word. */
@@ -176,7 +181,7 @@ static inline void padlock_store_cword(s
* should be used only inside the irq_ts_save/restore() context
*/
-static inline void padlock_xcrypt(const u8 *input, u8 *output, void *key,
+static inline void rep_xcrypt_ecb(const u8 *input, u8 *output, void *key,
struct cword *control_word, int count)
{
asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
@@ -184,32 +189,65 @@ static inline void padlock_xcrypt(const
: "d"(control_word), "b"(key), "c"(count));
}
-static void aes_crypt_copy(const u8 *in, u8 *out, u32 *key,
+static inline u8 *rep_xcrypt_cbc(const u8 *input, u8 *output, void *key,
+ u8 *iv, struct cword *control_word, int count)
+{
+ asm volatile (".byte 0xf3,0x0f,0xa7,0xd0" /* rep xcryptcbc */
+ : "+S" (input), "+D" (output), "+a" (iv)
+ : "d" (control_word), "b" (key), "c" (count));
+ return iv;
+}
+
+static void ecb_crypt_copy(const u8 *in, u8 *out, u32 *key,
struct cword *cword, int count)
{
/*
* Padlock prefetches extra data so we must provide mapped input buffers.
* Assume there are at least 16 bytes of stack already in use.
*/
- u8 buf[AES_BLOCK_SIZE * 7 + PADLOCK_ALIGNMENT - 1];
+ u8 buf[AES_BLOCK_SIZE * (MAX_ECB_FETCH_BLOCKS - 1) + PADLOCK_ALIGNMENT - 1];
u8 *tmp = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);
memcpy(tmp, in, count * AES_BLOCK_SIZE);
- padlock_xcrypt(tmp, out, key, cword, count);
+ rep_xcrypt_ecb(tmp, out, key, cword, count);
}
-static inline void aes_crypt(const u8 *in, u8 *out, u32 *key,
+static u8 *cbc_crypt_copy(const u8 *in, u8 *out, u32 *key,
+ u8 *iv, struct cword *cword, int count)
+{
+ /*
+ * Padlock prefetches extra data so we must provide mapped input buffers.
+ * Assume there are at least 16 bytes of stack already in use.
+ */
+ u8 buf[AES_BLOCK_SIZE * (MAX_CBC_FETCH_BLOCKS - 1) + PADLOCK_ALIGNMENT - 1];
+ u8 *tmp = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);
+
+ memcpy(tmp, in, count * AES_BLOCK_SIZE);
+ return rep_xcrypt_cbc(tmp, out, key, iv, cword, count);
+}
+
+static inline void ecb_crypt(const u8 *in, u8 *out, u32 *key,
struct cword *cword, int count)
{
/* Padlock in ECB mode fetches at least ecb_fetch_bytes of data.
* We could avoid some copying here but it's probably not worth it.
*/
if (unlikely(((unsigned long)in & PAGE_SIZE) + ecb_fetch_bytes > PAGE_SIZE)) {
- aes_crypt_copy(in, out, key, cword, count);
+ ecb_crypt_copy(in, out, key, cword, count);
return;
}
- padlock_xcrypt(in, out, key, cword, count);
+ rep_xcrypt_ecb(in, out, key, cword, count);
+}
+
+static inline u8 *cbc_crypt(const u8 *in, u8 *out, u32 *key,
+ u8 *iv, struct cword *cword, int count)
+{
+ /* Padlock in CBC mode fetches at least cbc_fetch_bytes of data. */
+ if (unlikely(((unsigned long)in & PAGE_SIZE) + cbc_fetch_bytes > PAGE_SIZE))
+ return cbc_crypt_copy(in, out, key, iv, cword, count);
+
+ return rep_xcrypt_cbc(in, out, key, iv, cword, count);
}
static inline void padlock_xcrypt_ecb(const u8 *input, u8 *output, void *key,
@@ -218,7 +256,7 @@ static inline void padlock_xcrypt_ecb(co
u32 initial = count & (ecb_fetch_blocks - 1);
if (count < ecb_fetch_blocks) {
- aes_crypt(input, output, key, control_word, count);
+ ecb_crypt(input, output, key, control_word, count);
return;
}
@@ -235,10 +273,19 @@ static inline void padlock_xcrypt_ecb(co
static inline u8 *padlock_xcrypt_cbc(const u8 *input, u8 *output, void *key,
u8 *iv, void *control_word, u32 count)
{
- /* rep xcryptcbc */
- asm volatile (".byte 0xf3,0x0f,0xa7,0xd0"
+ u32 initial = count & (cbc_fetch_blocks - 1);
+
+ if (count < cbc_fetch_blocks)
+ return cbc_crypt(input, output, key, iv, control_word, count);
+
+ if (initial)
+ asm volatile (".byte 0xf3,0x0f,0xa7,0xd0" /* rep xcryptcbc */
+ : "+S" (input), "+D" (output), "+a" (iv)
+ : "d" (control_word), "b" (key), "c" (count));
+
+ asm volatile (".byte 0xf3,0x0f,0xa7,0xd0" /* rep xcryptcbc */
: "+S" (input), "+D" (output), "+a" (iv)
- : "d" (control_word), "b" (key), "c" (count));
+ : "d" (control_word), "b" (key), "c" (count-initial));
return iv;
}
@@ -249,7 +296,7 @@ static void aes_encrypt(struct crypto_tf
padlock_reset_key(&ctx->cword.encrypt);
ts_state = irq_ts_save();
- aes_crypt(in, out, ctx->E, &ctx->cword.encrypt, 1);
+ ecb_crypt(in, out, ctx->E, &ctx->cword.encrypt, 1);
irq_ts_restore(ts_state);
padlock_store_cword(&ctx->cword.encrypt);
}
@@ -261,7 +308,7 @@ static void aes_decrypt(struct crypto_tf
padlock_reset_key(&ctx->cword.encrypt);
ts_state = irq_ts_save();
- aes_crypt(in, out, ctx->D, &ctx->cword.decrypt, 1);
+ ecb_crypt(in, out, ctx->D, &ctx->cword.decrypt, 1);
irq_ts_restore(ts_state);
padlock_store_cword(&ctx->cword.encrypt);
}
@@ -478,8 +525,8 @@ static int __init padlock_init(void)
printk(KERN_NOTICE PFX "Using VIA PadLock ACE for AES algorithm.\n");
if (c->x86 == 6 && c->x86_model == 15 && c->x86_mask == 2) {
- ecb_fetch_blocks = 8;
- cbc_fetch_blocks = 4; /* NOTE: notused */
+ ecb_fetch_blocks = MAX_ECB_FETCH_BLOCKS;
+ cbc_fetch_blocks = MAX_CBC_FETCH_BLOCKS;
printk(KERN_NOTICE PFX "VIA Nano stepping 2 detected: enabling workaround.\n");
}
* Chuck Ebbert | 2009-06-10 15:11:22 [-0400]:
>From: Chuck Ebbert <[email protected]>
>crypto: padlock-aes: work around Nano CPU errata in ECB mode
>
>The VIA Nano processor has a bug that makes it prefetch extra data
>during encryption operations, causing spurious page faults. Extend
>existing workarounds for ECB mode to copy the data to an temporary
>buffer to avoid the problem.
>
>Acked-by: HaraldWelte <[email protected]>
>Signed-off-by: Chuck Ebbert <[email protected]>
>---
>Against cryptodev.git as requested.
thx for the rebase. Those two patches are a nice cleanup of the ifdef
mess I've made earlier.
Sebastian
Hi Herbert,
do you think we will still be able to get this Nano errata handling
into 2.6.31? I've noticed that crpyto-2.6 and cryptodev-2.6 also don't seem
to have those patches. Are there any issues? I would want to help in case
there is any to-be-resolved issues with this patchset.
Once we have it in mainline, I woul also ask about your opinion on whether
you think they're applicable for the -stable series.
Thanks in advance,
--
- Harald Welte <[email protected]> http://linux.via.com.tw/
============================================================================
VIA Free and Open Source Software Liaison
On Thu, Jun 18, 2009 at 11:23:09AM +0200, Harald Welte wrote:
> Hi Herbert,
>
> do you think we will still be able to get this Nano errata handling
> into 2.6.31? I've noticed that crpyto-2.6 and cryptodev-2.6 also don't seem
> to have those patches. Are there any issues? I would want to help in case
> there is any to-be-resolved issues with this patchset.
>
> Once we have it in mainline, I woul also ask about your opinion on whether
> you think they're applicable for the -stable series.
Don't worry, I'll send them off to crypto-2.6 followed by stable.
Cheers,
--
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <[email protected]>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
On Wed, Jun 10, 2009 at 03:11:22PM -0400, Chuck Ebbert wrote:
> From: Chuck Ebbert <[email protected]>
> crypto: padlock-aes: work around Nano CPU errata in ECB mode
>
> The VIA Nano processor has a bug that makes it prefetch extra data
> during encryption operations, causing spurious page faults. Extend
> existing workarounds for ECB mode to copy the data to an temporary
> buffer to avoid the problem.
>
> Acked-by: HaraldWelte <[email protected]>
> Signed-off-by: Chuck Ebbert <[email protected]>
All applied to crypt-2.6. Thanks a lot!
--
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <[email protected]>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt