Hi,
Here is a set of patches which provides fixes and improvements.
Based on Herbert feedback it also includes fixes so that calling final()
is not mandatory.
BR, Dmitry
Dmitry Kasatkin (8):
omap-sham: uses digest buffer in request context
omap-sham: DMA initialization fixes for off mode
omap-sham: error handling improved
omap-sham: removed redundunt locking
omap-sham: crypto_ahash_final() now not need to be called.
omap-sham: hash-in-progress is stored in hw format
omap-sham: FLAGS_FIRST is redundant and removed
omap-sham: zero-copy scatterlist handling
drivers/crypto/omap-sham.c | 379 +++++++++++++++++++++++++-------------------
1 files changed, 218 insertions(+), 161 deletions(-)
DMA parameters for constant data were initialized during driver probe().
It seems that those settings sometimes are lost when devices goes to off mode.
This patch makes DMA initialization just before use.
It solves off mode problems.
Fixes: NB#202786 - Aegis & SHA1 block off mode changes
Signed-off-by: Dmitry Kasatkin <[email protected]>
---
drivers/crypto/omap-sham.c | 19 ++++++++++---------
1 files changed, 10 insertions(+), 9 deletions(-)
diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c
index 2222370..9dfbc4a 100644
--- a/drivers/crypto/omap-sham.c
+++ b/drivers/crypto/omap-sham.c
@@ -318,6 +318,16 @@ static int omap_sham_xmit_dma(struct omap_sham_dev *dd, dma_addr_t dma_addr,
omap_set_dma_src_params(dd->dma_lch, 0, OMAP_DMA_AMODE_POST_INC,
dma_addr, 0, 0);
+ omap_set_dma_dest_params(dd->dma_lch, 0,
+ OMAP_DMA_AMODE_CONSTANT,
+ dd->phys_base + SHA_REG_DIN(0), 0, 16);
+
+ omap_set_dma_dest_burst_mode(dd->dma_lch,
+ OMAP_DMA_DATA_BURST_16);
+
+ omap_set_dma_src_burst_mode(dd->dma_lch,
+ OMAP_DMA_DATA_BURST_4);
+
err = omap_sham_write_ctrl(dd, length, final, 1);
if (err)
return err;
@@ -1071,15 +1081,6 @@ static int omap_sham_dma_init(struct omap_sham_dev *dd)
dev_err(dd->dev, "Unable to request DMA channel\n");
return err;
}
- omap_set_dma_dest_params(dd->dma_lch, 0,
- OMAP_DMA_AMODE_CONSTANT,
- dd->phys_base + SHA_REG_DIN(0), 0, 16);
-
- omap_set_dma_dest_burst_mode(dd->dma_lch,
- OMAP_DMA_DATA_BURST_16);
-
- omap_set_dma_src_burst_mode(dd->dma_lch,
- OMAP_DMA_DATA_BURST_4);
return 0;
}
--
1.7.0.4
Locking for queuing and dequeuing is combined.
test_and_set_bit() is also replaced with checking under dd->lock.
Signed-off-by: Dmitry Kasatkin <[email protected]>
---
drivers/crypto/omap-sham.c | 47 +++++++++++++++++++------------------------
1 files changed, 21 insertions(+), 26 deletions(-)
diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c
index db20628..6340c5e 100644
--- a/drivers/crypto/omap-sham.c
+++ b/drivers/crypto/omap-sham.c
@@ -84,9 +84,7 @@
#define FLAGS_CPU 0x0200
#define FLAGS_HMAC 0x0400
#define FLAGS_ERROR 0x0800
-
-/* 3rd byte */
-#define FLAGS_BUSY 16
+#define FLAGS_BUSY 0x1000
#define OP_UPDATE 1
#define OP_FINAL 2
@@ -629,32 +627,37 @@ static void omap_sham_finish_req(struct ahash_request *req, int err)
if ((ctx->flags & FLAGS_FINAL) || err)
omap_sham_cleanup(req);
- clear_bit(FLAGS_BUSY, &ctx->dd->flags);
+ ctx->dd->flags &= ~FLAGS_BUSY;
if (req->base.complete)
req->base.complete(&req->base, err);
}
-static int omap_sham_handle_queue(struct omap_sham_dev *dd)
+static int omap_sham_handle_queue(struct omap_sham_dev *dd,
+ struct ahash_request *req)
{
struct crypto_async_request *async_req, *backlog;
struct omap_sham_reqctx *ctx;
- struct ahash_request *req, *prev_req;
+ struct ahash_request *prev_req;
unsigned long flags;
- int err = 0;
-
- if (test_and_set_bit(FLAGS_BUSY, &dd->flags))
- return 0;
+ int err = 0, ret = 0;
spin_lock_irqsave(&dd->lock, flags);
- backlog = crypto_get_backlog(&dd->queue);
+ if (req)
+ ret = ahash_enqueue_request(&dd->queue, req);
+ if (dd->flags & FLAGS_BUSY) {
+ spin_unlock_irqrestore(&dd->lock, flags);
+ return ret;
+ }
async_req = crypto_dequeue_request(&dd->queue);
- if (!async_req)
- clear_bit(FLAGS_BUSY, &dd->flags);
+ if (async_req) {
+ dd->flags |= FLAGS_BUSY;
+ backlog = crypto_get_backlog(&dd->queue);
+ }
spin_unlock_irqrestore(&dd->lock, flags);
if (!async_req)
- return 0;
+ return ret;
if (backlog)
backlog->complete(backlog, -EINPROGRESS);
@@ -690,7 +693,7 @@ static int omap_sham_handle_queue(struct omap_sham_dev *dd)
dev_dbg(dd->dev, "exit, err: %d\n", err);
- return err;
+ return ret;
}
static int omap_sham_enqueue(struct ahash_request *req, unsigned int op)
@@ -698,18 +701,10 @@ static int omap_sham_enqueue(struct ahash_request *req, unsigned int op)
struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
struct omap_sham_dev *dd = tctx->dd;
- unsigned long flags;
- int err;
ctx->op = op;
- spin_lock_irqsave(&dd->lock, flags);
- err = ahash_enqueue_request(&dd->queue, req);
- spin_unlock_irqrestore(&dd->lock, flags);
-
- omap_sham_handle_queue(dd);
-
- return err;
+ return omap_sham_handle_queue(dd, req);
}
static int omap_sham_update(struct ahash_request *req)
@@ -1041,7 +1036,7 @@ static void omap_sham_done_task(unsigned long data)
/* finish curent request */
omap_sham_finish_req(req, err);
/* start new request */
- omap_sham_handle_queue(dd);
+ omap_sham_handle_queue(dd, NULL);
}
}
@@ -1049,7 +1044,7 @@ static void omap_sham_queue_task(unsigned long data)
{
struct omap_sham_dev *dd = (struct omap_sham_dev *)data;
- omap_sham_handle_queue(dd);
+ omap_sham_handle_queue(dd, NULL);
}
static irqreturn_t omap_sham_irq(int irq, void *dev_id)
--
1.7.0.4
Introduces DMA error handling.
DMA error is returned as a result code of the hash request.
Clients needs to handle error codes and may repeat hash calculation attempt.
Also in the case of DMA error, SHAM module is set to be re-initialized again.
It significantly improves stability against possible HW failures.
Signed-off-by: Dmitry Kasatkin <[email protected]>
---
drivers/crypto/omap-sham.c | 67 +++++++++++++++++++++++++++++---------------
1 files changed, 44 insertions(+), 23 deletions(-)
diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c
index 9dfbc4a..db20628 100644
--- a/drivers/crypto/omap-sham.c
+++ b/drivers/crypto/omap-sham.c
@@ -83,6 +83,7 @@
#define FLAGS_INIT 0x0100
#define FLAGS_CPU 0x0200
#define FLAGS_HMAC 0x0400
+#define FLAGS_ERROR 0x0800
/* 3rd byte */
#define FLAGS_BUSY 16
@@ -137,6 +138,7 @@ struct omap_sham_dev {
int irq;
struct clk *iclk;
spinlock_t lock;
+ int err;
int dma;
int dma_lch;
struct tasklet_struct done_task;
@@ -234,10 +236,12 @@ static int omap_sham_write_ctrl(struct omap_sham_dev *dd, size_t length,
SHA_REG_MASK_SOFTRESET, SHA_REG_MASK_SOFTRESET);
if (omap_sham_wait(dd, SHA_REG_SYSSTATUS,
- SHA_REG_SYSSTATUS_RESETDONE))
+ SHA_REG_SYSSTATUS_RESETDONE)) {
+ clk_disable(dd->iclk);
return -ETIMEDOUT;
-
+ }
dd->flags |= FLAGS_INIT;
+ dd->err = 0;
}
} else {
omap_sham_write(dd, SHA_REG_DIGCNT, ctx->digcnt);
@@ -279,11 +283,12 @@ static int omap_sham_xmit_cpu(struct omap_sham_dev *dd, const u8 *buf,
if (err)
return err;
+ /* should be non-zero before next lines to disable clocks later */
+ ctx->digcnt += length;
+
if (omap_sham_wait(dd, SHA_REG_CTRL, SHA_REG_CTRL_INPUT_READY))
return -ETIMEDOUT;
- ctx->digcnt += length;
-
if (final)
ctx->flags |= FLAGS_FINAL; /* catch last interrupt */
@@ -303,7 +308,6 @@ static int omap_sham_xmit_dma(struct omap_sham_dev *dd, dma_addr_t dma_addr,
dev_dbg(dd->dev, "xmit_dma: digcnt: %d, length: %d, final: %d\n",
ctx->digcnt, length, final);
-
/* flush cache entries related to our page */
if (dma_addr == ctx->dma_addr)
dma_sync_single_for_device(dd->dev, dma_addr, length,
@@ -411,6 +415,7 @@ static int omap_sham_update_dma_fast(struct omap_sham_dev *dd)
{
struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
unsigned int length;
+ int err;
ctx->flags |= FLAGS_FAST;
@@ -424,7 +429,11 @@ static int omap_sham_update_dma_fast(struct omap_sham_dev *dd)
ctx->total -= length;
- return omap_sham_xmit_dma(dd, sg_dma_address(ctx->sg), length, 1);
+ err = omap_sham_xmit_dma(dd, sg_dma_address(ctx->sg), length, 1);
+ if (err != -EINPROGRESS)
+ dma_unmap_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE);
+
+ return err;
}
static int omap_sham_update_cpu(struct omap_sham_dev *dd)
@@ -580,9 +589,6 @@ static int omap_sham_final_req(struct omap_sham_dev *dd)
ctx->bufcnt = 0;
- if (err != -EINPROGRESS)
- omap_sham_cleanup(req);
-
dev_dbg(dd->dev, "final_req: err: %d\n", err);
return err;
@@ -616,9 +622,11 @@ static void omap_sham_finish_req(struct ahash_request *req, int err)
omap_sham_copy_hash(ctx->dd->req, 1);
if (ctx->flags & FLAGS_HMAC)
err = omap_sham_finish_req_hmac(req);
+ } else {
+ ctx->flags |= FLAGS_ERROR;
}
- if (ctx->flags & FLAGS_FINAL)
+ if ((ctx->flags & FLAGS_FINAL) || err)
omap_sham_cleanup(req);
clear_bit(FLAGS_BUSY, &ctx->dd->flags);
@@ -776,12 +784,14 @@ static int omap_sham_final(struct ahash_request *req)
ctx->flags |= FLAGS_FINUP;
- /* OMAP HW accel works only with buffers >= 9 */
- /* HMAC is always >= 9 because of ipad */
- if ((ctx->digcnt + ctx->bufcnt) < 9)
- err = omap_sham_final_shash(req);
- else if (ctx->bufcnt)
- return omap_sham_enqueue(req, OP_FINAL);
+ if (!(ctx->flags & FLAGS_ERROR)) {
+ /* OMAP HW accel works only with buffers >= 9 */
+ /* HMAC is always >= 9 because of ipad */
+ if ((ctx->digcnt + ctx->bufcnt) < 9)
+ err = omap_sham_final_shash(req);
+ else if (ctx->bufcnt)
+ return omap_sham_enqueue(req, OP_FINAL);
+ }
omap_sham_cleanup(req);
@@ -851,6 +861,8 @@ static int omap_sham_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base)
struct omap_sham_ctx *tctx = crypto_tfm_ctx(tfm);
const char *alg_name = crypto_tfm_alg_name(tfm);
+ pr_info("enter\n");
+
/* Allocate a fallback and abort if it failed. */
tctx->fallback = crypto_alloc_shash(alg_name, 0,
CRYPTO_ALG_NEED_FALLBACK);
@@ -1008,7 +1020,7 @@ static void omap_sham_done_task(unsigned long data)
struct omap_sham_dev *dd = (struct omap_sham_dev *)data;
struct ahash_request *req = dd->req;
struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
- int ready = 1;
+ int ready = 0, err = 0;
if (ctx->flags & FLAGS_OUTPUT_READY) {
ctx->flags &= ~FLAGS_OUTPUT_READY;
@@ -1018,13 +1030,16 @@ static void omap_sham_done_task(unsigned long data)
if (dd->flags & FLAGS_DMA_ACTIVE) {
dd->flags &= ~FLAGS_DMA_ACTIVE;
omap_sham_update_dma_stop(dd);
- omap_sham_update_dma_slow(dd);
+ if (!dd->err)
+ err = omap_sham_update_dma_slow(dd);
}
- if (ready && !(dd->flags & FLAGS_DMA_ACTIVE)) {
- dev_dbg(dd->dev, "update done\n");
+ err = dd->err ? : err;
+
+ if (err != -EINPROGRESS && (ready || err)) {
+ dev_dbg(dd->dev, "update done: err: %d\n", err);
/* finish curent request */
- omap_sham_finish_req(req, 0);
+ omap_sham_finish_req(req, err);
/* start new request */
omap_sham_handle_queue(dd);
}
@@ -1056,6 +1071,7 @@ static irqreturn_t omap_sham_irq(int irq, void *dev_id)
omap_sham_read(dd, SHA_REG_CTRL);
ctx->flags |= FLAGS_OUTPUT_READY;
+ dd->err = 0;
tasklet_schedule(&dd->done_task);
return IRQ_HANDLED;
@@ -1065,8 +1081,13 @@ static void omap_sham_dma_callback(int lch, u16 ch_status, void *data)
{
struct omap_sham_dev *dd = data;
- if (likely(lch == dd->dma_lch))
- tasklet_schedule(&dd->done_task);
+ if (ch_status != OMAP_DMA_BLOCK_IRQ) {
+ pr_err("omap-sham DMA error status: 0x%hx\n", ch_status);
+ dd->err = -EIO;
+ dd->flags &= ~FLAGS_INIT; /* request to re-initialize */
+ }
+
+ tasklet_schedule(&dd->done_task);
}
static int omap_sham_dma_init(struct omap_sham_dev *dd)
--
1.7.0.4
Hash-in-progress is now stored in hw format.
Only on final call, hash is converted to correct format.
Speedup copy procedure and will allow to use OMAP burst mode.
Signed-off-by: Dmitry Kasatkin <[email protected]>
---
drivers/crypto/omap-sham.c | 38 ++++++++++++++++++++++++--------------
1 files changed, 24 insertions(+), 14 deletions(-)
diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c
index 85d6277..c8d30eb 100644
--- a/drivers/crypto/omap-sham.c
+++ b/drivers/crypto/omap-sham.c
@@ -204,24 +204,35 @@ static void omap_sham_copy_hash(struct ahash_request *req, int out)
u32 *hash = (u32 *)ctx->digest;
int i;
+ /* MD5 is almost unused. So copy sha1 size to reduce code */
+ for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(u32); i++) {
+ if (out)
+ hash[i] = omap_sham_read(ctx->dd,
+ SHA_REG_DIGEST(i));
+ else
+ omap_sham_write(ctx->dd,
+ SHA_REG_DIGEST(i), hash[i]);
+ }
+}
+
+static void omap_sham_copy_ready_hash(struct ahash_request *req)
+{
+ struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
+ u32 *in = (u32 *)ctx->digest;
+ u32 *hash = (u32 *)req->result;
+ int i;
+
+ if (!hash)
+ return;
+
if (likely(ctx->flags & FLAGS_SHA1)) {
/* SHA1 results are in big endian */
for (i = 0; i < SHA1_DIGEST_SIZE / sizeof(u32); i++)
- if (out)
- hash[i] = be32_to_cpu(omap_sham_read(ctx->dd,
- SHA_REG_DIGEST(i)));
- else
- omap_sham_write(ctx->dd, SHA_REG_DIGEST(i),
- cpu_to_be32(hash[i]));
+ hash[i] = be32_to_cpu(in[i]);
} else {
/* MD5 results are in little endian */
for (i = 0; i < MD5_DIGEST_SIZE / sizeof(u32); i++)
- if (out)
- hash[i] = le32_to_cpu(omap_sham_read(ctx->dd,
- SHA_REG_DIGEST(i)));
- else
- omap_sham_write(ctx->dd, SHA_REG_DIGEST(i),
- cpu_to_le32(hash[i]));
+ hash[i] = le32_to_cpu(in[i]);
}
}
@@ -474,8 +485,7 @@ static void omap_sham_cleanup(struct ahash_request *req)
spin_unlock_irqrestore(&dd->lock, flags);
if (ctx->digcnt)
- memcpy(req->result, ctx->digest, (ctx->flags & FLAGS_SHA1) ?
- SHA1_DIGEST_SIZE : MD5_DIGEST_SIZE);
+ omap_sham_copy_ready_hash(req);
dev_dbg(dd->dev, "digcnt: %d, bufcnt: %d\n", ctx->digcnt, ctx->bufcnt);
}
--
1.7.0.4
Currently driver storred digest results in req->results
provided by the client. But some clients do not set it
until final() call. It leads to crash.
Changed to use internal buffer to store temporary digest results.
Signed-off-by: Dmitry Kasatkin <[email protected]>
---
drivers/crypto/omap-sham.c | 11 ++++++++---
1 files changed, 8 insertions(+), 3 deletions(-)
diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c
index a081c7c..2222370 100644
--- a/drivers/crypto/omap-sham.c
+++ b/drivers/crypto/omap-sham.c
@@ -97,6 +97,7 @@ struct omap_sham_reqctx {
unsigned long flags;
unsigned long op;
+ u8 digest[SHA1_DIGEST_SIZE];
size_t digcnt;
u8 *buffer;
size_t bufcnt;
@@ -194,7 +195,7 @@ static inline int omap_sham_wait(struct omap_sham_dev *dd, u32 offset, u32 bit)
static void omap_sham_copy_hash(struct ahash_request *req, int out)
{
struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
- u32 *hash = (u32 *)req->result;
+ u32 *hash = (u32 *)ctx->digest;
int i;
if (likely(ctx->flags & FLAGS_SHA1)) {
@@ -453,8 +454,11 @@ static void omap_sham_cleanup(struct ahash_request *req)
ctx->flags |= FLAGS_CLEAN;
spin_unlock_irqrestore(&dd->lock, flags);
- if (ctx->digcnt)
+ if (ctx->digcnt) {
clk_disable(dd->iclk);
+ memcpy(req->result, ctx->digest, (ctx->flags & FLAGS_SHA1) ?
+ SHA1_DIGEST_SIZE : MD5_DIGEST_SIZE);
+ }
if (ctx->dma_addr)
dma_unmap_single(dd->dev, ctx->dma_addr, ctx->buflen,
@@ -576,6 +580,7 @@ static int omap_sham_final_req(struct omap_sham_dev *dd)
static int omap_sham_finish_req_hmac(struct ahash_request *req)
{
+ struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm);
struct omap_sham_hmac_ctx *bctx = tctx->base;
int bs = crypto_shash_blocksize(bctx->shash);
@@ -590,7 +595,7 @@ static int omap_sham_finish_req_hmac(struct ahash_request *req)
return crypto_shash_init(&desc.shash) ?:
crypto_shash_update(&desc.shash, bctx->opad, bs) ?:
- crypto_shash_finup(&desc.shash, req->result, ds, req->result);
+ crypto_shash_finup(&desc.shash, ctx->digest, ds, ctx->digest);
}
static void omap_sham_finish_req(struct ahash_request *req, int err)
--
1.7.0.4
If scatterlist have more than one entry, current driver uses
aligned buffer to copy data to to accelerator to tackle possible
issues with DMA and SHA buffer alignment.
This commit adds more intelligence to verify SG alignment and
possibility to use DMA directly on the data without using copy
buffer.
Signed-off-by: Dmitry Kasatkin <[email protected]>
---
drivers/crypto/omap-sham.c | 87 +++++++++++++++++++++++++++++++-------------
1 files changed, 61 insertions(+), 26 deletions(-)
diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c
index d88d7eb..eb988e7 100644
--- a/drivers/crypto/omap-sham.c
+++ b/drivers/crypto/omap-sham.c
@@ -74,7 +74,7 @@
#define FLAGS_FINUP 0x0002
#define FLAGS_FINAL 0x0004
-#define FLAGS_FAST 0x0008
+#define FLAGS_SG 0x0008
#define FLAGS_SHA1 0x0010
#define FLAGS_DMA_ACTIVE 0x0020
#define FLAGS_OUTPUT_READY 0x0040
@@ -393,6 +393,8 @@ static int omap_sham_xmit_dma_map(struct omap_sham_dev *dd,
return -EINVAL;
}
+ ctx->flags &= ~FLAGS_SG;
+
/* next call does not fail... so no unmap in the case of error */
return omap_sham_xmit_dma(dd, ctx->dma_addr, length, final);
}
@@ -403,9 +405,6 @@ static int omap_sham_update_dma_slow(struct omap_sham_dev *dd)
unsigned int final;
size_t count;
- if (!ctx->total)
- return 0;
-
omap_sham_append_sg(ctx);
final = (ctx->flags & FLAGS_FINUP) && !ctx->total;
@@ -422,25 +421,62 @@ static int omap_sham_update_dma_slow(struct omap_sham_dev *dd)
return 0;
}
-static int omap_sham_update_dma_fast(struct omap_sham_dev *dd)
+/* Start address alignment */
+#define SG_AA(sg) (IS_ALIGNED(sg->offset, sizeof(u32)))
+/* SHA1 block size alignment */
+#define SG_SA(sg) (IS_ALIGNED(sg->length, SHA1_MD5_BLOCK_SIZE))
+
+static int omap_sham_update_dma_start(struct omap_sham_dev *dd)
{
struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
- unsigned int length;
+ unsigned int length, final, tail;
+ struct scatterlist *sg;
+
+ if (!ctx->total)
+ return 0;
+
+ if (ctx->bufcnt || ctx->offset)
+ return omap_sham_update_dma_slow(dd);
+
+ dev_dbg(dd->dev, "fast: digcnt: %d, bufcnt: %u, total: %u\n",
+ ctx->digcnt, ctx->bufcnt, ctx->total);
+
+ sg = ctx->sg;
- ctx->flags |= FLAGS_FAST;
+ if (!SG_AA(sg))
+ return omap_sham_update_dma_slow(dd);
- length = min(ctx->total, sg_dma_len(ctx->sg));
- ctx->total = length;
+ if (!sg_is_last(sg) && !SG_SA(sg))
+ /* size is not SHA1_BLOCK_SIZE aligned */
+ return omap_sham_update_dma_slow(dd);
+
+ length = min(ctx->total, sg->length);
+
+ if (sg_is_last(sg)) {
+ if (!(ctx->flags & FLAGS_FINUP)) {
+ /* not last sg must be SHA1_MD5_BLOCK_SIZE aligned */
+ tail = length & (SHA1_MD5_BLOCK_SIZE - 1);
+ /* without finup() we need one block to close hash */
+ if (!tail)
+ tail = SHA1_MD5_BLOCK_SIZE;
+ length -= tail;
+ }
+ }
if (!dma_map_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE)) {
dev_err(dd->dev, "dma_map_sg error\n");
return -EINVAL;
}
+ ctx->flags |= FLAGS_SG;
+
ctx->total -= length;
+ ctx->offset = length; /* offset where to start slow */
+
+ final = (ctx->flags & FLAGS_FINUP) && !ctx->total;
/* next call does not fail... so no unmap in the case of error */
- return omap_sham_xmit_dma(dd, sg_dma_address(ctx->sg), length, 1);
+ return omap_sham_xmit_dma(dd, sg_dma_address(ctx->sg), length, final);
}
static int omap_sham_update_cpu(struct omap_sham_dev *dd)
@@ -460,11 +496,17 @@ static int omap_sham_update_dma_stop(struct omap_sham_dev *dd)
struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
omap_stop_dma(dd->dma_lch);
- if (ctx->flags & FLAGS_FAST)
+ if (ctx->flags & FLAGS_SG) {
dma_unmap_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE);
- else
+ if (ctx->sg->length == ctx->offset) {
+ ctx->sg = sg_next(ctx->sg);
+ if (ctx->sg)
+ ctx->offset = 0;
+ }
+ } else {
dma_unmap_single(dd->dev, ctx->dma_addr, ctx->buflen,
DMA_TO_DEVICE);
+ }
return 0;
}
@@ -545,10 +587,8 @@ static int omap_sham_update_req(struct omap_sham_dev *dd)
if (ctx->flags & FLAGS_CPU)
err = omap_sham_update_cpu(dd);
- else if (ctx->flags & FLAGS_FAST)
- err = omap_sham_update_dma_fast(dd);
else
- err = omap_sham_update_dma_slow(dd);
+ err = omap_sham_update_dma_start(dd);
/* wait for dma completion before can take more data */
dev_dbg(dd->dev, "update: err: %d, digcnt: %d\n", err, ctx->digcnt);
@@ -730,18 +770,13 @@ static int omap_sham_update(struct ahash_request *req)
*/
omap_sham_append_sg(ctx);
return 0;
- } else if (ctx->bufcnt + ctx->total <= 64) {
+ } else if (ctx->bufcnt + ctx->total <= SHA1_MD5_BLOCK_SIZE) {
+ /*
+ * faster to use CPU for short transfers
+ */
ctx->flags |= FLAGS_CPU;
- } else if (!ctx->bufcnt && sg_is_last(ctx->sg)) {
- /* may be can use faster functions */
- int aligned = IS_ALIGNED((u32)ctx->sg->offset,
- sizeof(u32));
- if (aligned)
- /* digest: first and final */
- ctx->flags |= FLAGS_FAST;
}
- } else if (ctx->bufcnt + ctx->total <= ctx->buflen) {
- /* if not finaup -> not fast */
+ } else if (ctx->bufcnt + ctx->total < ctx->buflen) {
omap_sham_append_sg(ctx);
return 0;
}
@@ -1026,7 +1061,7 @@ static void omap_sham_done_task(unsigned long data)
dd->flags &= ~FLAGS_DMA_ACTIVE;
omap_sham_update_dma_stop(dd);
if (!dd->err)
- err = omap_sham_update_dma_slow(dd);
+ err = omap_sham_update_dma_start(dd);
}
err = dd->err ? : err;
--
1.7.0.4
According to the Herbert Xu, client may not always call
crypto_ahash_final().
In the case of error in hash calculation resources will be
automatically cleaned up.
But if no hash calculation error happens and client will not call
crypto_ahash_final() at all, then internal buffer will not be freed,
and clocks will not be disabled.
This patch provides support for atomic crypto_ahash_update() call.
Clocks are now enabled and disabled per update request.
Data buffer is now allocated as a part of request context.
Client is obligated to free it with crypto_free_ahash().
Signed-off-by: Dmitry Kasatkin <[email protected]>
---
drivers/crypto/omap-sham.c | 168 +++++++++++++++++++++----------------------
1 files changed, 82 insertions(+), 86 deletions(-)
diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c
index 6340c5e..85d6277 100644
--- a/drivers/crypto/omap-sham.c
+++ b/drivers/crypto/omap-sham.c
@@ -89,6 +89,11 @@
#define OP_UPDATE 1
#define OP_FINAL 2
+#define OMAP_ALIGN_MASK (sizeof(u32)-1)
+#define OMAP_ALIGNED __attribute__((aligned(sizeof(u32))))
+
+#define BUFLEN PAGE_SIZE
+
struct omap_sham_dev;
struct omap_sham_reqctx {
@@ -96,9 +101,8 @@ struct omap_sham_reqctx {
unsigned long flags;
unsigned long op;
- u8 digest[SHA1_DIGEST_SIZE];
+ u8 digest[SHA1_DIGEST_SIZE] OMAP_ALIGNED;
size_t digcnt;
- u8 *buffer;
size_t bufcnt;
size_t buflen;
dma_addr_t dma_addr;
@@ -107,6 +111,8 @@ struct omap_sham_reqctx {
struct scatterlist *sg;
unsigned int offset; /* offset in current sg */
unsigned int total; /* total request */
+
+ u8 buffer[0] OMAP_ALIGNED;
};
struct omap_sham_hmac_ctx {
@@ -219,31 +225,33 @@ static void omap_sham_copy_hash(struct ahash_request *req, int out)
}
}
-static int omap_sham_write_ctrl(struct omap_sham_dev *dd, size_t length,
- int final, int dma)
+static int omap_sham_hw_init(struct omap_sham_dev *dd)
{
- struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
- u32 val = length << 5, mask;
+ clk_enable(dd->iclk);
- if (unlikely(!ctx->digcnt)) {
+ if (!(dd->flags & FLAGS_INIT)) {
+ omap_sham_write_mask(dd, SHA_REG_MASK,
+ SHA_REG_MASK_SOFTRESET, SHA_REG_MASK_SOFTRESET);
- clk_enable(dd->iclk);
+ if (omap_sham_wait(dd, SHA_REG_SYSSTATUS,
+ SHA_REG_SYSSTATUS_RESETDONE))
+ return -ETIMEDOUT;
- if (!(dd->flags & FLAGS_INIT)) {
- omap_sham_write_mask(dd, SHA_REG_MASK,
- SHA_REG_MASK_SOFTRESET, SHA_REG_MASK_SOFTRESET);
+ dd->flags |= FLAGS_INIT;
+ dd->err = 0;
+ }
- if (omap_sham_wait(dd, SHA_REG_SYSSTATUS,
- SHA_REG_SYSSTATUS_RESETDONE)) {
- clk_disable(dd->iclk);
- return -ETIMEDOUT;
- }
- dd->flags |= FLAGS_INIT;
- dd->err = 0;
- }
- } else {
+ return 0;
+}
+
+static void omap_sham_write_ctrl(struct omap_sham_dev *dd, size_t length,
+ int final, int dma)
+{
+ struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
+ u32 val = length << 5, mask;
+
+ if (likely(ctx->digcnt))
omap_sham_write(dd, SHA_REG_DIGCNT, ctx->digcnt);
- }
omap_sham_write_mask(dd, SHA_REG_MASK,
SHA_REG_MASK_IT_EN | (dma ? SHA_REG_MASK_DMA_EN : 0),
@@ -263,23 +271,19 @@ static int omap_sham_write_ctrl(struct omap_sham_dev *dd, size_t length,
SHA_REG_CTRL_ALGO | SHA_REG_CTRL_LENGTH;
omap_sham_write_mask(dd, SHA_REG_CTRL, val, mask);
-
- return 0;
}
static int omap_sham_xmit_cpu(struct omap_sham_dev *dd, const u8 *buf,
size_t length, int final)
{
struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
- int err, count, len32;
+ int count, len32;
const u32 *buffer = (const u32 *)buf;
dev_dbg(dd->dev, "xmit_cpu: digcnt: %d, length: %d, final: %d\n",
ctx->digcnt, length, final);
- err = omap_sham_write_ctrl(dd, length, final, 0);
- if (err)
- return err;
+ omap_sham_write_ctrl(dd, length, final, 0);
/* should be non-zero before next lines to disable clocks later */
ctx->digcnt += length;
@@ -302,14 +306,10 @@ static int omap_sham_xmit_dma(struct omap_sham_dev *dd, dma_addr_t dma_addr,
size_t length, int final)
{
struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
- int err, len32;
+ int len32;
dev_dbg(dd->dev, "xmit_dma: digcnt: %d, length: %d, final: %d\n",
ctx->digcnt, length, final);
- /* flush cache entries related to our page */
- if (dma_addr == ctx->dma_addr)
- dma_sync_single_for_device(dd->dev, dma_addr, length,
- DMA_TO_DEVICE);
len32 = DIV_ROUND_UP(length, sizeof(u32));
@@ -320,19 +320,7 @@ static int omap_sham_xmit_dma(struct omap_sham_dev *dd, dma_addr_t dma_addr,
omap_set_dma_src_params(dd->dma_lch, 0, OMAP_DMA_AMODE_POST_INC,
dma_addr, 0, 0);
- omap_set_dma_dest_params(dd->dma_lch, 0,
- OMAP_DMA_AMODE_CONSTANT,
- dd->phys_base + SHA_REG_DIN(0), 0, 16);
-
- omap_set_dma_dest_burst_mode(dd->dma_lch,
- OMAP_DMA_DATA_BURST_16);
-
- omap_set_dma_src_burst_mode(dd->dma_lch,
- OMAP_DMA_DATA_BURST_4);
-
- err = omap_sham_write_ctrl(dd, length, final, 1);
- if (err)
- return err;
+ omap_sham_write_ctrl(dd, length, final, 1);
ctx->digcnt += length;
@@ -384,6 +372,21 @@ static size_t omap_sham_append_sg(struct omap_sham_reqctx *ctx)
return 0;
}
+static int omap_sham_xmit_dma_map(struct omap_sham_dev *dd,
+ struct omap_sham_reqctx *ctx,
+ size_t length, int final)
+{
+ ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer, ctx->buflen,
+ DMA_TO_DEVICE);
+ if (dma_mapping_error(dd->dev, ctx->dma_addr)) {
+ dev_err(dd->dev, "dma %u bytes error\n", ctx->buflen);
+ return -EINVAL;
+ }
+
+ /* next call does not fail... so no unmap in the case of error */
+ return omap_sham_xmit_dma(dd, ctx->dma_addr, length, final);
+}
+
static int omap_sham_update_dma_slow(struct omap_sham_dev *dd)
{
struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
@@ -403,7 +406,7 @@ static int omap_sham_update_dma_slow(struct omap_sham_dev *dd)
if (final || (ctx->bufcnt == ctx->buflen && ctx->total)) {
count = ctx->bufcnt;
ctx->bufcnt = 0;
- return omap_sham_xmit_dma(dd, ctx->dma_addr, count, final);
+ return omap_sham_xmit_dma_map(dd, ctx, count, final);
}
return 0;
@@ -413,7 +416,6 @@ static int omap_sham_update_dma_fast(struct omap_sham_dev *dd)
{
struct omap_sham_reqctx *ctx = ahash_request_ctx(dd->req);
unsigned int length;
- int err;
ctx->flags |= FLAGS_FAST;
@@ -427,11 +429,8 @@ static int omap_sham_update_dma_fast(struct omap_sham_dev *dd)
ctx->total -= length;
- err = omap_sham_xmit_dma(dd, sg_dma_address(ctx->sg), length, 1);
- if (err != -EINPROGRESS)
- dma_unmap_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE);
-
- return err;
+ /* next call does not fail... so no unmap in the case of error */
+ return omap_sham_xmit_dma(dd, sg_dma_address(ctx->sg), length, 1);
}
static int omap_sham_update_cpu(struct omap_sham_dev *dd)
@@ -453,6 +452,9 @@ static int omap_sham_update_dma_stop(struct omap_sham_dev *dd)
omap_stop_dma(dd->dma_lch);
if (ctx->flags & FLAGS_FAST)
dma_unmap_sg(dd->dev, ctx->sg, 1, DMA_TO_DEVICE);
+ else
+ dma_unmap_single(dd->dev, ctx->dma_addr, ctx->buflen,
+ DMA_TO_DEVICE);
return 0;
}
@@ -471,18 +473,9 @@ static void omap_sham_cleanup(struct ahash_request *req)
ctx->flags |= FLAGS_CLEAN;
spin_unlock_irqrestore(&dd->lock, flags);
- if (ctx->digcnt) {
- clk_disable(dd->iclk);
+ if (ctx->digcnt)
memcpy(req->result, ctx->digest, (ctx->flags & FLAGS_SHA1) ?
SHA1_DIGEST_SIZE : MD5_DIGEST_SIZE);
- }
-
- if (ctx->dma_addr)
- dma_unmap_single(dd->dev, ctx->dma_addr, ctx->buflen,
- DMA_TO_DEVICE);
-
- if (ctx->buffer)
- free_page((unsigned long)ctx->buffer);
dev_dbg(dd->dev, "digcnt: %d, bufcnt: %d\n", ctx->digcnt, ctx->bufcnt);
}
@@ -520,21 +513,7 @@ static int omap_sham_init(struct ahash_request *req)
ctx->bufcnt = 0;
ctx->digcnt = 0;
-
- ctx->buflen = PAGE_SIZE;
- ctx->buffer = (void *)__get_free_page(
- (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ?
- GFP_KERNEL : GFP_ATOMIC);
- if (!ctx->buffer)
- return -ENOMEM;
-
- ctx->dma_addr = dma_map_single(dd->dev, ctx->buffer, ctx->buflen,
- DMA_TO_DEVICE);
- if (dma_mapping_error(dd->dev, ctx->dma_addr)) {
- dev_err(dd->dev, "dma %u bytes error\n", ctx->buflen);
- free_page((unsigned long)ctx->buffer);
- return -EINVAL;
- }
+ ctx->buflen = BUFLEN;
if (tctx->flags & FLAGS_HMAC) {
struct omap_sham_hmac_ctx *bctx = tctx->base;
@@ -581,7 +560,7 @@ static int omap_sham_final_req(struct omap_sham_dev *dd)
use_dma = 0;
if (use_dma)
- err = omap_sham_xmit_dma(dd, ctx->dma_addr, ctx->bufcnt, 1);
+ err = omap_sham_xmit_dma_map(dd, ctx, ctx->bufcnt, 1);
else
err = omap_sham_xmit_cpu(dd, ctx->buffer, ctx->bufcnt, 1);
@@ -615,6 +594,7 @@ static int omap_sham_finish_req_hmac(struct ahash_request *req)
static void omap_sham_finish_req(struct ahash_request *req, int err)
{
struct omap_sham_reqctx *ctx = ahash_request_ctx(req);
+ struct omap_sham_dev *dd = ctx->dd;
if (!err) {
omap_sham_copy_hash(ctx->dd->req, 1);
@@ -627,7 +607,8 @@ static void omap_sham_finish_req(struct ahash_request *req, int err)
if ((ctx->flags & FLAGS_FINAL) || err)
omap_sham_cleanup(req);
- ctx->dd->flags &= ~FLAGS_BUSY;
+ clk_disable(dd->iclk);
+ dd->flags &= ~FLAGS_BUSY;
if (req->base.complete)
req->base.complete(&req->base, err);
@@ -636,7 +617,7 @@ static void omap_sham_finish_req(struct ahash_request *req, int err)
static int omap_sham_handle_queue(struct omap_sham_dev *dd,
struct ahash_request *req)
{
- struct crypto_async_request *async_req, *backlog;
+ struct crypto_async_request *async_req, *backlog = 0;
struct omap_sham_reqctx *ctx;
struct ahash_request *prev_req;
unsigned long flags;
@@ -672,7 +653,22 @@ static int omap_sham_handle_queue(struct omap_sham_dev *dd,
dev_dbg(dd->dev, "handling new req, op: %lu, nbytes: %d\n",
ctx->op, req->nbytes);
- if (req != prev_req && ctx->digcnt)
+
+ err = omap_sham_hw_init(dd);
+ if (err)
+ goto err1;
+
+ omap_set_dma_dest_params(dd->dma_lch, 0,
+ OMAP_DMA_AMODE_CONSTANT,
+ dd->phys_base + SHA_REG_DIN(0), 0, 16);
+
+ omap_set_dma_dest_burst_mode(dd->dma_lch,
+ OMAP_DMA_DATA_BURST_16);
+
+ omap_set_dma_src_burst_mode(dd->dma_lch,
+ OMAP_DMA_DATA_BURST_4);
+
+ if (ctx->digcnt)
/* request has changed - restore hash */
omap_sham_copy_hash(req, 0);
@@ -684,7 +680,7 @@ static int omap_sham_handle_queue(struct omap_sham_dev *dd,
} else if (ctx->op == OP_FINAL) {
err = omap_sham_final_req(dd);
}
-
+err1:
if (err != -EINPROGRESS) {
/* done_task will not finish it, so do it here */
omap_sham_finish_req(req, err);
@@ -868,7 +864,7 @@ static int omap_sham_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base)
}
crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
- sizeof(struct omap_sham_reqctx));
+ sizeof(struct omap_sham_reqctx) + BUFLEN);
if (alg_base) {
struct omap_sham_hmac_ctx *bctx = tctx->base;
@@ -954,7 +950,7 @@ static struct ahash_alg algs[] = {
CRYPTO_ALG_NEED_FALLBACK,
.cra_blocksize = SHA1_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct omap_sham_ctx),
- .cra_alignmask = 0,
+ .cra_alignmask = OMAP_ALIGN_MASK,
.cra_module = THIS_MODULE,
.cra_init = omap_sham_cra_init,
.cra_exit = omap_sham_cra_exit,
@@ -978,7 +974,7 @@ static struct ahash_alg algs[] = {
.cra_blocksize = SHA1_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct omap_sham_ctx) +
sizeof(struct omap_sham_hmac_ctx),
- .cra_alignmask = 0,
+ .cra_alignmask = OMAP_ALIGN_MASK,
.cra_module = THIS_MODULE,
.cra_init = omap_sham_cra_sha1_init,
.cra_exit = omap_sham_cra_exit,
@@ -1002,7 +998,7 @@ static struct ahash_alg algs[] = {
.cra_blocksize = SHA1_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct omap_sham_ctx) +
sizeof(struct omap_sham_hmac_ctx),
- .cra_alignmask = 0,
+ .cra_alignmask = OMAP_ALIGN_MASK,
.cra_module = THIS_MODULE,
.cra_init = omap_sham_cra_md5_init,
.cra_exit = omap_sham_cra_exit,
--
1.7.0.4
bufcnt is 0 if it was no update requests before,
which is exact meaning of FLAGS_FIRST.
Signed-off-by: Dmitry Kasatkin <[email protected]>
---
drivers/crypto/omap-sham.c | 8 +-------
1 files changed, 1 insertions(+), 7 deletions(-)
diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c
index c8d30eb..d88d7eb 100644
--- a/drivers/crypto/omap-sham.c
+++ b/drivers/crypto/omap-sham.c
@@ -72,7 +72,6 @@
#define DEFAULT_TIMEOUT_INTERVAL HZ
-#define FLAGS_FIRST 0x0001
#define FLAGS_FINUP 0x0002
#define FLAGS_FINAL 0x0004
#define FLAGS_FAST 0x0008
@@ -513,8 +512,6 @@ static int omap_sham_init(struct ahash_request *req)
ctx->flags = 0;
- ctx->flags |= FLAGS_FIRST;
-
dev_dbg(dd->dev, "init: digest size: %d\n",
crypto_ahash_digestsize(tfm));
@@ -739,12 +736,9 @@ static int omap_sham_update(struct ahash_request *req)
/* may be can use faster functions */
int aligned = IS_ALIGNED((u32)ctx->sg->offset,
sizeof(u32));
-
- if (aligned && (ctx->flags & FLAGS_FIRST))
+ if (aligned)
/* digest: first and final */
ctx->flags |= FLAGS_FAST;
-
- ctx->flags &= ~FLAGS_FIRST;
}
} else if (ctx->bufcnt + ctx->total <= ctx->buflen) {
/* if not finaup -> not fast */
--
1.7.0.4
On Fri, Nov 19, 2010 at 04:04:21PM +0200, Dmitry Kasatkin wrote:
> Hi,
>
> Here is a set of patches which provides fixes and improvements.
>
> Based on Herbert feedback it also includes fixes so that calling final()
> is not mandatory.
All applied. Thanks Dmitry!
--
Email: Herbert Xu <[email protected]>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
Thanks.
On 27/11/10 10:39, ext Herbert Xu wrote:
> On Fri, Nov 19, 2010 at 04:04:21PM +0200, Dmitry Kasatkin wrote:
>
>> Hi,
>>
>> Here is a set of patches which provides fixes and improvements.
>>
>> Based on Herbert feedback it also includes fixes so that calling final()
>> is not mandatory.
>>
> All applied. Thanks Dmitry!
>