This set is an extension of a previous set called `crypto: qat - fix dm-crypt
related issues` which aims to re-enable the algorithms in the QAT driver
after [1].
This fixes a number of issues with the implementation of the QAT algs,
both symmetric and asymmetric.
In particular this set enables the QAT driver to handle correctly the
flags CRYPTO_TFM_REQ_MAY_BACKLOG and CRYPTO_TFM_REQ_MAY_SLEEP,
fixes an hidden issue in RSA and DH which appeared after commit f5ff79fddf0e,
related to the usage of dma_free_coherent() from a tasklet, and includes
important fixes in the akcipher algorithms.
One item to mention is that, differently from the previous set, this
one does not removes the flag CRYPTO_ALG_ALLOCATES_MEMORY which will
be removed after the conversation in [2] is closed.
[1] https://lore.kernel.org/linux-crypto/YiEyGoHacN80FcOL@silpixa00400314/
[2] https://lore.kernel.org/linux-crypto/Yl6PlqyucVLCzwF5@silpixa00400314/
Giovanni Cabiddu (12):
crypto: qat - use pre-allocated buffers in datapath
crypto: qat - refactor submission logic
crypto: qat - add backlog mechanism
crypto: qat - fix memory leak in RSA
crypto: qat - remove dma_free_coherent() for RSA
crypto: qat - remove dma_free_coherent() for DH
crypto: qat - set to zero DH parameters before free
crypto: qat - add param check for RSA
crypto: qat - add param check for DH
crypto: qat - use memzero_explicit() for algs
crypto: qat - honor CRYPTO_TFM_REQ_MAY_SLEEP flag
crypto: qat - re-enable registration of algorithms
drivers/crypto/qat/qat_4xxx/adf_drv.c | 7 -
drivers/crypto/qat/qat_common/Makefile | 1 +
drivers/crypto/qat/qat_common/adf_transport.c | 11 +
drivers/crypto/qat/qat_common/adf_transport.h | 1 +
.../qat/qat_common/adf_transport_internal.h | 1 +
drivers/crypto/qat/qat_common/qat_algs.c | 173 ++++-----
drivers/crypto/qat/qat_common/qat_algs_send.c | 86 +++++
drivers/crypto/qat/qat_common/qat_algs_send.h | 11 +
drivers/crypto/qat/qat_common/qat_asym_algs.c | 327 +++++++++---------
drivers/crypto/qat/qat_common/qat_crypto.c | 10 +-
drivers/crypto/qat/qat_common/qat_crypto.h | 44 +++
11 files changed, 415 insertions(+), 257 deletions(-)
create mode 100644 drivers/crypto/qat/qat_common/qat_algs_send.c
create mode 100644 drivers/crypto/qat/qat_common/qat_algs_send.h
--
2.35.1
Reject requests with a source buffer that is bigger than the size of the
key. This is to prevent a possible integer underflow that might happen
when copying the source scatterlist into a linear buffer.
Cc: [email protected]
Signed-off-by: Giovanni Cabiddu <[email protected]>
Reviewed-by: Adam Guerin <[email protected]>
Reviewed-by: Wojciech Ziemba <[email protected]>
---
drivers/crypto/qat/qat_common/qat_asym_algs.c | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/drivers/crypto/qat/qat_common/qat_asym_algs.c b/drivers/crypto/qat/qat_common/qat_asym_algs.c
index 2b2adeee5eee..b3badc5bd224 100644
--- a/drivers/crypto/qat/qat_common/qat_asym_algs.c
+++ b/drivers/crypto/qat/qat_common/qat_asym_algs.c
@@ -235,6 +235,10 @@ static int qat_dh_compute_value(struct kpp_request *req)
req->dst_len = ctx->p_size;
return -EOVERFLOW;
}
+
+ if (req->src_len > ctx->p_size)
+ return -EINVAL;
+
memset(msg, '\0', sizeof(*msg));
ICP_QAT_FW_PKE_HDR_VALID_FLAG_SET(msg->pke_hdr,
ICP_QAT_FW_COMN_REQ_FLAG_SET);
--
2.35.1
Reject requests with a source buffer that is bigger than the size of the
key. This is to prevent a possible integer underflow that might happen
when copying the source scatterlist into a linear buffer.
Cc: [email protected]
Signed-off-by: Giovanni Cabiddu <[email protected]>
Reviewed-by: Adam Guerin <[email protected]>
Reviewed-by: Wojciech Ziemba <[email protected]>
---
drivers/crypto/qat/qat_common/qat_asym_algs.c | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/drivers/crypto/qat/qat_common/qat_asym_algs.c b/drivers/crypto/qat/qat_common/qat_asym_algs.c
index 2fec89b8a188..2b2adeee5eee 100644
--- a/drivers/crypto/qat/qat_common/qat_asym_algs.c
+++ b/drivers/crypto/qat/qat_common/qat_asym_algs.c
@@ -656,6 +656,10 @@ static int qat_rsa_enc(struct akcipher_request *req)
req->dst_len = ctx->key_sz;
return -EOVERFLOW;
}
+
+ if (req->src_len > ctx->key_sz)
+ return -EINVAL;
+
memset(msg, '\0', sizeof(*msg));
ICP_QAT_FW_PKE_HDR_VALID_FLAG_SET(msg->pke_hdr,
ICP_QAT_FW_COMN_REQ_FLAG_SET);
@@ -785,6 +789,10 @@ static int qat_rsa_dec(struct akcipher_request *req)
req->dst_len = ctx->key_sz;
return -EOVERFLOW;
}
+
+ if (req->src_len > ctx->key_sz)
+ return -EINVAL;
+
memset(msg, '\0', sizeof(*msg));
ICP_QAT_FW_PKE_HDR_VALID_FLAG_SET(msg->pke_hdr,
ICP_QAT_FW_COMN_REQ_FLAG_SET);
--
2.35.1
When an RSA key represented in form 2 (as defined in PKCS #1 V2.1) is
used, some components of the private key persist even after the TFM is
released.
Replace the explicit calls to free the buffers in qat_rsa_exit_tfm()
with a call to qat_rsa_clear_ctx() which frees all buffers referenced in
the TFM context.
Cc: [email protected]
Fixes: 879f77e9071f ("crypto: qat - Add RSA CRT mode")
Signed-off-by: Giovanni Cabiddu <[email protected]>
Reviewed-by: Adam Guerin <[email protected]>
Reviewed-by: Wojciech Ziemba <[email protected]>
---
drivers/crypto/qat/qat_common/qat_asym_algs.c | 12 +-----------
1 file changed, 1 insertion(+), 11 deletions(-)
diff --git a/drivers/crypto/qat/qat_common/qat_asym_algs.c b/drivers/crypto/qat/qat_common/qat_asym_algs.c
index 306219a4b00e..9be972430f11 100644
--- a/drivers/crypto/qat/qat_common/qat_asym_algs.c
+++ b/drivers/crypto/qat/qat_common/qat_asym_algs.c
@@ -1254,18 +1254,8 @@ static void qat_rsa_exit_tfm(struct crypto_akcipher *tfm)
struct qat_rsa_ctx *ctx = akcipher_tfm_ctx(tfm);
struct device *dev = &GET_DEV(ctx->inst->accel_dev);
- if (ctx->n)
- dma_free_coherent(dev, ctx->key_sz, ctx->n, ctx->dma_n);
- if (ctx->e)
- dma_free_coherent(dev, ctx->key_sz, ctx->e, ctx->dma_e);
- if (ctx->d) {
- memset(ctx->d, '\0', ctx->key_sz);
- dma_free_coherent(dev, ctx->key_sz, ctx->d, ctx->dma_d);
- }
+ qat_rsa_clear_ctx(dev, ctx);
qat_crypto_put_instance(ctx->inst);
- ctx->n = NULL;
- ctx->e = NULL;
- ctx->d = NULL;
}
static struct akcipher_alg rsa = {
--
2.35.1
Re-enable the registration of algorithms after fixes to (1) use
pre-allocated buffers in the datapath and (2) support the
CRYPTO_TFM_REQ_MAY_BACKLOG flag.
This reverts commit 8893d27ffcaf6ec6267038a177cb87bcde4dd3de.
Cc: [email protected]
Signed-off-by: Giovanni Cabiddu <[email protected]>
Reviewed-by: Marco Chiappero <[email protected]>
Reviewed-by: Adam Guerin <[email protected]>
Reviewed-by: Wojciech Ziemba <[email protected]>
---
drivers/crypto/qat/qat_4xxx/adf_drv.c | 7 -------
drivers/crypto/qat/qat_common/qat_crypto.c | 7 -------
2 files changed, 14 deletions(-)
diff --git a/drivers/crypto/qat/qat_4xxx/adf_drv.c b/drivers/crypto/qat/qat_4xxx/adf_drv.c
index fa4c350c1bf9..a6c78b9c730b 100644
--- a/drivers/crypto/qat/qat_4xxx/adf_drv.c
+++ b/drivers/crypto/qat/qat_4xxx/adf_drv.c
@@ -75,13 +75,6 @@ static int adf_crypto_dev_config(struct adf_accel_dev *accel_dev)
if (ret)
goto err;
- /* Temporarily set the number of crypto instances to zero to avoid
- * registering the crypto algorithms.
- * This will be removed when the algorithms will support the
- * CRYPTO_TFM_REQ_MAY_BACKLOG flag
- */
- instances = 0;
-
for (i = 0; i < instances; i++) {
val = i;
bank = i * 2;
diff --git a/drivers/crypto/qat/qat_common/qat_crypto.c b/drivers/crypto/qat/qat_common/qat_crypto.c
index 80d905ed102e..9341d892533a 100644
--- a/drivers/crypto/qat/qat_common/qat_crypto.c
+++ b/drivers/crypto/qat/qat_common/qat_crypto.c
@@ -161,13 +161,6 @@ int qat_crypto_dev_config(struct adf_accel_dev *accel_dev)
if (ret)
goto err;
- /* Temporarily set the number of crypto instances to zero to avoid
- * registering the crypto algorithms.
- * This will be removed when the algorithms will support the
- * CRYPTO_TFM_REQ_MAY_BACKLOG flag
- */
- instances = 0;
-
for (i = 0; i < instances; i++) {
val = i;
snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_BANK_NUM, i);
--
2.35.1
The implementations of the crypto algorithms (aead, skcipher, etc) in
the QAT driver do not properly support requests with the
CRYPTO_TFM_REQ_MAY_BACKLOG flag set. If the HW queue is full, the driver
returns -EBUSY but does not enqueue the request. This can result in
applications like dm-crypt waiting indefinitely for the completion of a
request that was never submitted to the hardware.
Fix this by adding a software backlog queue: if the ring buffer is more
than eighty percent full, then the request is enqueued to a backlog
list and the error code -EBUSY is returned back to the caller.
Requests in the backlog queue are resubmitted at a later time, in the
context of the callback of a previously submitted request.
The request for which -EBUSY is returned is then marked as -EINPROGRESS
once submitted to the HW queues.
The submission loop inside the function qat_alg_send_message() has been
modified to decide which submission policy to use based on the request
flags. If the request does not have the CRYPTO_TFM_REQ_MAY_BACKLOG set,
the previous behaviour has been preserved.
Based on a patch by
Vishnu Das Ramachandran <[email protected]>
Cc: [email protected]
Fixes: d370cec32194 ("crypto: qat - Intel(R) QAT crypto interface")
Reported-by: Mikulas Patocka <[email protected]>
Reported-by: Kyle Sanderson <[email protected]>
Signed-off-by: Giovanni Cabiddu <[email protected]>
Reviewed-by: Marco Chiappero <[email protected]>
---
drivers/crypto/qat/qat_common/adf_transport.c | 11 +++
drivers/crypto/qat/qat_common/adf_transport.h | 1 +
.../qat/qat_common/adf_transport_internal.h | 1 +
drivers/crypto/qat/qat_common/qat_algs.c | 24 ++++---
drivers/crypto/qat/qat_common/qat_algs_send.c | 67 ++++++++++++++++++-
drivers/crypto/qat/qat_common/qat_algs_send.h | 1 +
drivers/crypto/qat/qat_common/qat_asym_algs.c | 23 ++++---
drivers/crypto/qat/qat_common/qat_crypto.c | 3 +
drivers/crypto/qat/qat_common/qat_crypto.h | 10 +++
9 files changed, 123 insertions(+), 18 deletions(-)
diff --git a/drivers/crypto/qat/qat_common/adf_transport.c b/drivers/crypto/qat/qat_common/adf_transport.c
index 8ba28409fb74..630d0483c4e0 100644
--- a/drivers/crypto/qat/qat_common/adf_transport.c
+++ b/drivers/crypto/qat/qat_common/adf_transport.c
@@ -8,6 +8,9 @@
#include "adf_cfg.h"
#include "adf_common_drv.h"
+#define ADF_MAX_RING_THRESHOLD 80
+#define ADF_PERCENT(tot, percent) (((tot) * (percent)) / 100)
+
static inline u32 adf_modulo(u32 data, u32 shift)
{
u32 div = data >> shift;
@@ -77,6 +80,11 @@ static void adf_disable_ring_irq(struct adf_etr_bank_data *bank, u32 ring)
bank->irq_mask);
}
+bool adf_ring_nearly_full(struct adf_etr_ring_data *ring)
+{
+ return atomic_read(ring->inflights) > ring->threshold;
+}
+
int adf_send_message(struct adf_etr_ring_data *ring, u32 *msg)
{
struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(ring->bank->accel_dev);
@@ -217,6 +225,7 @@ int adf_create_ring(struct adf_accel_dev *accel_dev, const char *section,
struct adf_etr_bank_data *bank;
struct adf_etr_ring_data *ring;
char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES];
+ int max_inflights;
u32 ring_num;
int ret;
@@ -263,6 +272,8 @@ int adf_create_ring(struct adf_accel_dev *accel_dev, const char *section,
ring->ring_size = adf_verify_ring_size(msg_size, num_msgs);
ring->head = 0;
ring->tail = 0;
+ max_inflights = ADF_MAX_INFLIGHTS(ring->ring_size, ring->msg_size);
+ ring->threshold = ADF_PERCENT(max_inflights, ADF_MAX_RING_THRESHOLD);
atomic_set(ring->inflights, 0);
ret = adf_init_ring(ring);
if (ret)
diff --git a/drivers/crypto/qat/qat_common/adf_transport.h b/drivers/crypto/qat/qat_common/adf_transport.h
index 2c95f1697c76..e6ef6f9b7691 100644
--- a/drivers/crypto/qat/qat_common/adf_transport.h
+++ b/drivers/crypto/qat/qat_common/adf_transport.h
@@ -14,6 +14,7 @@ int adf_create_ring(struct adf_accel_dev *accel_dev, const char *section,
const char *ring_name, adf_callback_fn callback,
int poll_mode, struct adf_etr_ring_data **ring_ptr);
+bool adf_ring_nearly_full(struct adf_etr_ring_data *ring);
int adf_send_message(struct adf_etr_ring_data *ring, u32 *msg);
void adf_remove_ring(struct adf_etr_ring_data *ring);
#endif
diff --git a/drivers/crypto/qat/qat_common/adf_transport_internal.h b/drivers/crypto/qat/qat_common/adf_transport_internal.h
index 501bcf0f1809..8b2c92ba7ca1 100644
--- a/drivers/crypto/qat/qat_common/adf_transport_internal.h
+++ b/drivers/crypto/qat/qat_common/adf_transport_internal.h
@@ -22,6 +22,7 @@ struct adf_etr_ring_data {
spinlock_t lock; /* protects ring data struct */
u16 head;
u16 tail;
+ u32 threshold;
u8 ring_number;
u8 ring_size;
u8 msg_size;
diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c
index 6017ae82c713..873533dc43a7 100644
--- a/drivers/crypto/qat/qat_common/qat_algs.c
+++ b/drivers/crypto/qat/qat_common/qat_algs.c
@@ -935,19 +935,25 @@ void qat_alg_callback(void *resp)
struct icp_qat_fw_la_resp *qat_resp = resp;
struct qat_crypto_request *qat_req =
(void *)(__force long)qat_resp->opaque_data;
+ struct qat_instance_backlog *backlog = qat_req->alg_req.backlog;
qat_req->cb(qat_resp, qat_req);
+
+ qat_alg_send_backlog(backlog);
}
static int qat_alg_send_sym_message(struct qat_crypto_request *qat_req,
- struct qat_crypto_instance *inst)
+ struct qat_crypto_instance *inst,
+ struct crypto_async_request *base)
{
- struct qat_alg_req req;
+ struct qat_alg_req *alg_req = &qat_req->alg_req;
- req.fw_req = (u32 *)&qat_req->req;
- req.tx_ring = inst->sym_tx;
+ alg_req->fw_req = (u32 *)&qat_req->req;
+ alg_req->tx_ring = inst->sym_tx;
+ alg_req->base = base;
+ alg_req->backlog = &inst->backlog;
- return qat_alg_send_message(&req);
+ return qat_alg_send_message(alg_req);
}
static int qat_alg_aead_dec(struct aead_request *areq)
@@ -987,7 +993,7 @@ static int qat_alg_aead_dec(struct aead_request *areq)
auth_param->auth_off = 0;
auth_param->auth_len = areq->assoclen + cipher_param->cipher_length;
- ret = qat_alg_send_sym_message(qat_req, ctx->inst);
+ ret = qat_alg_send_sym_message(qat_req, ctx->inst, &areq->base);
if (ret == -ENOSPC)
qat_alg_free_bufl(ctx->inst, qat_req);
@@ -1031,7 +1037,7 @@ static int qat_alg_aead_enc(struct aead_request *areq)
auth_param->auth_off = 0;
auth_param->auth_len = areq->assoclen + areq->cryptlen;
- ret = qat_alg_send_sym_message(qat_req, ctx->inst);
+ ret = qat_alg_send_sym_message(qat_req, ctx->inst, &areq->base);
if (ret == -ENOSPC)
qat_alg_free_bufl(ctx->inst, qat_req);
@@ -1212,7 +1218,7 @@ static int qat_alg_skcipher_encrypt(struct skcipher_request *req)
qat_alg_set_req_iv(qat_req);
- ret = qat_alg_send_sym_message(qat_req, ctx->inst);
+ ret = qat_alg_send_sym_message(qat_req, ctx->inst, &req->base);
if (ret == -ENOSPC)
qat_alg_free_bufl(ctx->inst, qat_req);
@@ -1278,7 +1284,7 @@ static int qat_alg_skcipher_decrypt(struct skcipher_request *req)
qat_alg_set_req_iv(qat_req);
qat_alg_update_iv(qat_req);
- ret = qat_alg_send_sym_message(qat_req, ctx->inst);
+ ret = qat_alg_send_sym_message(qat_req, ctx->inst, &req->base);
if (ret == -ENOSPC)
qat_alg_free_bufl(ctx->inst, qat_req);
diff --git a/drivers/crypto/qat/qat_common/qat_algs_send.c b/drivers/crypto/qat/qat_common/qat_algs_send.c
index 78f1bb8c26c0..ff5b4347f783 100644
--- a/drivers/crypto/qat/qat_common/qat_algs_send.c
+++ b/drivers/crypto/qat/qat_common/qat_algs_send.c
@@ -6,7 +6,7 @@
#define ADF_MAX_RETRIES 20
-int qat_alg_send_message(struct qat_alg_req *req)
+static int qat_alg_send_message_retry(struct qat_alg_req *req)
{
int ret = 0, ctr = 0;
@@ -19,3 +19,68 @@ int qat_alg_send_message(struct qat_alg_req *req)
return -EINPROGRESS;
}
+
+void qat_alg_send_backlog(struct qat_instance_backlog *backlog)
+{
+ struct qat_alg_req *req, *tmp;
+
+ spin_lock_bh(&backlog->lock);
+ list_for_each_entry_safe(req, tmp, &backlog->list, list) {
+ if (adf_send_message(req->tx_ring, req->fw_req)) {
+ /* The HW ring is full. Do nothing.
+ * qat_alg_send_backlog() will be invoked again by
+ * another callback.
+ */
+ break;
+ }
+ list_del(&req->list);
+ req->base->complete(req->base, -EINPROGRESS);
+ }
+ spin_unlock_bh(&backlog->lock);
+}
+
+static void qat_alg_backlog_req(struct qat_alg_req *req,
+ struct qat_instance_backlog *backlog)
+{
+ INIT_LIST_HEAD(&req->list);
+
+ spin_lock_bh(&backlog->lock);
+ list_add_tail(&req->list, &backlog->list);
+ spin_unlock_bh(&backlog->lock);
+}
+
+static int qat_alg_send_message_maybacklog(struct qat_alg_req *req)
+{
+ struct qat_instance_backlog *backlog = req->backlog;
+ struct adf_etr_ring_data *tx_ring = req->tx_ring;
+ u32 *fw_req = req->fw_req;
+
+ /* If any request is already backlogged, then add to backlog list */
+ if (!list_empty(&backlog->list))
+ goto enqueue;
+
+ /* If ring is nearly full, then add to backlog list */
+ if (adf_ring_nearly_full(tx_ring))
+ goto enqueue;
+
+ /* If adding request to HW ring fails, then add to backlog list */
+ if (adf_send_message(tx_ring, fw_req))
+ goto enqueue;
+
+ return -EINPROGRESS;
+
+enqueue:
+ qat_alg_backlog_req(req, backlog);
+
+ return -EBUSY;
+}
+
+int qat_alg_send_message(struct qat_alg_req *req)
+{
+ u32 flags = req->base->flags;
+
+ if (flags & CRYPTO_TFM_REQ_MAY_BACKLOG)
+ return qat_alg_send_message_maybacklog(req);
+ else
+ return qat_alg_send_message_retry(req);
+}
diff --git a/drivers/crypto/qat/qat_common/qat_algs_send.h b/drivers/crypto/qat/qat_common/qat_algs_send.h
index 3fa685d0c293..5ce9f4f69d8f 100644
--- a/drivers/crypto/qat/qat_common/qat_algs_send.h
+++ b/drivers/crypto/qat/qat_common/qat_algs_send.h
@@ -6,5 +6,6 @@
#include "qat_crypto.h"
int qat_alg_send_message(struct qat_alg_req *req);
+void qat_alg_send_backlog(struct qat_instance_backlog *backlog);
#endif
diff --git a/drivers/crypto/qat/qat_common/qat_asym_algs.c b/drivers/crypto/qat/qat_common/qat_asym_algs.c
index a3fdbcc08772..306219a4b00e 100644
--- a/drivers/crypto/qat/qat_common/qat_asym_algs.c
+++ b/drivers/crypto/qat/qat_common/qat_asym_algs.c
@@ -136,17 +136,21 @@ struct qat_asym_request {
} areq;
int err;
void (*cb)(struct icp_qat_fw_pke_resp *resp);
+ struct qat_alg_req alg_req;
} __aligned(64);
static int qat_alg_send_asym_message(struct qat_asym_request *qat_req,
- struct qat_crypto_instance *inst)
+ struct qat_crypto_instance *inst,
+ struct crypto_async_request *base)
{
- struct qat_alg_req req;
+ struct qat_alg_req *alg_req = &qat_req->alg_req;
- req.fw_req = (u32 *)&qat_req->req;
- req.tx_ring = inst->pke_tx;
+ alg_req->fw_req = (u32 *)&qat_req->req;
+ alg_req->tx_ring = inst->pke_tx;
+ alg_req->base = base;
+ alg_req->backlog = &inst->backlog;
- return qat_alg_send_message(&req);
+ return qat_alg_send_message(alg_req);
}
static void qat_dh_cb(struct icp_qat_fw_pke_resp *resp)
@@ -350,7 +354,7 @@ static int qat_dh_compute_value(struct kpp_request *req)
msg->input_param_count = n_input_params;
msg->output_param_count = 1;
- ret = qat_alg_send_asym_message(qat_req, ctx->inst);
+ ret = qat_alg_send_asym_message(qat_req, inst, &req->base);
if (ret == -ENOSPC)
goto unmap_all;
@@ -554,8 +558,11 @@ void qat_alg_asym_callback(void *_resp)
{
struct icp_qat_fw_pke_resp *resp = _resp;
struct qat_asym_request *areq = (void *)(__force long)resp->opaque;
+ struct qat_instance_backlog *backlog = areq->alg_req.backlog;
areq->cb(resp);
+
+ qat_alg_send_backlog(backlog);
}
#define PKE_RSA_EP_512 0x1c161b21
@@ -745,7 +752,7 @@ static int qat_rsa_enc(struct akcipher_request *req)
msg->input_param_count = 3;
msg->output_param_count = 1;
- ret = qat_alg_send_asym_message(qat_req, ctx->inst);
+ ret = qat_alg_send_asym_message(qat_req, inst, &req->base);
if (ret == -ENOSPC)
goto unmap_all;
@@ -898,7 +905,7 @@ static int qat_rsa_dec(struct akcipher_request *req)
msg->output_param_count = 1;
- ret = qat_alg_send_asym_message(qat_req, ctx->inst);
+ ret = qat_alg_send_asym_message(qat_req, inst, &req->base);
if (ret == -ENOSPC)
goto unmap_all;
diff --git a/drivers/crypto/qat/qat_common/qat_crypto.c b/drivers/crypto/qat/qat_common/qat_crypto.c
index 67c9588e89df..80d905ed102e 100644
--- a/drivers/crypto/qat/qat_common/qat_crypto.c
+++ b/drivers/crypto/qat/qat_common/qat_crypto.c
@@ -353,6 +353,9 @@ static int qat_crypto_create_instances(struct adf_accel_dev *accel_dev)
&inst->pke_rx);
if (ret)
goto err;
+
+ INIT_LIST_HEAD(&inst->backlog.list);
+ spin_lock_init(&inst->backlog.lock);
}
return 0;
err:
diff --git a/drivers/crypto/qat/qat_common/qat_crypto.h b/drivers/crypto/qat/qat_common/qat_crypto.h
index 0dcba6fc358c..245b6d9a3650 100644
--- a/drivers/crypto/qat/qat_common/qat_crypto.h
+++ b/drivers/crypto/qat/qat_common/qat_crypto.h
@@ -9,9 +9,17 @@
#include "adf_accel_devices.h"
#include "icp_qat_fw_la.h"
+struct qat_instance_backlog {
+ struct list_head list;
+ spinlock_t lock; /* protects backlog list */
+};
+
struct qat_alg_req {
u32 *fw_req;
struct adf_etr_ring_data *tx_ring;
+ struct crypto_async_request *base;
+ struct list_head list;
+ struct qat_instance_backlog *backlog;
};
struct qat_crypto_instance {
@@ -24,6 +32,7 @@ struct qat_crypto_instance {
unsigned long state;
int id;
atomic_t refctr;
+ struct qat_instance_backlog backlog;
};
#define QAT_MAX_BUFF_DESC 4
@@ -82,6 +91,7 @@ struct qat_crypto_request {
u8 iv[AES_BLOCK_SIZE];
};
bool encryption;
+ struct qat_alg_req alg_req;
};
static inline bool adf_hw_dev_has_crypto(struct adf_accel_dev *accel_dev)
--
2.35.1
If a request has the flag CRYPTO_TFM_REQ_MAY_SLEEP set, allocate memory
using the flag GFP_KERNEL otherwise use GFP_ATOMIC.
Cc: [email protected]
Signed-off-by: Giovanni Cabiddu <[email protected]>
Reviewed-by: Adam Guerin <[email protected]>
Reviewed-by: Wojciech Ziemba <[email protected]>
---
drivers/crypto/qat/qat_common/qat_algs.c | 19 ++++++++++++-------
drivers/crypto/qat/qat_common/qat_asym_algs.c | 17 ++++++++++-------
drivers/crypto/qat/qat_common/qat_crypto.h | 5 +++++
3 files changed, 27 insertions(+), 14 deletions(-)
diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c
index c42df18e02b2..42b72f1e301b 100644
--- a/drivers/crypto/qat/qat_common/qat_algs.c
+++ b/drivers/crypto/qat/qat_common/qat_algs.c
@@ -703,7 +703,8 @@ static void qat_alg_free_bufl(struct qat_crypto_instance *inst,
static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst,
struct scatterlist *sgl,
struct scatterlist *sglout,
- struct qat_crypto_request *qat_req)
+ struct qat_crypto_request *qat_req,
+ gfp_t flags)
{
struct device *dev = &GET_DEV(inst->accel_dev);
int i, sg_nctr = 0;
@@ -723,7 +724,7 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst,
qat_req->buf.sgl_dst_valid = false;
if (n > QAT_MAX_BUFF_DESC) {
- bufl = kzalloc_node(sz, GFP_ATOMIC, node);
+ bufl = kzalloc_node(sz, flags, node);
if (unlikely(!bufl))
return -ENOMEM;
} else {
@@ -765,7 +766,7 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst,
sg_nctr = 0;
if (n > QAT_MAX_BUFF_DESC) {
- buflout = kzalloc_node(sz_out, GFP_ATOMIC, node);
+ buflout = kzalloc_node(sz_out, flags, node);
if (unlikely(!buflout))
goto err_in;
} else {
@@ -966,6 +967,7 @@ static int qat_alg_aead_dec(struct aead_request *areq)
struct icp_qat_fw_la_auth_req_params *auth_param;
struct icp_qat_fw_la_bulk_req *msg;
int digst_size = crypto_aead_authsize(aead_tfm);
+ gfp_t f = qat_algs_alloc_flags(&areq->base);
int ret;
u32 cipher_len;
@@ -973,7 +975,7 @@ static int qat_alg_aead_dec(struct aead_request *areq)
if (cipher_len % AES_BLOCK_SIZE != 0)
return -EINVAL;
- ret = qat_alg_sgl_to_bufl(ctx->inst, areq->src, areq->dst, qat_req);
+ ret = qat_alg_sgl_to_bufl(ctx->inst, areq->src, areq->dst, qat_req, f);
if (unlikely(ret))
return ret;
@@ -1008,6 +1010,7 @@ static int qat_alg_aead_enc(struct aead_request *areq)
struct qat_crypto_request *qat_req = aead_request_ctx(areq);
struct icp_qat_fw_la_cipher_req_params *cipher_param;
struct icp_qat_fw_la_auth_req_params *auth_param;
+ gfp_t f = qat_algs_alloc_flags(&areq->base);
struct icp_qat_fw_la_bulk_req *msg;
u8 *iv = areq->iv;
int ret;
@@ -1015,7 +1018,7 @@ static int qat_alg_aead_enc(struct aead_request *areq)
if (areq->cryptlen % AES_BLOCK_SIZE != 0)
return -EINVAL;
- ret = qat_alg_sgl_to_bufl(ctx->inst, areq->src, areq->dst, qat_req);
+ ret = qat_alg_sgl_to_bufl(ctx->inst, areq->src, areq->dst, qat_req, f);
if (unlikely(ret))
return ret;
@@ -1193,13 +1196,14 @@ static int qat_alg_skcipher_encrypt(struct skcipher_request *req)
struct qat_alg_skcipher_ctx *ctx = crypto_tfm_ctx(tfm);
struct qat_crypto_request *qat_req = skcipher_request_ctx(req);
struct icp_qat_fw_la_cipher_req_params *cipher_param;
+ gfp_t f = qat_algs_alloc_flags(&req->base);
struct icp_qat_fw_la_bulk_req *msg;
int ret;
if (req->cryptlen == 0)
return 0;
- ret = qat_alg_sgl_to_bufl(ctx->inst, req->src, req->dst, qat_req);
+ ret = qat_alg_sgl_to_bufl(ctx->inst, req->src, req->dst, qat_req, f);
if (unlikely(ret))
return ret;
@@ -1258,13 +1262,14 @@ static int qat_alg_skcipher_decrypt(struct skcipher_request *req)
struct qat_alg_skcipher_ctx *ctx = crypto_tfm_ctx(tfm);
struct qat_crypto_request *qat_req = skcipher_request_ctx(req);
struct icp_qat_fw_la_cipher_req_params *cipher_param;
+ gfp_t f = qat_algs_alloc_flags(&req->base);
struct icp_qat_fw_la_bulk_req *msg;
int ret;
if (req->cryptlen == 0)
return 0;
- ret = qat_alg_sgl_to_bufl(ctx->inst, req->src, req->dst, qat_req);
+ ret = qat_alg_sgl_to_bufl(ctx->inst, req->src, req->dst, qat_req, f);
if (unlikely(ret))
return ret;
diff --git a/drivers/crypto/qat/qat_common/qat_asym_algs.c b/drivers/crypto/qat/qat_common/qat_asym_algs.c
index 86c7d07435c8..688cf6bc95f9 100644
--- a/drivers/crypto/qat/qat_common/qat_asym_algs.c
+++ b/drivers/crypto/qat/qat_common/qat_asym_algs.c
@@ -224,9 +224,10 @@ static int qat_dh_compute_value(struct kpp_request *req)
struct qat_asym_request *qat_req =
PTR_ALIGN(kpp_request_ctx(req), 64);
struct icp_qat_fw_pke_request *msg = &qat_req->req;
- int ret;
+ gfp_t flags = qat_algs_alloc_flags(&req->base);
int n_input_params = 0;
u8 *vaddr;
+ int ret;
if (unlikely(!ctx->xa))
return -EINVAL;
@@ -291,7 +292,7 @@ static int qat_dh_compute_value(struct kpp_request *req)
} else {
int shift = ctx->p_size - req->src_len;
- qat_req->src_align = kzalloc(ctx->p_size, GFP_KERNEL);
+ qat_req->src_align = kzalloc(ctx->p_size, flags);
if (unlikely(!qat_req->src_align))
return ret;
@@ -317,7 +318,7 @@ static int qat_dh_compute_value(struct kpp_request *req)
qat_req->dst_align = NULL;
vaddr = sg_virt(req->dst);
} else {
- qat_req->dst_align = kzalloc(ctx->p_size, GFP_KERNEL);
+ qat_req->dst_align = kzalloc(ctx->p_size, flags);
if (unlikely(!qat_req->dst_align))
goto unmap_src;
@@ -650,6 +651,7 @@ static int qat_rsa_enc(struct akcipher_request *req)
struct qat_asym_request *qat_req =
PTR_ALIGN(akcipher_request_ctx(req), 64);
struct icp_qat_fw_pke_request *msg = &qat_req->req;
+ gfp_t flags = qat_algs_alloc_flags(&req->base);
u8 *vaddr;
int ret;
@@ -696,7 +698,7 @@ static int qat_rsa_enc(struct akcipher_request *req)
} else {
int shift = ctx->key_sz - req->src_len;
- qat_req->src_align = kzalloc(ctx->key_sz, GFP_KERNEL);
+ qat_req->src_align = kzalloc(ctx->key_sz, flags);
if (unlikely(!qat_req->src_align))
return ret;
@@ -714,7 +716,7 @@ static int qat_rsa_enc(struct akcipher_request *req)
qat_req->dst_align = NULL;
vaddr = sg_virt(req->dst);
} else {
- qat_req->dst_align = kzalloc(ctx->key_sz, GFP_KERNEL);
+ qat_req->dst_align = kzalloc(ctx->key_sz, flags);
if (unlikely(!qat_req->dst_align))
goto unmap_src;
vaddr = qat_req->dst_align;
@@ -783,6 +785,7 @@ static int qat_rsa_dec(struct akcipher_request *req)
struct qat_asym_request *qat_req =
PTR_ALIGN(akcipher_request_ctx(req), 64);
struct icp_qat_fw_pke_request *msg = &qat_req->req;
+ gfp_t flags = qat_algs_alloc_flags(&req->base);
u8 *vaddr;
int ret;
@@ -839,7 +842,7 @@ static int qat_rsa_dec(struct akcipher_request *req)
} else {
int shift = ctx->key_sz - req->src_len;
- qat_req->src_align = kzalloc(ctx->key_sz, GFP_KERNEL);
+ qat_req->src_align = kzalloc(ctx->key_sz, flags);
if (unlikely(!qat_req->src_align))
return ret;
@@ -857,7 +860,7 @@ static int qat_rsa_dec(struct akcipher_request *req)
qat_req->dst_align = NULL;
vaddr = sg_virt(req->dst);
} else {
- qat_req->dst_align = kzalloc(ctx->key_sz, GFP_KERNEL);
+ qat_req->dst_align = kzalloc(ctx->key_sz, flags);
if (unlikely(!qat_req->dst_align))
goto unmap_src;
vaddr = qat_req->dst_align;
diff --git a/drivers/crypto/qat/qat_common/qat_crypto.h b/drivers/crypto/qat/qat_common/qat_crypto.h
index 245b6d9a3650..df3c738ce323 100644
--- a/drivers/crypto/qat/qat_common/qat_crypto.h
+++ b/drivers/crypto/qat/qat_common/qat_crypto.h
@@ -109,4 +109,9 @@ static inline bool adf_hw_dev_has_crypto(struct adf_accel_dev *accel_dev)
return true;
}
+static inline gfp_t qat_algs_alloc_flags(struct crypto_async_request *req)
+{
+ return req->flags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL : GFP_ATOMIC;
+}
+
#endif
--
2.35.1
After commit f5ff79fddf0e ("dma-mapping: remove CONFIG_DMA_REMAP"), if
the algorithms are enabled, the driver crashes with a BUG_ON while
executing vunmap() in the context of a tasklet. This is due to the fact
that the function dma_free_coherent() cannot be called in an interrupt
context (see Documentation/core-api/dma-api-howto.rst).
The functions qat_rsa_enc() and qat_rsa_dec() allocate memory with
dma_alloc_coherent() if the source or the destination buffers are made
of multiple flat buffers or of a size that is not compatible with the
hardware.
This memory is then freed with dma_free_coherent() in the context of a
tasklet invoked to handle the response for the corresponding request.
Replace allocations with dma_alloc_coherent() in the functions
qat_rsa_enc() and qat_rsa_dec() with kmalloc() + dma_map_single().
Cc: [email protected]
Fixes: a990532023b9 ("crypto: qat - Add support for RSA algorithm")
Signed-off-by: Giovanni Cabiddu <[email protected]>
Reviewed-by: Adam Guerin <[email protected]>
Reviewed-by: Wojciech Ziemba <[email protected]>
---
drivers/crypto/qat/qat_common/qat_asym_algs.c | 137 ++++++++----------
1 file changed, 60 insertions(+), 77 deletions(-)
diff --git a/drivers/crypto/qat/qat_common/qat_asym_algs.c b/drivers/crypto/qat/qat_common/qat_asym_algs.c
index 9be972430f11..bba4b4d99e94 100644
--- a/drivers/crypto/qat/qat_common/qat_asym_algs.c
+++ b/drivers/crypto/qat/qat_common/qat_asym_algs.c
@@ -526,25 +526,22 @@ static void qat_rsa_cb(struct icp_qat_fw_pke_resp *resp)
err = (err == ICP_QAT_FW_COMN_STATUS_FLAG_OK) ? 0 : -EINVAL;
- if (req->src_align)
- dma_free_coherent(dev, req->ctx.rsa->key_sz, req->src_align,
- req->in.rsa.enc.m);
- else
- dma_unmap_single(dev, req->in.rsa.enc.m, req->ctx.rsa->key_sz,
- DMA_TO_DEVICE);
+ kfree_sensitive(req->src_align);
+
+ dma_unmap_single(dev, req->in.rsa.enc.m, req->ctx.rsa->key_sz,
+ DMA_TO_DEVICE);
areq->dst_len = req->ctx.rsa->key_sz;
if (req->dst_align) {
scatterwalk_map_and_copy(req->dst_align, areq->dst, 0,
areq->dst_len, 1);
- dma_free_coherent(dev, req->ctx.rsa->key_sz, req->dst_align,
- req->out.rsa.enc.c);
- } else {
- dma_unmap_single(dev, req->out.rsa.enc.c, req->ctx.rsa->key_sz,
- DMA_FROM_DEVICE);
+ kfree_sensitive(req->dst_align);
}
+ dma_unmap_single(dev, req->out.rsa.enc.c, req->ctx.rsa->key_sz,
+ DMA_FROM_DEVICE);
+
dma_unmap_single(dev, req->phy_in, sizeof(struct qat_rsa_input_params),
DMA_TO_DEVICE);
dma_unmap_single(dev, req->phy_out,
@@ -661,6 +658,7 @@ static int qat_rsa_enc(struct akcipher_request *req)
struct qat_asym_request *qat_req =
PTR_ALIGN(akcipher_request_ctx(req), 64);
struct icp_qat_fw_pke_request *msg = &qat_req->req;
+ u8 *vaddr;
int ret;
if (unlikely(!ctx->n || !ctx->e))
@@ -698,40 +696,39 @@ static int qat_rsa_enc(struct akcipher_request *req)
*/
if (sg_is_last(req->src) && req->src_len == ctx->key_sz) {
qat_req->src_align = NULL;
- qat_req->in.rsa.enc.m = dma_map_single(dev, sg_virt(req->src),
- req->src_len, DMA_TO_DEVICE);
- if (unlikely(dma_mapping_error(dev, qat_req->in.rsa.enc.m)))
- return ret;
-
+ vaddr = sg_virt(req->src);
} else {
int shift = ctx->key_sz - req->src_len;
- qat_req->src_align = dma_alloc_coherent(dev, ctx->key_sz,
- &qat_req->in.rsa.enc.m,
- GFP_KERNEL);
+ qat_req->src_align = kzalloc(ctx->key_sz, GFP_KERNEL);
if (unlikely(!qat_req->src_align))
return ret;
scatterwalk_map_and_copy(qat_req->src_align + shift, req->src,
0, req->src_len, 0);
+ vaddr = qat_req->src_align;
}
- if (sg_is_last(req->dst) && req->dst_len == ctx->key_sz) {
- qat_req->dst_align = NULL;
- qat_req->out.rsa.enc.c = dma_map_single(dev, sg_virt(req->dst),
- req->dst_len,
- DMA_FROM_DEVICE);
- if (unlikely(dma_mapping_error(dev, qat_req->out.rsa.enc.c)))
- goto unmap_src;
+ qat_req->in.rsa.enc.m = dma_map_single(dev, vaddr, ctx->key_sz,
+ DMA_TO_DEVICE);
+ if (unlikely(dma_mapping_error(dev, qat_req->in.rsa.enc.m)))
+ goto unmap_src;
+ if (sg_is_last(req->dst) && req->dst_len == ctx->key_sz) {
+ qat_req->dst_align = NULL;
+ vaddr = sg_virt(req->dst);
} else {
- qat_req->dst_align = dma_alloc_coherent(dev, ctx->key_sz,
- &qat_req->out.rsa.enc.c,
- GFP_KERNEL);
+ qat_req->dst_align = kzalloc(ctx->key_sz, GFP_KERNEL);
if (unlikely(!qat_req->dst_align))
goto unmap_src;
-
+ vaddr = qat_req->dst_align;
}
+
+ qat_req->out.rsa.enc.c = dma_map_single(dev, vaddr, ctx->key_sz,
+ DMA_FROM_DEVICE);
+ if (unlikely(dma_mapping_error(dev, qat_req->out.rsa.enc.c)))
+ goto unmap_dst;
+
qat_req->in.rsa.in_tab[3] = 0;
qat_req->out.rsa.out_tab[1] = 0;
qat_req->phy_in = dma_map_single(dev, &qat_req->in.rsa.enc.m,
@@ -769,21 +766,15 @@ static int qat_rsa_enc(struct akcipher_request *req)
sizeof(struct qat_rsa_input_params),
DMA_TO_DEVICE);
unmap_dst:
- if (qat_req->dst_align)
- dma_free_coherent(dev, ctx->key_sz, qat_req->dst_align,
- qat_req->out.rsa.enc.c);
- else
- if (!dma_mapping_error(dev, qat_req->out.rsa.enc.c))
- dma_unmap_single(dev, qat_req->out.rsa.enc.c,
- ctx->key_sz, DMA_FROM_DEVICE);
+ if (!dma_mapping_error(dev, qat_req->out.rsa.enc.c))
+ dma_unmap_single(dev, qat_req->out.rsa.enc.c,
+ ctx->key_sz, DMA_FROM_DEVICE);
+ kfree_sensitive(qat_req->dst_align);
unmap_src:
- if (qat_req->src_align)
- dma_free_coherent(dev, ctx->key_sz, qat_req->src_align,
- qat_req->in.rsa.enc.m);
- else
- if (!dma_mapping_error(dev, qat_req->in.rsa.enc.m))
- dma_unmap_single(dev, qat_req->in.rsa.enc.m,
- ctx->key_sz, DMA_TO_DEVICE);
+ if (!dma_mapping_error(dev, qat_req->in.rsa.enc.m))
+ dma_unmap_single(dev, qat_req->in.rsa.enc.m, ctx->key_sz,
+ DMA_TO_DEVICE);
+ kfree_sensitive(qat_req->src_align);
return ret;
}
@@ -796,6 +787,7 @@ static int qat_rsa_dec(struct akcipher_request *req)
struct qat_asym_request *qat_req =
PTR_ALIGN(akcipher_request_ctx(req), 64);
struct icp_qat_fw_pke_request *msg = &qat_req->req;
+ u8 *vaddr;
int ret;
if (unlikely(!ctx->n || !ctx->d))
@@ -843,40 +835,37 @@ static int qat_rsa_dec(struct akcipher_request *req)
*/
if (sg_is_last(req->src) && req->src_len == ctx->key_sz) {
qat_req->src_align = NULL;
- qat_req->in.rsa.dec.c = dma_map_single(dev, sg_virt(req->src),
- req->dst_len, DMA_TO_DEVICE);
- if (unlikely(dma_mapping_error(dev, qat_req->in.rsa.dec.c)))
- return ret;
-
+ vaddr = sg_virt(req->src);
} else {
int shift = ctx->key_sz - req->src_len;
- qat_req->src_align = dma_alloc_coherent(dev, ctx->key_sz,
- &qat_req->in.rsa.dec.c,
- GFP_KERNEL);
+ qat_req->src_align = kzalloc(ctx->key_sz, GFP_KERNEL);
if (unlikely(!qat_req->src_align))
return ret;
scatterwalk_map_and_copy(qat_req->src_align + shift, req->src,
0, req->src_len, 0);
+ vaddr = qat_req->src_align;
}
- if (sg_is_last(req->dst) && req->dst_len == ctx->key_sz) {
- qat_req->dst_align = NULL;
- qat_req->out.rsa.dec.m = dma_map_single(dev, sg_virt(req->dst),
- req->dst_len,
- DMA_FROM_DEVICE);
- if (unlikely(dma_mapping_error(dev, qat_req->out.rsa.dec.m)))
- goto unmap_src;
+ qat_req->in.rsa.dec.c = dma_map_single(dev, vaddr, ctx->key_sz,
+ DMA_TO_DEVICE);
+ if (unlikely(dma_mapping_error(dev, qat_req->in.rsa.dec.c)))
+ goto unmap_src;
+ if (sg_is_last(req->dst) && req->dst_len == ctx->key_sz) {
+ qat_req->dst_align = NULL;
+ vaddr = sg_virt(req->dst);
} else {
- qat_req->dst_align = dma_alloc_coherent(dev, ctx->key_sz,
- &qat_req->out.rsa.dec.m,
- GFP_KERNEL);
+ qat_req->dst_align = kzalloc(ctx->key_sz, GFP_KERNEL);
if (unlikely(!qat_req->dst_align))
goto unmap_src;
-
+ vaddr = qat_req->dst_align;
}
+ qat_req->out.rsa.dec.m = dma_map_single(dev, vaddr, ctx->key_sz,
+ DMA_FROM_DEVICE);
+ if (unlikely(dma_mapping_error(dev, qat_req->out.rsa.dec.m)))
+ goto unmap_dst;
if (ctx->crt_mode)
qat_req->in.rsa.in_tab[6] = 0;
@@ -922,21 +911,15 @@ static int qat_rsa_dec(struct akcipher_request *req)
sizeof(struct qat_rsa_input_params),
DMA_TO_DEVICE);
unmap_dst:
- if (qat_req->dst_align)
- dma_free_coherent(dev, ctx->key_sz, qat_req->dst_align,
- qat_req->out.rsa.dec.m);
- else
- if (!dma_mapping_error(dev, qat_req->out.rsa.dec.m))
- dma_unmap_single(dev, qat_req->out.rsa.dec.m,
- ctx->key_sz, DMA_FROM_DEVICE);
+ if (!dma_mapping_error(dev, qat_req->out.rsa.dec.m))
+ dma_unmap_single(dev, qat_req->out.rsa.dec.m,
+ ctx->key_sz, DMA_FROM_DEVICE);
+ kfree_sensitive(qat_req->dst_align);
unmap_src:
- if (qat_req->src_align)
- dma_free_coherent(dev, ctx->key_sz, qat_req->src_align,
- qat_req->in.rsa.dec.c);
- else
- if (!dma_mapping_error(dev, qat_req->in.rsa.dec.c))
- dma_unmap_single(dev, qat_req->in.rsa.dec.c,
- ctx->key_sz, DMA_TO_DEVICE);
+ if (!dma_mapping_error(dev, qat_req->in.rsa.dec.c))
+ dma_unmap_single(dev, qat_req->in.rsa.dec.c, ctx->key_sz,
+ DMA_TO_DEVICE);
+ kfree_sensitive(qat_req->src_align);
return ret;
}
--
2.35.1