Marvell OcteonTX2's next gen platform CN10KB/CN10KA B0
introduced changes in CPT SG input format(SGv2) to make
it compatibile with NIX SG input format to support inline
IPsec in SG mode.
This patchset modifies the CPT driver code to support SGv2
format for CN10KB/CN10KA B0. And also adds code to configure
newly introduced HW registers.
This patchset also implements SW workaround for couple of
HW erratas.
Nithin Dabilpuram (2):
crypto/octeontx2: register error interrupts for inline cptlf
crypto: octeontx2: support setting ctx ilen for inline CPT LF
Srujana Challa (8):
crypto: octeontx2: remove CPT block reset
crypto: octeontx2: add SGv2 support for CN10KB or CN10KA B0
crypto: octeontx2: add devlink option to set max_rxc_icb_cnt
crypto: octeontx2: add devlink option to set t106 mode
crypto: octeontx2: remove errata workaround for CN10KB or CN10KA B0
chip.
crypto: octeontx2: add LF reset on queue disable
octeontx2-af: update CPT inbound inline IPsec mailbox
crypto: octeontx2: add ctx_val workaround
drivers/crypto/marvell/octeontx2/cn10k_cpt.c | 87 +++++-
drivers/crypto/marvell/octeontx2/cn10k_cpt.h | 25 ++
.../marvell/octeontx2/otx2_cpt_common.h | 68 +++-
.../marvell/octeontx2/otx2_cpt_devlink.c | 89 +++++-
.../marvell/octeontx2/otx2_cpt_hw_types.h | 9 +-
.../marvell/octeontx2/otx2_cpt_mbox_common.c | 26 ++
.../marvell/octeontx2/otx2_cpt_reqmgr.h | 293 ++++++++++++++++++
drivers/crypto/marvell/octeontx2/otx2_cptlf.c | 131 +++++---
drivers/crypto/marvell/octeontx2/otx2_cptlf.h | 102 ++++--
drivers/crypto/marvell/octeontx2/otx2_cptpf.h | 4 +
.../marvell/octeontx2/otx2_cptpf_main.c | 76 ++---
.../marvell/octeontx2/otx2_cptpf_mbox.c | 81 ++++-
.../marvell/octeontx2/otx2_cptpf_ucode.c | 49 +--
.../marvell/octeontx2/otx2_cptpf_ucode.h | 3 +-
drivers/crypto/marvell/octeontx2/otx2_cptvf.h | 2 +
.../marvell/octeontx2/otx2_cptvf_algs.c | 31 ++
.../marvell/octeontx2/otx2_cptvf_algs.h | 5 +
.../marvell/octeontx2/otx2_cptvf_main.c | 25 +-
.../marvell/octeontx2/otx2_cptvf_mbox.c | 27 ++
.../marvell/octeontx2/otx2_cptvf_reqmgr.c | 162 +---------
.../ethernet/marvell/octeontx2/af/rvu_cpt.c | 14 +
.../ethernet/marvell/octeontx2/af/rvu_reg.h | 1 +
22 files changed, 1005 insertions(+), 305 deletions(-)
--
2.25.1
From: Nithin Dabilpuram <[email protected]>
Provide an option in Inline IPsec configure mailbox to configure the
CPT_AF_LFX_CTL:CTX_ILEN for inline CPT LF attached to CPT RVU PF.
This is needed to set the ctx ilen to size of inbound SA for
HW errata IPBUCPT-38756. Not setting this would lead to new context's
not being fetched.
Also set FLR_FLUSH in CPT_LF_CTX_CTL for CPT LF's as workaround
for same errata.
Signed-off-by: Nithin Dabilpuram <[email protected]>
---
.../marvell/octeontx2/otx2_cpt_common.h | 2 ++
.../marvell/octeontx2/otx2_cpt_hw_types.h | 4 ++-
drivers/crypto/marvell/octeontx2/otx2_cptlf.c | 33 +++++++++++++++++++
drivers/crypto/marvell/octeontx2/otx2_cptlf.h | 17 ++++++++++
.../marvell/octeontx2/otx2_cptpf_mbox.c | 5 +++
5 files changed, 60 insertions(+), 1 deletion(-)
diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h b/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h
index 6b8110208e27..6f5b85fe7d44 100644
--- a/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h
+++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_common.h
@@ -59,6 +59,8 @@ struct otx2_cpt_rx_inline_lf_cfg {
u32 credit_th;
u16 bpid;
u32 reserved;
+ u8 ctx_ilen_valid : 1;
+ u8 ctx_ilen : 7;
};
/*
diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_hw_types.h b/drivers/crypto/marvell/octeontx2/otx2_cpt_hw_types.h
index 06bcf49ee379..7e746a4def86 100644
--- a/drivers/crypto/marvell/octeontx2/otx2_cpt_hw_types.h
+++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_hw_types.h
@@ -102,6 +102,7 @@
#define OTX2_CPT_LF_Q_INST_PTR (0x110)
#define OTX2_CPT_LF_Q_GRP_PTR (0x120)
#define OTX2_CPT_LF_NQX(a) (0x400 | (a) << 3)
+#define OTX2_CPT_LF_CTX_CTL (0x500)
#define OTX2_CPT_LF_CTX_FLUSH (0x510)
#define OTX2_CPT_LF_CTX_ERR (0x520)
#define OTX2_CPT_RVU_FUNC_BLKADDR_SHIFT 20
@@ -472,7 +473,8 @@ union otx2_cptx_af_lf_ctrl {
u64 cont_err:1;
u64 reserved_11_15:5;
u64 nixtx_en:1;
- u64 reserved_17_47:31;
+ u64 ctx_ilen:3;
+ u64 reserved_17_47:28;
u64 grp:8;
u64 reserved_56_63:8;
} s;
diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptlf.c b/drivers/crypto/marvell/octeontx2/otx2_cptlf.c
index d7805e672047..571a8ec154e9 100644
--- a/drivers/crypto/marvell/octeontx2/otx2_cptlf.c
+++ b/drivers/crypto/marvell/octeontx2/otx2_cptlf.c
@@ -106,6 +106,33 @@ static int cptlf_set_grp_and_pri(struct otx2_cptlfs_info *lfs,
return ret;
}
+static int cptlf_set_ctx_ilen(struct otx2_cptlfs_info *lfs, int ctx_ilen)
+{
+ union otx2_cptx_af_lf_ctrl lf_ctrl;
+ struct otx2_cptlf_info *lf;
+ int slot, ret = 0;
+
+ for (slot = 0; slot < lfs->lfs_num; slot++) {
+ lf = &lfs->lf[slot];
+
+ ret = otx2_cpt_read_af_reg(lfs->mbox, lfs->pdev,
+ CPT_AF_LFX_CTL(lf->slot),
+ &lf_ctrl.u, lfs->blkaddr);
+ if (ret)
+ return ret;
+
+ lf_ctrl.s.ctx_ilen = ctx_ilen;
+
+ ret = otx2_cpt_write_af_reg(lfs->mbox, lfs->pdev,
+ CPT_AF_LFX_CTL(lf->slot),
+ lf_ctrl.u, lfs->blkaddr);
+ if (ret)
+ return ret;
+
+ }
+ return ret;
+}
+
static void cptlf_hw_init(struct otx2_cptlfs_info *lfs)
{
/* Disable instruction queues */
@@ -440,6 +467,12 @@ int otx2_cptlf_init(struct otx2_cptlfs_info *lfs, u8 eng_grp_mask, int pri,
if (ret)
goto free_iq;
+ if (lfs->ctx_ilen_ovrd) {
+ ret = cptlf_set_ctx_ilen(lfs, lfs->ctx_ilen);
+ if (ret)
+ goto free_iq;
+ }
+
return 0;
free_iq:
diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptlf.h b/drivers/crypto/marvell/octeontx2/otx2_cptlf.h
index f6138da945e9..cf4c055c50f4 100644
--- a/drivers/crypto/marvell/octeontx2/otx2_cptlf.h
+++ b/drivers/crypto/marvell/octeontx2/otx2_cptlf.h
@@ -120,6 +120,8 @@ struct otx2_cptlfs_info {
atomic_t state; /* LF's state. started/reset */
int blkaddr; /* CPT blkaddr: BLKADDR_CPT0/BLKADDR_CPT1 */
int global_slot; /* Global slot across the blocks */
+ u8 ctx_ilen;
+ u8 ctx_ilen_ovrd;
};
static inline void otx2_cpt_free_instruction_queues(
@@ -309,6 +311,17 @@ static inline void otx2_cptlf_set_iqueue_exec(struct otx2_cptlf_info *lf,
OTX2_CPT_LF_INPROG, lf_inprog.u);
}
+static inline void otx2_cptlf_set_ctx_flr_flush(struct otx2_cptlf_info *lf)
+{
+ u8 blkaddr = lf->lfs->blkaddr;
+ u64 val;
+
+ val = otx2_cpt_read64(lf->lfs->reg_base, blkaddr, lf->slot, OTX2_CPT_LF_CTX_CTL);
+ val |= BIT_ULL(0);
+
+ otx2_cpt_write64(lf->lfs->reg_base, blkaddr, lf->slot, OTX2_CPT_LF_CTX_CTL, val);
+}
+
static inline void otx2_cptlf_enable_iqueue_exec(struct otx2_cptlf_info *lf)
{
otx2_cptlf_set_iqueue_exec(lf, true);
@@ -324,6 +337,10 @@ static inline void otx2_cptlf_enable_iqueues(struct otx2_cptlfs_info *lfs)
int slot;
for (slot = 0; slot < lfs->lfs_num; slot++) {
+ /* Enable flush on FLR for Errata */
+ if (is_dev_cn10kb(lfs->pdev))
+ otx2_cptlf_set_ctx_flr_flush(&lfs->lf[slot]);
+
otx2_cptlf_enable_iqueue_exec(&lfs->lf[slot]);
otx2_cptlf_enable_iqueue_enq(&lfs->lf[slot]);
}
diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_mbox.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_mbox.c
index f7cb4ec74153..fd9632d2de03 100644
--- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_mbox.c
+++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_mbox.c
@@ -267,6 +267,9 @@ static int handle_msg_rx_inline_ipsec_lf_cfg(struct otx2_cptpf_dev *cptpf,
otx2_cptlf_set_dev_info(&cptpf->lfs, cptpf->pdev, cptpf->reg_base,
&cptpf->afpf_mbox, BLKADDR_CPT0);
cptpf->lfs.global_slot = 0;
+ cptpf->lfs.ctx_ilen_ovrd = cfg_req->ctx_ilen_valid;
+ cptpf->lfs.ctx_ilen = cfg_req->ctx_ilen;
+
ret = otx2_inline_cptlf_setup(cptpf, &cptpf->lfs, egrp, num_lfs);
if (ret) {
dev_err(&cptpf->pdev->dev, "Inline CPT0 LF setup failed.\n");
@@ -279,6 +282,8 @@ static int handle_msg_rx_inline_ipsec_lf_cfg(struct otx2_cptpf_dev *cptpf,
cptpf->reg_base, &cptpf->afpf_mbox,
BLKADDR_CPT1);
cptpf->cpt1_lfs.global_slot = num_lfs;
+ cptpf->cpt1_lfs.ctx_ilen_ovrd = cfg_req->ctx_ilen_valid;
+ cptpf->cpt1_lfs.ctx_ilen = cfg_req->ctx_ilen;
ret = otx2_inline_cptlf_setup(cptpf, &cptpf->cpt1_lfs, egrp, num_lfs);
if (ret) {
dev_err(&cptpf->pdev->dev, "Inline CPT1 LF setup failed.\n");
--
2.25.1
From: Nithin Dabilpuram <[email protected]>
Register errors interrupts for inline cptlf attached to PF driver
so that SMMU faults and other errors can be reported.
Signed-off-by: Nithin Dabilpuram <[email protected]>
---
drivers/crypto/marvell/octeontx2/otx2_cptlf.c | 98 ++++++++++++-------
drivers/crypto/marvell/octeontx2/otx2_cptlf.h | 6 +-
drivers/crypto/marvell/octeontx2/otx2_cptpf.h | 4 +
.../marvell/octeontx2/otx2_cptpf_main.c | 17 +++-
.../marvell/octeontx2/otx2_cptpf_mbox.c | 67 +++++++++++--
.../marvell/octeontx2/otx2_cptvf_main.c | 12 ++-
6 files changed, 149 insertions(+), 55 deletions(-)
diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptlf.c b/drivers/crypto/marvell/octeontx2/otx2_cptlf.c
index 6edd27ff8c4e..d7805e672047 100644
--- a/drivers/crypto/marvell/octeontx2/otx2_cptlf.c
+++ b/drivers/crypto/marvell/octeontx2/otx2_cptlf.c
@@ -151,26 +151,13 @@ static void cptlf_set_misc_intrs(struct otx2_cptlfs_info *lfs, u8 enable)
irq_misc.u);
}
-static void cptlf_enable_intrs(struct otx2_cptlfs_info *lfs)
+static void cptlf_set_done_intrs(struct otx2_cptlfs_info *lfs, u8 enable)
{
+ u64 reg = enable ? OTX2_CPT_LF_DONE_INT_ENA_W1S : OTX2_CPT_LF_DONE_INT_ENA_W1C;
int slot;
- /* Enable done interrupts */
for (slot = 0; slot < lfs->lfs_num; slot++)
- otx2_cpt_write64(lfs->reg_base, lfs->blkaddr, slot,
- OTX2_CPT_LF_DONE_INT_ENA_W1S, 0x1);
- /* Enable Misc interrupts */
- cptlf_set_misc_intrs(lfs, true);
-}
-
-static void cptlf_disable_intrs(struct otx2_cptlfs_info *lfs)
-{
- int slot;
-
- for (slot = 0; slot < lfs->lfs_num; slot++)
- otx2_cpt_write64(lfs->reg_base, lfs->blkaddr, slot,
- OTX2_CPT_LF_DONE_INT_ENA_W1C, 0x1);
- cptlf_set_misc_intrs(lfs, false);
+ otx2_cpt_write64(lfs->reg_base, lfs->blkaddr, slot, reg, 0x1);
}
static inline int cptlf_read_done_cnt(struct otx2_cptlf_info *lf)
@@ -257,24 +244,44 @@ static irqreturn_t cptlf_done_intr_handler(int irq, void *arg)
return IRQ_HANDLED;
}
-void otx2_cptlf_unregister_interrupts(struct otx2_cptlfs_info *lfs)
+void otx2_cptlf_unregister_misc_interrupts(struct otx2_cptlfs_info *lfs)
{
- int i, offs, vector;
+ int i, irq_offs, vector;
+ irq_offs = OTX2_CPT_LF_INT_VEC_E_MISC;
for (i = 0; i < lfs->lfs_num; i++) {
- for (offs = 0; offs < OTX2_CPT_LF_MSIX_VECTORS; offs++) {
- if (!lfs->lf[i].is_irq_reg[offs])
- continue;
+ if (!lfs->lf[i].is_irq_reg[irq_offs])
+ continue;
- vector = pci_irq_vector(lfs->pdev,
- lfs->lf[i].msix_offset + offs);
- free_irq(vector, &lfs->lf[i]);
- lfs->lf[i].is_irq_reg[offs] = false;
- }
+ vector = pci_irq_vector(lfs->pdev,
+ lfs->lf[i].msix_offset + irq_offs);
+ free_irq(vector, &lfs->lf[i]);
+ lfs->lf[i].is_irq_reg[irq_offs] = false;
+ }
+
+ cptlf_set_misc_intrs(lfs, false);
+}
+EXPORT_SYMBOL_NS_GPL(otx2_cptlf_unregister_misc_interrupts,
+ CRYPTO_DEV_OCTEONTX2_CPT);
+
+void otx2_cptlf_unregister_done_interrupts(struct otx2_cptlfs_info *lfs)
+{
+ int i, irq_offs, vector;
+
+ irq_offs = OTX2_CPT_LF_INT_VEC_E_DONE;
+ for (i = 0; i < lfs->lfs_num; i++) {
+ if (!lfs->lf[i].is_irq_reg[irq_offs])
+ continue;
+
+ vector = pci_irq_vector(lfs->pdev,
+ lfs->lf[i].msix_offset + irq_offs);
+ free_irq(vector, &lfs->lf[i]);
+ lfs->lf[i].is_irq_reg[irq_offs] = false;
}
- cptlf_disable_intrs(lfs);
+
+ cptlf_set_done_intrs(lfs, false);
}
-EXPORT_SYMBOL_NS_GPL(otx2_cptlf_unregister_interrupts,
+EXPORT_SYMBOL_NS_GPL(otx2_cptlf_unregister_done_interrupts,
CRYPTO_DEV_OCTEONTX2_CPT);
static int cptlf_do_register_interrrupts(struct otx2_cptlfs_info *lfs,
@@ -296,34 +303,51 @@ static int cptlf_do_register_interrrupts(struct otx2_cptlfs_info *lfs,
return ret;
}
-int otx2_cptlf_register_interrupts(struct otx2_cptlfs_info *lfs)
+int otx2_cptlf_register_misc_interrupts(struct otx2_cptlfs_info *lfs)
{
+ bool is_cpt1 = (lfs->blkaddr == BLKADDR_CPT1);
int irq_offs, ret, i;
+ irq_offs = OTX2_CPT_LF_INT_VEC_E_MISC;
for (i = 0; i < lfs->lfs_num; i++) {
- irq_offs = OTX2_CPT_LF_INT_VEC_E_MISC;
- snprintf(lfs->lf[i].irq_name[irq_offs], 32, "CPTLF Misc%d", i);
+ snprintf(lfs->lf[i].irq_name[irq_offs], 32, "CPT%dLF Misc%d",
+ is_cpt1, i);
ret = cptlf_do_register_interrrupts(lfs, i, irq_offs,
cptlf_misc_intr_handler);
if (ret)
goto free_irq;
+ }
+ cptlf_set_misc_intrs(lfs, true);
+ return 0;
- irq_offs = OTX2_CPT_LF_INT_VEC_E_DONE;
- snprintf(lfs->lf[i].irq_name[irq_offs], 32, "OTX2_CPTLF Done%d",
- i);
+free_irq:
+ otx2_cptlf_unregister_misc_interrupts(lfs);
+ return ret;
+}
+EXPORT_SYMBOL_NS_GPL(otx2_cptlf_register_misc_interrupts, CRYPTO_DEV_OCTEONTX2_CPT);
+
+int otx2_cptlf_register_done_interrupts(struct otx2_cptlfs_info *lfs)
+{
+ bool is_cpt1 = (lfs->blkaddr == BLKADDR_CPT1);
+ int irq_offs, ret, i;
+
+ irq_offs = OTX2_CPT_LF_INT_VEC_E_DONE;
+ for (i = 0; i < lfs->lfs_num; i++) {
+ snprintf(lfs->lf[i].irq_name[irq_offs], 32,
+ "OTX2_CPT%dLF Done%d", is_cpt1, i);
ret = cptlf_do_register_interrrupts(lfs, i, irq_offs,
cptlf_done_intr_handler);
if (ret)
goto free_irq;
}
- cptlf_enable_intrs(lfs);
+ cptlf_set_done_intrs(lfs, true);
return 0;
free_irq:
- otx2_cptlf_unregister_interrupts(lfs);
+ otx2_cptlf_unregister_done_interrupts(lfs);
return ret;
}
-EXPORT_SYMBOL_NS_GPL(otx2_cptlf_register_interrupts, CRYPTO_DEV_OCTEONTX2_CPT);
+EXPORT_SYMBOL_NS_GPL(otx2_cptlf_register_done_interrupts, CRYPTO_DEV_OCTEONTX2_CPT);
void otx2_cptlf_free_irqs_affinity(struct otx2_cptlfs_info *lfs)
{
diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptlf.h b/drivers/crypto/marvell/octeontx2/otx2_cptlf.h
index 4ce24aa1e941..f6138da945e9 100644
--- a/drivers/crypto/marvell/octeontx2/otx2_cptlf.h
+++ b/drivers/crypto/marvell/octeontx2/otx2_cptlf.h
@@ -409,8 +409,10 @@ static inline void otx2_cptlf_set_dev_info(struct otx2_cptlfs_info *lfs,
int otx2_cptlf_init(struct otx2_cptlfs_info *lfs, u8 eng_grp_msk, int pri,
int lfs_num);
void otx2_cptlf_shutdown(struct otx2_cptlfs_info *lfs);
-int otx2_cptlf_register_interrupts(struct otx2_cptlfs_info *lfs);
-void otx2_cptlf_unregister_interrupts(struct otx2_cptlfs_info *lfs);
+int otx2_cptlf_register_misc_interrupts(struct otx2_cptlfs_info *lfs);
+int otx2_cptlf_register_done_interrupts(struct otx2_cptlfs_info *lfs);
+void otx2_cptlf_unregister_misc_interrupts(struct otx2_cptlfs_info *lfs);
+void otx2_cptlf_unregister_done_interrupts(struct otx2_cptlfs_info *lfs);
void otx2_cptlf_free_irqs_affinity(struct otx2_cptlfs_info *lfs);
int otx2_cptlf_set_irqs_affinity(struct otx2_cptlfs_info *lfs);
diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf.h b/drivers/crypto/marvell/octeontx2/otx2_cptpf.h
index a209ec5af381..e5859a1e1c60 100644
--- a/drivers/crypto/marvell/octeontx2/otx2_cptpf.h
+++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf.h
@@ -71,4 +71,8 @@ void otx2_cptpf_afpf_mbox_up_handler(struct work_struct *work);
irqreturn_t otx2_cptpf_vfpf_mbox_intr(int irq, void *arg);
void otx2_cptpf_vfpf_mbox_handler(struct work_struct *work);
+int otx2_inline_cptlf_setup(struct otx2_cptpf_dev *cptpf,
+ struct otx2_cptlfs_info *lfs, u8 egrp, int num_lfs);
+void otx2_inline_cptlf_cleanup(struct otx2_cptlfs_info *lfs);
+
#endif /* __OTX2_CPTPF_H */
diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c
index 7d44b54659bf..79afa3a451a7 100644
--- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c
+++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c
@@ -725,7 +725,7 @@ static int otx2_cptpf_probe(struct pci_dev *pdev,
{
struct device *dev = &pdev->dev;
struct otx2_cptpf_dev *cptpf;
- int err;
+ int err, num_vec;
cptpf = devm_kzalloc(dev, sizeof(*cptpf), GFP_KERNEL);
if (!cptpf)
@@ -760,8 +760,11 @@ static int otx2_cptpf_probe(struct pci_dev *pdev,
if (err)
goto clear_drvdata;
- err = pci_alloc_irq_vectors(pdev, RVU_PF_INT_VEC_CNT,
- RVU_PF_INT_VEC_CNT, PCI_IRQ_MSIX);
+ num_vec = pci_msix_vec_count(cptpf->pdev);
+ if (num_vec <= 0)
+ goto clear_drvdata;
+
+ err = pci_alloc_irq_vectors(pdev, num_vec, num_vec, PCI_IRQ_MSIX);
if (err < 0) {
dev_err(dev, "Request for %d msix vectors failed\n",
RVU_PF_INT_VEC_CNT);
@@ -825,6 +828,14 @@ static void otx2_cptpf_remove(struct pci_dev *pdev)
cptpf_sriov_disable(pdev);
otx2_cpt_unregister_dl(cptpf);
+
+ /* Cleanup Inline CPT LF's if attached */
+ if (cptpf->lfs.lfs_num)
+ otx2_inline_cptlf_cleanup(&cptpf->lfs);
+
+ if (cptpf->cpt1_lfs.lfs_num)
+ otx2_inline_cptlf_cleanup(&cptpf->cpt1_lfs);
+
/* Delete sysfs entry created for kernel VF limits */
sysfs_remove_group(&pdev->dev.kobj, &cptpf_sysfs_group);
/* Cleanup engine groups */
diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_mbox.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_mbox.c
index bbabb57b4665..f7cb4ec74153 100644
--- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_mbox.c
+++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_mbox.c
@@ -199,6 +199,47 @@ static int rx_inline_ipsec_lf_cfg(struct otx2_cptpf_dev *cptpf, u8 egrp,
return send_inline_ipsec_inbound_msg(cptpf, req->sso_pf_func, 0);
}
+int
+otx2_inline_cptlf_setup(struct otx2_cptpf_dev *cptpf,
+ struct otx2_cptlfs_info *lfs, u8 egrp, int num_lfs)
+{
+ int ret;
+
+ ret = otx2_cptlf_init(lfs, 1 << egrp, OTX2_CPT_QUEUE_HI_PRIO, 1);
+ if (ret) {
+ dev_err(&cptpf->pdev->dev,
+ "LF configuration failed for RX inline ipsec.\n");
+ return ret;
+ }
+
+ /* Get msix offsets for attached LFs */
+ ret = otx2_cpt_msix_offset_msg(lfs);
+ if (ret)
+ goto cleanup_lf;
+
+ /* Register for CPT LF Misc interrupts */
+ ret = otx2_cptlf_register_misc_interrupts(lfs);
+ if (ret)
+ goto free_irq;
+
+ return 0;
+free_irq:
+ otx2_cptlf_unregister_misc_interrupts(lfs);
+cleanup_lf:
+ otx2_cptlf_shutdown(lfs);
+ return ret;
+}
+
+void
+otx2_inline_cptlf_cleanup(struct otx2_cptlfs_info *lfs)
+{
+ /* Unregister misc interrupt */
+ otx2_cptlf_unregister_misc_interrupts(lfs);
+
+ /* Cleanup LFs */
+ otx2_cptlf_shutdown(lfs);
+}
+
static int handle_msg_rx_inline_ipsec_lf_cfg(struct otx2_cptpf_dev *cptpf,
struct mbox_msghdr *req)
{
@@ -226,11 +267,9 @@ static int handle_msg_rx_inline_ipsec_lf_cfg(struct otx2_cptpf_dev *cptpf,
otx2_cptlf_set_dev_info(&cptpf->lfs, cptpf->pdev, cptpf->reg_base,
&cptpf->afpf_mbox, BLKADDR_CPT0);
cptpf->lfs.global_slot = 0;
- ret = otx2_cptlf_init(&cptpf->lfs, 1 << egrp, OTX2_CPT_QUEUE_HI_PRIO,
- num_lfs);
+ ret = otx2_inline_cptlf_setup(cptpf, &cptpf->lfs, egrp, num_lfs);
if (ret) {
- dev_err(&cptpf->pdev->dev,
- "LF configuration failed for RX inline ipsec.\n");
+ dev_err(&cptpf->pdev->dev, "Inline CPT0 LF setup failed.\n");
return ret;
}
@@ -240,11 +279,9 @@ static int handle_msg_rx_inline_ipsec_lf_cfg(struct otx2_cptpf_dev *cptpf,
cptpf->reg_base, &cptpf->afpf_mbox,
BLKADDR_CPT1);
cptpf->cpt1_lfs.global_slot = num_lfs;
- ret = otx2_cptlf_init(&cptpf->cpt1_lfs, 1 << egrp,
- OTX2_CPT_QUEUE_HI_PRIO, num_lfs);
+ ret = otx2_inline_cptlf_setup(cptpf, &cptpf->cpt1_lfs, egrp, num_lfs);
if (ret) {
- dev_err(&cptpf->pdev->dev,
- "LF configuration failed for RX inline ipsec.\n");
+ dev_err(&cptpf->pdev->dev, "Inline CPT1 LF setup failed.\n");
goto lf_cleanup;
}
cptpf->rsrc_req_blkaddr = 0;
@@ -257,9 +294,9 @@ static int handle_msg_rx_inline_ipsec_lf_cfg(struct otx2_cptpf_dev *cptpf,
return 0;
lf1_cleanup:
- otx2_cptlf_shutdown(&cptpf->cpt1_lfs);
+ otx2_inline_cptlf_cleanup(&cptpf->cpt1_lfs);
lf_cleanup:
- otx2_cptlf_shutdown(&cptpf->lfs);
+ otx2_inline_cptlf_cleanup(&cptpf->lfs);
return ret;
}
@@ -414,6 +451,8 @@ static void process_afpf_mbox_msg(struct otx2_cptpf_dev *cptpf,
struct otx2_cptlfs_info *lfs = &cptpf->lfs;
struct device *dev = &cptpf->pdev->dev;
struct cpt_rd_wr_reg_msg *rsp_rd_wr;
+ struct msix_offset_rsp *rsp_msix;
+ int i;
if (msg->id >= MBOX_MSG_MAX) {
dev_err(dev, "MBOX msg with unknown ID %d\n", msg->id);
@@ -432,6 +471,14 @@ static void process_afpf_mbox_msg(struct otx2_cptpf_dev *cptpf,
cptpf->pf_id = (msg->pcifunc >> RVU_PFVF_PF_SHIFT) &
RVU_PFVF_PF_MASK;
break;
+ case MBOX_MSG_MSIX_OFFSET:
+ rsp_msix = (struct msix_offset_rsp *) msg;
+ for (i = 0; i < rsp_msix->cptlfs; i++)
+ lfs->lf[i].msix_offset = rsp_msix->cptlf_msixoff[i];
+
+ for (i = 0; i < rsp_msix->cpt1_lfs; i++)
+ lfs->lf[i].msix_offset = rsp_msix->cpt1_lf_msixoff[i];
+ break;
case MBOX_MSG_CPT_RD_WR_REGISTER:
rsp_rd_wr = (struct cpt_rd_wr_reg_msg *)msg;
if (msg->rc) {
diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c
index 5d1e11135c17..83c3fd8e2e57 100644
--- a/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c
+++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_main.c
@@ -246,7 +246,8 @@ static void cptvf_lf_shutdown(struct otx2_cptlfs_info *lfs)
/* Unregister crypto algorithms */
otx2_cpt_crypto_exit(lfs->pdev, THIS_MODULE);
/* Unregister LFs interrupts */
- otx2_cptlf_unregister_interrupts(lfs);
+ otx2_cptlf_unregister_misc_interrupts(lfs);
+ otx2_cptlf_unregister_done_interrupts(lfs);
/* Cleanup LFs software side */
lf_sw_cleanup(lfs);
/* Send request to detach LFs */
@@ -298,7 +299,11 @@ static int cptvf_lf_init(struct otx2_cptvf_dev *cptvf)
goto cleanup_lf;
/* Register LFs interrupts */
- ret = otx2_cptlf_register_interrupts(lfs);
+ ret = otx2_cptlf_register_misc_interrupts(lfs);
+ if (ret)
+ goto cleanup_lf_sw;
+
+ ret = otx2_cptlf_register_done_interrupts(lfs);
if (ret)
goto cleanup_lf_sw;
@@ -319,7 +324,8 @@ static int cptvf_lf_init(struct otx2_cptvf_dev *cptvf)
disable_irqs:
otx2_cptlf_free_irqs_affinity(lfs);
unregister_intr:
- otx2_cptlf_unregister_interrupts(lfs);
+ otx2_cptlf_unregister_misc_interrupts(lfs);
+ otx2_cptlf_unregister_done_interrupts(lfs);
cleanup_lf_sw:
lf_sw_cleanup(lfs);
cleanup_lf:
--
2.25.1
HW has a errata that CPT HW may hit an issue, while processing CPT
instructions with CTX_VAL set and CTX_VAL not set. So, this patch
adds the code to always set the CTX_VAL as a workaround.
Signed-off-by: Srujana Challa <[email protected]>
---
drivers/crypto/marvell/octeontx2/cn10k_cpt.c | 68 +++++++++++++++++++
drivers/crypto/marvell/octeontx2/cn10k_cpt.h | 24 +++++++
.../marvell/octeontx2/otx2_cpt_hw_types.h | 2 +
.../marvell/octeontx2/otx2_cpt_reqmgr.h | 2 +
.../marvell/octeontx2/otx2_cptvf_algs.c | 31 +++++++++
.../marvell/octeontx2/otx2_cptvf_algs.h | 5 ++
.../marvell/octeontx2/otx2_cptvf_reqmgr.c | 2 +-
7 files changed, 133 insertions(+), 1 deletion(-)
diff --git a/drivers/crypto/marvell/octeontx2/cn10k_cpt.c b/drivers/crypto/marvell/octeontx2/cn10k_cpt.c
index b23ae3a020e0..a646aa01d5ef 100644
--- a/drivers/crypto/marvell/octeontx2/cn10k_cpt.c
+++ b/drivers/crypto/marvell/octeontx2/cn10k_cpt.c
@@ -96,6 +96,74 @@ int cn10k_cptvf_lmtst_init(struct otx2_cptvf_dev *cptvf)
}
EXPORT_SYMBOL_NS_GPL(cn10k_cptvf_lmtst_init, CRYPTO_DEV_OCTEONTX2_CPT);
+void cn10k_cpt_hw_ctx_clear(struct pci_dev *pdev, struct cn10k_cpt_errata_ctx *er_ctx)
+{
+ u64 cptr_dma;
+
+ if (!is_dev_cn10ka_ax(pdev))
+ return;
+
+ cptr_dma = er_ctx->cptr_dma & ~(BIT_ULL(60));
+ cn10k_cpt_ctx_flush(pdev, cptr_dma, true);
+ dma_unmap_single(&pdev->dev, cptr_dma, CN10K_CPT_HW_CTX_SIZE,
+ DMA_BIDIRECTIONAL);
+ kfree(er_ctx->hw_ctx);
+}
+EXPORT_SYMBOL_NS_GPL(cn10k_cpt_hw_ctx_clear, CRYPTO_DEV_OCTEONTX2_CPT);
+
+void cn10k_cpt_hw_ctx_set(union cn10k_cpt_hw_ctx *hctx, u16 ctx_sz)
+{
+ hctx->w0.aop_valid = 1;
+ hctx->w0.ctx_hdr_sz = 0;
+ hctx->w0.ctx_sz = ctx_sz;
+ hctx->w0.ctx_push_sz = 1;
+}
+EXPORT_SYMBOL_NS_GPL(cn10k_cpt_hw_ctx_set, CRYPTO_DEV_OCTEONTX2_CPT);
+
+int cn10k_cpt_hw_ctx_init(struct pci_dev *pdev, struct cn10k_cpt_errata_ctx *er_ctx)
+{
+ union cn10k_cpt_hw_ctx *hctx;
+ u64 cptr_dma;
+
+ er_ctx->cptr_dma = 0;
+ er_ctx->hw_ctx = NULL;
+
+ if (!is_dev_cn10ka_ax(pdev))
+ return 0;
+
+ hctx = kmalloc(CN10K_CPT_HW_CTX_SIZE, GFP_KERNEL);
+ if (unlikely(!hctx))
+ return -ENOMEM;
+ cptr_dma = dma_map_single(&pdev->dev, hctx, CN10K_CPT_HW_CTX_SIZE,
+ DMA_BIDIRECTIONAL);
+
+ cn10k_cpt_hw_ctx_set(hctx, 1);
+ er_ctx->hw_ctx = hctx;
+ er_ctx->cptr_dma = cptr_dma | BIT_ULL(60);
+
+ return 0;
+}
+EXPORT_SYMBOL_NS_GPL(cn10k_cpt_hw_ctx_init, CRYPTO_DEV_OCTEONTX2_CPT);
+
+void cn10k_cpt_ctx_flush(struct pci_dev *pdev, u64 cptr, bool inval)
+{
+ struct otx2_cptvf_dev *cptvf = pci_get_drvdata(pdev);
+ struct otx2_cptlfs_info *lfs = &cptvf->lfs;
+ u64 reg;
+
+ reg = (uintptr_t)cptr >> 7;
+ if (inval)
+ reg = reg | BIT_ULL(46);
+
+ otx2_cpt_write64(lfs->reg_base, lfs->blkaddr, lfs->lf[0].slot,
+ OTX2_CPT_LF_CTX_FLUSH, reg);
+ /* Make sure that the FLUSH operation is complete */
+ wmb();
+ otx2_cpt_read64(lfs->reg_base, lfs->blkaddr, lfs->lf[0].slot,
+ OTX2_CPT_LF_CTX_ERR);
+}
+EXPORT_SYMBOL_NS_GPL(cn10k_cpt_ctx_flush, CRYPTO_DEV_OCTEONTX2_CPT);
+
int cptvf_hw_ops_get(struct otx2_cptvf_dev *cptvf)
{
if (!test_bit(CN10K_LMTST, &cptvf->cap_flag)) {
diff --git a/drivers/crypto/marvell/octeontx2/cn10k_cpt.h b/drivers/crypto/marvell/octeontx2/cn10k_cpt.h
index 0f714ee564f5..9d47387a7669 100644
--- a/drivers/crypto/marvell/octeontx2/cn10k_cpt.h
+++ b/drivers/crypto/marvell/octeontx2/cn10k_cpt.h
@@ -8,6 +8,26 @@
#include "otx2_cptpf.h"
#include "otx2_cptvf.h"
+#define CN10K_CPT_HW_CTX_SIZE 256
+
+union cn10k_cpt_hw_ctx {
+ u64 u;
+ struct {
+ u64 reserved_0_47:48;
+ u64 ctx_push_sz:7;
+ u64 reserved_55:1;
+ u64 ctx_hdr_sz:2;
+ u64 aop_valid:1;
+ u64 reserved_59:1;
+ u64 ctx_sz:4;
+ } w0;
+};
+
+struct cn10k_cpt_errata_ctx {
+ union cn10k_cpt_hw_ctx *hw_ctx;
+ u64 cptr_dma;
+};
+
static inline u8 cn10k_cpt_get_compcode(union otx2_cpt_res_s *result)
{
return ((struct cn10k_cpt_res_s *)result)->compcode;
@@ -30,6 +50,10 @@ static inline u8 otx2_cpt_get_uc_compcode(union otx2_cpt_res_s *result)
int cn10k_cptpf_lmtst_init(struct otx2_cptpf_dev *cptpf);
int cn10k_cptvf_lmtst_init(struct otx2_cptvf_dev *cptvf);
+void cn10k_cpt_ctx_flush(struct pci_dev *pdev, u64 cptr, bool inval);
+int cn10k_cpt_hw_ctx_init(struct pci_dev *pdev, struct cn10k_cpt_errata_ctx *er_ctx);
+void cn10k_cpt_hw_ctx_clear(struct pci_dev *pdev, struct cn10k_cpt_errata_ctx *er_ctx);
+void cn10k_cpt_hw_ctx_set(union cn10k_cpt_hw_ctx *hctx, u16 ctx_sz);
int cptvf_hw_ops_get(struct otx2_cptvf_dev *cptvf);
#endif /* __CN10K_CPTLF_H */
diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_hw_types.h b/drivers/crypto/marvell/octeontx2/otx2_cpt_hw_types.h
index 756aee0c2b05..06bcf49ee379 100644
--- a/drivers/crypto/marvell/octeontx2/otx2_cpt_hw_types.h
+++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_hw_types.h
@@ -102,6 +102,8 @@
#define OTX2_CPT_LF_Q_INST_PTR (0x110)
#define OTX2_CPT_LF_Q_GRP_PTR (0x120)
#define OTX2_CPT_LF_NQX(a) (0x400 | (a) << 3)
+#define OTX2_CPT_LF_CTX_FLUSH (0x510)
+#define OTX2_CPT_LF_CTX_ERR (0x520)
#define OTX2_CPT_RVU_FUNC_BLKADDR_SHIFT 20
/* LMT LF registers */
#define OTX2_CPT_LMT_LFBASE BIT_ULL(OTX2_CPT_RVU_FUNC_BLKADDR_SHIFT)
diff --git a/drivers/crypto/marvell/octeontx2/otx2_cpt_reqmgr.h b/drivers/crypto/marvell/octeontx2/otx2_cpt_reqmgr.h
index fc5aca209837..7c990bb69ac5 100644
--- a/drivers/crypto/marvell/octeontx2/otx2_cpt_reqmgr.h
+++ b/drivers/crypto/marvell/octeontx2/otx2_cpt_reqmgr.h
@@ -40,6 +40,8 @@ struct otx2_cptvf_request {
u32 param2;
u16 dlen;
union otx2_cpt_opcode opcode;
+ dma_addr_t cptr_dma;
+ void *cptr;
};
/*
diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c
index e27ddd3c4e55..1604fc58dc13 100644
--- a/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c
+++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c
@@ -17,6 +17,7 @@
#include "otx2_cptvf.h"
#include "otx2_cptvf_algs.h"
#include "otx2_cpt_reqmgr.h"
+#include "cn10k_cpt.h"
/* Size of salt in AES GCM mode */
#define AES_GCM_SALT_SIZE 4
@@ -384,6 +385,9 @@ static inline int cpt_enc_dec(struct skcipher_request *req, u32 enc)
req_info->is_trunc_hmac = false;
req_info->ctrl.s.grp = otx2_cpt_get_kcrypto_eng_grp_num(pdev);
+ req_info->req.cptr = ctx->er_ctx.hw_ctx;
+ req_info->req.cptr_dma = ctx->er_ctx.cptr_dma;
+
/*
* We perform an asynchronous send and once
* the request is completed the driver would
@@ -530,6 +534,8 @@ static int otx2_cpt_enc_dec_init(struct crypto_skcipher *stfm)
struct otx2_cpt_enc_ctx *ctx = crypto_skcipher_ctx(stfm);
struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm);
struct crypto_alg *alg = tfm->__crt_alg;
+ struct pci_dev *pdev;
+ int ret, cpu_num;
memset(ctx, 0, sizeof(*ctx));
/*
@@ -541,6 +547,15 @@ static int otx2_cpt_enc_dec_init(struct crypto_skcipher *stfm)
stfm, sizeof(struct otx2_cpt_req_ctx) +
sizeof(struct skcipher_request));
+ ret = get_se_device(&pdev, &cpu_num);
+ if (ret)
+ return ret;
+
+ ctx->pdev = pdev;
+ ret = cn10k_cpt_hw_ctx_init(pdev, &ctx->er_ctx);
+ if (ret)
+ return ret;
+
return cpt_skcipher_fallback_init(ctx, alg);
}
@@ -552,6 +567,7 @@ static void otx2_cpt_skcipher_exit(struct crypto_skcipher *tfm)
crypto_free_skcipher(ctx->fbk_cipher);
ctx->fbk_cipher = NULL;
}
+ cn10k_cpt_hw_ctx_clear(ctx->pdev, &ctx->er_ctx);
}
static int cpt_aead_fallback_init(struct otx2_cpt_aead_ctx *ctx,
@@ -576,6 +592,8 @@ static int cpt_aead_init(struct crypto_aead *atfm, u8 cipher_type, u8 mac_type)
struct otx2_cpt_aead_ctx *ctx = crypto_aead_ctx_dma(atfm);
struct crypto_tfm *tfm = crypto_aead_tfm(atfm);
struct crypto_alg *alg = tfm->__crt_alg;
+ struct pci_dev *pdev;
+ int ret, cpu_num;
ctx->cipher_type = cipher_type;
ctx->mac_type = mac_type;
@@ -632,6 +650,15 @@ static int cpt_aead_init(struct crypto_aead *atfm, u8 cipher_type, u8 mac_type)
}
crypto_aead_set_reqsize_dma(atfm, sizeof(struct otx2_cpt_req_ctx));
+ ret = get_se_device(&pdev, &cpu_num);
+ if (ret)
+ return ret;
+
+ ctx->pdev = pdev;
+ ret = cn10k_cpt_hw_ctx_init(pdev, &ctx->er_ctx);
+ if (ret)
+ return ret;
+
return cpt_aead_fallback_init(ctx, alg);
}
@@ -694,6 +721,7 @@ static void otx2_cpt_aead_exit(struct crypto_aead *tfm)
crypto_free_aead(ctx->fbk_cipher);
ctx->fbk_cipher = NULL;
}
+ cn10k_cpt_hw_ctx_clear(ctx->pdev, &ctx->er_ctx);
}
static int otx2_cpt_aead_gcm_set_authsize(struct crypto_aead *tfm,
@@ -1299,6 +1327,9 @@ static int cpt_aead_enc_dec(struct aead_request *req, u8 reg_type, u8 enc)
req_info->is_enc = enc;
req_info->is_trunc_hmac = false;
+ req_info->req.cptr = ctx->er_ctx.hw_ctx;
+ req_info->req.cptr_dma = ctx->er_ctx.cptr_dma;
+
switch (reg_type) {
case OTX2_CPT_AEAD_ENC_DEC_REQ:
status = create_aead_input_list(req, enc);
diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.h b/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.h
index f04184bd1744..d29f84f01cee 100644
--- a/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.h
+++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.h
@@ -9,6 +9,7 @@
#include <crypto/skcipher.h>
#include <crypto/aead.h>
#include "otx2_cpt_common.h"
+#include "cn10k_cpt.h"
#define OTX2_CPT_MAX_ENC_KEY_SIZE 32
#define OTX2_CPT_MAX_HASH_KEY_SIZE 64
@@ -123,6 +124,8 @@ struct otx2_cpt_enc_ctx {
u8 key_type;
u8 enc_align_len;
struct crypto_skcipher *fbk_cipher;
+ struct pci_dev *pdev;
+ struct cn10k_cpt_errata_ctx er_ctx;
};
union otx2_cpt_offset_ctrl {
@@ -161,6 +164,8 @@ struct otx2_cpt_aead_ctx {
struct crypto_shash *hashalg;
struct otx2_cpt_sdesc *sdesc;
struct crypto_aead *fbk_cipher;
+ struct cn10k_cpt_errata_ctx er_ctx;
+ struct pci_dev *pdev;
u8 *ipad;
u8 *opad;
u32 enc_key_len;
diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c b/drivers/crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c
index 997a2eb60c66..5387c68f3c9d 100644
--- a/drivers/crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c
+++ b/drivers/crypto/marvell/octeontx2/otx2_cptvf_reqmgr.c
@@ -159,7 +159,7 @@ static int process_request(struct pci_dev *pdev, struct otx2_cpt_req_info *req,
cpu_to_be64s(&iq_cmd.cmd.u);
iq_cmd.dptr = info->dptr_baddr | info->gthr_sz << 60;
iq_cmd.rptr = info->rptr_baddr | info->sctr_sz << 60;
- iq_cmd.cptr.u = 0;
+ iq_cmd.cptr.s.cptr = cpt_req->cptr_dma;
iq_cmd.cptr.s.grp = ctrl->s.grp;
/* Fill in the CPT_INST_S type command for HW interpretation */
--
2.25.1