Received-SPF: pass (google.com: domain of linux-kernel-owner@vger.kernel.org designates 2620:137:e000::1:20 as permitted sender) client-ip=2620:137:e000::1:20;
Message-ID: <2ed4ecca-f36b-d7d9-cc4d-49c284cdedc1@huawei.com>
Date:   Fri, 5 Aug 2022 15:41:56 +0800
MIME-Version: 1.0
User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:91.0) Gecko/20100101
 Thunderbird/91.9.1
Subject: Re: [PATCH v4 5/7] dmaengine: hisilicon: Adapt DMA driver to
 HiSilicon IP09
To:     <vkoul@kernel.org>, <wangzhou1@hisilicon.com>
CC:     <liudongdong3@huawei.com>, <dmaengine@vger.kernel.org>,
        <linux-kernel@vger.kernel.org>
References: <20220625074422.3479591-1-haijie1@huawei.com>
 <20220802101226.9005-1-haijie1@huawei.com>
 <20220802101226.9005-6-haijie1@huawei.com>
From:   Jie Hai <haijie1@huawei.com>
In-Reply-To: <20220802101226.9005-6-haijie1@huawei.com>
Content-Type: text/plain; charset="UTF-8"; format=flowed
Content-Transfer-Encoding: 7bit
Precedence: bulk

On 2022/8/2 18:12, Jie Hai wrote:
> The HiSilicon IP08 and HiSilicon IP09 are DMA iEPs, they
> have the same pci device id but different pci revision.
> Unfortunately, they have different register layouts, so
> the origin driver cannot run on HiSilicon IP09 correctly.
> 
> This patch enables the driver to adapt to HiSilicon IP09.
> HiSilicon IP09 offers 4 channels, each channel has a send
> queue, a complete queue and an interrupt to help to do tasks.
> This DMA engine can do memory copy between memory blocks.
> 
> Signed-off-by: Jie Hai <haijie1@huawei.com>
> Acked-by: Zhou Wang <wangzhou1@hisilicon.com>
> ---
>   drivers/dma/hisi_dma.c | 379 ++++++++++++++++++++++++++++++++---------
>   1 file changed, 298 insertions(+), 81 deletions(-)
> 
> diff --git a/drivers/dma/hisi_dma.c b/drivers/dma/hisi_dma.c
> index 5d62fe62ba00..5fa3b6fa0529 100644
> --- a/drivers/dma/hisi_dma.c
> +++ b/drivers/dma/hisi_dma.c
> @@ -1,5 +1,6 @@
>   // SPDX-License-Identifier: GPL-2.0-only
> -/* Copyright(c) 2019 HiSilicon Limited. */
> +/* Copyright(c) 2019-2022 HiSilicon Limited. */
> +
>   #include <linux/bitfield.h>
>   #include <linux/dmaengine.h>
>   #include <linux/init.h>
> @@ -9,35 +10,85 @@
>   #include <linux/spinlock.h>
>   #include "virt-dma.h"
>   
> -#define HISI_DMA_SQ_BASE_L		0x0
> -#define HISI_DMA_SQ_BASE_H		0x4
> -#define HISI_DMA_SQ_DEPTH		0x8
> -#define HISI_DMA_SQ_TAIL_PTR		0xc
> -#define HISI_DMA_CQ_BASE_L		0x10
> -#define HISI_DMA_CQ_BASE_H		0x14
> -#define HISI_DMA_CQ_DEPTH		0x18
> -#define HISI_DMA_CQ_HEAD_PTR		0x1c
> -#define HISI_DMA_CTRL0			0x20
> -#define HISI_DMA_CTRL0_QUEUE_EN_S	0
> -#define HISI_DMA_CTRL0_QUEUE_PAUSE_S	4
> -#define HISI_DMA_CTRL1			0x24
> -#define HISI_DMA_CTRL1_QUEUE_RESET_S	0
> -#define HISI_DMA_Q_FSM_STS		0x30
> -#define HISI_DMA_FSM_STS_MASK		GENMASK(3, 0)
> -#define HISI_DMA_INT_STS		0x40
> -#define HISI_DMA_INT_STS_MASK		GENMASK(12, 0)
> -#define HISI_DMA_INT_MSK		0x44
> -#define HISI_DMA_MODE			0x217c
> -#define HISI_DMA_OFFSET			0x100
> -
> -#define HISI_DMA_MSI_NUM		32
> -#define HISI_DMA_CHAN_NUM		30
> -#define HISI_DMA_Q_DEPTH_VAL		1024
> -
> -#define PCI_BAR_2			2
> -
> -#define HISI_DMA_POLL_Q_STS_DELAY_US	10
> -#define HISI_DMA_POLL_Q_STS_TIME_OUT_US	1000
> +/* HiSilicon DMA register common field define */
> +#define HISI_DMA_Q_SQ_BASE_L			0x0
> +#define HISI_DMA_Q_SQ_BASE_H			0x4
> +#define HISI_DMA_Q_SQ_DEPTH			0x8
> +#define HISI_DMA_Q_SQ_TAIL_PTR			0xc
> +#define HISI_DMA_Q_CQ_BASE_L			0x10
> +#define HISI_DMA_Q_CQ_BASE_H			0x14
> +#define HISI_DMA_Q_CQ_DEPTH			0x18
> +#define HISI_DMA_Q_CQ_HEAD_PTR			0x1c
> +#define HISI_DMA_Q_CTRL0			0x20
> +#define HISI_DMA_Q_CTRL0_QUEUE_EN		BIT(0)
> +#define HISI_DMA_Q_CTRL0_QUEUE_PAUSE		BIT(4)
> +#define HISI_DMA_Q_CTRL1			0x24
> +#define HISI_DMA_Q_CTRL1_QUEUE_RESET		BIT(0)
> +#define HISI_DMA_Q_FSM_STS			0x30
> +#define HISI_DMA_Q_FSM_STS_MASK			GENMASK(3, 0)
> +#define HISI_DMA_Q_ERR_INT_NUM0			0x84
> +#define HISI_DMA_Q_ERR_INT_NUM1			0x88
> +#define HISI_DMA_Q_ERR_INT_NUM2			0x8c
> +
> +/* HiSilicon IP08 DMA register and field define */
> +#define HISI_DMA_HIP08_MODE			0x217C
> +#define HISI_DMA_HIP08_Q_BASE			0x0
> +#define HISI_DMA_HIP08_Q_CTRL0_ERR_ABORT_EN	BIT(2)
> +#define HISI_DMA_HIP08_Q_INT_STS		0x40
> +#define HISI_DMA_HIP08_Q_INT_MSK		0x44
> +#define HISI_DMA_HIP08_Q_INT_STS_MASK		GENMASK(14, 0)
> +#define HISI_DMA_HIP08_Q_ERR_INT_NUM3		0x90
> +#define HISI_DMA_HIP08_Q_ERR_INT_NUM4		0x94
> +#define HISI_DMA_HIP08_Q_ERR_INT_NUM5		0x98
> +#define HISI_DMA_HIP08_Q_ERR_INT_NUM6		0x48
> +#define HISI_DMA_HIP08_Q_CTRL0_SQCQ_DRCT	BIT(24)
> +
> +/* HiSilicon IP09 DMA register and field define */
> +#define HISI_DMA_HIP09_DMA_FLR_DISABLE		0xA00
> +#define HISI_DMA_HIP09_DMA_FLR_DISABLE_B	BIT(0)
> +#define HISI_DMA_HIP09_Q_BASE			0x2000
> +#define HISI_DMA_HIP09_Q_CTRL0_ERR_ABORT_EN	GENMASK(31, 28)
> +#define HISI_DMA_HIP09_Q_CTRL0_SQ_DRCT		BIT(26)
> +#define HISI_DMA_HIP09_Q_CTRL0_CQ_DRCT		BIT(27)
> +#define HISI_DMA_HIP09_Q_CTRL1_VA_ENABLE	BIT(2)
> +#define HISI_DMA_HIP09_Q_INT_STS		0x40
> +#define HISI_DMA_HIP09_Q_INT_MSK		0x44
> +#define HISI_DMA_HIP09_Q_INT_STS_MASK		0x1
> +#define HISI_DMA_HIP09_Q_ERR_INT_STS		0x48
> +#define HISI_DMA_HIP09_Q_ERR_INT_MSK		0x4C
> +#define HISI_DMA_HIP09_Q_ERR_INT_STS_MASK	GENMASK(18, 1)
> +#define HISI_DMA_HIP09_PORT_CFG_REG(port_id)	(0x800 + \
> +						(port_id) * 0x20)
> +#define HISI_DMA_HIP09_PORT_CFG_LINK_DOWN_MASK_B	BIT(16)
> +
> +#define HISI_DMA_HIP09_MAX_PORT_NUM		16
> +
> +#define HISI_DMA_HIP08_MSI_NUM			32
> +#define HISI_DMA_HIP08_CHAN_NUM			30
> +#define HISI_DMA_HIP09_MSI_NUM			4
> +#define HISI_DMA_HIP09_CHAN_NUM			4
> +#define HISI_DMA_REVISION_HIP08B		0x21
> +#define HISI_DMA_REVISION_HIP09A		0x30
> +
> +#define HISI_DMA_Q_OFFSET			0x100
> +#define HISI_DMA_Q_DEPTH_VAL			1024
> +
> +#define PCI_BAR_2				2
> +
> +#define HISI_DMA_POLL_Q_STS_DELAY_US		10
> +#define HISI_DMA_POLL_Q_STS_TIME_OUT_US		1000
> +
> +/*
> + * The HIP08B(HiSilicon IP08) and HIP09A(HiSilicon IP09) are DMA iEPs, they
> + * have the same pci device id but different pci revision.
> + * Unfortunately, they have different register layouts, so two layout
> + * enumerations are defined.
> + */
> +enum hisi_dma_reg_layout {
> +	HISI_DMA_REG_LAYOUT_INVALID = 0,
> +	HISI_DMA_REG_LAYOUT_HIP08,
> +	HISI_DMA_REG_LAYOUT_HIP09
> +};
>   
>   enum hisi_dma_mode {
>   	EP = 0,
> @@ -108,9 +159,45 @@ struct hisi_dma_dev {
>   	struct dma_device dma_dev;
>   	u32 chan_num;
>   	u32 chan_depth;
> +	enum hisi_dma_reg_layout reg_layout;
> +	void __iomem *queue_base; /* queue region start of register */
>   	struct hisi_dma_chan chan[];
>   };
>   
> +static enum hisi_dma_reg_layout hisi_dma_get_reg_layout(struct pci_dev *pdev)
> +{
> +	if (pdev->revision == HISI_DMA_REVISION_HIP08B)
> +		return HISI_DMA_REG_LAYOUT_HIP08;
> +	else if (pdev->revision >= HISI_DMA_REVISION_HIP09A)
> +		return HISI_DMA_REG_LAYOUT_HIP09;
> +
> +	return HISI_DMA_REG_LAYOUT_INVALID;
> +}
> +
> +static u32 hisi_dma_get_chan_num(struct pci_dev *pdev)
> +{
> +	if (pdev->revision == HISI_DMA_REVISION_HIP08B)
> +		return HISI_DMA_HIP08_CHAN_NUM;
> +
> +	return HISI_DMA_HIP09_CHAN_NUM;
> +}
> +
> +static u32 hisi_dma_get_msi_num(struct pci_dev *pdev)
> +{
> +	if (pdev->revision == HISI_DMA_REVISION_HIP08B)
> +		return HISI_DMA_HIP08_MSI_NUM;
> +
> +	return HISI_DMA_HIP09_MSI_NUM;
> +}
> +
> +static u32 hisi_dma_get_queue_base(struct pci_dev *pdev)
> +{
> +	if (pdev->revision == HISI_DMA_REVISION_HIP08B)
> +		return HISI_DMA_HIP08_Q_BASE;
> +
> +	return HISI_DMA_HIP09_Q_BASE;
> +}
> +
>   static inline struct hisi_dma_chan *to_hisi_dma_chan(struct dma_chan *c)
>   {
>   	return container_of(c, struct hisi_dma_chan, vc.chan);
> @@ -124,7 +211,7 @@ static inline struct hisi_dma_desc *to_hisi_dma_desc(struct virt_dma_desc *vd)
>   static inline void hisi_dma_chan_write(void __iomem *base, u32 reg, u32 index,
>   				       u32 val)
>   {
> -	writel_relaxed(val, base + reg + index * HISI_DMA_OFFSET);
> +	writel_relaxed(val, base + reg + index * HISI_DMA_Q_OFFSET);
>   }
>   
>   static inline void hisi_dma_update_bit(void __iomem *addr, u32 pos, bool val)
> @@ -132,55 +219,83 @@ static inline void hisi_dma_update_bit(void __iomem *addr, u32 pos, bool val)
>   	u32 tmp;
>   
>   	tmp = readl_relaxed(addr);
> -	tmp = val ? tmp | BIT(pos) : tmp & ~BIT(pos);
> +	tmp = val ? tmp | pos : tmp & ~pos;
>   	writel_relaxed(tmp, addr);
>   }
>   
>   static void hisi_dma_pause_dma(struct hisi_dma_dev *hdma_dev, u32 index,
>   			       bool pause)
>   {
> -	void __iomem *addr = hdma_dev->base + HISI_DMA_CTRL0 + index *
> -			     HISI_DMA_OFFSET;
> +	void __iomem *addr;
>   
> -	hisi_dma_update_bit(addr, HISI_DMA_CTRL0_QUEUE_PAUSE_S, pause);
> +	addr = hdma_dev->queue_base + HISI_DMA_Q_CTRL0 +
> +	       index * HISI_DMA_Q_OFFSET;
> +	hisi_dma_update_bit(addr, HISI_DMA_Q_CTRL0_QUEUE_PAUSE, pause);
>   }
>   
>   static void hisi_dma_enable_dma(struct hisi_dma_dev *hdma_dev, u32 index,
>   				bool enable)
>   {
> -	void __iomem *addr = hdma_dev->base + HISI_DMA_CTRL0 + index *
> -			     HISI_DMA_OFFSET;
> +	void __iomem *addr;
>   
> -	hisi_dma_update_bit(addr, HISI_DMA_CTRL0_QUEUE_EN_S, enable);
> +	addr = hdma_dev->queue_base + HISI_DMA_Q_CTRL0 +
> +	       index * HISI_DMA_Q_OFFSET;
> +	hisi_dma_update_bit(addr, HISI_DMA_Q_CTRL0_QUEUE_EN, enable);
>   }
>   
>   static void hisi_dma_mask_irq(struct hisi_dma_dev *hdma_dev, u32 qp_index)
>   {
> -	hisi_dma_chan_write(hdma_dev->base, HISI_DMA_INT_MSK, qp_index,
> -			    HISI_DMA_INT_STS_MASK);
> +	void __iomem *q_base = hdma_dev->queue_base;
> +
> +	if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08)
> +		hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_INT_MSK,
> +				    qp_index, HISI_DMA_HIP08_Q_INT_STS_MASK);
> +	else {
> +		hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_INT_MSK,
> +				    qp_index, HISI_DMA_HIP09_Q_INT_STS_MASK);
> +		hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_ERR_INT_MSK,
> +				    qp_index,
> +				    HISI_DMA_HIP09_Q_ERR_INT_STS_MASK);
> +	}
>   }
>   
>   static void hisi_dma_unmask_irq(struct hisi_dma_dev *hdma_dev, u32 qp_index)
>   {
> -	void __iomem *base = hdma_dev->base;
> -
> -	hisi_dma_chan_write(base, HISI_DMA_INT_STS, qp_index,
> -			    HISI_DMA_INT_STS_MASK);
> -	hisi_dma_chan_write(base, HISI_DMA_INT_MSK, qp_index, 0);
> +	void __iomem *q_base = hdma_dev->queue_base;
> +
> +	if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) {
> +		hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_INT_STS,
> +				    qp_index, HISI_DMA_HIP08_Q_INT_STS_MASK);
> +		hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_INT_MSK,
> +				    qp_index, 0);
> +	} else {
> +		hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_INT_STS,
> +				    qp_index, HISI_DMA_HIP09_Q_INT_STS_MASK);
> +		hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_ERR_INT_STS,
> +				    qp_index,
> +				    HISI_DMA_HIP09_Q_ERR_INT_STS_MASK);
> +		hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_INT_MSK,
> +				    qp_index, 0);
> +		hisi_dma_chan_write(q_base, HISI_DMA_HIP09_Q_ERR_INT_MSK,
> +				    qp_index, 0);
> +	}
>   }
>   
>   static void hisi_dma_do_reset(struct hisi_dma_dev *hdma_dev, u32 index)
>   {
> -	void __iomem *addr = hdma_dev->base + HISI_DMA_CTRL1 + index *
> -			     HISI_DMA_OFFSET;
> +	void __iomem *addr;
>   
> -	hisi_dma_update_bit(addr, HISI_DMA_CTRL1_QUEUE_RESET_S, 1);
> +	addr = hdma_dev->queue_base +
> +	       HISI_DMA_Q_CTRL1 + index * HISI_DMA_Q_OFFSET;
> +	hisi_dma_update_bit(addr, HISI_DMA_Q_CTRL1_QUEUE_RESET, 1);
>   }
>   
>   static void hisi_dma_reset_qp_point(struct hisi_dma_dev *hdma_dev, u32 index)
>   {
> -	hisi_dma_chan_write(hdma_dev->base, HISI_DMA_SQ_TAIL_PTR, index, 0);
> -	hisi_dma_chan_write(hdma_dev->base, HISI_DMA_CQ_HEAD_PTR, index, 0);
> +	void __iomem *q_base = hdma_dev->queue_base;
> +
> +	hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_TAIL_PTR, index, 0);
> +	hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_HEAD_PTR, index, 0);
>   }
>   
>   static void hisi_dma_reset_or_disable_hw_chan(struct hisi_dma_chan *chan,
> @@ -196,10 +311,10 @@ static void hisi_dma_reset_or_disable_hw_chan(struct hisi_dma_chan *chan,
>   	hisi_dma_mask_irq(hdma_dev, index);
>   
>   	addr = hdma_dev->base +
> -	       HISI_DMA_Q_FSM_STS + index * HISI_DMA_OFFSET;
> +	       HISI_DMA_Q_FSM_STS + index * HISI_DMA_Q_OFFSET;
>   

This is a channel related register and the base address should be 
hdma_dev->queue_base. This problem was introduced in v3 and will be 
fixed in next version.

>   	ret = readl_relaxed_poll_timeout(addr, tmp,
> -		FIELD_GET(HISI_DMA_FSM_STS_MASK, tmp) != RUN,
> +		FIELD_GET(HISI_DMA_Q_FSM_STS_MASK, tmp) != RUN,
>   		HISI_DMA_POLL_Q_STS_DELAY_US, HISI_DMA_POLL_Q_STS_TIME_OUT_US);
>   	if (ret) {
>   		dev_err(&hdma_dev->pdev->dev, "disable channel timeout!\n");
> @@ -216,7 +331,7 @@ static void hisi_dma_reset_or_disable_hw_chan(struct hisi_dma_chan *chan,
>   	}
>   
>   	ret = readl_relaxed_poll_timeout(addr, tmp,
> -		FIELD_GET(HISI_DMA_FSM_STS_MASK, tmp) == IDLE,
> +		FIELD_GET(HISI_DMA_Q_FSM_STS_MASK, tmp) == IDLE,
>   		HISI_DMA_POLL_Q_STS_DELAY_US, HISI_DMA_POLL_Q_STS_TIME_OUT_US);
>   	if (ret) {
>   		dev_err(&hdma_dev->pdev->dev, "reset channel timeout!\n");
> @@ -298,8 +413,8 @@ static void hisi_dma_start_transfer(struct hisi_dma_chan *chan)
>   	chan->sq_tail = (chan->sq_tail + 1) % hdma_dev->chan_depth;
>   
>   	/* update sq_tail to trigger a new task */
> -	hisi_dma_chan_write(hdma_dev->base, HISI_DMA_SQ_TAIL_PTR, chan->qp_num,
> -			    chan->sq_tail);
> +	hisi_dma_chan_write(hdma_dev->queue_base, HISI_DMA_Q_SQ_TAIL_PTR,
> +			    chan->qp_num, chan->sq_tail);
>   }
>   
>   static void hisi_dma_issue_pending(struct dma_chan *c)
> @@ -373,26 +488,86 @@ static int hisi_dma_alloc_qps_mem(struct hisi_dma_dev *hdma_dev)
>   static void hisi_dma_init_hw_qp(struct hisi_dma_dev *hdma_dev, u32 index)
>   {
>   	struct hisi_dma_chan *chan = &hdma_dev->chan[index];
> +	void __iomem *q_base = hdma_dev->queue_base;
>   	u32 hw_depth = hdma_dev->chan_depth - 1;
> -	void __iomem *base = hdma_dev->base;
> +	void __iomem *addr;
> +	u32 tmp;
>   
>   	/* set sq, cq base */
> -	hisi_dma_chan_write(base, HISI_DMA_SQ_BASE_L, index,
> +	hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_BASE_L, index,
>   			    lower_32_bits(chan->sq_dma));
> -	hisi_dma_chan_write(base, HISI_DMA_SQ_BASE_H, index,
> +	hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_BASE_H, index,
>   			    upper_32_bits(chan->sq_dma));
> -	hisi_dma_chan_write(base, HISI_DMA_CQ_BASE_L, index,
> +	hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_BASE_L, index,
>   			    lower_32_bits(chan->cq_dma));
> -	hisi_dma_chan_write(base, HISI_DMA_CQ_BASE_H, index,
> +	hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_BASE_H, index,
>   			    upper_32_bits(chan->cq_dma));
>   
>   	/* set sq, cq depth */
> -	hisi_dma_chan_write(base, HISI_DMA_SQ_DEPTH, index, hw_depth);
> -	hisi_dma_chan_write(base, HISI_DMA_CQ_DEPTH, index, hw_depth);
> +	hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_DEPTH, index, hw_depth);
> +	hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_DEPTH, index, hw_depth);
>   
>   	/* init sq tail and cq head */
> -	hisi_dma_chan_write(base, HISI_DMA_SQ_TAIL_PTR, index, 0);
> -	hisi_dma_chan_write(base, HISI_DMA_CQ_HEAD_PTR, index, 0);
> +	hisi_dma_chan_write(q_base, HISI_DMA_Q_SQ_TAIL_PTR, index, 0);
> +	hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_HEAD_PTR, index, 0);
> +
> +	/* init error interrupt stats */
> +	hisi_dma_chan_write(q_base, HISI_DMA_Q_ERR_INT_NUM0, index, 0);
> +	hisi_dma_chan_write(q_base, HISI_DMA_Q_ERR_INT_NUM1, index, 0);
> +	hisi_dma_chan_write(q_base, HISI_DMA_Q_ERR_INT_NUM2, index, 0);
> +
> +	if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08) {
> +		hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_ERR_INT_NUM3,
> +				    index, 0);
> +		hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_ERR_INT_NUM4,
> +				    index, 0);
> +		hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_ERR_INT_NUM5,
> +				    index, 0);
> +		hisi_dma_chan_write(q_base, HISI_DMA_HIP08_Q_ERR_INT_NUM6,
> +				    index, 0);
> +		/*
> +		 * init SQ/CQ direction selecting register.
> +		 * "0" is to local side and "1" is to remote side.
> +		 */
> +		addr = q_base + HISI_DMA_Q_CTRL0 + index * HISI_DMA_Q_OFFSET;
> +		hisi_dma_update_bit(addr, HISI_DMA_HIP08_Q_CTRL0_SQCQ_DRCT, 0);
> +
> +		/*
> +		 * 0 - Continue to next descriptor if error occurs.
> +		 * 1 - Abort the DMA queue if error occurs.
> +		 */
> +		hisi_dma_update_bit(addr,
> +				    HISI_DMA_HIP08_Q_CTRL0_ERR_ABORT_EN, 0);
> +	} else {
> +		addr = q_base + HISI_DMA_Q_CTRL0 + index * HISI_DMA_Q_OFFSET;
> +
> +		/*
> +		 * init SQ/CQ direction selecting register.
> +		 * "0" is to local side and "1" is to remote side.
> +		 */
> +		hisi_dma_update_bit(addr, HISI_DMA_HIP09_Q_CTRL0_SQ_DRCT, 0);
> +		hisi_dma_update_bit(addr, HISI_DMA_HIP09_Q_CTRL0_CQ_DRCT, 0);
> +
> +		/*
> +		 * 0 - Continue to next descriptor if error occurs.
> +		 * 1 - Abort the DMA queue if error occurs.
> +		 */
> +
> +		tmp = readl_relaxed(addr);
> +		tmp &= ~HISI_DMA_HIP09_Q_CTRL0_ERR_ABORT_EN;
> +		writel_relaxed(tmp, addr);
> +
> +		/*
> +		 * 0 - dma should process FLR whith CPU.
> +		 * 1 - dma not process FLR, only cpu process FLR.
> +		 */
> +		addr = q_base + HISI_DMA_HIP09_DMA_FLR_DISABLE +
> +		       index * HISI_DMA_Q_OFFSET;
> +		hisi_dma_update_bit(addr, HISI_DMA_HIP09_DMA_FLR_DISABLE_B, 0);
> +
> +		addr = q_base + HISI_DMA_Q_CTRL1 + index * HISI_DMA_Q_OFFSET;
> +		hisi_dma_update_bit(addr, HISI_DMA_HIP09_Q_CTRL1_VA_ENABLE, 1);
> +	}
>   }
>   
>   static void hisi_dma_enable_qp(struct hisi_dma_dev *hdma_dev, u32 qp_index)
> @@ -436,14 +611,16 @@ static irqreturn_t hisi_dma_irq(int irq, void *data)
>   	struct hisi_dma_dev *hdma_dev = chan->hdma_dev;
>   	struct hisi_dma_desc *desc;
>   	struct hisi_dma_cqe *cqe;
> +	void __iomem *q_base;
>   
>   	spin_lock(&chan->vc.lock);
>   
>   	desc = chan->desc;
>   	cqe = chan->cq + chan->cq_head;
> +	q_base = hdma_dev->queue_base;
>   	if (desc) {
>   		chan->cq_head = (chan->cq_head + 1) % hdma_dev->chan_depth;
> -		hisi_dma_chan_write(hdma_dev->base, HISI_DMA_CQ_HEAD_PTR,
> +		hisi_dma_chan_write(q_base, HISI_DMA_Q_CQ_HEAD_PTR,
>   				    chan->qp_num, chan->cq_head);
>   		if (FIELD_GET(STATUS_MASK, cqe->w0) == STATUS_SUCC) {
>   			vchan_cookie_complete(&desc->vd);
> @@ -504,16 +681,58 @@ static void hisi_dma_disable_hw_channels(void *data)
>   static void hisi_dma_set_mode(struct hisi_dma_dev *hdma_dev,
>   			      enum hisi_dma_mode mode)
>   {
> -	writel_relaxed(mode == RC ? 1 : 0, hdma_dev->base + HISI_DMA_MODE);
> +	if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP08)
> +		writel_relaxed(mode == RC ? 1 : 0,
> +			       hdma_dev->base + HISI_DMA_HIP08_MODE);
> +}
> +
> +static void hisi_dma_init_hw(struct hisi_dma_dev *hdma_dev)
> +{
> +	void __iomem *addr;
> +	int i;
> +
> +	if (hdma_dev->reg_layout == HISI_DMA_REG_LAYOUT_HIP09) {
> +		for (i = 0; i < HISI_DMA_HIP09_MAX_PORT_NUM; i++) {
> +			addr = hdma_dev->base + HISI_DMA_HIP09_PORT_CFG_REG(i);
> +			hisi_dma_update_bit(addr,
> +				HISI_DMA_HIP09_PORT_CFG_LINK_DOWN_MASK_B, 1);
> +		}
> +	}
> +}
> +
> +static void hisi_dma_init_dma_dev(struct hisi_dma_dev *hdma_dev)
> +{
> +	struct dma_device *dma_dev;
> +
> +	dma_dev = &hdma_dev->dma_dev;
> +	dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
> +	dma_dev->device_free_chan_resources = hisi_dma_free_chan_resources;
> +	dma_dev->device_prep_dma_memcpy = hisi_dma_prep_dma_memcpy;
> +	dma_dev->device_tx_status = hisi_dma_tx_status;
> +	dma_dev->device_issue_pending = hisi_dma_issue_pending;
> +	dma_dev->device_terminate_all = hisi_dma_terminate_all;
> +	dma_dev->device_synchronize = hisi_dma_synchronize;
> +	dma_dev->directions = BIT(DMA_MEM_TO_MEM);
> +	dma_dev->dev = &hdma_dev->pdev->dev;
> +	INIT_LIST_HEAD(&dma_dev->channels);
>   }
>   
>   static int hisi_dma_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>   {
> +	enum hisi_dma_reg_layout reg_layout;
>   	struct device *dev = &pdev->dev;
>   	struct hisi_dma_dev *hdma_dev;
>   	struct dma_device *dma_dev;
> +	u32 chan_num;
> +	u32 msi_num;
>   	int ret;
>   
> +	reg_layout = hisi_dma_get_reg_layout(pdev);
> +	if (reg_layout == HISI_DMA_REG_LAYOUT_INVALID) {
> +		dev_err(dev, "unsupported device!\n");
> +		return -EINVAL;
> +	}
> +
>   	ret = pcim_enable_device(pdev);
>   	if (ret) {
>   		dev_err(dev, "failed to enable device mem!\n");
> @@ -530,40 +749,37 @@ static int hisi_dma_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>   	if (ret)
>   		return ret;
>   
> -	hdma_dev = devm_kzalloc(dev, struct_size(hdma_dev, chan, HISI_DMA_CHAN_NUM), GFP_KERNEL);
> +	chan_num = hisi_dma_get_chan_num(pdev);
> +	hdma_dev = devm_kzalloc(dev, struct_size(hdma_dev, chan, chan_num),
> +				GFP_KERNEL);
>   	if (!hdma_dev)
>   		return -EINVAL;
>   
>   	hdma_dev->base = pcim_iomap_table(pdev)[PCI_BAR_2];
>   	hdma_dev->pdev = pdev;
> -	hdma_dev->chan_num = HISI_DMA_CHAN_NUM;
>   	hdma_dev->chan_depth = HISI_DMA_Q_DEPTH_VAL;
> +	hdma_dev->chan_num = chan_num;
> +	hdma_dev->reg_layout = reg_layout;
> +	hdma_dev->queue_base = hdma_dev->base + hisi_dma_get_queue_base(pdev);
>   
>   	pci_set_drvdata(pdev, hdma_dev);
>   	pci_set_master(pdev);
>   
> +	msi_num = hisi_dma_get_msi_num(pdev);
> +
>   	/* This will be freed by 'pcim_release()'. See 'pcim_enable_device()' */
> -	ret = pci_alloc_irq_vectors(pdev, HISI_DMA_MSI_NUM, HISI_DMA_MSI_NUM,
> -				    PCI_IRQ_MSI);
> +	ret = pci_alloc_irq_vectors(pdev, msi_num, msi_num, PCI_IRQ_MSI);
>   	if (ret < 0) {
>   		dev_err(dev, "Failed to allocate MSI vectors!\n");
>   		return ret;
>   	}
>   
> -	dma_dev = &hdma_dev->dma_dev;
> -	dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
> -	dma_dev->device_free_chan_resources = hisi_dma_free_chan_resources;
> -	dma_dev->device_prep_dma_memcpy = hisi_dma_prep_dma_memcpy;
> -	dma_dev->device_tx_status = hisi_dma_tx_status;
> -	dma_dev->device_issue_pending = hisi_dma_issue_pending;
> -	dma_dev->device_terminate_all = hisi_dma_terminate_all;
> -	dma_dev->device_synchronize = hisi_dma_synchronize;
> -	dma_dev->directions = BIT(DMA_MEM_TO_MEM);
> -	dma_dev->dev = dev;
> -	INIT_LIST_HEAD(&dma_dev->channels);
> +	hisi_dma_init_dma_dev(hdma_dev);
>   
>   	hisi_dma_set_mode(hdma_dev, RC);
>   
> +	hisi_dma_init_hw(hdma_dev);
> +
>   	ret = hisi_dma_enable_hw_channels(hdma_dev);
>   	if (ret < 0) {
>   		dev_err(dev, "failed to enable hw channel!\n");
> @@ -575,6 +791,7 @@ static int hisi_dma_probe(struct pci_dev *pdev, const struct pci_device_id *id)
>   	if (ret)
>   		return ret;
>   
> +	dma_dev = &hdma_dev->dma_dev;
>   	ret = dmaenginem_async_device_register(dma_dev);
>   	if (ret < 0)
>   		dev_err(dev, "failed to register device!\n");