Received-SPF: pass (google.com: domain of linux-kernel-owner@vger.kernel.org designates 23.128.96.18 as permitted sender) client-ip=23.128.96.18;
Subject: Re: [PATCH v1] vdpa/mlx5: Fix memory key MTT population
To:     Eli Cohen <elic@nvidia.com>, mst@redhat.com,
        virtualization@lists.linux-foundation.org, netdev@vger.kernel.org,
        linux-kernel@vger.kernel.org
Cc:     lulu@redhat.com
References: <20210107071845.GA224876@mtl-vdi-166.wap.labs.mlnx>
From:   Jason Wang <jasowang@redhat.com>
Message-ID: <07d336a3-7fc2-5e4a-667a-495b5bb755da@redhat.com>
Date:   Fri, 8 Jan 2021 16:38:55 +0800
User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101
 Thunderbird/68.10.0
MIME-Version: 1.0
In-Reply-To: <20210107071845.GA224876@mtl-vdi-166.wap.labs.mlnx>
Content-Type: text/plain; charset=utf-8; format=flowed
Content-Transfer-Encoding: 8bit
Content-Language: en-US
Precedence: bulk


On 2021/1/7 下午3:18, Eli Cohen wrote:
> map_direct_mr() assumed that the number of scatter/gather entries
> returned by dma_map_sg_attrs() was equal to the number of segments in
> the sgl list. This led to wrong population of the mkey object. Fix this
> by properly referring to the returned value.
>
> The hardware expects each MTT entry to contain the DMA address of a
> contiguous block of memory of size (1 << mr->log_size) bytes.
> dma_map_sg_attrs() can coalesce several sg entries into a single
> scatter/gather entry of contiguous DMA range so we need to scan the list
> and refer to the size of each s/g entry.
>
> In addition, get rid of fill_sg() which effect is overwritten by
> populate_mtts().
>
> Fixes: 94abbccdf291 ("vdpa/mlx5: Add shared memory registration code")
> Signed-off-by: Eli Cohen <elic@nvidia.com>
> ---
> V0->V1:
> 1. Fix typos
> 2. Improve changelog


Acked-by: Jason Wang <jasowang@redhat.com>


>
>   drivers/vdpa/mlx5/core/mlx5_vdpa.h |  1 +
>   drivers/vdpa/mlx5/core/mr.c        | 28 ++++++++++++----------------
>   2 files changed, 13 insertions(+), 16 deletions(-)
>
> diff --git a/drivers/vdpa/mlx5/core/mlx5_vdpa.h b/drivers/vdpa/mlx5/core/mlx5_vdpa.h
> index 5c92a576edae..08f742fd2409 100644
> --- a/drivers/vdpa/mlx5/core/mlx5_vdpa.h
> +++ b/drivers/vdpa/mlx5/core/mlx5_vdpa.h
> @@ -15,6 +15,7 @@ struct mlx5_vdpa_direct_mr {
>   	struct sg_table sg_head;
>   	int log_size;
>   	int nsg;
> +	int nent;
>   	struct list_head list;
>   	u64 offset;
>   };
> diff --git a/drivers/vdpa/mlx5/core/mr.c b/drivers/vdpa/mlx5/core/mr.c
> index 4b6195666c58..d300f799efcd 100644
> --- a/drivers/vdpa/mlx5/core/mr.c
> +++ b/drivers/vdpa/mlx5/core/mr.c
> @@ -25,17 +25,6 @@ static int get_octo_len(u64 len, int page_shift)
>   	return (npages + 1) / 2;
>   }
>   
> -static void fill_sg(struct mlx5_vdpa_direct_mr *mr, void *in)
> -{
> -	struct scatterlist *sg;
> -	__be64 *pas;
> -	int i;
> -
> -	pas = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
> -	for_each_sg(mr->sg_head.sgl, sg, mr->nsg, i)
> -		(*pas) = cpu_to_be64(sg_dma_address(sg));
> -}
> -
>   static void mlx5_set_access_mode(void *mkc, int mode)
>   {
>   	MLX5_SET(mkc, mkc, access_mode_1_0, mode & 0x3);
> @@ -45,10 +34,18 @@ static void mlx5_set_access_mode(void *mkc, int mode)
>   static void populate_mtts(struct mlx5_vdpa_direct_mr *mr, __be64 *mtt)
>   {
>   	struct scatterlist *sg;
> +	int nsg = mr->nsg;
> +	u64 dma_addr;
> +	u64 dma_len;
> +	int j = 0;
>   	int i;
>   
> -	for_each_sg(mr->sg_head.sgl, sg, mr->nsg, i)
> -		mtt[i] = cpu_to_be64(sg_dma_address(sg));
> +	for_each_sg(mr->sg_head.sgl, sg, mr->nent, i) {
> +		for (dma_addr = sg_dma_address(sg), dma_len = sg_dma_len(sg);
> +		     nsg && dma_len;
> +		     nsg--, dma_addr += BIT(mr->log_size), dma_len -= BIT(mr->log_size))
> +			mtt[j++] = cpu_to_be64(dma_addr);
> +	}
>   }
>   
>   static int create_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr *mr)
> @@ -64,7 +61,6 @@ static int create_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct
>   		return -ENOMEM;
>   
>   	MLX5_SET(create_mkey_in, in, uid, mvdev->res.uid);
> -	fill_sg(mr, in);
>   	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
>   	MLX5_SET(mkc, mkc, lw, !!(mr->perm & VHOST_MAP_WO));
>   	MLX5_SET(mkc, mkc, lr, !!(mr->perm & VHOST_MAP_RO));
> @@ -276,8 +272,8 @@ static int map_direct_mr(struct mlx5_vdpa_dev *mvdev, struct mlx5_vdpa_direct_mr
>   done:
>   	mr->log_size = log_entity_size;
>   	mr->nsg = nsg;
> -	err = dma_map_sg_attrs(dma, mr->sg_head.sgl, mr->nsg, DMA_BIDIRECTIONAL, 0);
> -	if (!err)
> +	mr->nent = dma_map_sg_attrs(dma, mr->sg_head.sgl, mr->nsg, DMA_BIDIRECTIONAL, 0);
> +	if (!mr->nent)
>   		goto err_map;
>   
>   	err = create_direct_mr(mvdev, mr);