Received-SPF: pass (google.com: domain of linux-kernel-owner@vger.kernel.org designates 2620:137:e000::1:20 as permitted sender) client-ip=2620:137:e000::1:20;
Message-ID: <90a2c5891b435c7d3734e46a70cadb56e84865b6.camel@redhat.com>
Subject: Re: Re: [PATCH net-next] octeontx2-pf: Add support for page pool
From:   Paolo Abeni <pabeni@redhat.com>
To:     Ratheesh Kannoth <rkannoth@marvell.com>,
        Yunsheng Lin <linyunsheng@huawei.com>,
        "netdev@vger.kernel.org" <netdev@vger.kernel.org>,
        "linux-kernel@vger.kernel.org" <linux-kernel@vger.kernel.org>
Cc:     Sunil Kovvuri Goutham <sgoutham@marvell.com>,
        "davem@davemloft.net" <davem@davemloft.net>,
        "edumazet@google.com" <edumazet@google.com>,
        "kuba@kernel.org" <kuba@kernel.org>
Date:   Tue, 16 May 2023 10:51:01 +0200
In-Reply-To: <MWHPR1801MB191862CDE18A98A89E57E3D9D3799@MWHPR1801MB1918.namprd18.prod.outlook.com>
References: <20230515055607.651799-1-rkannoth@marvell.com>
         <c50a0969-4b17-f2c2-6ad6-b085b8ac4043@huawei.com>
         <MWHPR1801MB191862CDE18A98A89E57E3D9D3799@MWHPR1801MB1918.namprd18.prod.outlook.com>
Content-Type: text/plain; charset="UTF-8"
Content-Transfer-Encoding: quoted-printable
User-Agent: Evolution 3.46.4 (3.46.4-1.fc37) 
MIME-Version: 1.0
Precedence: bulk

On Tue, 2023-05-16 at 03:36 +0000, Ratheesh Kannoth wrote:
>=20
> > ....
> > > @@ -1170,15 +1199,24 @@ void otx2_free_aura_ptr(struct otx2_nic
> > > *pfvf,
> > int type)
> > > =C2=A0	/* Free SQB and RQB pointers from the aura pool */
> > > =C2=A0	for (pool_id =3D pool_start; pool_id < pool_end;
> > > pool_id++) {
> > > =C2=A0		iova =3D otx2_aura_allocptr(pfvf, pool_id);
> > > +		pool =3D &pfvf->qset.pool[pool_id];
> > > =C2=A0		while (iova) {
> > > =C2=A0			if (type =3D=3D AURA_NIX_RQ)
> > > =C2=A0				iova -=3D OTX2_HEAD_ROOM;
> > >=20
> > > =C2=A0			pa =3D otx2_iova_to_phys(pfvf-
> > > >iommu_domain,
> > iova);
> > > -			dma_unmap_page_attrs(pfvf->dev, iova,
> > > size,
> > > -					     DMA_FROM_DEVICE,
> > > -					   =20
> > > DMA_ATTR_SKIP_CPU_SYNC);
> > > -
> > > 			put_page(virt_to_page(phys_to_virt(pa)));
> > > +			page =3D virt_to_page(phys_to_virt(pa));
> >=20
> > virt_to_page() seems ok for order-0 page allocated from page pool
> > as it does
> > now, but it may break for order-1+ page as
> > page_pool_put_page() expects head page of compound page or base
> > page.
> > Maybe add a comment for that or use virt_to_head_page() explicitly.
> Thanks !!.=20
> >=20
> > > +
> > > +			if (pool->page_pool) {
> > > +				page_pool_put_page(pool-
> > > >page_pool,
> > page, size, true);
> >=20
> > page_pool_put_full_page() seems more appropriate here, as the
> > PP_FLAG_DMA_SYNC_DEV flag is not set, even if it is set, it seems
> > the whole
> > page need to be synced instead of a frag.
> Agree.=20
> >=20
> >=20
> > > +			} else {
> > > +				dma_unmap_page_attrs(pfvf->dev,
> > > iova,
> > size,
> > > +						   =20
> > > DMA_FROM_DEVICE,
> > > +
> > DMA_ATTR_SKIP_CPU_SYNC);
> > > +
> > > +				put_page(page);
> > > +			}
> > > +
> > > =C2=A0			iova =3D otx2_aura_allocptr(pfvf,
> > > pool_id);
> > > =C2=A0		}
> > > =C2=A0	}
> > > @@ -1196,6 +1234,8 @@ void otx2_aura_pool_free(struct otx2_nic
> > > *pfvf)
> > > =C2=A0		pool =3D &pfvf->qset.pool[pool_id];
> > > =C2=A0		qmem_free(pfvf->dev, pool->stack);
> > > =C2=A0		qmem_free(pfvf->dev, pool->fc_addr);
> > > +		page_pool_destroy(pool->page_pool);
> > > +		pool->page_pool =3D NULL;
> > > =C2=A0	}
> > > =C2=A0	devm_kfree(pfvf->dev, pfvf->qset.pool);
> > > =C2=A0	pfvf->qset.pool =3D NULL;
> > > @@ -1279,8 +1319,10 @@ static int otx2_aura_init(struct otx2_nic
> > > *pfvf, int aura_id,  }
> > >=20
> > > =C2=A0static int otx2_pool_init(struct otx2_nic *pfvf, u16 pool_id,
> > > -			  int stack_pages, int numptrs, int
> > > buf_size)
> > > +			  int stack_pages, int numptrs, int
> > > buf_size,
> > > +			  int type)
> > > =C2=A0{
> > > +	struct page_pool_params pp_params =3D { 0 };
> > > =C2=A0	struct npa_aq_enq_req *aq;
> > > =C2=A0	struct otx2_pool *pool;
> > > =C2=A0	int err;
> > > @@ -1324,6 +1366,22 @@ static int otx2_pool_init(struct otx2_nic
> > > *pfvf,
> > u16 pool_id,
> > > =C2=A0	aq->ctype =3D NPA_AQ_CTYPE_POOL;
> > > =C2=A0	aq->op =3D NPA_AQ_INSTOP_INIT;
> > >=20
> > > +	if (type !=3D AURA_NIX_RQ) {
> > > +		pool->page_pool =3D NULL;
> > > +		return 0;
> > > +	}
> > > +
> > > +	pp_params.flags =3D PP_FLAG_PAGE_FRAG | PP_FLAG_DMA_MAP;
> > > +	pp_params.pool_size =3D numptrs;
> > > +	pp_params.nid =3D NUMA_NO_NODE;
> > > +	pp_params.dev =3D pfvf->dev;
> > > +	pp_params.dma_dir =3D DMA_FROM_DEVICE;
> > > +	pool->page_pool =3D page_pool_create(&pp_params);
> > > +	if (!pool->page_pool) {
> > > +		netdev_err(pfvf->netdev, "Creation of page pool
> > > failed\n");
> > > +		return -EFAULT;
> > > +	}
> > > +
> > > =C2=A0	return 0;
> > > =C2=A0}
> > >=20
> > > @@ -1358,7 +1416,7 @@ int otx2_sq_aura_pool_init(struct otx2_nic
> > > *pfvf)
> > >=20
> > > =C2=A0		/* Initialize pool context */
> > > =C2=A0		err =3D otx2_pool_init(pfvf, pool_id, stack_pages,
> > > -				     num_sqbs, hw->sqb_size);
> > > +				     num_sqbs, hw->sqb_size,
> > > AURA_NIX_SQ);
> > > =C2=A0		if (err)
> > > =C2=A0			goto fail;
> > > =C2=A0	}
> > > @@ -1421,7 +1479,7 @@ int otx2_rq_aura_pool_init(struct otx2_nic
> > > *pfvf)
> > > =C2=A0	}
> > > =C2=A0	for (pool_id =3D 0; pool_id < hw->rqpool_cnt; pool_id++) {
> > > =C2=A0		err =3D otx2_pool_init(pfvf, pool_id, stack_pages,
> > > -				     num_ptrs, pfvf->rbsize);
> > > +				     num_ptrs, pfvf->rbsize,
> > > AURA_NIX_RQ);
> > > =C2=A0		if (err)
> > > =C2=A0			goto fail;
> > > =C2=A0	}
> >=20
> > ...
> >=20
> > > diff --git
> > > a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
> > > b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
> > > index 7045fedfd73a..df5f45aa6980 100644
> > > --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
> > > +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_txrx.c
> > > @@ -217,9 +217,10 @@ static bool otx2_skb_add_frag(struct
> > > otx2_nic
> > *pfvf, struct sk_buff *skb,
> > > =C2=A0		skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
> > > page,
> > > =C2=A0				va - page_address(page) + off,
> > > =C2=A0				len - off, pfvf->rbsize);
> > > -
> > > +#ifndef CONFIG_PAGE_POOL
> >=20
> > Most driver does 'select PAGE_POOL' in config when adding page pool
> > support, is there any reason it does not do it here?
> We thought about it. User should be able to use the driver without
> PAGE_POOL support.=20

Uhm... the above looks like a questionable choice, as page pull is a
small infra, and the performance delta should be relevant.

Anyway if you really want to use such strategy, please be consistent
and guard any relevant chunck of code with compiler guards. Likely it
would be better providing dummy helpers for the few page_pool functions
still missing them when !CONFIG_PAGE_POOL

>=20
Cheers,

Paolo