Received-SPF: pass (google.com: domain of linux-kernel-owner@vger.kernel.org designates 2620:137:e000::1:20 as permitted sender) client-ip=2620:137:e000::1:20;
Message-ID: <71b9e85d-960f-7403-0113-135746127f3b@linux.ibm.com>
Date:   Tue, 17 Jan 2023 10:09:13 -0500
User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:102.0) Gecko/20100101
 Thunderbird/102.6.0
Subject: Re: [PATCH v4 4/7] s390/pci: Use dma-iommu layer
Content-Language: en-US
To:     Niklas Schnelle <schnelle@linux.ibm.com>,
        Joerg Roedel <joro@8bytes.org>, Will Deacon <will@kernel.org>,
        Robin Murphy <robin.murphy@arm.com>,
        Jason Gunthorpe <jgg@nvidia.com>,
        Wenjia Zhang <wenjia@linux.ibm.com>
Cc:     Gerd Bayer <gbayer@linux.ibm.com>,
        Pierre Morel <pmorel@linux.ibm.com>, iommu@lists.linux.dev,
        linux-s390@vger.kernel.org, borntraeger@linux.ibm.com,
        hca@linux.ibm.com, gor@linux.ibm.com,
        gerald.schaefer@linux.ibm.com, agordeev@linux.ibm.com,
        svens@linux.ibm.com, linux-kernel@vger.kernel.org,
        Julian Ruess <julianr@linux.ibm.com>
References: <20230104120543.308933-1-schnelle@linux.ibm.com>
 <20230104120543.308933-5-schnelle@linux.ibm.com>
From:   Matthew Rosato <mjrosato@linux.ibm.com>
In-Reply-To: <20230104120543.308933-5-schnelle@linux.ibm.com>
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 7bit
MIME-Version: 1.0
Precedence: bulk

On 1/4/23 7:05 AM, Niklas Schnelle wrote:
> While s390 already has a standard IOMMU driver and previous changes have
> added I/O TLB flushing operations this driver is currently only used for
> user-space PCI access such as vfio-pci. For the DMA API s390 instead
> utilizes its own implementation in arch/s390/pci/pci_dma.c which drives
> the same hardware and shares some code but requires a complex and
> fragile hand over between DMA API and IOMMU API use of a device and
> despite code sharing still leads to significant duplication and
> maintenance effort. Let's utilize the common code DMAP API
> implementation from drivers/iommu/dma-iommu.c instead allowing us to
> get rid of arch/s390/pci/pci_dma.c.
> 
> Signed-off-by: Niklas Schnelle <schnelle@linux.ibm.com>
> diff --git a/arch/s390/pci/pci.c b/arch/s390/pci/pci.c
> index ef38b1514c77..6b0fe8761509 100644
> --- a/arch/s390/pci/pci.c
> +++ b/arch/s390/pci/pci.c
> @@ -124,7 +124,11 @@ int zpci_register_ioat(struct zpci_dev *zdev, u8 dmaas,
>  
>  	WARN_ON_ONCE(iota & 0x3fff);
>  	fib.pba = base;
> -	fib.pal = limit;
> +	/* Work around off by one in ISM virt device */
> +	if (zdev->pft == 0x5 && limit > base)

Nit: maybe a named #define for the ISM pft rather than hard-coding 0x5 here

[...]

> diff --git a/arch/s390/pci/pci_bus.c b/arch/s390/pci/pci_bus.c
> index 6a8da1b742ae..b15ad15999f8 100644
> --- a/arch/s390/pci/pci_bus.c
> +++ b/arch/s390/pci/pci_bus.c
> @@ -49,11 +49,6 @@ static int zpci_bus_prepare_device(struct zpci_dev *zdev)
>  		rc = zpci_enable_device(zdev);
>  		if (rc)
>  			return rc;
> -		rc = zpci_dma_init_device(zdev);
> -		if (rc) {
> -			zpci_disable_device(zdev);
> -			return rc;
> -		}
>  	}
>  
>  	if (!zdev->has_resources) {
> diff --git a/arch/s390/pci/pci_debug.c b/arch/s390/pci/pci_debug.c
> index ca6bd98eec13..60cec57a3907 100644
> --- a/arch/s390/pci/pci_debug.c
> +++ b/arch/s390/pci/pci_debug.c
> @@ -53,9 +53,12 @@ static char *pci_fmt3_names[] = {
>  };
>  
>  static char *pci_sw_names[] = {
> -	"Allocated pages",
> +/* TODO "Allocated pages", */

? Forgot to finish this?

>  	"Mapped pages",
>  	"Unmapped pages",
> +	"Global RPCITs",
> +	"Sync Map RPCITs",
> +	"Sync RPCITs",
>  };

[...]

> diff --git a/drivers/iommu/s390-iommu.c b/drivers/iommu/s390-iommu.c
> index 4dfa557270f4..73144ea0adfc 100644
> --- a/drivers/iommu/s390-iommu.c
> +++ b/drivers/iommu/s390-iommu.c
> @@ -14,16 +14,300 @@

> -static void __s390_iommu_detach_device(struct zpci_dev *zdev)
> +static void s390_iommu_detach_device(struct iommu_domain *domain,
> +				     struct device *dev)
>  {
> -	struct s390_domain *s390_domain = zdev->s390_domain;
> +	struct s390_domain *s390_domain = to_s390_domain(domain);
> +	struct zpci_dev *zdev = to_zpci_dev(dev);
>  	unsigned long flags;
>  
> +	WARN_ON(zdev->s390_domain != to_s390_domain(domain));
>  	if (!s390_domain)
>  		return;
>  
> @@ -120,9 +412,7 @@ static int s390_iommu_attach_device(struct iommu_domain *domain,
>  		return -EINVAL;
>  
>  	if (zdev->s390_domain)
> -		__s390_iommu_detach_device(zdev);
> -	else if (zdev->dma_table)
> -		zpci_dma_exit_device(zdev);
> +		s390_iommu_detach_device(&zdev->s390_domain->domain, dev);
>  
>  	cc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
>  				virt_to_phys(s390_domain->dma_table), &status);
> @@ -144,17 +434,6 @@ static int s390_iommu_attach_device(struct iommu_domain *domain,
>  	return 0;
>  }
>  
> -static void s390_iommu_detach_device(struct iommu_domain *domain,
> -				     struct device *dev)
> -{
> -	struct zpci_dev *zdev = to_zpci_dev(dev);
> -
> -	WARN_ON(zdev->s390_domain != to_s390_domain(domain));
> -
> -	__s390_iommu_detach_device(zdev);
> -	zpci_dma_init_device(zdev);
> -}
> ->  static void s390_iommu_get_resv_regions(struct device *dev,
>  					struct list_head *list)
>  {
> @@ -207,7 +486,7 @@ static void s390_iommu_release_device(struct device *dev)
>  	 * to the device, but keep it attached to other devices in the group.
>  	 */
>  	if (zdev)
> -		__s390_iommu_detach_device(zdev);
> +		s390_iommu_detach_device(&zdev->s390_domain->domain, dev);
>  }
>  
Looks good overall, but I think these hits above collide with Baolu's series that recently went into -next:

https://lore.kernel.org/linux-iommu/20230110025408.667767-1-baolu.lu@linux.intel.com/