Received-SPF: pass (google.com: domain of linux-kernel-owner@vger.kernel.org designates 2620:137:e000::1:20 as permitted sender) client-ip=2620:137:e000::1:20;
Date:   Wed, 16 Nov 2022 14:10:11 -0600
From:   Bjorn Helgaas <helgaas@kernel.org>
To:     Thomas Gleixner <tglx@linutronix.de>
Cc:     LKML <linux-kernel@vger.kernel.org>, x86@kernel.org,
        Joerg Roedel <joro@8bytes.org>, Will Deacon <will@kernel.org>,
        linux-pci@vger.kernel.org, Bjorn Helgaas <bhelgaas@google.com>,
        Lorenzo Pieralisi <lorenzo.pieralisi@arm.com>,
        Marc Zyngier <maz@kernel.org>,
        Greg Kroah-Hartman <gregkh@linuxfoundation.org>,
        Jason Gunthorpe <jgg@mellanox.com>,
        Dave Jiang <dave.jiang@intel.com>,
        Alex Williamson <alex.williamson@redhat.com>,
        Kevin Tian <kevin.tian@intel.com>,
        Dan Williams <dan.j.williams@intel.com>,
        Logan Gunthorpe <logang@deltatee.com>,
        Ashok Raj <ashok.raj@intel.com>, Jon Mason <jdmason@kudzu.us>,
        Allen Hubbe <allenbh@gmail.com>,
        "Ahmed S. Darwish" <darwi@linutronix.de>,
        Reinette Chatre <reinette.chatre@intel.com>
Subject: Re: [patch 10/33] PCI/MSI: Split __pci_write_msi_msg()
Message-ID: <20221116201011.GA1133404@bhelgaas>
MIME-Version: 1.0
Content-Type: text/plain; charset=us-ascii
Content-Disposition: inline
In-Reply-To: <20221111135205.836259395@linutronix.de>
Precedence: bulk

On Fri, Nov 11, 2022 at 02:58:27PM +0100, Thomas Gleixner wrote:
> The upcoming per device MSI domains will create different domains for MSI
> and MSI-X. Split the write message function into MSI and MSI-X helpers so
> they can be used by those new domain functions seperately.
> 
> Signed-off-by: Ahmed S. Darwish <darwi@linutronix.de>
> Signed-off-by: Thomas Gleixner <tglx@linutronix.de>

Acked-by: Bjorn Helgaas <bhelgaas@google.com>

> ---
>  drivers/pci/msi/msi.c |  104 +++++++++++++++++++++++++-------------------------
>  1 file changed, 54 insertions(+), 50 deletions(-)
> 
> --- a/drivers/pci/msi/msi.c
> +++ b/drivers/pci/msi/msi.c
> @@ -180,6 +180,58 @@ void __pci_read_msi_msg(struct msi_desc
>  	}
>  }
>  
> +static inline void pci_write_msg_msi(struct pci_dev *dev, struct msi_desc *desc,
> +				     struct msi_msg *msg)
> +{
> +	int pos = dev->msi_cap;
> +	u16 msgctl;
> +
> +	pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &msgctl);
> +	msgctl &= ~PCI_MSI_FLAGS_QSIZE;
> +	msgctl |= desc->pci.msi_attrib.multiple << 4;
> +	pci_write_config_word(dev, pos + PCI_MSI_FLAGS, msgctl);
> +
> +	pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_LO, msg->address_lo);
> +	if (desc->pci.msi_attrib.is_64) {
> +		pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_HI,  msg->address_hi);
> +		pci_write_config_word(dev, pos + PCI_MSI_DATA_64, msg->data);
> +	} else {
> +		pci_write_config_word(dev, pos + PCI_MSI_DATA_32, msg->data);
> +	}
> +	/* Ensure that the writes are visible in the device */
> +	pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &msgctl);
> +}
> +
> +static inline void pci_write_msg_msix(struct msi_desc *desc, struct msi_msg *msg)
> +{
> +	void __iomem *base = pci_msix_desc_addr(desc);
> +	u32 ctrl = desc->pci.msix_ctrl;
> +	bool unmasked = !(ctrl & PCI_MSIX_ENTRY_CTRL_MASKBIT);
> +
> +	if (desc->pci.msi_attrib.is_virtual)
> +		return;
> +	/*
> +	 * The specification mandates that the entry is masked
> +	 * when the message is modified:
> +	 *
> +	 * "If software changes the Address or Data value of an
> +	 * entry while the entry is unmasked, the result is
> +	 * undefined."
> +	 */
> +	if (unmasked)
> +		pci_msix_write_vector_ctrl(desc, ctrl | PCI_MSIX_ENTRY_CTRL_MASKBIT);
> +
> +	writel(msg->address_lo, base + PCI_MSIX_ENTRY_LOWER_ADDR);
> +	writel(msg->address_hi, base + PCI_MSIX_ENTRY_UPPER_ADDR);
> +	writel(msg->data, base + PCI_MSIX_ENTRY_DATA);
> +
> +	if (unmasked)
> +		pci_msix_write_vector_ctrl(desc, ctrl);
> +
> +	/* Ensure that the writes are visible in the device */
> +	readl(base + PCI_MSIX_ENTRY_DATA);
> +}
> +
>  void __pci_write_msi_msg(struct msi_desc *entry, struct msi_msg *msg)
>  {
>  	struct pci_dev *dev = msi_desc_to_pci_dev(entry);
> @@ -187,63 +239,15 @@ void __pci_write_msi_msg(struct msi_desc
>  	if (dev->current_state != PCI_D0 || pci_dev_is_disconnected(dev)) {
>  		/* Don't touch the hardware now */
>  	} else if (entry->pci.msi_attrib.is_msix) {
> -		void __iomem *base = pci_msix_desc_addr(entry);
> -		u32 ctrl = entry->pci.msix_ctrl;
> -		bool unmasked = !(ctrl & PCI_MSIX_ENTRY_CTRL_MASKBIT);
> -
> -		if (entry->pci.msi_attrib.is_virtual)
> -			goto skip;
> -
> -		/*
> -		 * The specification mandates that the entry is masked
> -		 * when the message is modified:
> -		 *
> -		 * "If software changes the Address or Data value of an
> -		 * entry while the entry is unmasked, the result is
> -		 * undefined."
> -		 */
> -		if (unmasked)
> -			pci_msix_write_vector_ctrl(entry, ctrl | PCI_MSIX_ENTRY_CTRL_MASKBIT);
> -
> -		writel(msg->address_lo, base + PCI_MSIX_ENTRY_LOWER_ADDR);
> -		writel(msg->address_hi, base + PCI_MSIX_ENTRY_UPPER_ADDR);
> -		writel(msg->data, base + PCI_MSIX_ENTRY_DATA);
> -
> -		if (unmasked)
> -			pci_msix_write_vector_ctrl(entry, ctrl);
> -
> -		/* Ensure that the writes are visible in the device */
> -		readl(base + PCI_MSIX_ENTRY_DATA);
> +		pci_write_msg_msix(entry, msg);
>  	} else {
> -		int pos = dev->msi_cap;
> -		u16 msgctl;
> -
> -		pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &msgctl);
> -		msgctl &= ~PCI_MSI_FLAGS_QSIZE;
> -		msgctl |= entry->pci.msi_attrib.multiple << 4;
> -		pci_write_config_word(dev, pos + PCI_MSI_FLAGS, msgctl);
> -
> -		pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_LO,
> -				       msg->address_lo);
> -		if (entry->pci.msi_attrib.is_64) {
> -			pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_HI,
> -					       msg->address_hi);
> -			pci_write_config_word(dev, pos + PCI_MSI_DATA_64,
> -					      msg->data);
> -		} else {
> -			pci_write_config_word(dev, pos + PCI_MSI_DATA_32,
> -					      msg->data);
> -		}
> -		/* Ensure that the writes are visible in the device */
> -		pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &msgctl);
> +		pci_write_msg_msi(dev, entry, msg);
>  	}
>  
> -skip:
>  	entry->msg = *msg;
>  
>  	if (entry->write_msi_msg)
>  		entry->write_msi_msg(entry, entry->write_msi_msg_data);
> -
>  }
>  
>  void pci_write_msi_msg(unsigned int irq, struct msi_msg *msg)
>