Received-SPF: pass (google.com: domain of linux-kernel-owner@vger.kernel.org designates 2620:137:e000::1:20 as permitted sender) client-ip=2620:137:e000::1:20;
From:   "Liu, Jing2" <jing2.liu@intel.com>
To:     "Chatre, Reinette" <reinette.chatre@intel.com>,
        "jgg@nvidia.com" <jgg@nvidia.com>,
        "yishaih@nvidia.com" <yishaih@nvidia.com>,
        "shameerali.kolothum.thodi@huawei.com" 
        <shameerali.kolothum.thodi@huawei.com>,
        "Tian, Kevin" <kevin.tian@intel.com>,
        "alex.williamson@redhat.com" <alex.williamson@redhat.com>
CC:     "tglx@linutronix.de" <tglx@linutronix.de>,
        "darwi@linutronix.de" <darwi@linutronix.de>,
        "kvm@vger.kernel.org" <kvm@vger.kernel.org>,
        "Jiang, Dave" <dave.jiang@intel.com>,
        "Raj, Ashok" <ashok.raj@intel.com>,
        "Yu, Fenghua" <fenghua.yu@intel.com>,
        "tom.zanussi@linux.intel.com" <tom.zanussi@linux.intel.com>,
        "linux-kernel@vger.kernel.org" <linux-kernel@vger.kernel.org>
Subject: RE: [PATCH V2 7/8] vfio/pci: Support dynamic MSI-x
Thread-Topic: [PATCH V2 7/8] vfio/pci: Support dynamic MSI-x
Thread-Index: AQHZYb/Rk3to+fQWNUmPjc7xA3v1Mq8UqOiQ
Date:   Fri, 31 Mar 2023 10:02:32 +0000
Message-ID: <MWHPR11MB12456636D269F997D1812FF8A98F9@MWHPR11MB1245.namprd11.prod.outlook.com>
References: <cover.1680038771.git.reinette.chatre@intel.com>
 <419f3ba2f732154d8ae079b3deb02d0fdbe3e258.1680038771.git.reinette.chatre@intel.com>
In-Reply-To: <419f3ba2f732154d8ae079b3deb02d0fdbe3e258.1680038771.git.reinette.chatre@intel.com>
Accept-Language: en-US
Content-Language: en-US
x-ms-exchange-antispam-messagedata-chunkcount: 1
x-ms-exchange-antispam-messagedata-0: =?us-ascii?Q?cWNODCImc4BjME25eZvA0XE396Afw7+Tw14/Zq9BfbJT85znGEGKT0jNcWTy?=
 =?us-ascii?Q?SN5b4jr06QEpywIIuzKoEsJkja6TMLxBtu3AGSJwflb0g555nGsfLKqjshqS?=
 =?us-ascii?Q?TEamCtJhPVLDIOvbbX3jtKHKglNVPGe2HZH+KQyD1HxxZRn2qyPXRaYyBowI?=
 =?us-ascii?Q?IMWmNYHx/q9nMv+le/Kf7lJGeXq4TIpSLOip5mkblYDAivpWNpMQRDJC7ov5?=
 =?us-ascii?Q?bExWV5YjBBJ5uzEupeScgnrj6tSxE1FplwvgxVk1qD9TTeYCPjAw1qsKGj6Z?=
 =?us-ascii?Q?49DcRAW2pJH37PBbxOboYE8vHWErQSr96Gg1Az61b2syBZozVZVl/e9mEWx3?=
 =?us-ascii?Q?eoUs0a9o5eGsS6X6dOU/416D0G/XLQpoyGTj2K3sdbBioRPTIpUmqgAPbcbl?=
 =?us-ascii?Q?YNw/v4dw6ii4r5Y0CEQSESZLuL3HxFdmpgGW+FNGvYn0VgLd0WSUgHqAEt4E?=
 =?us-ascii?Q?0ddATyvyqddlRUM6q5Tg4BfVw9gLjHfYKeN0GPbxmXsL0rKv9BO42qbSQvNz?=
 =?us-ascii?Q?iqWCFPI8EGX91VnjwiDXGS1oKeWOIazlXJq0V2pdjV23TRMMo25TwJ4Iw6Q5?=
 =?us-ascii?Q?7TAAmLrEJpE1BiN/1iDsAqh68DgqaWzwXvwPsz7gEOwBmJ46G7WycJD2H9tF?=
 =?us-ascii?Q?lenYxztmbD7gkSdK5BodVlwmhuyEu8dXCAbp96eh/F5/Th3wchjnURU8ggKy?=
 =?us-ascii?Q?k97HR2JHCHQtpICRGdGVQkYieJGj33Xlqxr1f+jROOzDNKqbpFKRlNdgAuPf?=
 =?us-ascii?Q?yaUGPfudP1zMZ/vk15IsSDyitFRFJuls6I7c4yfZbvuXEU89BMEvJRPs8glp?=
 =?us-ascii?Q?HrJSlfxqj9ok6fHHNQpfDdb3yCkWprszoI+lgASshmZ7h6rbObpPUe3p/Dim?=
 =?us-ascii?Q?qcWSPWxB3ueS9nio2V7q7ksGyO01YJEQ9TU5N4ISUM5qMvXzwzCP/Vbteuzi?=
 =?us-ascii?Q?9JtpGxr/ljfmc27mh3VUUgHR59xBzvzEm+1bypj09+/w5vIif/nX/iVGF+FP?=
 =?us-ascii?Q?/YsrRHvBsYxWnAnsUz2UL4q1/2tZGc7uSTQ1Lf840HyunlzBr2Fq64HJ3qUk?=
 =?us-ascii?Q?AjmBOLDoXEVP5/fYiDHQaJPusNolUxNLSb/QbZ/O6pCu5FT1D9gJikmEmIq8?=
 =?us-ascii?Q?3Uvc5HBE99vcgbzbUCNFm1DQImQ/0bgC9ivSXO21XTeXifkeyjrjumlPQ86l?=
 =?us-ascii?Q?Kh+W6ygJRMaWNPYeism2tFdYI58OBEqEoSd99MqfnJBoRPrcS8RfVRPRCB86?=
 =?us-ascii?Q?DxuxoZJxVF6eMOj0R/TWuqSi1/DnIkWdxTDQetbe3U4pxQJCMNMvvzyfA9+S?=
 =?us-ascii?Q?sYu2eS2bsB8UF7LLIJfHxeFQ21Pgx23uxA+y35cegVIjlvk1tKZBBiG4cJuA?=
 =?us-ascii?Q?l57Elh7Tt4g3Gj0PH+e0p/5vxPU5Gxa2nNLgPxiSTWHYVsx6con1iN9taJJM?=
 =?us-ascii?Q?DBLw31Pr+y1aNLPhCRYhLpzsDph5/++4Q/PPnPV+B41mmqYJZhEGGlKGBxt4?=
 =?us-ascii?Q?tsGF42WIzdOs8oNQHzOujSBUqjDnnFF2/dy+Pg3Y4dEyst02H/rKFndR1VI6?=
 =?us-ascii?Q?ei8wJRTF72nwRv343ZBErjxGriOx0fq2IG+cpozH?=
Content-Type: text/plain; charset="us-ascii"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0
X-MS-Exchange-CrossTenant-AuthAs: Internal
X-MS-Exchange-CrossTenant-AuthSource: MWHPR11MB1245.namprd11.prod.outlook.com
X-MS-Exchange-CrossTenant-Network-Message-Id: d4bfdc6e-b8e4-46de-9787-08db31cf0b9a
X-MS-Exchange-CrossTenant-originalarrivaltime: 31 Mar 2023 10:02:32.5545
 (UTC)
X-MS-Exchange-CrossTenant-fromentityheader: Hosted
X-MS-Exchange-CrossTenant-id: 46c98d88-e344-4ed4-8496-4ed7712e255d
X-MS-Exchange-CrossTenant-mailboxtype: HOSTED
X-MS-Exchange-CrossTenant-userprincipalname: upWJazJjIA66LJ1Lzi5ilXNZud5bSMWWcNyQKMDOxbhIZCn7AeXjNA7agUaSN3uBdzaxcq2qPWpNOG2m5bv9CQ==
X-MS-Exchange-Transport-CrossTenantHeadersStamped: BL3PR11MB6313
X-OriginatorOrg: intel.com
X-Spam-Status: No, score=-2.5 required=5.0 tests=DKIMWL_WL_HIGH,DKIM_SIGNED,
        DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF,RCVD_IN_DNSWL_MED,
        RCVD_IN_MSPIKE_H3,RCVD_IN_MSPIKE_WL,SPF_HELO_NONE,SPF_NONE
        autolearn=unavailable autolearn_force=no version=3.4.6
X-Spam-Checker-Version: SpamAssassin 3.4.6 (2021-04-09) on
        lindbergh.monkeyblade.net
Precedence: bulk
List-ID: <linux-kernel.vger.kernel.org>
X-Mailing-List: linux-kernel@vger.kernel.org

Hi Reinette,

> @@ -409,33 +416,62 @@ static int vfio_msi_set_vector_signal(struct
> vfio_pci_core_device *vdev,  {
>  	struct pci_dev *pdev =3D vdev->pdev;
>  	struct vfio_pci_irq_ctx *ctx;
> +	struct msi_map msix_map =3D {};
> +	bool allow_dyn_alloc =3D false;
>  	struct eventfd_ctx *trigger;
> +	bool new_ctx =3D false;
>  	int irq, ret;
>  	u16 cmd;
>=20
> +	/* Only MSI-X allows dynamic allocation. */
> +	if (msix && pci_msix_can_alloc_dyn(vdev->pdev))
> +		allow_dyn_alloc =3D true;
> +
>  	ctx =3D vfio_irq_ctx_get(vdev, vector);
> -	if (!ctx)
> +	if (!ctx && !allow_dyn_alloc)
>  		return -EINVAL;
> +
>  	irq =3D pci_irq_vector(pdev, vector);
> +	/* Context and interrupt are always allocated together. */
> +	WARN_ON((ctx && irq =3D=3D -EINVAL) || (!ctx && irq !=3D -EINVAL));
>=20
> -	if (ctx->trigger) {
> +	if (ctx && ctx->trigger) {
>  		irq_bypass_unregister_producer(&ctx->producer);
>=20
>  		cmd =3D vfio_pci_memory_lock_and_enable(vdev);
>  		free_irq(irq, ctx->trigger);
> +		if (allow_dyn_alloc) {
> +			msix_map.index =3D vector;
> +			msix_map.virq =3D irq;
> +			pci_msix_free_irq(pdev, msix_map);
> +			irq =3D -EINVAL;
> +		}
>  		vfio_pci_memory_unlock_and_restore(vdev, cmd);
>  		kfree(ctx->name);
>  		eventfd_ctx_put(ctx->trigger);
>  		ctx->trigger =3D NULL;
> +		if (allow_dyn_alloc) {
> +			vfio_irq_ctx_free(vdev, ctx, vector);
> +			ctx =3D NULL;
> +		}
>  	}
>=20
>  	if (fd < 0)
>  		return 0;
>=20

While looking at this piece of code, one thought comes to me:=20
It might be possible that userspace comes twice with the same valid fd for =
a specific
vector, this vfio code will free the resource in if(ctx && ctx->trigger) fo=
r the second
time, and then re-alloc again for the same fd given by userspace.=20

Would that help if we firstly check e.g. ctx->trigger with the given valid =
fd, to see if=20
the trigger is really changed or not?

Thanks,
Jing


> +	if (!ctx) {
> +		ctx =3D vfio_irq_ctx_alloc_single(vdev, vector);
> +		if (!ctx)
> +			return -ENOMEM;
> +		new_ctx =3D true;
> +	}
> +
>  	ctx->name =3D kasprintf(GFP_KERNEL_ACCOUNT, "vfio-msi%s[%d](%s)",
>  			      msix ? "x" : "", vector, pci_name(pdev));
> -	if (!ctx->name)
> -		return -ENOMEM;
> +	if (!ctx->name) {
> +		ret =3D -ENOMEM;
> +		goto out_free_ctx;
> +	}
>=20
>  	trigger =3D eventfd_ctx_fdget(fd);
>  	if (IS_ERR(trigger)) {
> @@ -443,25 +479,38 @@ static int vfio_msi_set_vector_signal(struct
> vfio_pci_core_device *vdev,
>  		goto out_free_name;
>  	}
>=20
> -	/*
> -	 * The MSIx vector table resides in device memory which may be cleared
> -	 * via backdoor resets. We don't allow direct access to the vector
> -	 * table so even if a userspace driver attempts to save/restore around
> -	 * such a reset it would be unsuccessful. To avoid this, restore the
> -	 * cached value of the message prior to enabling.
> -	 */
>  	cmd =3D vfio_pci_memory_lock_and_enable(vdev);
>  	if (msix) {
> -		struct msi_msg msg;
> -
> -		get_cached_msi_msg(irq, &msg);
> -		pci_write_msi_msg(irq, &msg);
> +		if (irq =3D=3D -EINVAL) {
> +			msix_map =3D pci_msix_alloc_irq_at(pdev, vector, NULL);
> +			if (msix_map.index < 0) {
> +				vfio_pci_memory_unlock_and_restore(vdev,
> cmd);
> +				ret =3D msix_map.index;
> +				goto out_put_eventfd_ctx;
> +			}
> +			irq =3D msix_map.virq;
> +		} else {
> +			/*
> +			 * The MSIx vector table resides in device memory
> which
> +			 * may be cleared via backdoor resets. We don't allow
> +			 * direct access to the vector table so even if a
> +			 * userspace driver attempts to save/restore around
> +			 * such a reset it would be unsuccessful. To avoid
> +			 * this, restore the cached value of the message prior
> +			 * to enabling.
> +			 */
> +			struct msi_msg msg;
> +
> +			get_cached_msi_msg(irq, &msg);
> +			pci_write_msi_msg(irq, &msg);
> +		}
>  	}
>=20
>  	ret =3D request_irq(irq, vfio_msihandler, 0, ctx->name, trigger);
> -	vfio_pci_memory_unlock_and_restore(vdev, cmd);
>  	if (ret)
> -		goto out_put_eventfd_ctx;
> +		goto out_free_irq_locked;
> +
> +	vfio_pci_memory_unlock_and_restore(vdev, cmd);
>=20
>  	ctx->producer.token =3D trigger;
>  	ctx->producer.irq =3D irq;
> @@ -477,11 +526,21 @@ static int vfio_msi_set_vector_signal(struct
> vfio_pci_core_device *vdev,
>=20
>  	return 0;
>=20
> +out_free_irq_locked:
> +	if (allow_dyn_alloc && new_ctx) {
> +		msix_map.index =3D vector;
> +		msix_map.virq =3D irq;
> +		pci_msix_free_irq(pdev, msix_map);
> +	}
> +	vfio_pci_memory_unlock_and_restore(vdev, cmd);
>  out_put_eventfd_ctx:
>  	eventfd_ctx_put(trigger);
>  out_free_name:
>  	kfree(ctx->name);
>  	ctx->name =3D NULL;
> +out_free_ctx:
> +	if (allow_dyn_alloc && new_ctx)
> +		vfio_irq_ctx_free(vdev, ctx, vector);
>  	return ret;
>  }
>=20
> --
> 2.34.1