Received-SPF: pass (google.com: domain of linux-kernel+bounces-116682-linux.lists.archive=gmail.com@vger.kernel.org designates 147.75.48.161 as permitted sender) client-ip=147.75.48.161;
From: Jens Wiklander <jens.wiklander@linaro.org>
To: linux-kernel@vger.kernel.org,
	linux-arm-kernel@lists.infradead.org
Cc: Sudeep Holla <sudeep.holla@arm.com>,
	Marc Bonnici <marc.bonnici@arm.com>,
	Olivier Deprez <Olivier.Deprez@arm.com>,
	Lorenzo Pieralisi <lpieralisi@kernel.org>,
	Bertrand Marquis <Bertrand.Marquis@arm.com>,
	Jens Wiklander <jens.wiklander@linaro.org>
Subject: [PATCH v2] firmware: arm_ffa: support running as a guest in a vm
Date: Mon, 25 Mar 2024 09:13:35 +0100
Message-Id: <20240325081335.2326979-1-jens.wiklander@linaro.org>
Precedence: bulk
MIME-Version: 1.0
Content-Transfer-Encoding: 8bit

Add support for running the driver in a guest to a hypervisor. The main
difference is introducing notification pending interrupt and that
FFA_NOTIFICATION_BITMAP_CREATE doesn't need to be called.

The guest may need to use a notification pending interrupt instead of or
in addition to the schedule receiver interrupt.
FFA_FEAT_NOTIFICATION_PENDING_INT gives the interrupt the hypervisor has
chosen to notify its guest of pending notifications.

Signed-off-by: Jens Wiklander <jens.wiklander@linaro.org>
---
Hi,

I've tested this with Xen with as hypervisor (Xen patches to be posted) and
the NPI handler delivers notifications as expected using
FFA_NOTIFICATION_GET. I've also tested without a hypervisor to see that the
SRI handler still works as expected.

Thanks,
Jens

v1->v2:
* Adds a Notification Pending Interrupt handler to retrieve notifications
  using for the interrupted CPU.
* The original Schedule Receiver Interrupt handler is initialized in parallel
  depending on whether FFA_FEAT_SCHEDULE_RECEIVER_INT is available.

 drivers/firmware/arm_ffa/driver.c | 117 +++++++++++++++++++++---------
 1 file changed, 82 insertions(+), 35 deletions(-)

diff --git a/drivers/firmware/arm_ffa/driver.c b/drivers/firmware/arm_ffa/driver.c
index f2556a8e9401..9732f0066c70 100644
--- a/drivers/firmware/arm_ffa/driver.c
+++ b/drivers/firmware/arm_ffa/driver.c
@@ -101,11 +101,12 @@ struct ffa_drv_info {
 	bool bitmap_created;
 	bool notif_enabled;
 	unsigned int sched_recv_irq;
+	unsigned int notif_pend_irq;
 	unsigned int cpuhp_state;
 	struct ffa_pcpu_irq __percpu *irq_pcpu;
 	struct workqueue_struct *notif_pcpu_wq;
 	struct work_struct notif_pcpu_work;
-	struct work_struct irq_work;
+	struct work_struct sched_recv_irq_work;
 	struct xarray partition_info;
 	DECLARE_HASHTABLE(notifier_hash, ilog2(FFA_MAX_NOTIFICATIONS));
 	struct mutex notify_lock; /* lock to protect notifier hashtable  */
@@ -1291,12 +1292,23 @@ static void ffa_partitions_cleanup(void)
 #define FFA_FEAT_SCHEDULE_RECEIVER_INT		(2)
 #define FFA_FEAT_MANAGED_EXIT_INT		(3)
 
-static irqreturn_t irq_handler(int irq, void *irq_data)
+static irqreturn_t sched_recv_irq_handler(int irq, void *irq_data)
 {
 	struct ffa_pcpu_irq *pcpu = irq_data;
 	struct ffa_drv_info *info = pcpu->info;
 
-	queue_work(info->notif_pcpu_wq, &info->irq_work);
+	queue_work(info->notif_pcpu_wq, &info->sched_recv_irq_work);
+
+	return IRQ_HANDLED;
+}
+
+static irqreturn_t notif_pend_irq_handler(int irq, void *irq_data)
+{
+	struct ffa_pcpu_irq *pcpu = irq_data;
+	struct ffa_drv_info *info = pcpu->info;
+
+	queue_work_on(smp_processor_id(), info->notif_pcpu_wq,
+		      &info->notif_pcpu_work);
 
 	return IRQ_HANDLED;
 }
@@ -1306,15 +1318,16 @@ static void ffa_sched_recv_irq_work_fn(struct work_struct *work)
 	ffa_notification_info_get();
 }
 
-static int ffa_sched_recv_irq_map(void)
+static int ffa_irq_map(u32 id)
 {
-	int ret, irq, sr_intid;
+	int ret, irq, intid;
 
-	/* The returned sr_intid is assumed to be SGI donated to NS world */
-	ret = ffa_features(FFA_FEAT_SCHEDULE_RECEIVER_INT, 0, &sr_intid, NULL);
+	/* The returned intid is assumed to be SGI donated to NS world */
+	ret = ffa_features(id, 0, &intid, NULL);
 	if (ret < 0) {
+		pr_err("Failed to retrieve FF-A id %u interrupt\n", id);
 		if (ret != -EOPNOTSUPP)
-			pr_err("Failed to retrieve scheduler Rx interrupt\n");
+			pr_err("Failed to retrieve FF-A id %u interrupt\n", id);
 		return ret;
 	}
 
@@ -1329,12 +1342,12 @@ static int ffa_sched_recv_irq_map(void)
 
 		oirq.np = gic;
 		oirq.args_count = 1;
-		oirq.args[0] = sr_intid;
+		oirq.args[0] = intid;
 		irq = irq_create_of_mapping(&oirq);
 		of_node_put(gic);
 #ifdef CONFIG_ACPI
 	} else {
-		irq = acpi_register_gsi(NULL, sr_intid, ACPI_EDGE_SENSITIVE,
+		irq = acpi_register_gsi(NULL, intid, ACPI_EDGE_SENSITIVE,
 					ACPI_ACTIVE_HIGH);
 #endif
 	}
@@ -1355,15 +1368,29 @@ static void ffa_sched_recv_irq_unmap(void)
 	}
 }
 
+static void ffa_notif_pend_irq_unmap(void)
+{
+	if (drv_info->notif_pend_irq) {
+		irq_dispose_mapping(drv_info->notif_pend_irq);
+		drv_info->notif_pend_irq = 0;
+	}
+}
+
 static int ffa_cpuhp_pcpu_irq_enable(unsigned int cpu)
 {
-	enable_percpu_irq(drv_info->sched_recv_irq, IRQ_TYPE_NONE);
+	if (drv_info->sched_recv_irq)
+		enable_percpu_irq(drv_info->sched_recv_irq, IRQ_TYPE_NONE);
+	if (drv_info->notif_pend_irq)
+		enable_percpu_irq(drv_info->notif_pend_irq, IRQ_TYPE_NONE);
 	return 0;
 }
 
 static int ffa_cpuhp_pcpu_irq_disable(unsigned int cpu)
 {
-	disable_percpu_irq(drv_info->sched_recv_irq);
+	if (drv_info->sched_recv_irq)
+		disable_percpu_irq(drv_info->sched_recv_irq);
+	if (drv_info->notif_pend_irq)
+		disable_percpu_irq(drv_info->notif_pend_irq);
 	return 0;
 }
 
@@ -1382,13 +1409,16 @@ static void ffa_uninit_pcpu_irq(void)
 	if (drv_info->sched_recv_irq)
 		free_percpu_irq(drv_info->sched_recv_irq, drv_info->irq_pcpu);
 
+	if (drv_info->notif_pend_irq)
+		free_percpu_irq(drv_info->notif_pend_irq, drv_info->irq_pcpu);
+
 	if (drv_info->irq_pcpu) {
 		free_percpu(drv_info->irq_pcpu);
 		drv_info->irq_pcpu = NULL;
 	}
 }
 
-static int ffa_init_pcpu_irq(unsigned int irq)
+static int ffa_init_pcpu_irq(void)
 {
 	struct ffa_pcpu_irq __percpu *irq_pcpu;
 	int ret, cpu;
@@ -1402,13 +1432,29 @@ static int ffa_init_pcpu_irq(unsigned int irq)
 
 	drv_info->irq_pcpu = irq_pcpu;
 
-	ret = request_percpu_irq(irq, irq_handler, "ARM-FFA", irq_pcpu);
-	if (ret) {
-		pr_err("Error registering notification IRQ %d: %d\n", irq, ret);
-		return ret;
+	if (drv_info->sched_recv_irq) {
+		ret = request_percpu_irq(drv_info->sched_recv_irq,
+					 sched_recv_irq_handler,
+					 "ARM-FFA-SR", irq_pcpu);
+		if (ret) {
+			pr_err("Error registering schedule receiver nIRQ %d: %d\n",
+			       drv_info->sched_recv_irq, ret);
+			return ret;
+		}
+	}
+
+	if (drv_info->notif_pend_irq) {
+		ret = request_percpu_irq(drv_info->notif_pend_irq,
+					 notif_pend_irq_handler,
+					 "ARM-FFA-NP", irq_pcpu);
+		if (ret) {
+			pr_err("Error registering notification pendig nIRQ %d: %d\n",
+			       drv_info->notif_pend_irq, ret);
+			return ret;
+		}
 	}
 
-	INIT_WORK(&drv_info->irq_work, ffa_sched_recv_irq_work_fn);
+	INIT_WORK(&drv_info->sched_recv_irq_work, ffa_sched_recv_irq_work_fn);
 	INIT_WORK(&drv_info->notif_pcpu_work, notif_pcpu_irq_work_fn);
 	drv_info->notif_pcpu_wq = create_workqueue("ffa_pcpu_irq_notification");
 	if (!drv_info->notif_pcpu_wq)
@@ -1429,6 +1475,7 @@ static void ffa_notifications_cleanup(void)
 {
 	ffa_uninit_pcpu_irq();
 	ffa_sched_recv_irq_unmap();
+	ffa_notif_pend_irq_unmap();
 
 	if (drv_info->bitmap_created) {
 		ffa_notification_bitmap_destroy();
@@ -1439,30 +1486,30 @@ static void ffa_notifications_cleanup(void)
 
 static void ffa_notifications_setup(void)
 {
-	int ret, irq;
+	int ret;
 
 	ret = ffa_features(FFA_NOTIFICATION_BITMAP_CREATE, 0, NULL, NULL);
-	if (ret) {
-		pr_info("Notifications not supported, continuing with it ..\n");
-		return;
+	if (!ret) {
+		ret = ffa_notification_bitmap_create();
+		if (ret) {
+			pr_info("Notification bitmap create error %d\n", ret);
+			return;
+		}
+		drv_info->bitmap_created = true;
 	}
 
-	ret = ffa_notification_bitmap_create();
-	if (ret) {
-		pr_info("Notification bitmap create error %d\n", ret);
-		return;
-	}
-	drv_info->bitmap_created = true;
+	ret = ffa_irq_map(FFA_FEAT_SCHEDULE_RECEIVER_INT);
+	if (ret > 0)
+		drv_info->sched_recv_irq = ret;
 
-	irq = ffa_sched_recv_irq_map();
-	if (irq <= 0) {
-		ret = irq;
-		goto cleanup;
-	}
+	ret = ffa_irq_map(FFA_FEAT_NOTIFICATION_PENDING_INT);
+	if (ret > 0)
+		drv_info->notif_pend_irq = ret;
 
-	drv_info->sched_recv_irq = irq;
+	if (!drv_info->sched_recv_irq && !drv_info->notif_pend_irq)
+		goto cleanup;
 
-	ret = ffa_init_pcpu_irq(irq);
+	ret = ffa_init_pcpu_irq();
 	if (ret)
 		goto cleanup;
 
-- 
2.34.1