Most of the existing RISC-V platforms use SiFive CLINT to provide M-level
timer and IPI support whereas S-level uses SBI calls for timer and IPI
support. Also, the SiFive CLINT device is a single device providing both
timer and IPI functionality so RISC-V platforms can't partially implement
SiFive CLINT device and provide alternate mechanism for timer and IPI.
The RISC-V Advacned Core Local Interruptor (ACLINT) tries to address the
limitations of the SiFive CLINT by:
1) Taking modular approach and defining timer and IPI functionality as
separate devices so that RISC-V platforms can include only required
devices
2) Providing dedicated MMIO device for S-level IPIs so that SBI calls
can be avoided for IPIs in Linux RISC-V
3) Allowing multiple instances of timer and IPI devices for a
multi-socket (or multi-die) NUMA systems
The RISC-V ACLINT specification is backward compatible to the SiFive CLINT
so existing RISC-V platforms with SiFive CLINT are already compliant.
Latest RISC-V ACLINT specification (is stable and will be frozen soon)
can be found at:
https://github.com/riscv/riscv-aclint/releases/download/v1.0-rc2/riscv-aclint-1.0-rc2.pdf
This series adds RISC-V ACLINT support and can be found in the
riscv_aclint_v4 branch at: https://github.com/avpatel/linux
This series is tested on QEMU virt machine with both MMU and NoMMU
Linux RISC-V kernel.
Changes since v3:
- Dropped PATCH which was updating SiFive CLINT DT bindings
- Updated MTIMER, MSWI, and SSWI DT bindings to mandate a implmentation
specific compatible strings
- Added MTIMER and MSWI implementation specific compatible strings for
existing platforms with SiFive CLINT
- Call aclint_swi_init() from CLINT timer driver in PATCH9 when we are
dealing with SiFive CLINT device
Changes since v2:
- Addresed Rob's comments on [M|S]SWI DT bindings
- Dropped PATCH2 because it was not a required change
- Addressed Marc's comments on ACLINT SWI driver added by PATCH7
- Added a separate PATCH6 to update SiFive CLINT DT bindings
Changes since v1:
- Added a new PATCH3 to treat IPIs as normal Linux IRQs for RISC-V kernel
- New SBI IPI call based irqchip driver in PATCH3 which is only initialized
by riscv_ipi_setup() when no Linux IRQ numbers are available for IPIs
- Moved DT bindings patches before corresponding driver patches
- Implemented ACLINT SWI driver as a irqchip driver in PATCH7
- Minor nit fixes pointed by Bin Meng
Anup Patel (10):
RISC-V: Clear SIP bit only when using SBI IPI operations
RISC-V: Treat IPIs as normal Linux IRQs
RISC-V: Allow marking IPIs as suitable for remote FENCEs
RISC-V: Use IPIs for remote TLB flush when possible
dt-bindings: interrupt-controller: Add ACLINT MSWI and SSWI bindings
irqchip: Add ACLINT software interrupt driver
RISC-V: Select ACLINT SWI driver for virt machine
dt-bindings: timer: Add ACLINT MTIMER bindings
clocksource: clint: Add support for ACLINT MTIMER device
MAINTAINERS: Add entry for RISC-V ACLINT drivers
.../riscv,aclint-swi.yaml | 97 ++++++
.../bindings/timer/riscv,aclint-mtimer.yaml | 67 +++++
MAINTAINERS | 10 +
arch/riscv/Kconfig | 1 +
arch/riscv/Kconfig.socs | 1 +
arch/riscv/include/asm/sbi.h | 2 +
arch/riscv/include/asm/smp.h | 49 ++-
arch/riscv/kernel/Makefile | 1 +
arch/riscv/kernel/cpu-hotplug.c | 3 +-
arch/riscv/kernel/irq.c | 3 +-
arch/riscv/kernel/sbi-ipi.c | 218 ++++++++++++++
arch/riscv/kernel/sbi.c | 15 -
arch/riscv/kernel/smp.c | 167 ++++++-----
arch/riscv/kernel/smpboot.c | 5 +-
arch/riscv/mm/tlbflush.c | 91 ++++--
drivers/clocksource/timer-clint.c | 80 ++---
drivers/irqchip/Kconfig | 9 +
drivers/irqchip/Makefile | 1 +
drivers/irqchip/irq-riscv-aclint-swi.c | 282 ++++++++++++++++++
drivers/irqchip/irq-riscv-intc.c | 55 ++--
include/linux/irqchip/irq-riscv-aclint-swi.h | 19 ++
21 files changed, 974 insertions(+), 202 deletions(-)
create mode 100644 Documentation/devicetree/bindings/interrupt-controller/riscv,aclint-swi.yaml
create mode 100644 Documentation/devicetree/bindings/timer/riscv,aclint-mtimer.yaml
create mode 100644 arch/riscv/kernel/sbi-ipi.c
create mode 100644 drivers/irqchip/irq-riscv-aclint-swi.c
create mode 100644 include/linux/irqchip/irq-riscv-aclint-swi.h
--
2.25.1
The software interrupt pending (i.e. [M|S]SIP) bit is writeable for
S-mode but read-only for M-mode so we clear this bit only when using
SBI IPI operations.
Signed-off-by: Anup Patel <[email protected]>
Reviewed-by: Bin Meng <[email protected]>
---
arch/riscv/kernel/sbi.c | 8 +++++++-
arch/riscv/kernel/smp.c | 2 --
2 files changed, 7 insertions(+), 3 deletions(-)
diff --git a/arch/riscv/kernel/sbi.c b/arch/riscv/kernel/sbi.c
index 9a84f0cb5175..8aeca26198f2 100644
--- a/arch/riscv/kernel/sbi.c
+++ b/arch/riscv/kernel/sbi.c
@@ -598,8 +598,14 @@ static void sbi_send_cpumask_ipi(const struct cpumask *target)
sbi_send_ipi(cpumask_bits(&hartid_mask));
}
+static void sbi_ipi_clear(void)
+{
+ csr_clear(CSR_IP, IE_SIE);
+}
+
static const struct riscv_ipi_ops sbi_ipi_ops = {
- .ipi_inject = sbi_send_cpumask_ipi
+ .ipi_inject = sbi_send_cpumask_ipi,
+ .ipi_clear = sbi_ipi_clear
};
void __init sbi_init(void)
diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c
index 921d9d7df400..547dc508f7d1 100644
--- a/arch/riscv/kernel/smp.c
+++ b/arch/riscv/kernel/smp.c
@@ -99,8 +99,6 @@ void riscv_clear_ipi(void)
{
if (ipi_ops && ipi_ops->ipi_clear)
ipi_ops->ipi_clear();
-
- csr_clear(CSR_IP, IE_SIE);
}
EXPORT_SYMBOL_GPL(riscv_clear_ipi);
--
2.25.1
The RISC-V ACLINT specification is a modular specification and the
ACLINT MTIMER device is backward compatible with the M-mode timer
functionality of the CLINT device. This patch extends the CLINT
timer driver to support both CLINT device and ACLINT MTIMER device.
Signed-off-by: Anup Patel <[email protected]>
Reviewed-by: Bin Meng <[email protected]>
---
drivers/clocksource/timer-clint.c | 59 ++++++++++++++++++++++++-------
1 file changed, 47 insertions(+), 12 deletions(-)
diff --git a/drivers/clocksource/timer-clint.c b/drivers/clocksource/timer-clint.c
index 3b68ed53fe4a..9d38dfce374e 100644
--- a/drivers/clocksource/timer-clint.c
+++ b/drivers/clocksource/timer-clint.c
@@ -2,8 +2,16 @@
/*
* Copyright (C) 2020 Western Digital Corporation or its affiliates.
*
- * Most of the M-mode (i.e. NoMMU) RISC-V systems usually have a
- * CLINT MMIO timer device.
+ * Most of the M-mode (i.e. NoMMU) RISC-V systems usually have a CLINT
+ * MMIO device which is a composite device capable of injecting M-mode
+ * software interrupts and M-mode timer interrupts.
+ *
+ * The RISC-V ACLINT specification is modular in nature and defines
+ * separate devices for M-mode software interrupt (MSWI), M-mode timer
+ * (MTIMER) and S-mode software interrupt (SSWI).
+ *
+ * This is a common timer driver for the CLINT device and the ACLINT
+ * MTIMER device.
*/
#define pr_fmt(fmt) "clint: " fmt
@@ -17,12 +25,16 @@
#include <linux/sched_clock.h>
#include <linux/io-64-nonatomic-lo-hi.h>
#include <linux/interrupt.h>
+#include <linux/irqchip/irq-riscv-aclint-swi.h>
#include <linux/of_irq.h>
#include <linux/smp.h>
#include <linux/timex.h>
-#ifndef CONFIG_RISCV_M_MODE
+#ifdef CONFIG_RISCV_M_MODE
#include <asm/clint.h>
+
+u64 __iomem *clint_time_val;
+EXPORT_SYMBOL(clint_time_val);
#endif
#define CLINT_IPI_OFF 0
@@ -35,11 +47,6 @@ static u64 __iomem *clint_timer_val;
static unsigned long clint_timer_freq;
static unsigned int clint_timer_irq;
-#ifdef CONFIG_RISCV_M_MODE
-u64 __iomem *clint_time_val;
-EXPORT_SYMBOL(clint_time_val);
-#endif
-
#ifdef CONFIG_64BIT
#define clint_get_cycles() readq_relaxed(clint_timer_val)
#else
@@ -129,8 +136,10 @@ static int __init clint_timer_init_dt(struct device_node *np)
{
int rc;
u32 i, nr_irqs;
- void __iomem *base;
+ void __iomem *base = NULL;
+ void __iomem *base1 = NULL;
struct of_phandle_args oirq;
+ bool is_aclint = of_device_is_compatible(np, "riscv,aclint-mtimer");
/*
* Ensure that CLINT device interrupts are either RV_IRQ_TIMER or
@@ -170,8 +179,19 @@ static int __init clint_timer_init_dt(struct device_node *np)
return -ENODEV;
}
- clint_timer_cmp = base + CLINT_TIMER_CMP_OFF;
- clint_timer_val = base + CLINT_TIMER_VAL_OFF;
+ if (is_aclint) {
+ clint_timer_val = base;
+ base1 = of_iomap(np, 1);
+ if (!base1) {
+ rc = -ENODEV;
+ pr_err("%pOFP: could not map registers\n", np);
+ goto fail_iounmap;
+ }
+ clint_timer_cmp = base1;
+ } else {
+ clint_timer_cmp = base + CLINT_TIMER_CMP_OFF;
+ clint_timer_val = base + CLINT_TIMER_VAL_OFF;
+ }
clint_timer_freq = riscv_timebase;
#ifdef CONFIG_RISCV_M_MODE
@@ -208,14 +228,29 @@ static int __init clint_timer_init_dt(struct device_node *np)
goto fail_free_irq;
}
+ if (!is_aclint) {
+ rc = aclint_swi_init(np, base + CLINT_IPI_OFF);
+ if (rc) {
+ pr_err("%pOFP: aclint swi init failed [%d]\n",
+ np, rc);
+ goto fail_remove_cpuhp;
+ }
+ }
+
return 0;
+fail_remove_cpuhp:
+ cpuhp_remove_state(CPUHP_AP_CLINT_TIMER_STARTING);
fail_free_irq:
free_irq(clint_timer_irq, &clint_clock_event);
fail_iounmap:
- iounmap(base);
+ if (base1)
+ iounmap(base1);
+ if (base)
+ iounmap(base);
return rc;
}
TIMER_OF_DECLARE(clint_timer, "riscv,clint0", clint_timer_init_dt);
TIMER_OF_DECLARE(clint_timer1, "sifive,clint0", clint_timer_init_dt);
+TIMER_OF_DECLARE(clint_timer2, "riscv,aclint-mtimer", clint_timer_init_dt);
--
2.25.1
If IPI calls are injected using SBI IPI calls then remote TLB flush
using SBI RFENCE calls is much faster because using IPIs for remote
TLB flush would still endup as SBI IPI calls with extra processing
on kernel side.
It is now possible to have specialized hardware (such as RISC-V AIA
and RISC-V ACLINT) which allows S-mode software to directly inject
IPIs without any assistance from M-mode runtime firmware.
This patch extends remote TLB flush functions to use IPIs whenever
underlying IPI operations are suitable for remote FENCEs.
Signed-off-by: Anup Patel <[email protected]>
---
arch/riscv/mm/tlbflush.c | 91 +++++++++++++++++++++++++++++++---------
1 file changed, 72 insertions(+), 19 deletions(-)
diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c
index 64f8201237c2..f96f02ed29ef 100644
--- a/arch/riscv/mm/tlbflush.c
+++ b/arch/riscv/mm/tlbflush.c
@@ -23,13 +23,60 @@ static inline void local_flush_tlb_page_asid(unsigned long addr,
: "memory");
}
+static inline void local_flush_tlb_range(unsigned long start,
+ unsigned long size, unsigned long stride)
+{
+ if (size <= stride)
+ local_flush_tlb_page(start);
+ else
+ local_flush_tlb_all();
+}
+
+static inline void local_flush_tlb_range_asid(unsigned long start,
+ unsigned long size, unsigned long stride, unsigned long asid)
+{
+ if (size <= stride)
+ local_flush_tlb_page_asid(start, asid);
+ else
+ local_flush_tlb_all_asid(asid);
+}
+
+static void __ipi_flush_tlb_all(void *info)
+{
+ local_flush_tlb_all();
+}
+
void flush_tlb_all(void)
{
- sbi_remote_sfence_vma(NULL, 0, -1);
+ if (riscv_use_ipi_for_rfence())
+ on_each_cpu(__ipi_flush_tlb_all, NULL, 1);
+ else
+ sbi_remote_sfence_vma(NULL, 0, -1);
+}
+
+struct flush_tlb_range_data {
+ unsigned long asid;
+ unsigned long start;
+ unsigned long size;
+ unsigned long stride;
+};
+
+static void __ipi_flush_tlb_range_asid(void *info)
+{
+ struct flush_tlb_range_data *d = info;
+
+ local_flush_tlb_range_asid(d->start, d->size, d->stride, d->asid);
+}
+
+static void __ipi_flush_tlb_range(void *info)
+{
+ struct flush_tlb_range_data *d = info;
+
+ local_flush_tlb_range(d->start, d->size, d->stride);
}
-static void __sbi_tlb_flush_range(struct mm_struct *mm, unsigned long start,
- unsigned long size, unsigned long stride)
+static void __flush_tlb_range(struct mm_struct *mm, unsigned long start,
+ unsigned long size, unsigned long stride)
{
struct cpumask *cmask = mm_cpumask(mm);
struct cpumask hmask;
@@ -46,23 +93,29 @@ static void __sbi_tlb_flush_range(struct mm_struct *mm, unsigned long start,
unsigned long asid = atomic_long_read(&mm->context.id);
if (broadcast) {
- riscv_cpuid_to_hartid_mask(cmask, &hmask);
- sbi_remote_sfence_vma_asid(cpumask_bits(&hmask),
- start, size, asid);
- } else if (size <= stride) {
- local_flush_tlb_page_asid(start, asid);
+ if (riscv_use_ipi_for_rfence()) {
+ on_each_cpu(__ipi_flush_tlb_range_asid,
+ cmask, 1);
+ } else {
+ riscv_cpuid_to_hartid_mask(cmask, &hmask);
+ sbi_remote_sfence_vma_asid(
+ cpumask_bits(&hmask),
+ start, size, asid);
+ }
} else {
- local_flush_tlb_all_asid(asid);
+ local_flush_tlb_range_asid(start, size, stride, asid);
}
} else {
if (broadcast) {
- riscv_cpuid_to_hartid_mask(cmask, &hmask);
- sbi_remote_sfence_vma(cpumask_bits(&hmask),
- start, size);
- } else if (size <= stride) {
- local_flush_tlb_page(start);
+ if (riscv_use_ipi_for_rfence()) {
+ on_each_cpu(__ipi_flush_tlb_range, cmask, 1);
+ } else {
+ riscv_cpuid_to_hartid_mask(cmask, &hmask);
+ sbi_remote_sfence_vma(cpumask_bits(&hmask),
+ start, size);
+ }
} else {
- local_flush_tlb_all();
+ local_flush_tlb_range(start, size, stride);
}
}
@@ -71,23 +124,23 @@ static void __sbi_tlb_flush_range(struct mm_struct *mm, unsigned long start,
void flush_tlb_mm(struct mm_struct *mm)
{
- __sbi_tlb_flush_range(mm, 0, -1, PAGE_SIZE);
+ __flush_tlb_range(mm, 0, -1, PAGE_SIZE);
}
void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
{
- __sbi_tlb_flush_range(vma->vm_mm, addr, PAGE_SIZE, PAGE_SIZE);
+ __flush_tlb_range(vma->vm_mm, addr, PAGE_SIZE, PAGE_SIZE);
}
void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
unsigned long end)
{
- __sbi_tlb_flush_range(vma->vm_mm, start, end - start, PAGE_SIZE);
+ __flush_tlb_range(vma->vm_mm, start, end - start, PAGE_SIZE);
}
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start,
unsigned long end)
{
- __sbi_tlb_flush_range(vma->vm_mm, start, end - start, PMD_SIZE);
+ __flush_tlb_range(vma->vm_mm, start, end - start, PMD_SIZE);
}
#endif
--
2.25.1
The RISC-V ACLINT defines MSWI and SSWI devices for M-mode and
S-mode software interrupts respectively. We add irqchip driver
which provide IPI operations based on ACLINT [M|S]SWI devices
to the Linux RISC-V kernel.
Signed-off-by: Anup Patel <[email protected]>
---
drivers/irqchip/Kconfig | 9 +
drivers/irqchip/Makefile | 1 +
drivers/irqchip/irq-riscv-aclint-swi.c | 282 +++++++++++++++++++
include/linux/irqchip/irq-riscv-aclint-swi.h | 19 ++
4 files changed, 311 insertions(+)
create mode 100644 drivers/irqchip/irq-riscv-aclint-swi.c
create mode 100644 include/linux/irqchip/irq-riscv-aclint-swi.h
diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig
index aca7b595c4c7..c354d0576b1a 100644
--- a/drivers/irqchip/Kconfig
+++ b/drivers/irqchip/Kconfig
@@ -509,6 +509,15 @@ config RISCV_INTC
If you don't know what to do here, say Y.
+config RISCV_ACLINT_SWI
+ bool "RISC-V Advanced Core Local Interruptor Software Interrupts"
+ depends on RISCV
+ help
+ This enables support for software interrupts using the Advanced
+ Core Local Interruptor (ACLINT) found in RISC-V systems. The
+ RISC-V ACLINT provides devices for inter-process interrupt and
+ timer functionality.
+
config SIFIVE_PLIC
bool "SiFive Platform-Level Interrupt Controller"
depends on RISCV
diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile
index f88cbf36a9d2..81306d560972 100644
--- a/drivers/irqchip/Makefile
+++ b/drivers/irqchip/Makefile
@@ -97,6 +97,7 @@ obj-$(CONFIG_QCOM_PDC) += qcom-pdc.o
obj-$(CONFIG_CSKY_MPINTC) += irq-csky-mpintc.o
obj-$(CONFIG_CSKY_APB_INTC) += irq-csky-apb-intc.o
obj-$(CONFIG_RISCV_INTC) += irq-riscv-intc.o
+obj-$(CONFIG_RISCV_ACLINT_SWI) += irq-riscv-aclint-swi.o
obj-$(CONFIG_SIFIVE_PLIC) += irq-sifive-plic.o
obj-$(CONFIG_IMX_IRQSTEER) += irq-imx-irqsteer.o
obj-$(CONFIG_IMX_INTMUX) += irq-imx-intmux.o
diff --git a/drivers/irqchip/irq-riscv-aclint-swi.c b/drivers/irqchip/irq-riscv-aclint-swi.c
new file mode 100644
index 000000000000..ffd16cb795ba
--- /dev/null
+++ b/drivers/irqchip/irq-riscv-aclint-swi.c
@@ -0,0 +1,282 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2021 Western Digital Corporation or its affiliates.
+ */
+
+#define pr_fmt(fmt) "aclint-swi: " fmt
+#include <linux/cpu.h>
+#include <linux/cpumask.h>
+#include <linux/io.h>
+#include <linux/init.h>
+#include <linux/irq.h>
+#include <linux/irqchip.h>
+#include <linux/irqchip/chained_irq.h>
+#include <linux/irqchip/irq-riscv-aclint-swi.h>
+#include <linux/irqdomain.h>
+#include <linux/module.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/smp.h>
+
+struct aclint_swi {
+ void __iomem *sip_reg;
+ unsigned long bits;
+};
+
+static int aclint_swi_parent_irq __ro_after_init;
+static struct irq_domain *aclint_swi_domain __ro_after_init;
+static DEFINE_PER_CPU(struct aclint_swi, aclint_swis);
+
+static void aclint_swi_dummy(struct irq_data *d)
+{
+}
+
+static void aclint_swi_send_mask(struct irq_data *d,
+ const struct cpumask *mask)
+{
+ int cpu;
+ struct aclint_swi *swi;
+
+ /* Barrier before doing atomic bit update to IPI bits */
+ smp_mb__before_atomic();
+
+ for_each_cpu(cpu, mask) {
+ swi = per_cpu_ptr(&aclint_swis, cpu);
+ set_bit(d->hwirq, &swi->bits);
+ writel(1, swi->sip_reg);
+ }
+
+ /* Barrier after doing atomic bit update to IPI bits */
+ smp_mb__after_atomic();
+}
+
+static struct irq_chip aclint_swi_chip = {
+ .name = "RISC-V ACLINT SWI",
+ .irq_mask = aclint_swi_dummy,
+ .irq_unmask = aclint_swi_dummy,
+ .ipi_send_mask = aclint_swi_send_mask,
+};
+
+static int aclint_swi_domain_map(struct irq_domain *d, unsigned int irq,
+ irq_hw_number_t hwirq)
+{
+ irq_set_percpu_devid(irq);
+ irq_domain_set_info(d, irq, hwirq, &aclint_swi_chip, d->host_data,
+ handle_percpu_devid_irq, NULL, NULL);
+
+ return 0;
+}
+
+static int aclint_swi_domain_alloc(struct irq_domain *d, unsigned int virq,
+ unsigned int nr_irqs, void *arg)
+{
+ int i, ret;
+ irq_hw_number_t hwirq;
+ unsigned int type = IRQ_TYPE_NONE;
+ struct irq_fwspec *fwspec = arg;
+
+ ret = irq_domain_translate_onecell(d, fwspec, &hwirq, &type);
+ if (ret)
+ return ret;
+
+ for (i = 0; i < nr_irqs; i++) {
+ ret = aclint_swi_domain_map(d, virq + i, hwirq + i);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+
+static const struct irq_domain_ops aclint_swi_domain_ops = {
+ .translate = irq_domain_translate_onecell,
+ .alloc = aclint_swi_domain_alloc,
+ .free = irq_domain_free_irqs_top,
+};
+
+static void aclint_swi_handle_irq(struct irq_desc *desc)
+{
+ int err;
+ unsigned long irqs;
+ irq_hw_number_t hwirq;
+ struct irq_chip *chip = irq_desc_get_chip(desc);
+ struct aclint_swi *swi = this_cpu_ptr(&aclint_swis);
+
+ chained_irq_enter(chip, desc);
+
+ while (true) {
+#ifdef CONFIG_RISCV_M_MODE
+ writel(0, swi->sip_reg);
+#else
+ csr_clear(CSR_IP, IE_SIE);
+#endif
+
+ /* Order bit clearing and data access. */
+ mb();
+
+ irqs = xchg(&swi->bits, 0);
+ if (!irqs)
+ goto done;
+
+ for_each_set_bit(hwirq, &irqs, BITS_PER_LONG) {
+ err = generic_handle_domain_irq(aclint_swi_domain,
+ hwirq);
+ if (unlikely(err))
+ pr_warn_ratelimited(
+ "can't find mapping for hwirq %lu\n",
+ hwirq);
+ }
+ }
+
+done:
+ chained_irq_exit(chip, desc);
+}
+
+static int aclint_swi_dying_cpu(unsigned int cpu)
+{
+ disable_percpu_irq(aclint_swi_parent_irq);
+ return 0;
+}
+
+static int aclint_swi_starting_cpu(unsigned int cpu)
+{
+ enable_percpu_irq(aclint_swi_parent_irq,
+ irq_get_trigger_type(aclint_swi_parent_irq));
+ return 0;
+}
+
+static int __init aclint_swi_domain_init(struct device_node *node)
+{
+ int virq;
+ struct irq_fwspec ipi;
+
+ /*
+ * We can have multiple ACLINT SWI devices but we only need
+ * one IRQ domain for providing per-HART (or per-CPU) IPIs.
+ */
+ if (aclint_swi_domain)
+ return 0;
+
+ aclint_swi_domain = irq_domain_add_linear(node, BITS_PER_LONG,
+ &aclint_swi_domain_ops, NULL);
+ if (!aclint_swi_domain) {
+ pr_err("unable to add ACLINT SWI IRQ domain\n");
+ return -ENOMEM;
+ }
+
+ ipi.fwnode = aclint_swi_domain->fwnode;
+ ipi.param_count = 1;
+ ipi.param[0] = 0;
+ virq = __irq_domain_alloc_irqs(aclint_swi_domain, -1, BITS_PER_LONG,
+ NUMA_NO_NODE, &ipi,
+ false, NULL);
+ if (virq <= 0) {
+ pr_err("unable to alloc IRQs from ACLINT SWI IRQ domain\n");
+ return -ENOMEM;
+ }
+
+ riscv_ipi_set_virq_range(virq, BITS_PER_LONG, true);
+
+ return 0;
+}
+
+/**
+ * aclint_swi_init() - Function to initialize ACLINT from SiFive CLINT
+ * timer driver used by NoMMU (M-mode) kernel.
+ * @node: device node pointer
+ * @base: base address of the [M|S]SWI registers
+ */
+int __init aclint_swi_init(struct device_node *node, void __iomem *base)
+{
+ int rc;
+ struct aclint_swi *swi;
+ u32 i, nr_irqs, nr_cpus = 0;
+
+ /* Iterate over each target CPU connected with this ACLINT */
+ nr_irqs = of_irq_count(node);
+ for (i = 0; i < nr_irqs; i++) {
+ struct of_phandle_args parent;
+ int cpu, hartid;
+
+ if (of_irq_parse_one(node, i, &parent)) {
+ pr_err("%pOFP: failed to parse irq %d.\n",
+ node, i);
+ continue;
+ }
+
+ if (parent.args[0] != RV_IRQ_SOFT)
+ continue;
+
+ hartid = riscv_of_parent_hartid(parent.np);
+ if (hartid < 0) {
+ pr_warn("failed to parse hart ID for irq %d.\n", i);
+ continue;
+ }
+
+ cpu = riscv_hartid_to_cpuid(hartid);
+ if (cpu < 0) {
+ pr_warn("Invalid cpuid for irq %d\n", i);
+ continue;
+ }
+
+ /* Find parent domain and register chained handler */
+ if (!aclint_swi_parent_irq && irq_find_host(parent.np)) {
+ aclint_swi_parent_irq = irq_of_parse_and_map(node, i);
+ if (aclint_swi_parent_irq) {
+ irq_set_chained_handler(aclint_swi_parent_irq,
+ aclint_swi_handle_irq);
+ cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
+ "irqchip/riscv/aclint-swi:starting",
+ aclint_swi_starting_cpu,
+ aclint_swi_dying_cpu);
+ }
+ }
+
+ swi = per_cpu_ptr(&aclint_swis, cpu);
+ swi->sip_reg = base + nr_cpus * sizeof(u32);
+ writel(0, swi->sip_reg);
+
+ nr_cpus++;
+ }
+
+ /* Create the IPI domain for ACLINT SWI device */
+ rc = aclint_swi_domain_init(node);
+ if (rc)
+ return rc;
+
+ /* Announce the ACLINT SWI device */
+ pr_info("%pOFP: providing IPIs for %d CPUs\n", node, nr_cpus);
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(aclint_swi_init);
+
+static int __init aclint_swi_of_init(struct device_node *node,
+ struct device_node *parent)
+{
+ int rc;
+ void __iomem *base;
+
+ /* Map the registers */
+ base = of_iomap(node, 0);
+ if (!base) {
+ pr_err("%pOFP: could not map registers\n", node);
+ return -ENODEV;
+ }
+
+ /* Call common/exported ACLINT SWI initialization */
+ rc = aclint_swi_init(node, base);
+ if (rc) {
+ iounmap(base);
+ return rc;
+ }
+
+ return 0;
+}
+
+#ifdef CONFIG_RISCV_M_MODE
+IRQCHIP_DECLARE(riscv_aclint_swi, "riscv,aclint-mswi", aclint_swi_of_init);
+#else
+IRQCHIP_DECLARE(riscv_aclint_swi, "riscv,aclint-sswi", aclint_swi_of_init);
+#endif
diff --git a/include/linux/irqchip/irq-riscv-aclint-swi.h b/include/linux/irqchip/irq-riscv-aclint-swi.h
new file mode 100644
index 000000000000..766b7ec26667
--- /dev/null
+++ b/include/linux/irqchip/irq-riscv-aclint-swi.h
@@ -0,0 +1,19 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef __IRQ_RISCV_ACLINT_SWI_H
+#define __IRQ_RISCV_ACLINT_SWI_H
+
+#include <linux/types.h>
+
+struct device_node;
+
+#ifdef CONFIG_RISCV_ACLINT_SWI
+int aclint_swi_init(struct device_node *node, void __iomem *base);
+#else
+static inline int aclint_swi_init(struct device_node *node,
+ void __iomem *base)
+{
+ return 0;
+}
+#endif
+
+#endif /* __IRQ_RISCV_ACLINT_SWI_H */
--
2.25.1
Currently, the RISC-V kernel provides arch specific hooks (i.e.
struct riscv_ipi_ops) to register IPI handling methods. The stats
gathering of IPIs is also arch specific in the RISC-V kernel.
Other architectures (such as ARM, ARM64, and MIPS) have moved away
from custom arch specific IPI handling methods. Currently, these
architectures have Linux irqchip drivers providing a range of Linux
IRQ numbers to be used as IPIs and IPI triggering is done using
generic IPI APIs. This approach allows architectures to treat IPIs
as normal Linux IRQs and IPI stats gathering is done by the generic
Linux IRQ subsystem.
We extend the RISC-V IPI handling as-per above approach so that
arch specific IPI handling methods (struct riscv_ipi_ops) can be
removed and the IPI handling is totally contained within Linux
irqchip drivers.
Signed-off-by: Anup Patel <[email protected]>
---
arch/riscv/Kconfig | 1 +
arch/riscv/include/asm/sbi.h | 2 +
arch/riscv/include/asm/smp.h | 35 +++--
arch/riscv/kernel/Makefile | 1 +
arch/riscv/kernel/cpu-hotplug.c | 3 +-
arch/riscv/kernel/irq.c | 3 +-
arch/riscv/kernel/sbi-ipi.c | 218 ++++++++++++++++++++++++++++++
arch/riscv/kernel/sbi.c | 21 ---
arch/riscv/kernel/smp.c | 156 ++++++++++-----------
arch/riscv/kernel/smpboot.c | 5 +-
drivers/clocksource/timer-clint.c | 23 ----
drivers/irqchip/irq-riscv-intc.c | 55 ++++----
12 files changed, 347 insertions(+), 176 deletions(-)
create mode 100644 arch/riscv/kernel/sbi-ipi.c
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 2251f27aba53..8de2afb460f7 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -54,6 +54,7 @@ config RISCV
select GENERIC_GETTIMEOFDAY if HAVE_GENERIC_VDSO
select GENERIC_IDLE_POLL_SETUP
select GENERIC_IOREMAP if MMU
+ select GENERIC_IRQ_IPI
select GENERIC_IRQ_MULTI_HANDLER
select GENERIC_IRQ_SHOW
select GENERIC_IRQ_SHOW_LEVEL
diff --git a/arch/riscv/include/asm/sbi.h b/arch/riscv/include/asm/sbi.h
index 289621da4a2a..a992faeded7e 100644
--- a/arch/riscv/include/asm/sbi.h
+++ b/arch/riscv/include/asm/sbi.h
@@ -106,6 +106,7 @@ struct sbiret {
};
void sbi_init(void);
+void sbi_ipi_init(void);
struct sbiret sbi_ecall(int ext, int fid, unsigned long arg0,
unsigned long arg1, unsigned long arg2,
unsigned long arg3, unsigned long arg4,
@@ -175,6 +176,7 @@ static inline unsigned long sbi_mk_version(unsigned long major,
int sbi_err_map_linux_errno(int err);
#else /* CONFIG_RISCV_SBI */
static inline int sbi_remote_fence_i(const unsigned long *hart_mask) { return -1; }
+static inline void sbi_ipi_init(void) { }
static inline void sbi_init(void) {}
#endif /* CONFIG_RISCV_SBI */
#endif /* _ASM_RISCV_SBI_H */
diff --git a/arch/riscv/include/asm/smp.h b/arch/riscv/include/asm/smp.h
index a7d2811f3536..e5480d75b2d9 100644
--- a/arch/riscv/include/asm/smp.h
+++ b/arch/riscv/include/asm/smp.h
@@ -15,11 +15,6 @@
struct seq_file;
extern unsigned long boot_cpu_hartid;
-struct riscv_ipi_ops {
- void (*ipi_inject)(const struct cpumask *target);
- void (*ipi_clear)(void);
-};
-
#ifdef CONFIG_SMP
/*
* Mapping between linux logical cpu index and hartid.
@@ -33,9 +28,6 @@ void show_ipi_stats(struct seq_file *p, int prec);
/* SMP initialization hook for setup_arch */
void __init setup_smp(void);
-/* Called from C code, this handles an IPI. */
-void handle_IPI(struct pt_regs *regs);
-
/* Hook for the generic smp_call_function_many() routine. */
void arch_send_call_function_ipi_mask(struct cpumask *mask);
@@ -45,11 +37,17 @@ void arch_send_call_function_single_ipi(int cpu);
int riscv_hartid_to_cpuid(int hartid);
void riscv_cpuid_to_hartid_mask(const struct cpumask *in, struct cpumask *out);
-/* Set custom IPI operations */
-void riscv_set_ipi_ops(const struct riscv_ipi_ops *ops);
+/* Enable IPI for CPU hotplug */
+void riscv_ipi_enable(void);
+
+/* Disable IPI for CPU hotplug */
+void riscv_ipi_disable(void);
-/* Clear IPI for current CPU */
-void riscv_clear_ipi(void);
+/* Check if IPI interrupt numbers are available */
+bool riscv_ipi_have_virq_range(void);
+
+/* Set the IPI interrupt numbers for arch (called by irqchip drivers) */
+void riscv_ipi_set_virq_range(int virq, int nr_irqs);
/* Secondary hart entry */
asmlinkage void smp_callin(void);
@@ -92,11 +90,20 @@ static inline void riscv_cpuid_to_hartid_mask(const struct cpumask *in,
cpumask_set_cpu(boot_cpu_hartid, out);
}
-static inline void riscv_set_ipi_ops(const struct riscv_ipi_ops *ops)
+static inline void riscv_ipi_enable(void)
{
}
-static inline void riscv_clear_ipi(void)
+static inline void riscv_ipi_disable(void)
+{
+}
+
+static inline bool riscv_ipi_have_virq_range(void)
+{
+ return false;
+}
+
+static inline void riscv_ipi_set_virq_range(int virq, int nr)
{
}
diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile
index 3397ddac1a30..38b555edb2ee 100644
--- a/arch/riscv/kernel/Makefile
+++ b/arch/riscv/kernel/Makefile
@@ -54,6 +54,7 @@ obj-$(CONFIG_RISCV_BASE_PMU) += perf_event.o
obj-$(CONFIG_PERF_EVENTS) += perf_callchain.o
obj-$(CONFIG_HAVE_PERF_REGS) += perf_regs.o
obj-$(CONFIG_RISCV_SBI) += sbi.o
+obj-$(CONFIG_RISCV_SBI) += sbi-ipi.o
ifeq ($(CONFIG_RISCV_SBI), y)
obj-$(CONFIG_SMP) += cpu_ops_sbi.o
endif
diff --git a/arch/riscv/kernel/cpu-hotplug.c b/arch/riscv/kernel/cpu-hotplug.c
index df84e0c13db1..141f8a01d706 100644
--- a/arch/riscv/kernel/cpu-hotplug.c
+++ b/arch/riscv/kernel/cpu-hotplug.c
@@ -12,7 +12,7 @@
#include <linux/sched/hotplug.h>
#include <asm/irq.h>
#include <asm/cpu_ops.h>
-#include <asm/sbi.h>
+#include <asm/smp.h>
void cpu_stop(void);
void arch_cpu_idle_dead(void)
@@ -47,6 +47,7 @@ int __cpu_disable(void)
remove_cpu_topology(cpu);
set_cpu_online(cpu, false);
+ riscv_ipi_disable();
irq_migrate_all_off_this_cpu();
return ret;
diff --git a/arch/riscv/kernel/irq.c b/arch/riscv/kernel/irq.c
index 7207fa08d78f..a179350d2208 100644
--- a/arch/riscv/kernel/irq.c
+++ b/arch/riscv/kernel/irq.c
@@ -8,7 +8,7 @@
#include <linux/interrupt.h>
#include <linux/irqchip.h>
#include <linux/seq_file.h>
-#include <asm/smp.h>
+#include <asm/sbi.h>
int arch_show_interrupts(struct seq_file *p, int prec)
{
@@ -21,4 +21,5 @@ void __init init_IRQ(void)
irqchip_init();
if (!handle_arch_irq)
panic("No interrupt controller found.");
+ sbi_ipi_init();
}
diff --git a/arch/riscv/kernel/sbi-ipi.c b/arch/riscv/kernel/sbi-ipi.c
new file mode 100644
index 000000000000..8d06b1e3d682
--- /dev/null
+++ b/arch/riscv/kernel/sbi-ipi.c
@@ -0,0 +1,218 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * SBI based IPI support.
+ *
+ * Copyright (c) 2021 Western Digital Corporation or its affiliates.
+ */
+
+#define pr_fmt(fmt) "riscv-sbi-ipi: " fmt
+#include <linux/cpu.h>
+#include <linux/cpumask.h>
+#include <linux/init.h>
+#include <linux/irq.h>
+#include <linux/irqchip.h>
+#include <linux/irqchip/chained_irq.h>
+#include <linux/irqdomain.h>
+#include <linux/of.h>
+#include <linux/smp.h>
+#include <asm/sbi.h>
+
+static int intc_parent_irq __ro_after_init;
+static struct irq_domain *sbi_ipi_domain __ro_after_init;
+static DEFINE_PER_CPU(unsigned long, sbi_ipi_bits);
+
+static void sbi_ipi_dummy(struct irq_data *d)
+{
+}
+
+static void sbi_ipi_send_mask(struct irq_data *d, const struct cpumask *mask)
+{
+ int cpu;
+ struct cpumask hartid_mask;
+
+ /* Barrier before doing atomic bit update to IPI bits */
+ smp_mb__before_atomic();
+ for_each_cpu(cpu, mask)
+ set_bit(d->hwirq, per_cpu_ptr(&sbi_ipi_bits, cpu));
+ /* Barrier after doing atomic bit update to IPI bits */
+ smp_mb__after_atomic();
+
+ riscv_cpuid_to_hartid_mask(mask, &hartid_mask);
+
+ sbi_send_ipi(cpumask_bits(&hartid_mask));
+}
+
+static struct irq_chip sbi_ipi_chip = {
+ .name = "RISC-V SBI IPI",
+ .irq_mask = sbi_ipi_dummy,
+ .irq_unmask = sbi_ipi_dummy,
+ .ipi_send_mask = sbi_ipi_send_mask,
+};
+
+static int sbi_ipi_domain_map(struct irq_domain *d, unsigned int irq,
+ irq_hw_number_t hwirq)
+{
+ irq_set_percpu_devid(irq);
+ irq_domain_set_info(d, irq, hwirq, &sbi_ipi_chip, d->host_data,
+ handle_percpu_devid_irq, NULL, NULL);
+
+ return 0;
+}
+
+static int sbi_ipi_domain_alloc(struct irq_domain *d, unsigned int virq,
+ unsigned int nr_irqs, void *arg)
+{
+ int i, ret;
+ irq_hw_number_t hwirq;
+ unsigned int type = IRQ_TYPE_NONE;
+ struct irq_fwspec *fwspec = arg;
+
+ ret = irq_domain_translate_onecell(d, fwspec, &hwirq, &type);
+ if (ret)
+ return ret;
+
+ for (i = 0; i < nr_irqs; i++) {
+ ret = sbi_ipi_domain_map(d, virq + i, hwirq + i);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+
+static const struct irq_domain_ops sbi_ipi_domain_ops = {
+ .translate = irq_domain_translate_onecell,
+ .alloc = sbi_ipi_domain_alloc,
+ .free = irq_domain_free_irqs_top,
+};
+
+static void sbi_ipi_handle_irq(struct irq_desc *desc)
+{
+ int err;
+ struct irq_chip *chip = irq_desc_get_chip(desc);
+ unsigned long irqs, *bits = this_cpu_ptr(&sbi_ipi_bits);
+ irq_hw_number_t hwirq;
+
+ chained_irq_enter(chip, desc);
+
+ while (true) {
+ csr_clear(CSR_IP, IE_SIE);
+
+ /* Order bit clearing and data access. */
+ mb();
+
+ irqs = xchg(bits, 0);
+ if (!irqs)
+ goto done;
+
+ for_each_set_bit(hwirq, &irqs, BITS_PER_LONG) {
+ err = generic_handle_domain_irq(sbi_ipi_domain,
+ hwirq);
+ if (unlikely(err))
+ pr_warn_ratelimited(
+ "can't find mapping for hwirq %lu\n",
+ hwirq);
+ }
+ }
+
+done:
+ chained_irq_exit(chip, desc);
+}
+
+static int sbi_ipi_dying_cpu(unsigned int cpu)
+{
+ disable_percpu_irq(intc_parent_irq);
+ return 0;
+}
+
+static int sbi_ipi_starting_cpu(unsigned int cpu)
+{
+ enable_percpu_irq(intc_parent_irq,
+ irq_get_trigger_type(intc_parent_irq));
+ return 0;
+}
+
+static int __init sbi_ipi_set_virq(void)
+{
+ int virq;
+ struct irq_fwspec ipi = {
+ .fwnode = sbi_ipi_domain->fwnode,
+ .param_count = 1,
+ .param[0] = 0,
+ };
+
+ virq = __irq_domain_alloc_irqs(sbi_ipi_domain, -1, BITS_PER_LONG,
+ NUMA_NO_NODE, &ipi,
+ false, NULL);
+ if (virq <= 0) {
+ pr_err("unable to alloc IRQs from SBI IPI IRQ domain\n");
+ return -ENOMEM;
+ }
+
+ riscv_ipi_set_virq_range(virq, BITS_PER_LONG);
+
+ return 0;
+}
+
+static int __init sbi_ipi_domain_init(struct irq_domain *domain)
+{
+ struct irq_fwspec swi = {
+ .fwnode = domain->fwnode,
+ .param_count = 1,
+ .param[0] = RV_IRQ_SOFT,
+ };
+
+ intc_parent_irq = __irq_domain_alloc_irqs(domain, -1, 1,
+ NUMA_NO_NODE, &swi,
+ false, NULL);
+ if (intc_parent_irq <= 0) {
+ pr_err("unable to alloc IRQ from INTC IRQ domain\n");
+ return -ENOMEM;
+ }
+
+ irq_set_chained_handler(intc_parent_irq, sbi_ipi_handle_irq);
+
+ sbi_ipi_domain = irq_domain_add_linear(NULL, BITS_PER_LONG,
+ &sbi_ipi_domain_ops, NULL);
+ if (!sbi_ipi_domain) {
+ pr_err("unable to add SBI IPI IRQ domain\n");
+ return -ENOMEM;
+ }
+
+ cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
+ "irqchip/riscv/sbi-ipi:starting",
+ sbi_ipi_starting_cpu, sbi_ipi_dying_cpu);
+
+ return sbi_ipi_set_virq();
+}
+
+void __init sbi_ipi_init(void)
+{
+ struct irq_domain *domain = NULL;
+ struct device_node *cpu, *child;
+
+ if (riscv_ipi_have_virq_range())
+ return;
+
+ for_each_of_cpu_node(cpu) {
+ child = of_get_compatible_child(cpu, "riscv,cpu-intc");
+ if (!child) {
+ pr_err("failed to find INTC node [%pOF]\n", cpu);
+ return;
+ }
+
+ domain = irq_find_host(child);
+ of_node_put(child);
+ if (domain)
+ break;
+ }
+ if (!domain) {
+ pr_err("can't find INTC IRQ domain\n");
+ return;
+ }
+
+ if (sbi_ipi_domain_init(domain))
+ pr_err("failed to register IPI domain\n");
+ else
+ pr_info("registered IPI domain\n");
+}
diff --git a/arch/riscv/kernel/sbi.c b/arch/riscv/kernel/sbi.c
index 8aeca26198f2..372aa7e181d5 100644
--- a/arch/riscv/kernel/sbi.c
+++ b/arch/riscv/kernel/sbi.c
@@ -589,25 +589,6 @@ long sbi_get_mimpid(void)
return __sbi_base_ecall(SBI_EXT_BASE_GET_MIMPID);
}
-static void sbi_send_cpumask_ipi(const struct cpumask *target)
-{
- struct cpumask hartid_mask;
-
- riscv_cpuid_to_hartid_mask(target, &hartid_mask);
-
- sbi_send_ipi(cpumask_bits(&hartid_mask));
-}
-
-static void sbi_ipi_clear(void)
-{
- csr_clear(CSR_IP, IE_SIE);
-}
-
-static const struct riscv_ipi_ops sbi_ipi_ops = {
- .ipi_inject = sbi_send_cpumask_ipi,
- .ipi_clear = sbi_ipi_clear
-};
-
void __init sbi_init(void)
{
int ret;
@@ -654,6 +635,4 @@ void __init sbi_init(void)
__sbi_send_ipi = __sbi_send_ipi_v01;
__sbi_rfence = __sbi_rfence_v01;
}
-
- riscv_set_ipi_ops(&sbi_ipi_ops);
}
diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c
index 547dc508f7d1..53902cd1671f 100644
--- a/arch/riscv/kernel/smp.c
+++ b/arch/riscv/kernel/smp.c
@@ -17,9 +17,9 @@
#include <linux/sched.h>
#include <linux/seq_file.h>
#include <linux/delay.h>
+#include <linux/irq.h>
#include <linux/irq_work.h>
-#include <asm/sbi.h>
#include <asm/tlbflush.h>
#include <asm/cacheflush.h>
@@ -41,11 +41,9 @@ void __init smp_setup_processor_id(void)
cpuid_to_hartid_map(0) = boot_cpu_hartid;
}
-/* A collection of single bit ipi messages. */
-static struct {
- unsigned long stats[IPI_MAX] ____cacheline_aligned;
- unsigned long bits ____cacheline_aligned;
-} ipi_data[NR_CPUS] __cacheline_aligned;
+static int ipi_virq_base __ro_after_init;
+static int nr_ipi __ro_after_init = IPI_MAX;
+static struct irq_desc *ipi_desc[IPI_MAX] __read_mostly;
int riscv_hartid_to_cpuid(int hartid)
{
@@ -87,46 +85,14 @@ static void ipi_stop(void)
wait_for_interrupt();
}
-static const struct riscv_ipi_ops *ipi_ops __ro_after_init;
-
-void riscv_set_ipi_ops(const struct riscv_ipi_ops *ops)
-{
- ipi_ops = ops;
-}
-EXPORT_SYMBOL_GPL(riscv_set_ipi_ops);
-
-void riscv_clear_ipi(void)
-{
- if (ipi_ops && ipi_ops->ipi_clear)
- ipi_ops->ipi_clear();
-}
-EXPORT_SYMBOL_GPL(riscv_clear_ipi);
-
static void send_ipi_mask(const struct cpumask *mask, enum ipi_message_type op)
{
- int cpu;
-
- smp_mb__before_atomic();
- for_each_cpu(cpu, mask)
- set_bit(op, &ipi_data[cpu].bits);
- smp_mb__after_atomic();
-
- if (ipi_ops && ipi_ops->ipi_inject)
- ipi_ops->ipi_inject(mask);
- else
- pr_warn("SMP: IPI inject method not available\n");
+ __ipi_send_mask(ipi_desc[op], mask);
}
static void send_ipi_single(int cpu, enum ipi_message_type op)
{
- smp_mb__before_atomic();
- set_bit(op, &ipi_data[cpu].bits);
- smp_mb__after_atomic();
-
- if (ipi_ops && ipi_ops->ipi_inject)
- ipi_ops->ipi_inject(cpumask_of(cpu));
- else
- pr_warn("SMP: IPI inject method not available\n");
+ __ipi_send_mask(ipi_desc[op], cpumask_of(cpu));
}
#ifdef CONFIG_IRQ_WORK
@@ -136,62 +102,88 @@ void arch_irq_work_raise(void)
}
#endif
-void handle_IPI(struct pt_regs *regs)
+static irqreturn_t handle_IPI(int irq, void *data)
+{
+ int ipi = irq - ipi_virq_base;
+
+ switch (ipi) {
+ case IPI_RESCHEDULE:
+ scheduler_ipi();
+ break;
+ case IPI_CALL_FUNC:
+ generic_smp_call_function_interrupt();
+ break;
+ case IPI_CPU_STOP:
+ ipi_stop();
+ break;
+ case IPI_IRQ_WORK:
+ irq_work_run();
+ break;
+#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
+ case IPI_TIMER:
+ tick_receive_broadcast();
+ break;
+#endif
+ default:
+ pr_warn("CPU%d: unhandled IPI%d\n", smp_processor_id(), ipi);
+ break;
+ };
+
+ return IRQ_HANDLED;
+}
+
+void riscv_ipi_enable(void)
{
- struct pt_regs *old_regs = set_irq_regs(regs);
- unsigned long *pending_ipis = &ipi_data[smp_processor_id()].bits;
- unsigned long *stats = ipi_data[smp_processor_id()].stats;
+ int i;
- irq_enter();
+ if (WARN_ON_ONCE(!ipi_virq_base))
+ return;
- riscv_clear_ipi();
+ for (i = 0; i < nr_ipi; i++)
+ enable_percpu_irq(ipi_virq_base + i, 0);
+}
- while (true) {
- unsigned long ops;
+void riscv_ipi_disable(void)
+{
+ int i;
- /* Order bit clearing and data access. */
- mb();
+ if (WARN_ON_ONCE(!ipi_virq_base))
+ return;
- ops = xchg(pending_ipis, 0);
- if (ops == 0)
- goto done;
+ for (i = 0; i < nr_ipi; i++)
+ disable_percpu_irq(ipi_virq_base + i);
+}
- if (ops & (1 << IPI_RESCHEDULE)) {
- stats[IPI_RESCHEDULE]++;
- scheduler_ipi();
- }
+bool riscv_ipi_have_virq_range(void)
+{
+ return (ipi_virq_base) ? true : false;
+}
- if (ops & (1 << IPI_CALL_FUNC)) {
- stats[IPI_CALL_FUNC]++;
- generic_smp_call_function_interrupt();
- }
+void riscv_ipi_set_virq_range(int virq, int nr)
+{
+ int i, err;
- if (ops & (1 << IPI_CPU_STOP)) {
- stats[IPI_CPU_STOP]++;
- ipi_stop();
- }
+ if (WARN_ON(ipi_virq_base))
+ return;
- if (ops & (1 << IPI_IRQ_WORK)) {
- stats[IPI_IRQ_WORK]++;
- irq_work_run();
- }
+ WARN_ON(nr < IPI_MAX);
+ nr_ipi = min(nr, IPI_MAX);
+ ipi_virq_base = virq;
-#ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
- if (ops & (1 << IPI_TIMER)) {
- stats[IPI_TIMER]++;
- tick_receive_broadcast();
- }
-#endif
- BUG_ON((ops >> IPI_MAX) != 0);
+ /* Request IPIs */
+ for (i = 0; i < nr_ipi; i++) {
+ err = request_percpu_irq(ipi_virq_base + i, handle_IPI,
+ "IPI", &ipi_virq_base);
+ WARN_ON(err);
- /* Order data access and bit testing. */
- mb();
+ ipi_desc[i] = irq_to_desc(ipi_virq_base + i);
+ irq_set_status_flags(ipi_virq_base + i, IRQ_HIDDEN);
}
-done:
- irq_exit();
- set_irq_regs(old_regs);
+ /* Enabled IPIs for boot CPU immediately */
+ riscv_ipi_enable();
}
+EXPORT_SYMBOL_GPL(riscv_ipi_set_virq_range);
static const char * const ipi_names[] = {
[IPI_RESCHEDULE] = "Rescheduling interrupts",
@@ -209,7 +201,7 @@ void show_ipi_stats(struct seq_file *p, int prec)
seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i,
prec >= 4 ? " " : "");
for_each_online_cpu(cpu)
- seq_printf(p, "%10lu ", ipi_data[cpu].stats[i]);
+ seq_printf(p, "%10u ", irq_desc_kstat_cpu(ipi_desc[i], cpu));
seq_printf(p, " %s\n", ipi_names[i]);
}
}
diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c
index bd82375db51a..775d0026efd4 100644
--- a/arch/riscv/kernel/smpboot.c
+++ b/arch/riscv/kernel/smpboot.c
@@ -30,7 +30,6 @@
#include <asm/numa.h>
#include <asm/tlbflush.h>
#include <asm/sections.h>
-#include <asm/sbi.h>
#include <asm/smp.h>
#include <asm/alternative.h>
@@ -159,12 +158,12 @@ asmlinkage __visible void smp_callin(void)
struct mm_struct *mm = &init_mm;
unsigned int curr_cpuid = smp_processor_id();
- riscv_clear_ipi();
-
/* All kernel threads share the same mm context. */
mmgrab(mm);
current->active_mm = mm;
+ riscv_ipi_enable();
+
notify_cpu_starting(curr_cpuid);
numa_add_cpu(curr_cpuid);
update_siblings_masks(curr_cpuid);
diff --git a/drivers/clocksource/timer-clint.c b/drivers/clocksource/timer-clint.c
index 6cfe2ab73eb0..3b68ed53fe4a 100644
--- a/drivers/clocksource/timer-clint.c
+++ b/drivers/clocksource/timer-clint.c
@@ -30,7 +30,6 @@
#define CLINT_TIMER_VAL_OFF 0xbff8
/* CLINT manages IPI and Timer for RISC-V M-mode */
-static u32 __iomem *clint_ipi_base;
static u64 __iomem *clint_timer_cmp;
static u64 __iomem *clint_timer_val;
static unsigned long clint_timer_freq;
@@ -41,24 +40,6 @@ u64 __iomem *clint_time_val;
EXPORT_SYMBOL(clint_time_val);
#endif
-static void clint_send_ipi(const struct cpumask *target)
-{
- unsigned int cpu;
-
- for_each_cpu(cpu, target)
- writel(1, clint_ipi_base + cpuid_to_hartid_map(cpu));
-}
-
-static void clint_clear_ipi(void)
-{
- writel(0, clint_ipi_base + cpuid_to_hartid_map(smp_processor_id()));
-}
-
-static struct riscv_ipi_ops clint_ipi_ops = {
- .ipi_inject = clint_send_ipi,
- .ipi_clear = clint_clear_ipi,
-};
-
#ifdef CONFIG_64BIT
#define clint_get_cycles() readq_relaxed(clint_timer_val)
#else
@@ -189,7 +170,6 @@ static int __init clint_timer_init_dt(struct device_node *np)
return -ENODEV;
}
- clint_ipi_base = base + CLINT_IPI_OFF;
clint_timer_cmp = base + CLINT_TIMER_CMP_OFF;
clint_timer_val = base + CLINT_TIMER_VAL_OFF;
clint_timer_freq = riscv_timebase;
@@ -228,9 +208,6 @@ static int __init clint_timer_init_dt(struct device_node *np)
goto fail_free_irq;
}
- riscv_set_ipi_ops(&clint_ipi_ops);
- clint_clear_ipi();
-
return 0;
fail_free_irq:
diff --git a/drivers/irqchip/irq-riscv-intc.c b/drivers/irqchip/irq-riscv-intc.c
index 8017f6d32d52..65d9c5b0ddb8 100644
--- a/drivers/irqchip/irq-riscv-intc.c
+++ b/drivers/irqchip/irq-riscv-intc.c
@@ -26,20 +26,7 @@ static asmlinkage void riscv_intc_irq(struct pt_regs *regs)
if (unlikely(cause >= BITS_PER_LONG))
panic("unexpected interrupt cause");
- switch (cause) {
-#ifdef CONFIG_SMP
- case RV_IRQ_SOFT:
- /*
- * We only use software interrupts to pass IPIs, so if a
- * non-SMP system gets one, then we don't know what to do.
- */
- handle_IPI(regs);
- break;
-#endif
- default:
- handle_domain_irq(intc_domain, cause, regs);
- break;
- }
+ handle_domain_irq(intc_domain, cause, regs);
}
/*
@@ -59,18 +46,6 @@ static void riscv_intc_irq_unmask(struct irq_data *d)
csr_set(CSR_IE, BIT(d->hwirq));
}
-static int riscv_intc_cpu_starting(unsigned int cpu)
-{
- csr_set(CSR_IE, BIT(RV_IRQ_SOFT));
- return 0;
-}
-
-static int riscv_intc_cpu_dying(unsigned int cpu)
-{
- csr_clear(CSR_IE, BIT(RV_IRQ_SOFT));
- return 0;
-}
-
static struct irq_chip riscv_intc_chip = {
.name = "RISC-V INTC",
.irq_mask = riscv_intc_irq_mask,
@@ -87,9 +62,32 @@ static int riscv_intc_domain_map(struct irq_domain *d, unsigned int irq,
return 0;
}
+static int riscv_intc_domain_alloc(struct irq_domain *domain,
+ unsigned int virq, unsigned int nr_irqs,
+ void *arg)
+{
+ int i, ret;
+ irq_hw_number_t hwirq;
+ unsigned int type = IRQ_TYPE_NONE;
+ struct irq_fwspec *fwspec = arg;
+
+ ret = irq_domain_translate_onecell(domain, fwspec, &hwirq, &type);
+ if (ret)
+ return ret;
+
+ for (i = 0; i < nr_irqs; i++) {
+ ret = riscv_intc_domain_map(domain, virq + i, hwirq + i);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+
static const struct irq_domain_ops riscv_intc_domain_ops = {
.map = riscv_intc_domain_map,
.xlate = irq_domain_xlate_onecell,
+ .alloc = riscv_intc_domain_alloc
};
static int __init riscv_intc_init(struct device_node *node,
@@ -125,11 +123,6 @@ static int __init riscv_intc_init(struct device_node *node,
return rc;
}
- cpuhp_setup_state(CPUHP_AP_IRQ_RISCV_STARTING,
- "irqchip/riscv/intc:starting",
- riscv_intc_cpu_starting,
- riscv_intc_cpu_dying);
-
pr_info("%d local interrupts mapped\n", BITS_PER_LONG);
return 0;
--
2.25.1
To do remote FENCEs (i.e. remote TLB flushes) using IPI calls on
the RISC-V kernel, we need hardware mechanism to directly inject
IPI from the RISC-V kernel instead of using SBI calls.
The upcoming ACLINT [M|S]SWI devices and AIA IMSIC devices allow
direct IPI injection from the RISC-V kernel. To support this, we
extend the riscv_ipi_set_virq_range() function so that irqchip
drivers can mark IPIs as suitable for remote FENCEs.
Signed-off-by: Anup Patel <[email protected]>
---
arch/riscv/include/asm/smp.h | 18 ++++++++++++++++--
arch/riscv/kernel/sbi-ipi.c | 2 +-
arch/riscv/kernel/smp.c | 11 ++++++++++-
3 files changed, 27 insertions(+), 4 deletions(-)
diff --git a/arch/riscv/include/asm/smp.h b/arch/riscv/include/asm/smp.h
index e5480d75b2d9..42569f1130f0 100644
--- a/arch/riscv/include/asm/smp.h
+++ b/arch/riscv/include/asm/smp.h
@@ -16,6 +16,9 @@ struct seq_file;
extern unsigned long boot_cpu_hartid;
#ifdef CONFIG_SMP
+
+#include <linux/jump_label.h>
+
/*
* Mapping between linux logical cpu index and hartid.
*/
@@ -47,7 +50,12 @@ void riscv_ipi_disable(void);
bool riscv_ipi_have_virq_range(void);
/* Set the IPI interrupt numbers for arch (called by irqchip drivers) */
-void riscv_ipi_set_virq_range(int virq, int nr_irqs);
+void riscv_ipi_set_virq_range(int virq, int nr_irqs, bool use_for_rfence);
+
+/* Check if we can use IPIs for remote FENCEs */
+DECLARE_STATIC_KEY_FALSE(riscv_ipi_for_rfence);
+#define riscv_use_ipi_for_rfence() \
+ static_branch_unlikely(&riscv_ipi_for_rfence)
/* Secondary hart entry */
asmlinkage void smp_callin(void);
@@ -103,10 +111,16 @@ static inline bool riscv_ipi_have_virq_range(void)
return false;
}
-static inline void riscv_ipi_set_virq_range(int virq, int nr)
+static inline void riscv_ipi_set_virq_range(int virq, int nr,
+ bool use_for_rfence)
{
}
+static inline bool riscv_use_ipi_for_rfence(void)
+{
+ return false;
+}
+
#endif /* CONFIG_SMP */
#if defined(CONFIG_HOTPLUG_CPU) && (CONFIG_SMP)
diff --git a/arch/riscv/kernel/sbi-ipi.c b/arch/riscv/kernel/sbi-ipi.c
index 8d06b1e3d682..2728be0439a2 100644
--- a/arch/riscv/kernel/sbi-ipi.c
+++ b/arch/riscv/kernel/sbi-ipi.c
@@ -149,7 +149,7 @@ static int __init sbi_ipi_set_virq(void)
return -ENOMEM;
}
- riscv_ipi_set_virq_range(virq, BITS_PER_LONG);
+ riscv_ipi_set_virq_range(virq, BITS_PER_LONG, false);
return 0;
}
diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c
index 53902cd1671f..b9ea5f88ca0e 100644
--- a/arch/riscv/kernel/smp.c
+++ b/arch/riscv/kernel/smp.c
@@ -159,7 +159,10 @@ bool riscv_ipi_have_virq_range(void)
return (ipi_virq_base) ? true : false;
}
-void riscv_ipi_set_virq_range(int virq, int nr)
+DEFINE_STATIC_KEY_FALSE(riscv_ipi_for_rfence);
+EXPORT_SYMBOL_GPL(riscv_ipi_for_rfence);
+
+void riscv_ipi_set_virq_range(int virq, int nr, bool use_for_rfence)
{
int i, err;
@@ -182,6 +185,12 @@ void riscv_ipi_set_virq_range(int virq, int nr)
/* Enabled IPIs for boot CPU immediately */
riscv_ipi_enable();
+
+ /* Update RFENCE static key */
+ if (use_for_rfence)
+ static_branch_enable(&riscv_ipi_for_rfence);
+ else
+ static_branch_disable(&riscv_ipi_for_rfence);
}
EXPORT_SYMBOL_GPL(riscv_ipi_set_virq_range);
--
2.25.1