This series primarily does two things:
1. Allows RISC-V IPI provider to specificy whether IPI operations are
suitable for remote TLB flush (PATCH1)
2. Improve remote TLB flush to use IPIs whenever possible (PATCH2)
3. Allow irqchip drivers to handle IPIs from chained IRQ handlers (PATCH3)
This series also a preparatory series for upcoming RISC-V advanced
interrupt architecture (AIA) support.
These patches can be found in riscv_ipi_imp_v1 branch at
https://github.com/avpatel/linux
Anup Patel (3):
RISC-V: IPI provider should specify if we can use IPI for remote FENCE
RISC-V: Use IPIs for remote TLB flush when possible
RISC-V: Add handle_IPI_noregs() for irqchip drivers
arch/riscv/include/asm/smp.h | 19 +++++++++-
arch/riscv/kernel/sbi.c | 2 +-
arch/riscv/kernel/smp.c | 30 +++++++++++----
arch/riscv/mm/cacheflush.c | 2 +-
arch/riscv/mm/tlbflush.c | 62 ++++++++++++++++++++++++-------
drivers/clocksource/timer-clint.c | 2 +-
6 files changed, 91 insertions(+), 26 deletions(-)
--
2.25.1
We extend riscv_set_ipi_ops() so that IPI providers (such as SBI, CLINT
driver, etc) can specify whether IPIs are suitable for doing remote
FENCEs (i.e remote TLB shoot down).
The upcoming AIA specification allows IPI injection directly from S-mode
(or VS-mode) using IMSIC controller so the extended riscv_set_ipi_ops()
will be useful to AIA IMSIC driver as well.
Signed-off-by: Anup Patel <[email protected]>
---
arch/riscv/include/asm/smp.h | 13 +++++++++++--
arch/riscv/kernel/sbi.c | 2 +-
arch/riscv/kernel/smp.c | 10 +++++++++-
arch/riscv/mm/cacheflush.c | 2 +-
drivers/clocksource/timer-clint.c | 2 +-
5 files changed, 23 insertions(+), 6 deletions(-)
diff --git a/arch/riscv/include/asm/smp.h b/arch/riscv/include/asm/smp.h
index df1f7c4cd433..82c23e5f22f6 100644
--- a/arch/riscv/include/asm/smp.h
+++ b/arch/riscv/include/asm/smp.h
@@ -45,8 +45,11 @@ void arch_send_call_function_single_ipi(int cpu);
int riscv_hartid_to_cpuid(int hartid);
void riscv_cpuid_to_hartid_mask(const struct cpumask *in, struct cpumask *out);
+/* Check if we can use IPIs for remote FENCE */
+bool riscv_use_ipi_for_rfence(void);
+
/* Set custom IPI operations */
-void riscv_set_ipi_ops(struct riscv_ipi_ops *ops);
+void riscv_set_ipi_ops(struct riscv_ipi_ops *ops, bool use_for_rfence);
/* Clear IPI for current CPU */
void riscv_clear_ipi(void);
@@ -92,7 +95,13 @@ static inline void riscv_cpuid_to_hartid_mask(const struct cpumask *in,
cpumask_set_cpu(boot_cpu_hartid, out);
}
-static inline void riscv_set_ipi_ops(struct riscv_ipi_ops *ops)
+static inline bool riscv_use_ipi_for_rfence(void)
+{
+ return false;
+}
+
+static inline void riscv_set_ipi_ops(struct riscv_ipi_ops *ops,
+ bool use_for_rfence)
{
}
diff --git a/arch/riscv/kernel/sbi.c b/arch/riscv/kernel/sbi.c
index 49155588e56c..15a09680fdb6 100644
--- a/arch/riscv/kernel/sbi.c
+++ b/arch/riscv/kernel/sbi.c
@@ -634,5 +634,5 @@ void __init sbi_init(void)
__sbi_rfence = __sbi_rfence_v01;
}
- riscv_set_ipi_ops(&sbi_ipi_ops);
+ riscv_set_ipi_ops(&sbi_ipi_ops, false);
}
diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c
index ea028d9e0d24..9258e3eaa8c6 100644
--- a/arch/riscv/kernel/smp.c
+++ b/arch/riscv/kernel/smp.c
@@ -85,11 +85,19 @@ static void ipi_stop(void)
wait_for_interrupt();
}
+static bool ipi_for_rfence;
static struct riscv_ipi_ops *ipi_ops;
-void riscv_set_ipi_ops(struct riscv_ipi_ops *ops)
+bool riscv_use_ipi_for_rfence(void)
+{
+ return ipi_for_rfence;
+}
+EXPORT_SYMBOL_GPL(riscv_use_ipi_for_rfence);
+
+void riscv_set_ipi_ops(struct riscv_ipi_ops *ops, bool use_for_rfence)
{
ipi_ops = ops;
+ ipi_for_rfence = use_for_rfence;
}
EXPORT_SYMBOL_GPL(riscv_set_ipi_ops);
diff --git a/arch/riscv/mm/cacheflush.c b/arch/riscv/mm/cacheflush.c
index 094118663285..0ffe7d560dc8 100644
--- a/arch/riscv/mm/cacheflush.c
+++ b/arch/riscv/mm/cacheflush.c
@@ -16,7 +16,7 @@ static void ipi_remote_fence_i(void *info)
void flush_icache_all(void)
{
- if (IS_ENABLED(CONFIG_RISCV_SBI))
+ if (!riscv_use_ipi_for_rfence())
sbi_remote_fence_i(NULL);
else
on_each_cpu(ipi_remote_fence_i, NULL, 1);
diff --git a/drivers/clocksource/timer-clint.c b/drivers/clocksource/timer-clint.c
index 6cfe2ab73eb0..fe018a2c008f 100644
--- a/drivers/clocksource/timer-clint.c
+++ b/drivers/clocksource/timer-clint.c
@@ -228,7 +228,7 @@ static int __init clint_timer_init_dt(struct device_node *np)
goto fail_free_irq;
}
- riscv_set_ipi_ops(&clint_ipi_ops);
+ riscv_set_ipi_ops(&clint_ipi_ops, true);
clint_clear_ipi();
return 0;
--
2.25.1
If IPI calls are injected using SBI IPI calls then remote TLB flush
using SBI RFENCE calls is much faster because using IPIs for remote
TLB flush would still endup as SBI IPI calls with extra processing
on kernel side.
It is now possible to have specialized hardware (such as RISC-V AIA)
which allows S-mode software to directly inject IPIs without any
assistance from M-mode runtime firmware.
This patch extends remote TLB flush functions to use IPIs whenever
underlying IPI operations are suitable for remote FENCEs.
Signed-off-by: Anup Patel <[email protected]>
---
arch/riscv/mm/tlbflush.c | 62 +++++++++++++++++++++++++++++++---------
1 file changed, 48 insertions(+), 14 deletions(-)
diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c
index 720b443c4528..009c56fa102d 100644
--- a/arch/riscv/mm/tlbflush.c
+++ b/arch/riscv/mm/tlbflush.c
@@ -1,39 +1,73 @@
// SPDX-License-Identifier: GPL-2.0
+/*
+ * TLB flush implementation.
+ *
+ * Copyright (c) 2021 Western Digital Corporation or its affiliates.
+ */
#include <linux/mm.h>
#include <linux/smp.h>
#include <linux/sched.h>
#include <asm/sbi.h>
+static void ipi_flush_tlb_all(void *info)
+{
+ local_flush_tlb_all();
+}
+
void flush_tlb_all(void)
{
- sbi_remote_sfence_vma(NULL, 0, -1);
+ if (!riscv_use_ipi_for_rfence())
+ sbi_remote_sfence_vma(NULL, 0, -1);
+ else
+ on_each_cpu(ipi_flush_tlb_all, NULL, 1);
+}
+
+struct flush_range_data {
+ unsigned long start;
+ unsigned long size;
+};
+
+static void ipi_flush_range(void *info)
+{
+ struct flush_range_data *data = info;
+
+ /* local cpu is the only cpu present in cpumask */
+ if (data->size <= PAGE_SIZE)
+ local_flush_tlb_page(data->start);
+ else
+ local_flush_tlb_all();
}
/*
- * This function must not be called with cmask being null.
+ * This function must not be called with NULL cpumask.
* Kernel may panic if cmask is NULL.
*/
-static void __sbi_tlb_flush_range(struct cpumask *cmask, unsigned long start,
- unsigned long size)
+static void flush_range(struct cpumask *cmask, unsigned long start,
+ unsigned long size)
{
+ struct flush_range_data info;
struct cpumask hmask;
unsigned int cpuid;
if (cpumask_empty(cmask))
return;
+ info.start = start;
+ info.size = size;
+
cpuid = get_cpu();
if (cpumask_any_but(cmask, cpuid) >= nr_cpu_ids) {
- /* local cpu is the only cpu present in cpumask */
- if (size <= PAGE_SIZE)
- local_flush_tlb_page(start);
- else
- local_flush_tlb_all();
+ ipi_flush_range(&info);
} else {
- riscv_cpuid_to_hartid_mask(cmask, &hmask);
- sbi_remote_sfence_vma(cpumask_bits(&hmask), start, size);
+ if (!riscv_use_ipi_for_rfence()) {
+ riscv_cpuid_to_hartid_mask(cmask, &hmask);
+ sbi_remote_sfence_vma(cpumask_bits(&hmask),
+ start, size);
+ } else {
+ on_each_cpu_mask(cmask, ipi_flush_range, &info, 1);
+ }
}
put_cpu();
@@ -41,16 +75,16 @@ static void __sbi_tlb_flush_range(struct cpumask *cmask, unsigned long start,
void flush_tlb_mm(struct mm_struct *mm)
{
- __sbi_tlb_flush_range(mm_cpumask(mm), 0, -1);
+ flush_range(mm_cpumask(mm), 0, -1);
}
void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
{
- __sbi_tlb_flush_range(mm_cpumask(vma->vm_mm), addr, PAGE_SIZE);
+ flush_range(mm_cpumask(vma->vm_mm), addr, PAGE_SIZE);
}
void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
unsigned long end)
{
- __sbi_tlb_flush_range(mm_cpumask(vma->vm_mm), start, end - start);
+ flush_range(mm_cpumask(vma->vm_mm), start, end - start);
}
--
2.25.1
We will be having IPI handled through nested interrupt controllers
(such as AIA IMSIC). The irqchip driver of such nested interrupt
controller will not do irq_enter() and save pt_regs because this
would have been already done by the irqchip driver of the parent
interrupt controller.
This patch adds handle_IPI_noregs() for nested irqchip drivers.
Signed-off-by: Anup Patel <[email protected]>
---
arch/riscv/include/asm/smp.h | 6 ++++++
arch/riscv/kernel/smp.c | 20 ++++++++++++++------
2 files changed, 20 insertions(+), 6 deletions(-)
diff --git a/arch/riscv/include/asm/smp.h b/arch/riscv/include/asm/smp.h
index 82c23e5f22f6..b31d3ec2f71b 100644
--- a/arch/riscv/include/asm/smp.h
+++ b/arch/riscv/include/asm/smp.h
@@ -33,6 +33,12 @@ void show_ipi_stats(struct seq_file *p, int prec);
/* SMP initialization hook for setup_arch */
void __init setup_smp(void);
+/*
+ * Called from C code, this handles an IPI assuming irq_enter() and
+ * pt_regs already saved by caller.
+ */
+void handle_IPI_noregs(void);
+
/* Called from C code, this handles an IPI. */
void handle_IPI(struct pt_regs *regs);
diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c
index 9258e3eaa8c6..19e102e2d5e6 100644
--- a/arch/riscv/kernel/smp.c
+++ b/arch/riscv/kernel/smp.c
@@ -144,14 +144,11 @@ void arch_irq_work_raise(void)
}
#endif
-void handle_IPI(struct pt_regs *regs)
+void handle_IPI_noregs(void)
{
- struct pt_regs *old_regs = set_irq_regs(regs);
unsigned long *pending_ipis = &ipi_data[smp_processor_id()].bits;
unsigned long *stats = ipi_data[smp_processor_id()].stats;
- irq_enter();
-
riscv_clear_ipi();
while (true) {
@@ -162,7 +159,7 @@ void handle_IPI(struct pt_regs *regs)
ops = xchg(pending_ipis, 0);
if (ops == 0)
- goto done;
+ break;
if (ops & (1 << IPI_RESCHEDULE)) {
stats[IPI_RESCHEDULE]++;
@@ -189,9 +186,20 @@ void handle_IPI(struct pt_regs *regs)
/* Order data access and bit testing. */
mb();
}
+}
+
+void handle_IPI(struct pt_regs *regs)
+{
+ struct pt_regs *old_regs = set_irq_regs(regs);
+
+ irq_enter();
+
+ handle_IPI_noregs();
+
+ riscv_clear_ipi();
-done:
irq_exit();
+
set_irq_regs(old_regs);
}
--
2.25.1
On Thu, 11 Mar 2021 08:47:09 PST (-0800), Anup Patel wrote:
> This series primarily does two things:
> 1. Allows RISC-V IPI provider to specificy whether IPI operations are
> suitable for remote TLB flush (PATCH1)
> 2. Improve remote TLB flush to use IPIs whenever possible (PATCH2)
> 3. Allow irqchip drivers to handle IPIs from chained IRQ handlers (PATCH3)
IIUC this last one isn't technically used in both forms, as we don't have any
drivers that behave that way yet? I'm OK taking it, under the assumption it
makes keeping the out-of-tree driver for the draft interrupt controller easier,
but I was wrong then it's probably out of order so I figured I'd check.
Aside from that this generally LGTM. We are making quite a bit of mess in
here, but I don't really see a way around that as we need to support the old
hardware. We can always do a cleanup when the specifications settle down.
Oddly enough this did come up in IRC recently and there may be some new bits in
the CLINT on the FU740 that allow S-mode SW interrupts to show up directly --
there's at least a "delegate supervisor software interrupt" bit now, but the
manual only calls out machine mode as being able to set it (though IIUC it's
memory mapped, so not sure how that would be enforced). Not saying we need
that in order to take the last patch, but if it is possible it's probably worth
giving it a shot when the boards show up.
> This series also a preparatory series for upcoming RISC-V advanced
> interrupt architecture (AIA) support.
>
> These patches can be found in riscv_ipi_imp_v1 branch at
> https://github.com/avpatel/linux
>
> Anup Patel (3):
> RISC-V: IPI provider should specify if we can use IPI for remote FENCE
> RISC-V: Use IPIs for remote TLB flush when possible
> RISC-V: Add handle_IPI_noregs() for irqchip drivers
>
> arch/riscv/include/asm/smp.h | 19 +++++++++-
> arch/riscv/kernel/sbi.c | 2 +-
> arch/riscv/kernel/smp.c | 30 +++++++++++----
> arch/riscv/mm/cacheflush.c | 2 +-
> arch/riscv/mm/tlbflush.c | 62 ++++++++++++++++++++++++-------
> drivers/clocksource/timer-clint.c | 2 +-
> 6 files changed, 91 insertions(+), 26 deletions(-)
On Wed, Mar 17, 2021 at 10:23 AM Palmer Dabbelt
<[email protected]> wrote:
>
> On Thu, 11 Mar 2021 08:47:09 PST (-0800), Anup Patel wrote:
> > This series primarily does two things:
> > 1. Allows RISC-V IPI provider to specificy whether IPI operations are
> > suitable for remote TLB flush (PATCH1)
> > 2. Improve remote TLB flush to use IPIs whenever possible (PATCH2)
> > 3. Allow irqchip drivers to handle IPIs from chained IRQ handlers (PATCH3)
>
> IIUC this last one isn't technically used in both forms, as we don't have any
> drivers that behave that way yet? I'm OK taking it, under the assumption it
> makes keeping the out-of-tree driver for the draft interrupt controller easier,
> but I was wrong then it's probably out of order so I figured I'd check.
The last patch is for RISC-V AIA drivers I am working on.
The draft RISC-V AIA specification is available at:
http://www.jhauser.us/RISCV/riscv-interrupts-019.pdf
>
> Aside from that this generally LGTM. We are making quite a bit of mess in
> here, but I don't really see a way around that as we need to support the old
> hardware. We can always do a cleanup when the specifications settle down.
Not all RISC-V platforms will have a mechanism for direct IPI injection from
S-mode so to maintain backward compatibility for older platforms (where
IPI injection will be always through SBI calls) we have chosen the current
approach.
The RISC-V AIA spec is trying to solve this in a way which works for both
S-mode (or HS-mode) and VS-mode. The current RISC-V AIA plan is to
provide IPIs as software injected MSIs between HARTs and this will work
fine for Guest/VM as well.
>
> Oddly enough this did come up in IRC recently and there may be some new bits in
> the CLINT on the FU740 that allow S-mode SW interrupts to show up directly --
> there's at least a "delegate supervisor software interrupt" bit now, but the
> manual only calls out machine mode as being able to set it (though IIUC it's
> memory mapped, so not sure how that would be enforced). Not saying we need
> that in order to take the last patch, but if it is possible it's probably worth
> giving it a shot when the boards show up.
Adding a few bits in CLINT will not be a complete solution because we also need
a mechanism where a Guest/VM can directly inject IPIs without SBI calls to the
hypervisor.
>
> > This series also a preparatory series for upcoming RISC-V advanced
> > interrupt architecture (AIA) support.
> >
> > These patches can be found in riscv_ipi_imp_v1 branch at
> > https://github.com/avpatel/linux
> >
> > Anup Patel (3):
> > RISC-V: IPI provider should specify if we can use IPI for remote FENCE
> > RISC-V: Use IPIs for remote TLB flush when possible
> > RISC-V: Add handle_IPI_noregs() for irqchip drivers
> >
> > arch/riscv/include/asm/smp.h | 19 +++++++++-
> > arch/riscv/kernel/sbi.c | 2 +-
> > arch/riscv/kernel/smp.c | 30 +++++++++++----
> > arch/riscv/mm/cacheflush.c | 2 +-
> > arch/riscv/mm/tlbflush.c | 62 ++++++++++++++++++++++++-------
> > drivers/clocksource/timer-clint.c | 2 +-
> > 6 files changed, 91 insertions(+), 26 deletions(-)
Regards,
Anup