This series is to add PREEMPT_RT support to riscv:
patch1 adds the missing number of signal exits in vCPU stat
patch2 switches to the generic guest entry infrastructure
patch3 select HAVE_POSIX_CPU_TIMERS_TASK_WORK which is a requirement for
RT
patch4 adds lazy preempt support
patch5 allows to enable PREEMPT_RT
I assume patch1, patch2 and patch3 can be reviewed and merged for
riscv-next, patch4 and patch5 can be reviewed and maintained in rt tree,
and finally merged once the remaining patches in rt tree are all
mainlined.
Since v1:
- send to related maillist, I press ENTER too quickly when sending v1
- remove the signal_pending() handling because that's covered by
generic guest entry infrastructure
Jisheng Zhang (5):
RISC-V: KVM: Record number of signal exits as a vCPU stat
RISC-V: KVM: Use generic guest entry infrastructure
riscv: select HAVE_POSIX_CPU_TIMERS_TASK_WORK
riscv: add lazy preempt support
riscv: Allow to enable RT
arch/riscv/Kconfig | 3 +++
arch/riscv/include/asm/kvm_host.h | 1 +
arch/riscv/include/asm/thread_info.h | 7 +++++--
arch/riscv/kernel/asm-offsets.c | 1 +
arch/riscv/kernel/entry.S | 9 +++++++--
arch/riscv/kvm/Kconfig | 1 +
arch/riscv/kvm/vcpu.c | 18 +++++++-----------
7 files changed, 25 insertions(+), 15 deletions(-)
--
2.34.1
Implement the lazy preempt for riscv.
Signed-off-by: Jisheng Zhang <[email protected]>
---
arch/riscv/Kconfig | 1 +
arch/riscv/include/asm/thread_info.h | 7 +++++--
arch/riscv/kernel/asm-offsets.c | 1 +
arch/riscv/kernel/entry.S | 9 +++++++--
4 files changed, 14 insertions(+), 4 deletions(-)
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 7a8134fd7ec9..9f2f1936b1b5 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -105,6 +105,7 @@ config RISCV
select HAVE_PERF_REGS
select HAVE_PERF_USER_STACK_DUMP
select HAVE_POSIX_CPU_TIMERS_TASK_WORK
+ select HAVE_PREEMPT_LAZY
select HAVE_REGS_AND_STACK_ACCESS_API
select HAVE_FUNCTION_ARG_ACCESS_API
select HAVE_STACKPROTECTOR
diff --git a/arch/riscv/include/asm/thread_info.h b/arch/riscv/include/asm/thread_info.h
index 78933ac04995..471915b179a2 100644
--- a/arch/riscv/include/asm/thread_info.h
+++ b/arch/riscv/include/asm/thread_info.h
@@ -56,6 +56,7 @@
struct thread_info {
unsigned long flags; /* low level flags */
int preempt_count; /* 0=>preemptible, <0=>BUG */
+ int preempt_lazy_count; /* 0=>preemptible, <0=>BUG */
/*
* These stack pointers are overwritten on every system call or
* exception. SP is also saved to the stack it can be recovered when
@@ -90,7 +91,7 @@ struct thread_info {
#define TIF_NOTIFY_RESUME 1 /* callback before returning to user */
#define TIF_SIGPENDING 2 /* signal pending */
#define TIF_NEED_RESCHED 3 /* rescheduling necessary */
-#define TIF_RESTORE_SIGMASK 4 /* restore signal mask in do_signal() */
+#define TIF_NEED_RESCHED_LAZY 4 /* lazy rescheduling */
#define TIF_MEMDIE 5 /* is terminating due to OOM killer */
#define TIF_SYSCALL_TRACEPOINT 6 /* syscall tracepoint instrumentation */
#define TIF_SYSCALL_AUDIT 7 /* syscall auditing */
@@ -98,6 +99,7 @@ struct thread_info {
#define TIF_NOTIFY_SIGNAL 9 /* signal notifications exist */
#define TIF_UPROBE 10 /* uprobe breakpoint or singlestep */
#define TIF_32BIT 11 /* compat-mode 32bit process */
+#define TIF_RESTORE_SIGMASK 12 /* restore signal mask in do_signal() */
#define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE)
#define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME)
@@ -108,10 +110,11 @@ struct thread_info {
#define _TIF_SECCOMP (1 << TIF_SECCOMP)
#define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL)
#define _TIF_UPROBE (1 << TIF_UPROBE)
+#define _TIF_NEED_RESCHED_LAZY (1 << TIF_NEED_RESCHED_LAZY)
#define _TIF_WORK_MASK \
(_TIF_NOTIFY_RESUME | _TIF_SIGPENDING | _TIF_NEED_RESCHED | \
- _TIF_NOTIFY_SIGNAL | _TIF_UPROBE)
+ _TIF_NEED_RESCHED_LAZY | _TIF_NOTIFY_SIGNAL | _TIF_UPROBE)
#define _TIF_SYSCALL_WORK \
(_TIF_SYSCALL_TRACE | _TIF_SYSCALL_TRACEPOINT | _TIF_SYSCALL_AUDIT | \
diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c
index df9444397908..e38e33822f72 100644
--- a/arch/riscv/kernel/asm-offsets.c
+++ b/arch/riscv/kernel/asm-offsets.c
@@ -35,6 +35,7 @@ void asm_offsets(void)
OFFSET(TASK_THREAD_S11, task_struct, thread.s[11]);
OFFSET(TASK_TI_FLAGS, task_struct, thread_info.flags);
OFFSET(TASK_TI_PREEMPT_COUNT, task_struct, thread_info.preempt_count);
+ OFFSET(TASK_TI_PREEMPT_LAZY_COUNT, task_struct, thread_info.preempt_lazy_count);
OFFSET(TASK_TI_KERNEL_SP, task_struct, thread_info.kernel_sp);
OFFSET(TASK_TI_USER_SP, task_struct, thread_info.user_sp);
diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S
index b9eda3fcbd6d..595100a4c2c7 100644
--- a/arch/riscv/kernel/entry.S
+++ b/arch/riscv/kernel/entry.S
@@ -361,9 +361,14 @@ restore_all:
resume_kernel:
REG_L s0, TASK_TI_PREEMPT_COUNT(tp)
bnez s0, restore_all
- REG_L s0, TASK_TI_FLAGS(tp)
- andi s0, s0, _TIF_NEED_RESCHED
+ REG_L s1, TASK_TI_FLAGS(tp)
+ andi s0, s1, _TIF_NEED_RESCHED
+ bnez s0, 1f
+ REG_L s0, TASK_TI_PREEMPT_LAZY_COUNT(tp)
+ bnez s0, restore_all
+ andi s0, s1, _TIF_NEED_RESCHED_LAZY
beqz s0, restore_all
+1:
call preempt_schedule_irq
j restore_all
#endif
--
2.34.1
Move POSIX CPU timer expiry and signal delivery into task context to
allow PREEMPT_RT setups to coexist with KVM.
Signed-off-by: Jisheng Zhang <[email protected]>
---
arch/riscv/Kconfig | 1 +
1 file changed, 1 insertion(+)
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 79e52441e18b..7a8134fd7ec9 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -104,6 +104,7 @@ config RISCV
select HAVE_PERF_EVENTS
select HAVE_PERF_REGS
select HAVE_PERF_USER_STACK_DUMP
+ select HAVE_POSIX_CPU_TIMERS_TASK_WORK
select HAVE_REGS_AND_STACK_ACCESS_API
select HAVE_FUNCTION_ARG_ACCESS_API
select HAVE_STACKPROTECTOR
--
2.34.1
Record a statistic indicating the number of times a vCPU has exited
due to a pending signal.
Signed-off-by: Jisheng Zhang <[email protected]>
---
arch/riscv/include/asm/kvm_host.h | 1 +
arch/riscv/kvm/vcpu.c | 2 ++
2 files changed, 3 insertions(+)
diff --git a/arch/riscv/include/asm/kvm_host.h b/arch/riscv/include/asm/kvm_host.h
index 60c517e4d576..dbbf43d52623 100644
--- a/arch/riscv/include/asm/kvm_host.h
+++ b/arch/riscv/include/asm/kvm_host.h
@@ -67,6 +67,7 @@ struct kvm_vcpu_stat {
u64 mmio_exit_kernel;
u64 csr_exit_user;
u64 csr_exit_kernel;
+ u64 signal_exits;
u64 exits;
};
diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c
index d0f08d5b4282..3da459fedc28 100644
--- a/arch/riscv/kvm/vcpu.c
+++ b/arch/riscv/kvm/vcpu.c
@@ -28,6 +28,7 @@ const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = {
STATS_DESC_COUNTER(VCPU, mmio_exit_kernel),
STATS_DESC_COUNTER(VCPU, csr_exit_user),
STATS_DESC_COUNTER(VCPU, csr_exit_kernel),
+ STATS_DESC_COUNTER(VCPU, signal_exits),
STATS_DESC_COUNTER(VCPU, exits)
};
@@ -973,6 +974,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
if (signal_pending(current)) {
ret = -EINTR;
run->exit_reason = KVM_EXIT_INTR;
+ ++vcpu->stat.signal_exits;
}
/*
--
2.34.1
Use generic guest entry infrastructure to properly handle
TIF_NOTIFY_RESUME.
Signed-off-by: Jisheng Zhang <[email protected]>
---
arch/riscv/kvm/Kconfig | 1 +
arch/riscv/kvm/vcpu.c | 18 ++++++------------
2 files changed, 7 insertions(+), 12 deletions(-)
diff --git a/arch/riscv/kvm/Kconfig b/arch/riscv/kvm/Kconfig
index f5a342fa1b1d..f36a737d5f96 100644
--- a/arch/riscv/kvm/Kconfig
+++ b/arch/riscv/kvm/Kconfig
@@ -24,6 +24,7 @@ config KVM
select PREEMPT_NOTIFIERS
select KVM_MMIO
select KVM_GENERIC_DIRTYLOG_READ_PROTECT
+ select KVM_XFER_TO_GUEST_WORK
select HAVE_KVM_VCPU_ASYNC_IOCTL
select HAVE_KVM_EVENTFD
select SRCU
diff --git a/arch/riscv/kvm/vcpu.c b/arch/riscv/kvm/vcpu.c
index 3da459fedc28..e3e6b8608288 100644
--- a/arch/riscv/kvm/vcpu.c
+++ b/arch/riscv/kvm/vcpu.c
@@ -7,6 +7,7 @@
*/
#include <linux/bitops.h>
+#include <linux/entry-kvm.h>
#include <linux/errno.h>
#include <linux/err.h>
#include <linux/kdebug.h>
@@ -959,7 +960,9 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
run->exit_reason = KVM_EXIT_UNKNOWN;
while (ret > 0) {
/* Check conditions before entering the guest */
- cond_resched();
+ ret = xfer_to_guest_mode_handle_work(vcpu);
+ if (!ret)
+ ret = 1;
kvm_riscv_gstage_vmid_update(vcpu);
@@ -967,16 +970,6 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
local_irq_disable();
- /*
- * Exit if we have a signal pending so that we can deliver
- * the signal to user space.
- */
- if (signal_pending(current)) {
- ret = -EINTR;
- run->exit_reason = KVM_EXIT_INTR;
- ++vcpu->stat.signal_exits;
- }
-
/*
* Ensure we set mode to IN_GUEST_MODE after we disable
* interrupts and before the final VCPU requests check.
@@ -999,7 +992,8 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
if (ret <= 0 ||
kvm_riscv_gstage_vmid_ver_changed(&vcpu->kvm->arch.vmid) ||
- kvm_request_pending(vcpu)) {
+ kvm_request_pending(vcpu) ||
+ xfer_to_guest_mode_work_pending()) {
vcpu->mode = OUTSIDE_GUEST_MODE;
local_irq_enable();
kvm_vcpu_srcu_read_lock(vcpu);
--
2.34.1
Allow to select RT.
Signed-off-by: Jisheng Zhang <[email protected]>
---
arch/riscv/Kconfig | 1 +
1 file changed, 1 insertion(+)
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index 9f2f1936b1b5..69cdcb3cf251 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -39,6 +39,7 @@ config RISCV
select ARCH_SUPPORTS_DEBUG_PAGEALLOC if MMU
select ARCH_SUPPORTS_HUGETLBFS if MMU
select ARCH_SUPPORTS_PAGE_TABLE_CHECK if MMU
+ select ARCH_SUPPORTS_RT
select ARCH_USE_MEMTEST
select ARCH_USE_QUEUED_RWLOCKS
select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT if MMU
--
2.34.1
On 2022-09-01 01:59:15 [+0800], Jisheng Zhang wrote:
> I assume patch1, patch2 and patch3 can be reviewed and merged for
> riscv-next, patch4 and patch5 can be reviewed and maintained in rt tree,
> and finally merged once the remaining patches in rt tree are all
> mainlined.
I would say so, yes.
What about JUMP_LABEL support? Do you halt all CPUs while patching the
code?
Sebastian
On Thu, Sep 01, 2022 at 09:04:05AM +0200, Sebastian Andrzej Siewior wrote:
> On 2022-09-01 01:59:15 [+0800], Jisheng Zhang wrote:
> > I assume patch1, patch2 and patch3 can be reviewed and merged for
> > riscv-next, patch4 and patch5 can be reviewed and maintained in rt tree,
> > and finally merged once the remaining patches in rt tree are all
> > mainlined.
>
> I would say so, yes.
>
> What about JUMP_LABEL support? Do you halt all CPUs while patching the
> code?
>
FWICT, riscv JUMP_LABEL implementation doesn't rely on stop all cpus while
patching text.
Thanks
On 31/08/2022 18:59, Jisheng Zhang wrote:
> EXTERNAL EMAIL: Do not click links or open attachments unless you know the content is safe
>
> This series is to add PREEMPT_RT support to riscv:
> patch1 adds the missing number of signal exits in vCPU stat
> patch2 switches to the generic guest entry infrastructure
> patch3 select HAVE_POSIX_CPU_TIMERS_TASK_WORK which is a requirement for
> RT
> patch4 adds lazy preempt support
> patch5 allows to enable PREEMPT_RT
>
What version of the preempt_rt patch did you test this with?
Maybe I am missing something, but I gave this a whirl with
v6.0-rc3 + v6.0-rc3-rt5 & was meant by a bunch of complaints.
I am not familiar with the preempt_rt patch, so I am not sure what
level of BUG()s or WARNING()s are to be expected, but I saw a fair
few...
Thanks,
Conor.
> I assume patch1, patch2 and patch3 can be reviewed and merged for
> riscv-next, patch4 and patch5 can be reviewed and maintained in rt tree,
> and finally merged once the remaining patches in rt tree are all
> mainlined.
>
> Since v1:
> - send to related maillist, I press ENTER too quickly when sending v1
> - remove the signal_pending() handling because that's covered by
> generic guest entry infrastructure
>
> Jisheng Zhang (5):
> RISC-V: KVM: Record number of signal exits as a vCPU stat
> RISC-V: KVM: Use generic guest entry infrastructure
> riscv: select HAVE_POSIX_CPU_TIMERS_TASK_WORK
> riscv: add lazy preempt support
> riscv: Allow to enable RT
>
> arch/riscv/Kconfig | 3 +++
> arch/riscv/include/asm/kvm_host.h | 1 +
> arch/riscv/include/asm/thread_info.h | 7 +++++--
> arch/riscv/kernel/asm-offsets.c | 1 +
> arch/riscv/kernel/entry.S | 9 +++++++--
> arch/riscv/kvm/Kconfig | 1 +
> arch/riscv/kvm/vcpu.c | 18 +++++++-----------
> 7 files changed, 25 insertions(+), 15 deletions(-)
>
> --
> 2.34.1
>
>
> _______________________________________________
> linux-riscv mailing list
> [email protected]
> http://lists.infradead.org/mailman/listinfo/linux-riscv
On Thu, Sep 01, 2022 at 04:41:52PM +0000, [email protected] wrote:
> On 31/08/2022 18:59, Jisheng Zhang wrote:
> > EXTERNAL EMAIL: Do not click links or open attachments unless you know the content is safe
> >
> > This series is to add PREEMPT_RT support to riscv:
> > patch1 adds the missing number of signal exits in vCPU stat
> > patch2 switches to the generic guest entry infrastructure
> > patch3 select HAVE_POSIX_CPU_TIMERS_TASK_WORK which is a requirement for
> > RT
> > patch4 adds lazy preempt support
> > patch5 allows to enable PREEMPT_RT
> >
>
> What version of the preempt_rt patch did you test this with?
v6.0-rc1 + v6.0-rc1-rt patch
>
> Maybe I am missing something, but I gave this a whirl with
> v6.0-rc3 + v6.0-rc3-rt5 & was meant by a bunch of complaints.
> I am not familiar with the preempt_rt patch, so I am not sure what
> level of BUG()s or WARNING()s are to be expected, but I saw a fair
> few...
Could you please provide corresponding log? Usually, this means there's
a bug in related drivers, so it's better to fix them now rather than
wait for RT patches mainlined.
PS: which HW are you using?
Thanks
On 02/09/2022 14:09, Jisheng Zhang wrote:
> EXTERNAL EMAIL: Do not click links or open attachments unless you know the content is safe
>
> On Thu, Sep 01, 2022 at 04:41:52PM +0000, [email protected] wrote:
>> On 31/08/2022 18:59, Jisheng Zhang wrote:
>>> EXTERNAL EMAIL: Do not click links or open attachments unless you know the content is safe
>>>
>>> This series is to add PREEMPT_RT support to riscv:
>>> patch1 adds the missing number of signal exits in vCPU stat
>>> patch2 switches to the generic guest entry infrastructure
>>> patch3 select HAVE_POSIX_CPU_TIMERS_TASK_WORK which is a requirement for
>>> RT
>>> patch4 adds lazy preempt support
>>> patch5 allows to enable PREEMPT_RT
>>>
>>
>> What version of the preempt_rt patch did you test this with?
>
> v6.0-rc1 + v6.0-rc1-rt patch
>
>>
>> Maybe I am missing something, but I gave this a whirl with
>> v6.0-rc3 + v6.0-rc3-rt5 & was meant by a bunch of complaints.
>> I am not familiar with the preempt_rt patch, so I am not sure what
>> level of BUG()s or WARNING()s are to be expected, but I saw a fair
>> few...
>
> Could you please provide corresponding log? Usually, this means there's
> a bug in related drivers, so it's better to fix them now rather than
> wait for RT patches mainlined.
I tried it on PolarFire SoC. I know that at least one of the problems
I found is down to drivers - specifically the system controller & hwrng.
The first issue that comes up is in early smp setup code - we call out
to update_siblings_masks() which does an alloc with preemption. It's
the same backtrace from here:
https://lore.kernel.org/all/[email protected]/
I'll give it a run through tonight or tomorrow & give you a full log
of what I saw. There's some splats all over the place for me, but I
can't tell if that's just knock-on from the other issues.
Thanks,
Conor.
On Thu, Sep 1, 2022 at 2:08 AM Jisheng Zhang <[email protected]> wrote:
>
> Implement the lazy preempt for riscv.
>
> Signed-off-by: Jisheng Zhang <[email protected]>
> ---
> arch/riscv/Kconfig | 1 +
> arch/riscv/include/asm/thread_info.h | 7 +++++--
> arch/riscv/kernel/asm-offsets.c | 1 +
> arch/riscv/kernel/entry.S | 9 +++++++--
> 4 files changed, 14 insertions(+), 4 deletions(-)
>
> diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
> index 7a8134fd7ec9..9f2f1936b1b5 100644
> --- a/arch/riscv/Kconfig
> +++ b/arch/riscv/Kconfig
> @@ -105,6 +105,7 @@ config RISCV
> select HAVE_PERF_REGS
> select HAVE_PERF_USER_STACK_DUMP
> select HAVE_POSIX_CPU_TIMERS_TASK_WORK
> + select HAVE_PREEMPT_LAZY
> select HAVE_REGS_AND_STACK_ACCESS_API
> select HAVE_FUNCTION_ARG_ACCESS_API
> select HAVE_STACKPROTECTOR
> diff --git a/arch/riscv/include/asm/thread_info.h b/arch/riscv/include/asm/thread_info.h
> index 78933ac04995..471915b179a2 100644
> --- a/arch/riscv/include/asm/thread_info.h
> +++ b/arch/riscv/include/asm/thread_info.h
> @@ -56,6 +56,7 @@
> struct thread_info {
> unsigned long flags; /* low level flags */
> int preempt_count; /* 0=>preemptible, <0=>BUG */
> + int preempt_lazy_count; /* 0=>preemptible, <0=>BUG */
> /*
> * These stack pointers are overwritten on every system call or
> * exception. SP is also saved to the stack it can be recovered when
> @@ -90,7 +91,7 @@ struct thread_info {
> #define TIF_NOTIFY_RESUME 1 /* callback before returning to user */
> #define TIF_SIGPENDING 2 /* signal pending */
> #define TIF_NEED_RESCHED 3 /* rescheduling necessary */
> -#define TIF_RESTORE_SIGMASK 4 /* restore signal mask in do_signal() */
> +#define TIF_NEED_RESCHED_LAZY 4 /* lazy rescheduling */
> #define TIF_MEMDIE 5 /* is terminating due to OOM killer */
> #define TIF_SYSCALL_TRACEPOINT 6 /* syscall tracepoint instrumentation */
> #define TIF_SYSCALL_AUDIT 7 /* syscall auditing */
> @@ -98,6 +99,7 @@ struct thread_info {
> #define TIF_NOTIFY_SIGNAL 9 /* signal notifications exist */
> #define TIF_UPROBE 10 /* uprobe breakpoint or singlestep */
> #define TIF_32BIT 11 /* compat-mode 32bit process */
> +#define TIF_RESTORE_SIGMASK 12 /* restore signal mask in do_signal() */
>
> #define _TIF_SYSCALL_TRACE (1 << TIF_SYSCALL_TRACE)
> #define _TIF_NOTIFY_RESUME (1 << TIF_NOTIFY_RESUME)
> @@ -108,10 +110,11 @@ struct thread_info {
> #define _TIF_SECCOMP (1 << TIF_SECCOMP)
> #define _TIF_NOTIFY_SIGNAL (1 << TIF_NOTIFY_SIGNAL)
> #define _TIF_UPROBE (1 << TIF_UPROBE)
> +#define _TIF_NEED_RESCHED_LAZY (1 << TIF_NEED_RESCHED_LAZY)
>
> #define _TIF_WORK_MASK \
> (_TIF_NOTIFY_RESUME | _TIF_SIGPENDING | _TIF_NEED_RESCHED | \
> - _TIF_NOTIFY_SIGNAL | _TIF_UPROBE)
> + _TIF_NEED_RESCHED_LAZY | _TIF_NOTIFY_SIGNAL | _TIF_UPROBE)
>
> #define _TIF_SYSCALL_WORK \
> (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_TRACEPOINT | _TIF_SYSCALL_AUDIT | \
> diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c
> index df9444397908..e38e33822f72 100644
> --- a/arch/riscv/kernel/asm-offsets.c
> +++ b/arch/riscv/kernel/asm-offsets.c
> @@ -35,6 +35,7 @@ void asm_offsets(void)
> OFFSET(TASK_THREAD_S11, task_struct, thread.s[11]);
> OFFSET(TASK_TI_FLAGS, task_struct, thread_info.flags);
> OFFSET(TASK_TI_PREEMPT_COUNT, task_struct, thread_info.preempt_count);
> + OFFSET(TASK_TI_PREEMPT_LAZY_COUNT, task_struct, thread_info.preempt_lazy_count);
> OFFSET(TASK_TI_KERNEL_SP, task_struct, thread_info.kernel_sp);
> OFFSET(TASK_TI_USER_SP, task_struct, thread_info.user_sp);
>
> diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S
> index b9eda3fcbd6d..595100a4c2c7 100644
> --- a/arch/riscv/kernel/entry.S
> +++ b/arch/riscv/kernel/entry.S
> @@ -361,9 +361,14 @@ restore_all:
> resume_kernel:
> REG_L s0, TASK_TI_PREEMPT_COUNT(tp)
> bnez s0, restore_all
> - REG_L s0, TASK_TI_FLAGS(tp)
> - andi s0, s0, _TIF_NEED_RESCHED
> + REG_L s1, TASK_TI_FLAGS(tp)
> + andi s0, s1, _TIF_NEED_RESCHED
> + bnez s0, 1f
> + REG_L s0, TASK_TI_PREEMPT_LAZY_COUNT(tp)
> + bnez s0, restore_all
> + andi s0, s1, _TIF_NEED_RESCHED_LAZY
Can you tell me, who increased/decreased the PREEMPT_LAZY_COUNT? And
who set NEED_RESCHED_LAZY?
> beqz s0, restore_all
> +1:
> call preempt_schedule_irq
> j restore_all
> #endif
> --
> 2.34.1
>
>
> --
> kvm-riscv mailing list
> [email protected]
> http://lists.infradead.org/mailman/listinfo/kvm-riscv
--
Best Regards
Guo Ren
On 2022-09-04 23:16:12 [+0800], Guo Ren wrote:
> > diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S
> > index b9eda3fcbd6d..595100a4c2c7 100644
> > --- a/arch/riscv/kernel/entry.S
> > +++ b/arch/riscv/kernel/entry.S
> > @@ -361,9 +361,14 @@ restore_all:
> > resume_kernel:
> > REG_L s0, TASK_TI_PREEMPT_COUNT(tp)
> > bnez s0, restore_all
> > - REG_L s0, TASK_TI_FLAGS(tp)
> > - andi s0, s0, _TIF_NEED_RESCHED
> > + REG_L s1, TASK_TI_FLAGS(tp)
> > + andi s0, s1, _TIF_NEED_RESCHED
> > + bnez s0, 1f
> > + REG_L s0, TASK_TI_PREEMPT_LAZY_COUNT(tp)
> > + bnez s0, restore_all
> > + andi s0, s1, _TIF_NEED_RESCHED_LAZY
> Can you tell me, who increased/decreased the PREEMPT_LAZY_COUNT? And
> who set NEED_RESCHED_LAZY?
There is "generic" code in the PREEMPT_RT patch doing that. The counter
is incremented/ decremented via preempt_lazy_enable()/disable() and one
of the user is migrate_disable()/enable().
Basically if a task is task_is_realtime() then NEED_RESCHED is set for
the wakeup. For the remaining states (SCHED_OTHER, …) NEED_RESCHED_LAZY
is set for the wakeup. This can be delayed if the task is in a "preempt
disable lazy" section (similar to a preempt_disable() section) but a
task_is_realtime() can still be scheduled if needed.
See details at
https://git.kernel.org/pub/scm/linux/kernel/git/rt/linux-rt-devel.git/plain/patches/sched__Add_support_for_lazy_preemption.patch?h=linux-6.0.y-rt-patches
Sebastian
On Mon, Sep 5, 2022 at 2:34 PM Sebastian Andrzej Siewior
<[email protected]> wrote:
>
> On 2022-09-04 23:16:12 [+0800], Guo Ren wrote:
> > > diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S
> > > index b9eda3fcbd6d..595100a4c2c7 100644
> > > --- a/arch/riscv/kernel/entry.S
> > > +++ b/arch/riscv/kernel/entry.S
> > > @@ -361,9 +361,14 @@ restore_all:
> > > resume_kernel:
> > > REG_L s0, TASK_TI_PREEMPT_COUNT(tp)
> > > bnez s0, restore_all
> > > - REG_L s0, TASK_TI_FLAGS(tp)
> > > - andi s0, s0, _TIF_NEED_RESCHED
> > > + REG_L s1, TASK_TI_FLAGS(tp)
> > > + andi s0, s1, _TIF_NEED_RESCHED
> > > + bnez s0, 1f
> > > + REG_L s0, TASK_TI_PREEMPT_LAZY_COUNT(tp)
> > > + bnez s0, restore_all
> > > + andi s0, s1, _TIF_NEED_RESCHED_LAZY
> > Can you tell me, who increased/decreased the PREEMPT_LAZY_COUNT? And
> > who set NEED_RESCHED_LAZY?
>
> There is "generic" code in the PREEMPT_RT patch doing that. The counter
> is incremented/ decremented via preempt_lazy_enable()/disable() and one
> of the user is migrate_disable()/enable().
> Basically if a task is task_is_realtime() then NEED_RESCHED is set for
> the wakeup. For the remaining states (SCHED_OTHER, …) NEED_RESCHED_LAZY
> is set for the wakeup. This can be delayed if the task is in a "preempt
> disable lazy" section (similar to a preempt_disable() section) but a
> task_is_realtime() can still be scheduled if needed.
Okay, It should be [PATCH RT]. RISC-V would also move to GENERIC_ENTRY
[1], so above assembly code would be replaced by generic one, right?
[1]: https://lore.kernel.org/linux-riscv/[email protected]/T/#u
> See details at
> https://git.kernel.org/pub/scm/linux/kernel/git/rt/linux-rt-devel.git/plain/patches/sched__Add_support_for_lazy_preemption.patch?h=linux-6.0.y-rt-patches
>
> Sebastian
--
Best Regards
Guo Ren
On 2022-09-05 16:33:54 [+0800], Guo Ren wrote:
> > There is "generic" code in the PREEMPT_RT patch doing that. The counter
> > is incremented/ decremented via preempt_lazy_enable()/disable() and one
> > of the user is migrate_disable()/enable().
> > Basically if a task is task_is_realtime() then NEED_RESCHED is set for
> > the wakeup. For the remaining states (SCHED_OTHER, …) NEED_RESCHED_LAZY
> > is set for the wakeup. This can be delayed if the task is in a "preempt
> > disable lazy" section (similar to a preempt_disable() section) but a
> > task_is_realtime() can still be scheduled if needed.
> Okay, It should be [PATCH RT]. RISC-V would also move to GENERIC_ENTRY
> [1], so above assembly code would be replaced by generic one, right?
correct.
Sebastian
On Mon, Sep 05, 2022 at 04:33:54PM +0800, Guo Ren wrote:
> On Mon, Sep 5, 2022 at 2:34 PM Sebastian Andrzej Siewior
> <[email protected]> wrote:
> >
> > On 2022-09-04 23:16:12 [+0800], Guo Ren wrote:
> > > > diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S
> > > > index b9eda3fcbd6d..595100a4c2c7 100644
> > > > --- a/arch/riscv/kernel/entry.S
> > > > +++ b/arch/riscv/kernel/entry.S
> > > > @@ -361,9 +361,14 @@ restore_all:
> > > > resume_kernel:
> > > > REG_L s0, TASK_TI_PREEMPT_COUNT(tp)
> > > > bnez s0, restore_all
> > > > - REG_L s0, TASK_TI_FLAGS(tp)
> > > > - andi s0, s0, _TIF_NEED_RESCHED
> > > > + REG_L s1, TASK_TI_FLAGS(tp)
> > > > + andi s0, s1, _TIF_NEED_RESCHED
> > > > + bnez s0, 1f
> > > > + REG_L s0, TASK_TI_PREEMPT_LAZY_COUNT(tp)
> > > > + bnez s0, restore_all
> > > > + andi s0, s1, _TIF_NEED_RESCHED_LAZY
> > > Can you tell me, who increased/decreased the PREEMPT_LAZY_COUNT? And
> > > who set NEED_RESCHED_LAZY?
> >
> > There is "generic" code in the PREEMPT_RT patch doing that. The counter
> > is incremented/ decremented via preempt_lazy_enable()/disable() and one
> > of the user is migrate_disable()/enable().
> > Basically if a task is task_is_realtime() then NEED_RESCHED is set for
> > the wakeup. For the remaining states (SCHED_OTHER, …) NEED_RESCHED_LAZY
> > is set for the wakeup. This can be delayed if the task is in a "preempt
> > disable lazy" section (similar to a preempt_disable() section) but a
> > task_is_realtime() can still be scheduled if needed.
> Okay, It should be [PATCH RT]. RISC-V would also move to GENERIC_ENTRY
As said in the cover letter, this patch is expected to reviewed and
maintained in RT tree. If your GENERIC_ENTRY patches are merged, I will
send an updated patch.
> [1], so above assembly code would be replaced by generic one, right?
>
> [1]: https://lore.kernel.org/linux-riscv/[email protected]/T/#u
>
>
> > See details at
> > https://git.kernel.org/pub/scm/linux/kernel/git/rt/linux-rt-devel.git/plain/patches/sched__Add_support_for_lazy_preemption.patch?h=linux-6.0.y-rt-patches
>
>
> >
> > Sebastian
>
>
>
> --
> Best Regards
> Guo Ren
On Mon, Sep 5, 2022 at 4:46 PM Sebastian Andrzej Siewior
<[email protected]> wrote:
>
> On 2022-09-05 16:33:54 [+0800], Guo Ren wrote:
> > > There is "generic" code in the PREEMPT_RT patch doing that. The counter
> > > is incremented/ decremented via preempt_lazy_enable()/disable() and one
> > > of the user is migrate_disable()/enable().
> > > Basically if a task is task_is_realtime() then NEED_RESCHED is set for
> > > the wakeup. For the remaining states (SCHED_OTHER, …) NEED_RESCHED_LAZY
> > > is set for the wakeup. This can be delayed if the task is in a "preempt
> > > disable lazy" section (similar to a preempt_disable() section) but a
> > > task_is_realtime() can still be scheduled if needed.
> > Okay, It should be [PATCH RT]. RISC-V would also move to GENERIC_ENTRY
> > [1], so above assembly code would be replaced by generic one, right?
>
> correct.
Maybe TIF_XXX_RESCHED also could be merged into GENERIC_ENTRY, just
like what you've done in syscall.
struct thread_info {
unsigned long flags;
unsigned long syscall_work; /* SYSCALL_WORK_ flags */
+ unsigned long resched_work; /* RESCHED flags */
Or merge them into one:
struct thread_info {
unsigned long flags;
- unsigned long syscall_work; /* SYSCALL_WORK_ flags */
+ unsigned long ge_flags; /* GENERIC_ENTRY flags */
>
> Sebastian
--
Best Regards
Guo Ren
On 11/11/2022 14:32, Sebastian Andrzej Siewior wrote:
> On 2022-09-02 13:29:23 [+0000], [email protected] wrote:
>> I'll give it a run through tonight or tomorrow & give you a full log
>> of what I saw. There's some splats all over the place for me, but I
>> can't tell if that's just knock-on from the other issues.
>
> Is there an update to this or the series as a whole?
Not from me.. completely forgot about it.
I'll put it back in my todo list, sorry.
On 2022-09-02 13:29:23 [+0000], [email protected] wrote:
> I'll give it a run through tonight or tomorrow & give you a full log
> of what I saw. There's some splats all over the place for me, but I
> can't tell if that's just knock-on from the other issues.
Is there an update to this or the series as a whole?
> Thanks,
> Conor.
Sebastian
On 11/11/2022 14:34, Conor Dooley - M52691 wrote:
> On 11/11/2022 14:32, Sebastian Andrzej Siewior wrote:
>> On 2022-09-02 13:29:23 [+0000], [email protected] wrote:
>>> I'll give it a run through tonight or tomorrow & give you a full log
>>> of what I saw. There's some splats all over the place for me, but I
>>> can't tell if that's just knock-on from the other issues.
>>
>> Is there an update to this or the series as a whole?
>
> Not from me.. completely forgot about it.
> I'll put it back in my todo list, sorry.
>
I tried out v6.0.5-rc5 + this patchset (it doesnt apply to v6.1-rcN)
and rt14, got the following:
[ 4.036667] smp: Bringing up secondary CPUs ...
[ 4.069365] BUG: sleeping function called from invalid context at kernel/locking/spinlock_rt.c:46
[ 4.069389] in_atomic(): 1, irqs_disabled(): 1, non_block: 0, pid: 0, name: swapper/1
[ 4.069410] preempt_count: 1, expected: 0
[ 4.069422] RCU nest depth: 1, expected: 1
[ 4.069434] 3 locks held by swapper/1/0:
[ 4.069449] #0: ffffffd82cda3b58 (&pcp->lock){+.+.}-{2:2}, at: get_page_from_freelist+0x220/0x1094
[ 4.069537] #1: ffffffff8129b178 (rcu_read_lock){....}-{1:2}, at: rcu_lock_acquire+0x0/0x2e
[ 4.069602] #2: ffffffff813a3e38 (&zone->lock){+.+.}-{2:2}, at: __rmqueue_pcplist+0x156/0xc28
[ 4.069662] irq event stamp: 0
[ 4.069670] hardirqs last enabled at (0): [<0000000000000000>] 0x0
[ 4.069690] hardirqs last disabled at (0): [<ffffffff8000f0c8>] copy_process+0x50c/0xdaa
[ 4.069727] softirqs last enabled at (0): [<ffffffff8000f0d6>] copy_process+0x51a/0xdaa
[ 4.069756] softirqs last disabled at (0): [<0000000000000000>] 0x0
[ 4.069776] Preemption disabled at:
[ 4.069782] [<ffffffff80041346>] migrate_enable+0x32/0x124
[ 4.069819] CPU: 1 PID: 0 Comm: swapper/1 Not tainted 6.0.5-rt14-00006-g0fda08a972f4-dirty #1
[ 4.069848] Hardware name: Microchip PolarFire-SoC Icicle Kit (DT)
[ 4.069861] Call Trace:
[ 4.069870] [<ffffffff80006628>] show_stack+0x2c/0x38
[ 4.069907] [<ffffffff80900ad4>] dump_stack_lvl+0x64/0x86
[ 4.069935] [<ffffffff80900b0a>] dump_stack+0x14/0x1c
[ 4.069959] [<ffffffff80047534>] __might_resched+0x1bc/0x1c6
[ 4.069995] [<ffffffff80908f7a>] rt_spin_lock+0x42/0xb8
[ 4.070033] [<ffffffff801cab7a>] __rmqueue_pcplist+0x156/0xc28
[ 4.070061] [<ffffffff801cbade>] get_page_from_freelist+0x24e/0x1094
[ 4.070088] [<ffffffff801cb712>] __alloc_pages+0xc6/0x244
[ 4.070113] [<ffffffff801ede42>] new_slab+0x8c/0x4a8
[ 4.070153] [<ffffffff801e955a>] ___slab_alloc+0x5d4/0x9a4
[ 4.070181] [<ffffffff801ea206>] __kmalloc+0xc0/0x1fc
[ 4.070209] [<ffffffff80578296>] detect_cache_attributes+0xb4/0x470
[ 4.070238] [<ffffffff80590520>] update_siblings_masks+0x2c/0x202
[ 4.070270] [<ffffffff80590aa0>] store_cpu_topology+0x30/0x6a
[ 4.070296] [<ffffffff80007756>] smp_callin+0x38/0x66
[ 4.231582] smp: Brought up 1 node, 4 CPUs
There's other stuff that goes awry later on too:
https://gist.githubusercontent.com/ConchuOD/47fd47dfa1f49eb4b0f2fb2a68852a7c/raw/b109b83eec6caa1d67cbb156c6f3e671c10aefe9/gistfile1.txt
The SDHCI stuff I am seeing without rt & in v6.1-rc4 so is
unrelated, but the rest resembles what I saw previously.
idk anything about -rt so if there's something blatant that
I've missed here, please lmk.
On 31/08/2022 18:59, Jisheng Zhang wrote:
> This series is to add PREEMPT_RT support to riscv:
> patch1 adds the missing number of signal exits in vCPU stat
> patch2 switches to the generic guest entry infrastructure
> patch3 select HAVE_POSIX_CPU_TIMERS_TASK_WORK which is a requirement for
> RT
> patch4 adds lazy preempt support
> patch5 allows to enable PREEMPT_RT
>
> I assume patch1, patch2 and patch3 can be reviewed and merged for
> riscv-next, patch4 and patch5 can be reviewed and maintained in rt tree,
> and finally merged once the remaining patches in rt tree are all
> mainlined.
I tested the last two patches on a StarFive VisionFive V2 (DT) board
with 6.1.12-rt7-gdfa52cc14f3b today and the results looked pretty good
for a first run.
root@StarFive:~# lscpu
Architecture: riscv64
Byte Order: Little Endian
CPU(s): 4
On-line CPU(s) list: 0-3
root@StarFive:~# uname -a
Linux StarFive 6.1.12-rt7-gdfa52cc14f3b #1 SMP PREEMPT_RT Thu, 01 Jan
1970 01:00:00 +0000 riscv64 GNU/Linuxb
root@StarFive:~# cat /proc/cmdline
initrd=\initrd.img-6.1.12-rt7-gdfa52cc14f3b LABEL=Boot
root=PARTUUID=7176479f-eeea-46ac-afb6-7ec47ff7c390 console=tty0
console=ttyS0,115200 earlycon rootwait isolcpus=2-3 rcu_nocbs=2-3
nohz_full=2-3 irqaffinity=0-1
root@StarFive:~# cyclictest -m -S -p 90 -i 50 -d 0 -q -D 10m
WARN: stat /dev/cpu_dma_latency failed: No such file or directory
T: 0 ( 358) P:90 I:50 C:11999999 Min: 11 Act: 11 Avg: 11 Max:
55
T: 1 ( 359) P:90 I:50 C:11999241 Min: 11 Act: 11 Avg: 11 Max:
60
Feel free to reach out for further tests or logs.
Best,
Tobias
> Since v1:
> - send to related maillist, I press ENTER too quickly when sending v1
> - remove the signal_pending() handling because that's covered by
> generic guest entry infrastructure
>
> Jisheng Zhang (5):
> RISC-V: KVM: Record number of signal exits as a vCPU stat
> RISC-V: KVM: Use generic guest entry infrastructure
> riscv: select HAVE_POSIX_CPU_TIMERS_TASK_WORK
> riscv: add lazy preempt support
> riscv: Allow to enable RT
>
> arch/riscv/Kconfig | 3 +++
> arch/riscv/include/asm/kvm_host.h | 1 +
> arch/riscv/include/asm/thread_info.h | 7 +++++--
> arch/riscv/kernel/asm-offsets.c | 1 +
> arch/riscv/kernel/entry.S | 9 +++++++--
> arch/riscv/kvm/Kconfig | 1 +
> arch/riscv/kvm/vcpu.c | 18 +++++++-----------
> 7 files changed, 25 insertions(+), 15 deletions(-)
>
> --
> 2.34.1
>
>
> _______________________________________________
> linux-riscv mailing list
> [email protected]
> http://lists.infradead.org/mailman/listinfo/linux-riscv