It turns out that we weren't actually hooking sys_riscv_flush_icache
into the syscall table, which results in any flush_icache() call that
escapes the vDSO to silently do nothing.
Changes since v1:
* sys_riscv_flush_icache is now defined even when SMP=n, which allows
this patch set to build against SMP=n and SMP=y.
This would be necessary to make non-SMP builds work, but there is
another error in the implementation of our syscall linkage that actually
just causes sys_riscv_flush_icache to never build. I've build tested
this on allnoconfig and allnoconfig+SMP=y, as well as defconfig like
normal.
CC: Christoph Hellwig <[email protected]>
CC: Guenter Roeck <[email protected]>
In-Reply-To: <[email protected]>
In-Reply-To: <[email protected]>
Signed-off-by: Palmer Dabbelt <[email protected]>
---
arch/riscv/include/asm/vdso.h | 2 --
arch/riscv/kernel/sys_riscv.c | 10 ++++++++--
2 files changed, 8 insertions(+), 4 deletions(-)
diff --git a/arch/riscv/include/asm/vdso.h b/arch/riscv/include/asm/vdso.h
index 541544d64c33..ec6180a4b55d 100644
--- a/arch/riscv/include/asm/vdso.h
+++ b/arch/riscv/include/asm/vdso.h
@@ -38,8 +38,6 @@ struct vdso_data {
(void __user *)((unsigned long)(base) + __vdso_##name); \
})
-#ifdef CONFIG_SMP
asmlinkage long sys_riscv_flush_icache(uintptr_t, uintptr_t, uintptr_t);
-#endif
#endif /* _ASM_RISCV_VDSO_H */
diff --git a/arch/riscv/kernel/sys_riscv.c b/arch/riscv/kernel/sys_riscv.c
index f7181ed8aafc..180da8d4e14a 100644
--- a/arch/riscv/kernel/sys_riscv.c
+++ b/arch/riscv/kernel/sys_riscv.c
@@ -48,7 +48,6 @@ SYSCALL_DEFINE6(mmap2, unsigned long, addr, unsigned long, len,
}
#endif /* !CONFIG_64BIT */
-#ifdef CONFIG_SMP
/*
* Allows the instruction cache to be flushed from userspace. Despite RISC-V
* having a direct 'fence.i' instruction available to userspace (which we
@@ -66,15 +65,22 @@ SYSCALL_DEFINE6(mmap2, unsigned long, addr, unsigned long, len,
SYSCALL_DEFINE3(riscv_flush_icache, uintptr_t, start, uintptr_t, end,
uintptr_t, flags)
{
+#ifdef CONFIG_SMP
struct mm_struct *mm = current->mm;
bool local = (flags & SYS_RISCV_FLUSH_ICACHE_LOCAL) != 0;
+#endif
/* Check the reserved flags. */
if (unlikely(flags & ~SYS_RISCV_FLUSH_ICACHE_ALL))
return -EINVAL;
+ /*
+ * Without CONFIG_SMP flush_icache_mm is a NOP, which generates unused
+ * variable warnings all over this function.
+ */
+#ifdef CONFIG_SMP
flush_icache_mm(mm, local);
+#endif
return 0;
}
-#endif
--
2.16.4
This file is expected to be included multiple times in the same file in
order to allow the __SYSCALL macro to generate system call tables. With
a global include guard we end up missing __NR_riscv_flush_icache in the
syscall table, which results in icache flushes that escape the vDSO call
to not actually do anything.
The fix is to move to per-#define include guards, which allows the
system call tables to actually be populated. Thanks to Macrus Comstedt
for finding and fixing the bug!
I also went ahead and fixed the SPDX header to use a //-style comment,
which I've been told is the canonical way to do it.
Cc: Marcus Comstedt <[email protected]>
Signed-off-by: Palmer Dabbelt <[email protected]>
---
arch/riscv/include/asm/unistd.h | 5 +++++
arch/riscv/include/uapi/asm/syscalls.h | 15 +++++++++------
2 files changed, 14 insertions(+), 6 deletions(-)
diff --git a/arch/riscv/include/asm/unistd.h b/arch/riscv/include/asm/unistd.h
index 080fb28061de..0caea01d5cca 100644
--- a/arch/riscv/include/asm/unistd.h
+++ b/arch/riscv/include/asm/unistd.h
@@ -11,6 +11,11 @@
* GNU General Public License for more details.
*/
+/*
+ * There is explicitly no include guard here because this file is expected to
+ * be included multiple times. See uapi/asm/syscalls.h for more info.
+ */
+
#define __ARCH_WANT_SYS_CLONE
#include <uapi/asm/unistd.h>
#include <uapi/asm/syscalls.h>
diff --git a/arch/riscv/include/uapi/asm/syscalls.h b/arch/riscv/include/uapi/asm/syscalls.h
index 818655b0d535..690beb002d1d 100644
--- a/arch/riscv/include/uapi/asm/syscalls.h
+++ b/arch/riscv/include/uapi/asm/syscalls.h
@@ -1,10 +1,13 @@
-/* SPDX-License-Identifier: GPL-2.0 */
+// SPDX-License-Identifier: GPL-2.0
/*
- * Copyright (C) 2017 SiFive
+ * Copyright (C) 2017-2018 SiFive
*/
-#ifndef _ASM__UAPI__SYSCALLS_H
-#define _ASM__UAPI__SYSCALLS_H
+/*
+ * There is explicitly no include guard here because this file is expected to
+ * be included multiple times in order to define the syscall macros via
+ * __SYSCALL.
+ */
/*
* Allows the instruction cache to be flushed from userspace. Despite RISC-V
@@ -20,7 +23,7 @@
* caller. We don't currently do anything with the address range, that's just
* in there for forwards compatibility.
*/
+#ifndef __NR_riscv_flush_icache
#define __NR_riscv_flush_icache (__NR_arch_specific_syscall + 15)
-__SYSCALL(__NR_riscv_flush_icache, sys_riscv_flush_icache)
-
#endif
+__SYSCALL(__NR_riscv_flush_icache, sys_riscv_flush_icache)
--
2.16.4
On Thu, Aug 09, 2018 at 01:25:23PM -0700, Palmer Dabbelt wrote:
> This would be necessary to make non-SMP builds work, but there is
> another error in the implementation of our syscall linkage that actually
> just causes sys_riscv_flush_icache to never build. I've build tested
> this on allnoconfig and allnoconfig+SMP=y, as well as defconfig like
> normal.
>
> CC: Christoph Hellwig <[email protected]>
> CC: Guenter Roeck <[email protected]>
> In-Reply-To: <[email protected]>
> In-Reply-To: <[email protected]>
> Signed-off-by: Palmer Dabbelt <[email protected]>
[Compile-]Tested-by: Guenter Roeck <[email protected]>
> ---
> arch/riscv/include/asm/vdso.h | 2 --
> arch/riscv/kernel/sys_riscv.c | 10 ++++++++--
> 2 files changed, 8 insertions(+), 4 deletions(-)
>
> diff --git a/arch/riscv/include/asm/vdso.h b/arch/riscv/include/asm/vdso.h
> index 541544d64c33..ec6180a4b55d 100644
> --- a/arch/riscv/include/asm/vdso.h
> +++ b/arch/riscv/include/asm/vdso.h
> @@ -38,8 +38,6 @@ struct vdso_data {
> (void __user *)((unsigned long)(base) + __vdso_##name); \
> })
>
> -#ifdef CONFIG_SMP
> asmlinkage long sys_riscv_flush_icache(uintptr_t, uintptr_t, uintptr_t);
> -#endif
>
> #endif /* _ASM_RISCV_VDSO_H */
> diff --git a/arch/riscv/kernel/sys_riscv.c b/arch/riscv/kernel/sys_riscv.c
> index f7181ed8aafc..180da8d4e14a 100644
> --- a/arch/riscv/kernel/sys_riscv.c
> +++ b/arch/riscv/kernel/sys_riscv.c
> @@ -48,7 +48,6 @@ SYSCALL_DEFINE6(mmap2, unsigned long, addr, unsigned long, len,
> }
> #endif /* !CONFIG_64BIT */
>
> -#ifdef CONFIG_SMP
> /*
> * Allows the instruction cache to be flushed from userspace. Despite RISC-V
> * having a direct 'fence.i' instruction available to userspace (which we
> @@ -66,15 +65,22 @@ SYSCALL_DEFINE6(mmap2, unsigned long, addr, unsigned long, len,
> SYSCALL_DEFINE3(riscv_flush_icache, uintptr_t, start, uintptr_t, end,
> uintptr_t, flags)
> {
> +#ifdef CONFIG_SMP
> struct mm_struct *mm = current->mm;
> bool local = (flags & SYS_RISCV_FLUSH_ICACHE_LOCAL) != 0;
> +#endif
>
> /* Check the reserved flags. */
> if (unlikely(flags & ~SYS_RISCV_FLUSH_ICACHE_ALL))
> return -EINVAL;
>
> + /*
> + * Without CONFIG_SMP flush_icache_mm is a NOP, which generates unused
> + * variable warnings all over this function.
> + */
> +#ifdef CONFIG_SMP
> flush_icache_mm(mm, local);
> +#endif
>
> return 0;
> }
> -#endif
> --
> 2.16.4
>
On Thu, Aug 09, 2018 at 01:25:24PM -0700, Palmer Dabbelt wrote:
> This file is expected to be included multiple times in the same file in
> order to allow the __SYSCALL macro to generate system call tables. With
> a global include guard we end up missing __NR_riscv_flush_icache in the
> syscall table, which results in icache flushes that escape the vDSO call
> to not actually do anything.
>
> The fix is to move to per-#define include guards, which allows the
> system call tables to actually be populated. Thanks to Macrus Comstedt
> for finding and fixing the bug!
>
> I also went ahead and fixed the SPDX header to use a //-style comment,
> which I've been told is the canonical way to do it.
>
> Cc: Marcus Comstedt <[email protected]>
> Signed-off-by: Palmer Dabbelt <[email protected]>
[Compile-]Tested-by: Guenter Roeck <[email protected]>
on top of linux-next after reverting the version of the patch there.
I also tried to run the resulting image (defconfig) with qemu (built
from https://github.com/riscv/riscv-qemu.git), but that still doesn't
work. I assume there are still some patches missing ?
Thanks,
Guenter
On Thu, 09 Aug 2018 14:24:22 PDT (-0700), [email protected] wrote:
> On Thu, Aug 09, 2018 at 01:25:24PM -0700, Palmer Dabbelt wrote:
>> This file is expected to be included multiple times in the same file in
>> order to allow the __SYSCALL macro to generate system call tables. With
>> a global include guard we end up missing __NR_riscv_flush_icache in the
>> syscall table, which results in icache flushes that escape the vDSO call
>> to not actually do anything.
>>
>> The fix is to move to per-#define include guards, which allows the
>> system call tables to actually be populated. Thanks to Macrus Comstedt
>> for finding and fixing the bug!
>>
>> I also went ahead and fixed the SPDX header to use a //-style comment,
>> which I've been told is the canonical way to do it.
>>
>> Cc: Marcus Comstedt <[email protected]>
>> Signed-off-by: Palmer Dabbelt <[email protected]>
>
> [Compile-]Tested-by: Guenter Roeck <[email protected]>
>
> on top of linux-next after reverting the version of the patch there.
>
> I also tried to run the resulting image (defconfig) with qemu (built
> from https://github.com/riscv/riscv-qemu.git), but that still doesn't
> work. I assume there are still some patches missing ?
Do you have the PLIC patches? They'll be necessary to make this all work, and
there's a v4 out now that when combined with for-next should get you to
userspace.
https://lore.kernel.org/lkml/[email protected]/T/#u
Also, what is your methodology? I follow
https://wiki.qemu.org/Documentation/Platforms/RISCV
and could could natively compile and run hello world with an earlier version of
Christoph's patch set, which is really only cosmetically different than the v4.
I use qemu's master branch as well, which when I tried was exactly 3.0.0-rc3.
On 08/09/2018 06:03 PM, Palmer Dabbelt wrote:
> On Thu, 09 Aug 2018 14:24:22 PDT (-0700), [email protected] wrote:
>> On Thu, Aug 09, 2018 at 01:25:24PM -0700, Palmer Dabbelt wrote:
>>> This file is expected to be included multiple times in the same file in
>>> order to allow the __SYSCALL macro to generate system call tables. With
>>> a global include guard we end up missing __NR_riscv_flush_icache in the
>>> syscall table, which results in icache flushes that escape the vDSO call
>>> to not actually do anything.
>>>
>>> The fix is to move to per-#define include guards, which allows the
>>> system call tables to actually be populated. Thanks to Macrus Comstedt
>>> for finding and fixing the bug!
>>>
>>> I also went ahead and fixed the SPDX header to use a //-style comment,
>>> which I've been told is the canonical way to do it.
>>>
>>> Cc: Marcus Comstedt <[email protected]>
>>> Signed-off-by: Palmer Dabbelt <[email protected]>
>>
>> [Compile-]Tested-by: Guenter Roeck <[email protected]>
>>
>> on top of linux-next after reverting the version of the patch there.
>>
>> I also tried to run the resulting image (defconfig) with qemu (built
>> from https://github.com/riscv/riscv-qemu.git), but that still doesn't
>> work. I assume there are still some patches missing ?
>
> Do you have the PLIC patches? They'll be necessary to make this all work, and there's a v4 out now that when combined with for-next should get you to userspace.
>
> https://lore.kernel.org/lkml/[email protected]/T/#u
>
Yes, after merging that branch on top of linux-next I can boot into Linux.
If I add my "riscv: Drop setup_initrd" patch as well, I can boot using
initrd, otherwise I have to use virtio-blk-device.
> Also, what is your methodology? I follow
>
> https://wiki.qemu.org/Documentation/Platforms/RISCV
>
> and could could natively compile and run hello world with an earlier version of Christoph's patch set, which is really only cosmetically different than the v4. I use qemu's master branch as well, which when I tried was exactly 3.0.0-rc3.
>
That doesn't work for me, possibly because I don't specify a bbl
image with -kernel but vmlinux (using -bios for the bbl image).
I use branch qemu-for-upstream of https://github.com/riscv/riscv-qemu.git.
Guenter
On Thu, 09 Aug 2018 19:40:55 PDT (-0700), [email protected] wrote:
> On 08/09/2018 06:03 PM, Palmer Dabbelt wrote:
>> On Thu, 09 Aug 2018 14:24:22 PDT (-0700), [email protected] wrote:
>>> On Thu, Aug 09, 2018 at 01:25:24PM -0700, Palmer Dabbelt wrote:
>>>> This file is expected to be included multiple times in the same file in
>>>> order to allow the __SYSCALL macro to generate system call tables. With
>>>> a global include guard we end up missing __NR_riscv_flush_icache in the
>>>> syscall table, which results in icache flushes that escape the vDSO call
>>>> to not actually do anything.
>>>>
>>>> The fix is to move to per-#define include guards, which allows the
>>>> system call tables to actually be populated. Thanks to Macrus Comstedt
>>>> for finding and fixing the bug!
>>>>
>>>> I also went ahead and fixed the SPDX header to use a //-style comment,
>>>> which I've been told is the canonical way to do it.
>>>>
>>>> Cc: Marcus Comstedt <[email protected]>
>>>> Signed-off-by: Palmer Dabbelt <[email protected]>
>>>
>>> [Compile-]Tested-by: Guenter Roeck <[email protected]>
>>>
>>> on top of linux-next after reverting the version of the patch there.
>>>
>>> I also tried to run the resulting image (defconfig) with qemu (built
>>> from https://github.com/riscv/riscv-qemu.git), but that still doesn't
>>> work. I assume there are still some patches missing ?
>>
>> Do you have the PLIC patches? They'll be necessary to make this all work, and there's a v4 out now that when combined with for-next should get you to userspace.
>>
>> https://lore.kernel.org/lkml/[email protected]/T/#u
>>
> Yes, after merging that branch on top of linux-next I can boot into Linux.
> If I add my "riscv: Drop setup_initrd" patch as well, I can boot using
> initrd, otherwise I have to use virtio-blk-device.
Awesome! If you patch isn't on for-next then I must have missed it, do you
mind sending me a pointer? I can't find any references in my email.
>> Also, what is your methodology? I follow
>>
>> https://wiki.qemu.org/Documentation/Platforms/RISCV
>>
>> and could could natively compile and run hello world with an earlier version of Christoph's patch set, which is really only cosmetically different than the v4. I use qemu's master branch as well, which when I tried was exactly 3.0.0-rc3.
>>
>
> That doesn't work for me, possibly because I don't specify a bbl
> image with -kernel but vmlinux (using -bios for the bbl image).
> I use branch qemu-for-upstream of https://github.com/riscv/riscv-qemu.git.
>
> Guenter
On 08/09/2018 08:59 PM, Palmer Dabbelt wrote:
> On Thu, 09 Aug 2018 19:40:55 PDT (-0700), [email protected] wrote:
>> On 08/09/2018 06:03 PM, Palmer Dabbelt wrote:
>>> On Thu, 09 Aug 2018 14:24:22 PDT (-0700), [email protected] wrote:
>>>> On Thu, Aug 09, 2018 at 01:25:24PM -0700, Palmer Dabbelt wrote:
>>>>> This file is expected to be included multiple times in the same file in
>>>>> order to allow the __SYSCALL macro to generate system call tables. With
>>>>> a global include guard we end up missing __NR_riscv_flush_icache in the
>>>>> syscall table, which results in icache flushes that escape the vDSO call
>>>>> to not actually do anything.
>>>>>
>>>>> The fix is to move to per-#define include guards, which allows the
>>>>> system call tables to actually be populated. Thanks to Macrus Comstedt
>>>>> for finding and fixing the bug!
>>>>>
>>>>> I also went ahead and fixed the SPDX header to use a //-style comment,
>>>>> which I've been told is the canonical way to do it.
>>>>>
>>>>> Cc: Marcus Comstedt <[email protected]>
>>>>> Signed-off-by: Palmer Dabbelt <[email protected]>
>>>>
>>>> [Compile-]Tested-by: Guenter Roeck <[email protected]>
>>>>
>>>> on top of linux-next after reverting the version of the patch there.
>>>>
>>>> I also tried to run the resulting image (defconfig) with qemu (built
>>>> from https://github.com/riscv/riscv-qemu.git), but that still doesn't
>>>> work. I assume there are still some patches missing ?
>>>
>>> Do you have the PLIC patches? They'll be necessary to make this all work, and there's a v4 out now that when combined with for-next should get you to userspace.
>>>
>>> https://lore.kernel.org/lkml/[email protected]/T/#u
>>>
>> Yes, after merging that branch on top of linux-next I can boot into Linux.
>> If I add my "riscv: Drop setup_initrd" patch as well, I can boot using
>> initrd, otherwise I have to use virtio-blk-device.
>
> Awesome! If you patch isn't on for-next then I must have missed it, do you mind sending me a pointer? I can't find any references in my email.
>
Hmm ... weird. I don't find it either. Maybe I sent it only in my dreams.
I'll send it out for review.
Guenter
>>> Also, what is your methodology? I follow
>>>
>>> https://wiki.qemu.org/Documentation/Platforms/RISCV
>>>
>>> and could could natively compile and run hello world with an earlier version of Christoph's patch set, which is really only cosmetically different than the v4. I use qemu's master branch as well, which when I tried was exactly 3.0.0-rc3.
>>>
>>
>> That doesn't work for me, possibly because I don't specify a bbl
>> image with -kernel but vmlinux (using -bios for the bbl image).
>> I use branch qemu-for-upstream of https://github.com/riscv/riscv-qemu.git.
>>
>> Guenter
>
On 08/09/2018 08:59 PM, Palmer Dabbelt wrote:
[ ... ]
>>> Also, what is your methodology? I follow
>>>
>>> https://wiki.qemu.org/Documentation/Platforms/RISCV
Here are my qemu command lines:
qemu-system-riscv64 -M virt -m 512M -no-reboot -bios bbl \
-kernel vmlinux -netdev user,id=net0 -device virtio-net-device,netdev=net0 \
-initrd rootfs.cpio \
-append 'rdinit=/sbin/init earlycon console=ttyS0,115200' \
-nographic -monitor none
qemu-system-riscv64 -M virt -m 512M -no-reboot -bios bbl \
-kernel vmlinux -netdev user,id=net0 -device virtio-net-device,netdev=net0 \
-device virtio-blk-device,drive=d0 \
-drive file=rootfs.ext2,if=none,id=d0,format=raw \
-append 'root=/dev/vda rw earlycon console=ttyS0,115200' \
-nographic -monitor none
Root file systems and the bbl binary are published at
https://github.com/groeck/linux-build-test/tree/master/rootfs/riscv64
Hint: If you specify "noreboot" as additional command line option, you'll end up in a shell.
Qemu version:
https://github.com/riscv/riscv-qemu.git branch 'v3.0.0-local-riscv'
Guenter