From: Arnd Bergmann <[email protected]>
In some configurations __const_iowrite32_copy() does not get inlined
and gcc runs into the BUILD_BUG():
In file included from <command-line>:
In function '__const_memcpy_toio_aligned32',
inlined from '__const_iowrite32_copy' at arch/arm64/include/asm/io.h:203:3,
inlined from '__const_iowrite32_copy' at arch/arm64/include/asm/io.h:199:20:
include/linux/compiler_types.h:487:45: error: call to '__compiletime_assert_538' declared with attribute error: BUILD_BUG failed
487 | _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
| ^
include/linux/compiler_types.h:468:25: note: in definition of macro '__compiletime_assert'
468 | prefix ## suffix(); \
| ^~~~~~
include/linux/compiler_types.h:487:9: note: in expansion of macro '_compiletime_assert'
487 | _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
| ^~~~~~~~~~~~~~~~~~~
include/linux/build_bug.h:39:37: note: in expansion of macro 'compiletime_assert'
39 | #define BUILD_BUG_ON_MSG(cond, msg) compiletime_assert(!(cond), msg)
| ^~~~~~~~~~~~~~~~~~
include/linux/build_bug.h:59:21: note: in expansion of macro 'BUILD_BUG_ON_MSG'
59 | #define BUILD_BUG() BUILD_BUG_ON_MSG(1, "BUILD_BUG failed")
| ^~~~~~~~~~~~~~~~
arch/arm64/include/asm/io.h:193:17: note: in expansion of macro 'BUILD_BUG'
193 | BUILD_BUG();
| ^~~~~~~~~
Move the check for constant arguments into the inline function to ensure
it is still constant if the compiler decides against inlining it, and
mark them as __always_inline to override the logic that sometimes leads
to the compiler not producing the simplified output.
Note that either the __always_inline annotation or the check for a
constant value are sufficient here, but combining the two looks cleaner
as it also avoids the macro. With clang-8 and older, the macro was still
needed, but all versions of gcc and clang can reliably perform constant
folding here.
Fixes: ead79118dae6 ("arm64/io: Provide a WC friendly __iowriteXX_copy()")
Signed-off-by: Arnd Bergmann <[email protected]>
---
v3:
- also mark functions as __always_inline
v2:
- fix both 32-bit and 64-bit copies
- remove now-redundant macros
---
arch/arm64/include/asm/io.h | 36 ++++++++++++++++--------------------
1 file changed, 16 insertions(+), 20 deletions(-)
diff --git a/arch/arm64/include/asm/io.h b/arch/arm64/include/asm/io.h
index 4ff0ae3f6d66..bc239371323a 100644
--- a/arch/arm64/include/asm/io.h
+++ b/arch/arm64/include/asm/io.h
@@ -153,8 +153,9 @@ extern void __memset_io(volatile void __iomem *, int, size_t);
* emit the large TLP from the CPU.
*/
-static inline void __const_memcpy_toio_aligned32(volatile u32 __iomem *to,
- const u32 *from, size_t count)
+static __always_inline void
+__const_memcpy_toio_aligned32(volatile u32 __iomem *to, const u32 *from,
+ size_t count)
{
switch (count) {
case 8:
@@ -196,24 +197,22 @@ static inline void __const_memcpy_toio_aligned32(volatile u32 __iomem *to,
void __iowrite32_copy_full(void __iomem *to, const void *from, size_t count);
-static inline void __const_iowrite32_copy(void __iomem *to, const void *from,
- size_t count)
+static __always_inline void
+__iowrite32_copy(void __iomem *to, const void *from, size_t count)
{
- if (count == 8 || count == 4 || count == 2 || count == 1) {
+ if (__builtin_constant_p(count) &&
+ (count == 8 || count == 4 || count == 2 || count == 1)) {
__const_memcpy_toio_aligned32(to, from, count);
dgh();
} else {
__iowrite32_copy_full(to, from, count);
}
}
+#define __iowrite32_copy(to, from, count) __iowrite32_copy(to, from, count)
-#define __iowrite32_copy(to, from, count) \
- (__builtin_constant_p(count) ? \
- __const_iowrite32_copy(to, from, count) : \
- __iowrite32_copy_full(to, from, count))
-
-static inline void __const_memcpy_toio_aligned64(volatile u64 __iomem *to,
- const u64 *from, size_t count)
+static __always_inline void
+__const_memcpy_toio_aligned64(volatile u64 __iomem *to, const u64 *from,
+ size_t count)
{
switch (count) {
case 8:
@@ -255,21 +254,18 @@ static inline void __const_memcpy_toio_aligned64(volatile u64 __iomem *to,
void __iowrite64_copy_full(void __iomem *to, const void *from, size_t count);
-static inline void __const_iowrite64_copy(void __iomem *to, const void *from,
- size_t count)
+static __always_inline void
+__iowrite64_copy(void __iomem *to, const void *from, size_t count)
{
- if (count == 8 || count == 4 || count == 2 || count == 1) {
+ if (__builtin_constant_p(count) &&
+ (count == 8 || count == 4 || count == 2 || count == 1)) {
__const_memcpy_toio_aligned64(to, from, count);
dgh();
} else {
__iowrite64_copy_full(to, from, count);
}
}
-
-#define __iowrite64_copy(to, from, count) \
- (__builtin_constant_p(count) ? \
- __const_iowrite64_copy(to, from, count) : \
- __iowrite64_copy_full(to, from, count))
+#define __iowrite64_copy(to, from, count) __iowrite64_copy(to, from, count)
/*
* I/O memory mapping functions.
--
2.39.2
On Tue, Jun 04, 2024 at 10:59:57PM +0200, Arnd Bergmann wrote:
> From: Arnd Bergmann <[email protected]>
>
> In some configurations __const_iowrite32_copy() does not get inlined
> and gcc runs into the BUILD_BUG():
>
> In file included from <command-line>:
> In function '__const_memcpy_toio_aligned32',
> inlined from '__const_iowrite32_copy' at arch/arm64/include/asm/io.h:203:3,
> inlined from '__const_iowrite32_copy' at arch/arm64/include/asm/io.h:199:20:
> include/linux/compiler_types.h:487:45: error: call to '__compiletime_assert_538' declared with attribute error: BUILD_BUG failed
> 487 | _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
> | ^
> include/linux/compiler_types.h:468:25: note: in definition of macro '__compiletime_assert'
> 468 | prefix ## suffix(); \
> | ^~~~~~
> include/linux/compiler_types.h:487:9: note: in expansion of macro '_compiletime_assert'
> 487 | _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
> | ^~~~~~~~~~~~~~~~~~~
> include/linux/build_bug.h:39:37: note: in expansion of macro 'compiletime_assert'
> 39 | #define BUILD_BUG_ON_MSG(cond, msg) compiletime_assert(!(cond), msg)
> | ^~~~~~~~~~~~~~~~~~
> include/linux/build_bug.h:59:21: note: in expansion of macro 'BUILD_BUG_ON_MSG'
> 59 | #define BUILD_BUG() BUILD_BUG_ON_MSG(1, "BUILD_BUG failed")
> | ^~~~~~~~~~~~~~~~
> arch/arm64/include/asm/io.h:193:17: note: in expansion of macro 'BUILD_BUG'
> 193 | BUILD_BUG();
> | ^~~~~~~~~
>
> Move the check for constant arguments into the inline function to ensure
> it is still constant if the compiler decides against inlining it, and
> mark them as __always_inline to override the logic that sometimes leads
> to the compiler not producing the simplified output.
>
> Note that either the __always_inline annotation or the check for a
> constant value are sufficient here, but combining the two looks cleaner
> as it also avoids the macro. With clang-8 and older, the macro was still
> needed, but all versions of gcc and clang can reliably perform constant
> folding here.
>
> Fixes: ead79118dae6 ("arm64/io: Provide a WC friendly __iowriteXX_copy()")
> Signed-off-by: Arnd Bergmann <[email protected]>
I have a trivial nit below, but either way this looks good to me, so
regardless of that:
Reviewed-by: Mark Rutland <[email protected]>
> +static __always_inline void
> +__iowrite32_copy(void __iomem *to, const void *from, size_t count)
> {
> - if (count == 8 || count == 4 || count == 2 || count == 1) {
> + if (__builtin_constant_p(count) &&
> + (count == 8 || count == 4 || count == 2 || count == 1)) {
> __const_memcpy_toio_aligned32(to, from, count);
> dgh();
> } else {
> __iowrite32_copy_full(to, from, count);
> }
> }
> +#define __iowrite32_copy(to, from, count) __iowrite32_copy(to, from, count)
Normally we'd make this:
#define __iowrite32_copy __iowrite32_copy
... so that it's clear it's just providing the preprocessor symbol, and
doesn't have to be updated if the prototype changes.
[...]
> +#define __iowrite64_copy(to, from, count) __iowrite64_copy(to, from, count)
Likewise here.
Mark.
On Wed, Jun 05, 2024 at 10:15:38AM +0100, Mark Rutland wrote:
> On Tue, Jun 04, 2024 at 10:59:57PM +0200, Arnd Bergmann wrote:
> > Move the check for constant arguments into the inline function to ensure
> > it is still constant if the compiler decides against inlining it, and
> > mark them as __always_inline to override the logic that sometimes leads
> > to the compiler not producing the simplified output.
> >
> > Note that either the __always_inline annotation or the check for a
> > constant value are sufficient here, but combining the two looks cleaner
> > as it also avoids the macro. With clang-8 and older, the macro was still
> > needed, but all versions of gcc and clang can reliably perform constant
> > folding here.
> >
> > Fixes: ead79118dae6 ("arm64/io: Provide a WC friendly __iowriteXX_copy()")
> > Signed-off-by: Arnd Bergmann <[email protected]>
>
> I have a trivial nit below, but either way this looks good to me, so
> regardless of that:
>
> Reviewed-by: Mark Rutland <[email protected]>
>
> > +static __always_inline void
> > +__iowrite32_copy(void __iomem *to, const void *from, size_t count)
> > {
> > - if (count == 8 || count == 4 || count == 2 || count == 1) {
> > + if (__builtin_constant_p(count) &&
> > + (count == 8 || count == 4 || count == 2 || count == 1)) {
> > __const_memcpy_toio_aligned32(to, from, count);
> > dgh();
> > } else {
> > __iowrite32_copy_full(to, from, count);
> > }
> > }
> > +#define __iowrite32_copy(to, from, count) __iowrite32_copy(to, from, count)
>
> Normally we'd make this:
>
> #define __iowrite32_copy __iowrite32_copy
>
> ... so that it's clear it's just providing the preprocessor symbol, and
> doesn't have to be updated if the prototype changes.
>
> [...]
>
> > +#define __iowrite64_copy(to, from, count) __iowrite64_copy(to, from, count)
>
> Likewise here.
I can fold these two changes in.
Will
On Wed, Jun 5, 2024, at 13:10, Will Deacon wrote:
> On Wed, Jun 05, 2024 at 10:15:38AM +0100, Mark Rutland wrote:
>> > +#define __iowrite32_copy(to, from, count) __iowrite32_copy(to, from, count)
>>
>> Normally we'd make this:
>>
>> #define __iowrite32_copy __iowrite32_copy
>>
>
> I can fold these two changes in.
Sounds good, thanks!
Arnd
On Wed, Jun 05, 2024 at 10:15:38AM +0100, Mark Rutland wrote:
> On Tue, Jun 04, 2024 at 10:59:57PM +0200, Arnd Bergmann wrote:
> > From: Arnd Bergmann <[email protected]>
> >
> > In some configurations __const_iowrite32_copy() does not get inlined
> > and gcc runs into the BUILD_BUG():
> >
> > In file included from <command-line>:
> > In function '__const_memcpy_toio_aligned32',
> > inlined from '__const_iowrite32_copy' at arch/arm64/include/asm/io.h:203:3,
> > inlined from '__const_iowrite32_copy' at arch/arm64/include/asm/io.h:199:20:
> > include/linux/compiler_types.h:487:45: error: call to '__compiletime_assert_538' declared with attribute error: BUILD_BUG failed
> > 487 | _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
> > | ^
> > include/linux/compiler_types.h:468:25: note: in definition of macro '__compiletime_assert'
> > 468 | prefix ## suffix(); \
> > | ^~~~~~
> > include/linux/compiler_types.h:487:9: note: in expansion of macro '_compiletime_assert'
> > 487 | _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
> > | ^~~~~~~~~~~~~~~~~~~
> > include/linux/build_bug.h:39:37: note: in expansion of macro 'compiletime_assert'
> > 39 | #define BUILD_BUG_ON_MSG(cond, msg) compiletime_assert(!(cond), msg)
> > | ^~~~~~~~~~~~~~~~~~
> > include/linux/build_bug.h:59:21: note: in expansion of macro 'BUILD_BUG_ON_MSG'
> > 59 | #define BUILD_BUG() BUILD_BUG_ON_MSG(1, "BUILD_BUG failed")
> > | ^~~~~~~~~~~~~~~~
> > arch/arm64/include/asm/io.h:193:17: note: in expansion of macro 'BUILD_BUG'
> > 193 | BUILD_BUG();
> > | ^~~~~~~~~
> >
> > Move the check for constant arguments into the inline function to ensure
> > it is still constant if the compiler decides against inlining it, and
> > mark them as __always_inline to override the logic that sometimes leads
> > to the compiler not producing the simplified output.
> >
> > Note that either the __always_inline annotation or the check for a
> > constant value are sufficient here, but combining the two looks cleaner
> > as it also avoids the macro. With clang-8 and older, the macro was still
> > needed, but all versions of gcc and clang can reliably perform constant
> > folding here.
> >
> > Fixes: ead79118dae6 ("arm64/io: Provide a WC friendly __iowriteXX_copy()")
> > Signed-off-by: Arnd Bergmann <[email protected]>
>
> I have a trivial nit below, but either way this looks good to me, so
> regardless of that:
>
> Reviewed-by: Mark Rutland <[email protected]>
Reviewed-by: Jason Gunthorpe <[email protected]>
Still codegens what I expect on clang-17 at least, agree with Mark's
note
Thanks,
Jason
On Tue, 04 Jun 2024 22:59:57 +0200, Arnd Bergmann wrote:
> In some configurations __const_iowrite32_copy() does not get inlined
> and gcc runs into the BUILD_BUG():
>
> In file included from <command-line>:
> In function '__const_memcpy_toio_aligned32',
> inlined from '__const_iowrite32_copy' at arch/arm64/include/asm/io.h:203:3,
> inlined from '__const_iowrite32_copy' at arch/arm64/include/asm/io.h:199:20:
> include/linux/compiler_types.h:487:45: error: call to '__compiletime_assert_538' declared with attribute error: BUILD_BUG failed
> 487 | _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
> | ^
> include/linux/compiler_types.h:468:25: note: in definition of macro '__compiletime_assert'
> 468 | prefix ## suffix(); \
> | ^~~~~~
> include/linux/compiler_types.h:487:9: note: in expansion of macro '_compiletime_assert'
> 487 | _compiletime_assert(condition, msg, __compiletime_assert_, __COUNTER__)
> | ^~~~~~~~~~~~~~~~~~~
> include/linux/build_bug.h:39:37: note: in expansion of macro 'compiletime_assert'
> 39 | #define BUILD_BUG_ON_MSG(cond, msg) compiletime_assert(!(cond), msg)
> | ^~~~~~~~~~~~~~~~~~
> include/linux/build_bug.h:59:21: note: in expansion of macro 'BUILD_BUG_ON_MSG'
> 59 | #define BUILD_BUG() BUILD_BUG_ON_MSG(1, "BUILD_BUG failed")
> | ^~~~~~~~~~~~~~~~
> arch/arm64/include/asm/io.h:193:17: note: in expansion of macro 'BUILD_BUG'
> 193 | BUILD_BUG();
> | ^~~~~~~~~
>
> [...]
Applied to arm64 (for-next/fixes), thanks!
[1/1] arm64/io: add constant-argument check
https://git.kernel.org/arm64/c/5c40e428aea6
Cheers,
--
Will
https://fixes.arm64.dev
https://next.arm64.dev
https://will.arm64.dev