As of today we enable unaligned access unconditionally on ARCv2.
Lets move it under Kconfig option and use it actively in SW if it is
enabled.
While I'm at it fix and optimise ARCv2 memcpy implementaion.
Changes v1->v2:
* Rebase onto last ARC changes.
* Don't add dummy symbol to ARC Kconfig
* Print info about unaligned access status in kernel log
Eugeniy Paltsev (5):
ARCv2: lib: memcpy: fix doing prefetchw outside of buffer
ARCv2: introduce unaligned access under a Kconfig option
ARCv2: use unaligned access in SW
ARCv2: LIB: MEMCPY: fixed and optimised routine
ARC: boot log: print unaligned memory access details
arch/arc/Kconfig | 9 +++++++
arch/arc/Makefile | 6 +++++
arch/arc/include/asm/irqflags-arcv2.h | 4 +++
arch/arc/kernel/head.S | 14 +++++++----
arch/arc/kernel/intc-arcv2.c | 2 +-
arch/arc/kernel/setup.c | 13 +++++++---
arch/arc/lib/Makefile | 8 +++++-
arch/arc/lib/memcpy-archs-unaligned.S | 46 +++++++++++++++++++++++++++++++++++
arch/arc/lib/memcpy-archs.S | 14 -----------
9 files changed, 92 insertions(+), 24 deletions(-)
create mode 100644 arch/arc/lib/memcpy-archs-unaligned.S
--
2.14.5
Optimise code to use efficient unaligned memory access which is
available on ARCv2. This allows us to really simplify memcpy code
and speed up the code one and a half times (in case of unaligned
source or destination).
Signed-off-by: Eugeniy Paltsev <[email protected]>
---
arch/arc/lib/Makefile | 8 +++++-
arch/arc/lib/memcpy-archs-unaligned.S | 47 +++++++++++++++++++++++++++++++++++
2 files changed, 54 insertions(+), 1 deletion(-)
create mode 100644 arch/arc/lib/memcpy-archs-unaligned.S
diff --git a/arch/arc/lib/Makefile b/arch/arc/lib/Makefile
index b1656d156097..f7537b466b23 100644
--- a/arch/arc/lib/Makefile
+++ b/arch/arc/lib/Makefile
@@ -8,4 +8,10 @@
lib-y := strchr-700.o strcpy-700.o strlen.o memcmp.o
lib-$(CONFIG_ISA_ARCOMPACT) += memcpy-700.o memset.o strcmp.o
-lib-$(CONFIG_ISA_ARCV2) += memcpy-archs.o memset-archs.o strcmp-archs.o
+lib-$(CONFIG_ISA_ARCV2) += memset-archs.o strcmp-archs.o
+
+ifdef CONFIG_ARC_USE_UNALIGNED_MEM_ACCESS
+lib-$(CONFIG_ISA_ARCV2) +=memcpy-archs-unaligned.o
+else
+lib-$(CONFIG_ISA_ARCV2) +=memcpy-archs.o
+endif
diff --git a/arch/arc/lib/memcpy-archs-unaligned.S b/arch/arc/lib/memcpy-archs-unaligned.S
new file mode 100644
index 000000000000..28993a73fdde
--- /dev/null
+++ b/arch/arc/lib/memcpy-archs-unaligned.S
@@ -0,0 +1,47 @@
+/* SPDX-License-Identifier: GPL-2.0+ */
+/*
+ * ARCv2 memcpy implementation optimized for unaligned memory access using.
+ *
+ * Copyright (C) 2019 Synopsys
+ * Author: Eugeniy Paltsev <[email protected]>
+ */
+
+#include <linux/linkage.h>
+
+#ifdef CONFIG_ARC_HAS_LL64
+# define LOADX(DST,RX) ldd.ab DST, [RX, 8]
+# define STOREX(SRC,RX) std.ab SRC, [RX, 8]
+# define ZOLSHFT 5
+# define ZOLAND 0x1F
+#else
+# define LOADX(DST,RX) ld.ab DST, [RX, 4]
+# define STOREX(SRC,RX) st.ab SRC, [RX, 4]
+# define ZOLSHFT 4
+# define ZOLAND 0xF
+#endif
+
+ENTRY_CFI(memcpy)
+ mov r3, r0 ; don;t clobber ret val
+
+ lsr.f lp_count, r2, ZOLSHFT
+ lpnz @.Lcopy32_64bytes
+ ;; LOOP START
+ LOADX (r6, r1)
+ LOADX (r8, r1)
+ LOADX (r10, r1)
+ LOADX (r4, r1)
+ STOREX (r6, r3)
+ STOREX (r8, r3)
+ STOREX (r10, r3)
+ STOREX (r4, r3)
+.Lcopy32_64bytes:
+
+ and.f lp_count, r2, ZOLAND ;Last remaining 31 bytes
+ lpnz @.Lcopyremainingbytes
+ ;; LOOP START
+ ldb.ab r5, [r1, 1]
+ stb.ab r5, [r3, 1]
+.Lcopyremainingbytes:
+
+ j [blink]
+END_CFI(memcpy)
--
2.14.5
ARCv2 optimized memcpy uses PREFETCHW instruction for prefetching the
next cache line but doesn't ensure that the line is not past the end of
the buffer. PRETECHW changes the line ownership and marks it dirty,
which can cause data corruption if this area is used for DMA IO.
Fix the issue by avoiding the PREFETCHW. This leads to performance
degradation but it is OK as we'll introduce new memcpy implementation
optimized for unaligned memory access using.
We also cut off all PREFETCH instructions at they are quite useless
here:
* we call PREFETCH right before LOAD instruction call.
* we copy 16 or 32 bytes of data (depending on CONFIG_ARC_HAS_LL64)
in a main logical loop. so we call PREFETCH 4 times (or 2 times)
for each L1 cache line (in case of 64B L1 cache Line which is
default case). Obviously this is not optimal.
Signed-off-by: Eugeniy Paltsev <[email protected]>
---
arch/arc/lib/memcpy-archs.S | 14 --------------
1 file changed, 14 deletions(-)
diff --git a/arch/arc/lib/memcpy-archs.S b/arch/arc/lib/memcpy-archs.S
index d61044dd8b58..ea14b0bf3116 100644
--- a/arch/arc/lib/memcpy-archs.S
+++ b/arch/arc/lib/memcpy-archs.S
@@ -25,15 +25,11 @@
#endif
#ifdef CONFIG_ARC_HAS_LL64
-# define PREFETCH_READ(RX) prefetch [RX, 56]
-# define PREFETCH_WRITE(RX) prefetchw [RX, 64]
# define LOADX(DST,RX) ldd.ab DST, [RX, 8]
# define STOREX(SRC,RX) std.ab SRC, [RX, 8]
# define ZOLSHFT 5
# define ZOLAND 0x1F
#else
-# define PREFETCH_READ(RX) prefetch [RX, 28]
-# define PREFETCH_WRITE(RX) prefetchw [RX, 32]
# define LOADX(DST,RX) ld.ab DST, [RX, 4]
# define STOREX(SRC,RX) st.ab SRC, [RX, 4]
# define ZOLSHFT 4
@@ -41,8 +37,6 @@
#endif
ENTRY_CFI(memcpy)
- prefetch [r1] ; Prefetch the read location
- prefetchw [r0] ; Prefetch the write location
mov.f 0, r2
;;; if size is zero
jz.d [blink]
@@ -72,8 +66,6 @@ ENTRY_CFI(memcpy)
lpnz @.Lcopy32_64bytes
;; LOOP START
LOADX (r6, r1)
- PREFETCH_READ (r1)
- PREFETCH_WRITE (r3)
LOADX (r8, r1)
LOADX (r10, r1)
LOADX (r4, r1)
@@ -117,9 +109,7 @@ ENTRY_CFI(memcpy)
lpnz @.Lcopy8bytes_1
;; LOOP START
ld.ab r6, [r1, 4]
- prefetch [r1, 28] ;Prefetch the next read location
ld.ab r8, [r1,4]
- prefetchw [r3, 32] ;Prefetch the next write location
SHIFT_1 (r7, r6, 24)
or r7, r7, r5
@@ -162,9 +152,7 @@ ENTRY_CFI(memcpy)
lpnz @.Lcopy8bytes_2
;; LOOP START
ld.ab r6, [r1, 4]
- prefetch [r1, 28] ;Prefetch the next read location
ld.ab r8, [r1,4]
- prefetchw [r3, 32] ;Prefetch the next write location
SHIFT_1 (r7, r6, 16)
or r7, r7, r5
@@ -204,9 +192,7 @@ ENTRY_CFI(memcpy)
lpnz @.Lcopy8bytes_3
;; LOOP START
ld.ab r6, [r1, 4]
- prefetch [r1, 28] ;Prefetch the next read location
ld.ab r8, [r1,4]
- prefetchw [r3, 32] ;Prefetch the next write location
SHIFT_1 (r7, r6, 8)
or r7, r7, r5
--
2.14.5
As of today we enable unaligned access unconditionally on ARCv2.
Lets move it under Kconfig option so we can disable it in case of
using HW configuration which lacks of it.
Signed-off-by: Eugeniy Paltsev <[email protected]>
---
arch/arc/Kconfig | 8 ++++++++
arch/arc/include/asm/irqflags-arcv2.h | 4 ++++
arch/arc/kernel/head.S | 14 +++++++++-----
arch/arc/kernel/intc-arcv2.c | 2 +-
4 files changed, 22 insertions(+), 6 deletions(-)
diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig
index 376366a7db81..37c8aeefa3a5 100644
--- a/arch/arc/Kconfig
+++ b/arch/arc/Kconfig
@@ -387,6 +387,14 @@ config ARC_HAS_SWAPE
if ISA_ARCV2
+config ARC_USE_UNALIGNED_MEM_ACCESS
+ bool "Handle unaligned access in HW and use it"
+ default y
+ help
+ The ARC HS architecture supports unaligned memory access
+ which is disabled by default. Enable unaligned access in
+ hardware and use it in software.
+
config ARC_HAS_LL64
bool "Insn: 64bit LDD/STD"
help
diff --git a/arch/arc/include/asm/irqflags-arcv2.h b/arch/arc/include/asm/irqflags-arcv2.h
index 8a4f77ea3238..9b911e2c6b31 100644
--- a/arch/arc/include/asm/irqflags-arcv2.h
+++ b/arch/arc/include/asm/irqflags-arcv2.h
@@ -44,8 +44,12 @@
#define ARCV2_IRQ_DEF_PRIO 1
/* seed value for status register */
+#ifdef CONFIG_ARC_USE_UNALIGNED_MEM_ACCESS
#define ISA_INIT_STATUS_BITS (STATUS_IE_MASK | STATUS_AD_MASK | \
(ARCV2_IRQ_DEF_PRIO << 1))
+#else
+#define ISA_INIT_STATUS_BITS (STATUS_IE_MASK | (ARCV2_IRQ_DEF_PRIO << 1))
+#endif /* CONFIG_ARC_USE_UNALIGNED_MEM_ACCESS */
#ifndef __ASSEMBLY__
diff --git a/arch/arc/kernel/head.S b/arch/arc/kernel/head.S
index 25b3a247e11c..bd24ba0e0264 100644
--- a/arch/arc/kernel/head.S
+++ b/arch/arc/kernel/head.S
@@ -49,11 +49,15 @@
1:
-#ifdef CONFIG_ISA_ARCV2
- ; Enable unaligned access as disabled by default in hw while
- ; gcc 8.1.0 onwards (ARC GNU 2018.03) unconditionally generates
- ; code for unaligned accesses
- flag 1 << STATUS_AD_BIT
+ ; Enable / disable HW handling of unaligned access in the CPU.
+#ifdef CONFIG_ARC_USE_UNALIGNED_MEM_ACCESS
+ kflag STATUS_AD_MASK
+#else
+ ; Handling of unaligned access is disabled by default but we disable it
+ ; manually in case of any bootloader enabled it earlier.
+ lr r5, [ARC_REG_STATUS32]
+ bclr r5, r5, STATUS_AD_BIT
+ kflag r5
#endif
.endm
diff --git a/arch/arc/kernel/intc-arcv2.c b/arch/arc/kernel/intc-arcv2.c
index 067ea362fb3e..6359896da1ea 100644
--- a/arch/arc/kernel/intc-arcv2.c
+++ b/arch/arc/kernel/intc-arcv2.c
@@ -93,7 +93,7 @@ void arc_init_IRQ(void)
/* setup status32, don't enable intr yet as kernel doesn't want */
tmp = read_aux_reg(ARC_REG_STATUS32);
- tmp |= STATUS_AD_MASK | (ARCV2_IRQ_DEF_PRIO << 1);
+ tmp |= ARCV2_IRQ_DEF_PRIO << 1;
tmp &= ~STATUS_IE_MASK;
asm volatile("kflag %0 \n"::"r"(tmp));
}
--
2.14.5
Select HAVE_EFFICIENT_UNALIGNED_ACCESS and allow GCC to generate
unaligned data if we enable enable unaligned access in HW.
Signed-off-by: Eugeniy Paltsev <[email protected]>
---
arch/arc/Kconfig | 1 +
arch/arc/Makefile | 6 ++++++
2 files changed, 7 insertions(+)
diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig
index 37c8aeefa3a5..a1d976c612a6 100644
--- a/arch/arc/Kconfig
+++ b/arch/arc/Kconfig
@@ -390,6 +390,7 @@ if ISA_ARCV2
config ARC_USE_UNALIGNED_MEM_ACCESS
bool "Handle unaligned access in HW and use it"
default y
+ select HAVE_EFFICIENT_UNALIGNED_ACCESS
help
The ARC HS architecture supports unaligned memory access
which is disabled by default. Enable unaligned access in
diff --git a/arch/arc/Makefile b/arch/arc/Makefile
index df00578c279d..e2b991f75bc5 100644
--- a/arch/arc/Makefile
+++ b/arch/arc/Makefile
@@ -28,6 +28,12 @@ cflags-$(CONFIG_ARC_HAS_SWAPE) += -mswape
ifdef CONFIG_ISA_ARCV2
+ifdef CONFIG_ARC_USE_UNALIGNED_MEM_ACCESS
+cflags-y += -munaligned-access
+else
+cflags-y += -mno-unaligned-access
+endif
+
ifndef CONFIG_ARC_HAS_LL64
cflags-y += -mno-ll64
endif
--
2.14.5
This now prints both HW feature status (exists, enabled / disabled)
and SW status (used / not used).
Signed-off-by: Eugeniy Paltsev <[email protected]>
---
arch/arc/kernel/setup.c | 13 ++++++++++---
1 file changed, 10 insertions(+), 3 deletions(-)
diff --git a/arch/arc/kernel/setup.c b/arch/arc/kernel/setup.c
index feb90093e6b1..e465cae3f661 100644
--- a/arch/arc/kernel/setup.c
+++ b/arch/arc/kernel/setup.c
@@ -270,10 +270,17 @@ static char *arc_cpu_mumbojumbo(int cpu_id, char *buf, int len)
#ifdef __ARC_UNALIGNED__
ua = 1;
#endif
- n += i = scnprintf(buf + n, len - n, "%s%s%s%s%s%s",
+ n += i = scnprintf(buf + n, len - n, "%s%s%s%s",
IS_AVAIL2(cpu->isa.atomic, "atomic ", CONFIG_ARC_HAS_LLSC),
- IS_AVAIL2(cpu->isa.ldd, "ll64 ", CONFIG_ARC_HAS_LL64),
- IS_AVAIL1(cpu->isa.unalign, "unalign "), IS_USED_RUN(ua));
+ IS_AVAIL2(cpu->isa.ldd, "ll64 ", CONFIG_ARC_HAS_LL64));
+
+ if (cpu->isa.unalign) {
+ i += scnprintf(buf + n, len - n, "unalign (%s, %s)",
+ IS_ENABLED(CONFIG_ARC_USE_UNALIGNED_MEM_ACCESS) ?
+ "enabled" : "disabled",
+ ua ? "used" : "not used");
+ n += i;
+ }
if (i)
n += scnprintf(buf + n, len - n, "\n\t\t: ");
--
2.14.5
On 1/30/19 8:32 AM, Eugeniy Paltsev wrote:
> As of today we enable unaligned access unconditionally on ARCv2.
> Lets move it under Kconfig option so we can disable it in case of
> using HW configuration which lacks of it.
>
> Signed-off-by: Eugeniy Paltsev <[email protected]>
> ---
> arch/arc/Kconfig | 8 ++++++++
> arch/arc/include/asm/irqflags-arcv2.h | 4 ++++
> arch/arc/kernel/head.S | 14 +++++++++-----
> arch/arc/kernel/intc-arcv2.c | 2 +-
> 4 files changed, 22 insertions(+), 6 deletions(-)
>
> diff --git a/arch/arc/Kconfig b/arch/arc/Kconfig
> index 376366a7db81..37c8aeefa3a5 100644
> --- a/arch/arc/Kconfig
> +++ b/arch/arc/Kconfig
> @@ -387,6 +387,14 @@ config ARC_HAS_SWAPE
>
> if ISA_ARCV2
>
> +config ARC_USE_UNALIGNED_MEM_ACCESS
> + bool "Handle unaligned access in HW and use it"
> + default y
> + help
> + The ARC HS architecture supports unaligned memory access
> + which is disabled by default. Enable unaligned access in
> + hardware and use it in software.
> +
> config ARC_HAS_LL64
> bool "Insn: 64bit LDD/STD"
> help
> diff --git a/arch/arc/include/asm/irqflags-arcv2.h b/arch/arc/include/asm/irqflags-arcv2.h
> index 8a4f77ea3238..9b911e2c6b31 100644
> --- a/arch/arc/include/asm/irqflags-arcv2.h
> +++ b/arch/arc/include/asm/irqflags-arcv2.h
> @@ -44,8 +44,12 @@
> #define ARCV2_IRQ_DEF_PRIO 1
>
> /* seed value for status register */
> +#ifdef CONFIG_ARC_USE_UNALIGNED_MEM_ACCESS
> #define ISA_INIT_STATUS_BITS (STATUS_IE_MASK | STATUS_AD_MASK | \
> (ARCV2_IRQ_DEF_PRIO << 1))
> +#else
> +#define ISA_INIT_STATUS_BITS (STATUS_IE_MASK | (ARCV2_IRQ_DEF_PRIO << 1))
> +#endif /* CONFIG_ARC_USE_UNALIGNED_MEM_ACCESS */
>
> #ifndef __ASSEMBLY__
>
> diff --git a/arch/arc/kernel/head.S b/arch/arc/kernel/head.S
> index 25b3a247e11c..bd24ba0e0264 100644
> --- a/arch/arc/kernel/head.S
> +++ b/arch/arc/kernel/head.S
> @@ -49,11 +49,15 @@
>
> 1:
>
> -#ifdef CONFIG_ISA_ARCV2
> - ; Enable unaligned access as disabled by default in hw while
> - ; gcc 8.1.0 onwards (ARC GNU 2018.03) unconditionally generates
> - ; code for unaligned accesses
> - flag 1 << STATUS_AD_BIT
> + ; Enable / disable HW handling of unaligned access in the CPU.
> +#ifdef CONFIG_ARC_USE_UNALIGNED_MEM_ACCESS
> + kflag STATUS_AD_MASK
> +#else
> + ; Handling of unaligned access is disabled by default but we disable it
> + ; manually in case of any bootloader enabled it earlier.
> + lr r5, [ARC_REG_STATUS32]
> + bclr r5, r5, STATUS_AD_BIT
> + kflag r5
> #endif
> .endm
I tested these patches and we have a little problem when
CONFIG_ARC_USE_UNALIGNED_MEM_ACCESS is disabled. AD bit is cleared, but current
gcc (2018.09 release) generates unaligned accesses still.
Misaligned Access
Path: (null)
CPU: 0 PID: 0 Comm: swapper Not tainted 5.0.0-rc4+ #484
[ECR ]: 0x000d0000 => Check Programmer's Manual
[EFA ]: 0x808293cf
[BLINK ]: device_node_gen_full_name+0x3e/0xd4
[ERET ]: string.constprop.11+0x3e/0x6c
[STAT32]: 0x00000002 : K
BTA: 0x807e8768 SP: 0x808d9e94 FP: 0x00000000
LPS: 0x807ede44 LPE: 0x807ede54 LPC: 0x00000000
r00: 0x00000001 r01: 0x00000001 r02: 0x00000000
...
Stack Trace:
string.constprop.11+0x3e/0x6c
device_node_gen_full_name+0x3e/0xd4
device_node_string+0x128/0x32c
vsnprintf+0xfa/0x3c4
kvasprintf+0x24/0x78
kasprintf+0x16/0x1c
__irq_domain_add+0x72/0x1b8
init_onchip_IRQ+0x38/0x60
Guess there's no solution to this. We could force select the option in relevant
Kconfig, but this applies to pretty much every platform and defeats the purpose of
option in first place.
>
> diff --git a/arch/arc/kernel/intc-arcv2.c b/arch/arc/kernel/intc-arcv2.c
> index 067ea362fb3e..6359896da1ea 100644
> --- a/arch/arc/kernel/intc-arcv2.c
> +++ b/arch/arc/kernel/intc-arcv2.c
> @@ -93,7 +93,7 @@ void arc_init_IRQ(void)
>
> /* setup status32, don't enable intr yet as kernel doesn't want */
> tmp = read_aux_reg(ARC_REG_STATUS32);
> - tmp |= STATUS_AD_MASK | (ARCV2_IRQ_DEF_PRIO << 1);
> + tmp |= ARCV2_IRQ_DEF_PRIO << 1;
> tmp &= ~STATUS_IE_MASK;
> asm volatile("kflag %0 \n"::"r"(tmp));
> }
>
On 2/1/19 3:28 PM, Vineet Gupta wrote:
> Guess there's no solution to this. We could force select the option in relevant
> Kconfig, but this applies to pretty much every platform and defeats the purpose of
> option in first place.
Answering my own question:
#if defined(__ARC_UNALIGNED__) && !defined(CONFIG_ARC_USE_UNALIGNED_MEM_ACCESS)
BUILD_BUG_ON_MSG(1, "gcc doesn't support disabling unaligned accesses");
#endif
On 1/30/19 8:32 AM, Eugeniy Paltsev wrote:
> As of today we enable unaligned access unconditionally on ARCv2.
> Lets move it under Kconfig option and use it actively in SW if it is
> enabled.
>
> While I'm at it fix and optimise ARCv2 memcpy implementaion.
>
> Changes v1->v2:
> * Rebase onto last ARC changes.
> * Don't add dummy symbol to ARC Kconfig
> * Print info about unaligned access status in kernel log
>
> Eugeniy Paltsev (5):
> ARCv2: lib: memcpy: fix doing prefetchw outside of buffer
> ARCv2: introduce unaligned access under a Kconfig option
> ARCv2: use unaligned access in SW
> ARCv2: LIB: MEMCPY: fixed and optimised routine
> ARC: boot log: print unaligned memory access details
I made some mods, squashed some and pushed to for-next (it will land in next release)
Thx,
-vineet
>
> arch/arc/Kconfig | 9 +++++++
> arch/arc/Makefile | 6 +++++
> arch/arc/include/asm/irqflags-arcv2.h | 4 +++
> arch/arc/kernel/head.S | 14 +++++++----
> arch/arc/kernel/intc-arcv2.c | 2 +-
> arch/arc/kernel/setup.c | 13 +++++++---
> arch/arc/lib/Makefile | 8 +++++-
> arch/arc/lib/memcpy-archs-unaligned.S | 46 +++++++++++++++++++++++++++++++++++
> arch/arc/lib/memcpy-archs.S | 14 -----------
> 9 files changed, 92 insertions(+), 24 deletions(-)
> create mode 100644 arch/arc/lib/memcpy-archs-unaligned.S
>