Despite the architecture spec requiring that reserved registers in the GIC
distributor memory map are RES0 (and thus are not allowed to generate
an exception), the Cavium ThunderX (aka TX1) SoC explodes as such:
[ 0.000000] GICv3: GIC: Using split EOI/Deactivate mode
[ 0.000000] GICv3: 128 SPIs implemented
[ 0.000000] GICv3: 0 Extended SPIs implemented
[ 0.000000] Internal error: synchronous external abort: 96000210 [#1] SMP
[ 0.000000] Modules linked in:
[ 0.000000] CPU: 0 PID: 0 Comm: swapper/0 Not tainted 5.4.0-rc4-00035-g3cf6a3d5725f #7956
[ 0.000000] Hardware name: cavium,thunder-88xx (DT)
[ 0.000000] pstate: 60000085 (nZCv daIf -PAN -UAO)
[ 0.000000] pc : __raw_readl+0x0/0x8
[ 0.000000] lr : gic_init_bases+0x110/0x560
[ 0.000000] sp : ffff800011243d90
[ 0.000000] x29: ffff800011243d90 x28: 0000000000000000
[ 0.000000] x27: 0000000000000018 x26: 0000000000000002
[ 0.000000] x25: ffff8000116f0000 x24: ffff000fbe6a2c80
[ 0.000000] x23: 0000000000000000 x22: ffff010fdc322b68
[ 0.000000] x21: ffff800010a7a208 x20: 00000000009b0404
[ 0.000000] x19: ffff80001124dad0 x18: 0000000000000010
[ 0.000000] x17: 000000004d8d492b x16: 00000000f67eb9af
[ 0.000000] x15: ffffffffffffffff x14: ffff800011249908
[ 0.000000] x13: ffff800091243ae7 x12: ffff800011243af4
[ 0.000000] x11: ffff80001126e000 x10: ffff800011243a70
[ 0.000000] x9 : 00000000ffffffd0 x8 : ffff80001069c828
[ 0.000000] x7 : 0000000000000059 x6 : ffff8000113fb4d1
[ 0.000000] x5 : 0000000000000001 x4 : 0000000000000000
[ 0.000000] x3 : 0000000000000000 x2 : 0000000000000000
[ 0.000000] x1 : 0000000000000000 x0 : ffff8000116f000c
[ 0.000000] Call trace:
[ 0.000000] __raw_readl+0x0/0x8
[ 0.000000] gic_of_init+0x188/0x224
[ 0.000000] of_irq_init+0x200/0x3cc
[ 0.000000] irqchip_init+0x1c/0x40
[ 0.000000] init_IRQ+0x160/0x1d0
[ 0.000000] start_kernel+0x2ec/0x4b8
[ 0.000000] Code: a8c47bfd d65f03c0 d538d080 d65f03c0 (b9400000)
when reading the GICv4.1 GICD_TYPER2 register, which is unexpected...
Work around it by adding a new quirk for the following variants:
ThunderX: CN88xx
OCTEON TX: CN83xx, CN81xx
OCTEON TX2: CN93xx, CN96xx, CN98xx, CNF95xx*
and use this flag to avoid accessing GICD_TYPER2. Note that all
reserved registers (including redistributors and ITS) are impacted
by this erratum, but that only GICD_TYPER2 has to be worked around
so far.
Signed-off-by: Marc Zyngier <[email protected]>
Cc: Mark Salter <[email protected]>
Cc: Robert Richter <[email protected]>
---
This is a respin of [1], with the erratum number and affected
platform list provided by Robert.
[1] https://lore.kernel.org/lkml/[email protected]/
Documentation/arm64/silicon-errata.rst | 2 ++
drivers/irqchip/irq-gic-v3.c | 30 +++++++++++++++++++++++++-
2 files changed, 31 insertions(+), 1 deletion(-)
diff --git a/Documentation/arm64/silicon-errata.rst b/Documentation/arm64/silicon-errata.rst
index 9120e59578dc..2c08c628febd 100644
--- a/Documentation/arm64/silicon-errata.rst
+++ b/Documentation/arm64/silicon-errata.rst
@@ -110,6 +110,8 @@ stable kernels.
+----------------+-----------------+-----------------+-----------------------------+
| Cavium | ThunderX GICv3 | #23154 | CAVIUM_ERRATUM_23154 |
+----------------+-----------------+-----------------+-----------------------------+
+| Cavium | ThunderX GICv3 | #38539 | N/A |
++----------------+-----------------+-----------------+-----------------------------+
| Cavium | ThunderX Core | #27456 | CAVIUM_ERRATUM_27456 |
+----------------+-----------------+-----------------+-----------------------------+
| Cavium | ThunderX Core | #30115 | CAVIUM_ERRATUM_30115 |
diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c
index c1f7af9d9ae7..1eec9d4649d5 100644
--- a/drivers/irqchip/irq-gic-v3.c
+++ b/drivers/irqchip/irq-gic-v3.c
@@ -34,6 +34,7 @@
#define GICD_INT_NMI_PRI (GICD_INT_DEF_PRI & ~0x80)
#define FLAGS_WORKAROUND_GICR_WAKER_MSM8996 (1ULL << 0)
+#define FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539 (1ULL << 1)
struct redist_region {
void __iomem *redist_base;
@@ -1464,6 +1465,15 @@ static bool gic_enable_quirk_msm8996(void *data)
return true;
}
+static bool gic_enable_quirk_cavium_38539(void *data)
+{
+ struct gic_chip_data *d = data;
+
+ d->flags |= FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539;
+
+ return true;
+}
+
static bool gic_enable_quirk_hip06_07(void *data)
{
struct gic_chip_data *d = data;
@@ -1502,6 +1512,19 @@ static const struct gic_quirk gic_quirks[] = {
.mask = 0xffffffff,
.init = gic_enable_quirk_hip06_07,
},
+ {
+ /*
+ * Reserved register accesses generate a Synchronous
+ * External Abort. This erratum applies to:
+ * - ThunderX: CN88xx
+ * - OCTEON TX: CN83xx, CN81xx
+ * - OCTEON TX2: CN93xx, CN96xx, CN98xx, CNF95xx*
+ */
+ .desc = "GICv3: Cavium erratum 38539",
+ .iidr = 0xa000034c,
+ .mask = 0xe8f00fff,
+ .init = gic_enable_quirk_cavium_38539,
+ },
{
}
};
@@ -1577,7 +1600,12 @@ static int __init gic_init_bases(void __iomem *dist_base,
pr_info("%d SPIs implemented\n", GIC_LINE_NR - 32);
pr_info("%d Extended SPIs implemented\n", GIC_ESPI_NR);
- gic_data.rdists.gicd_typer2 = readl_relaxed(gic_data.dist_base + GICD_TYPER2);
+ /*
+ * ThunderX1 explodes on reading GICD_TYPER2, in violation of the
+ * architecture spec (which says that reserved registers are RES0).
+ */
+ if (!(gic_data.flags & FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539))
+ gic_data.rdists.gicd_typer2 = readl_relaxed(gic_data.dist_base + GICD_TYPER2);
gic_data.domain = irq_domain_create_tree(handle, &gic_irq_domain_ops,
&gic_data);
--
2.20.1
On Wed, Mar 11, 2020 at 11:56:49AM +0000, Marc Zyngier wrote:
> Despite the architecture spec requiring that reserved registers in the GIC
> distributor memory map are RES0 (and thus are not allowed to generate
> an exception), the Cavium ThunderX (aka TX1) SoC explodes as such:
>
> [ 0.000000] GICv3: GIC: Using split EOI/Deactivate mode
> [ 0.000000] GICv3: 128 SPIs implemented
> [ 0.000000] GICv3: 0 Extended SPIs implemented
> [ 0.000000] Internal error: synchronous external abort: 96000210 [#1] SMP
> [ 0.000000] Modules linked in:
> [ 0.000000] CPU: 0 PID: 0 Comm: swapper/0 Not tainted 5.4.0-rc4-00035-g3cf6a3d5725f #7956
> [ 0.000000] Hardware name: cavium,thunder-88xx (DT)
> [ 0.000000] pstate: 60000085 (nZCv daIf -PAN -UAO)
> [ 0.000000] pc : __raw_readl+0x0/0x8
> [ 0.000000] lr : gic_init_bases+0x110/0x560
> [ 0.000000] sp : ffff800011243d90
> [ 0.000000] x29: ffff800011243d90 x28: 0000000000000000
> [ 0.000000] x27: 0000000000000018 x26: 0000000000000002
> [ 0.000000] x25: ffff8000116f0000 x24: ffff000fbe6a2c80
> [ 0.000000] x23: 0000000000000000 x22: ffff010fdc322b68
> [ 0.000000] x21: ffff800010a7a208 x20: 00000000009b0404
> [ 0.000000] x19: ffff80001124dad0 x18: 0000000000000010
> [ 0.000000] x17: 000000004d8d492b x16: 00000000f67eb9af
> [ 0.000000] x15: ffffffffffffffff x14: ffff800011249908
> [ 0.000000] x13: ffff800091243ae7 x12: ffff800011243af4
> [ 0.000000] x11: ffff80001126e000 x10: ffff800011243a70
> [ 0.000000] x9 : 00000000ffffffd0 x8 : ffff80001069c828
> [ 0.000000] x7 : 0000000000000059 x6 : ffff8000113fb4d1
> [ 0.000000] x5 : 0000000000000001 x4 : 0000000000000000
> [ 0.000000] x3 : 0000000000000000 x2 : 0000000000000000
> [ 0.000000] x1 : 0000000000000000 x0 : ffff8000116f000c
> [ 0.000000] Call trace:
> [ 0.000000] __raw_readl+0x0/0x8
> [ 0.000000] gic_of_init+0x188/0x224
> [ 0.000000] of_irq_init+0x200/0x3cc
> [ 0.000000] irqchip_init+0x1c/0x40
> [ 0.000000] init_IRQ+0x160/0x1d0
> [ 0.000000] start_kernel+0x2ec/0x4b8
> [ 0.000000] Code: a8c47bfd d65f03c0 d538d080 d65f03c0 (b9400000)
>
> when reading the GICv4.1 GICD_TYPER2 register, which is unexpected...
>
> Work around it by adding a new quirk for the following variants:
>
> ThunderX: CN88xx
> OCTEON TX: CN83xx, CN81xx
> OCTEON TX2: CN93xx, CN96xx, CN98xx, CNF95xx*
>
> and use this flag to avoid accessing GICD_TYPER2. Note that all
> reserved registers (including redistributors and ITS) are impacted
> by this erratum, but that only GICD_TYPER2 has to be worked around
> so far.
>
> Signed-off-by: Marc Zyngier <[email protected]>
> Cc: Mark Salter <[email protected]>
> Cc: Robert Richter <[email protected]>
> ---
> This is a respin of [1], with the erratum number and affected
> platform list provided by Robert.
>
> [1] https://lore.kernel.org/lkml/[email protected]/
>
> Documentation/arm64/silicon-errata.rst | 2 ++
For the arm64 documentation:
Acked-by: Catalin Marinas <[email protected]>
On Wed, 2020-03-11 at 11:56 +0000, Marc Zyngier wrote:
> Despite the architecture spec requiring that reserved registers in the GIC
> distributor memory map are RES0 (and thus are not allowed to generate
> an exception), the Cavium ThunderX (aka TX1) SoC explodes as such:
>
> [ 0.000000] GICv3: GIC: Using split EOI/Deactivate mode
> [ 0.000000] GICv3: 128 SPIs implemented
> [ 0.000000] GICv3: 0 Extended SPIs implemented
> [ 0.000000] Internal error: synchronous external abort: 96000210 [#1] SMP
> [ 0.000000] Modules linked in:
> [ 0.000000] CPU: 0 PID: 0 Comm: swapper/0 Not tainted 5.4.0-rc4-00035-g3cf6a3d5725f #7956
> [ 0.000000] Hardware name: cavium,thunder-88xx (DT)
> [ 0.000000] pstate: 60000085 (nZCv daIf -PAN -UAO)
> [ 0.000000] pc : __raw_readl+0x0/0x8
> [ 0.000000] lr : gic_init_bases+0x110/0x560
> [ 0.000000] sp : ffff800011243d90
> [ 0.000000] x29: ffff800011243d90 x28: 0000000000000000
> [ 0.000000] x27: 0000000000000018 x26: 0000000000000002
> [ 0.000000] x25: ffff8000116f0000 x24: ffff000fbe6a2c80
> [ 0.000000] x23: 0000000000000000 x22: ffff010fdc322b68
> [ 0.000000] x21: ffff800010a7a208 x20: 00000000009b0404
> [ 0.000000] x19: ffff80001124dad0 x18: 0000000000000010
> [ 0.000000] x17: 000000004d8d492b x16: 00000000f67eb9af
> [ 0.000000] x15: ffffffffffffffff x14: ffff800011249908
> [ 0.000000] x13: ffff800091243ae7 x12: ffff800011243af4
> [ 0.000000] x11: ffff80001126e000 x10: ffff800011243a70
> [ 0.000000] x9 : 00000000ffffffd0 x8 : ffff80001069c828
> [ 0.000000] x7 : 0000000000000059 x6 : ffff8000113fb4d1
> [ 0.000000] x5 : 0000000000000001 x4 : 0000000000000000
> [ 0.000000] x3 : 0000000000000000 x2 : 0000000000000000
> [ 0.000000] x1 : 0000000000000000 x0 : ffff8000116f000c
> [ 0.000000] Call trace:
> [ 0.000000] __raw_readl+0x0/0x8
> [ 0.000000] gic_of_init+0x188/0x224
> [ 0.000000] of_irq_init+0x200/0x3cc
> [ 0.000000] irqchip_init+0x1c/0x40
> [ 0.000000] init_IRQ+0x160/0x1d0
> [ 0.000000] start_kernel+0x2ec/0x4b8
> [ 0.000000] Code: a8c47bfd d65f03c0 d538d080 d65f03c0 (b9400000)
>
> when reading the GICv4.1 GICD_TYPER2 register, which is unexpected...
>
> Work around it by adding a new quirk for the following variants:
>
> ThunderX: CN88xx
> OCTEON TX: CN83xx, CN81xx
> OCTEON TX2: CN93xx, CN96xx, CN98xx, CNF95xx*
>
> and use this flag to avoid accessing GICD_TYPER2. Note that all
> reserved registers (including redistributors and ITS) are impacted
> by this erratum, but that only GICD_TYPER2 has to be worked around
> so far.
>
> Signed-off-by: Marc Zyngier <[email protected]>
> Cc: Mark Salter <[email protected]>
> Cc: Robert Richter <[email protected]>
> ---
> This is a respin of [1], with the erratum number and affected
> platform list provided by Robert.
>
> [1] https://lore.kernel.org/lkml/[email protected]/
Tested-by: Mark Salter <[email protected]>
This resolves failure to boot on OcteonTX (TX1).
Fixes: f2d8340 ("irqchip/gic-v3: Add GICv4.1 VPEID size discovery")
Tested-by: Tim Harvey <[email protected]>
Tim
Tim,
On 2020-03-12 16:56, Tim Harvey wrote:
> This resolves failure to boot on OcteonTX (TX1).
>
> Fixes: f2d8340 ("irqchip/gic-v3: Add GICv4.1 VPEID size discovery")
I disagree here. There's nothing wrong with that initial patch,
and the bug is squarely with the hardware.
> Tested-by: Tim Harvey <[email protected]>
Thanks for that. I'll try and send a pull request tomorrow.
M.
--
Jazz is not dead. It just smells funny...
On 11.03.20 11:56:49, Marc Zyngier wrote:
> Despite the architecture spec requiring that reserved registers in the GIC
> distributor memory map are RES0 (and thus are not allowed to generate
> an exception), the Cavium ThunderX (aka TX1) SoC explodes as such:
>
> [ 0.000000] GICv3: GIC: Using split EOI/Deactivate mode
> [ 0.000000] GICv3: 128 SPIs implemented
> [ 0.000000] GICv3: 0 Extended SPIs implemented
> [ 0.000000] Internal error: synchronous external abort: 96000210 [#1] SMP
> [ 0.000000] Modules linked in:
> [ 0.000000] CPU: 0 PID: 0 Comm: swapper/0 Not tainted 5.4.0-rc4-00035-g3cf6a3d5725f #7956
> [ 0.000000] Hardware name: cavium,thunder-88xx (DT)
> [ 0.000000] pstate: 60000085 (nZCv daIf -PAN -UAO)
> [ 0.000000] pc : __raw_readl+0x0/0x8
> [ 0.000000] lr : gic_init_bases+0x110/0x560
> [ 0.000000] sp : ffff800011243d90
> [ 0.000000] x29: ffff800011243d90 x28: 0000000000000000
> [ 0.000000] x27: 0000000000000018 x26: 0000000000000002
> [ 0.000000] x25: ffff8000116f0000 x24: ffff000fbe6a2c80
> [ 0.000000] x23: 0000000000000000 x22: ffff010fdc322b68
> [ 0.000000] x21: ffff800010a7a208 x20: 00000000009b0404
> [ 0.000000] x19: ffff80001124dad0 x18: 0000000000000010
> [ 0.000000] x17: 000000004d8d492b x16: 00000000f67eb9af
> [ 0.000000] x15: ffffffffffffffff x14: ffff800011249908
> [ 0.000000] x13: ffff800091243ae7 x12: ffff800011243af4
> [ 0.000000] x11: ffff80001126e000 x10: ffff800011243a70
> [ 0.000000] x9 : 00000000ffffffd0 x8 : ffff80001069c828
> [ 0.000000] x7 : 0000000000000059 x6 : ffff8000113fb4d1
> [ 0.000000] x5 : 0000000000000001 x4 : 0000000000000000
> [ 0.000000] x3 : 0000000000000000 x2 : 0000000000000000
> [ 0.000000] x1 : 0000000000000000 x0 : ffff8000116f000c
> [ 0.000000] Call trace:
> [ 0.000000] __raw_readl+0x0/0x8
> [ 0.000000] gic_of_init+0x188/0x224
> [ 0.000000] of_irq_init+0x200/0x3cc
> [ 0.000000] irqchip_init+0x1c/0x40
> [ 0.000000] init_IRQ+0x160/0x1d0
> [ 0.000000] start_kernel+0x2ec/0x4b8
> [ 0.000000] Code: a8c47bfd d65f03c0 d538d080 d65f03c0 (b9400000)
>
> when reading the GICv4.1 GICD_TYPER2 register, which is unexpected...
>
> Work around it by adding a new quirk for the following variants:
>
> ThunderX: CN88xx
> OCTEON TX: CN83xx, CN81xx
> OCTEON TX2: CN93xx, CN96xx, CN98xx, CNF95xx*
>
> and use this flag to avoid accessing GICD_TYPER2. Note that all
> reserved registers (including redistributors and ITS) are impacted
> by this erratum, but that only GICD_TYPER2 has to be worked around
> so far.
>
> Signed-off-by: Marc Zyngier <[email protected]>
> Cc: Mark Salter <[email protected]>
> Cc: Robert Richter <[email protected]>
Tested on a Cavium Thunder cn88xx.
Acked-by: Robert Richter <[email protected]>
Tested-by: Robert Richter <[email protected]>
Thanks,
-Robert