Impact: Eliminates an issue that can leave the system in an
unusable state.
This patch addresses an issue where device generated IRQs
are no longer seen by the kernel following IRQ affinity
migration while the device is generating IRQs at a high rate.
We have seen this problem happen when IRQ affinities are
adjusted in response to CPU offlining but I believe it
could also happen in during user initiated IRQ affinity
changes unrelated to CPU offlining. e.g. while the
irqbalance daemon is adjusting IRQ affinities when the
system is heavily loaded.
I have been able to consistently reproduce the problem on
some of our systems by running the following script (VICTIM_IRQ
specifies the IRQ for the aic94xx device) while a single instance
of the command
# while true; do find / -exec file {} \;; done
is keeping the filesystem activity and IRQ rate reasonably high.
#!/bin/sh
SYS_CPU_DIR=/sys/devices/system/cpu
VICTIM_IRQ=25
IRQ_MASK=f0
iteration=0
while true; do
echo $iteration
echo $IRQ_MASK > /proc/irq/$VICTIM_IRQ/smp_affinity
for cpudir in $SYS_CPU_DIR/cpu[1-9] $SYS_CPU_DIR/cpu??; do
echo 0 > $cpudir/online
done
for cpudir in $SYS_CPU_DIR/cpu[1-9] $SYS_CPU_DIR/cpu??; do
echo 1 > $cpudir/online
done
iteration=`expr $iteration + 1`
done
The root cause is a known issue already addressed for some
code paths [e.g. ack_apic_level() and the now obsolete
migrate_irq_remapped_level_desc()] where the ioapic can
misbehave when the I/O redirection table register is written
while the Remote IRR bit is set.
The proposed fix uses the same avoidance method and much
of same code that the Interrupt Remapping code previously
used to avoid the same problem.
Signed-off-by: Gary Hade <[email protected]>
---
arch/x86/kernel/apic/io_apic.c | 72 ++++++++++++++++++++++++++++++-
1 file changed, 71 insertions(+), 1 deletion(-)
Index: linux-2.6.30-rc1/arch/x86/kernel/apic/io_apic.c
===================================================================
--- linux-2.6.30-rc1.orig/arch/x86/kernel/apic/io_apic.c 2009-04-08 09:24:11.000000000 -0700
+++ linux-2.6.30-rc1/arch/x86/kernel/apic/io_apic.c 2009-04-08 09:24:23.000000000 -0700
@@ -2331,7 +2331,8 @@ set_desc_affinity(struct irq_desc *desc,
}
static void
-set_ioapic_affinity_irq_desc(struct irq_desc *desc, const struct cpumask *mask)
+set_ioapic_irq_affinity_desc(struct irq_desc *desc,
+ const struct cpumask *mask)
{
struct irq_cfg *cfg;
unsigned long flags;
@@ -2352,6 +2353,75 @@ set_ioapic_affinity_irq_desc(struct irq_
}
static void
+delayed_irq_move(struct work_struct *work)
+{
+ unsigned int irq;
+ struct irq_desc *desc;
+
+ for_each_irq_desc(irq, desc) {
+ if (desc->status & IRQ_MOVE_PENDING) {
+ unsigned long flags;
+
+ spin_lock_irqsave(&desc->lock, flags);
+ if (!desc->chip->set_affinity ||
+ !(desc->status & IRQ_MOVE_PENDING)) {
+ desc->status &= ~IRQ_MOVE_PENDING;
+ spin_unlock_irqrestore(&desc->lock, flags);
+ continue;
+ }
+
+ desc->chip->set_affinity(irq, desc->pending_mask);
+ spin_unlock_irqrestore(&desc->lock, flags);
+ }
+ }
+}
+
+static DECLARE_DELAYED_WORK(delayed_irq_move_work, delayed_irq_move);
+
+static void
+set_ioapic_irq_affinity_level_desc(struct irq_desc *desc)
+{
+
+ struct irq_cfg *cfg = desc->chip_data;
+
+ mask_IO_APIC_irq_desc(desc);
+
+ if (io_apic_level_ack_pending(cfg)) {
+ /*
+ * Interrupt in progress. Migrating irq now will change
+ * the vector information in the IO-APIC RTE which will
+ * confuse the EOI broadcast performed by cpu.
+ * So, we delay the irq migration.
+ */
+ schedule_delayed_work(&delayed_irq_move_work, 1);
+ goto unmask;
+ }
+
+ /* Interrupt not in progress. we can change the vector
+ * information in the IO-APIC RTE. */
+ set_ioapic_irq_affinity_desc(desc, desc->pending_mask);
+
+ desc->status &= ~IRQ_MOVE_PENDING;
+ cpumask_clear(desc->pending_mask);
+
+unmask:
+ unmask_IO_APIC_irq_desc(desc);
+}
+
+static void
+set_ioapic_affinity_irq_desc(struct irq_desc *desc,
+ const struct cpumask *mask)
+{
+ if (desc->status & IRQ_LEVEL) {
+ desc->status |= IRQ_MOVE_PENDING;
+ cpumask_copy(desc->pending_mask, mask);
+ set_ioapic_irq_affinity_level_desc(desc);
+ return;
+ }
+ set_ioapic_irq_affinity_desc(desc, mask);
+}
+
+static void
set_ioapic_affinity_irq(unsigned int irq, const struct cpumask *mask)
{
struct irq_desc *desc;
On Wed, Apr 8, 2009 at 2:07 PM, Gary Hade <[email protected]> wrote:
> Impact: Eliminates an issue that can leave the system in an
> ? ? ? ?unusable state.
>
> This patch addresses an issue where device generated IRQs
> are no longer seen by the kernel following IRQ affinity
> migration while the device is generating IRQs at a high rate.
> We have seen this problem happen when IRQ affinities are
> adjusted in response to CPU offlining but I believe it
> could also happen in during user initiated IRQ affinity
> changes unrelated to CPU offlining. e.g. while the
> irqbalance daemon is adjusting IRQ affinities when the
> system is heavily loaded.
>
> I have been able to consistently reproduce the problem on
> some of our systems by running the following script (VICTIM_IRQ
> specifies the IRQ for the aic94xx device) while a single instance
> of the command
> ?# while true; do find / -exec file {} \;; done
> is keeping the filesystem activity and IRQ rate reasonably high.
>
> #!/bin/sh
>
> SYS_CPU_DIR=/sys/devices/system/cpu
> VICTIM_IRQ=25
> IRQ_MASK=f0
>
> iteration=0
> while true; do
> ?echo $iteration
> ?echo $IRQ_MASK > /proc/irq/$VICTIM_IRQ/smp_affinity
> ?for cpudir in $SYS_CPU_DIR/cpu[1-9] $SYS_CPU_DIR/cpu??; do
> ? ?echo 0 > $cpudir/online
> ?done
> ?for cpudir in $SYS_CPU_DIR/cpu[1-9] $SYS_CPU_DIR/cpu??; do
> ? ?echo 1 > $cpudir/online
> ?done
> ?iteration=`expr $iteration + 1`
> done
>
> The root cause is a known issue already addressed for some
> code paths [e.g. ack_apic_level() and the now obsolete
> migrate_irq_remapped_level_desc()] where the ioapic can
> misbehave when the I/O redirection table register is written
> while the Remote IRR bit is set.
>
> The proposed fix uses the same avoidance method and much
> of same code that the Interrupt Remapping code previously
> used to avoid the same problem.
>
> Signed-off-by: Gary Hade <[email protected]>
>
> ---
> ?arch/x86/kernel/apic/io_apic.c | ? 72 ++++++++++++++++++++++++++++++-
> ?1 file changed, 71 insertions(+), 1 deletion(-)
>
> Index: linux-2.6.30-rc1/arch/x86/kernel/apic/io_apic.c
> ===================================================================
> --- linux-2.6.30-rc1.orig/arch/x86/kernel/apic/io_apic.c ? ? ? ?2009-04-08 09:24:11.000000000 -0700
> +++ linux-2.6.30-rc1/arch/x86/kernel/apic/io_apic.c ? ? 2009-04-08 09:24:23.000000000 -0700
> @@ -2331,7 +2331,8 @@ set_desc_affinity(struct irq_desc *desc,
> ?}
>
> ?static void
> -set_ioapic_affinity_irq_desc(struct irq_desc *desc, const struct cpumask *mask)
> +set_ioapic_irq_affinity_desc(struct irq_desc *desc,
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ?const struct cpumask *mask)
> ?{
> ? ? ? ?struct irq_cfg *cfg;
> ? ? ? ?unsigned long flags;
> @@ -2352,6 +2353,75 @@ set_ioapic_affinity_irq_desc(struct irq_
> ?}
>
> ?static void
> +delayed_irq_move(struct work_struct *work)
> +{
> + ? ? ? unsigned int irq;
> + ? ? ? struct irq_desc *desc;
> +
> + ? ? ? for_each_irq_desc(irq, desc) {
> + ? ? ? ? ? ? ? if (desc->status & IRQ_MOVE_PENDING) {
> + ? ? ? ? ? ? ? ? ? ? ? unsigned long flags;
> +
> + ? ? ? ? ? ? ? ? ? ? ? spin_lock_irqsave(&desc->lock, flags);
> + ? ? ? ? ? ? ? ? ? ? ? if (!desc->chip->set_affinity ||
> + ? ? ? ? ? ? ? ? ? ? ? ? ? !(desc->status & IRQ_MOVE_PENDING)) {
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? desc->status &= ~IRQ_MOVE_PENDING;
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? spin_unlock_irqrestore(&desc->lock, flags);
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? continue;
> + ? ? ? ? ? ? ? ? ? ? ? }
> +
> + ? ? ? ? ? ? ? ? ? ? ? desc->chip->set_affinity(irq, desc->pending_mask);
> + ? ? ? ? ? ? ? ? ? ? ? spin_unlock_irqrestore(&desc->lock, flags);
> + ? ? ? ? ? ? ? }
> + ? ? ? }
> +}
> +
> +static DECLARE_DELAYED_WORK(delayed_irq_move_work, delayed_irq_move);
> +
> +static void
> +set_ioapic_irq_affinity_level_desc(struct irq_desc *desc)
> +{
> +
> + ? ? ? struct irq_cfg *cfg = desc->chip_data;
> +
> + ? ? ? mask_IO_APIC_irq_desc(desc);
> +
> + ? ? ? if (io_apic_level_ack_pending(cfg)) {
> + ? ? ? ? ? ? ? /*
> + ? ? ? ? ? ? ? ?* Interrupt in progress. Migrating irq now will change
> + ? ? ? ? ? ? ? ?* the vector information in the IO-APIC RTE which will
> + ? ? ? ? ? ? ? ?* confuse the EOI broadcast performed by cpu.
> + ? ? ? ? ? ? ? ?* So, we delay the irq migration.
> + ? ? ? ? ? ? ? ?*/
> + ? ? ? ? ? ? ? schedule_delayed_work(&delayed_irq_move_work, 1);
> + ? ? ? ? ? ? ? goto unmask;
> + ? ? ? }
> +
> + ? ? ? /* Interrupt not in progress. we can change the vector
> + ? ? ? ?* information in the IO-APIC RTE. */
> + ? ? ? set_ioapic_irq_affinity_desc(desc, desc->pending_mask);
> +
> + ? ? ? desc->status &= ~IRQ_MOVE_PENDING;
> + ? ? ? cpumask_clear(desc->pending_mask);
> +
> +unmask:
> + ? ? ? unmask_IO_APIC_irq_desc(desc);
> +}
> +
> +static void
> +set_ioapic_affinity_irq_desc(struct irq_desc *desc,
> + ? ? ? ? ? ? ? ? ? ? ? ? ? ?const struct cpumask *mask)
> +{
> + ? ? ? if (desc->status & IRQ_LEVEL) {
> + ? ? ? ? ? ? ? desc->status |= IRQ_MOVE_PENDING;
> + ? ? ? ? ? ? ? cpumask_copy(desc->pending_mask, mask);
> + ? ? ? ? ? ? ? set_ioapic_irq_affinity_level_desc(desc);
> + ? ? ? ? ? ? ? return;
> + ? ? ? }
> + ? ? ? set_ioapic_irq_affinity_desc(desc, mask);
> +}
> +
> +static void
> ?set_ioapic_affinity_irq(unsigned int irq, const struct cpumask *mask)
> ?{
> ? ? ? ?struct irq_desc *desc;
> --
it seems, ack_apic_level() already checked io_apic_level_ack_pending()
cfg = desc->chip_data;
if (!io_apic_level_ack_pending(cfg))
move_masked_irq(irq);
unmask_IO_APIC_irq_desc(desc);
also calling path without irremap - for level triggered.
1. user use /proc/irq/xx/smp_affinity to set affinity
2. kernel/irq/manager.c::irq_set_affinity() is called, and
desc->pending_mask is changed
3a. one irq is arrived: arch/x86/kernel/irq.c::do_IRQ() is called,
hand_irq==>desc->handle_irq
==>kernel/irq/chip.c::handle_fasteoi_irq() ==> call desc->chip->eoi()
3b. desc->chip->eoi() aka
arch/x86/kernel/apic/io_apic.c::ack_apic_level() will call
mask_IO_APIC_irq_desc() and kernel/irq/migration.c::move_masked_irq()
3c. move_masked_irq will call desc->set_affinity(), vector and target
in io_apic controller and IDT are changed
desc->affinity is changed at last. ( for desc moving,
move_in_progress and extra_move flag is set too.)
3d then ack_apic_level() will unmask the irq.
3e. finished other call in handle_fasteoi_irq()
YH
On Wed, Apr 08, 2009 at 03:03:49PM -0700, Yinghai Lu wrote:
> On Wed, Apr 8, 2009 at 2:07 PM, Gary Hade <[email protected]> wrote:
> > Impact: Eliminates an issue that can leave the system in an
> > ? ? ? ?unusable state.
> >
> > This patch addresses an issue where device generated IRQs
> > are no longer seen by the kernel following IRQ affinity
> > migration while the device is generating IRQs at a high rate.
> > We have seen this problem happen when IRQ affinities are
> > adjusted in response to CPU offlining but I believe it
> > could also happen in during user initiated IRQ affinity
> > changes unrelated to CPU offlining. e.g. while the
> > irqbalance daemon is adjusting IRQ affinities when the
> > system is heavily loaded.
> >
> > I have been able to consistently reproduce the problem on
> > some of our systems by running the following script (VICTIM_IRQ
> > specifies the IRQ for the aic94xx device) while a single instance
> > of the command
> > ?# while true; do find / -exec file {} \;; done
> > is keeping the filesystem activity and IRQ rate reasonably high.
> >
> > #!/bin/sh
> >
> > SYS_CPU_DIR=/sys/devices/system/cpu
> > VICTIM_IRQ=25
> > IRQ_MASK=f0
> >
> > iteration=0
> > while true; do
> > ?echo $iteration
> > ?echo $IRQ_MASK > /proc/irq/$VICTIM_IRQ/smp_affinity
> > ?for cpudir in $SYS_CPU_DIR/cpu[1-9] $SYS_CPU_DIR/cpu??; do
> > ? ?echo 0 > $cpudir/online
> > ?done
> > ?for cpudir in $SYS_CPU_DIR/cpu[1-9] $SYS_CPU_DIR/cpu??; do
> > ? ?echo 1 > $cpudir/online
> > ?done
> > ?iteration=`expr $iteration + 1`
> > done
> >
> > The root cause is a known issue already addressed for some
> > code paths [e.g. ack_apic_level() and the now obsolete
> > migrate_irq_remapped_level_desc()] where the ioapic can
> > misbehave when the I/O redirection table register is written
> > while the Remote IRR bit is set.
> >
> > The proposed fix uses the same avoidance method and much
> > of same code that the Interrupt Remapping code previously
> > used to avoid the same problem.
> >
> > Signed-off-by: Gary Hade <[email protected]>
> >
> > ---
> > ?arch/x86/kernel/apic/io_apic.c | ? 72 ++++++++++++++++++++++++++++++-
> > ?1 file changed, 71 insertions(+), 1 deletion(-)
> >
> > Index: linux-2.6.30-rc1/arch/x86/kernel/apic/io_apic.c
> > ===================================================================
> > --- linux-2.6.30-rc1.orig/arch/x86/kernel/apic/io_apic.c ? ? ? ?2009-04-08 09:24:11.000000000 -0700
> > +++ linux-2.6.30-rc1/arch/x86/kernel/apic/io_apic.c ? ? 2009-04-08 09:24:23.000000000 -0700
> > @@ -2331,7 +2331,8 @@ set_desc_affinity(struct irq_desc *desc,
> > ?}
> >
> > ?static void
> > -set_ioapic_affinity_irq_desc(struct irq_desc *desc, const struct cpumask *mask)
> > +set_ioapic_irq_affinity_desc(struct irq_desc *desc,
> > + ? ? ? ? ? ? ? ? ? ? ? ? ? ?const struct cpumask *mask)
> > ?{
> > ? ? ? ?struct irq_cfg *cfg;
> > ? ? ? ?unsigned long flags;
> > @@ -2352,6 +2353,75 @@ set_ioapic_affinity_irq_desc(struct irq_
> > ?}
> >
> > ?static void
> > +delayed_irq_move(struct work_struct *work)
> > +{
> > + ? ? ? unsigned int irq;
> > + ? ? ? struct irq_desc *desc;
> > +
> > + ? ? ? for_each_irq_desc(irq, desc) {
> > + ? ? ? ? ? ? ? if (desc->status & IRQ_MOVE_PENDING) {
> > + ? ? ? ? ? ? ? ? ? ? ? unsigned long flags;
> > +
> > + ? ? ? ? ? ? ? ? ? ? ? spin_lock_irqsave(&desc->lock, flags);
> > + ? ? ? ? ? ? ? ? ? ? ? if (!desc->chip->set_affinity ||
> > + ? ? ? ? ? ? ? ? ? ? ? ? ? !(desc->status & IRQ_MOVE_PENDING)) {
> > + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? desc->status &= ~IRQ_MOVE_PENDING;
> > + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? spin_unlock_irqrestore(&desc->lock, flags);
> > + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? continue;
> > + ? ? ? ? ? ? ? ? ? ? ? }
> > +
> > + ? ? ? ? ? ? ? ? ? ? ? desc->chip->set_affinity(irq, desc->pending_mask);
> > + ? ? ? ? ? ? ? ? ? ? ? spin_unlock_irqrestore(&desc->lock, flags);
> > + ? ? ? ? ? ? ? }
> > + ? ? ? }
> > +}
> > +
> > +static DECLARE_DELAYED_WORK(delayed_irq_move_work, delayed_irq_move);
> > +
> > +static void
> > +set_ioapic_irq_affinity_level_desc(struct irq_desc *desc)
> > +{
> > +
> > + ? ? ? struct irq_cfg *cfg = desc->chip_data;
> > +
> > + ? ? ? mask_IO_APIC_irq_desc(desc);
> > +
> > + ? ? ? if (io_apic_level_ack_pending(cfg)) {
> > + ? ? ? ? ? ? ? /*
> > + ? ? ? ? ? ? ? ?* Interrupt in progress. Migrating irq now will change
> > + ? ? ? ? ? ? ? ?* the vector information in the IO-APIC RTE which will
> > + ? ? ? ? ? ? ? ?* confuse the EOI broadcast performed by cpu.
> > + ? ? ? ? ? ? ? ?* So, we delay the irq migration.
> > + ? ? ? ? ? ? ? ?*/
> > + ? ? ? ? ? ? ? schedule_delayed_work(&delayed_irq_move_work, 1);
> > + ? ? ? ? ? ? ? goto unmask;
> > + ? ? ? }
> > +
> > + ? ? ? /* Interrupt not in progress. we can change the vector
> > + ? ? ? ?* information in the IO-APIC RTE. */
> > + ? ? ? set_ioapic_irq_affinity_desc(desc, desc->pending_mask);
> > +
> > + ? ? ? desc->status &= ~IRQ_MOVE_PENDING;
> > + ? ? ? cpumask_clear(desc->pending_mask);
> > +
> > +unmask:
> > + ? ? ? unmask_IO_APIC_irq_desc(desc);
> > +}
> > +
> > +static void
> > +set_ioapic_affinity_irq_desc(struct irq_desc *desc,
> > + ? ? ? ? ? ? ? ? ? ? ? ? ? ?const struct cpumask *mask)
> > +{
> > + ? ? ? if (desc->status & IRQ_LEVEL) {
> > + ? ? ? ? ? ? ? desc->status |= IRQ_MOVE_PENDING;
> > + ? ? ? ? ? ? ? cpumask_copy(desc->pending_mask, mask);
> > + ? ? ? ? ? ? ? set_ioapic_irq_affinity_level_desc(desc);
> > + ? ? ? ? ? ? ? return;
> > + ? ? ? }
> > + ? ? ? set_ioapic_irq_affinity_desc(desc, mask);
> > +}
> > +
> > +static void
> > ?set_ioapic_affinity_irq(unsigned int irq, const struct cpumask *mask)
> > ?{
> > ? ? ? ?struct irq_desc *desc;
> > --
>
> it seems, ack_apic_level() already checked io_apic_level_ack_pending()
>
> cfg = desc->chip_data;
> if (!io_apic_level_ack_pending(cfg))
> move_masked_irq(irq);
> unmask_IO_APIC_irq_desc(desc);
Yes, I have actually observed instances where the command
`echo $IRQ_MASK > /proc/irq/$VICTIM_IRQ/smp_affinity`
in the above test script failed to modify the affinity due to
a debug printk confirmed non-zero return from io_apic_level_ack_pending()
at this location. However, this is definitely not catching all the cases.
This was confirmed this with a printk in __target_IO_APIC_irq() that would
sometimes shows the Remote IRR bit set before the io_apic_modify() call.
e.g.
__target_IO_APIC_irq: XXX before io_apic_modify irq=25 vector=0xd9 reg=0x1e0d9
>
> also calling path without irremap - for level triggered.
> 1. user use /proc/irq/xx/smp_affinity to set affinity
> 2. kernel/irq/manager.c::irq_set_affinity() is called, and
> desc->pending_mask is changed
> 3a. one irq is arrived: arch/x86/kernel/irq.c::do_IRQ() is called,
> hand_irq==>desc->handle_irq
> ==>kernel/irq/chip.c::handle_fasteoi_irq() ==> call desc->chip->eoi()
> 3b. desc->chip->eoi() aka
> arch/x86/kernel/apic/io_apic.c::ack_apic_level() will call
> mask_IO_APIC_irq_desc() and kernel/irq/migration.c::move_masked_irq()
> 3c. move_masked_irq will call desc->set_affinity(), vector and target
> in io_apic controller and IDT are changed
> desc->affinity is changed at last. ( for desc moving,
> move_in_progress and extra_move flag is set too.)
> 3d then ack_apic_level() will unmask the irq.
> 3e. finished other call in handle_fasteoi_irq()
Need to think about this. I'll get back to you.
Thanks!
Gary
--
Gary Hade
System x Enablement
IBM Linux Technology Center
503-578-4503 IBM T/L: 775-4503
[email protected]
http://www.ibm.com/linux/ltc
On Wed, Apr 08, 2009 at 03:03:49PM -0700, Yinghai Lu wrote:
> On Wed, Apr 8, 2009 at 2:07 PM, Gary Hade <[email protected]> wrote:
> > Impact: Eliminates an issue that can leave the system in an
> > ? ? ? ?unusable state.
> >
> > This patch addresses an issue where device generated IRQs
> > are no longer seen by the kernel following IRQ affinity
> > migration while the device is generating IRQs at a high rate.
> > We have seen this problem happen when IRQ affinities are
> > adjusted in response to CPU offlining but I believe it
> > could also happen in during user initiated IRQ affinity
> > changes unrelated to CPU offlining. e.g. while the
> > irqbalance daemon is adjusting IRQ affinities when the
> > system is heavily loaded.
> >
> > I have been able to consistently reproduce the problem on
> > some of our systems by running the following script (VICTIM_IRQ
> > specifies the IRQ for the aic94xx device) while a single instance
> > of the command
> > ?# while true; do find / -exec file {} \;; done
> > is keeping the filesystem activity and IRQ rate reasonably high.
> >
> > #!/bin/sh
> >
> > SYS_CPU_DIR=/sys/devices/system/cpu
> > VICTIM_IRQ=25
> > IRQ_MASK=f0
> >
> > iteration=0
> > while true; do
> > ?echo $iteration
> > ?echo $IRQ_MASK > /proc/irq/$VICTIM_IRQ/smp_affinity
> > ?for cpudir in $SYS_CPU_DIR/cpu[1-9] $SYS_CPU_DIR/cpu??; do
> > ? ?echo 0 > $cpudir/online
> > ?done
> > ?for cpudir in $SYS_CPU_DIR/cpu[1-9] $SYS_CPU_DIR/cpu??; do
> > ? ?echo 1 > $cpudir/online
> > ?done
> > ?iteration=`expr $iteration + 1`
> > done
> >
> > The root cause is a known issue already addressed for some
> > code paths [e.g. ack_apic_level() and the now obsolete
> > migrate_irq_remapped_level_desc()] where the ioapic can
> > misbehave when the I/O redirection table register is written
> > while the Remote IRR bit is set.
> >
> > The proposed fix uses the same avoidance method and much
> > of same code that the Interrupt Remapping code previously
> > used to avoid the same problem.
> >
> > Signed-off-by: Gary Hade <[email protected]>
> >
> > ---
> > ?arch/x86/kernel/apic/io_apic.c | ? 72 ++++++++++++++++++++++++++++++-
> > ?1 file changed, 71 insertions(+), 1 deletion(-)
> >
> > Index: linux-2.6.30-rc1/arch/x86/kernel/apic/io_apic.c
> > ===================================================================
> > --- linux-2.6.30-rc1.orig/arch/x86/kernel/apic/io_apic.c ? ? ? ?2009-04-08 09:24:11.000000000 -0700
> > +++ linux-2.6.30-rc1/arch/x86/kernel/apic/io_apic.c ? ? 2009-04-08 09:24:23.000000000 -0700
> > @@ -2331,7 +2331,8 @@ set_desc_affinity(struct irq_desc *desc,
> > ?}
> >
> > ?static void
> > -set_ioapic_affinity_irq_desc(struct irq_desc *desc, const struct cpumask *mask)
> > +set_ioapic_irq_affinity_desc(struct irq_desc *desc,
> > + ? ? ? ? ? ? ? ? ? ? ? ? ? ?const struct cpumask *mask)
> > ?{
> > ? ? ? ?struct irq_cfg *cfg;
> > ? ? ? ?unsigned long flags;
> > @@ -2352,6 +2353,75 @@ set_ioapic_affinity_irq_desc(struct irq_
> > ?}
> >
> > ?static void
> > +delayed_irq_move(struct work_struct *work)
> > +{
> > + ? ? ? unsigned int irq;
> > + ? ? ? struct irq_desc *desc;
> > +
> > + ? ? ? for_each_irq_desc(irq, desc) {
> > + ? ? ? ? ? ? ? if (desc->status & IRQ_MOVE_PENDING) {
> > + ? ? ? ? ? ? ? ? ? ? ? unsigned long flags;
> > +
> > + ? ? ? ? ? ? ? ? ? ? ? spin_lock_irqsave(&desc->lock, flags);
> > + ? ? ? ? ? ? ? ? ? ? ? if (!desc->chip->set_affinity ||
> > + ? ? ? ? ? ? ? ? ? ? ? ? ? !(desc->status & IRQ_MOVE_PENDING)) {
> > + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? desc->status &= ~IRQ_MOVE_PENDING;
> > + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? spin_unlock_irqrestore(&desc->lock, flags);
> > + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? continue;
> > + ? ? ? ? ? ? ? ? ? ? ? }
> > +
> > + ? ? ? ? ? ? ? ? ? ? ? desc->chip->set_affinity(irq, desc->pending_mask);
> > + ? ? ? ? ? ? ? ? ? ? ? spin_unlock_irqrestore(&desc->lock, flags);
> > + ? ? ? ? ? ? ? }
> > + ? ? ? }
> > +}
> > +
> > +static DECLARE_DELAYED_WORK(delayed_irq_move_work, delayed_irq_move);
> > +
> > +static void
> > +set_ioapic_irq_affinity_level_desc(struct irq_desc *desc)
> > +{
> > +
> > + ? ? ? struct irq_cfg *cfg = desc->chip_data;
> > +
> > + ? ? ? mask_IO_APIC_irq_desc(desc);
> > +
> > + ? ? ? if (io_apic_level_ack_pending(cfg)) {
> > + ? ? ? ? ? ? ? /*
> > + ? ? ? ? ? ? ? ?* Interrupt in progress. Migrating irq now will change
> > + ? ? ? ? ? ? ? ?* the vector information in the IO-APIC RTE which will
> > + ? ? ? ? ? ? ? ?* confuse the EOI broadcast performed by cpu.
> > + ? ? ? ? ? ? ? ?* So, we delay the irq migration.
> > + ? ? ? ? ? ? ? ?*/
> > + ? ? ? ? ? ? ? schedule_delayed_work(&delayed_irq_move_work, 1);
> > + ? ? ? ? ? ? ? goto unmask;
> > + ? ? ? }
> > +
> > + ? ? ? /* Interrupt not in progress. we can change the vector
> > + ? ? ? ?* information in the IO-APIC RTE. */
> > + ? ? ? set_ioapic_irq_affinity_desc(desc, desc->pending_mask);
> > +
> > + ? ? ? desc->status &= ~IRQ_MOVE_PENDING;
> > + ? ? ? cpumask_clear(desc->pending_mask);
> > +
> > +unmask:
> > + ? ? ? unmask_IO_APIC_irq_desc(desc);
> > +}
> > +
> > +static void
> > +set_ioapic_affinity_irq_desc(struct irq_desc *desc,
> > + ? ? ? ? ? ? ? ? ? ? ? ? ? ?const struct cpumask *mask)
> > +{
> > + ? ? ? if (desc->status & IRQ_LEVEL) {
> > + ? ? ? ? ? ? ? desc->status |= IRQ_MOVE_PENDING;
> > + ? ? ? ? ? ? ? cpumask_copy(desc->pending_mask, mask);
> > + ? ? ? ? ? ? ? set_ioapic_irq_affinity_level_desc(desc);
> > + ? ? ? ? ? ? ? return;
> > + ? ? ? }
> > + ? ? ? set_ioapic_irq_affinity_desc(desc, mask);
> > +}
> > +
> > +static void
> > ?set_ioapic_affinity_irq(unsigned int irq, const struct cpumask *mask)
> > ?{
> > ? ? ? ?struct irq_desc *desc;
> > --
>
> it seems, ack_apic_level() already checked io_apic_level_ack_pending()
>
> cfg = desc->chip_data;
> if (!io_apic_level_ack_pending(cfg))
> move_masked_irq(irq);
> unmask_IO_APIC_irq_desc(desc);
>
> also calling path without irremap - for level triggered.
> 1. user use /proc/irq/xx/smp_affinity to set affinity
> 2. kernel/irq/manager.c::irq_set_affinity() is called, and
> desc->pending_mask is changed
> 3a. one irq is arrived: arch/x86/kernel/irq.c::do_IRQ() is called,
> hand_irq==>desc->handle_irq
> ==>kernel/irq/chip.c::handle_fasteoi_irq() ==> call desc->chip->eoi()
> 3b. desc->chip->eoi() aka
> arch/x86/kernel/apic/io_apic.c::ack_apic_level() will call
> mask_IO_APIC_irq_desc() and kernel/irq/migration.c::move_masked_irq()
> 3c. move_masked_irq will call desc->set_affinity(), vector and target
> in io_apic controller and IDT are changed
> desc->affinity is changed at last. ( for desc moving,
> move_in_progress and extra_move flag is set too.)
> 3d then ack_apic_level() will unmask the irq.
> 3e. finished other call in handle_fasteoi_irq()
I considered possible conflicts between my proposed changes
and this code path before I posted the patch. I had concluded
that they should play well together but your comment caused me
to take a much more detailed look at this code. The additional
review (some detail included below) and my testing have not
uncovered anything of concern but please let me know if you
(or anyone else that is listening) see a something that I may
have overlooked.
move_masked_irq() clears the IRQ_MOVE_PENDING desc->status
bit before calling the function pointed to by
desc->chip->set_affinity [e.g. set_ioapic_affinity_irq()]
AND the content of desc->pending_mask is not used after
cpumask_and(desc->affinity, desc->pending_mask, cpu_online_mask);
which precedes the set_affinity function call. So, I believe
this provides the necessary separation between the
IRQ_MOVE_PENDING/desc->pending_mask useage in move_masked_irq()
and the IRQ_MOVE_PENDING/desc->pending_mask useage in the code
that the patch adds.
One interesting observation is that since the IRQ remains
masked from before the io_apic_level_ack_pending() check in
ack_apic_level() until after the set_affinity function call
returns it seems that my speculation in the patch header
that the problem could happen when an IRQ affinity change is
requested from user level on a heavily loaded system is very
likely incorrect.
Thanks,
Gary
--
Gary Hade
System x Enablement
IBM Linux Technology Center
503-578-4503 IBM T/L: 775-4503
[email protected]
http://www.ibm.com/linux/ltc
On Wed, Apr 8, 2009 at 4:08 PM, Gary Hade <[email protected]> wrote:
> On Wed, Apr 08, 2009 at 03:03:49PM -0700, Yinghai Lu wrote:
>> On Wed, Apr 8, 2009 at 2:07 PM, Gary Hade <[email protected]> wrote:
>> > Impact: Eliminates an issue that can leave the system in an
>> > ? ? ? ?unusable state.
>> >
>> > This patch addresses an issue where device generated IRQs
>> > are no longer seen by the kernel following IRQ affinity
>> > migration while the device is generating IRQs at a high rate.
>> > We have seen this problem happen when IRQ affinities are
>> > adjusted in response to CPU offlining but I believe it
>> > could also happen in during user initiated IRQ affinity
>> > changes unrelated to CPU offlining. e.g. while the
>> > irqbalance daemon is adjusting IRQ affinities when the
>> > system is heavily loaded.
>> >
>> > I have been able to consistently reproduce the problem on
>> > some of our systems by running the following script (VICTIM_IRQ
>> > specifies the IRQ for the aic94xx device) while a single instance
>> > of the command
>> > ?# while true; do find / -exec file {} \;; done
>> > is keeping the filesystem activity and IRQ rate reasonably high.
>> >
>> > #!/bin/sh
>> >
>> > SYS_CPU_DIR=/sys/devices/system/cpu
>> > VICTIM_IRQ=25
>> > IRQ_MASK=f0
>> >
>> > iteration=0
>> > while true; do
>> > ?echo $iteration
>> > ?echo $IRQ_MASK > /proc/irq/$VICTIM_IRQ/smp_affinity
>> > ?for cpudir in $SYS_CPU_DIR/cpu[1-9] $SYS_CPU_DIR/cpu??; do
>> > ? ?echo 0 > $cpudir/online
>> > ?done
>> > ?for cpudir in $SYS_CPU_DIR/cpu[1-9] $SYS_CPU_DIR/cpu??; do
>> > ? ?echo 1 > $cpudir/online
>> > ?done
>> > ?iteration=`expr $iteration + 1`
>> > done
>> >
>> > The root cause is a known issue already addressed for some
>> > code paths [e.g. ack_apic_level() and the now obsolete
>> > migrate_irq_remapped_level_desc()] where the ioapic can
>> > misbehave when the I/O redirection table register is written
>> > while the Remote IRR bit is set.
>> >
>> > The proposed fix uses the same avoidance method and much
>> > of same code that the Interrupt Remapping code previously
>> > used to avoid the same problem.
>> >
>> > Signed-off-by: Gary Hade <[email protected]>
>> >
>> > ---
>> > ?arch/x86/kernel/apic/io_apic.c | ? 72 ++++++++++++++++++++++++++++++-
>> > ?1 file changed, 71 insertions(+), 1 deletion(-)
>> >
>> > Index: linux-2.6.30-rc1/arch/x86/kernel/apic/io_apic.c
>> > ===================================================================
>> > --- linux-2.6.30-rc1.orig/arch/x86/kernel/apic/io_apic.c ? ? ? ?2009-04-08 09:24:11.000000000 -0700
>> > +++ linux-2.6.30-rc1/arch/x86/kernel/apic/io_apic.c ? ? 2009-04-08 09:24:23.000000000 -0700
>> > @@ -2331,7 +2331,8 @@ set_desc_affinity(struct irq_desc *desc,
>> > ?}
>> >
>> > ?static void
>> > -set_ioapic_affinity_irq_desc(struct irq_desc *desc, const struct cpumask *mask)
>> > +set_ioapic_irq_affinity_desc(struct irq_desc *desc,
>> > + ? ? ? ? ? ? ? ? ? ? ? ? ? ?const struct cpumask *mask)
>> > ?{
>> > ? ? ? ?struct irq_cfg *cfg;
>> > ? ? ? ?unsigned long flags;
>> > @@ -2352,6 +2353,75 @@ set_ioapic_affinity_irq_desc(struct irq_
>> > ?}
>> >
>> > ?static void
>> > +delayed_irq_move(struct work_struct *work)
>> > +{
>> > + ? ? ? unsigned int irq;
>> > + ? ? ? struct irq_desc *desc;
>> > +
>> > + ? ? ? for_each_irq_desc(irq, desc) {
>> > + ? ? ? ? ? ? ? if (desc->status & IRQ_MOVE_PENDING) {
>> > + ? ? ? ? ? ? ? ? ? ? ? unsigned long flags;
>> > +
>> > + ? ? ? ? ? ? ? ? ? ? ? spin_lock_irqsave(&desc->lock, flags);
>> > + ? ? ? ? ? ? ? ? ? ? ? if (!desc->chip->set_affinity ||
>> > + ? ? ? ? ? ? ? ? ? ? ? ? ? !(desc->status & IRQ_MOVE_PENDING)) {
>> > + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? desc->status &= ~IRQ_MOVE_PENDING;
>> > + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? spin_unlock_irqrestore(&desc->lock, flags);
>> > + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? continue;
>> > + ? ? ? ? ? ? ? ? ? ? ? }
>> > +
>> > + ? ? ? ? ? ? ? ? ? ? ? desc->chip->set_affinity(irq, desc->pending_mask);
>> > + ? ? ? ? ? ? ? ? ? ? ? spin_unlock_irqrestore(&desc->lock, flags);
>> > + ? ? ? ? ? ? ? }
>> > + ? ? ? }
>> > +}
>> > +
>> > +static DECLARE_DELAYED_WORK(delayed_irq_move_work, delayed_irq_move);
>> > +
>> > +static void
>> > +set_ioapic_irq_affinity_level_desc(struct irq_desc *desc)
>> > +{
>> > +
>> > + ? ? ? struct irq_cfg *cfg = desc->chip_data;
>> > +
>> > + ? ? ? mask_IO_APIC_irq_desc(desc);
>> > +
>> > + ? ? ? if (io_apic_level_ack_pending(cfg)) {
>> > + ? ? ? ? ? ? ? /*
>> > + ? ? ? ? ? ? ? ?* Interrupt in progress. Migrating irq now will change
>> > + ? ? ? ? ? ? ? ?* the vector information in the IO-APIC RTE which will
>> > + ? ? ? ? ? ? ? ?* confuse the EOI broadcast performed by cpu.
>> > + ? ? ? ? ? ? ? ?* So, we delay the irq migration.
>> > + ? ? ? ? ? ? ? ?*/
>> > + ? ? ? ? ? ? ? schedule_delayed_work(&delayed_irq_move_work, 1);
>> > + ? ? ? ? ? ? ? goto unmask;
>> > + ? ? ? }
>> > +
>> > + ? ? ? /* Interrupt not in progress. we can change the vector
>> > + ? ? ? ?* information in the IO-APIC RTE. */
>> > + ? ? ? set_ioapic_irq_affinity_desc(desc, desc->pending_mask);
>> > +
>> > + ? ? ? desc->status &= ~IRQ_MOVE_PENDING;
>> > + ? ? ? cpumask_clear(desc->pending_mask);
>> > +
>> > +unmask:
>> > + ? ? ? unmask_IO_APIC_irq_desc(desc);
>> > +}
>> > +
>> > +static void
>> > +set_ioapic_affinity_irq_desc(struct irq_desc *desc,
>> > + ? ? ? ? ? ? ? ? ? ? ? ? ? ?const struct cpumask *mask)
>> > +{
>> > + ? ? ? if (desc->status & IRQ_LEVEL) {
>> > + ? ? ? ? ? ? ? desc->status |= IRQ_MOVE_PENDING;
>> > + ? ? ? ? ? ? ? cpumask_copy(desc->pending_mask, mask);
>> > + ? ? ? ? ? ? ? set_ioapic_irq_affinity_level_desc(desc);
>> > + ? ? ? ? ? ? ? return;
>> > + ? ? ? }
>> > + ? ? ? set_ioapic_irq_affinity_desc(desc, mask);
>> > +}
>> > +
>> > +static void
>> > ?set_ioapic_affinity_irq(unsigned int irq, const struct cpumask *mask)
>> > ?{
>> > ? ? ? ?struct irq_desc *desc;
>> > --
>>
>> it seems, ack_apic_level() already checked io_apic_level_ack_pending()
>>
>> ? ? ? ? ? ? ? ? cfg = desc->chip_data;
>> ? ? ? ? ? ? ? ? if (!io_apic_level_ack_pending(cfg))
>> ? ? ? ? ? ? ? ? ? ? ? ? move_masked_irq(irq);
>> ? ? ? ? ? ? ? ? unmask_IO_APIC_irq_desc(desc);
>
> Yes, I have actually observed instances where the command
> ? `echo $IRQ_MASK > /proc/irq/$VICTIM_IRQ/smp_affinity`
> in the above test script failed to modify the affinity due to
> a debug printk confirmed non-zero return from io_apic_level_ack_pending()
> at this location. ?However, this is definitely not catching all the cases.
> This was confirmed this with a printk in __target_IO_APIC_irq() that would
> sometimes shows the Remote IRR bit set before the io_apic_modify() call.
> e.g.
> __target_IO_APIC_irq: XXX before io_apic_modify irq=25 vector=0xd9 reg=0x1e0d9
that is strange, before that irq is masked,
#ifdef CONFIG_GENERIC_PENDING_IRQ
/* If we are moving the irq we need to mask it */
if (unlikely(desc->status & IRQ_MOVE_PENDING)) {
do_unmask_irq = 1;
mask_IO_APIC_irq_desc(desc);
}
#endif
YH
>
>>
>> also calling path without irremap - for level triggered.
>> 1. user use /proc/irq/xx/smp_affinity to set affinity
>> 2. kernel/irq/manager.c::irq_set_affinity() is called, and
>> desc->pending_mask is changed
>> 3a. one irq is arrived: arch/x86/kernel/irq.c::do_IRQ() is called,
>> hand_irq==>desc->handle_irq
>> ? ? ==>kernel/irq/chip.c::handle_fasteoi_irq() ==> call desc->chip->eoi()
>> 3b. desc->chip->eoi() aka
>> arch/x86/kernel/apic/io_apic.c::ack_apic_level() will call
>> mask_IO_APIC_irq_desc() and kernel/irq/migration.c::move_masked_irq()
>> 3c. move_masked_irq will call desc->set_affinity(), vector and target
>> in io_apic controller and IDT are changed
>> ? ? desc->affinity is changed at last. ( for desc moving,
>> move_in_progress and extra_move flag is set too.)
>> 3d then ack_apic_level() will unmask the irq.
>> 3e. finished other call in handle_fasteoi_irq()
>
> Need to think about this. ?I'll get back to you.
>
> Thanks!
>
> Gary
>
> --
> Gary Hade
> System x Enablement
> IBM Linux Technology Center
> 503-578-4503 ?IBM T/L: 775-4503
> [email protected]
> http://www.ibm.com/linux/ltc
>
>
On Fri, Apr 10, 2009 at 2:39 PM, Gary Hade <[email protected]> wrote:
> move_masked_irq() clears the IRQ_MOVE_PENDING desc->status
> bit before calling the function pointed to by
> desc->chip->set_affinity [e.g. set_ioapic_affinity_irq()]
> AND the content of desc->pending_mask is not used after
> ?cpumask_and(desc->affinity, desc->pending_mask, cpu_online_mask);
> which precedes the set_affinity function call. ?So, I believe
> this provides the necessary separation between the
> IRQ_MOVE_PENDING/desc->pending_mask useage in move_masked_irq()
> and the IRQ_MOVE_PENDING/desc->pending_mask useage in the code
> that the patch adds.
>
that should be ok, they are proteced by spin_lock(&desc->lock)
YH
On Fri, Apr 10, 2009 at 11:46:39PM -0700, Yinghai Lu wrote:
> On Wed, Apr 8, 2009 at 4:08 PM, Gary Hade <[email protected]> wrote:
> > On Wed, Apr 08, 2009 at 03:03:49PM -0700, Yinghai Lu wrote:
> >> On Wed, Apr 8, 2009 at 2:07 PM, Gary Hade <[email protected]> wrote:
> >> > Impact: Eliminates an issue that can leave the system in an
> >> > ? ? ? ?unusable state.
> >> >
> >> > This patch addresses an issue where device generated IRQs
> >> > are no longer seen by the kernel following IRQ affinity
> >> > migration while the device is generating IRQs at a high rate.
> >> > We have seen this problem happen when IRQ affinities are
> >> > adjusted in response to CPU offlining but I believe it
> >> > could also happen in during user initiated IRQ affinity
> >> > changes unrelated to CPU offlining. e.g. while the
> >> > irqbalance daemon is adjusting IRQ affinities when the
> >> > system is heavily loaded.
> >> >
> >> > I have been able to consistently reproduce the problem on
> >> > some of our systems by running the following script (VICTIM_IRQ
> >> > specifies the IRQ for the aic94xx device) while a single instance
> >> > of the command
> >> > ?# while true; do find / -exec file {} \;; done
> >> > is keeping the filesystem activity and IRQ rate reasonably high.
> >> >
> >> > #!/bin/sh
> >> >
> >> > SYS_CPU_DIR=/sys/devices/system/cpu
> >> > VICTIM_IRQ=25
> >> > IRQ_MASK=f0
> >> >
> >> > iteration=0
> >> > while true; do
> >> > ?echo $iteration
> >> > ?echo $IRQ_MASK > /proc/irq/$VICTIM_IRQ/smp_affinity
> >> > ?for cpudir in $SYS_CPU_DIR/cpu[1-9] $SYS_CPU_DIR/cpu??; do
> >> > ? ?echo 0 > $cpudir/online
> >> > ?done
> >> > ?for cpudir in $SYS_CPU_DIR/cpu[1-9] $SYS_CPU_DIR/cpu??; do
> >> > ? ?echo 1 > $cpudir/online
> >> > ?done
> >> > ?iteration=`expr $iteration + 1`
> >> > done
> >> >
> >> > The root cause is a known issue already addressed for some
> >> > code paths [e.g. ack_apic_level() and the now obsolete
> >> > migrate_irq_remapped_level_desc()] where the ioapic can
> >> > misbehave when the I/O redirection table register is written
> >> > while the Remote IRR bit is set.
> >> >
> >> > The proposed fix uses the same avoidance method and much
> >> > of same code that the Interrupt Remapping code previously
> >> > used to avoid the same problem.
> >> >
> >> > Signed-off-by: Gary Hade <[email protected]>
> >> >
> >> > ---
> >> > ?arch/x86/kernel/apic/io_apic.c | ? 72 ++++++++++++++++++++++++++++++-
> >> > ?1 file changed, 71 insertions(+), 1 deletion(-)
> >> >
> >> > Index: linux-2.6.30-rc1/arch/x86/kernel/apic/io_apic.c
> >> > ===================================================================
> >> > --- linux-2.6.30-rc1.orig/arch/x86/kernel/apic/io_apic.c ? ? ? ?2009-04-08 09:24:11.000000000 -0700
> >> > +++ linux-2.6.30-rc1/arch/x86/kernel/apic/io_apic.c ? ? 2009-04-08 09:24:23.000000000 -0700
> >> > @@ -2331,7 +2331,8 @@ set_desc_affinity(struct irq_desc *desc,
> >> > ?}
> >> >
> >> > ?static void
> >> > -set_ioapic_affinity_irq_desc(struct irq_desc *desc, const struct cpumask *mask)
> >> > +set_ioapic_irq_affinity_desc(struct irq_desc *desc,
> >> > + ? ? ? ? ? ? ? ? ? ? ? ? ? ?const struct cpumask *mask)
> >> > ?{
> >> > ? ? ? ?struct irq_cfg *cfg;
> >> > ? ? ? ?unsigned long flags;
> >> > @@ -2352,6 +2353,75 @@ set_ioapic_affinity_irq_desc(struct irq_
> >> > ?}
> >> >
> >> > ?static void
> >> > +delayed_irq_move(struct work_struct *work)
> >> > +{
> >> > + ? ? ? unsigned int irq;
> >> > + ? ? ? struct irq_desc *desc;
> >> > +
> >> > + ? ? ? for_each_irq_desc(irq, desc) {
> >> > + ? ? ? ? ? ? ? if (desc->status & IRQ_MOVE_PENDING) {
> >> > + ? ? ? ? ? ? ? ? ? ? ? unsigned long flags;
> >> > +
> >> > + ? ? ? ? ? ? ? ? ? ? ? spin_lock_irqsave(&desc->lock, flags);
> >> > + ? ? ? ? ? ? ? ? ? ? ? if (!desc->chip->set_affinity ||
> >> > + ? ? ? ? ? ? ? ? ? ? ? ? ? !(desc->status & IRQ_MOVE_PENDING)) {
> >> > + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? desc->status &= ~IRQ_MOVE_PENDING;
> >> > + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? spin_unlock_irqrestore(&desc->lock, flags);
> >> > + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? continue;
> >> > + ? ? ? ? ? ? ? ? ? ? ? }
> >> > +
> >> > + ? ? ? ? ? ? ? ? ? ? ? desc->chip->set_affinity(irq, desc->pending_mask);
> >> > + ? ? ? ? ? ? ? ? ? ? ? spin_unlock_irqrestore(&desc->lock, flags);
> >> > + ? ? ? ? ? ? ? }
> >> > + ? ? ? }
> >> > +}
> >> > +
> >> > +static DECLARE_DELAYED_WORK(delayed_irq_move_work, delayed_irq_move);
> >> > +
> >> > +static void
> >> > +set_ioapic_irq_affinity_level_desc(struct irq_desc *desc)
> >> > +{
> >> > +
> >> > + ? ? ? struct irq_cfg *cfg = desc->chip_data;
> >> > +
> >> > + ? ? ? mask_IO_APIC_irq_desc(desc);
> >> > +
> >> > + ? ? ? if (io_apic_level_ack_pending(cfg)) {
> >> > + ? ? ? ? ? ? ? /*
> >> > + ? ? ? ? ? ? ? ?* Interrupt in progress. Migrating irq now will change
> >> > + ? ? ? ? ? ? ? ?* the vector information in the IO-APIC RTE which will
> >> > + ? ? ? ? ? ? ? ?* confuse the EOI broadcast performed by cpu.
> >> > + ? ? ? ? ? ? ? ?* So, we delay the irq migration.
> >> > + ? ? ? ? ? ? ? ?*/
> >> > + ? ? ? ? ? ? ? schedule_delayed_work(&delayed_irq_move_work, 1);
> >> > + ? ? ? ? ? ? ? goto unmask;
> >> > + ? ? ? }
> >> > +
> >> > + ? ? ? /* Interrupt not in progress. we can change the vector
> >> > + ? ? ? ?* information in the IO-APIC RTE. */
> >> > + ? ? ? set_ioapic_irq_affinity_desc(desc, desc->pending_mask);
> >> > +
> >> > + ? ? ? desc->status &= ~IRQ_MOVE_PENDING;
> >> > + ? ? ? cpumask_clear(desc->pending_mask);
> >> > +
> >> > +unmask:
> >> > + ? ? ? unmask_IO_APIC_irq_desc(desc);
> >> > +}
> >> > +
> >> > +static void
> >> > +set_ioapic_affinity_irq_desc(struct irq_desc *desc,
> >> > + ? ? ? ? ? ? ? ? ? ? ? ? ? ?const struct cpumask *mask)
> >> > +{
> >> > + ? ? ? if (desc->status & IRQ_LEVEL) {
> >> > + ? ? ? ? ? ? ? desc->status |= IRQ_MOVE_PENDING;
> >> > + ? ? ? ? ? ? ? cpumask_copy(desc->pending_mask, mask);
> >> > + ? ? ? ? ? ? ? set_ioapic_irq_affinity_level_desc(desc);
> >> > + ? ? ? ? ? ? ? return;
> >> > + ? ? ? }
> >> > + ? ? ? set_ioapic_irq_affinity_desc(desc, mask);
> >> > +}
> >> > +
> >> > +static void
> >> > ?set_ioapic_affinity_irq(unsigned int irq, const struct cpumask *mask)
> >> > ?{
> >> > ? ? ? ?struct irq_desc *desc;
> >> > --
> >>
> >> it seems, ack_apic_level() already checked io_apic_level_ack_pending()
> >>
> >> ? ? ? ? ? ? ? ? cfg = desc->chip_data;
> >> ? ? ? ? ? ? ? ? if (!io_apic_level_ack_pending(cfg))
> >> ? ? ? ? ? ? ? ? ? ? ? ? move_masked_irq(irq);
> >> ? ? ? ? ? ? ? ? unmask_IO_APIC_irq_desc(desc);
> >
> > Yes, I have actually observed instances where the command
> > ? `echo $IRQ_MASK > /proc/irq/$VICTIM_IRQ/smp_affinity`
> > in the above test script failed to modify the affinity due to
> > a debug printk confirmed non-zero return from io_apic_level_ack_pending()
> > at this location. ?However, this is definitely not catching all the cases.
> > This was confirmed this with a printk in __target_IO_APIC_irq() that would
> > sometimes shows the Remote IRR bit set before the io_apic_modify() call.
> > e.g.
> > __target_IO_APIC_irq: XXX before io_apic_modify irq=25 vector=0xd9 reg=0x1e0d9
>
>
> that is strange, before that irq is masked,
>
> #ifdef CONFIG_GENERIC_PENDING_IRQ
> /* If we are moving the irq we need to mask it */
> if (unlikely(desc->status & IRQ_MOVE_PENDING)) {
> do_unmask_irq = 1;
> mask_IO_APIC_irq_desc(desc);
> }
> #endif
When the device is very actively generating IRQs I suspect
that __target_IO_APIC_irq() is sometimes visited (via CPU
offline motivated path) before the IRQ is masked at this location.
I believe the vulnerability window is actually a bit larger
because it spans this mask_IO_APIC_irq_desc() call.
Gary
--
Gary Hade
System x Enablement
IBM Linux Technology Center
503-578-4503 IBM T/L: 775-4503
[email protected]
http://www.ibm.com/linux/ltc
Gary Hade <[email protected]> writes:
> On Fri, Apr 10, 2009 at 11:46:39PM -0700, Yinghai Lu wrote:
>> On Wed, Apr 8, 2009 at 4:08 PM, Gary Hade <[email protected]> wrote:
>> > On Wed, Apr 08, 2009 at 03:03:49PM -0700, Yinghai Lu wrote:
>> >> On Wed, Apr 8, 2009 at 2:07 PM, Gary Hade <[email protected]> wrote:
>> >> > Impact: Eliminates an issue that can leave the system in an
>> >> > Â Â Â Â unusable state.
>> >> >
>> >> > This patch addresses an issue where device generated IRQs
>> >> > are no longer seen by the kernel following IRQ affinity
>> >> > migration while the device is generating IRQs at a high rate.
>> >> > We have seen this problem happen when IRQ affinities are
>> >> > adjusted in response to CPU offlining but I believe it
>> >> > could also happen in during user initiated IRQ affinity
>> >> > changes unrelated to CPU offlining. e.g. while the
>> >> > irqbalance daemon is adjusting IRQ affinities when the
>> >> > system is heavily loaded.
>> >> >
>> >> > I have been able to consistently reproduce the problem on
>> >> > some of our systems by running the following script (VICTIM_IRQ
>> >> > specifies the IRQ for the aic94xx device) while a single instance
>> >> > of the command
>> >> > Â # while true; do find / -exec file {} \;; done
>> >> > is keeping the filesystem activity and IRQ rate reasonably high.
>> >> >
>> >> > #!/bin/sh
>> >> >
>> >> > SYS_CPU_DIR=/sys/devices/system/cpu
>> >> > VICTIM_IRQ=25
>> >> > IRQ_MASK=f0
>> >> >
>> >> > iteration=0
>> >> > while true; do
>> >> > Â echo $iteration
>> >> > Â echo $IRQ_MASK > /proc/irq/$VICTIM_IRQ/smp_affinity
>> >> > Â for cpudir in $SYS_CPU_DIR/cpu[1-9] $SYS_CPU_DIR/cpu??; do
>> >> > Â Â echo 0 > $cpudir/online
>> >> > Â done
>> >> > Â for cpudir in $SYS_CPU_DIR/cpu[1-9] $SYS_CPU_DIR/cpu??; do
>> >> > Â Â echo 1 > $cpudir/online
>> >> > Â done
>> >> > Â iteration=`expr $iteration + 1`
>> >> > done
>> >> >
>> >> > The root cause is a known issue already addressed for some
>> >> > code paths [e.g. ack_apic_level() and the now obsolete
>> >> > migrate_irq_remapped_level_desc()] where the ioapic can
>> >> > misbehave when the I/O redirection table register is written
>> >> > while the Remote IRR bit is set.
>> >> >
>> >> > The proposed fix uses the same avoidance method and much
>> >> > of same code that the Interrupt Remapping code previously
>> >> > used to avoid the same problem.
>> >> >
>> >> > Signed-off-by: Gary Hade <[email protected]>
>> >> >
>> >> > ---
>> >> > Â arch/x86/kernel/apic/io_apic.c | Â 72 ++++++++++++++++++++++++++++++-
>> >> > Â 1 file changed, 71 insertions(+), 1 deletion(-)
>> >> >
>> >> > Index: linux-2.6.30-rc1/arch/x86/kernel/apic/io_apic.c
>> >> > ===================================================================
>> >> > --- linux-2.6.30-rc1.orig/arch/x86/kernel/apic/io_apic.c     2009-04-08 09:24:11.000000000 -0700
>> >> > +++ linux-2.6.30-rc1/arch/x86/kernel/apic/io_apic.c   2009-04-08 09:24:23.000000000 -0700
>> >> > @@ -2331,7 +2331,8 @@ set_desc_affinity(struct irq_desc *desc,
>> >> > Â }
>> >> >
>> >> > Â static void
>> >> > -set_ioapic_affinity_irq_desc(struct irq_desc *desc, const struct cpumask *mask)
>> >> > +set_ioapic_irq_affinity_desc(struct irq_desc *desc,
>> >> > + Â Â Â Â Â Â Â Â Â Â Â Â Â Â const struct cpumask *mask)
>> >> > Â {
>> >> > Â Â Â Â struct irq_cfg *cfg;
>> >> > Â Â Â Â unsigned long flags;
>> >> > @@ -2352,6 +2353,75 @@ set_ioapic_affinity_irq_desc(struct irq_
>> >> > Â }
>> >> >
>> >> > Â static void
>> >> > +delayed_irq_move(struct work_struct *work)
>> >> > +{
>> >> > + Â Â Â unsigned int irq;
>> >> > + Â Â Â struct irq_desc *desc;
>> >> > +
>> >> > + Â Â Â for_each_irq_desc(irq, desc) {
>> >> > + Â Â Â Â Â Â Â if (desc->status & IRQ_MOVE_PENDING) {
>> >> > + Â Â Â Â Â Â Â Â Â Â Â unsigned long flags;
>> >> > +
>> >> > + Â Â Â Â Â Â Â Â Â Â Â spin_lock_irqsave(&desc->lock, flags);
>> >> > + Â Â Â Â Â Â Â Â Â Â Â if (!desc->chip->set_affinity ||
>> >> > + Â Â Â Â Â Â Â Â Â Â Â Â Â !(desc->status & IRQ_MOVE_PENDING)) {
>> >> > + Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â desc->status &= ~IRQ_MOVE_PENDING;
>> >> > + Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â spin_unlock_irqrestore(&desc->lock, flags);
>> >> > + Â Â Â Â Â Â Â Â Â Â Â Â Â Â Â continue;
>> >> > + Â Â Â Â Â Â Â Â Â Â Â }
>> >> > +
>> >> > + Â Â Â Â Â Â Â Â Â Â Â desc->chip->set_affinity(irq, desc->pending_mask);
>> >> > + Â Â Â Â Â Â Â Â Â Â Â spin_unlock_irqrestore(&desc->lock, flags);
>> >> > + Â Â Â Â Â Â Â }
>> >> > + Â Â Â }
>> >> > +}
>> >> > +
>> >> > +static DECLARE_DELAYED_WORK(delayed_irq_move_work, delayed_irq_move);
>> >> > +
>> >> > +static void
>> >> > +set_ioapic_irq_affinity_level_desc(struct irq_desc *desc)
>> >> > +{
>> >> > +
>> >> > + Â Â Â struct irq_cfg *cfg = desc->chip_data;
>> >> > +
>> >> > + Â Â Â mask_IO_APIC_irq_desc(desc);
>> >> > +
>> >> > + Â Â Â if (io_apic_level_ack_pending(cfg)) {
>> >> > + Â Â Â Â Â Â Â /*
>> >> > + Â Â Â Â Â Â Â Â * Interrupt in progress. Migrating irq now will change
>> >> > + Â Â Â Â Â Â Â Â * the vector information in the IO-APIC RTE which will
>> >> > + Â Â Â Â Â Â Â Â * confuse the EOI broadcast performed by cpu.
>> >> > + Â Â Â Â Â Â Â Â * So, we delay the irq migration.
>> >> > + Â Â Â Â Â Â Â Â */
>> >> > + Â Â Â Â Â Â Â schedule_delayed_work(&delayed_irq_move_work, 1);
>> >> > + Â Â Â Â Â Â Â goto unmask;
>> >> > + Â Â Â }
>> >> > +
>> >> > + Â Â Â /* Interrupt not in progress. we can change the vector
>> >> > + Â Â Â Â * information in the IO-APIC RTE. */
>> >> > + Â Â Â set_ioapic_irq_affinity_desc(desc, desc->pending_mask);
>> >> > +
>> >> > + Â Â Â desc->status &= ~IRQ_MOVE_PENDING;
>> >> > + Â Â Â cpumask_clear(desc->pending_mask);
>> >> > +
>> >> > +unmask:
>> >> > + Â Â Â unmask_IO_APIC_irq_desc(desc);
>> >> > +}
>> >> > +
>> >> > +static void
>> >> > +set_ioapic_affinity_irq_desc(struct irq_desc *desc,
>> >> > + Â Â Â Â Â Â Â Â Â Â Â Â Â Â const struct cpumask *mask)
>> >> > +{
>> >> > + Â Â Â if (desc->status & IRQ_LEVEL) {
>> >> > + Â Â Â Â Â Â Â desc->status |= IRQ_MOVE_PENDING;
>> >> > + Â Â Â Â Â Â Â cpumask_copy(desc->pending_mask, mask);
>> >> > + Â Â Â Â Â Â Â set_ioapic_irq_affinity_level_desc(desc);
>> >> > + Â Â Â Â Â Â Â return;
>> >> > + Â Â Â }
>> >> > + Â Â Â set_ioapic_irq_affinity_desc(desc, mask);
>> >> > +}
>> >> > +
>> >> > +static void
>> >> > Â set_ioapic_affinity_irq(unsigned int irq, const struct cpumask *mask)
>> >> > Â {
>> >> > Â Â Â Â struct irq_desc *desc;
>> >> > --
>> >>
>> >> it seems, ack_apic_level() already checked io_apic_level_ack_pending()
>> >>
>> >> Â Â Â Â Â Â Â Â cfg = desc->chip_data;
>> >> Â Â Â Â Â Â Â Â if (!io_apic_level_ack_pending(cfg))
>> >> Â Â Â Â Â Â Â Â Â Â Â Â move_masked_irq(irq);
>> >> Â Â Â Â Â Â Â Â unmask_IO_APIC_irq_desc(desc);
>> >
>> > Yes, I have actually observed instances where the command
>> > Â `echo $IRQ_MASK > /proc/irq/$VICTIM_IRQ/smp_affinity`
>> > in the above test script failed to modify the affinity due to
>> > a debug printk confirmed non-zero return from io_apic_level_ack_pending()
>> > at this location. Â However, this is definitely not catching all the cases.
>> > This was confirmed this with a printk in __target_IO_APIC_irq() that would
>> > sometimes shows the Remote IRR bit set before the io_apic_modify() call.
>> > e.g.
>> > __target_IO_APIC_irq: XXX before io_apic_modify irq=25 vector=0xd9 reg=0x1e0d9
>>
>>
>> that is strange, before that irq is masked,
>>
>> #ifdef CONFIG_GENERIC_PENDING_IRQ
>> /* If we are moving the irq we need to mask it */
>> if (unlikely(desc->status & IRQ_MOVE_PENDING)) {
>> do_unmask_irq = 1;
>> mask_IO_APIC_irq_desc(desc);
>> }
>> #endif
>
> When the device is very actively generating IRQs I suspect
> that __target_IO_APIC_irq() is sometimes visited (via CPU
> offline motivated path) before the IRQ is masked at this location.
> I believe the vulnerability window is actually a bit larger
> because it spans this mask_IO_APIC_irq_desc() call.
I don't think it makes sense to debate how broken fixup_irqs is.
The code is fundamentally wrong, if you don't have code that
can migrate an irq in process context.
Not playing games when Remote_IRR is set is the easy part. As
that is required on correctly functioning hardware. Unless
your ioapic has the magic acknowledge register.
The hard part is that ioapic to put it politely are fragile
things so you have to handle them just so. It is possible to
wedge the state machines into a unrecoverable mess on most
common ioapic implementations. Not sending the ack when
the ioapic is ready to receive it is easy by comparison.
Eric
On Mon, Apr 13, 2009 at 01:17:29PM -0700, Eric W. Biederman wrote:
> Gary Hade <[email protected]> writes:
>
> > On Fri, Apr 10, 2009 at 11:46:39PM -0700, Yinghai Lu wrote:
> >> On Wed, Apr 8, 2009 at 4:08 PM, Gary Hade <[email protected]> wrote:
> >> > On Wed, Apr 08, 2009 at 03:03:49PM -0700, Yinghai Lu wrote:
> >> >> On Wed, Apr 8, 2009 at 2:07 PM, Gary Hade <[email protected]> wrote:
> >> >> > Impact: Eliminates an issue that can leave the system in an
> >> >> > ? ? ? ?unusable state.
> >> >> >
> >> >> > This patch addresses an issue where device generated IRQs
> >> >> > are no longer seen by the kernel following IRQ affinity
> >> >> > migration while the device is generating IRQs at a high rate.
> >> >> > We have seen this problem happen when IRQ affinities are
> >> >> > adjusted in response to CPU offlining but I believe it
> >> >> > could also happen in during user initiated IRQ affinity
> >> >> > changes unrelated to CPU offlining. e.g. while the
> >> >> > irqbalance daemon is adjusting IRQ affinities when the
> >> >> > system is heavily loaded.
> >> >> >
> >> >> > I have been able to consistently reproduce the problem on
> >> >> > some of our systems by running the following script (VICTIM_IRQ
> >> >> > specifies the IRQ for the aic94xx device) while a single instance
> >> >> > of the command
> >> >> > ?# while true; do find / -exec file {} \;; done
> >> >> > is keeping the filesystem activity and IRQ rate reasonably high.
> >> >> >
> >> >> > #!/bin/sh
> >> >> >
> >> >> > SYS_CPU_DIR=/sys/devices/system/cpu
> >> >> > VICTIM_IRQ=25
> >> >> > IRQ_MASK=f0
> >> >> >
> >> >> > iteration=0
> >> >> > while true; do
> >> >> > ?echo $iteration
> >> >> > ?echo $IRQ_MASK > /proc/irq/$VICTIM_IRQ/smp_affinity
> >> >> > ?for cpudir in $SYS_CPU_DIR/cpu[1-9] $SYS_CPU_DIR/cpu??; do
> >> >> > ? ?echo 0 > $cpudir/online
> >> >> > ?done
> >> >> > ?for cpudir in $SYS_CPU_DIR/cpu[1-9] $SYS_CPU_DIR/cpu??; do
> >> >> > ? ?echo 1 > $cpudir/online
> >> >> > ?done
> >> >> > ?iteration=`expr $iteration + 1`
> >> >> > done
> >> >> >
> >> >> > The root cause is a known issue already addressed for some
> >> >> > code paths [e.g. ack_apic_level() and the now obsolete
> >> >> > migrate_irq_remapped_level_desc()] where the ioapic can
> >> >> > misbehave when the I/O redirection table register is written
> >> >> > while the Remote IRR bit is set.
> >> >> >
> >> >> > The proposed fix uses the same avoidance method and much
> >> >> > of same code that the Interrupt Remapping code previously
> >> >> > used to avoid the same problem.
> >> >> >
> >> >> > Signed-off-by: Gary Hade <[email protected]>
> >> >> >
> >> >> > ---
> >> >> > ?arch/x86/kernel/apic/io_apic.c | ? 72 ++++++++++++++++++++++++++++++-
> >> >> > ?1 file changed, 71 insertions(+), 1 deletion(-)
> >> >> >
> >> >> > Index: linux-2.6.30-rc1/arch/x86/kernel/apic/io_apic.c
> >> >> > ===================================================================
> >> >> > --- linux-2.6.30-rc1.orig/arch/x86/kernel/apic/io_apic.c ? ? ? ?2009-04-08 09:24:11.000000000 -0700
> >> >> > +++ linux-2.6.30-rc1/arch/x86/kernel/apic/io_apic.c ? ? 2009-04-08 09:24:23.000000000 -0700
> >> >> > @@ -2331,7 +2331,8 @@ set_desc_affinity(struct irq_desc *desc,
> >> >> > ?}
> >> >> >
> >> >> > ?static void
> >> >> > -set_ioapic_affinity_irq_desc(struct irq_desc *desc, const struct cpumask *mask)
> >> >> > +set_ioapic_irq_affinity_desc(struct irq_desc *desc,
> >> >> > + ? ? ? ? ? ? ? ? ? ? ? ? ? ?const struct cpumask *mask)
> >> >> > ?{
> >> >> > ? ? ? ?struct irq_cfg *cfg;
> >> >> > ? ? ? ?unsigned long flags;
> >> >> > @@ -2352,6 +2353,75 @@ set_ioapic_affinity_irq_desc(struct irq_
> >> >> > ?}
> >> >> >
> >> >> > ?static void
> >> >> > +delayed_irq_move(struct work_struct *work)
> >> >> > +{
> >> >> > + ? ? ? unsigned int irq;
> >> >> > + ? ? ? struct irq_desc *desc;
> >> >> > +
> >> >> > + ? ? ? for_each_irq_desc(irq, desc) {
> >> >> > + ? ? ? ? ? ? ? if (desc->status & IRQ_MOVE_PENDING) {
> >> >> > + ? ? ? ? ? ? ? ? ? ? ? unsigned long flags;
> >> >> > +
> >> >> > + ? ? ? ? ? ? ? ? ? ? ? spin_lock_irqsave(&desc->lock, flags);
> >> >> > + ? ? ? ? ? ? ? ? ? ? ? if (!desc->chip->set_affinity ||
> >> >> > + ? ? ? ? ? ? ? ? ? ? ? ? ? !(desc->status & IRQ_MOVE_PENDING)) {
> >> >> > + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? desc->status &= ~IRQ_MOVE_PENDING;
> >> >> > + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? spin_unlock_irqrestore(&desc->lock, flags);
> >> >> > + ? ? ? ? ? ? ? ? ? ? ? ? ? ? ? continue;
> >> >> > + ? ? ? ? ? ? ? ? ? ? ? }
> >> >> > +
> >> >> > + ? ? ? ? ? ? ? ? ? ? ? desc->chip->set_affinity(irq, desc->pending_mask);
> >> >> > + ? ? ? ? ? ? ? ? ? ? ? spin_unlock_irqrestore(&desc->lock, flags);
> >> >> > + ? ? ? ? ? ? ? }
> >> >> > + ? ? ? }
> >> >> > +}
> >> >> > +
> >> >> > +static DECLARE_DELAYED_WORK(delayed_irq_move_work, delayed_irq_move);
> >> >> > +
> >> >> > +static void
> >> >> > +set_ioapic_irq_affinity_level_desc(struct irq_desc *desc)
> >> >> > +{
> >> >> > +
> >> >> > + ? ? ? struct irq_cfg *cfg = desc->chip_data;
> >> >> > +
> >> >> > + ? ? ? mask_IO_APIC_irq_desc(desc);
> >> >> > +
> >> >> > + ? ? ? if (io_apic_level_ack_pending(cfg)) {
> >> >> > + ? ? ? ? ? ? ? /*
> >> >> > + ? ? ? ? ? ? ? ?* Interrupt in progress. Migrating irq now will change
> >> >> > + ? ? ? ? ? ? ? ?* the vector information in the IO-APIC RTE which will
> >> >> > + ? ? ? ? ? ? ? ?* confuse the EOI broadcast performed by cpu.
> >> >> > + ? ? ? ? ? ? ? ?* So, we delay the irq migration.
> >> >> > + ? ? ? ? ? ? ? ?*/
> >> >> > + ? ? ? ? ? ? ? schedule_delayed_work(&delayed_irq_move_work, 1);
> >> >> > + ? ? ? ? ? ? ? goto unmask;
> >> >> > + ? ? ? }
> >> >> > +
> >> >> > + ? ? ? /* Interrupt not in progress. we can change the vector
> >> >> > + ? ? ? ?* information in the IO-APIC RTE. */
> >> >> > + ? ? ? set_ioapic_irq_affinity_desc(desc, desc->pending_mask);
> >> >> > +
> >> >> > + ? ? ? desc->status &= ~IRQ_MOVE_PENDING;
> >> >> > + ? ? ? cpumask_clear(desc->pending_mask);
> >> >> > +
> >> >> > +unmask:
> >> >> > + ? ? ? unmask_IO_APIC_irq_desc(desc);
> >> >> > +}
> >> >> > +
> >> >> > +static void
> >> >> > +set_ioapic_affinity_irq_desc(struct irq_desc *desc,
> >> >> > + ? ? ? ? ? ? ? ? ? ? ? ? ? ?const struct cpumask *mask)
> >> >> > +{
> >> >> > + ? ? ? if (desc->status & IRQ_LEVEL) {
> >> >> > + ? ? ? ? ? ? ? desc->status |= IRQ_MOVE_PENDING;
> >> >> > + ? ? ? ? ? ? ? cpumask_copy(desc->pending_mask, mask);
> >> >> > + ? ? ? ? ? ? ? set_ioapic_irq_affinity_level_desc(desc);
> >> >> > + ? ? ? ? ? ? ? return;
> >> >> > + ? ? ? }
> >> >> > + ? ? ? set_ioapic_irq_affinity_desc(desc, mask);
> >> >> > +}
> >> >> > +
> >> >> > +static void
> >> >> > ?set_ioapic_affinity_irq(unsigned int irq, const struct cpumask *mask)
> >> >> > ?{
> >> >> > ? ? ? ?struct irq_desc *desc;
> >> >> > --
> >> >>
> >> >> it seems, ack_apic_level() already checked io_apic_level_ack_pending()
> >> >>
> >> >> ? ? ? ? ? ? ? ? cfg = desc->chip_data;
> >> >> ? ? ? ? ? ? ? ? if (!io_apic_level_ack_pending(cfg))
> >> >> ? ? ? ? ? ? ? ? ? ? ? ? move_masked_irq(irq);
> >> >> ? ? ? ? ? ? ? ? unmask_IO_APIC_irq_desc(desc);
> >> >
> >> > Yes, I have actually observed instances where the command
> >> > ? `echo $IRQ_MASK > /proc/irq/$VICTIM_IRQ/smp_affinity`
> >> > in the above test script failed to modify the affinity due to
> >> > a debug printk confirmed non-zero return from io_apic_level_ack_pending()
> >> > at this location. ?However, this is definitely not catching all the cases.
> >> > This was confirmed this with a printk in __target_IO_APIC_irq() that would
> >> > sometimes shows the Remote IRR bit set before the io_apic_modify() call.
> >> > e.g.
> >> > __target_IO_APIC_irq: XXX before io_apic_modify irq=25 vector=0xd9 reg=0x1e0d9
> >>
> >>
> >> that is strange, before that irq is masked,
> >>
> >> #ifdef CONFIG_GENERIC_PENDING_IRQ
> >> /* If we are moving the irq we need to mask it */
> >> if (unlikely(desc->status & IRQ_MOVE_PENDING)) {
> >> do_unmask_irq = 1;
> >> mask_IO_APIC_irq_desc(desc);
> >> }
> >> #endif
> >
> > When the device is very actively generating IRQs I suspect
> > that __target_IO_APIC_irq() is sometimes visited (via CPU
> > offline motivated path) before the IRQ is masked at this location.
> > I believe the vulnerability window is actually a bit larger
> > because it spans this mask_IO_APIC_irq_desc() call.
>
> I don't think it makes sense to debate how broken fixup_irqs is.
> The code is fundamentally wrong, if you don't have code that
> can migrate an irq in process context.
>
> Not playing games when Remote_IRR is set is the easy part. As
> that is required on correctly functioning hardware. Unless
> your ioapic has the magic acknowledge register.
>
> The hard part is that ioapic to put it politely are fragile
> things so you have to handle them just so. It is possible to
> wedge the state machines into a unrecoverable mess on most
> common ioapic implementations. Not sending the ack when
> the ioapic is ready to receive it is easy by comparison.
The I/O redirection table register write with the remote
IRR set issue has reproduced on every IBM System x server
I have tried including the x460, x3850, x3550 M2, and x3950 M2.
Nobody has responded to my request to test for the presence
of this issue on non-IBM systems but I think it is pretty safe
to assume that the problem is not IBM specific.
After incorporating the fix that avoids writes to the
the I/O redirection table registers while the remote IRR
bit is set, I have _not_ observed any other issues that
might be related to the ioapic fragility that you mentioned.
This of course does not prove that you are wrong and that
there is not a need for the changes you are suggesting.
However, until someone has the bandwidth to tackle the difficult
changes that you suggest, I propose that we continue to repair
problems that are found in the current implementation with fixes
such as those that I provided.
Thanks,
Gary
--
Gary Hade
System x Enablement
IBM Linux Technology Center
503-578-4503 IBM T/L: 775-4503
[email protected]
http://www.ibm.com/linux/ltc
Gary Hade <[email protected]> writes:
> The I/O redirection table register write with the remote
> IRR set issue has reproduced on every IBM System x server
> I have tried including the x460, x3850, x3550 M2, and x3950 M2.
> Nobody has responded to my request to test for the presence
> of this issue on non-IBM systems but I think it is pretty safe
> to assume that the problem is not IBM specific.
There is no question. The problem can be verified from a simple
code inspection. It results from the brokenness of the current
fixup_irqs.
Your suggested change at the very least is likely to result in
dropped irqs at runtime. Something some drivers do not
tolerate well.
> After incorporating the fix that avoids writes to the
> the I/O redirection table registers while the remote IRR
> bit is set, I have _not_ observed any other issues that
> might be related to the ioapic fragility that you mentioned.
> This of course does not prove that you are wrong and that
> there is not a need for the changes you are suggesting.
> However, until someone has the bandwidth to tackle the difficult
> changes that you suggest, I propose that we continue to repair
> problems that are found in the current implementation with fixes
> such as those that I provided.
The changes I suggest are not difficult.
How is APIC routing setup on your hardware?
"Setting APIC routing to flat" Is what my laptop reports.
My apologies for asking it badly last time. For understanding what
you are testing that is a critical piece of information.
Below is my draft patch that stops cpu shutdown when irqs can not be
migrated off. The code to do that is trivial, and is guaranteed
to fix all of your lost irq problems.
Beyond that everything I am proposing is very localized.
You have proposed making interrupts behave like they can be migrated
in process context when in fact extensive testing over the years have
shown in the general case interrupts can only be migrated from the irq
handler, from a location where the irqs are naturally disabled.
I propose detecting the cases that we know are safe to migrate in
process context, aka logical deliver with less than 8 cpus aka "flat"
routing mode and modifying the code so that those work in process
context and simply deny cpu hotplug in all of the rest of the cases.
That gives us: Code that always works. An incremental easily testable
path. No need to loose sleep at night worrying about the weird failure
modes. Ultimately less complex code.
Eric
diff --git a/arch/x86/include/asm/irq.h b/arch/x86/include/asm/irq.h
index f38481b..c585a0e 100644
--- a/arch/x86/include/asm/irq.h
+++ b/arch/x86/include/asm/irq.h
@@ -47,4 +47,12 @@ extern unsigned int do_IRQ(struct pt_regs *regs);
extern DECLARE_BITMAP(used_vectors, NR_VECTORS);
extern int vector_used_by_percpu_irq(unsigned int vector);
+#ifdef CONFIG_X86_IO_APIC
+extern void cleanup_pending_irqs(unsigned int cpu);
+#else
+static inline void cleanup_pending_irqs(unsigned int cpu)
+{
+}
+#endif
+
#endif /* _ASM_X86_IRQ_H */
diff --git a/arch/x86/kernel/apic/io_apic.c b/arch/x86/kernel/apic/io_apic.c
index 767fe7e..88a5235 100644
--- a/arch/x86/kernel/apic/io_apic.c
+++ b/arch/x86/kernel/apic/io_apic.c
@@ -4202,3 +4202,57 @@ static int __init ioapic_insert_resources(void)
/* Insert the IO APIC resources after PCI initialization has occured to handle
* IO APICS that are mapped in on a BAR in PCI space. */
late_initcall(ioapic_insert_resources);
+
+#ifndef CONFIG_HOTPLUG_CPUS
+void cleanup_pending_irqs(unsigned me)
+{
+ /* Called with irqs disabled */
+ unsigned vector;
+
+ for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
+ unsigned long flags;
+ unsigned int irq;
+ unsigned int irr;
+ struct irq_desc *desc;
+ struct irq_cfg *cfg;
+ int bug = 0;
+
+ irq = __get_cpu_var(vector_irq)[vector];
+
+ if (irq == -1)
+ continue;
+
+ desc = irq_to_desc(irq);
+ if (!desc)
+ continue;
+
+ cfg = irq_cfg(irq);
+
+ spin_lock_irqsave(&desc->lock, flags);
+ bug = 1;
+ if (!cfg->move_cleanup_count)
+ goto unlock;
+
+ if (vector == cfg->vector && cpumask_test_cpu(me, cfg->domain))
+ goto unlock;
+
+ bug = 0; /* It's not a bug only if a move is pending */
+
+ irr = apic_read(APIC_IRR + (vector / 32 * 0x10));
+ /*
+ * Check if the vector that needs to be cleaned up is
+ * registered at the cpu's IRR. If so then resend the
+ * irq so we don't loose it.
+ */
+ if (irr & (1 << (vector % 32)))
+ desc->chip->retrigger(irq);
+ __get_cpu_var(vector_irq)[vector] = -1;
+ cfg->move_cleanup_count--;
+ unlock:
+ spin_unlock_irqrestore(&desc->lock, flags);
+
+ if (bug)
+ printk(KERN_EMERG "irq: %i left on downed cpu %d\n", irq, me);
+ }
+}
+#endif
diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c
index c3fe010..e2082ad 100644
--- a/arch/x86/kernel/irq.c
+++ b/arch/x86/kernel/irq.c
@@ -253,3 +253,98 @@ void smp_generic_interrupt(struct pt_regs *regs)
}
EXPORT_SYMBOL_GPL(vector_used_by_percpu_irq);
+
+#ifdef CONFIG_HOTPLUG_CPU
+static int irq_cpu_notify(struct notifier_block *self, unsigned long action,
+ void *hcpu)
+{
+ int cpu = (long)hcpu;
+ unsigned int irq;
+ struct irq_desc *desc;
+ int ret = NOTIFY_OK;
+
+ if ((action != CPU_DOWN_PREPARE) && (action != CPU_DOWN_PREPARE_FROZEN))
+ goto out;
+
+ for_each_irq_desc(irq, desc) {
+ if (!desc)
+ continue;
+ if (irq == 2)
+ continue;
+ if (desc->status & IRQ_MOVE_PCNTXT)
+ continue;
+ if (!cpu_isset(cpu, *desc->affinity))
+ continue;
+#if 0
+ /* Should I allow these? */
+ if (!desc->action)
+ continue;
+ if (desc->status & IRQ_MASKED)
+ continue;
+#endif
+ ret = NOTIFY_BAD;
+ goto out;
+ }
+out:
+#if 1
+ printk(KERN_DEBUG "%s action: %lx cpu_down_prepare: %d cpu_down_prepare_frozen: %d done\n",
+ __func__, action,
+ (action == CPU_DOWN_PREPARE), (action == CPU_DOWN_PREPARE_FROZEN));
+#endif
+ return ret;
+}
+
+static struct notifier_block irq_cpu_notifier = {
+ .notifier_call = irq_cpu_notify,
+};
+
+static __init int setup_irq_cpu_notifier(void)
+{
+#if 1
+ printk(KERN_DEBUG "%s\n",__func__);
+#endif
+ return register_cpu_notifier(&irq_cpu_notifier);
+}
+module_init(setup_irq_cpu_notifier);
+
+
+/* The current cpu has been removed from cpu_online_mask. Reset irq affinities. */
+void fixup_irqs(void)
+{
+ unsigned int irq;
+ static int warned;
+ struct irq_desc *desc;
+ int cpu = smp_processor_id();
+
+ for_each_irq_desc(irq, desc) {
+ const struct cpumask *affinity;
+ int ret;
+
+ if (!desc)
+ continue;
+ if (irq == 2)
+ continue;
+ affinity = desc->affinity;
+ if (!cpu_isset(cpu, *affinity))
+ continue;
+ if (cpumask_any_and(affinity, cpu_online_mask) >= nr_cpu_ids) {
+ printk("Breaking affinity for irq %i\n", irq);
+ affinity = cpu_all_mask;
+ }
+ ret = -EINVAL;
+ if (desc->status & IRQ_MOVE_PCNTXT)
+ ret = irq_set_affinity(irq, affinity);
+ if (ret && desc->action && !(warned++))
+ printk("Cannot set affinity for irq %i\n", irq);
+#if 1
+ printk(KERN_DEBUG "irq: %i moved: %d action: %p\n",
+ irq, ret == 0, desc->action);
+#endif
+ }
+ cleanup_pending_irqs(cpu);
+#if 1
+ printk(KERN_DEBUG "%s done\n", __func__);
+#endif
+}
+
+#endif /* CONFIG_HOTPLUG_CPU */
diff --git a/arch/x86/kernel/irq_32.c b/arch/x86/kernel/irq_32.c
index 3b09634..e11eea7 100644
--- a/arch/x86/kernel/irq_32.c
+++ b/arch/x86/kernel/irq_32.c
@@ -212,48 +212,3 @@ bool handle_irq(unsigned irq, struct pt_regs *regs)
return true;
}
-#ifdef CONFIG_HOTPLUG_CPU
-
-/* A cpu has been removed from cpu_online_mask. Reset irq affinities. */
-void fixup_irqs(void)
-{
- unsigned int irq;
- static int warned;
- struct irq_desc *desc;
-
- for_each_irq_desc(irq, desc) {
- const struct cpumask *affinity;
-
- if (!desc)
- continue;
- if (irq == 2)
- continue;
-
- affinity = desc->affinity;
- if (cpumask_any_and(affinity, cpu_online_mask) >= nr_cpu_ids) {
- printk("Breaking affinity for irq %i\n", irq);
- affinity = cpu_all_mask;
- }
- if (desc->chip->set_affinity)
- desc->chip->set_affinity(irq, affinity);
- else if (desc->action && !(warned++))
- printk("Cannot set affinity for irq %i\n", irq);
- }
-
-#if 0
- barrier();
- /* Ingo Molnar says: "after the IO-APIC masks have been redirected
- [note the nop - the interrupt-enable boundary on x86 is two
- instructions from sti] - to flush out pending hardirqs and
- IPIs. After this point nothing is supposed to reach this CPU." */
- __asm__ __volatile__("sti; nop; cli");
- barrier();
-#else
- /* That doesn't seem sufficient. Give it 1ms. */
- local_irq_enable();
- mdelay(1);
- local_irq_disable();
-#endif
-}
-#endif
-
diff --git a/arch/x86/kernel/irq_64.c b/arch/x86/kernel/irq_64.c
index 977d8b4..c2f9bd5 100644
--- a/arch/x86/kernel/irq_64.c
+++ b/arch/x86/kernel/irq_64.c
@@ -62,65 +62,6 @@ bool handle_irq(unsigned irq, struct pt_regs *regs)
return true;
}
-#ifdef CONFIG_HOTPLUG_CPU
-/* A cpu has been removed from cpu_online_mask. Reset irq affinities. */
-void fixup_irqs(void)
-{
- unsigned int irq;
- static int warned;
- struct irq_desc *desc;
-
- for_each_irq_desc(irq, desc) {
- int break_affinity = 0;
- int set_affinity = 1;
- const struct cpumask *affinity;
-
- if (!desc)
- continue;
- if (irq == 2)
- continue;
-
- /* interrupt's are disabled at this point */
- spin_lock(&desc->lock);
-
- affinity = desc->affinity;
- if (!irq_has_action(irq) ||
- cpumask_equal(affinity, cpu_online_mask)) {
- spin_unlock(&desc->lock);
- continue;
- }
-
- if (cpumask_any_and(affinity, cpu_online_mask) >= nr_cpu_ids) {
- break_affinity = 1;
- affinity = cpu_all_mask;
- }
-
- if (desc->chip->mask)
- desc->chip->mask(irq);
-
- if (desc->chip->set_affinity)
- desc->chip->set_affinity(irq, affinity);
- else if (!(warned++))
- set_affinity = 0;
-
- if (desc->chip->unmask)
- desc->chip->unmask(irq);
-
- spin_unlock(&desc->lock);
-
- if (break_affinity && set_affinity)
- printk("Broke affinity for irq %i\n", irq);
- else if (!set_affinity)
- printk("Cannot set affinity for irq %i\n", irq);
- }
-
- /* That doesn't seem sufficient. Give it 1ms. */
- local_irq_enable();
- mdelay(1);
- local_irq_disable();
-}
-#endif
-
extern void call_softirq(void);
asmlinkage void do_softirq(void)
On Tue, Apr 28, 2009 at 03:27:36AM -0700, Eric W. Biederman wrote:
> Gary Hade <[email protected]> writes:
>
> > The I/O redirection table register write with the remote
> > IRR set issue has reproduced on every IBM System x server
> > I have tried including the x460, x3850, x3550 M2, and x3950 M2.
> > Nobody has responded to my request to test for the presence
> > of this issue on non-IBM systems but I think it is pretty safe
> > to assume that the problem is not IBM specific.
>
> There is no question. The problem can be verified from a simple
> code inspection. It results from the brokenness of the current
> fixup_irqs.
>
> Your suggested change at the very least is likely to result in
> dropped irqs at runtime.
Which of the two patches (2/3 or 3/3) could cause the dropped IRQs
and exactly how can this happen? If you are possibly referring to
the concerns about IRQs being migrated in process context during CPU
offlining, that design pre-dates my patches.
> Something some drivers do not
> tolerate well.
>
> > After incorporating the fix that avoids writes to the
> > the I/O redirection table registers while the remote IRR
> > bit is set, I have _not_ observed any other issues that
> > might be related to the ioapic fragility that you mentioned.
> > This of course does not prove that you are wrong and that
> > there is not a need for the changes you are suggesting.
> > However, until someone has the bandwidth to tackle the difficult
> > changes that you suggest, I propose that we continue to repair
> > problems that are found in the current implementation with fixes
> > such as those that I provided.
>
> The changes I suggest are not difficult.
>
> How is APIC routing setup on your hardware?
> "Setting APIC routing to flat" Is what my laptop reports.
Oh, thats easy. On the IBM x3550 M2 where I have confirmed that
both bugs exist and where I did the below described testing of
your patch, the APIC routing is shown as physical flat:
"Setting APIC routing to physical flat"
>
> My apologies for asking it badly last time.
No problem! Badly probably depends on the audience and
I'm probably not a particularily good one. :)
> For understanding what
> you are testing that is a critical piece of information.
>
> Below is my draft patch that stops cpu shutdown when irqs can not be
> migrated off. The code to do that is trivial, and is guaranteed
> to fix all of your lost irq problems.
This didn't help. Using 2.6.30-rc3 plus your patch both bugs
are unfortunately still present.
With respect to the race fixed by patch 2/3 that occurs when
the device is not very active I have found that a printk in
__assign_irq_vector() showing -EBUSY returns is a reliable
indicator of when things go bad and the CPU handling the IRQ
is offlined without the affinity being migrated to an online CPU.
The following was logged during the failure.
__assign_irq_vector: XXX irq=16 returning -EBUSY
__assign_irq_vector: XXX irq=16 returning -EBUSY
__assign_irq_vector: XXX irq=16 returning -EBUSY
__assign_irq_vector: XXX irq=16 returning -EBUSY
__assign_irq_vector: XXX irq=16 returning -EBUSY
__assign_irq_vector: XXX irq=16 returning -EBUSY
__assign_irq_vector: XXX irq=16 returning -EBUSY
__assign_irq_vector: XXX irq=16 returning -EBUSY
__assign_irq_vector: XXX irq=16 returning -EBUSY
__assign_irq_vector: XXX irq=16 returning -EBUSY
__assign_irq_vector: XXX irq=16 returning -EBUSY
I am not sure if I totally understand what your new
cleanup_pending_irqs() is doing but I *think* it is
only broadcasting an IPI when the same IRQ is being
handled on the IRQ move destination CPU. If this understanding
is correct, I don't think it covers the case where the device
is generating IRQs at such an extremely slow rate that
absolutely none are received between the time that the
move_in_progress flag was set to 1 and the IRQ move
destination CPU is offlined. I actually played around with
broadcasting an IPI every time the move_in_progress flag
was set to 1 which repaired the problem but did not feel
like a good solution especially after I discovered that there
was no need to even set move_in_progress to 1 if all CPUs in
the old domain were going to be offline when the cleanup code
was run i.e. cleanup code does nothing when it eventually runs.
I then incorporated my fix (patch 2/3) for the above issue
plus a printk in __target_IO_APIC_irq() to display values
written to the I/O redirection table register for the IRQ
of interest and tried again. With a low incoming IRQ rate
the above problem no longer reproduced but after increasing
the incoming IRQ rate I started seeing an increasing number
of writes to the I/O redirection table register while the
remote IRR bit was set (see below) before IRQs from the
device were no longer being seen by the kernel.
>
> Beyond that everything I am proposing is very localized.
>
> You have proposed making interrupts behave like they can be migrated
> in process context when in fact extensive testing over the years have
> shown in the general case interrupts can only be migrated from the irq
> handler, from a location where the irqs are naturally disabled.
To be clear, IRQs were already being migrated in process
context during CPU offlining before I even looked at the code.
This was not my doing. I believe it has been this way for
quite some time.
>
> I propose detecting the cases that we know are safe to migrate in
> process context, aka logical deliver with less than 8 cpus aka "flat"
> routing mode and modifying the code so that those work in process
> context and simply deny cpu hotplug in all of the rest of the cases.
Humm, are you suggesting that CPU offlining/onlining would not
be possible at all on systems with >8 logical CPUs (i.e. most
of our systems) or would this just force users to separately
migrate IRQ affinities away from a CPU (e.g. by shutting down
the irqbalance daemon and writing to /proc/irq/<irq>/smp_affinity)
before attempting to offline it?
Thanks,
Gary
--
Gary Hade
System x Enablement
IBM Linux Technology Center
503-578-4503 IBM T/L: 775-4503
[email protected]
http://www.ibm.com/linux/ltc
ioc0: LSISAS1ities={Initiatorities={Initiator}IC_irq: XXX irqPIC_irq: XXX irq=y with reg=0xa0fy with reg=0xa061:00.0: setting01:00.0: setting 64
melm3b55:~ # m__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a069
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a06a
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a06a
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a06a
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a072
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a07a
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a082
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a08a
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a08a
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a08a
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a08a
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a08a
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a08a
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a08a
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a08a
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a08a
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a08a
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a092
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a092
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a092
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a09a
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a2
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0aa
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0b2
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0b2
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0b2
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0b2
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0b2
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0b2
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0b2
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0b2
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0b2
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ba
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0ba
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ba
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ba
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0c2
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ca
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0d2
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0da
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0da
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0da
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0da
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0da
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0da
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0da
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0da
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0da
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0e2
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0e2
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0e2
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0e2
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ea
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a043
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a04b
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a053
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a053
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a053
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a053
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a053
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a053
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a053
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a053
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a053
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a05b
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e05b
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a05b
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a05b
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a063
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a06b
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a073
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a07b
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a07b
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a07b
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a07b
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a07b
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a07b
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a07b
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a07b
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a07b
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a083
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a083
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a083
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a083
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a08b
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a093
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a09b
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a3
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a3
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a3
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a3
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a3
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a3
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a3
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a3
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a3
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ab
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0ab
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ab
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ab
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0b3
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0bb
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0c3
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0cb
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0cb
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0cb
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0cb
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0cb
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0cb
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0cb
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0cb
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0cb
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0d3
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0d3
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0d3
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0d3
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0db
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0e3
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0eb
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a044
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a044
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a044
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a044
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a044
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a044
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e044
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a044
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a044
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a04c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a04c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a04c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a04c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a054
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a05c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a064
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a06c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e06c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e06c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a06c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e06c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e06c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e06c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a06c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a06c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a074
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a074
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a074
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a074
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a07c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a084
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a08c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a094
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a094
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a094
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a094
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a094
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a094
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a094
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a094
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a094
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a09c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e09c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a09c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e09c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ac
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0b4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0bc
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0bc
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0bc
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0bc
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0bc
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0bc
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0bc
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0bc
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0bc
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0cc
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0d4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0dc
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0e4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0e4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0e4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0e4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0e4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0e4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0e4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0e4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0e4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ec
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0ec
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ec
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ec
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a045
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a04d
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a055
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a05d
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e05d
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a05d
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a05d
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a05d
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a05d
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a05d
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a05d
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a05d
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a065
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a065
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a065
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e065
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a06d
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a075
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a07d
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a085
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a085
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e085
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a085
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e085
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a085
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a085
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a085
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e085
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a08d
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a08d
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e08d
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a08d
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a095
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a09d
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a5
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ad
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ad
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ad
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ad
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ad
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ad
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0ad
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0ad
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ad
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0b5
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0b5
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0b5
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0b5
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0bd
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0c5
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0cd
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0d5
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0d5
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0d5
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0d5
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0d5
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0d5
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0d5
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0d5
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0d5
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0dd
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0dd
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0dd
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0dd
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0e5
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a046
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a04e
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a056
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e056
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a056
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e056
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a056
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a056
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a056
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a056
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a056
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a05e
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a05e
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a05e
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a05e
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a066
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a06e
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a076
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a07e
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e07e
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a07e
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a07e
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a07e
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a07e
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e07e
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a07e
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e07e
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a086
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a086
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a086
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a086
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a08e
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a096
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a09e
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a6
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0a6
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a6
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a6
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a6
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a6
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0a6
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a6
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a6
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ae
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ae
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ae
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ae
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0b6
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0be
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0c6
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ce
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ce
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ce
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ce
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ce
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ce
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ce
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ce
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ce
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0d6
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0d6
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0d6
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0d6
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0de
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0e6
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a047
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a04f
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a04f
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a04f
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a04f
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a04f
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a04f
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a04f
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a04f
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a04f
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a057
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a057
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a057
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a057
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a05f
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a067
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a06f
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a077
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a077
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a077
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a077
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a077
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a077
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a077
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a077
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a077
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a07f
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e07f
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a07f
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a07f
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a087
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a08f
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a097
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a09f
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a09f
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a09f
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a09f
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a09f
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a09f
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a09f
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a09f
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a09f
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a7
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a7
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a7
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a7
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0af
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0b7
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0bf
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0c7
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0c7
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0c7
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0c7
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0c7
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0c7
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0c7
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0c7
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0c7
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0cf
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0cf
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0cf
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0cf
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0d7
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0df
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0e7
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a048
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a048
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a048
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a048
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a048
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a048
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a048
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a048
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a048
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a050
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a050
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a050
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a050
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a058
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a060
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a068
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a070
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a070
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a070
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a070
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a070
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a070
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a070
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a070
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a070
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a078
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a078
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a078
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a078
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a088
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a090
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a098
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a0
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a0
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a0
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a0
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a0
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a0
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a0
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a0
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0a0
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a8
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0a8
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a8
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a8
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0b0
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0b8
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0c0
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0c8
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0c8
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0c8
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0c8
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c8
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0c8
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0c8
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0c8
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0c8
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0d0
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0d0
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0d0
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0d0
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0d8
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0e0
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0e8
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a041
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a041
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a041
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a041
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a041
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a041
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a041
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a041
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a041
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a049
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a049
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a049
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a049
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a051
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a059
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a061
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a069
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a069
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a069
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a069
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a069
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a069
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a069
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a069
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a069
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a071
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e071
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a071
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a071
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a079
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a081
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a089
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a06a
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a06a
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a06a
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a06a
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a06a
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a06a
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a06a
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e06a
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a06a
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a072
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a072
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a072
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a072
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a07a
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a082
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a08a
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a092
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a092
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a092
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a092
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a092
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e092
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a092
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a092
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a092
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a09a
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a09a
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a09a
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a09a
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a2
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0aa
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0b2
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ba
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ba
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0ba
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ba
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ba
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0ba
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ba
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ba
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ba
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0c2
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0c2
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c2
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0c2
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ca
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0d2
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0da
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0e2
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0e2
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0e2
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0e2
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0e2
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0e2
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0e2
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0e2
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0e2
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ea
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0ea
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ea
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ea
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a043
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a04b
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a053
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a05b
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a05b
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a05b
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a05b
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a05b
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a05b
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a05b
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a05b
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e05b
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a063
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e063
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e063
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a063
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a06b
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a073
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a07b
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a083
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e083
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a083
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a083
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a083
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a083
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a083
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a083
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a083
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a08b
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a08b
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a08b
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a08b
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a093
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a09b
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a3
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ab
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ab
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ab
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ab
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ab
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ab
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ab
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ab
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0ab
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0b3
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0b3
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0b3
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0b3
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0bb
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0c3
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0cb
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0d3
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0d3
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0d3
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0d3
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0d3
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0d3
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0d3
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0d3
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0d3
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0db
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0db
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0db
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0db
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0e3
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0eb
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a044
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a04c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a04c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a04c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e04c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a04c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a04c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e04c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a04c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e04c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a054
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e054
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a054
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a054
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a05c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a064
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a06c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a074
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a074
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a074
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a074
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a074
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a074
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e074
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a074
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a074
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a07c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a07c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a07c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a07c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a084
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a08c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a094
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a09c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a09c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a09c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a09c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a09c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a09c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a09c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a09c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a09c
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0a4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1a0a4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0a4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0ac
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0b4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0bc
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
__target_IO_APIC_irq: XXX irq=16 io_apic_modify with reg=0x1e0c4
Gary Hade <[email protected]> writes:
> On Tue, Apr 28, 2009 at 03:27:36AM -0700, Eric W. Biederman wrote:
>> Gary Hade <[email protected]> writes:
>>
>> > The I/O redirection table register write with the remote
>> > IRR set issue has reproduced on every IBM System x server
>> > I have tried including the x460, x3850, x3550 M2, and x3950 M2.
>> > Nobody has responded to my request to test for the presence
>> > of this issue on non-IBM systems but I think it is pretty safe
>> > to assume that the problem is not IBM specific.
>>
>> There is no question. The problem can be verified from a simple
>> code inspection. It results from the brokenness of the current
>> fixup_irqs.
>>
>> Your suggested change at the very least is likely to result in
>> dropped irqs at runtime.
>
> Which of the two patches (2/3 or 3/3) could cause the dropped IRQs
> and exactly how can this happen? If you are possibly referring to
> the concerns about IRQs being migrated in process context during CPU
> offlining, that design pre-dates my patches.
I am referring to some of the side effects of a cpu being migrated in
process context during CPU offlining. That code has been
fundamentally broken since it was merged in 2005. Various changes
have changed the odds of how likely it is to fail, and how likely
people are to encounter problems.
The practice of masking an irq while it is actively being used and
we don't have that irq software pending means we will drop that
irq at some point, and that is what the code in irq_64.c:fixup_irqs
does today.
>From what I can tell your patches simply apply more code dubious
code to the fixup_irqs path.
>> Something some drivers do not
>> tolerate well.
>>
>> > After incorporating the fix that avoids writes to the
>> > the I/O redirection table registers while the remote IRR
>> > bit is set, I have _not_ observed any other issues that
>> > might be related to the ioapic fragility that you mentioned.
>> > This of course does not prove that you are wrong and that
>> > there is not a need for the changes you are suggesting.
>> > However, until someone has the bandwidth to tackle the difficult
>> > changes that you suggest, I propose that we continue to repair
>> > problems that are found in the current implementation with fixes
>> > such as those that I provided.
>>
>> The changes I suggest are not difficult.
>>
>> How is APIC routing setup on your hardware?
>> "Setting APIC routing to flat" Is what my laptop reports.
>
> Oh, thats easy. On the IBM x3550 M2 where I have confirmed that
> both bugs exist and where I did the below described testing of
> your patch, the APIC routing is shown as physical flat:
> "Setting APIC routing to physical flat"
>
>>
>> My apologies for asking it badly last time.
>
> No problem! Badly probably depends on the audience and
> I'm probably not a particularily good one. :)
>
>> For understanding what
>> you are testing that is a critical piece of information.
>>
>> Below is my draft patch that stops cpu shutdown when irqs can not be
>> migrated off. The code to do that is trivial, and is guaranteed
>> to fix all of your lost irq problems.
>
> This didn't help. Using 2.6.30-rc3 plus your patch both bugs
> are unfortunately still present.
You could offline the cpus? I know when I tested it on my
laptop I could not offline the cpus.
If you can offline cpus because your irqs have IRQ_MOVE_PCNT
set, then you must have an iommu and the work that needs to
be done to get things stable on your system is different.
> With respect to the race fixed by patch 2/3 that occurs when
> the device is not very active I have found that a printk in
> __assign_irq_vector() showing -EBUSY returns is a reliable
> indicator of when things go bad and the CPU handling the IRQ
> is offlined without the affinity being migrated to an online CPU.
> The following was logged during the failure.
> __assign_irq_vector: XXX irq=16 returning -EBUSY
> __assign_irq_vector: XXX irq=16 returning -EBUSY
> __assign_irq_vector: XXX irq=16 returning -EBUSY
> __assign_irq_vector: XXX irq=16 returning -EBUSY
> __assign_irq_vector: XXX irq=16 returning -EBUSY
> __assign_irq_vector: XXX irq=16 returning -EBUSY
> __assign_irq_vector: XXX irq=16 returning -EBUSY
> __assign_irq_vector: XXX irq=16 returning -EBUSY
> __assign_irq_vector: XXX irq=16 returning -EBUSY
> __assign_irq_vector: XXX irq=16 returning -EBUSY
> __assign_irq_vector: XXX irq=16 returning -EBUSY
I have not been looking closely at your reported bugs. So
far everything seems to be related to the totally broken moving
irqs in process context code in fixup_irqs, and making the
races smaller does not interest me when it looks like we can
make the races go away.
> I am not sure if I totally understand what your new
> cleanup_pending_irqs() is doing but I *think* it is
> only broadcasting an IPI when the same IRQ is being
> handled on the IRQ move destination CPU.
My new cleanup_pending_irqs() as written still has a number of bugs,
it is more of a placeholder than solid tested code at the moment.
cleanup_pending_irqs() is supposed to happen after a synchronous
irq migration (so no irqs should be hitting the cpu) and
as such it should be possible to cleanup any irq pending state
on the current cpu, and if an irq is pending to simply redirect
it to some other cpu that still has the irq active.
cleanup_pending_irqs() currently does not have the code to ack the
irqs that are pending in the irr, which could have a correctness
consequence with the hardware. Beyond that it is just a matter of
reflecting the irq to a different cpu (so we don't miss one).
> If this understanding
> is correct, I don't think it covers the case where the device
> is generating IRQs at such an extremely slow rate that
> absolutely none are received between the time that the
> move_in_progress flag was set to 1 and the IRQ move
> destination CPU is offlined. I actually played around with
> broadcasting an IPI every time the move_in_progress flag
> was set to 1 which repaired the problem but did not feel
> like a good solution especially after I discovered that there
> was no need to even set move_in_progress to 1 if all CPUs in
> the old domain were going to be offline when the cleanup code
> was run i.e. cleanup code does nothing when it eventually runs.
That case should be handled simply by denying cpu down event.
And that code was working when I tested it on my laptop.
I am wondering how you managed to get the cpu to go down.
> I then incorporated my fix (patch 2/3) for the above issue
> plus a printk in __target_IO_APIC_irq() to display values
> written to the I/O redirection table register for the IRQ
> of interest and tried again. With a low incoming IRQ rate
> the above problem no longer reproduced but after increasing
> the incoming IRQ rate I started seeing an increasing number
> of writes to the I/O redirection table register while the
> remote IRR bit was set (see below) before IRQs from the
> device were no longer being seen by the kernel.
>
>>
>> Beyond that everything I am proposing is very localized.
>>
>> You have proposed making interrupts behave like they can be migrated
>> in process context when in fact extensive testing over the years have
>> shown in the general case interrupts can only be migrated from the irq
>> handler, from a location where the irqs are naturally disabled.
>
> To be clear, IRQs were already being migrated in process
> context during CPU offlining before I even looked at the code.
> This was not my doing. I believe it has been this way for
> quite some time.
Yep. It doesn't mean it has ever been correct, and I am trying
to development in ways so that we only take down a cpu when it
is possible to migrate all of it's irqs in process context.
>> I propose detecting thpe cases that we know are safe to migrate in
>> process context, aka logical deliver with less than 8 cpus aka "flat"
>> routing mode and modifying the code so that those work in process
>> context and simply deny cpu hotplug in all of the rest of the cases.
>
> Humm, are you suggesting that CPU offlining/onlining would not
> be possible at all on systems with >8 logical CPUs (i.e. most
> of our systems) or would this just force users to separately
> migrate IRQ affinities away from a CPU (e.g. by shutting down
> the irqbalance daemon and writing to /proc/irq/<irq>/smp_affinity)
> before attempting to offline it?
A separate migration, for those hard to handle irqs.
The newest systems have iommus that irqs go through or are using MSIs
for the important irqs, and as such can be migrated in process
context. So this is not a restriction for future systems.
Eric
On Tue, Apr 28, 2009 at 06:44:56PM -0700, Eric W. Biederman wrote:
> Gary Hade <[email protected]> writes:
>
> > On Tue, Apr 28, 2009 at 03:27:36AM -0700, Eric W. Biederman wrote:
> >> Gary Hade <[email protected]> writes:
> >>
> >> > The I/O redirection table register write with the remote
> >> > IRR set issue has reproduced on every IBM System x server
> >> > I have tried including the x460, x3850, x3550 M2, and x3950 M2.
> >> > Nobody has responded to my request to test for the presence
> >> > of this issue on non-IBM systems but I think it is pretty safe
> >> > to assume that the problem is not IBM specific.
> >>
> >> There is no question. The problem can be verified from a simple
> >> code inspection. It results from the brokenness of the current
> >> fixup_irqs.
> >>
> >> Your suggested change at the very least is likely to result in
> >> dropped irqs at runtime.
> >
> > Which of the two patches (2/3 or 3/3) could cause the dropped IRQs
> > and exactly how can this happen? If you are possibly referring to
> > the concerns about IRQs being migrated in process context during CPU
> > offlining, that design pre-dates my patches.
>
> I am referring to some of the side effects of a cpu being migrated in
> process context during CPU offlining. That code has been
> fundamentally broken since it was merged in 2005. Various changes
> have changed the odds of how likely it is to fail, and how likely
> people are to encounter problems.
IMHO, my proposed fixes clearly reduce the odds of failure.
>
> The practice of masking an irq while it is actively being used and
> we don't have that irq software pending means we will drop that
> irq at some point, and that is what the code in irq_64.c:fixup_irqs
> does today.
>
> >From what I can tell your patches simply apply more code dubious
> code to the fixup_irqs path.
Well, they also fix two insidious problems that can render
the system unuseable.
>
> >> Something some drivers do not
> >> tolerate well.
> >>
> >> > After incorporating the fix that avoids writes to the
> >> > the I/O redirection table registers while the remote IRR
> >> > bit is set, I have _not_ observed any other issues that
> >> > might be related to the ioapic fragility that you mentioned.
> >> > This of course does not prove that you are wrong and that
> >> > there is not a need for the changes you are suggesting.
> >> > However, until someone has the bandwidth to tackle the difficult
> >> > changes that you suggest, I propose that we continue to repair
> >> > problems that are found in the current implementation with fixes
> >> > such as those that I provided.
> >>
> >> The changes I suggest are not difficult.
> >>
> >> How is APIC routing setup on your hardware?
> >> "Setting APIC routing to flat" Is what my laptop reports.
> >
> > Oh, thats easy. On the IBM x3550 M2 where I have confirmed that
> > both bugs exist and where I did the below described testing of
> > your patch, the APIC routing is shown as physical flat:
> > "Setting APIC routing to physical flat"
> >
> >>
> >> My apologies for asking it badly last time.
> >
> > No problem! Badly probably depends on the audience and
> > I'm probably not a particularily good one. :)
> >
> >> For understanding what
> >> you are testing that is a critical piece of information.
> >>
> >> Below is my draft patch that stops cpu shutdown when irqs can not be
> >> migrated off. The code to do that is trivial, and is guaranteed
> >> to fix all of your lost irq problems.
> >
> > This didn't help. Using 2.6.30-rc3 plus your patch both bugs
> > are unfortunately still present.
>
> You could offline the cpus? I know when I tested it on my
> laptop I could not offline the cpus.
Eric, I'm sorry! This was due to my stupid mistake. When I
went to apply your patch I included --dry-run to test it but
apparently got distracted and never actually ran patch(1)
without --dry-run. <SIGH>
So, I just rebuilt after _really_ applying the patch and got
the following result which probably to be what you intended.
elm3b55:/home/garyh/kernel # cat ./cpus_offline_all.sh
#!/bin/sh
##
# Offline all offlineable CPUs
##
SYS_CPU_DIR=/sys/devices/system/cpu
for d in $SYS_CPU_DIR/cpu[1-9] $SYS_CPU_DIR/cpu??; do
cpu="`basename $d`"
state=`cat $d/online`
if [ "$state" = 1 ]; then
echo $cpu: offlining
echo 0 > $d/online
else
echo $cpu: already offline
fi
done
elm3b55:/home/garyh/kernel # ./cpus_offline_all.sh
cpu1: offlining
./cpus_offline_all.sh: line 14: echo: write error: Invalid argument
cpu2: offlining
./cpus_offline_all.sh: line 14: echo: write error: Invalid argument
cpu3: offlining
./cpus_offline_all.sh: line 14: echo: write error: Invalid argument
cpu4: offlining
./cpus_offline_all.sh: line 14: echo: write error: Invalid argument
cpu5: offlining
./cpus_offline_all.sh: line 14: echo: write error: Invalid argument
cpu6: offlining
./cpus_offline_all.sh: line 14: echo: write error: Invalid argument
cpu7: offlining
./cpus_offline_all.sh: line 14: echo: write error: Invalid argument
cpu8: offlining
./cpus_offline_all.sh: line 14: echo: write error: Invalid argument
cpu9: offlining
./cpus_offline_all.sh: line 14: echo: write error: Invalid argument
cpu10: offlining
./cpus_offline_all.sh: line 14: echo: write error: Invalid argument
cpu11: offlining
./cpus_offline_all.sh: line 14: echo: write error: Invalid argument
cpu12: offlining
./cpus_offline_all.sh: line 14: echo: write error: Invalid argument
cpu13: offlining
./cpus_offline_all.sh: line 14: echo: write error: Invalid argument
cpu14: offlining
./cpus_offline_all.sh: line 14: echo: write error: Invalid argument
cpu15: offlining
./cpus_offline_all.sh: line 14: echo: write error: Invalid argument
>
> If you can offline cpus because your irqs have IRQ_MOVE_PCNT
> set, then you must have an iommu and the work that needs to
> be done to get things stable on your system is different.
>
> > With respect to the race fixed by patch 2/3 that occurs when
> > the device is not very active I have found that a printk in
> > __assign_irq_vector() showing -EBUSY returns is a reliable
> > indicator of when things go bad and the CPU handling the IRQ
> > is offlined without the affinity being migrated to an online CPU.
> > The following was logged during the failure.
> > __assign_irq_vector: XXX irq=16 returning -EBUSY
> > __assign_irq_vector: XXX irq=16 returning -EBUSY
> > __assign_irq_vector: XXX irq=16 returning -EBUSY
> > __assign_irq_vector: XXX irq=16 returning -EBUSY
> > __assign_irq_vector: XXX irq=16 returning -EBUSY
> > __assign_irq_vector: XXX irq=16 returning -EBUSY
> > __assign_irq_vector: XXX irq=16 returning -EBUSY
> > __assign_irq_vector: XXX irq=16 returning -EBUSY
> > __assign_irq_vector: XXX irq=16 returning -EBUSY
> > __assign_irq_vector: XXX irq=16 returning -EBUSY
> > __assign_irq_vector: XXX irq=16 returning -EBUSY
>
> I have not been looking closely at your reported bugs. So
> far everything seems to be related to the totally broken moving
> irqs in process context code in fixup_irqs, and making the
> races smaller does not interest me when it looks like we can
> make the races go away.
>
> > I am not sure if I totally understand what your new
> > cleanup_pending_irqs() is doing but I *think* it is
> > only broadcasting an IPI when the same IRQ is being
> > handled on the IRQ move destination CPU.
>
> My new cleanup_pending_irqs() as written still has a number of bugs,
> it is more of a placeholder than solid tested code at the moment.
>
> cleanup_pending_irqs() is supposed to happen after a synchronous
> irq migration (so no irqs should be hitting the cpu) and
> as such it should be possible to cleanup any irq pending state
> on the current cpu, and if an irq is pending to simply redirect
> it to some other cpu that still has the irq active.
>
> cleanup_pending_irqs() currently does not have the code to ack the
> irqs that are pending in the irr, which could have a correctness
> consequence with the hardware. Beyond that it is just a matter of
> reflecting the irq to a different cpu (so we don't miss one).
>
> > If this understanding
> > is correct, I don't think it covers the case where the device
> > is generating IRQs at such an extremely slow rate that
> > absolutely none are received between the time that the
> > move_in_progress flag was set to 1 and the IRQ move
> > destination CPU is offlined. I actually played around with
> > broadcasting an IPI every time the move_in_progress flag
> > was set to 1 which repaired the problem but did not feel
> > like a good solution especially after I discovered that there
> > was no need to even set move_in_progress to 1 if all CPUs in
> > the old domain were going to be offline when the cleanup code
> > was run i.e. cleanup code does nothing when it eventually runs.
>
> That case should be handled simply by denying cpu down event.
> And that code was working when I tested it on my laptop.
>
> I am wondering how you managed to get the cpu to go down.
>
> > I then incorporated my fix (patch 2/3) for the above issue
> > plus a printk in __target_IO_APIC_irq() to display values
> > written to the I/O redirection table register for the IRQ
> > of interest and tried again. With a low incoming IRQ rate
> > the above problem no longer reproduced but after increasing
> > the incoming IRQ rate I started seeing an increasing number
> > of writes to the I/O redirection table register while the
> > remote IRR bit was set (see below) before IRQs from the
> > device were no longer being seen by the kernel.
> >
> >>
> >> Beyond that everything I am proposing is very localized.
> >>
> >> You have proposed making interrupts behave like they can be migrated
> >> in process context when in fact extensive testing over the years have
> >> shown in the general case interrupts can only be migrated from the irq
> >> handler, from a location where the irqs are naturally disabled.
> >
> > To be clear, IRQs were already being migrated in process
> > context during CPU offlining before I even looked at the code.
> > This was not my doing. I believe it has been this way for
> > quite some time.
>
> Yep. It doesn't mean it has ever been correct, and I am trying
> to development in ways so that we only take down a cpu when it
> is possible to migrate all of it's irqs in process context.
>
> >> I propose detecting thpe cases that we know are safe to migrate in
> >> process context, aka logical deliver with less than 8 cpus aka "flat"
> >> routing mode and modifying the code so that those work in process
> >> context and simply deny cpu hotplug in all of the rest of the cases.
> >
> > Humm, are you suggesting that CPU offlining/onlining would not
> > be possible at all on systems with >8 logical CPUs (i.e. most
> > of our systems) or would this just force users to separately
> > migrate IRQ affinities away from a CPU (e.g. by shutting down
> > the irqbalance daemon and writing to /proc/irq/<irq>/smp_affinity)
> > before attempting to offline it?
>
> A separate migration, for those hard to handle irqs.
>
> The newest systems have iommus that irqs go through or are using MSIs
> for the important irqs, and as such can be migrated in process
> context. So this is not a restriction for future systems.
I understand your concerns but we need a solution for the
earlier systems that does NOT remove or cripple the existing
CPU hotplug functionality. If you can come up with a way to
retain CPU hotplug function while doing all IRQ migration in
interrupt context I would certainly be willing to try to find
some time to help test and debug your changes on our systems.
Thanks,
Gary
--
Gary Hade
System x Enablement
IBM Linux Technology Center
503-578-4503 IBM T/L: 775-4503
[email protected]
http://www.ibm.com/linux/ltc
Gary Hade <[email protected]> writes:
>> > This didn't help. Using 2.6.30-rc3 plus your patch both bugs
>> > are unfortunately still present.
>>
>> You could offline the cpus? I know when I tested it on my
>> laptop I could not offline the cpus.
>
> Eric, I'm sorry! This was due to my stupid mistake. When I
> went to apply your patch I included --dry-run to test it but
> apparently got distracted and never actually ran patch(1)
> without --dry-run. <SIGH>
>
> So, I just rebuilt after _really_ applying the patch and got
> the following result which probably to be what you intended.
Ok. Good to see.
>> >> I propose detecting thpe cases that we know are safe to migrate in
>> >> process context, aka logical deliver with less than 8 cpus aka "flat"
>> >> routing mode and modifying the code so that those work in process
>> >> context and simply deny cpu hotplug in all of the rest of the cases.
>> >
>> > Humm, are you suggesting that CPU offlining/onlining would not
>> > be possible at all on systems with >8 logical CPUs (i.e. most
>> > of our systems) or would this just force users to separately
>> > migrate IRQ affinities away from a CPU (e.g. by shutting down
>> > the irqbalance daemon and writing to /proc/irq/<irq>/smp_affinity)
>> > before attempting to offline it?
>>
>> A separate migration, for those hard to handle irqs.
>>
>> The newest systems have iommus that irqs go through or are using MSIs
>> for the important irqs, and as such can be migrated in process
>> context. So this is not a restriction for future systems.
>
> I understand your concerns but we need a solution for the
> earlier systems that does NOT remove or cripple the existing
> CPU hotplug functionality. If you can come up with a way to
> retain CPU hotplug function while doing all IRQ migration in
> interrupt context I would certainly be willing to try to find
> some time to help test and debug your changes on our systems.
Well that is ultimately what I am looking towards.
How do we move to a system that works by design, instead of
one with design goals that are completely conflicting.
Thinking about it, we should be able to preemptively migrate
irqs in the hook I am using that denies cpu hotplug.
If they don't migrate after a short while I expect we should
still fail but that would relieve some of the pain, and certainly
prevent a non-working system.
There are little bits we can tweak like special casing irqs that
no-one is using.
My preference here is that I would rather deny cpu hotplug unplug than
have the non-working system problems that you have seen.
All of that said I have some questions about your hardware.
- How many sockets and how many cores do you have?
- How many irqs do you have?
- Do you have an iommu that irqs can go through?
If you have <= 8 cores this problem is totally solvable.
Other cases may be but I don't know what the tradeoffs are.
For very large systems we don't have enough irqs without
limiting running in physical flat mode which makes things
even more of a challenge.
It may also be that your ioapics don't have the bugs that
intel and amd ioapics have and we could have a way to recognize
high quality ioapics.
Eric
On Wed, Apr 29, 2009 at 10:46:29AM -0700, Eric W. Biederman wrote:
> Gary Hade <[email protected]> writes:
>
> >> > This didn't help. Using 2.6.30-rc3 plus your patch both bugs
> >> > are unfortunately still present.
> >>
> >> You could offline the cpus? I know when I tested it on my
> >> laptop I could not offline the cpus.
> >
> > Eric, I'm sorry! This was due to my stupid mistake. When I
> > went to apply your patch I included --dry-run to test it but
> > apparently got distracted and never actually ran patch(1)
> > without --dry-run. <SIGH>
> >
> > So, I just rebuilt after _really_ applying the patch and got
> > the following result which probably to be what you intended.
>
> Ok. Good to see.
>
> >> >> I propose detecting thpe cases that we know are safe to migrate in
> >> >> process context, aka logical deliver with less than 8 cpus aka "flat"
> >> >> routing mode and modifying the code so that those work in process
> >> >> context and simply deny cpu hotplug in all of the rest of the cases.
> >> >
> >> > Humm, are you suggesting that CPU offlining/onlining would not
> >> > be possible at all on systems with >8 logical CPUs (i.e. most
> >> > of our systems) or would this just force users to separately
> >> > migrate IRQ affinities away from a CPU (e.g. by shutting down
> >> > the irqbalance daemon and writing to /proc/irq/<irq>/smp_affinity)
> >> > before attempting to offline it?
> >>
> >> A separate migration, for those hard to handle irqs.
> >>
> >> The newest systems have iommus that irqs go through or are using MSIs
> >> for the important irqs, and as such can be migrated in process
> >> context. So this is not a restriction for future systems.
> >
> > I understand your concerns but we need a solution for the
> > earlier systems that does NOT remove or cripple the existing
> > CPU hotplug functionality. If you can come up with a way to
> > retain CPU hotplug function while doing all IRQ migration in
> > interrupt context I would certainly be willing to try to find
> > some time to help test and debug your changes on our systems.
>
> Well that is ultimately what I am looking towards.
>
> How do we move to a system that works by design, instead of
> one with design goals that are completely conflicting.
>
> Thinking about it, we should be able to preemptively migrate
> irqs in the hook I am using that denies cpu hotplug.
>
> If they don't migrate after a short while I expect we should
> still fail but that would relieve some of the pain, and certainly
> prevent a non-working system.
>
> There are little bits we can tweak like special casing irqs that
> no-one is using.
>
> My preference here is that I would rather deny cpu hotplug unplug than
> have the non-working system problems that you have seen.
>
> All of that said I have some questions about your hardware.
> - How many sockets and how many cores do you have?
The largest is the x3950 M2 with up to 16 sockets and
96 cores in currently supported configurations and I
expect that there could be at least double those numbers
in the future.
http://www-03.ibm.com/systems/x/hardware/enterprise/x3950m2/index.html
> - How many irqs do you have?
On the single node x3950 M2 that I have been using with
all of it's 7 PCIe slots vacant I see:
[root@elm3c160 ~]# cat /proc/interrupts | wc -l
21
Up to 4 nodes are currently supported and I expect
that there could be at least double that number in
the future.
> - Do you have an iommu that irqs can go through?
Only a subset of our systems (e.g. x460, x3850, x3950
w/Calgary iommu) have this.
>
> If you have <= 8 cores this problem is totally solvable.
Dreamer :-)
>
> Other cases may be but I don't know what the tradeoffs are.
> For very large systems we don't have enough irqs without
> limiting running in physical flat mode which makes things
> even more of a challenge.
>
> It may also be that your ioapics don't have the bugs that
> intel and amd ioapics have and we could have a way to recognize
> high quality ioapics.
I believe all our System x boxes have Intel and AMD ioapics.
Gary
--
Gary Hade
System x Enablement
IBM Linux Technology Center
503-578-4503 IBM T/L: 775-4503
[email protected]
http://www.ibm.com/linux/ltc
On Thu, Apr 30, 2009 at 11:15:46AM -0700, Gary Hade wrote:
> On Wed, Apr 29, 2009 at 10:46:29AM -0700, Eric W. Biederman wrote:
> > Gary Hade <[email protected]> writes:
> >
> > >> > This didn't help. Using 2.6.30-rc3 plus your patch both bugs
> > >> > are unfortunately still present.
> > >>
> > >> You could offline the cpus? I know when I tested it on my
> > >> laptop I could not offline the cpus.
> > >
> > > Eric, I'm sorry! This was due to my stupid mistake. When I
> > > went to apply your patch I included --dry-run to test it but
> > > apparently got distracted and never actually ran patch(1)
> > > without --dry-run. <SIGH>
> > >
> > > So, I just rebuilt after _really_ applying the patch and got
> > > the following result which probably to be what you intended.
> >
> > Ok. Good to see.
> >
> > >> >> I propose detecting thpe cases that we know are safe to migrate in
> > >> >> process context, aka logical deliver with less than 8 cpus aka "flat"
> > >> >> routing mode and modifying the code so that those work in process
> > >> >> context and simply deny cpu hotplug in all of the rest of the cases.
> > >> >
> > >> > Humm, are you suggesting that CPU offlining/onlining would not
> > >> > be possible at all on systems with >8 logical CPUs (i.e. most
> > >> > of our systems) or would this just force users to separately
> > >> > migrate IRQ affinities away from a CPU (e.g. by shutting down
> > >> > the irqbalance daemon and writing to /proc/irq/<irq>/smp_affinity)
> > >> > before attempting to offline it?
> > >>
> > >> A separate migration, for those hard to handle irqs.
> > >>
> > >> The newest systems have iommus that irqs go through or are using MSIs
> > >> for the important irqs, and as such can be migrated in process
> > >> context. So this is not a restriction for future systems.
> > >
> > > I understand your concerns but we need a solution for the
> > > earlier systems that does NOT remove or cripple the existing
> > > CPU hotplug functionality. If you can come up with a way to
> > > retain CPU hotplug function while doing all IRQ migration in
> > > interrupt context I would certainly be willing to try to find
> > > some time to help test and debug your changes on our systems.
> >
> > Well that is ultimately what I am looking towards.
> >
> > How do we move to a system that works by design, instead of
> > one with design goals that are completely conflicting.
> >
> > Thinking about it, we should be able to preemptively migrate
> > irqs in the hook I am using that denies cpu hotplug.
> >
> > If they don't migrate after a short while I expect we should
> > still fail but that would relieve some of the pain, and certainly
> > prevent a non-working system.
> >
> > There are little bits we can tweak like special casing irqs that
> > no-one is using.
> >
> > My preference here is that I would rather deny cpu hotplug unplug than
> > have the non-working system problems that you have seen.
> >
> > All of that said I have some questions about your hardware.
> > - How many sockets and how many cores do you have?
>
> The largest is the x3950 M2 with up to 16 sockets and
> 96 cores in currently supported configurations and I
> expect that there could be at least double those numbers
> in the future.
> http://www-03.ibm.com/systems/x/hardware/enterprise/x3950m2/index.html
>
> > - How many irqs do you have?
>
> On the single node x3950 M2 that I have been using with
> all of it's 7 PCIe slots vacant I see:
> [root@elm3c160 ~]# cat /proc/interrupts | wc -l
> 21
> Up to 4 nodes are currently supported and I expect
> that there could be at least double that number in
> the future.
>
> > - Do you have an iommu that irqs can go through?
>
> Only a subset of our systems (e.g. x460, x3850, x3950
> w/Calgary iommu) have this.
>
> >
> > If you have <= 8 cores this problem is totally solvable.
>
> Dreamer :-)
>
> >
> > Other cases may be but I don't know what the tradeoffs are.
> > For very large systems we don't have enough irqs without
> > limiting running in physical flat mode which makes things
> > even more of a challenge.
> >
> > It may also be that your ioapics don't have the bugs that
> > intel and amd ioapics have and we could have a way to recognize
> > high quality ioapics.
>
> I believe all our System x boxes have Intel and AMD ioapics.
Actually, I should have said that many System x boxes have
Intel or AMD ioapics. It is my understanding that the ioapic
function on some of the high-end systems is integrated into
the IBM chipset (e.g. Calgary and CalIOC2). However, since
I have also seen the I/O redirection table register write with
remote IRR bit set issue on some of those systems it probably
doesn't make sense treat them any differently.
Gary
--
Gary Hade
System x Enablement
IBM Linux Technology Center
503-578-4503 IBM T/L: 775-4503
[email protected]
http://www.ibm.com/linux/ltc