This patchset adds a new CPU mask for SMP systems to the irq_desc
struct. It also exposes an API for underlying device drivers to
assist irqbalance in making smarter decisions when balancing, especially
in a NUMA environment. For example, an ethernet driver with MSI-X may
wish to limit the CPUs that an interrupt can be balanced within to
stay on a single NUMA node. Current irqbalance operation can move the
interrupt off the node, resulting in cross-node memory accesses and
locks.
The API is a get/set API within the kernel, along with a /proc entry
for the interrupt.
Signed-off-by: Peter P Waskiewicz Jr <[email protected]>
---
include/linux/interrupt.h | 8 ++++++
include/linux/irq.h | 2 ++
kernel/irq/manage.c | 32 +++++++++++++++++++++++++
kernel/irq/proc.c | 57 +++++++++++++++++++++++++++++++++++++++++++++
4 files changed, 99 insertions(+), 0 deletions(-)
diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
index 75f3f00..9fd08aa 100644
--- a/include/linux/interrupt.h
+++ b/include/linux/interrupt.h
@@ -208,6 +208,8 @@ extern cpumask_var_t irq_default_affinity;
extern int irq_set_affinity(unsigned int irq, const struct cpumask *cpumask);
extern int irq_can_set_affinity(unsigned int irq);
extern int irq_select_affinity(unsigned int irq);
+extern int irq_set_node_affinity(unsigned int irq,
+ const struct cpumask *cpumask);
#else /* CONFIG_SMP */
@@ -223,6 +225,12 @@ static inline int irq_can_set_affinity(unsigned int irq)
static inline int irq_select_affinity(unsigned int irq) { return 0; }
+static inline int irq_set_node_affinity(unsigned int irq,
+ const struct cpumask *m)
+{
+ return -EINVAL;
+}
+
#endif /* CONFIG_SMP && CONFIG_GENERIC_HARDIRQS */
#ifdef CONFIG_GENERIC_HARDIRQS
diff --git a/include/linux/irq.h b/include/linux/irq.h
index ae9653d..26d7d07 100644
--- a/include/linux/irq.h
+++ b/include/linux/irq.h
@@ -166,6 +166,7 @@ struct irq_2_iommu;
* @lock: locking for SMP
* @affinity: IRQ affinity on SMP
* @node: node index useful for balancing
+ * @node_affinity: irq mask hints for irqbalance
* @pending_mask: pending rebalanced interrupts
* @threads_active: number of irqaction threads currently running
* @wait_for_threads: wait queue for sync_irq to wait for threaded handlers
@@ -196,6 +197,7 @@ struct irq_desc {
#ifdef CONFIG_SMP
cpumask_var_t affinity;
unsigned int node;
+ cpumask_var_t node_affinity;
#ifdef CONFIG_GENERIC_PENDING_IRQ
cpumask_var_t pending_mask;
#endif
diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c
index 7305b29..9e80783 100644
--- a/kernel/irq/manage.c
+++ b/kernel/irq/manage.c
@@ -138,6 +138,38 @@ int irq_set_affinity(unsigned int irq, const struct cpumask *cpumask)
return 0;
}
+/**
+ * irq_set_node_affinity - Set the CPU mask this interrupt can run on
+ * @irq: Interrupt to modify
+ * @cpumask: CPU mask to assign to the interrupt
+ *
+ */
+int irq_set_node_affinity(unsigned int irq, const struct cpumask *cpumask)
+{
+ struct irq_desc *desc = irq_to_desc(irq);
+ unsigned long flags;
+
+ spin_lock_irqsave(&desc->lock, flags);
+ cpumask_copy(desc->node_affinity, cpumask);
+ spin_unlock_irqrestore(&desc->lock, flags);
+
+ return 0;
+}
+EXPORT_SYMBOL(irq_set_node_affinity);
+
+/**
+ * irq_get_node_affinity - Get the CPU mask this interrupt can run on
+ * @irq: Interrupt to get information
+ *
+ */
+struct cpumask *irq_get_node_affinity(unsigned int irq)
+{
+ struct irq_desc *desc = irq_to_desc(irq);
+
+ return desc->node_affinity;
+}
+EXPORT_SYMBOL(irq_get_node_affinity);
+
#ifndef CONFIG_AUTO_IRQ_AFFINITY
/*
* Generic version of the affinity autoselector.
diff --git a/kernel/irq/proc.c b/kernel/irq/proc.c
index 0832145..192e3fb 100644
--- a/kernel/irq/proc.c
+++ b/kernel/irq/proc.c
@@ -31,6 +31,16 @@ static int irq_affinity_proc_show(struct seq_file *m, void *v)
return 0;
}
+static int irq_node_affinity_proc_show(struct seq_file *m, void *v)
+{
+ struct irq_desc *desc = irq_to_desc((long)m->private);
+ const struct cpumask *mask = desc->node_affinity;
+
+ seq_cpumask(m, mask);
+ seq_putc(m, '\n');
+ return 0;
+}
+
#ifndef is_affinity_mask_valid
#define is_affinity_mask_valid(val) 1
#endif
@@ -78,11 +88,46 @@ free_cpumask:
return err;
}
+static ssize_t irq_node_affinity_proc_write(struct file *file,
+ const char __user *buffer, size_t count, loff_t *pos)
+{
+ unsigned int irq = (int)(long)PDE(file->f_path.dentry->d_inode)->data;
+ cpumask_var_t new_value;
+ int err;
+
+ if (no_irq_affinity || irq_balancing_disabled(irq))
+ return -EIO;
+
+ if (!alloc_cpumask_var(&new_value, GFP_KERNEL))
+ return -ENOMEM;
+
+ err = cpumask_parse_user(buffer, count, new_value);
+ if (err)
+ goto free_cpumask;
+
+ if (!is_affinity_mask_valid(new_value)) {
+ err = -EINVAL;
+ goto free_cpumask;
+ }
+
+ irq_set_node_affinity(irq, new_value);
+ err = count;
+
+free_cpumask:
+ free_cpumask_var(new_value);
+ return err;
+}
+
static int irq_affinity_proc_open(struct inode *inode, struct file *file)
{
return single_open(file, irq_affinity_proc_show, PDE(inode)->data);
}
+static int irq_node_affinity_proc_open(struct inode *inode, struct file *f)
+{
+ return single_open(f, irq_node_affinity_proc_show, PDE(inode)->data);
+}
+
static const struct file_operations irq_affinity_proc_fops = {
.open = irq_affinity_proc_open,
.read = seq_read,
@@ -91,6 +136,14 @@ static const struct file_operations irq_affinity_proc_fops = {
.write = irq_affinity_proc_write,
};
+static const struct file_operations irq_node_affinity_proc_fops = {
+ .open = irq_node_affinity_proc_open,
+ .read = seq_read,
+ .llseek = seq_lseek,
+ .release = single_release,
+ .write = irq_node_affinity_proc_write,
+};
+
static int default_affinity_show(struct seq_file *m, void *v)
{
seq_cpumask(m, irq_default_affinity);
@@ -230,6 +283,10 @@ void register_irq_proc(unsigned int irq, struct irq_desc *desc)
/* create /proc/irq/<irq>/smp_affinity */
proc_create_data("smp_affinity", 0600, desc->dir,
&irq_affinity_proc_fops, (void *)(long)irq);
+
+ /* create /proc/irq/<irq>/node_affinity */
+ proc_create_data("node_affinity", 0600, desc->dir,
+ &irq_node_affinity_proc_fops, (void *)(long)irq);
#endif
proc_create_data("spurious", 0444, desc->dir,
On Mon, Nov 23, 2009 at 2:46 PM, Peter P Waskiewicz Jr
<[email protected]> wrote:
> This patchset adds a new CPU mask for SMP systems to the irq_desc
> struct. It also exposes an API for underlying device drivers to
> assist irqbalance in making smarter decisions when balancing, especially
> in a NUMA environment. For example, an ethernet driver with MSI-X may
> wish to limit the CPUs that an interrupt can be balanced within to
> stay on a single NUMA node. Current irqbalance operation can move the
> interrupt off the node, resulting in cross-node memory accesses and
> locks.
>
> The API is a get/set API within the kernel, along with a /proc entry
> for the interrupt.
>
> Signed-off-by: Peter P Waskiewicz Jr <[email protected]>
> ---
1) I think you should consider CONFIG_CPUMASK_OFFSTACK which will affect
node_affinity.
2) It seems like this patch can't work with SPARSE_IRQ.
Thanks,
Yong
On Sun, 2009-11-22 at 23:32 -0800, Yong Zhang wrote:
> On Mon, Nov 23, 2009 at 2:46 PM, Peter P Waskiewicz Jr
> <[email protected]> wrote:
> > This patchset adds a new CPU mask for SMP systems to the irq_desc
> > struct. It also exposes an API for underlying device drivers to
> > assist irqbalance in making smarter decisions when balancing, especially
> > in a NUMA environment. For example, an ethernet driver with MSI-X may
> > wish to limit the CPUs that an interrupt can be balanced within to
> > stay on a single NUMA node. Current irqbalance operation can move the
> > interrupt off the node, resulting in cross-node memory accesses and
> > locks.
> >
> > The API is a get/set API within the kernel, along with a /proc entry
> > for the interrupt.
> >
> > Signed-off-by: Peter P Waskiewicz Jr <[email protected]>
> > ---
>
> 1) I think you should consider CONFIG_CPUMASK_OFFSTACK which will affect
> node_affinity.
> 2) It seems like this patch can't work with SPARSE_IRQ.
This mechanism isn't going to be used by any internal kernel mechanism
for determining interrupt placement or operation. It's purely something
that either a driver can modify, or external script (through /proc),
that irqbalance will make use of. If irqbalance isn't running, or the
current version of irqbalance doesn't support reading node_affinity,
then it won't affect the system's operation.
If irqbalance does support it, it'll read whatever the supplied mask is,
and then will try and balance interrupts within that mask. It will bail
if the mask is invalid, or won't apply to the running system, just like
how putting a bogus mask into smp_affinity is ignored.
If there's something I'm missing beyond this with the two suggestions
you've made (I looked into those two parameters and tried to draw
conclusions), please let me know.
Cheers,
-PJ Waskiewicz
On Mon, 2009-11-23 at 01:36 -0800, Peter P Waskiewicz Jr wrote:
> This mechanism isn't going to be used by any internal kernel mechanism
> for determining interrupt placement or operation. It's purely something
> that either a driver can modify, or external script (through /proc),
> that irqbalance will make use of. If irqbalance isn't running, or the
> current version of irqbalance doesn't support reading node_affinity,
> then it won't affect the system's operation.
>
> If irqbalance does support it, it'll read whatever the supplied mask is,
> and then will try and balance interrupts within that mask. It will bail
> if the mask is invalid, or won't apply to the running system, just like
> how putting a bogus mask into smp_affinity is ignored.
>
> If there's something I'm missing beyond this with the two suggestions
> you've made (I looked into those two parameters and tried to draw
> conclusions), please let me know.
I don't see the point in adding it, if the driver wants to set a node
cpu mask it can already do that using the regular smp affinity settings.
Same for userspace.
On Mon, 23 Nov 2009, Peter Zijlstra wrote:
> On Mon, 2009-11-23 at 01:36 -0800, Peter P Waskiewicz Jr wrote:
>
> > This mechanism isn't going to be used by any internal kernel mechanism
> > for determining interrupt placement or operation. It's purely something
> > that either a driver can modify, or external script (through /proc),
> > that irqbalance will make use of. If irqbalance isn't running, or the
> > current version of irqbalance doesn't support reading node_affinity,
> > then it won't affect the system's operation.
> >
> > If irqbalance does support it, it'll read whatever the supplied mask is,
> > and then will try and balance interrupts within that mask. It will bail
> > if the mask is invalid, or won't apply to the running system, just like
> > how putting a bogus mask into smp_affinity is ignored.
> >
> > If there's something I'm missing beyond this with the two suggestions
> > you've made (I looked into those two parameters and tried to draw
> > conclusions), please let me know.
>
> I don't see the point in adding it, if the driver wants to set a node
> cpu mask it can already do that using the regular smp affinity settings.
Unfortunately, a driver can't. The irq_set_affinity() function isn't
exported. I proposed a patch on netdev to export it, and then to tie down
an interrupt using IRQF_NOBALANCING, so irqbalance won't touch it. That
was rejected, since the driver is enforcing policy of the interrupt
balancing, not irqbalance.
I and Jesse Brandeburg had a meeting with Arjan about this. What we came
up with was this interface, so drivers can set what they'd like to see, if
irqbalance decides to honor it. That way interrupt affinity policies are
set only by irqbalance, but this interface gives us a mechanism to hint to
irqbalance what we'd like it to do.
Also, if you use the /proc interface to change smp_affinity on an
interrupt without any of these changes, irqbalance will override it on its
next poll interval. This also is not desirable.
Cheers,
-PJ
[snip]
>>
>> 1) I think you should consider CONFIG_CPUMASK_OFFSTACK which will affect
>> node_affinity.
>> 2) It seems like this patch can't work with SPARSE_IRQ.
>
> This mechanism isn't going to be used by any internal kernel mechanism
> for determining interrupt placement or operation. It's purely something
> that either a driver can modify, or external script (through /proc),
> that irqbalance will make use of. If irqbalance isn't running, or the
> current version of irqbalance doesn't support reading node_affinity,
> then it won't affect the system's operation.
>
> If irqbalance does support it, it'll read whatever the supplied mask is,
> and then will try and balance interrupts within that mask. It will bail
> if the mask is invalid, or won't apply to the running system, just like
> how putting a bogus mask into smp_affinity is ignored.
>
> If there's something I'm missing beyond this with the two suggestions
> you've made (I looked into those two parameters and tried to draw
> conclusions), please let me know.
My two suggestions are both about your adding node_affinity. Before you can
use this element, you must initialise it firstly. You can refer how
irq_desc::affinity
is used in function alloc_desc_masks().
include/linux/irq.h:
static inline bool alloc_desc_masks(struct irq_desc *desc, int node,
bool boot)
{
gfp_t gfp = GFP_ATOMIC;
if (boot)
gfp = GFP_NOWAIT;
#ifdef CONFIG_CPUMASK_OFFSTACK
if (!alloc_cpumask_var_node(&desc->affinity, gfp, node))
return false;
#ifdef CONFIG_GENERIC_PENDING_IRQ
if (!alloc_cpumask_var_node(&desc->pending_mask, gfp, node)) {
free_cpumask_var(desc->affinity);
return false;
}
#endif
#endif
return true;
}
Thanks,
Yong
>
> Cheers,
> -PJ Waskiewicz
>
>
Peter Zijlstra wrote:
> On Mon, 2009-11-23 at 01:36 -0800, Peter P Waskiewicz Jr wrote:
>
>> This mechanism isn't going to be used by any internal kernel mechanism
>> for determining interrupt placement or operation. It's purely something
>> that either a driver can modify, or external script (through /proc),
>> that irqbalance will make use of. If irqbalance isn't running, or the
>> current version of irqbalance doesn't support reading node_affinity,
>> then it won't affect the system's operation.
>>
>> If irqbalance does support it, it'll read whatever the supplied mask is,
>> and then will try and balance interrupts within that mask. It will bail
>> if the mask is invalid, or won't apply to the running system, just like
>> how putting a bogus mask into smp_affinity is ignored.
>>
>> If there's something I'm missing beyond this with the two suggestions
>> you've made (I looked into those two parameters and tried to draw
>> conclusions), please let me know.
>
> I don't see the point in adding it, if the driver wants to set a node
> cpu mask it can already do that using the regular smp affinity settings.
>
> Same for userspace.
the problem is that there is no way currently that the driver can communicate
"I allocated all my metadata on THIS numa node". irqbalance and sysadmins need
that to not make really stupid decisions.....
On Mon, 2009-11-23 at 15:32 -0800, Waskiewicz Jr, Peter P wrote:
> Unfortunately, a driver can't. The irq_set_affinity() function isn't
> exported. I proposed a patch on netdev to export it, and then to tie down
> an interrupt using IRQF_NOBALANCING, so irqbalance won't touch it. That
> was rejected, since the driver is enforcing policy of the interrupt
> balancing, not irqbalance.
Why would a patch touching the irq subsystem go to netdev?
What is wrong with exporting irq_set_affinity(), and wtf do you need
IRQF_NOBALANCING for?
> I and Jesse Brandeburg had a meeting with Arjan about this. What we came
> up with was this interface, so drivers can set what they'd like to see, if
> irqbalance decides to honor it. That way interrupt affinity policies are
> set only by irqbalance, but this interface gives us a mechanism to hint to
> irqbalance what we'd like it to do.
If all you want is to expose policy to userspace then you don't need any
of this, simply expose the NICs home node through a sysfs device thingy
(I was under the impression its already there somewhere, but I can't
ever find anything in /sys).
No need what so ever to poke at the IRQ subsystem.
> Also, if you use the /proc interface to change smp_affinity on an
> interrupt without any of these changes, irqbalance will override it on its
> next poll interval. This also is not desirable.
This all sounds backwards.. we've got a perfectly functional interface
for affinity -- which people object to being used for some reason. So
you add another interface on top, and that is ok?
All the while not CC'ing the IRQ folks,.. brilliant approach.
On Mon, 2009-11-23 at 22:17 -0700, Yong Zhang wrote:
> [snip]
> >>
> >> 1) I think you should consider CONFIG_CPUMASK_OFFSTACK which will affect
> >> node_affinity.
> >> 2) It seems like this patch can't work with SPARSE_IRQ.
> >
> > This mechanism isn't going to be used by any internal kernel mechanism
> > for determining interrupt placement or operation. It's purely something
> > that either a driver can modify, or external script (through /proc),
> > that irqbalance will make use of. If irqbalance isn't running, or the
> > current version of irqbalance doesn't support reading node_affinity,
> > then it won't affect the system's operation.
> >
> > If irqbalance does support it, it'll read whatever the supplied mask is,
> > and then will try and balance interrupts within that mask. It will bail
> > if the mask is invalid, or won't apply to the running system, just like
> > how putting a bogus mask into smp_affinity is ignored.
> >
> > If there's something I'm missing beyond this with the two suggestions
> > you've made (I looked into those two parameters and tried to draw
> > conclusions), please let me know.
>
> My two suggestions are both about your adding node_affinity. Before you can
> use this element, you must initialise it firstly. You can refer how
> irq_desc::affinity
> is used in function alloc_desc_masks().
> include/linux/irq.h:
> static inline bool alloc_desc_masks(struct irq_desc *desc, int node,
> bool boot)
> {
> gfp_t gfp = GFP_ATOMIC;
>
> if (boot)
> gfp = GFP_NOWAIT;
>
> #ifdef CONFIG_CPUMASK_OFFSTACK
> if (!alloc_cpumask_var_node(&desc->affinity, gfp, node))
> return false;
>
> #ifdef CONFIG_GENERIC_PENDING_IRQ
> if (!alloc_cpumask_var_node(&desc->pending_mask, gfp, node)) {
> free_cpumask_var(desc->affinity);
> return false;
> }
> #endif
> #endif
> return true;
> }
>
Ah, ok. I see what you were referring to now. Let me respin the patch
and send a second version.
Thanks Yong,
-PJ Waskiewicz
On Mon, 2009-11-23 at 22:07 -0800, Arjan van de Ven wrote:
> Peter Zijlstra wrote:
> > On Mon, 2009-11-23 at 01:36 -0800, Peter P Waskiewicz Jr wrote:
> >
> >> This mechanism isn't going to be used by any internal kernel mechanism
> >> for determining interrupt placement or operation. It's purely something
> >> that either a driver can modify, or external script (through /proc),
> >> that irqbalance will make use of. If irqbalance isn't running, or the
> >> current version of irqbalance doesn't support reading node_affinity,
> >> then it won't affect the system's operation.
> >>
> >> If irqbalance does support it, it'll read whatever the supplied mask is,
> >> and then will try and balance interrupts within that mask. It will bail
> >> if the mask is invalid, or won't apply to the running system, just like
> >> how putting a bogus mask into smp_affinity is ignored.
> >>
> >> If there's something I'm missing beyond this with the two suggestions
> >> you've made (I looked into those two parameters and tried to draw
> >> conclusions), please let me know.
> >
> > I don't see the point in adding it, if the driver wants to set a node
> > cpu mask it can already do that using the regular smp affinity settings.
> >
> > Same for userspace.
>
> the problem is that there is no way currently that the driver can communicate
> "I allocated all my metadata on THIS numa node". irqbalance and sysadmins need
> that to not make really stupid decisions.....
And what exactly is struct device::numa_node good for then?
On Tue, 2009-11-24 at 01:38 -0700, Peter Zijlstra wrote:
> On Mon, 2009-11-23 at 15:32 -0800, Waskiewicz Jr, Peter P wrote:
>
> > Unfortunately, a driver can't. The irq_set_affinity() function isn't
> > exported. I proposed a patch on netdev to export it, and then to tie down
> > an interrupt using IRQF_NOBALANCING, so irqbalance won't touch it. That
> > was rejected, since the driver is enforcing policy of the interrupt
> > balancing, not irqbalance.
>
> Why would a patch touching the irq subsystem go to netdev?
The only change to the IRQ subsystem was:
EXPORT_SYMBOL(irq_set_affinity);
The majority of the changeset was for the ixgbe driver.
> What is wrong with exporting irq_set_affinity(), and wtf do you need
> IRQF_NOBALANCING for?
>
Again, the pushback I received was with allowing anything other than
irqbalance to dictate interrupt affinity policy.
And if I set interrupt affinity from the driver or from /proc,
irqbalance will happily rebalance the interrupt elsewhere. The
IRQF_NOBALANCING flag will prevent irqbalance from being able to move
the interrupt.
> > I and Jesse Brandeburg had a meeting with Arjan about this. What we came
> > up with was this interface, so drivers can set what they'd like to see, if
> > irqbalance decides to honor it. That way interrupt affinity policies are
> > set only by irqbalance, but this interface gives us a mechanism to hint to
> > irqbalance what we'd like it to do.
>
> If all you want is to expose policy to userspace then you don't need any
> of this, simply expose the NICs home node through a sysfs device thingy
> (I was under the impression its already there somewhere, but I can't
> ever find anything in /sys).
>
> No need what so ever to poke at the IRQ subsystem.
The point is we need something common that the kernel side (whether a
driver or /proc can modify) that irqbalance can use.
> > Also, if you use the /proc interface to change smp_affinity on an
> > interrupt without any of these changes, irqbalance will override it on its
> > next poll interval. This also is not desirable.
>
> This all sounds backwards.. we've got a perfectly functional interface
> for affinity -- which people object to being used for some reason. So
> you add another interface on top, and that is ok?
>
But it's not functional. If I set the affinity in smp_affinity, then
irqbalance will override it 10 seconds later.
> All the while not CC'ing the IRQ folks,.. brilliant approach.
If I knew who I should CC, I'd be happy to add them. Can you provide
email addresses please?
Cheers,
-PJ Waskiewicz
On Tue, 2009-11-24 at 00:59 -0800, Peter P Waskiewicz Jr wrote:
> > This all sounds backwards.. we've got a perfectly functional interface
> > for affinity -- which people object to being used for some reason. So
> > you add another interface on top, and that is ok?
> >
>
> But it's not functional. If I set the affinity in smp_affinity, then
> irqbalance will override it 10 seconds later.
And here I was thinking the kernel round-robins IRQ delivery on the mask
specified there. Are you talking about some daft userspace thing that
writes into the irq smp_affinity to effect irq balancing?
On Tue, 2009-11-24 at 02:08 -0700, Peter Zijlstra wrote:
> On Tue, 2009-11-24 at 00:59 -0800, Peter P Waskiewicz Jr wrote:
> > > This all sounds backwards.. we've got a perfectly functional interface
> > > for affinity -- which people object to being used for some reason. So
> > > you add another interface on top, and that is ok?
> > >
> >
> > But it's not functional. If I set the affinity in smp_affinity, then
> > irqbalance will override it 10 seconds later.
>
> And here I was thinking the kernel round-robins IRQ delivery on the mask
> specified there. Are you talking about some daft userspace thing that
> writes into the irq smp_affinity to effect irq balancing?
>
Yep. That's exactly what irqbalance does.
Cheers,
-PJ
On Tue, 2009-11-24 at 00:59 -0800, Peter P Waskiewicz Jr wrote:
>
> > All the while not CC'ing the IRQ folks,.. brilliant approach.
>
> If I knew who I should CC, I'd be happy to add them. Can you provide
> email addresses please?
Since most people can't seen to read a simple MAINTAINERS file, some
other people wrote a script to read it for you:
# scripts/get_maintainer.pl -f kernel/irq/manage.c
Ingo Molnar <[email protected]>
Thomas Gleixner <[email protected]>
[email protected]
Another option is to do something like:
# git log kernel/irq/manage.c | grep Author | head -30 | awk
'{ t[$0]++; } END { for (i in t) { print t[i] " " i; }}' | sort -rn
10 Author: Thomas Gleixner <[email protected]>
9 Author: Ingo Molnar <[email protected]>
3 Author: Magnus Damm <[email protected]>
2 Author: Linus Torvalds <[email protected]>
...
On Tue, 24 Nov 2009, Peter P Waskiewicz Jr wrote:
> On Tue, 2009-11-24 at 01:38 -0700, Peter Zijlstra wrote:
> > On Mon, 2009-11-23 at 15:32 -0800, Waskiewicz Jr, Peter P wrote:
> >
> > > Unfortunately, a driver can't. The irq_set_affinity() function isn't
> > > exported. I proposed a patch on netdev to export it, and then to tie down
> > > an interrupt using IRQF_NOBALANCING, so irqbalance won't touch it. That
> > > was rejected, since the driver is enforcing policy of the interrupt
> > > balancing, not irqbalance.
> >
> > Why would a patch touching the irq subsystem go to netdev?
>
> The only change to the IRQ subsystem was:
>
> EXPORT_SYMBOL(irq_set_affinity);
Which is still touching the generic irq subsystem and needs the ack of
the relevant maintainer. If there is a need to expose such an
interface to drivers then the maintainer wants to know exactly why and
needs to be part of the discussion of alternative solutions. Otherwise
you waste time on implementing stuff like the current patch which is
definitely not going anywhere near the irq subsystem.
> > If all you want is to expose policy to userspace then you don't need any
> > of this, simply expose the NICs home node through a sysfs device thingy
> > (I was under the impression its already there somewhere, but I can't
> > ever find anything in /sys).
> >
> > No need what so ever to poke at the IRQ subsystem.
>
> The point is we need something common that the kernel side (whether a
> driver or /proc can modify) that irqbalance can use.
/sys/class/net/ethX/device/numa_node
perhaps ?
> > > Also, if you use the /proc interface to change smp_affinity on an
> > > interrupt without any of these changes, irqbalance will override it on its
> > > next poll interval. This also is not desirable.
> >
> > This all sounds backwards.. we've got a perfectly functional interface
> > for affinity -- which people object to being used for some reason. So
> > you add another interface on top, and that is ok?
> >
>
> But it's not functional. If I set the affinity in smp_affinity, then
> irqbalance will override it 10 seconds later.
And to work around the brain wreckage of irqbalanced you want to
fiddle in the irq code instead of teaching irqbalanced to handle node
affinities ?
The only thing which is worth to investigate is whether the irq core
code should honour the dev->numa_node setting and restrict the
possible irq affinity settings to that node. If a device is tied to a
node it makes a certain amount of sense to do that.
But such a change would not need a new interface in the irq core and
definitely not a new cpumask_t member in the irq_desc structure to
store a node affinity which can be expressed with a simple
integer.
But this needs more thoughts and I want to know more about the
background and the reasoning for such a change.
Thanks,
tglx
Peter Zijlstra wrote:
>>> Same for userspace.
>> the problem is that there is no way currently that the driver can communicate
>> "I allocated all my metadata on THIS numa node". irqbalance and sysadmins need
>> that to not make really stupid decisions.....
>
> And what exactly is struct device::numa_node good for then?
>
and that is exported to userspace.. how?
... that has nothing to do with an irq, and also falls flat for a driver that
supports multiple irqs, and assigns one to each numa node.
Peter Zijlstra wrote:
> On Tue, 2009-11-24 at 00:59 -0800, Peter P Waskiewicz Jr wrote:
>>> This all sounds backwards.. we've got a perfectly functional interface
>>> for affinity -- which people object to being used for some reason. So
>>> you add another interface on top, and that is ok?
>>>
>> But it's not functional. If I set the affinity in smp_affinity, then
>> irqbalance will override it 10 seconds later.
>
> And here I was thinking the kernel round-robins IRQ delivery on the mask
> specified there.
the kernel does no such thing, nor has code to do so.
> Are you talking about some daft userspace thing that
> writes into the irq smp_affinity to effect irq balancing?
thanks ;)
From: Peter Zijlstra <[email protected]>
Date: Tue, 24 Nov 2009 09:39:46 +0100
> On Mon, 2009-11-23 at 22:07 -0800, Arjan van de Ven wrote:
>> the problem is that there is no way currently that the driver can communicate
>> "I allocated all my metadata on THIS numa node". irqbalance and sysadmins need
>> that to not make really stupid decisions.....
>
> And what exactly is struct device::numa_node good for then?
device->numa_node just says where the device is.
For better performance, it can make sense to, for example, allocate the ring
buffers for different device queues on other NUMA nodes.
That's the kind of thing PJ is trying to make available.
On Tue, 2009-11-24 at 03:07 -0700, Thomas Gleixner wrote:
> On Tue, 24 Nov 2009, Peter P Waskiewicz Jr wrote:
> > On Tue, 2009-11-24 at 01:38 -0700, Peter Zijlstra wrote:
> > > On Mon, 2009-11-23 at 15:32 -0800, Waskiewicz Jr, Peter P wrote:
> > >
> > > > Unfortunately, a driver can't. The irq_set_affinity() function isn't
> > > > exported. I proposed a patch on netdev to export it, and then to tie down
> > > > an interrupt using IRQF_NOBALANCING, so irqbalance won't touch it. That
> > > > was rejected, since the driver is enforcing policy of the interrupt
> > > > balancing, not irqbalance.
> > >
> > > Why would a patch touching the irq subsystem go to netdev?
> >
> > The only change to the IRQ subsystem was:
> >
> > EXPORT_SYMBOL(irq_set_affinity);
>
> Which is still touching the generic irq subsystem and needs the ack of
> the relevant maintainer. If there is a need to expose such an
> interface to drivers then the maintainer wants to know exactly why and
> needs to be part of the discussion of alternative solutions. Otherwise
> you waste time on implementing stuff like the current patch which is
> definitely not going anywhere near the irq subsystem.
>
Understood, and duly noted.
> > > If all you want is to expose policy to userspace then you don't need any
> > > of this, simply expose the NICs home node through a sysfs device thingy
> > > (I was under the impression its already there somewhere, but I can't
> > > ever find anything in /sys).
> > >
> > > No need what so ever to poke at the IRQ subsystem.
> >
> > The point is we need something common that the kernel side (whether a
> > driver or /proc can modify) that irqbalance can use.
>
> /sys/class/net/ethX/device/numa_node
>
> perhaps ?
What I'm trying to do though is one to many NUMA node assignments. See
below for a better overview of what the issue is we're trying to solve.
>
> > > > Also, if you use the /proc interface to change smp_affinity on an
> > > > interrupt without any of these changes, irqbalance will override it on its
> > > > next poll interval. This also is not desirable.
> > >
> > > This all sounds backwards.. we've got a perfectly functional interface
> > > for affinity -- which people object to being used for some reason. So
> > > you add another interface on top, and that is ok?
> > >
> >
> > But it's not functional. If I set the affinity in smp_affinity, then
> > irqbalance will override it 10 seconds later.
>
> And to work around the brain wreckage of irqbalanced you want to
> fiddle in the irq code instead of teaching irqbalanced to handle node
> affinities ?
>
> The only thing which is worth to investigate is whether the irq core
> code should honour the dev->numa_node setting and restrict the
> possible irq affinity settings to that node. If a device is tied to a
> node it makes a certain amount of sense to do that.
>
> But such a change would not need a new interface in the irq core and
> definitely not a new cpumask_t member in the irq_desc structure to
> store a node affinity which can be expressed with a simple
> integer.
>
> But this needs more thoughts and I want to know more about the
> background and the reasoning for such a change.
>
I'll use the ixgbe driver as my example, since that is where my
immediate problems are. This is our 10GbE device, and supports 128 Rx
queues, 128 Tx queues, and has a maximum of 64 MSI-X vectors. In a
typical case, let's say an 8-core machine (Nehalem-EP with
hyperthreading off) brings one port online. We'll allocate 8 Rx and 8
Tx queues. When these allocations occur, we want to allocate the memory
for our descriptor rings and buffer structs and DMA areas onto the
various NUMA nodes. This will promote spreading of the load not just
across CPUs, but also the memory controllers.
If we were to just run like that and have irqbalance move our vectors to
a single node, then we'd have half of our network resources creating
cross-node traffic, which is undesirable, since the OS may have to take
locks node to node to get the memory it's looking for.
The bottom line is we need some mechanism that allows a driver/user to
deterministically assign the underlying interrupt resources to the
correct NUMA node for each interrupt. And in the example above, we may
have more than one NUMA node we need to balance into.
Please let me know if I've explained this well enough. I appreciate the
time.
Cheers,
-PJ Waskiewicz
On Tue, 2009-11-24 at 09:39 -0800, David Miller wrote:
> From: Peter Zijlstra <[email protected]>
> Date: Tue, 24 Nov 2009 09:39:46 +0100
>
> > On Mon, 2009-11-23 at 22:07 -0800, Arjan van de Ven wrote:
> >> the problem is that there is no way currently that the driver can communicate
> >> "I allocated all my metadata on THIS numa node". irqbalance and sysadmins need
> >> that to not make really stupid decisions.....
> >
> > And what exactly is struct device::numa_node good for then?
>
> device->numa_node just says where the device is.
>
> For better performance, it can make sense to, for example, allocate the ring
> buffers for different device queues on other NUMA nodes.
>
> That's the kind of thing PJ is trying to make available.
Yes, that's exactly what I'm trying to do. Even further, we want to
allocate the ring SW struct itself and descriptor structures on other
NUMA nodes, and make sure the interrupt lines up with those allocations.
Cheers,
-PJ
Peter P Waskiewicz Jr a ?crit :
That's the kind of thing PJ is trying to make available.
>
> Yes, that's exactly what I'm trying to do. Even further, we want to
> allocate the ring SW struct itself and descriptor structures on other
> NUMA nodes, and make sure the interrupt lines up with those allocations.
>
Say you allocate ring buffers on NUMA node of the CPU handling interrupt
on a particular queue.
If irqbalance or an admin changes /proc/irq/{number}/smp_affinities,
do you want to realloc ring buffer to another NUMA node ?
It seems complex to me, maybe optimal thing would be to use a NUMA policy to
spread vmalloc() allocations to all nodes to get a good bandwidth...
On Tue, 2009-11-24 at 10:26 -0800, Eric Dumazet wrote:
> Peter P Waskiewicz Jr a écrit :
> That's the kind of thing PJ is trying to make available.
> >
> > Yes, that's exactly what I'm trying to do. Even further, we want to
> > allocate the ring SW struct itself and descriptor structures on other
> > NUMA nodes, and make sure the interrupt lines up with those allocations.
> >
>
> Say you allocate ring buffers on NUMA node of the CPU handling interrupt
> on a particular queue.
>
> If irqbalance or an admin changes /proc/irq/{number}/smp_affinities,
> do you want to realloc ring buffer to another NUMA node ?
>
That's why I'm trying to add the node_affinity mechanism that irqbalance
can use to prevent the interrupt being moved to another node.
> It seems complex to me, maybe optimal thing would be to use a NUMA policy to
> spread vmalloc() allocations to all nodes to get a good bandwidth...
That's exactly what we're doing in our 10GbE driver right now (isn't
pushed upstream yet, still finalizing our testing). We spread to all
NUMA nodes in a semi-intelligent fashion when allocating our rings and
buffers. The last piece is ensuring the interrupts tied to the various
queues all route to the NUMA nodes those CPUs belong to. irqbalance
needs some kind of hint to make sure it does the right thing, which
today it does not.
I don't see how this is complex though. Driver loads, allocates across
the NUMA nodes for optimal throughput, then writes CPU masks for the
NUMA nodes each interrupt belongs to. irqbalance comes along and looks
at the new mask "hint," and then balances that interrupt within that
hinted mask.
Cheers,
-PJ
From: Eric Dumazet <[email protected]>
Date: Tue, 24 Nov 2009 19:26:15 +0100
> It seems complex to me, maybe optimal thing would be to use a NUMA policy to
> spread vmalloc() allocations to all nodes to get a good bandwidth...
vmalloc() and sk_buff's don't currently mix and I really don't see us
every allowing them to :-)
David Miller a ?crit :
> From: Eric Dumazet <[email protected]>
> Date: Tue, 24 Nov 2009 19:26:15 +0100
>
>> It seems complex to me, maybe optimal thing would be to use a NUMA policy to
>> spread vmalloc() allocations to all nodes to get a good bandwidth...
>
> vmalloc() and sk_buff's don't currently mix and I really don't see us
> every allowing them to :-)
I think Peter was referring to tx/rx rings buffers, not sk_buffs.
They (ring buffers) are allocated with vmalloc() at driver init time.
And Tom pointed out that our rx sk_buff allocation should be using the node
of requester, no need to hardcode node number per rx queue (or per device as of today)
Peter P Waskiewicz Jr a écrit :
> That's exactly what we're doing in our 10GbE driver right now (isn't
> pushed upstream yet, still finalizing our testing). We spread to all
> NUMA nodes in a semi-intelligent fashion when allocating our rings and
> buffers. The last piece is ensuring the interrupts tied to the various
> queues all route to the NUMA nodes those CPUs belong to. irqbalance
> needs some kind of hint to make sure it does the right thing, which
> today it does not.
sk_buff allocations should be done on the node of the cpu handling rx interrupts.
For rings, I am ok for irqbalance and driver cooperation, in case admin
doesnt want to change the defaults.
>
> I don't see how this is complex though. Driver loads, allocates across
> the NUMA nodes for optimal throughput, then writes CPU masks for the
> NUMA nodes each interrupt belongs to. irqbalance comes along and looks
> at the new mask "hint," and then balances that interrupt within that
> hinted mask.
So NUMA policy is given by the driver at load time ?
An admin might chose to direct all NIC trafic to a given node, because
its machine has mixed workload. 3 nodes out of 4 for database workload,
one node for network IO...
So if an admin changes smp_affinity, is your driver able to reconfigure itself
and re-allocate all its rings to be on NUMA node chosen by admin ? This is
what I qualify as complex.
On Tue, 2009-11-24 at 11:01 -0800, Eric Dumazet wrote:
> Peter P Waskiewicz Jr a écrit :
>
> > That's exactly what we're doing in our 10GbE driver right now (isn't
> > pushed upstream yet, still finalizing our testing). We spread to all
> > NUMA nodes in a semi-intelligent fashion when allocating our rings and
> > buffers. The last piece is ensuring the interrupts tied to the various
> > queues all route to the NUMA nodes those CPUs belong to. irqbalance
> > needs some kind of hint to make sure it does the right thing, which
> > today it does not.
>
> sk_buff allocations should be done on the node of the cpu handling rx interrupts.
Yes, but we preallocate the buffers to minimize overhead when running
our interrupt routines. Regardless, whatever queue we're filling with
those sk_buff's has an interrupt vector attached. So wherever the
descriptor ring/queue and its associated buffers were allocated, that is
where the interrupt's affinity needs to be set to.
> For rings, I am ok for irqbalance and driver cooperation, in case admin
> doesnt want to change the defaults.
>
> >
> > I don't see how this is complex though. Driver loads, allocates across
> > the NUMA nodes for optimal throughput, then writes CPU masks for the
> > NUMA nodes each interrupt belongs to. irqbalance comes along and looks
> > at the new mask "hint," and then balances that interrupt within that
> > hinted mask.
>
> So NUMA policy is given by the driver at load time ?
I think it would have to. Nobody else has insight how the driver
allocated its resources. So the driver can be told where to allocate
(see below), or the driver needs to indicate upwards how it allocated
resources.
> An admin might chose to direct all NIC trafic to a given node, because
> its machine has mixed workload. 3 nodes out of 4 for database workload,
> one node for network IO...
>
> So if an admin changes smp_affinity, is your driver able to reconfigure itself
> and re-allocate all its rings to be on NUMA node chosen by admin ? This is
> what I qualify as complex.
No, we don't want to go this route of reallocation. This, I agree, is
very complex, and can be very devastating. We'd basically be resetting
the driver whenever an interrupt moved, so this could be a terrible DoS
vulnerability.
Jesse Brandeburg has a set of patches he's working on that will allow us
to bind an interface to a single node. So in your example of 3 nodes
for DB workload and 1 for network I/O, the driver can be loaded and
directly bound to that 4th node. Then the node_affinity mask would be
set by the driver for the CPU mask of that single node. But in these
deployments, a sysadmin changing affinity that will fly directly in the
face of how resources are laid out is poor system administration. I
know it will happen, but I don't know how far we need to protect the
sysadmin from shooting themselves in the foot in terms of performance
tuning.
Cheers,
-PJ
Eric Dumazet <[email protected]> writes:
> David Miller a ?crit :
>> From: Eric Dumazet <[email protected]>
>> Date: Tue, 24 Nov 2009 19:26:15 +0100
>>
>>> It seems complex to me, maybe optimal thing would be to use a NUMA policy to
>>> spread vmalloc() allocations to all nodes to get a good bandwidth...
>>
>> vmalloc() and sk_buff's don't currently mix and I really don't see us
>> every allowing them to :-)
>
> I think Peter was referring to tx/rx rings buffers, not sk_buffs.
>
> They (ring buffers) are allocated with vmalloc() at driver init time.
They are typically allocated with dma_alloc_coherent(), which does
allocate a continuous area. In theory you could do interleaving
with IOMMus, but just putting it on the same node as the device
is probably better.
-Andi
--
[email protected] -- Speaking for myself only.
Andi Kleen a ?crit :
> They are typically allocated with dma_alloc_coherent(), which does
> allocate a continuous area. In theory you could do interleaving
> with IOMMus, but just putting it on the same node as the device
> is probably better.
There are two parts, biggest one allocated with vmalloc()
(to hold struct ixgbe_rx_buffer array, 32 bytes or more per entry),
only used by driver (not adapter)
and one allocated with pci_alloc_consistent()
(to hold ixgbe_adv_tx_desc array, 16 bytes per entry)
vmalloc() one could be spreaded on many nodes.
I am not speaking about the pci_alloc_consistent() one :)
Eric Dumazet a ?crit :
> Andi Kleen a ?crit :
>> They are typically allocated with dma_alloc_coherent(), which does
>> allocate a continuous area. In theory you could do interleaving
>> with IOMMus, but just putting it on the same node as the device
>> is probably better.
>
> There are two parts, biggest one allocated with vmalloc()
> (to hold struct ixgbe_rx_buffer array, 32 bytes or more per entry),
> only used by driver (not adapter)
>
> and one allocated with pci_alloc_consistent()
> (to hold ixgbe_adv_tx_desc array, 16 bytes per entry)
>
> vmalloc() one could be spreaded on many nodes.
> I am not speaking about the pci_alloc_consistent() one :)
>
BTW, I found my Nehalem dev machine behaves strangly, defeating all
my NUMA tweaks. (This is an HP DL380 G6)
It has two sockets, populated with two E5530 @2.4GH.
Each cpu has 2x4GB RAM modules.
It claims having two memory nodes, but all cpus are on Node 0
dmesg | grep -i node
[ 0.000000] SRAT: PXM 0 -> APIC 0 -> Node 0
[ 0.000000] SRAT: PXM 0 -> APIC 1 -> Node 0
[ 0.000000] SRAT: PXM 0 -> APIC 2 -> Node 0
[ 0.000000] SRAT: PXM 0 -> APIC 3 -> Node 0
[ 0.000000] SRAT: PXM 0 -> APIC 4 -> Node 0
[ 0.000000] SRAT: PXM 0 -> APIC 5 -> Node 0
[ 0.000000] SRAT: PXM 0 -> APIC 6 -> Node 0
[ 0.000000] SRAT: PXM 0 -> APIC 7 -> Node 0
[ 0.000000] SRAT: Node 0 PXM 0 0-e0000000
[ 0.000000] SRAT: Node 0 PXM 0 100000000-220000000
[ 0.000000] SRAT: Node 1 PXM 1 220000000-420000000
[ 0.000000] Bootmem setup node 0 0000000000000000-0000000220000000
[ 0.000000] NODE_DATA [0000000000001000 - 0000000000004fff]
[ 0.000000] Bootmem setup node 1 0000000220000000-000000041ffff000
[ 0.000000] NODE_DATA [0000000220000000 - 0000000220003fff]
[ 0.000000] [ffffea0000000000-ffffea00087fffff] PMD -> [ffff880028600000-ffff8800305fffff] on node 0
[ 0.000000] [ffffea0008800000-ffffea00107fffff] PMD -> [ffff880220200000-ffff8802281fffff] on node 1
[ 0.000000] Movable zone start PFN for each node
[ 0.000000] early_node_map[5] active PFN ranges
[ 0.000000] On node 0 totalpages: 2094543
[ 0.000000] On node 1 totalpages: 2097151
[ 0.000000] NR_CPUS:16 nr_cpumask_bits:16 nr_cpu_ids:16 nr_node_ids:2
[ 0.000000] SLUB: Genslabs=14, HWalign=64, Order=0-3, MinObjects=0, CPUs=16, Nodes=2
[ 0.004756] Inode-cache hash table entries: 1048576 (order: 11, 8388608 bytes)
[ 0.007213] CPU 0/0x0 -> Node 0
[ 0.398104] CPU 1/0x10 -> Node 0
[ 0.557854] CPU 2/0x4 -> Node 0
[ 0.717606] CPU 3/0x14 -> Node 0
[ 0.877357] CPU 4/0x2 -> Node 0
[ 1.037109] CPU 5/0x12 -> Node 0
[ 1.196860] CPU 6/0x6 -> Node 0
[ 1.356611] CPU 7/0x16 -> Node 0
[ 1.516365] CPU 8/0x1 -> Node 0
[ 1.676114] CPU 9/0x11 -> Node 0
[ 1.835865] CPU 10/0x5 -> Node 0
[ 1.995616] CPU 11/0x15 -> Node 0
[ 2.155367] CPU 12/0x3 -> Node 0
[ 2.315119] CPU 13/0x13 -> Node 0
[ 2.474870] CPU 14/0x7 -> Node 0
[ 2.634621] CPU 15/0x17 -> Node 0
# cat /proc/buddyinfo
Node 0, zone DMA 2 2 2 1 1 1 1 0 1 1 3
Node 0, zone DMA32 5 11 4 5 4 12 1 4 4 5 834
Node 0, zone Normal 4109 120 98 153 67 35 21 15 11 10 109
Node 1, zone Normal 7 17 10 12 7 14 5 7 6 5 2004
This is with net-next-2.6, I'll try linux-2.6
Works here
>
> dmesg | grep -i node
> [ 0.000000] SRAT: PXM 0 -> APIC 0 -> Node 0
> [ 0.000000] SRAT: PXM 0 -> APIC 1 -> Node 0
> [ 0.000000] SRAT: PXM 0 -> APIC 2 -> Node 0
> [ 0.000000] SRAT: PXM 0 -> APIC 3 -> Node 0
> [ 0.000000] SRAT: PXM 0 -> APIC 4 -> Node 0
> [ 0.000000] SRAT: PXM 0 -> APIC 5 -> Node 0
> [ 0.000000] SRAT: PXM 0 -> APIC 6 -> Node 0
> [ 0.000000] SRAT: PXM 0 -> APIC 7 -> Node 0
You seem to only have 8 CPUs (one socket) Normally a dual socket nehalem
should have 16 with HyperThreading enabled.
For some reason the BIOS is not reporting the other CPU.
You could double check with acpidmp / iasl -d if that's
what the BIOS really reports, but normally it should work.
> [ 0.000000] SRAT: Node 0 PXM 0 0-e0000000
> [ 0.000000] SRAT: Node 0 PXM 0 100000000-220000000
> [ 0.000000] SRAT: Node 1 PXM 1 220000000-420000000
-Andi
While we're dicking about with irq affinities, maybe we ought to
consider cpuset integration as well. There's various folks who want to
use exclusive cpusets to isolate workloads.
Andi Kleen a ?crit :
> Works here
>> dmesg | grep -i node
>> [ 0.000000] SRAT: PXM 0 -> APIC 0 -> Node 0
>> [ 0.000000] SRAT: PXM 0 -> APIC 1 -> Node 0
>> [ 0.000000] SRAT: PXM 0 -> APIC 2 -> Node 0
>> [ 0.000000] SRAT: PXM 0 -> APIC 3 -> Node 0
>> [ 0.000000] SRAT: PXM 0 -> APIC 4 -> Node 0
>> [ 0.000000] SRAT: PXM 0 -> APIC 5 -> Node 0
>> [ 0.000000] SRAT: PXM 0 -> APIC 6 -> Node 0
>> [ 0.000000] SRAT: PXM 0 -> APIC 7 -> Node 0
>
> You seem to only have 8 CPUs (one socket) Normally a dual socket nehalem
> should have 16 with HyperThreading enabled.
>
> For some reason the BIOS is not reporting the other CPU.
>
> You could double check with acpidmp / iasl -d if that's
> what the BIOS really reports, but normally it should work.
>
Good Lord, I had a CONFIG_NR_CPUS=16 in my .config.
Changing to to 32 or 64 seems better :)
# dmesg | grep -i node
[ 0.000000] SRAT: PXM 0 -> APIC 0 -> Node 0
[ 0.000000] SRAT: PXM 0 -> APIC 1 -> Node 0
[ 0.000000] SRAT: PXM 0 -> APIC 2 -> Node 0
[ 0.000000] SRAT: PXM 0 -> APIC 3 -> Node 0
[ 0.000000] SRAT: PXM 0 -> APIC 4 -> Node 0
[ 0.000000] SRAT: PXM 0 -> APIC 5 -> Node 0
[ 0.000000] SRAT: PXM 0 -> APIC 6 -> Node 0
[ 0.000000] SRAT: PXM 0 -> APIC 7 -> Node 0
[ 0.000000] SRAT: PXM 1 -> APIC 16 -> Node 1
[ 0.000000] SRAT: PXM 1 -> APIC 17 -> Node 1
[ 0.000000] SRAT: PXM 1 -> APIC 18 -> Node 1
[ 0.000000] SRAT: PXM 1 -> APIC 19 -> Node 1
[ 0.000000] SRAT: PXM 1 -> APIC 20 -> Node 1
[ 0.000000] SRAT: PXM 1 -> APIC 21 -> Node 1
[ 0.000000] SRAT: PXM 1 -> APIC 22 -> Node 1
[ 0.000000] SRAT: PXM 1 -> APIC 23 -> Node 1
[ 0.000000] SRAT: Node 0 PXM 0 0-e0000000
[ 0.000000] SRAT: Node 0 PXM 0 100000000-220000000
[ 0.000000] SRAT: Node 1 PXM 1 220000000-420000000
[ 0.000000] Bootmem setup node 0 0000000000000000-0000000220000000
[ 0.000000] NODE_DATA [0000000000001000 - 0000000000005fff]
[ 0.000000] Bootmem setup node 1 0000000220000000-000000041ffff000
[ 0.000000] NODE_DATA [0000000220000000 - 0000000220004fff]
[ 0.000000] [ffffea0000000000-ffffea00087fffff] PMD -> [ffff880028600000-ffff8800305fffff] on node 0
[ 0.000000] [ffffea0008800000-ffffea00107fffff] PMD -> [ffff880220200000-ffff8802281fffff] on node 1
[ 0.000000] Movable zone start PFN for each node
[ 0.000000] early_node_map[5] active PFN ranges
[ 0.000000] On node 0 totalpages: 2094543
[ 0.000000] On node 1 totalpages: 2097151
[ 0.000000] NR_CPUS:32 nr_cpumask_bits:32 nr_cpu_ids:32 nr_node_ids:2
[ 0.000000] SLUB: Genslabs=14, HWalign=64, Order=0-3, MinObjects=0, CPUs=32, Nodes=2
[ 0.004830] Inode-cache hash table entries: 1048576 (order: 11, 8388608 bytes)
[ 0.007291] CPU 0/0x0 -> Node 0
[ 0.398106] CPU 1/0x10 -> Node 1
[ 0.557857] CPU 2/0x4 -> Node 0
[ 0.717609] CPU 3/0x14 -> Node 1
[ 0.877359] CPU 4/0x2 -> Node 0
[ 1.037112] CPU 5/0x12 -> Node 1
[ 1.196862] CPU 6/0x6 -> Node 0
[ 1.356614] CPU 7/0x16 -> Node 1
[ 1.516368] CPU 8/0x1 -> Node 0
[ 1.676117] CPU 9/0x11 -> Node 1
[ 1.835867] CPU 10/0x5 -> Node 0
[ 1.995619] CPU 11/0x15 -> Node 1
[ 2.155370] CPU 12/0x3 -> Node 0
[ 2.315122] CPU 13/0x13 -> Node 1
[ 2.474873] CPU 14/0x7 -> Node 0
[ 2.634624] CPU 15/0x17 -> Node 1
Thanks Andi
On Wed, Nov 25, 2009 at 12:35:03PM +0100, Eric Dumazet wrote:
> > You seem to only have 8 CPUs (one socket) Normally a dual socket nehalem
> > should have 16 with HyperThreading enabled.
> >
> > For some reason the BIOS is not reporting the other CPU.
> >
> > You could double check with acpidmp / iasl -d if that's
> > what the BIOS really reports, but normally it should work.
> >
>
> Good Lord, I had a CONFIG_NR_CPUS=16 in my .config.
That should be enough for a two socket (2S x 4C x 2T) today,
but of course that will eventually change too.
> Changing to to 32 or 64 seems better :)
That looks weird. It should have worked with CONFIG_NR_CPUS==16 too,
because you only have 16 CPUs and the NR_CPUS should affect APIC
ranges etc.
Something still fishy. I would properly report it.
BTW kernel should give some error message in any case when
there are not enough CPUs I guess.
-Andi
--
[email protected] -- Speaking for myself only.
Andi Kleen a ?crit :
> That looks weird. It should have worked with CONFIG_NR_CPUS==16 too,
> because you only have 16 CPUs and the NR_CPUS should affect APIC
> ranges etc.
>
> Something still fishy. I would properly report it.
>
> BTW kernel should give some error message in any case when
> there are not enough CPUs I guess.
Problem comes from acpi_numa_init(), calling
acpi_table_parse_srat(ACPI_SRAT_TYPE_CPU_AFFINITY, acpi_parse_processor_affinity, NR_CPUS);
But on this machine SRAT table contains 64 processor_affinity entries,
part of them enabled and other parts disabled
[8 ENABLED], [8 disabled], [8 ENABLED], [40 disabled]
So if NR_CPUS = 16, we 'see' only 8 enabled entries and 8 disabled entries.
# acpidump -t SRAT >SRAT.dump
# acpixtract -a SRAT.dump
# iasl -d SRAT.dat
# cat SRAT.dsl
/*
* Intel ACPI Component Architecture
* AML Disassembler version 20090521
*
* Disassembly of SRAT.dat, Thu Nov 26 12:29:34 2009
*
* ACPI Data Table [SRAT]
*
* Format: [HexOffset DecimalOffset ByteLength] FieldName : FieldValue
*/
[000h 0000 4] Signature : "SRAT" /* System Resource Affinity Table */
[004h 0004 4] Table Length : 00000570
[008h 0008 1] Revision : 01
[009h 0009 1] Checksum : D9
[00Ah 0010 6] Oem ID : "HP "
[010h 0016 8] Oem Table ID : "Proliant"
[018h 0024 4] Oem Revision : 00000001
[01Ch 0028 4] Asl Compiler ID : " "
[020h 0032 4] Asl Compiler Revision : 0000162E
[024h 0036 4] Table Revision : 00000001
[028h 0040 8] Reserved : 0000000000000000
[030h 0048 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[031h 0049 1] Length : 10
[032h 0050 1] Proximity Domain Low(8) : 00
[033h 0051 1] Apic ID : 00
[034h 0052 4] Flags (decoded below) : 00000001
Enabled : 1
[038h 0056 1] Local Sapic EID : 00
[039h 0057 3] Proximity Domain High(24) : 000000
[03Ch 0060 4] Reserved : 00000000
[040h 0064 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[041h 0065 1] Length : 10
[042h 0066 1] Proximity Domain Low(8) : 00
[043h 0067 1] Apic ID : 01
[044h 0068 4] Flags (decoded below) : 00000001
Enabled : 1
[048h 0072 1] Local Sapic EID : 00
[049h 0073 3] Proximity Domain High(24) : 000000
[04Ch 0076 4] Reserved : 00000000
[050h 0080 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[051h 0081 1] Length : 10
[052h 0082 1] Proximity Domain Low(8) : 00
[053h 0083 1] Apic ID : 02
[054h 0084 4] Flags (decoded below) : 00000001
Enabled : 1
[058h 0088 1] Local Sapic EID : 00
[059h 0089 3] Proximity Domain High(24) : 000000
[05Ch 0092 4] Reserved : 00000000
[060h 0096 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[061h 0097 1] Length : 10
[062h 0098 1] Proximity Domain Low(8) : 00
[063h 0099 1] Apic ID : 03
[064h 0100 4] Flags (decoded below) : 00000001
Enabled : 1
[068h 0104 1] Local Sapic EID : 00
[069h 0105 3] Proximity Domain High(24) : 000000
[06Ch 0108 4] Reserved : 00000000
[070h 0112 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[071h 0113 1] Length : 10
[072h 0114 1] Proximity Domain Low(8) : 00
[073h 0115 1] Apic ID : 04
[074h 0116 4] Flags (decoded below) : 00000001
Enabled : 1
[078h 0120 1] Local Sapic EID : 00
[079h 0121 3] Proximity Domain High(24) : 000000
[07Ch 0124 4] Reserved : 00000000
[080h 0128 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[081h 0129 1] Length : 10
[082h 0130 1] Proximity Domain Low(8) : 00
[083h 0131 1] Apic ID : 05
[084h 0132 4] Flags (decoded below) : 00000001
Enabled : 1
[088h 0136 1] Local Sapic EID : 00
[089h 0137 3] Proximity Domain High(24) : 000000
[08Ch 0140 4] Reserved : 00000000
[090h 0144 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[091h 0145 1] Length : 10
[092h 0146 1] Proximity Domain Low(8) : 00
[093h 0147 1] Apic ID : 06
[094h 0148 4] Flags (decoded below) : 00000001
Enabled : 1
[098h 0152 1] Local Sapic EID : 00
[099h 0153 3] Proximity Domain High(24) : 000000
[09Ch 0156 4] Reserved : 00000000
[0A0h 0160 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[0A1h 0161 1] Length : 10
[0A2h 0162 1] Proximity Domain Low(8) : 00
[0A3h 0163 1] Apic ID : 07
[0A4h 0164 4] Flags (decoded below) : 00000001
Enabled : 1
[0A8h 0168 1] Local Sapic EID : 00
[0A9h 0169 3] Proximity Domain High(24) : 000000
[0ACh 0172 4] Reserved : 00000000
[0B0h 0176 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[0B1h 0177 1] Length : 10
[0B2h 0178 1] Proximity Domain Low(8) : 00
[0B3h 0179 1] Apic ID : 08
[0B4h 0180 4] Flags (decoded below) : 00000000
Enabled : 0
[0B8h 0184 1] Local Sapic EID : 00
[0B9h 0185 3] Proximity Domain High(24) : 000000
[0BCh 0188 4] Reserved : 00000000
[0C0h 0192 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[0C1h 0193 1] Length : 10
[0C2h 0194 1] Proximity Domain Low(8) : 00
[0C3h 0195 1] Apic ID : 09
[0C4h 0196 4] Flags (decoded below) : 00000000
Enabled : 0
[0C8h 0200 1] Local Sapic EID : 00
[0C9h 0201 3] Proximity Domain High(24) : 000000
[0CCh 0204 4] Reserved : 00000000
[0D0h 0208 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[0D1h 0209 1] Length : 10
[0D2h 0210 1] Proximity Domain Low(8) : 00
[0D3h 0211 1] Apic ID : 0A
[0D4h 0212 4] Flags (decoded below) : 00000000
Enabled : 0
[0D8h 0216 1] Local Sapic EID : 00
[0D9h 0217 3] Proximity Domain High(24) : 000000
[0DCh 0220 4] Reserved : 00000000
[0E0h 0224 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[0E1h 0225 1] Length : 10
[0E2h 0226 1] Proximity Domain Low(8) : 00
[0E3h 0227 1] Apic ID : 0B
[0E4h 0228 4] Flags (decoded below) : 00000000
Enabled : 0
[0E8h 0232 1] Local Sapic EID : 00
[0E9h 0233 3] Proximity Domain High(24) : 000000
[0ECh 0236 4] Reserved : 00000000
[0F0h 0240 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[0F1h 0241 1] Length : 10
[0F2h 0242 1] Proximity Domain Low(8) : 00
[0F3h 0243 1] Apic ID : 0C
[0F4h 0244 4] Flags (decoded below) : 00000000
Enabled : 0
[0F8h 0248 1] Local Sapic EID : 00
[0F9h 0249 3] Proximity Domain High(24) : 000000
[0FCh 0252 4] Reserved : 00000000
[100h 0256 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[101h 0257 1] Length : 10
[102h 0258 1] Proximity Domain Low(8) : 00
[103h 0259 1] Apic ID : 0D
[104h 0260 4] Flags (decoded below) : 00000000
Enabled : 0
[108h 0264 1] Local Sapic EID : 00
[109h 0265 3] Proximity Domain High(24) : 000000
[10Ch 0268 4] Reserved : 00000000
[110h 0272 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[111h 0273 1] Length : 10
[112h 0274 1] Proximity Domain Low(8) : 00
[113h 0275 1] Apic ID : 0E
[114h 0276 4] Flags (decoded below) : 00000000
Enabled : 0
[118h 0280 1] Local Sapic EID : 00
[119h 0281 3] Proximity Domain High(24) : 000000
[11Ch 0284 4] Reserved : 00000000
[120h 0288 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[121h 0289 1] Length : 10
[122h 0290 1] Proximity Domain Low(8) : 00
[123h 0291 1] Apic ID : 0F
[124h 0292 4] Flags (decoded below) : 00000000
Enabled : 0
[128h 0296 1] Local Sapic EID : 00
[129h 0297 3] Proximity Domain High(24) : 000000
[12Ch 0300 4] Reserved : 00000000
[130h 0304 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[131h 0305 1] Length : 10
[132h 0306 1] Proximity Domain Low(8) : 01
[133h 0307 1] Apic ID : 10
[134h 0308 4] Flags (decoded below) : 00000001
Enabled : 1
[138h 0312 1] Local Sapic EID : 00
[139h 0313 3] Proximity Domain High(24) : 000000
[13Ch 0316 4] Reserved : 00000000
[140h 0320 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[141h 0321 1] Length : 10
[142h 0322 1] Proximity Domain Low(8) : 01
[143h 0323 1] Apic ID : 11
[144h 0324 4] Flags (decoded below) : 00000001
Enabled : 1
[148h 0328 1] Local Sapic EID : 00
[149h 0329 3] Proximity Domain High(24) : 000000
[14Ch 0332 4] Reserved : 00000000
[150h 0336 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[151h 0337 1] Length : 10
[152h 0338 1] Proximity Domain Low(8) : 01
[153h 0339 1] Apic ID : 12
[154h 0340 4] Flags (decoded below) : 00000001
Enabled : 1
[158h 0344 1] Local Sapic EID : 00
[159h 0345 3] Proximity Domain High(24) : 000000
[15Ch 0348 4] Reserved : 00000000
[160h 0352 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[161h 0353 1] Length : 10
[162h 0354 1] Proximity Domain Low(8) : 01
[163h 0355 1] Apic ID : 13
[164h 0356 4] Flags (decoded below) : 00000001
Enabled : 1
[168h 0360 1] Local Sapic EID : 00
[169h 0361 3] Proximity Domain High(24) : 000000
[16Ch 0364 4] Reserved : 00000000
[170h 0368 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[171h 0369 1] Length : 10
[172h 0370 1] Proximity Domain Low(8) : 01
[173h 0371 1] Apic ID : 14
[174h 0372 4] Flags (decoded below) : 00000001
Enabled : 1
[178h 0376 1] Local Sapic EID : 00
[179h 0377 3] Proximity Domain High(24) : 000000
[17Ch 0380 4] Reserved : 00000000
[180h 0384 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[181h 0385 1] Length : 10
[182h 0386 1] Proximity Domain Low(8) : 01
[183h 0387 1] Apic ID : 15
[184h 0388 4] Flags (decoded below) : 00000001
Enabled : 1
[188h 0392 1] Local Sapic EID : 00
[189h 0393 3] Proximity Domain High(24) : 000000
[18Ch 0396 4] Reserved : 00000000
[190h 0400 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[191h 0401 1] Length : 10
[192h 0402 1] Proximity Domain Low(8) : 01
[193h 0403 1] Apic ID : 16
[194h 0404 4] Flags (decoded below) : 00000001
Enabled : 1
[198h 0408 1] Local Sapic EID : 00
[199h 0409 3] Proximity Domain High(24) : 000000
[19Ch 0412 4] Reserved : 00000000
[1A0h 0416 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[1A1h 0417 1] Length : 10
[1A2h 0418 1] Proximity Domain Low(8) : 01
[1A3h 0419 1] Apic ID : 17
[1A4h 0420 4] Flags (decoded below) : 00000001
Enabled : 1
[1A8h 0424 1] Local Sapic EID : 00
[1A9h 0425 3] Proximity Domain High(24) : 000000
[1ACh 0428 4] Reserved : 00000000
[1B0h 0432 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[1B1h 0433 1] Length : 10
[1B2h 0434 1] Proximity Domain Low(8) : 00
[1B3h 0435 1] Apic ID : 18
[1B4h 0436 4] Flags (decoded below) : 00000000
Enabled : 0
[1B8h 0440 1] Local Sapic EID : 00
[1B9h 0441 3] Proximity Domain High(24) : 000000
[1BCh 0444 4] Reserved : 00000000
[1C0h 0448 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[1C1h 0449 1] Length : 10
[1C2h 0450 1] Proximity Domain Low(8) : 00
[1C3h 0451 1] Apic ID : 19
[1C4h 0452 4] Flags (decoded below) : 00000000
Enabled : 0
[1C8h 0456 1] Local Sapic EID : 00
[1C9h 0457 3] Proximity Domain High(24) : 000000
[1CCh 0460 4] Reserved : 00000000
[1D0h 0464 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[1D1h 0465 1] Length : 10
[1D2h 0466 1] Proximity Domain Low(8) : 00
[1D3h 0467 1] Apic ID : 1A
[1D4h 0468 4] Flags (decoded below) : 00000000
Enabled : 0
[1D8h 0472 1] Local Sapic EID : 00
[1D9h 0473 3] Proximity Domain High(24) : 000000
[1DCh 0476 4] Reserved : 00000000
[1E0h 0480 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[1E1h 0481 1] Length : 10
[1E2h 0482 1] Proximity Domain Low(8) : 00
[1E3h 0483 1] Apic ID : 1B
[1E4h 0484 4] Flags (decoded below) : 00000000
Enabled : 0
[1E8h 0488 1] Local Sapic EID : 00
[1E9h 0489 3] Proximity Domain High(24) : 000000
[1ECh 0492 4] Reserved : 00000000
[1F0h 0496 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[1F1h 0497 1] Length : 10
[1F2h 0498 1] Proximity Domain Low(8) : 00
[1F3h 0499 1] Apic ID : 1C
[1F4h 0500 4] Flags (decoded below) : 00000000
Enabled : 0
[1F8h 0504 1] Local Sapic EID : 00
[1F9h 0505 3] Proximity Domain High(24) : 000000
[1FCh 0508 4] Reserved : 00000000
[200h 0512 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[201h 0513 1] Length : 10
[202h 0514 1] Proximity Domain Low(8) : 00
[203h 0515 1] Apic ID : 1D
[204h 0516 4] Flags (decoded below) : 00000000
Enabled : 0
[208h 0520 1] Local Sapic EID : 00
[209h 0521 3] Proximity Domain High(24) : 000000
[20Ch 0524 4] Reserved : 00000000
[210h 0528 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[211h 0529 1] Length : 10
[212h 0530 1] Proximity Domain Low(8) : 00
[213h 0531 1] Apic ID : 1E
[214h 0532 4] Flags (decoded below) : 00000000
Enabled : 0
[218h 0536 1] Local Sapic EID : 00
[219h 0537 3] Proximity Domain High(24) : 000000
[21Ch 0540 4] Reserved : 00000000
[220h 0544 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[221h 0545 1] Length : 10
[222h 0546 1] Proximity Domain Low(8) : 00
[223h 0547 1] Apic ID : 1F
[224h 0548 4] Flags (decoded below) : 00000000
Enabled : 0
[228h 0552 1] Local Sapic EID : 00
[229h 0553 3] Proximity Domain High(24) : 000000
[22Ch 0556 4] Reserved : 00000000
[230h 0560 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[231h 0561 1] Length : 10
[232h 0562 1] Proximity Domain Low(8) : 00
[233h 0563 1] Apic ID : 20
[234h 0564 4] Flags (decoded below) : 00000000
Enabled : 0
[238h 0568 1] Local Sapic EID : 00
[239h 0569 3] Proximity Domain High(24) : 000000
[23Ch 0572 4] Reserved : 00000000
[240h 0576 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[241h 0577 1] Length : 10
[242h 0578 1] Proximity Domain Low(8) : 00
[243h 0579 1] Apic ID : 21
[244h 0580 4] Flags (decoded below) : 00000000
Enabled : 0
[248h 0584 1] Local Sapic EID : 00
[249h 0585 3] Proximity Domain High(24) : 000000
[24Ch 0588 4] Reserved : 00000000
[250h 0592 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[251h 0593 1] Length : 10
[252h 0594 1] Proximity Domain Low(8) : 00
[253h 0595 1] Apic ID : 22
[254h 0596 4] Flags (decoded below) : 00000000
Enabled : 0
[258h 0600 1] Local Sapic EID : 00
[259h 0601 3] Proximity Domain High(24) : 000000
[25Ch 0604 4] Reserved : 00000000
[260h 0608 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[261h 0609 1] Length : 10
[262h 0610 1] Proximity Domain Low(8) : 00
[263h 0611 1] Apic ID : 23
[264h 0612 4] Flags (decoded below) : 00000000
Enabled : 0
[268h 0616 1] Local Sapic EID : 00
[269h 0617 3] Proximity Domain High(24) : 000000
[26Ch 0620 4] Reserved : 00000000
[270h 0624 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[271h 0625 1] Length : 10
[272h 0626 1] Proximity Domain Low(8) : 00
[273h 0627 1] Apic ID : 24
[274h 0628 4] Flags (decoded below) : 00000000
Enabled : 0
[278h 0632 1] Local Sapic EID : 00
[279h 0633 3] Proximity Domain High(24) : 000000
[27Ch 0636 4] Reserved : 00000000
[280h 0640 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[281h 0641 1] Length : 10
[282h 0642 1] Proximity Domain Low(8) : 00
[283h 0643 1] Apic ID : 25
[284h 0644 4] Flags (decoded below) : 00000000
Enabled : 0
[288h 0648 1] Local Sapic EID : 00
[289h 0649 3] Proximity Domain High(24) : 000000
[28Ch 0652 4] Reserved : 00000000
[290h 0656 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[291h 0657 1] Length : 10
[292h 0658 1] Proximity Domain Low(8) : 00
[293h 0659 1] Apic ID : 26
[294h 0660 4] Flags (decoded below) : 00000000
Enabled : 0
[298h 0664 1] Local Sapic EID : 00
[299h 0665 3] Proximity Domain High(24) : 000000
[29Ch 0668 4] Reserved : 00000000
[2A0h 0672 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[2A1h 0673 1] Length : 10
[2A2h 0674 1] Proximity Domain Low(8) : 00
[2A3h 0675 1] Apic ID : 27
[2A4h 0676 4] Flags (decoded below) : 00000000
Enabled : 0
[2A8h 0680 1] Local Sapic EID : 00
[2A9h 0681 3] Proximity Domain High(24) : 000000
[2ACh 0684 4] Reserved : 00000000
[2B0h 0688 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[2B1h 0689 1] Length : 10
[2B2h 0690 1] Proximity Domain Low(8) : 00
[2B3h 0691 1] Apic ID : 28
[2B4h 0692 4] Flags (decoded below) : 00000000
Enabled : 0
[2B8h 0696 1] Local Sapic EID : 00
[2B9h 0697 3] Proximity Domain High(24) : 000000
[2BCh 0700 4] Reserved : 00000000
[2C0h 0704 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[2C1h 0705 1] Length : 10
[2C2h 0706 1] Proximity Domain Low(8) : 00
[2C3h 0707 1] Apic ID : 29
[2C4h 0708 4] Flags (decoded below) : 00000000
Enabled : 0
[2C8h 0712 1] Local Sapic EID : 00
[2C9h 0713 3] Proximity Domain High(24) : 000000
[2CCh 0716 4] Reserved : 00000000
[2D0h 0720 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[2D1h 0721 1] Length : 10
[2D2h 0722 1] Proximity Domain Low(8) : 00
[2D3h 0723 1] Apic ID : 2A
[2D4h 0724 4] Flags (decoded below) : 00000000
Enabled : 0
[2D8h 0728 1] Local Sapic EID : 00
[2D9h 0729 3] Proximity Domain High(24) : 000000
[2DCh 0732 4] Reserved : 00000000
[2E0h 0736 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[2E1h 0737 1] Length : 10
[2E2h 0738 1] Proximity Domain Low(8) : 00
[2E3h 0739 1] Apic ID : 2B
[2E4h 0740 4] Flags (decoded below) : 00000000
Enabled : 0
[2E8h 0744 1] Local Sapic EID : 00
[2E9h 0745 3] Proximity Domain High(24) : 000000
[2ECh 0748 4] Reserved : 00000000
[2F0h 0752 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[2F1h 0753 1] Length : 10
[2F2h 0754 1] Proximity Domain Low(8) : 00
[2F3h 0755 1] Apic ID : 2C
[2F4h 0756 4] Flags (decoded below) : 00000000
Enabled : 0
[2F8h 0760 1] Local Sapic EID : 00
[2F9h 0761 3] Proximity Domain High(24) : 000000
[2FCh 0764 4] Reserved : 00000000
[300h 0768 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[301h 0769 1] Length : 10
[302h 0770 1] Proximity Domain Low(8) : 00
[303h 0771 1] Apic ID : 2D
[304h 0772 4] Flags (decoded below) : 00000000
Enabled : 0
[308h 0776 1] Local Sapic EID : 00
[309h 0777 3] Proximity Domain High(24) : 000000
[30Ch 0780 4] Reserved : 00000000
[310h 0784 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[311h 0785 1] Length : 10
[312h 0786 1] Proximity Domain Low(8) : 00
[313h 0787 1] Apic ID : 2E
[314h 0788 4] Flags (decoded below) : 00000000
Enabled : 0
[318h 0792 1] Local Sapic EID : 00
[319h 0793 3] Proximity Domain High(24) : 000000
[31Ch 0796 4] Reserved : 00000000
[320h 0800 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[321h 0801 1] Length : 10
[322h 0802 1] Proximity Domain Low(8) : 00
[323h 0803 1] Apic ID : 2F
[324h 0804 4] Flags (decoded below) : 00000000
Enabled : 0
[328h 0808 1] Local Sapic EID : 00
[329h 0809 3] Proximity Domain High(24) : 000000
[32Ch 0812 4] Reserved : 00000000
[330h 0816 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[331h 0817 1] Length : 10
[332h 0818 1] Proximity Domain Low(8) : 00
[333h 0819 1] Apic ID : 30
[334h 0820 4] Flags (decoded below) : 00000000
Enabled : 0
[338h 0824 1] Local Sapic EID : 00
[339h 0825 3] Proximity Domain High(24) : 000000
[33Ch 0828 4] Reserved : 00000000
[340h 0832 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[341h 0833 1] Length : 10
[342h 0834 1] Proximity Domain Low(8) : 00
[343h 0835 1] Apic ID : 31
[344h 0836 4] Flags (decoded below) : 00000000
Enabled : 0
[348h 0840 1] Local Sapic EID : 00
[349h 0841 3] Proximity Domain High(24) : 000000
[34Ch 0844 4] Reserved : 00000000
[350h 0848 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[351h 0849 1] Length : 10
[352h 0850 1] Proximity Domain Low(8) : 00
[353h 0851 1] Apic ID : 32
[354h 0852 4] Flags (decoded below) : 00000000
Enabled : 0
[358h 0856 1] Local Sapic EID : 00
[359h 0857 3] Proximity Domain High(24) : 000000
[35Ch 0860 4] Reserved : 00000000
[360h 0864 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[361h 0865 1] Length : 10
[362h 0866 1] Proximity Domain Low(8) : 00
[363h 0867 1] Apic ID : 33
[364h 0868 4] Flags (decoded below) : 00000000
Enabled : 0
[368h 0872 1] Local Sapic EID : 00
[369h 0873 3] Proximity Domain High(24) : 000000
[36Ch 0876 4] Reserved : 00000000
[370h 0880 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[371h 0881 1] Length : 10
[372h 0882 1] Proximity Domain Low(8) : 00
[373h 0883 1] Apic ID : 34
[374h 0884 4] Flags (decoded below) : 00000000
Enabled : 0
[378h 0888 1] Local Sapic EID : 00
[379h 0889 3] Proximity Domain High(24) : 000000
[37Ch 0892 4] Reserved : 00000000
[380h 0896 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[381h 0897 1] Length : 10
[382h 0898 1] Proximity Domain Low(8) : 00
[383h 0899 1] Apic ID : 35
[384h 0900 4] Flags (decoded below) : 00000000
Enabled : 0
[388h 0904 1] Local Sapic EID : 00
[389h 0905 3] Proximity Domain High(24) : 000000
[38Ch 0908 4] Reserved : 00000000
[390h 0912 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[391h 0913 1] Length : 10
[392h 0914 1] Proximity Domain Low(8) : 00
[393h 0915 1] Apic ID : 36
[394h 0916 4] Flags (decoded below) : 00000000
Enabled : 0
[398h 0920 1] Local Sapic EID : 00
[399h 0921 3] Proximity Domain High(24) : 000000
[39Ch 0924 4] Reserved : 00000000
[3A0h 0928 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[3A1h 0929 1] Length : 10
[3A2h 0930 1] Proximity Domain Low(8) : 00
[3A3h 0931 1] Apic ID : 37
[3A4h 0932 4] Flags (decoded below) : 00000000
Enabled : 0
[3A8h 0936 1] Local Sapic EID : 00
[3A9h 0937 3] Proximity Domain High(24) : 000000
[3ACh 0940 4] Reserved : 00000000
[3B0h 0944 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[3B1h 0945 1] Length : 10
[3B2h 0946 1] Proximity Domain Low(8) : 00
[3B3h 0947 1] Apic ID : 38
[3B4h 0948 4] Flags (decoded below) : 00000000
Enabled : 0
[3B8h 0952 1] Local Sapic EID : 00
[3B9h 0953 3] Proximity Domain High(24) : 000000
[3BCh 0956 4] Reserved : 00000000
[3C0h 0960 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[3C1h 0961 1] Length : 10
[3C2h 0962 1] Proximity Domain Low(8) : 00
[3C3h 0963 1] Apic ID : 39
[3C4h 0964 4] Flags (decoded below) : 00000000
Enabled : 0
[3C8h 0968 1] Local Sapic EID : 00
[3C9h 0969 3] Proximity Domain High(24) : 000000
[3CCh 0972 4] Reserved : 00000000
[3D0h 0976 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[3D1h 0977 1] Length : 10
[3D2h 0978 1] Proximity Domain Low(8) : 00
[3D3h 0979 1] Apic ID : 3A
[3D4h 0980 4] Flags (decoded below) : 00000000
Enabled : 0
[3D8h 0984 1] Local Sapic EID : 00
[3D9h 0985 3] Proximity Domain High(24) : 000000
[3DCh 0988 4] Reserved : 00000000
[3E0h 0992 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[3E1h 0993 1] Length : 10
[3E2h 0994 1] Proximity Domain Low(8) : 00
[3E3h 0995 1] Apic ID : 3B
[3E4h 0996 4] Flags (decoded below) : 00000000
Enabled : 0
[3E8h 1000 1] Local Sapic EID : 00
[3E9h 1001 3] Proximity Domain High(24) : 000000
[3ECh 1004 4] Reserved : 00000000
[3F0h 1008 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[3F1h 1009 1] Length : 10
[3F2h 1010 1] Proximity Domain Low(8) : 00
[3F3h 1011 1] Apic ID : 3C
[3F4h 1012 4] Flags (decoded below) : 00000000
Enabled : 0
[3F8h 1016 1] Local Sapic EID : 00
[3F9h 1017 3] Proximity Domain High(24) : 000000
[3FCh 1020 4] Reserved : 00000000
[400h 1024 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[401h 1025 1] Length : 10
[402h 1026 1] Proximity Domain Low(8) : 00
[403h 1027 1] Apic ID : 3D
[404h 1028 4] Flags (decoded below) : 00000000
Enabled : 0
[408h 1032 1] Local Sapic EID : 00
[409h 1033 3] Proximity Domain High(24) : 000000
[40Ch 1036 4] Reserved : 00000000
[410h 1040 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[411h 1041 1] Length : 10
[412h 1042 1] Proximity Domain Low(8) : 00
[413h 1043 1] Apic ID : 3E
[414h 1044 4] Flags (decoded below) : 00000000
Enabled : 0
[418h 1048 1] Local Sapic EID : 00
[419h 1049 3] Proximity Domain High(24) : 000000
[41Ch 1052 4] Reserved : 00000000
[420h 1056 1] Subtable Type : 00 <Processor Local APIC/SAPIC Affinity>
[421h 1057 1] Length : 10
[422h 1058 1] Proximity Domain Low(8) : 00
[423h 1059 1] Apic ID : 3F
[424h 1060 4] Flags (decoded below) : 00000000
Enabled : 0
[428h 1064 1] Local Sapic EID : 00
[429h 1065 3] Proximity Domain High(24) : 000000
[42Ch 1068 4] Reserved : 00000000
[430h 1072 1] Subtable Type : 01 <Memory Affinity>
[431h 1073 1] Length : 28
[432h 1074 4] Proximity Domain : 00000000
[436h 1078 2] Reserved : 0000
[438h 1080 8] Base Address : 0000000000000000
[440h 1088 8] Address Length : 00000000E0000000
[448h 1096 4] Reserved : 00000000
[44Ch 1100 4] Flags (decoded below) : 00000001
Enabled : 1
Hot Pluggable : 0
Non-Volatile : 0
[450h 1104 8] Reserved : 0000000000000000
[458h 1112 1] Subtable Type : 01 <Memory Affinity>
[459h 1113 1] Length : 28
[45Ah 1114 4] Proximity Domain : 00000000
[45Eh 1118 2] Reserved : 0000
[460h 1120 8] Base Address : 00000000E0000000
[468h 1128 8] Address Length : 0000000020000000
[470h 1136 4] Reserved : 00000000
[474h 1140 4] Flags (decoded below) : 00000000
Enabled : 0
Hot Pluggable : 0
Non-Volatile : 0
[478h 1144 8] Reserved : 0000000000000000
[480h 1152 1] Subtable Type : 01 <Memory Affinity>
[481h 1153 1] Length : 28
[482h 1154 4] Proximity Domain : 00000000
[486h 1158 2] Reserved : 0000
[488h 1160 8] Base Address : 0000000100000000
[490h 1168 8] Address Length : 0000000120000000
[498h 1176 4] Reserved : 00000000
[49Ch 1180 4] Flags (decoded below) : 00000001
Enabled : 1
Hot Pluggable : 0
Non-Volatile : 0
[4A0h 1184 8] Reserved : 0000000000000000
[4A8h 1192 1] Subtable Type : 01 <Memory Affinity>
[4A9h 1193 1] Length : 28
[4AAh 1194 4] Proximity Domain : 00000001
[4AEh 1198 2] Reserved : 0000
[4B0h 1200 8] Base Address : 0000000220000000
[4B8h 1208 8] Address Length : 0000000200000000
[4C0h 1216 4] Reserved : 00000000
[4C4h 1220 4] Flags (decoded below) : 00000001
Enabled : 1
Hot Pluggable : 0
Non-Volatile : 0
[4C8h 1224 8] Reserved : 0000000000000000
[4D0h 1232 1] Subtable Type : 01 <Memory Affinity>
[4D1h 1233 1] Length : 28
[4D2h 1234 4] Proximity Domain : 00000000
[4D6h 1238 2] Reserved : 0000
[4D8h 1240 8] Base Address : 0000000420000000
[4E0h 1248 8] Address Length : 0000000000000000
[4E8h 1256 4] Reserved : 00000000
[4ECh 1260 4] Flags (decoded below) : 00000000
Enabled : 0
Hot Pluggable : 0
Non-Volatile : 0
[4F0h 1264 8] Reserved : 0000000000000000
[4F8h 1272 1] Subtable Type : 01 <Memory Affinity>
[4F9h 1273 1] Length : 28
[4FAh 1274 4] Proximity Domain : 00000000
[4FEh 1278 2] Reserved : 0000
[500h 1280 8] Base Address : 0000000420000000
[508h 1288 8] Address Length : 0000000000000000
[510h 1296 4] Reserved : 00000000
[514h 1300 4] Flags (decoded below) : 00000000
Enabled : 0
Hot Pluggable : 0
Non-Volatile : 0
[518h 1304 8] Reserved : 0000000000000000
[520h 1312 1] Subtable Type : 01 <Memory Affinity>
[521h 1313 1] Length : 28
[522h 1314 4] Proximity Domain : 00000000
[526h 1318 2] Reserved : 0000
[528h 1320 8] Base Address : 0000000420000000
[530h 1328 8] Address Length : 0000000000000000
[538h 1336 4] Reserved : 00000000
[53Ch 1340 4] Flags (decoded below) : 00000000
Enabled : 0
Hot Pluggable : 0
Non-Volatile : 0
[540h 1344 8] Reserved : 0000000000000000
[548h 1352 1] Subtable Type : 01 <Memory Affinity>
[549h 1353 1] Length : 28
[54Ah 1354 4] Proximity Domain : 00000000
[54Eh 1358 2] Reserved : 0000
[550h 1360 8] Base Address : 0000000420000000
[558h 1368 8] Address Length : 0000000000000000
[560h 1376 4] Reserved : 00000000
[564h 1380 4] Flags (decoded below) : 00000000
Enabled : 0
Hot Pluggable : 0
Non-Volatile : 0
[568h 1384 8] Reserved : 0000000000000000
Raw Table Data
0000: 53 52 41 54 70 05 00 00 01 D9 48 50 20 20 20 20 SRATp.....HP
0010: 50 72 6F 6C 69 61 6E 74 01 00 00 00 20 20 20 20 Proliant....
0020: 2E 16 00 00 01 00 00 00 00 00 00 00 00 00 00 00 ................
0030: 00 10 00 00 01 00 00 00 00 00 00 00 00 00 00 00 ................
0040: 00 10 00 01 01 00 00 00 00 00 00 00 00 00 00 00 ................
0050: 00 10 00 02 01 00 00 00 00 00 00 00 00 00 00 00 ................
0060: 00 10 00 03 01 00 00 00 00 00 00 00 00 00 00 00 ................
0070: 00 10 00 04 01 00 00 00 00 00 00 00 00 00 00 00 ................
0080: 00 10 00 05 01 00 00 00 00 00 00 00 00 00 00 00 ................
0090: 00 10 00 06 01 00 00 00 00 00 00 00 00 00 00 00 ................
00A0: 00 10 00 07 01 00 00 00 00 00 00 00 00 00 00 00 ................
00B0: 00 10 00 08 00 00 00 00 00 00 00 00 00 00 00 00 ................
00C0: 00 10 00 09 00 00 00 00 00 00 00 00 00 00 00 00 ................
00D0: 00 10 00 0A 00 00 00 00 00 00 00 00 00 00 00 00 ................
00E0: 00 10 00 0B 00 00 00 00 00 00 00 00 00 00 00 00 ................
00F0: 00 10 00 0C 00 00 00 00 00 00 00 00 00 00 00 00 ................
0100: 00 10 00 0D 00 00 00 00 00 00 00 00 00 00 00 00 ................
0110: 00 10 00 0E 00 00 00 00 00 00 00 00 00 00 00 00 ................
0120: 00 10 00 0F 00 00 00 00 00 00 00 00 00 00 00 00 ................
0130: 00 10 01 10 01 00 00 00 00 00 00 00 00 00 00 00 ................
0140: 00 10 01 11 01 00 00 00 00 00 00 00 00 00 00 00 ................
0150: 00 10 01 12 01 00 00 00 00 00 00 00 00 00 00 00 ................
0160: 00 10 01 13 01 00 00 00 00 00 00 00 00 00 00 00 ................
0170: 00 10 01 14 01 00 00 00 00 00 00 00 00 00 00 00 ................
0180: 00 10 01 15 01 00 00 00 00 00 00 00 00 00 00 00 ................
0190: 00 10 01 16 01 00 00 00 00 00 00 00 00 00 00 00 ................
01A0: 00 10 01 17 01 00 00 00 00 00 00 00 00 00 00 00 ................
01B0: 00 10 00 18 00 00 00 00 00 00 00 00 00 00 00 00 ................
01C0: 00 10 00 19 00 00 00 00 00 00 00 00 00 00 00 00 ................
01D0: 00 10 00 1A 00 00 00 00 00 00 00 00 00 00 00 00 ................
01E0: 00 10 00 1B 00 00 00 00 00 00 00 00 00 00 00 00 ................
01F0: 00 10 00 1C 00 00 00 00 00 00 00 00 00 00 00 00 ................
0200: 00 10 00 1D 00 00 00 00 00 00 00 00 00 00 00 00 ................
0210: 00 10 00 1E 00 00 00 00 00 00 00 00 00 00 00 00 ................
0220: 00 10 00 1F 00 00 00 00 00 00 00 00 00 00 00 00 ................
0230: 00 10 00 20 00 00 00 00 00 00 00 00 00 00 00 00 ... ............
0240: 00 10 00 21 00 00 00 00 00 00 00 00 00 00 00 00 ...!............
0250: 00 10 00 22 00 00 00 00 00 00 00 00 00 00 00 00 ..."............
0260: 00 10 00 23 00 00 00 00 00 00 00 00 00 00 00 00 ...#............
0270: 00 10 00 24 00 00 00 00 00 00 00 00 00 00 00 00 ...$............
0280: 00 10 00 25 00 00 00 00 00 00 00 00 00 00 00 00 ...%............
0290: 00 10 00 26 00 00 00 00 00 00 00 00 00 00 00 00 ...&............
02A0: 00 10 00 27 00 00 00 00 00 00 00 00 00 00 00 00 ...'............
02B0: 00 10 00 28 00 00 00 00 00 00 00 00 00 00 00 00 ...(............
02C0: 00 10 00 29 00 00 00 00 00 00 00 00 00 00 00 00 ...)............
02D0: 00 10 00 2A 00 00 00 00 00 00 00 00 00 00 00 00 ...*............
02E0: 00 10 00 2B 00 00 00 00 00 00 00 00 00 00 00 00 ...+............
02F0: 00 10 00 2C 00 00 00 00 00 00 00 00 00 00 00 00 ...,............
0300: 00 10 00 2D 00 00 00 00 00 00 00 00 00 00 00 00 ...-............
0310: 00 10 00 2E 00 00 00 00 00 00 00 00 00 00 00 00 ................
0320: 00 10 00 2F 00 00 00 00 00 00 00 00 00 00 00 00 .../............
0330: 00 10 00 30 00 00 00 00 00 00 00 00 00 00 00 00 ...0............
0340: 00 10 00 31 00 00 00 00 00 00 00 00 00 00 00 00 ...1............
0350: 00 10 00 32 00 00 00 00 00 00 00 00 00 00 00 00 ...2............
0360: 00 10 00 33 00 00 00 00 00 00 00 00 00 00 00 00 ...3............
0370: 00 10 00 34 00 00 00 00 00 00 00 00 00 00 00 00 ...4............
0380: 00 10 00 35 00 00 00 00 00 00 00 00 00 00 00 00 ...5............
0390: 00 10 00 36 00 00 00 00 00 00 00 00 00 00 00 00 ...6............
03A0: 00 10 00 37 00 00 00 00 00 00 00 00 00 00 00 00 ...7............
03B0: 00 10 00 38 00 00 00 00 00 00 00 00 00 00 00 00 ...8............
03C0: 00 10 00 39 00 00 00 00 00 00 00 00 00 00 00 00 ...9............
03D0: 00 10 00 3A 00 00 00 00 00 00 00 00 00 00 00 00 ...:............
03E0: 00 10 00 3B 00 00 00 00 00 00 00 00 00 00 00 00 ...;............
03F0: 00 10 00 3C 00 00 00 00 00 00 00 00 00 00 00 00 ...<............
0400: 00 10 00 3D 00 00 00 00 00 00 00 00 00 00 00 00 ...=............
0410: 00 10 00 3E 00 00 00 00 00 00 00 00 00 00 00 00 ...>............
0420: 00 10 00 3F 00 00 00 00 00 00 00 00 00 00 00 00 ...?............
0430: 01 28 00 00 00 00 00 00 00 00 00 00 00 00 00 00 .(..............
0440: 00 00 00 E0 00 00 00 00 00 00 00 00 01 00 00 00 ................
0450: 00 00 00 00 00 00 00 00 01 28 00 00 00 00 00 00 .........(......
0460: 00 00 00 E0 00 00 00 00 00 00 00 20 00 00 00 00 ........... ....
0470: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
0480: 01 28 00 00 00 00 00 00 00 00 00 00 01 00 00 00 .(..............
0490: 00 00 00 20 01 00 00 00 00 00 00 00 01 00 00 00 ... ............
04A0: 00 00 00 00 00 00 00 00 01 28 01 00 00 00 00 00 .........(......
04B0: 00 00 00 20 02 00 00 00 00 00 00 00 02 00 00 00 ... ............
04C0: 00 00 00 00 01 00 00 00 00 00 00 00 00 00 00 00 ................
04D0: 01 28 00 00 00 00 00 00 00 00 00 20 04 00 00 00 .(......... ....
04E0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
04F0: 00 00 00 00 00 00 00 00 01 28 00 00 00 00 00 00 .........(......
0500: 00 00 00 20 04 00 00 00 00 00 00 00 00 00 00 00 ... ............
0510: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
0520: 01 28 00 00 00 00 00 00 00 00 00 20 04 00 00 00 .(......... ....
0530: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................
0540: 00 00 00 00 00 00 00 00 01 28 00 00 00 00 00 00 .........(......
0550: 00 00 00 20 04 00 00 00 00 00 00 00 00 00 00 00 ... ............
0560: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 ................