by Wen Congyang

[permalink] [raw]

Subject: Re: [PATCH v2 10/12] memory-hotplug: memory_hotplug: clear zone when removing the memory

At 10/29/2012 11:34 PM, Jianguo Wu Wrote:
> On 2012/10/23 18:30, [email protected] wrote:
>> From: Yasuaki Ishimatsu <[email protected]>
>>
>> When a memory is added, we update zone's and pgdat's start_pfn and
>> spanned_pages in the function __add_zone(). So we should revert them
>> when the memory is removed.
>>
>> The patch adds a new function __remove_zone() to do this.
>>
>> CC: David Rientjes <[email protected]>
>> CC: Jiang Liu <[email protected]>
>> CC: Len Brown <[email protected]>
>> CC: Christoph Lameter <[email protected]>
>> Cc: Minchan Kim <[email protected]>
>> CC: Andrew Morton <[email protected]>
>> CC: KOSAKI Motohiro <[email protected]>
>> Signed-off-by: Yasuaki Ishimatsu <[email protected]>
>> Signed-off-by: Wen Congyang <[email protected]>
>> ---
>> mm/memory_hotplug.c | 207 +++++++++++++++++++++++++++++++++++++++++++++++++++
>> 1 files changed, 207 insertions(+), 0 deletions(-)
>>
>> diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
>> index 03153cf..55a228d 100644
>> --- a/mm/memory_hotplug.c
>> +++ b/mm/memory_hotplug.c
>> @@ -312,10 +312,213 @@ static int __meminit __add_section(int nid, struct zone *zone,
>> return register_new_memory(nid, __pfn_to_section(phys_start_pfn));
>> }
>>
>> +/* find the smallest valid pfn in the range [start_pfn, end_pfn) */
>> +static int find_smallest_section_pfn(int nid, struct zone *zone,
>> + unsigned long start_pfn,
>> + unsigned long end_pfn)
>> +{
>> + struct mem_section *ms;
>> +
>> + for (; start_pfn < end_pfn; start_pfn += PAGES_PER_SECTION) {
>> + ms = __pfn_to_section(start_pfn);
>> +
>> + if (unlikely(!valid_section(ms)))
>> + continue;
>> +
>> + if (unlikely(pfn_to_nid(start_pfn)) != nid)
>
> if (unlikely(pfn_to_nid(start_pfn) != nid))

Thanks for pointing it out. I will fix it soon.

Wen Congyang

>
>> + continue;
>> +
>> + if (zone && zone != page_zone(pfn_to_page(start_pfn)))
>> + continue;
>> +
>> + return start_pfn;
>> + }
>> +
>> + return 0;
>> +}
>> +
>> +/* find the biggest valid pfn in the range [start_pfn, end_pfn). */
>> +static int find_biggest_section_pfn(int nid, struct zone *zone,
>> + unsigned long start_pfn,
>> + unsigned long end_pfn)
>> +{
>> + struct mem_section *ms;
>> + unsigned long pfn;
>> +
>> + /* pfn is the end pfn of a memory section. */
>> + pfn = end_pfn - 1;
>> + for (; pfn >= start_pfn; pfn -= PAGES_PER_SECTION) {
>> + ms = __pfn_to_section(pfn);
>> +
>> + if (unlikely(!valid_section(ms)))
>> + continue;
>> +
>> + if (unlikely(pfn_to_nid(pfn)) != nid)
>
> if (unlikely(pfn_to_nid(pfn) != nid))
>
>> + continue;
>> +
>> + if (zone && zone != page_zone(pfn_to_page(pfn)))
>> + continue;
>> +
>> + return pfn;
>> + }
>> +
>> + return 0;
>> +}
>> +
>> +static void shrink_zone_span(struct zone *zone, unsigned long start_pfn,
>> + unsigned long end_pfn)
>> +{
>> + unsigned long zone_start_pfn = zone->zone_start_pfn;
>> + unsigned long zone_end_pfn = zone->zone_start_pfn + zone->spanned_pages;
>> + unsigned long pfn;
>> + struct mem_section *ms;
>> + int nid = zone_to_nid(zone);
>> +
>> + zone_span_writelock(zone);
>> + if (zone_start_pfn == start_pfn) {
>> + /*
>> + * If the section is smallest section in the zone, it need
>> + * shrink zone->zone_start_pfn and zone->zone_spanned_pages.
>> + * In this case, we find second smallest valid mem_section
>> + * for shrinking zone.
>> + */
>> + pfn = find_smallest_section_pfn(nid, zone, end_pfn,
>> + zone_end_pfn);
>> + if (pfn) {
>> + zone->zone_start_pfn = pfn;
>> + zone->spanned_pages = zone_end_pfn - pfn;
>> + }
>> + } else if (zone_end_pfn == end_pfn) {
>> + /*
>> + * If the section is biggest section in the zone, it need
>> + * shrink zone->spanned_pages.
>> + * In this case, we find second biggest valid mem_section for
>> + * shrinking zone.
>> + */
>> + pfn = find_biggest_section_pfn(nid, zone, zone_start_pfn,
>> + start_pfn);
>> + if (pfn)
>> + zone->spanned_pages = pfn - zone_start_pfn + 1;
>> + }
>> +
>> + /*
>> + * The section is not biggest or smallest mem_section in the zone, it
>> + * only creates a hole in the zone. So in this case, we need not
>> + * change the zone. But perhaps, the zone has only hole data. Thus
>> + * it check the zone has only hole or not.
>> + */
>> + pfn = zone_start_pfn;
>> + for (; pfn < zone_end_pfn; pfn += PAGES_PER_SECTION) {
>> + ms = __pfn_to_section(pfn);
>> +
>> + if (unlikely(!valid_section(ms)))
>> + continue;
>> +
>> + if (page_zone(pfn_to_page(pfn)) != zone)
>> + continue;
>> +
>> + /* If the section is current section, it continues the loop */
>> + if (start_pfn == pfn)
>> + continue;
>> +
>> + /* If we find valid section, we have nothing to do */
>> + zone_span_writeunlock(zone);
>> + return;
>> + }
>> +
>> + /* The zone has no valid section */
>> + zone->zone_start_pfn = 0;
>> + zone->spanned_pages = 0;
>> + zone_span_writeunlock(zone);
>> +}
>> +
>> +static void shrink_pgdat_span(struct pglist_data *pgdat,
>> + unsigned long start_pfn, unsigned long end_pfn)
>> +{
>> + unsigned long pgdat_start_pfn = pgdat->node_start_pfn;
>> + unsigned long pgdat_end_pfn =
>> + pgdat->node_start_pfn + pgdat->node_spanned_pages;
>> + unsigned long pfn;
>> + struct mem_section *ms;
>> + int nid = pgdat->node_id;
>> +
>> + if (pgdat_start_pfn == start_pfn) {
>> + /*
>> + * If the section is smallest section in the pgdat, it need
>> + * shrink pgdat->node_start_pfn and pgdat->node_spanned_pages.
>> + * In this case, we find second smallest valid mem_section
>> + * for shrinking zone.
>> + */
>> + pfn = find_smallest_section_pfn(nid, NULL, end_pfn,
>> + pgdat_end_pfn);
>> + if (pfn) {
>> + pgdat->node_start_pfn = pfn;
>> + pgdat->node_spanned_pages = pgdat_end_pfn - pfn;
>> + }
>> + } else if (pgdat_end_pfn == end_pfn) {
>> + /*
>> + * If the section is biggest section in the pgdat, it need
>> + * shrink pgdat->node_spanned_pages.
>> + * In this case, we find second biggest valid mem_section for
>> + * shrinking zone.
>> + */
>> + pfn = find_biggest_section_pfn(nid, NULL, pgdat_start_pfn,
>> + start_pfn);
>> + if (pfn)
>> + pgdat->node_spanned_pages = pfn - pgdat_start_pfn + 1;
>> + }
>> +
>> + /*
>> + * If the section is not biggest or smallest mem_section in the pgdat,
>> + * it only creates a hole in the pgdat. So in this case, we need not
>> + * change the pgdat.
>> + * But perhaps, the pgdat has only hole data. Thus it check the pgdat
>> + * has only hole or not.
>> + */
>> + pfn = pgdat_start_pfn;
>> + for (; pfn < pgdat_end_pfn; pfn += PAGES_PER_SECTION) {
>> + ms = __pfn_to_section(pfn);
>> +
>> + if (unlikely(!valid_section(ms)))
>> + continue;
>> +
>> + if (pfn_to_nid(pfn) != nid)
>> + continue;
>> +
>> + /* If the section is current section, it continues the loop */
>> + if (start_pfn == pfn)
>> + continue;
>> +
>> + /* If we find valid section, we have nothing to do */
>> + return;
>> + }
>> +
>> + /* The pgdat has no valid section */
>> + pgdat->node_start_pfn = 0;
>> + pgdat->node_spanned_pages = 0;
>> +}
>> +
>> +static void __remove_zone(struct zone *zone, unsigned long start_pfn)
>> +{
>> + struct pglist_data *pgdat = zone->zone_pgdat;
>> + int nr_pages = PAGES_PER_SECTION;
>> + int zone_type;
>> + unsigned long flags;
>> +
>> + zone_type = zone - pgdat->node_zones;
>> +
>> + pgdat_resize_lock(zone->zone_pgdat, &flags);
>> + shrink_zone_span(zone, start_pfn, start_pfn + nr_pages);
>> + shrink_pgdat_span(pgdat, start_pfn, start_pfn + nr_pages);
>> + pgdat_resize_unlock(zone->zone_pgdat, &flags);
>> +}
>> +
>> static int __remove_section(struct zone *zone, struct mem_section *ms)
>> {
>> unsigned long flags;
>> struct pglist_data *pgdat = zone->zone_pgdat;
>> + unsigned long start_pfn;
>> + int scn_nr;
>> int ret = -EINVAL;
>>
>> if (!valid_section(ms))
>> @@ -325,6 +528,10 @@ static int __remove_section(struct zone *zone, struct mem_section *ms)
>> if (ret)
>> return ret;
>>
>> + scn_nr = __section_nr(ms);
>> + start_pfn = section_nr_to_pfn(scn_nr);
>> + __remove_zone(zone, start_pfn);
>> +
>> pgdat_resize_lock(pgdat, &flags);
>> sparse_remove_one_section(zone, ms);
>> pgdat_resize_unlock(pgdat, &flags);
>>
>
>

2012-11-07 07:42:04

by Tang Chen

[permalink] [raw]

Subject: [PATCH] Update start_pfn in zone and pg_data when spanned_pages == 0.

On 10/23/2012 06:30 PM, [email protected] wrote:
> From: Yasuaki Ishimatsu<[email protected]>
......
> + /* The zone has no valid section */
> + zone->zone_start_pfn = 0;
> + zone->spanned_pages = 0;
> + zone_span_writeunlock(zone);
> +}
> +
> +static void shrink_pgdat_span(struct pglist_data *pgdat,
> + unsigned long start_pfn, unsigned long end_pfn)
> +{
......
> + /* The pgdat has no valid section */
> + pgdat->node_start_pfn = 0;
> + pgdat->node_spanned_pages = 0;
> +}

Hi,

If we hot-remove memory only and leave the cpus alive, the corresponding
node will not be removed. But the node_start_pfn and node_spanned_pages
in pg_data will be reset to 0. In this case, when we hot-add the memory
back next time, the node_start_pfn will always be 0 because no pfn is less
than 0. After that, if we hot-remove the memory again, it will cause kernel
panic in function find_biggest_section_pfn() when it tries to scan all
the pfns.

The zone will also have the same problem.

This patch sets start_pfn to the start_pfn of the section being added when
spanned_pages of the zone or pg_data is 0.

---How to reproduce---

1. hot-add a container with some memory and cpus;
2. hot-remove the container's memory, and leave cpus there;
3. hot-add these memory again;
4. hot-remove them again;

then, the kernel will panic.

---Call trace---

[10530.646285] BUG: unable to handle kernel paging request at
00000fff82a8cc38
[10530.729670] IP: [<ffffffff811c0d55>] find_biggest_section_pfn+0xe5/0x180
......
[10533.064975] Call Trace:
[10533.094162] [<ffffffff811c0fcf>] ? __remove_zone+0x2f/0x1b0
[10533.161757] [<ffffffff811c1124>] __remove_zone+0x184/0x1b0
[10533.228318] [<ffffffff811c11dc>] __remove_section+0x8c/0xb0
[10533.295916] [<ffffffff811c12e7>] __remove_pages+0xe7/0x120
[10533.362476] [<ffffffff81654f7c>] arch_remove_memory+0x2c/0x80
[10533.432151] [<ffffffff81655bb6>] remove_memory+0x56/0x90
[10533.496633] [<ffffffff813da0c8>]
acpi_memory_device_remove_memory+0x48/0x73
[10533.580846] [<ffffffff813da55a>] acpi_memory_device_notify+0x153/0x274
[10533.659865] [<ffffffff813a63cf>] ? acpi_bus_get_device+0x2f/0x77
[10533.732653] [<ffffffff813a6589>] ? acpi_bus_notify+0xb5/0xec
[10533.801291] [<ffffffff813b6786>] acpi_ev_notify_dispatch+0x41/0x5f
[10533.876156] [<ffffffff813a3867>] acpi_os_execute_deferred+0x27/0x34
[10533.952062] [<ffffffff81090589>] process_one_work+0x219/0x680
[10534.021736] [<ffffffff81090528>] ? process_one_work+0x1b8/0x680
[10534.093488] [<ffffffff813a3840>] ?
acpi_os_wait_events_complete+0x23/0x23
[10534.175622] [<ffffffff810923be>] worker_thread+0x12e/0x320
[10534.242181] [<ffffffff81092290>] ? manage_workers+0x110/0x110
[10534.311855] [<ffffffff81098396>] kthread+0xc6/0xd0
[10534.370111] [<ffffffff8167c7c4>] kernel_thread_helper+0x4/0x10
[10534.440824] [<ffffffff81672230>] ? retint_restore_args+0x13/0x13
[10534.513612] [<ffffffff810982d0>] ? __init_kthread_worker+0x70/0x70
[10534.588480] [<ffffffff8167c7c0>] ? gs_change+0x13/0x13
......
[10535.045543] ---[ end trace 96d845dbf33fee11 ]---

Signed-off-by: Tang Chen <[email protected]>
---
mm/memory_hotplug.c | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index 56b758a..4aa313c 100644
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -212,7 +212,7 @@ static void grow_zone_span(struct zone *zone,
unsigned long start_pfn,
zone_span_writelock(zone);

old_zone_end_pfn = zone->zone_start_pfn + zone->spanned_pages;
- if (start_pfn < zone->zone_start_pfn)
+ if (!zone->spanned_pages || start_pfn < zone->zone_start_pfn)
zone->zone_start_pfn = start_pfn;

zone->spanned_pages = max(old_zone_end_pfn, end_pfn) -
@@ -227,7 +227,7 @@ static void grow_pgdat_span(struct pglist_data
*pgdat, unsigned long start_pfn,
unsigned long old_pgdat_end_pfn =
pgdat->node_start_pfn + pgdat->node_spanned_pages;

- if (start_pfn < pgdat->node_start_pfn)
+ if (!pgdat->node_spanned_pages || start_pfn < pgdat->node_start_pfn)
pgdat->node_start_pfn = start_pfn;

pgdat->node_spanned_pages = max(old_pgdat_end_pfn, end_pfn) -
--
1.7.10.1