Return-Path: Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1752586AbbBWM72 (ORCPT ); Mon, 23 Feb 2015 07:59:28 -0500 Received: from cantor2.suse.de ([195.135.220.15]:52599 "EHLO mx2.suse.de" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1752145AbbBWM7W (ORCPT ); Mon, 23 Feb 2015 07:59:22 -0500 From: Vlastimil Babka To: linux-mm@kvack.org Cc: linux-kernel@vger.kernel.org, Andrew Morton , Hugh Dickins , Andrea Arcangeli , "Kirill A. Shutemov" , Rik van Riel , Mel Gorman , Michal Hocko , Ebru Akagunduz , Alex Thorlton , David Rientjes , Peter Zijlstra , Ingo Molnar , Vlastimil Babka Subject: [RFC 5/6] mm, thp: wakeup khugepaged when THP allocation fails Date: Mon, 23 Feb 2015 13:58:41 +0100 Message-Id: <1424696322-21952-6-git-send-email-vbabka@suse.cz> X-Mailer: git-send-email 2.1.4 In-Reply-To: <1424696322-21952-1-git-send-email-vbabka@suse.cz> References: <1424696322-21952-1-git-send-email-vbabka@suse.cz> Sender: linux-kernel-owner@vger.kernel.org List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Length: 5925 Lines: 214 The previous patch has taken away the THP collapse scanning from khugepaged, leaving it only to maintain the thp_avail_nodes nodemask through heavyweight attempts to make a hugepage available on nodes where it could not be allocated from the process context, both through page fault or the collapse scanning. This patch improves the coordination between failed THP allocations and khugepaged by wakeups, repurposing the khugepaged_wait infrastructure. Instead of periodical sleeping and checking for work, khugepaged will now sleep at least alloc_sleep_millisecs after its last allocation attempt in order to prevent excessive activity, and then respond to a failed THP allocation immediately through khugepaged_wait. Signed-off-by: Vlastimil Babka --- mm/huge_memory.c | 77 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 44 insertions(+), 33 deletions(-) diff --git a/mm/huge_memory.c b/mm/huge_memory.c index 1c92edc..9172c7f 100644 --- a/mm/huge_memory.c +++ b/mm/huge_memory.c @@ -158,9 +158,6 @@ static int start_khugepaged(void) khugepaged_thread = NULL; } - if (!list_empty(&khugepaged_scan.mm_head)) - wake_up_interruptible(&khugepaged_wait); - set_recommended_min_free_kbytes(); } else if (khugepaged_thread) { kthread_stop(khugepaged_thread); @@ -430,7 +427,6 @@ static ssize_t scan_sleep_millisecs_store(struct kobject *kobj, return -EINVAL; khugepaged_scan_sleep_millisecs = msecs; - wake_up_interruptible(&khugepaged_wait); return count; } @@ -781,8 +777,10 @@ fault_alloc_hugepage(struct vm_area_struct *vma, unsigned long haddr) gfp = alloc_hugepage_gfpmask(transparent_hugepage_defrag(vma)); hpage = alloc_hugepage_vma(gfp, vma, haddr, HPAGE_PMD_ORDER); - if (!hpage) + if (!hpage) { node_clear(nid, thp_avail_nodes); + wake_up_interruptible(&khugepaged_wait); + } return hpage; } @@ -2054,8 +2052,6 @@ int __khugepaged_enter(struct mm_struct *mm) spin_unlock(&khugepaged_mm_lock); atomic_inc(&mm->mm_count); - if (wakeup) - wake_up_interruptible(&khugepaged_wait); return 0; } @@ -2252,12 +2248,6 @@ static void __collapse_huge_page_copy(pte_t *pte, struct page *page, } } -static void khugepaged_alloc_sleep(void) -{ - wait_event_freezable_timeout(khugepaged_wait, false, - msecs_to_jiffies(khugepaged_alloc_sleep_millisecs)); -} - static bool khugepaged_scan_abort(int nid, int *node_load) { int i; @@ -2358,6 +2348,7 @@ static struct page count_vm_event(THP_COLLAPSE_ALLOC_FAILED); *hpage = ERR_PTR(-ENOMEM); node_clear(node, thp_avail_nodes); + wake_up_interruptible(&khugepaged_wait); return NULL; } @@ -2365,7 +2356,7 @@ static struct page return *hpage; } -/* Return true, if THP should be allocatable on at least one node */ +/* Return true if we tried to allocate on at least one node */ static bool khugepaged_check_nodes(void) { bool ret = false; @@ -2375,15 +2366,14 @@ static bool khugepaged_check_nodes(void) for_each_online_node(nid) { if (node_isset(nid, thp_avail_nodes)) { - ret = true; continue; } newpage = alloc_hugepage_node(gfp, nid); + ret = true; if (newpage) { node_set(nid, thp_avail_nodes); - ret = true; put_page(newpage); } if (unlikely(kthread_should_stop() || freezing(current))) @@ -2393,6 +2383,19 @@ static bool khugepaged_check_nodes(void) return ret; } +/* Return true if hugepages are available on at least one node */ +static bool check_thp_avail(void) +{ + int nid; + + for_each_online_node(nid) { + if (node_isset(nid, thp_avail_nodes)) + return true; + } + + return false; +} + static bool hugepage_vma_check(struct vm_area_struct *vma) { if ((!(vma->vm_flags & VM_HUGEPAGE) && !khugepaged_always()) || @@ -2656,6 +2659,9 @@ bool khugepaged_scan_mm(struct mm_struct *mm, unsigned long *start, long pages) int ret; int *node_load; + if (!check_thp_avail()) + return false; + //TODO: #ifdef this for NUMA only node_load = kmalloc(sizeof(int) * MAX_NUMNODES, GFP_KERNEL | GFP_NOWAIT); @@ -2706,30 +2712,36 @@ out: return true; } -static int khugepaged_has_work(void) +static bool khugepaged_has_work(void) { - return !list_empty(&khugepaged_scan.mm_head) && - khugepaged_enabled(); + int nid; + + for_each_online_node(nid) { + if (!node_isset(nid, thp_avail_nodes)) + return true; + } + + return false; } -static int khugepaged_wait_event(void) +static bool khugepaged_wait_event(void) { - return !list_empty(&khugepaged_scan.mm_head) || - kthread_should_stop(); + return khugepaged_has_work() || kthread_should_stop(); } -static void khugepaged_wait_work(void) +static void khugepaged_wait_work(bool did_alloc) { + unsigned int msec_sleep; + try_to_freeze(); - if (khugepaged_has_work()) { - if (!khugepaged_scan_sleep_millisecs) - return; + if (did_alloc) { + msec_sleep = READ_ONCE(khugepaged_alloc_sleep_millisecs); - wait_event_freezable_timeout(khugepaged_wait, + if (msec_sleep) + wait_event_freezable_timeout(khugepaged_wait, kthread_should_stop(), - msecs_to_jiffies(khugepaged_scan_sleep_millisecs)); - return; + msecs_to_jiffies(msec_sleep)); } if (khugepaged_enabled()) @@ -2739,15 +2751,14 @@ static void khugepaged_wait_work(void) static int khugepaged(void *none) { struct mm_slot *mm_slot; + bool did_alloc; set_freezable(); set_user_nice(current, MAX_NICE); while (!kthread_should_stop()) { - if (khugepaged_check_nodes()) - khugepaged_wait_work(); - else - khugepaged_alloc_sleep(); + did_alloc = khugepaged_check_nodes(); + khugepaged_wait_work(did_alloc); } spin_lock(&khugepaged_mm_lock); -- 2.1.4 -- To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/