Return-Path: Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S965757Ab3HIJ12 (ORCPT ); Fri, 9 Aug 2013 05:27:28 -0400 Received: from LGEMRELSE7Q.lge.com ([156.147.1.151]:42785 "EHLO LGEMRELSE7Q.lge.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S966097Ab3HIJ1U (ORCPT ); Fri, 9 Aug 2013 05:27:20 -0400 X-AuditID: 9c930197-b7b44ae00000347f-c0-5204b5f11e09 From: Joonsoo Kim To: Andrew Morton Cc: Rik van Riel , Mel Gorman , Michal Hocko , "Aneesh Kumar K.V" , KAMEZAWA Hiroyuki , Hugh Dickins , Davidlohr Bueso , David Gibson , linux-mm@kvack.org, linux-kernel@vger.kernel.org, Joonsoo Kim , Wanpeng Li , Naoya Horiguchi , Hillf Danton , Joonsoo Kim Subject: [PATCH v2 19/20] mm, hugetlb: retry if failed to allocate and there is concurrent user Date: Fri, 9 Aug 2013 18:26:37 +0900 Message-Id: <1376040398-11212-20-git-send-email-iamjoonsoo.kim@lge.com> X-Mailer: git-send-email 1.7.9.5 In-Reply-To: <1376040398-11212-1-git-send-email-iamjoonsoo.kim@lge.com> References: <1376040398-11212-1-git-send-email-iamjoonsoo.kim@lge.com> X-Brightmail-Tracker: AAAAAA== Sender: linux-kernel-owner@vger.kernel.org List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Length: 6333 Lines: 212 If parallel fault occur, we can fail to allocate a hugepage, because many threads dequeue a hugepage to handle a fault of same address. This makes reserved pool shortage just for a little while and this cause faulting thread who can get hugepages to get a SIGBUS signal. To solve this problem, we already have a nice solution, that is, a hugetlb_instantiation_mutex. This blocks other threads to dive into a fault handler. This solve the problem clearly, but it introduce performance degradation, because it serialize all fault handling. Now, I try to remove a hugetlb_instantiation_mutex to get rid of performance degradation. For achieving it, at first, we should ensure that no one get a SIGBUS if there are enough hugepages. For this purpose, if we fail to allocate a new hugepage when there is concurrent user, we return just 0, instead of VM_FAULT_SIGBUS. With this, these threads defer to get a SIGBUS signal until there is no concurrent user, and so, we can ensure that no one get a SIGBUS if there are enough hugepages. Signed-off-by: Joonsoo Kim diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index e29e28f..981c539 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h @@ -242,6 +242,7 @@ struct hstate { int next_nid_to_free; unsigned int order; unsigned long mask; + unsigned long nr_dequeue_users; unsigned long max_huge_pages; unsigned long nr_huge_pages; unsigned long free_huge_pages; diff --git a/mm/hugetlb.c b/mm/hugetlb.c index 8743e5c..0501fe5 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -561,6 +561,7 @@ retry_cpuset: if (cpuset_zone_allowed_softwall(zone, htlb_alloc_mask)) { page = dequeue_huge_page_node(h, zone_to_nid(zone)); if (page) { + h->nr_dequeue_users++; if (!use_reserve) break; @@ -577,6 +578,16 @@ retry_cpuset: return page; } +static void commit_dequeued_huge_page(struct hstate *h, bool do_dequeue) +{ + if (!do_dequeue) + return; + + spin_lock(&hugetlb_lock); + h->nr_dequeue_users--; + spin_unlock(&hugetlb_lock); +} + static void update_and_free_page(struct hstate *h, struct page *page) { int i; @@ -1110,7 +1121,9 @@ static void vma_commit_reservation(struct hstate *h, } static struct page *alloc_huge_page(struct vm_area_struct *vma, - unsigned long addr, int use_reserve) + unsigned long addr, int use_reserve, + unsigned long *nr_dequeue_users, + bool *do_dequeue) { struct hugepage_subpool *spool = subpool_vma(vma); struct hstate *h = hstate_vma(vma); @@ -1138,8 +1151,11 @@ static struct page *alloc_huge_page(struct vm_area_struct *vma, return ERR_PTR(-ENOSPC); } spin_lock(&hugetlb_lock); + *do_dequeue = true; page = dequeue_huge_page_vma(h, vma, addr, use_reserve); if (!page) { + *nr_dequeue_users = h->nr_dequeue_users; + *do_dequeue = false; spin_unlock(&hugetlb_lock); page = alloc_buddy_huge_page(h, NUMA_NO_NODE); if (!page) { @@ -1894,6 +1910,7 @@ void __init hugetlb_add_hstate(unsigned order) h->mask = ~((1ULL << (order + PAGE_SHIFT)) - 1); h->nr_huge_pages = 0; h->free_huge_pages = 0; + h->nr_dequeue_users = 0; for (i = 0; i < MAX_NUMNODES; ++i) INIT_LIST_HEAD(&h->hugepage_freelists[i]); INIT_LIST_HEAD(&h->hugepage_activelist); @@ -2500,6 +2517,8 @@ static int hugetlb_cow(struct mm_struct *mm, struct vm_area_struct *vma, int outside_reserve = 0; long chg; bool use_reserve = false; + unsigned long nr_dequeue_users = 0; + bool do_dequeue = false; int ret = 0; unsigned long mmun_start; /* For mmu_notifiers */ unsigned long mmun_end; /* For mmu_notifiers */ @@ -2551,11 +2570,17 @@ retry_avoidcopy: use_reserve = !chg; } - new_page = alloc_huge_page(vma, address, use_reserve); + new_page = alloc_huge_page(vma, address, use_reserve, + &nr_dequeue_users, &do_dequeue); if (IS_ERR(new_page)) { page_cache_release(old_page); + if (nr_dequeue_users) { + ret = 0; + goto out_lock; + } + /* * If a process owning a MAP_PRIVATE mapping fails to COW, * it is due to references held by a child and an insufficient @@ -2580,6 +2605,9 @@ retry_avoidcopy: WARN_ON_ONCE(1); } + if (use_reserve) + WARN_ON_ONCE(1); + ret = VM_FAULT_SIGBUS; goto out_lock; } @@ -2614,6 +2642,7 @@ retry_avoidcopy: page_cache_release(new_page); out_old_page: page_cache_release(old_page); + commit_dequeued_huge_page(h, do_dequeue); out_lock: /* Caller expects lock to be held */ spin_lock(&mm->page_table_lock); @@ -2666,6 +2695,8 @@ static int hugetlb_no_page(struct mm_struct *mm, struct vm_area_struct *vma, pte_t new_pte; long chg; bool use_reserve; + unsigned long nr_dequeue_users = 0; + bool do_dequeue = false; /* * Currently, we are forced to kill the process in the event the @@ -2699,9 +2730,17 @@ retry: } use_reserve = !chg; - page = alloc_huge_page(vma, address, use_reserve); + page = alloc_huge_page(vma, address, use_reserve, + &nr_dequeue_users, &do_dequeue); if (IS_ERR(page)) { - ret = VM_FAULT_SIGBUS; + if (nr_dequeue_users) + ret = 0; + else { + if (use_reserve) + WARN_ON_ONCE(1); + + ret = VM_FAULT_SIGBUS; + } goto out; } clear_huge_page(page, address, pages_per_huge_page(h)); @@ -2714,22 +2753,24 @@ retry: err = add_to_page_cache(page, mapping, idx, GFP_KERNEL); if (err) { put_page(page); + commit_dequeued_huge_page(h, do_dequeue); if (err == -EEXIST) goto retry; goto out; } ClearPagePrivate(page); + commit_dequeued_huge_page(h, do_dequeue); spin_lock(&inode->i_lock); inode->i_blocks += blocks_per_huge_page(h); spin_unlock(&inode->i_lock); } else { lock_page(page); + anon_rmap = 1; if (unlikely(anon_vma_prepare(vma))) { ret = VM_FAULT_OOM; goto backout_unlocked; } - anon_rmap = 1; } } else { /* @@ -2783,6 +2824,8 @@ retry: spin_unlock(&mm->page_table_lock); unlock_page(page); out: + if (anon_rmap) + commit_dequeued_huge_page(h, do_dequeue); return ret; backout: -- 1.7.9.5 -- To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/