Return-Path: Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1754232AbdC1Fc4 (ORCPT ); Tue, 28 Mar 2017 01:32:56 -0400 Received: from mga06.intel.com ([134.134.136.31]:24585 "EHLO mga06.intel.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1752096AbdC1Fco (ORCPT ); Tue, 28 Mar 2017 01:32:44 -0400 X-ExtLoop1: 1 X-IronPort-AV: E=Sophos;i="5.36,235,1486454400"; d="scan'208";a="1147757240" From: "Huang, Ying" To: Andrew Morton Cc: linux-mm@kvack.org, linux-kernel@vger.kernel.org, Huang Ying , Hugh Dickins , Shaohua Li , Minchan Kim , Rik van Riel , Andrea Arcangeli , "Kirill A . Shutemov" Subject: [PATCH -mm -v7 6/9] mm, THP, swap: Support to add/delete THP to/from swap cache Date: Tue, 28 Mar 2017 13:32:06 +0800 Message-Id: <20170328053209.25876-7-ying.huang@intel.com> X-Mailer: git-send-email 2.11.0 In-Reply-To: <20170328053209.25876-1-ying.huang@intel.com> References: <20170328053209.25876-1-ying.huang@intel.com> Sender: linux-kernel-owner@vger.kernel.org List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Length: 5505 Lines: 173 From: Huang Ying With this patch, a THP (Transparent Huge Page) can be added/deleted to/from the swap cache as a set of (HPAGE_PMD_NR) sub-pages. This will be used for the THP (Transparent Huge Page) swap support. Where one THP may be added/delted to/from the swap cache. This will batch the swap cache operations to reduce the lock acquire/release times for the THP swap too. Cc: Hugh Dickins Cc: Shaohua Li Cc: Minchan Kim Cc: Rik van Riel Cc: Andrea Arcangeli Cc: Kirill A. Shutemov Signed-off-by: "Huang, Ying" --- include/linux/page-flags.h | 5 ++-- mm/swap_state.c | 64 ++++++++++++++++++++++++++++++---------------- 2 files changed, 45 insertions(+), 24 deletions(-) diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index 6b5818d6de32..f4acd6c4f808 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -326,11 +326,12 @@ PAGEFLAG_FALSE(HighMem) #ifdef CONFIG_SWAP static __always_inline int PageSwapCache(struct page *page) { + page = compound_head(page); return PageSwapBacked(page) && test_bit(PG_swapcache, &page->flags); } -SETPAGEFLAG(SwapCache, swapcache, PF_NO_COMPOUND) -CLEARPAGEFLAG(SwapCache, swapcache, PF_NO_COMPOUND) +SETPAGEFLAG(SwapCache, swapcache, PF_NO_TAIL) +CLEARPAGEFLAG(SwapCache, swapcache, PF_NO_TAIL) #else PAGEFLAG_FALSE(SwapCache) #endif diff --git a/mm/swap_state.c b/mm/swap_state.c index 199a07efc44d..504f67d73f67 100644 --- a/mm/swap_state.c +++ b/mm/swap_state.c @@ -38,6 +38,7 @@ struct address_space *swapper_spaces[MAX_SWAPFILES]; static unsigned int nr_swapper_spaces[MAX_SWAPFILES]; #define INC_CACHE_INFO(x) do { swap_cache_info.x++; } while (0) +#define ADD_CACHE_INFO(x, nr) do { swap_cache_info.x += (nr); } while (0) static struct { unsigned long add_total; @@ -90,39 +91,52 @@ void show_swap_cache_info(void) */ int __add_to_swap_cache(struct page *page, swp_entry_t entry) { - int error; + int error, i, nr = hpage_nr_pages(page); struct address_space *address_space; + struct page *cur_page; + swp_entry_t cur_entry; VM_BUG_ON_PAGE(!PageLocked(page), page); VM_BUG_ON_PAGE(PageSwapCache(page), page); VM_BUG_ON_PAGE(!PageSwapBacked(page), page); - get_page(page); + page_ref_add(page, nr); SetPageSwapCache(page); - set_page_private(page, entry.val); address_space = swap_address_space(entry); + cur_page = page; + cur_entry.val = entry.val; spin_lock_irq(&address_space->tree_lock); - error = radix_tree_insert(&address_space->page_tree, - swp_offset(entry), page); - if (likely(!error)) { - address_space->nrpages++; - __inc_node_page_state(page, NR_FILE_PAGES); - INC_CACHE_INFO(add_total); + for (i = 0; i < nr; i++, cur_page++, cur_entry.val++) { + set_page_private(cur_page, cur_entry.val); + error = radix_tree_insert(&address_space->page_tree, + swp_offset(cur_entry), cur_page); + if (unlikely(error)) + break; } - spin_unlock_irq(&address_space->tree_lock); - - if (unlikely(error)) { + if (likely(!error)) { + address_space->nrpages += nr; + __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr); + ADD_CACHE_INFO(add_total, nr); + } else { /* * Only the context which have set SWAP_HAS_CACHE flag * would call add_to_swap_cache(). * So add_to_swap_cache() doesn't returns -EEXIST. */ VM_BUG_ON(error == -EEXIST); - set_page_private(page, 0UL); + set_page_private(cur_page, 0UL); + while (i--) { + cur_page--; + cur_entry.val--; + radix_tree_delete(&address_space->page_tree, + swp_offset(cur_entry)); + set_page_private(cur_page, 0UL); + } ClearPageSwapCache(page); - put_page(page); + page_ref_sub(page, nr); } + spin_unlock_irq(&address_space->tree_lock); return error; } @@ -132,7 +146,7 @@ int add_to_swap_cache(struct page *page, swp_entry_t entry, gfp_t gfp_mask) { int error; - error = radix_tree_maybe_preload(gfp_mask); + error = radix_tree_maybe_preload_order(gfp_mask, compound_order(page)); if (!error) { error = __add_to_swap_cache(page, entry); radix_tree_preload_end(); @@ -148,6 +162,7 @@ void __delete_from_swap_cache(struct page *page) { swp_entry_t entry; struct address_space *address_space; + int i, nr = hpage_nr_pages(page); VM_BUG_ON_PAGE(!PageLocked(page), page); VM_BUG_ON_PAGE(!PageSwapCache(page), page); @@ -155,12 +170,17 @@ void __delete_from_swap_cache(struct page *page) entry.val = page_private(page); address_space = swap_address_space(entry); - radix_tree_delete(&address_space->page_tree, swp_offset(entry)); - set_page_private(page, 0); + for (i = 0; i < nr; i++, entry.val++) { + struct page *cur_page = page + i; + + radix_tree_delete(&address_space->page_tree, + swp_offset(entry)); + set_page_private(cur_page, 0); + } ClearPageSwapCache(page); - address_space->nrpages--; - __dec_node_page_state(page, NR_FILE_PAGES); - INC_CACHE_INFO(del_total); + address_space->nrpages -= nr; + __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, -nr); + ADD_CACHE_INFO(del_total, nr); } /** @@ -237,8 +257,8 @@ void delete_from_swap_cache(struct page *page) __delete_from_swap_cache(page); spin_unlock_irq(&address_space->tree_lock); - swapcache_free(entry); - put_page(page); + __swapcache_free(entry, PageTransHuge(page)); + page_ref_sub(page, hpage_nr_pages(page)); } /* -- 2.11.0