============== OVERVIEW ===========================
Now that many hugetlb helper functions that deal with hugetlb specific
flags[1] and hugetlb cgroups[2] are converted to folios, higher level
allocation, prep, and freeing functions within hugetlb can also be
converted to operate in folios.
Patch 1 of this series implements the wrapper functions around setting
the compound destructor and compound order for a folio. Besides the user
added in patch 1, patch 2 and patch 9 also use these helper functions.
Patches 2-10 convert the higher level hugetlb functions to folios.
============== TESTING ===========================
LTP:
Ran 10 back to back rounds of the LTP hugetlb test suite.
Gigantic Huge Pages:
Test allocation and freeing via hugeadm commands:
hugeadm --pool-pages-min 1GB:10
hugeadm --pool-pages-min 1GB:0
Demote:
Demote 1 1GB hugepages to 512 2MB hugepages
echo 1 > /sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages
echo 1 > /sys/kernel/mm/hugepages/hugepages-1048576kB/demote
cat /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
# 512
cat /sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages
# 0
Rebased on 10/17/2022 mm-unstable
[1] https://lore.kernel.org/lkml/[email protected]/
[2] https://lore.kernel.org/linux-mm/[email protected]/
v1 -> v2:
- fix conflict with "mm,thp,rmap: simplify compound page mapcount handling"
v2 -> v3:
- v2 contained wrong version of patch 1
v3 -> v4:
- change instances of folio_{clear, set}_head() to __folio_{clear, set}_head()
- rebase on top of hugetlb: Fix __prep_compound_gigantic_page page flag setting
Sidhartha Kumar (10):
mm: add folio dtor and order setter functions
mm/hugetlb: convert destroy_compound_gigantic_page() to folios
mm/hugetlb: convert dissolve_free_huge_page() to folios
mm/hugetlb: convert remove_hugetlb_page() to folios
mm/hugetlb: convert update_and_free_page() to folios
mm/hugetlb: convert add_hugetlb_page() to folios and add
hugetlb_cma_folio()
mm/hugetlb: convert enqueue_huge_page() to folios
mm/hugetlb: convert free_gigantic_page() to folios
mm/hugetlb: convert hugetlb prep functions to folios
mm/hugetlb: change hugetlb allocation functions to return a folio
include/linux/mm.h | 16 ++
mm/hugetlb.c | 385 ++++++++++++++++++++++-----------------------
2 files changed, 207 insertions(+), 194 deletions(-)
--
2.38.1
Convert add_hugetlb_page() to take in a folio, also convert
hugetlb_cma_page() to take in a folio.
Signed-off-by: Sidhartha Kumar <[email protected]>
---
mm/hugetlb.c | 40 ++++++++++++++++++++--------------------
1 file changed, 20 insertions(+), 20 deletions(-)
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 83777d1ccbf3..6acaa61571be 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -54,13 +54,13 @@ struct hstate hstates[HUGE_MAX_HSTATE];
#ifdef CONFIG_CMA
static struct cma *hugetlb_cma[MAX_NUMNODES];
static unsigned long hugetlb_cma_size_in_node[MAX_NUMNODES] __initdata;
-static bool hugetlb_cma_page(struct page *page, unsigned int order)
+static bool hugetlb_cma_folio(struct folio *folio, unsigned int order)
{
- return cma_pages_valid(hugetlb_cma[page_to_nid(page)], page,
+ return cma_pages_valid(hugetlb_cma[folio_nid(folio)], &folio->page,
1 << order);
}
#else
-static bool hugetlb_cma_page(struct page *page, unsigned int order)
+static bool hugetlb_cma_folio(struct folio *folio, unsigned int order)
{
return false;
}
@@ -1506,17 +1506,17 @@ static void remove_hugetlb_folio_for_demote(struct hstate *h, struct folio *foli
__remove_hugetlb_folio(h, folio, adjust_surplus, true);
}
-static void add_hugetlb_page(struct hstate *h, struct page *page,
+static void add_hugetlb_folio(struct hstate *h, struct folio *folio,
bool adjust_surplus)
{
int zeroed;
- int nid = page_to_nid(page);
+ int nid = folio_nid(folio);
- VM_BUG_ON_PAGE(!HPageVmemmapOptimized(page), page);
+ VM_BUG_ON_FOLIO(!folio_test_hugetlb_vmemmap_optimized(folio), folio);
lockdep_assert_held(&hugetlb_lock);
- INIT_LIST_HEAD(&page->lru);
+ INIT_LIST_HEAD(&folio->lru);
h->nr_huge_pages++;
h->nr_huge_pages_node[nid]++;
@@ -1525,21 +1525,21 @@ static void add_hugetlb_page(struct hstate *h, struct page *page,
h->surplus_huge_pages_node[nid]++;
}
- set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);
- set_page_private(page, 0);
+ folio_set_compound_dtor(folio, HUGETLB_PAGE_DTOR);
+ folio_change_private(folio, 0);
/*
* We have to set HPageVmemmapOptimized again as above
- * set_page_private(page, 0) cleared it.
+ * folio_change_private(folio, 0) cleared it.
*/
- SetHPageVmemmapOptimized(page);
+ folio_set_hugetlb_vmemmap_optimized(folio);
/*
- * This page is about to be managed by the hugetlb allocator and
+ * This folio is about to be managed by the hugetlb allocator and
* should have no users. Drop our reference, and check for others
* just in case.
*/
- zeroed = put_page_testzero(page);
- if (!zeroed)
+ zeroed = folio_put_testzero(folio);
+ if (unlikely(!zeroed))
/*
* It is VERY unlikely soneone else has taken a ref on
* the page. In this case, we simply return as the
@@ -1548,8 +1548,8 @@ static void add_hugetlb_page(struct hstate *h, struct page *page,
*/
return;
- arch_clear_hugepage_flags(page);
- enqueue_huge_page(h, page);
+ arch_clear_hugepage_flags(&folio->page);
+ enqueue_huge_page(h, &folio->page);
}
static void __update_and_free_page(struct hstate *h, struct page *page)
@@ -1575,7 +1575,7 @@ static void __update_and_free_page(struct hstate *h, struct page *page)
* page and put the page back on the hugetlb free list and treat
* as a surplus page.
*/
- add_hugetlb_page(h, page, true);
+ add_hugetlb_folio(h, page_folio(page), true);
spin_unlock_irq(&hugetlb_lock);
return;
}
@@ -1600,7 +1600,7 @@ static void __update_and_free_page(struct hstate *h, struct page *page)
* need to be given back to CMA in free_gigantic_page.
*/
if (hstate_is_gigantic(h) ||
- hugetlb_cma_page(page, huge_page_order(h))) {
+ hugetlb_cma_folio(folio, huge_page_order(h))) {
destroy_compound_gigantic_folio(folio, huge_page_order(h));
free_gigantic_page(page, huge_page_order(h));
} else {
@@ -2186,7 +2186,7 @@ int dissolve_free_huge_page(struct page *page)
update_and_free_hugetlb_folio(h, folio, false);
} else {
spin_lock_irq(&hugetlb_lock);
- add_hugetlb_page(h, &folio->page, false);
+ add_hugetlb_folio(h, folio, false);
h->max_huge_pages++;
spin_unlock_irq(&hugetlb_lock);
}
@@ -3453,7 +3453,7 @@ static int demote_free_huge_page(struct hstate *h, struct page *page)
/* Allocation of vmemmmap failed, we can not demote page */
spin_lock_irq(&hugetlb_lock);
set_page_refcounted(page);
- add_hugetlb_page(h, page, false);
+ add_hugetlb_folio(h, page_folio(page), false);
return rc;
}
--
2.38.1
Add folio equivalents for set_compound_order() and set_compound_page_dtor().
Also remove extra new-lines introduced by mm/hugetlb: convert
move_hugetlb_state() to folios and mm/hugetlb_cgroup: convert
hugetlb_cgroup_uncharge_page() to folios.
Suggested-by: Mike Kravetz <[email protected]>
Suggested-by: Muchun Song <[email protected]>
Signed-off-by: Sidhartha Kumar <[email protected]>
---
include/linux/mm.h | 16 ++++++++++++++++
mm/hugetlb.c | 4 +---
2 files changed, 17 insertions(+), 3 deletions(-)
diff --git a/include/linux/mm.h b/include/linux/mm.h
index a48c5ad16a5e..2bdef8a5298a 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -972,6 +972,13 @@ static inline void set_compound_page_dtor(struct page *page,
page[1].compound_dtor = compound_dtor;
}
+static inline void folio_set_compound_dtor(struct folio *folio,
+ enum compound_dtor_id compound_dtor)
+{
+ VM_BUG_ON_FOLIO(compound_dtor >= NR_COMPOUND_DTORS, folio);
+ folio->_folio_dtor = compound_dtor;
+}
+
void destroy_large_folio(struct folio *folio);
static inline int head_compound_pincount(struct page *head)
@@ -987,6 +994,15 @@ static inline void set_compound_order(struct page *page, unsigned int order)
#endif
}
+static inline void folio_set_compound_order(struct folio *folio,
+ unsigned int order)
+{
+ folio->_folio_order = order;
+#ifdef CONFIG_64BIT
+ folio->_folio_nr_pages = order ? 1U << order : 0;
+#endif
+}
+
/* Returns the number of pages in this potentially compound page. */
static inline unsigned long compound_nr(struct page *page)
{
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 4c2fe7fec475..6390de8975c5 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -1780,7 +1780,7 @@ static void __prep_new_hugetlb_folio(struct hstate *h, struct folio *folio)
{
hugetlb_vmemmap_optimize(h, &folio->page);
INIT_LIST_HEAD(&folio->lru);
- folio->_folio_dtor = HUGETLB_PAGE_DTOR;
+ folio_set_compound_dtor(folio, HUGETLB_PAGE_DTOR);
hugetlb_set_folio_subpool(folio, NULL);
set_hugetlb_cgroup(folio, NULL);
set_hugetlb_cgroup_rsvd(folio, NULL);
@@ -2938,7 +2938,6 @@ struct page *alloc_huge_page(struct vm_area_struct *vma,
* a reservation exists for the allocation.
*/
page = dequeue_huge_page_vma(h, vma, addr, avoid_reserve, gbl_chg);
-
if (!page) {
spin_unlock_irq(&hugetlb_lock);
page = alloc_buddy_huge_page_with_mpol(h, vma, addr);
@@ -7351,7 +7350,6 @@ void move_hugetlb_state(struct folio *old_folio, struct folio *new_folio, int re
int old_nid = folio_nid(old_folio);
int new_nid = folio_nid(new_folio);
-
folio_set_hugetlb_temporary(old_folio);
folio_clear_hugetlb_temporary(new_folio);
--
2.38.1
On 18.11.22 23:19, Sidhartha Kumar wrote:
> Convert add_hugetlb_page() to take in a folio, also convert
> hugetlb_cma_page() to take in a folio.
>
> Signed-off-by: Sidhartha Kumar <[email protected]>
[...]
>
> @@ -1525,21 +1525,21 @@ static void add_hugetlb_page(struct hstate *h, struct page *page,
> h->surplus_huge_pages_node[nid]++;
> }
>
> - set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);
> - set_page_private(page, 0);
> + folio_set_compound_dtor(folio, HUGETLB_PAGE_DTOR);
> + folio_change_private(folio, 0);
>
As raised (CC'ed you), this should be s/0/NULL/
--
Thanks,
David / dhildenb
On 11/23/22 1:29 AM, David Hildenbrand wrote:
> On 18.11.22 23:19, Sidhartha Kumar wrote:
>> Convert add_hugetlb_page() to take in a folio, also convert
>> hugetlb_cma_page() to take in a folio.
>>
>> Signed-off-by: Sidhartha Kumar <[email protected]>
>
> [...]
>
>> @@ -1525,21 +1525,21 @@ static void add_hugetlb_page(struct hstate
>> *h, struct page *page,
>> h->surplus_huge_pages_node[nid]++;
>> }
>> - set_compound_page_dtor(page, HUGETLB_PAGE_DTOR);
>> - set_page_private(page, 0);
>> + folio_set_compound_dtor(folio, HUGETLB_PAGE_DTOR);
>> + folio_change_private(folio, 0);
>>
>
> As raised (CC'ed you), this should be s/0/NULL/
Sure, I'll make this change in my next version.
Thanks,
Sidhartha Kumar