This patch set introduces folio_movable_ops() and converts 3 functions
in mm/migrate.c to use folios. It also introduces
folio_get_nontail_page() for folio conversions which may want to
distinguish between head and tail pages.
---
v2:
- Add folio_get_nontail_page() to fix behavior in isolate_movable_page()
- Rebase on latest mm-unstable
Vishal Moola (Oracle) (4):
mm: Add folio_get_nontail_page()
mm/migrate: Add folio_movable_ops()
mm/migrate: Convert isolate_movable_page() to use folios
mm/migrate: Convert putback_movable_pages() to use folios
include/linux/migrate.h | 9 +++++
include/linux/mm.h | 7 ++++
mm/migrate.c | 87 +++++++++++++++++++++--------------------
3 files changed, 60 insertions(+), 43 deletions(-)
--
2.38.1
folio_get_nontail_page() returns the folio associated with a head
page. This is necessary for folio conversions where the behavior of that
function differs between head pages and tail pages.
Signed-off-by: Vishal Moola (Oracle) <[email protected]>
---
include/linux/mm.h | 7 +++++++
1 file changed, 7 insertions(+)
diff --git a/include/linux/mm.h b/include/linux/mm.h
index dd295c020e85..5da260346406 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -879,6 +879,13 @@ static inline bool get_page_unless_zero(struct page *page)
return page_ref_add_unless(page, 1, 0);
}
+static inline struct folio *folio_get_nontail_page(struct page *page)
+{
+ if (unlikely(!get_page_unless_zero(page)))
+ return NULL;
+ return (struct folio *)page;
+}
+
extern int page_is_ram(unsigned long pfn);
enum {
--
2.38.1
Removes 6 calls to compound_head() and prepares the function to take in a
folio instead of page argument.
Signed-off-by: Vishal Moola (Oracle) <[email protected]>
---
mm/migrate.c | 39 ++++++++++++++++++++-------------------
1 file changed, 20 insertions(+), 19 deletions(-)
diff --git a/mm/migrate.c b/mm/migrate.c
index fcb2e9fa1953..c20d9f531db4 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -60,6 +60,7 @@
int isolate_movable_page(struct page *page, isolate_mode_t mode)
{
+ struct folio *folio = folio_get_nontail_page(page);
const struct movable_operations *mops;
/*
@@ -71,11 +72,11 @@ int isolate_movable_page(struct page *page, isolate_mode_t mode)
* the put_page() at the end of this block will take care of
* release this page, thus avoiding a nasty leakage.
*/
- if (unlikely(!get_page_unless_zero(page)))
+ if (!folio)
goto out;
- if (unlikely(PageSlab(page)))
- goto out_putpage;
+ if (unlikely(folio_test_slab(folio)))
+ goto out_putfolio;
/* Pairs with smp_wmb() in slab freeing, e.g. SLUB's __free_slab() */
smp_rmb();
/*
@@ -83,12 +84,12 @@ int isolate_movable_page(struct page *page, isolate_mode_t mode)
* we use non-atomic bitops on newly allocated page flags so
* unconditionally grabbing the lock ruins page's owner side.
*/
- if (unlikely(!__PageMovable(page)))
- goto out_putpage;
+ if (unlikely(!__folio_test_movable(folio)))
+ goto out_putfolio;
/* Pairs with smp_wmb() in slab allocation, e.g. SLUB's alloc_slab_page() */
smp_rmb();
- if (unlikely(PageSlab(page)))
- goto out_putpage;
+ if (unlikely(folio_test_slab(folio)))
+ goto out_putfolio;
/*
* As movable pages are not isolated from LRU lists, concurrent
@@ -101,29 +102,29 @@ int isolate_movable_page(struct page *page, isolate_mode_t mode)
* lets be sure we have the page lock
* before proceeding with the movable page isolation steps.
*/
- if (unlikely(!trylock_page(page)))
- goto out_putpage;
+ if (unlikely(!folio_trylock(folio)))
+ goto out_putfolio;
- if (!PageMovable(page) || PageIsolated(page))
+ if (!folio_test_movable(folio) || folio_test_isolated(folio))
goto out_no_isolated;
- mops = page_movable_ops(page);
- VM_BUG_ON_PAGE(!mops, page);
+ mops = folio_movable_ops(folio);
+ VM_BUG_ON_FOLIO(!mops, folio);
- if (!mops->isolate_page(page, mode))
+ if (!mops->isolate_page(&folio->page, mode))
goto out_no_isolated;
/* Driver shouldn't use PG_isolated bit of page->flags */
- WARN_ON_ONCE(PageIsolated(page));
- SetPageIsolated(page);
- unlock_page(page);
+ WARN_ON_ONCE(folio_test_isolated(folio));
+ folio_set_isolated(folio);
+ folio_unlock(folio);
return 0;
out_no_isolated:
- unlock_page(page);
-out_putpage:
- put_page(page);
+ folio_unlock(folio);
+out_putfolio:
+ folio_put(folio);
out:
return -EBUSY;
}
--
2.38.1
folio_movable_ops() does the same as page_movable_ops() except uses
folios instead of pages. This function will help make folio conversions
in migrate.c more readable.
Signed-off-by: Vishal Moola (Oracle) <[email protected]>
---
include/linux/migrate.h | 9 +++++++++
mm/migrate.c | 2 +-
2 files changed, 10 insertions(+), 1 deletion(-)
diff --git a/include/linux/migrate.h b/include/linux/migrate.h
index 7376074f2e1e..c88b96b48be7 100644
--- a/include/linux/migrate.h
+++ b/include/linux/migrate.h
@@ -123,6 +123,15 @@ static inline bool folio_test_movable(struct folio *folio)
return PageMovable(&folio->page);
}
+static inline
+const struct movable_operations *folio_movable_ops(struct folio *folio)
+{
+ VM_BUG_ON(!__folio_test_movable(folio));
+
+ return (const struct movable_operations *)
+ ((unsigned long)folio->mapping - PAGE_MAPPING_MOVABLE);
+}
+
static inline
const struct movable_operations *page_movable_ops(struct page *page)
{
diff --git a/mm/migrate.c b/mm/migrate.c
index b971edbf32fc..fcb2e9fa1953 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -990,7 +990,7 @@ static int move_to_new_folio(struct folio *dst, struct folio *src,
goto out;
}
- mops = page_movable_ops(&src->page);
+ mops = folio_movable_ops(src);
rc = mops->migrate_page(&dst->page, &src->page, mode);
WARN_ON_ONCE(rc == MIGRATEPAGE_SUCCESS &&
!folio_test_isolated(src));
--
2.38.1
Removes 6 calls to compound_head(), and replaces putback_movable_page()
with putback_movable_folio() as well.
Signed-off-by: Vishal Moola (Oracle) <[email protected]>
---
mm/migrate.c | 46 +++++++++++++++++++++++-----------------------
1 file changed, 23 insertions(+), 23 deletions(-)
diff --git a/mm/migrate.c b/mm/migrate.c
index c20d9f531db4..db480dc08bac 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -129,12 +129,12 @@ int isolate_movable_page(struct page *page, isolate_mode_t mode)
return -EBUSY;
}
-static void putback_movable_page(struct page *page)
+static void putback_movable_folio(struct folio *folio)
{
- const struct movable_operations *mops = page_movable_ops(page);
+ const struct movable_operations *mops = folio_movable_ops(folio);
- mops->putback_page(page);
- ClearPageIsolated(page);
+ mops->putback_page(&folio->page);
+ folio_clear_isolated(folio);
}
/*
@@ -147,33 +147,33 @@ static void putback_movable_page(struct page *page)
*/
void putback_movable_pages(struct list_head *l)
{
- struct page *page;
- struct page *page2;
+ struct folio *folio;
+ struct folio *folio2;
- list_for_each_entry_safe(page, page2, l, lru) {
- if (unlikely(PageHuge(page))) {
- folio_putback_active_hugetlb(page_folio(page));
+ list_for_each_entry_safe(folio, folio2, l, lru) {
+ if (unlikely(folio_test_hugetlb(folio))) {
+ folio_putback_active_hugetlb(folio);
continue;
}
- list_del(&page->lru);
+ list_del(&folio->lru);
/*
- * We isolated non-lru movable page so here we can use
- * __PageMovable because LRU page's mapping cannot have
+ * We isolated non-lru movable folio so here we can use
+ * __PageMovable because LRU folio's mapping cannot have
* PAGE_MAPPING_MOVABLE.
*/
- if (unlikely(__PageMovable(page))) {
- VM_BUG_ON_PAGE(!PageIsolated(page), page);
- lock_page(page);
- if (PageMovable(page))
- putback_movable_page(page);
+ if (unlikely(__folio_test_movable(folio))) {
+ VM_BUG_ON_FOLIO(!folio_test_isolated(folio), folio);
+ folio_lock(folio);
+ if (folio_test_movable(folio))
+ putback_movable_folio(folio);
else
- ClearPageIsolated(page);
- unlock_page(page);
- put_page(page);
+ folio_clear_isolated(folio);
+ folio_unlock(folio);
+ folio_put(folio);
} else {
- mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON +
- page_is_file_lru(page), -thp_nr_pages(page));
- putback_lru_page(page);
+ node_stat_mod_folio(folio, NR_ISOLATED_ANON +
+ folio_is_file_lru(folio), -folio_nr_pages(folio));
+ folio_putback_lru(folio);
}
}
}
--
2.38.1