Received-SPF: pass (google.com: domain of linux-kernel-owner@vger.kernel.org designates 23.128.96.37 as permitted sender) client-ip=23.128.96.37;
Feedback-ID: iccd040f4:Fastmail
From:   Zi Yan <zi.yan@sent.com>
To:     Huang Ying <ying.huang@intel.com>, linux-mm@kvack.org,
        linux-kernel@vger.kernel.org
Cc:     Zi Yan <ziy@nvidia.com>, Andrew Morton <akpm@linux-foundation.org>,
        "Matthew Wilcox (Oracle)" <willy@infradead.org>,
        David Hildenbrand <david@redhat.com>,
        Baolin Wang <baolin.wang@linux.alibaba.com>
Subject: [PATCH v2] mm/migrate: correct nr_failed in migrate_pages_sync()
Date:   Mon, 16 Oct 2023 11:41:56 -0400
Message-ID: <20231016154156.1948815-1-zi.yan@sent.com>
Reply-To: Zi Yan <ziy@nvidia.com>
MIME-Version: 1.0
Content-Transfer-Encoding: quoted-printable
Precedence: bulk

From: Zi Yan <ziy@nvidia.com>

nr_failed was missing the large folio splits from migrate_pages_batch()
and can cause a mismatch between migrate_pages() return value and the
number of not migrated pages, i.e., when the return value of
migrate_pages() is 0, there are still pages left in the from page list.
It will happen when a non-PMD THP large folio fails to migrate due to
-ENOMEM and is split successfully but not all the split pages are not
migrated, migrate_pages_batch() would return non-zero, but
astats.nr_thp_split =3D 0. nr_failed would be 0 and returned to the caller
of migrate_pages(), but the not migrated pages are left in the from page
list without being added back to LRU lists.

Fix it by adding a new nr_split counter for large folio splits and adding
it to nr_failed in migrate_page_sync() after migrate_pages_batch() is
done.

Fixes: 2ef7dbb26990 ("migrate_pages: try migrate in batch asynchronously fi=
rstly")
Signed-off-by: Zi Yan <ziy@nvidia.com>
---
 include/trace/events/migrate.h | 24 ++++++++++++++----------
 mm/migrate.c                   | 15 +++++++++++----
 2 files changed, 25 insertions(+), 14 deletions(-)

diff --git a/include/trace/events/migrate.h b/include/trace/events/migrate.h
index 061b5128f335..0190ef725b43 100644
--- a/include/trace/events/migrate.h
+++ b/include/trace/events/migrate.h
@@ -49,10 +49,11 @@ TRACE_EVENT(mm_migrate_pages,
=20
 	TP_PROTO(unsigned long succeeded, unsigned long failed,
 		 unsigned long thp_succeeded, unsigned long thp_failed,
-		 unsigned long thp_split, enum migrate_mode mode, int reason),
+		 unsigned long thp_split, unsigned long large_folio_split,
+		 enum migrate_mode mode, int reason),
=20
 	TP_ARGS(succeeded, failed, thp_succeeded, thp_failed,
-		thp_split, mode, reason),
+		thp_split, large_folio_split, mode, reason),
=20
 	TP_STRUCT__entry(
 		__field(	unsigned long,		succeeded)
@@ -60,26 +61,29 @@ TRACE_EVENT(mm_migrate_pages,
 		__field(	unsigned long,		thp_succeeded)
 		__field(	unsigned long,		thp_failed)
 		__field(	unsigned long,		thp_split)
+		__field(	unsigned long,		large_folio_split)
 		__field(	enum migrate_mode,	mode)
 		__field(	int,			reason)
 	),
=20
 	TP_fast_assign(
-		__entry->succeeded	=3D succeeded;
-		__entry->failed		=3D failed;
-		__entry->thp_succeeded	=3D thp_succeeded;
-		__entry->thp_failed	=3D thp_failed;
-		__entry->thp_split	=3D thp_split;
-		__entry->mode		=3D mode;
-		__entry->reason		=3D reason;
+		__entry->succeeded			=3D succeeded;
+		__entry->failed				=3D failed;
+		__entry->thp_succeeded		=3D thp_succeeded;
+		__entry->thp_failed			=3D thp_failed;
+		__entry->thp_split			=3D thp_split;
+		__entry->large_folio_split	=3D large_folio_split;
+		__entry->mode				=3D mode;
+		__entry->reason				=3D reason;
 	),
=20
-	TP_printk("nr_succeeded=3D%lu nr_failed=3D%lu nr_thp_succeeded=3D%lu nr_t=
hp_failed=3D%lu nr_thp_split=3D%lu mode=3D%s reason=3D%s",
+	TP_printk("nr_succeeded=3D%lu nr_failed=3D%lu nr_thp_succeeded=3D%lu nr_t=
hp_failed=3D%lu nr_thp_split=3D%lu nr_split=3D%lu mode=3D%s reason=3D%s",
 		__entry->succeeded,
 		__entry->failed,
 		__entry->thp_succeeded,
 		__entry->thp_failed,
 		__entry->thp_split,
+		__entry->large_folio_split,
 		__print_symbolic(__entry->mode, MIGRATE_MODE),
 		__print_symbolic(__entry->reason, MIGRATE_REASON))
 );
diff --git a/mm/migrate.c b/mm/migrate.c
index c602bf6dec97..4caf405b6504 100644
--- a/mm/migrate.c
+++ b/mm/migrate.c
@@ -1495,6 +1495,7 @@ struct migrate_pages_stats {
 	int nr_thp_succeeded;	/* THP migrated successfully */
 	int nr_thp_failed;	/* THP failed to be migrated */
 	int nr_thp_split;	/* THP split before migrating */
+	int nr_split;	/* Large folio (include THP) split before migrating */
 };
=20
 /*
@@ -1614,6 +1615,7 @@ static int migrate_pages_batch(struct list_head *from,
 	int nr_retry_pages =3D 0;
 	int pass =3D 0;
 	bool is_thp =3D false;
+	bool is_large =3D false;
 	struct folio *folio, *folio2, *dst =3D NULL, *dst2;
 	int rc, rc_saved =3D 0, nr_pages;
 	LIST_HEAD(unmap_folios);
@@ -1629,7 +1631,8 @@ static int migrate_pages_batch(struct list_head *from,
 		nr_retry_pages =3D 0;
=20
 		list_for_each_entry_safe(folio, folio2, from, lru) {
-			is_thp =3D folio_test_large(folio) && folio_test_pmd_mappable(folio);
+			is_large =3D folio_test_large(folio);
+			is_thp =3D is_large && folio_test_pmd_mappable(folio);
 			nr_pages =3D folio_nr_pages(folio);
=20
 			cond_resched();
@@ -1649,6 +1652,7 @@ static int migrate_pages_batch(struct list_head *from,
 				stats->nr_thp_failed++;
 				if (!try_split_folio(folio, split_folios)) {
 					stats->nr_thp_split++;
+					stats->nr_split++;
 					continue;
 				}
 				stats->nr_failed_pages +=3D nr_pages;
@@ -1677,11 +1681,12 @@ static int migrate_pages_batch(struct list_head *fr=
om,
 				nr_failed++;
 				stats->nr_thp_failed +=3D is_thp;
 				/* Large folio NUMA faulting doesn't split to retry. */
-				if (folio_test_large(folio) && !nosplit) {
+				if (is_large && !nosplit) {
 					int ret =3D try_split_folio(folio, split_folios);
=20
 					if (!ret) {
 						stats->nr_thp_split +=3D is_thp;
+						stats->nr_split++;
 						break;
 					} else if (reason =3D=3D MR_LONGTERM_PIN &&
 						   ret =3D=3D -EAGAIN) {
@@ -1827,6 +1832,7 @@ static int migrate_pages_sync(struct list_head *from,=
 new_folio_t get_new_folio,
 	stats->nr_succeeded +=3D astats.nr_succeeded;
 	stats->nr_thp_succeeded +=3D astats.nr_thp_succeeded;
 	stats->nr_thp_split +=3D astats.nr_thp_split;
+	stats->nr_split +=3D astats.nr_split;
 	if (rc < 0) {
 		stats->nr_failed_pages +=3D astats.nr_failed_pages;
 		stats->nr_thp_failed +=3D astats.nr_thp_failed;
@@ -1834,7 +1840,7 @@ static int migrate_pages_sync(struct list_head *from,=
 new_folio_t get_new_folio,
 		return rc;
 	}
 	stats->nr_thp_failed +=3D astats.nr_thp_split;
-	nr_failed +=3D astats.nr_thp_split;
+	nr_failed +=3D astats.nr_split + astats.nr_thp_split;
 	/*
 	 * Fall back to migrate all failed folios one by one synchronously. All
 	 * failed folios except split THPs will be retried, so their failure
@@ -1969,7 +1975,8 @@ int migrate_pages(struct list_head *from, new_folio_t=
 get_new_folio,
 	count_vm_events(THP_MIGRATION_SPLIT, stats.nr_thp_split);
 	trace_mm_migrate_pages(stats.nr_succeeded, stats.nr_failed_pages,
 			       stats.nr_thp_succeeded, stats.nr_thp_failed,
-			       stats.nr_thp_split, mode, reason);
+			       stats.nr_thp_split, stats.nr_split, mode,
+			       reason);
=20
 	if (ret_succeeded)
 		*ret_succeeded =3D stats.nr_succeeded;
--=20
2.42.0