Received-SPF: pass (google.com: domain of linux-kernel+bounces-200147-linux.lists.archive=gmail.com@vger.kernel.org designates 139.178.88.99 as permitted sender) client-ip=139.178.88.99;
From: Byungchul Park <byungchul@sk.com>
To: akpm@linux-foundation.org
Cc: linux-kernel@vger.kernel.org,
	linux-mm@kvack.org,
	kernel_team@skhynix.com,
	hannes@cmpxchg.org,
	iamjoonsoo.kim@lge.com,
	rientjes@google.com,
	ying.huang@intel.com
Subject: [PATCH v2] mm: let kswapd work again for node that used to be hopeless but may not now
Date: Tue,  4 Jun 2024 16:23:23 +0900
Message-Id: <20240604072323.10886-1-byungchul@sk.com>
Precedence: bulk

Changes from v1:
	1. Don't allow to resume kswapd if the system is under memory
	   pressure that might affect direct reclaim by any chance, like
	   if NR_FREE_PAGES is less than (low wmark + min wmark)/2.

--->8---
From 6c73fc16b75907f5da9e6b33aff86bf7d7c9dd64 Mon Sep 17 00:00:00 2001
From: Byungchul Park <byungchul@sk.com>
Date: Tue, 4 Jun 2024 15:27:56 +0900
Subject: [PATCH v2] mm: let kswapd work again for node that used to be hopeless but may not now

A system should run with kswapd running in background when under memory
pressure, such as when the available memory level is below the low water
mark and there are reclaimable folios.

However, the current code let the system run with kswapd stopped if
kswapd has been stopped due to more than MAX_RECLAIM_RETRIES failures
until direct reclaim will do for that, even if there are reclaimable
folios that can be reclaimed by kswapd.  This case was observed in the
following scenario:

   CONFIG_NUMA_BALANCING enabled
   sysctl_numa_balancing_mode set to NUMA_BALANCING_MEMORY_TIERING
   numa node0 (500GB local DRAM, 128 CPUs)
   numa node1 (100GB CXL memory, no CPUs)
   swap off

   1) Run a workload with big anon pages e.g. mmap(200GB).
   2) Continue adding the same workload to the system.
   3) The anon pages are placed in node0 by promotion/demotion.
   4) kswapd0 stops because of the unreclaimable anon pages in node0.
   5) Kill the memory hoggers to restore the system.

After restoring the system at 5), the system starts to run without
kswapd.  Even worse, tiering mechanism is no longer able to work since
the mechanism relies on kswapd for demotion.

However, the node0 has pages newly allocated after 5), that might or
might not be reclaimable.  Since those are potentially reclaimable, it's
worth hopefully trying reclaim by allowing kswapd to work again.

Signed-off-by: Byungchul Park <byungchul@sk.com>
---
 include/linux/mmzone.h |  4 ++++
 mm/page_alloc.c        | 12 ++++++++++
 mm/vmscan.c            | 52 ++++++++++++++++++++++++++++++++++++++----
 3 files changed, 63 insertions(+), 5 deletions(-)

diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index c11b7cde81ef..7c0ba90ea7b4 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -1331,6 +1331,10 @@ typedef struct pglist_data {
 	enum zone_type kswapd_highest_zoneidx;
 
 	int kswapd_failures;		/* Number of 'reclaimed == 0' runs */
+	int nr_may_reclaimable;		/* Number of pages that have been
+					   allocated since considered the
+					   node is hopeless due to too many
+					   kswapd_failures. */
 
 #ifdef CONFIG_COMPACTION
 	int kcompactd_max_order;
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 14d39f34d336..1dd2daede014 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -1538,8 +1538,20 @@ inline void post_alloc_hook(struct page *page, unsigned int order,
 static void prep_new_page(struct page *page, unsigned int order, gfp_t gfp_flags,
 							unsigned int alloc_flags)
 {
+	pg_data_t *pgdat = page_pgdat(page);
+
 	post_alloc_hook(page, order, gfp_flags);
 
+	/*
+	 * New pages might or might not be reclaimable depending on how
+	 * these pages are going to be used.  However, since these are
+	 * potentially reclaimable, it's worth hopefully trying reclaim
+	 * by allowing kswapd to work again even if there have been too
+	 * many ->kswapd_failures, if ->nr_may_reclaimable is big enough.
+	 */
+	if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES)
+		pgdat->nr_may_reclaimable += 1 << order;
+
 	if (order && (gfp_flags & __GFP_COMP))
 		prep_compound_page(page, order);
 
diff --git a/mm/vmscan.c b/mm/vmscan.c
index 3ef654addd44..6cf7ff164c2a 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -4943,6 +4943,7 @@ static void lru_gen_shrink_node(struct pglist_data *pgdat, struct scan_control *
 done:
 	/* kswapd should never fail */
 	pgdat->kswapd_failures = 0;
+	pgdat->nr_may_reclaimable = 0;
 }
 
 /******************************************************************************
@@ -5991,9 +5992,10 @@ static void shrink_node(pg_data_t *pgdat, struct scan_control *sc)
 	 * sleep. On reclaim progress, reset the failure counter. A
 	 * successful direct reclaim run will revive a dormant kswapd.
 	 */
-	if (reclaimable)
+	if (reclaimable) {
 		pgdat->kswapd_failures = 0;
-	else if (sc->cache_trim_mode)
+		pgdat->nr_may_reclaimable = 0;
+	} else if (sc->cache_trim_mode)
 		sc->cache_trim_mode_failed = 1;
 }
 
@@ -6636,6 +6638,42 @@ static void clear_pgdat_congested(pg_data_t *pgdat)
 	clear_bit(PGDAT_WRITEBACK, &pgdat->flags);
 }
 
+static bool may_reclaimable(pg_data_t *pgdat, int order,
+		int highest_zoneidx)
+{
+	int i;
+	bool may_reclaimable;
+
+	may_reclaimable = pgdat->nr_may_reclaimable >= 1 << order;
+	if (!may_reclaimable)
+		return false;
+
+	/*
+	 * Check watermarks bottom-up as lower zones are more likely to
+	 * meet watermarks.
+	 */
+	for (i = 0; i <= highest_zoneidx; i++) {
+		unsigned long mark;
+		struct zone *zone;
+
+		zone = pgdat->node_zones + i;
+		if (!managed_zone(zone))
+			continue;
+
+		/*
+		 * Don't bother the system by resuming kswapd if the
+		 * system is under memory pressure that might affect
+		 * direct reclaim by any chance.  Conservatively allow it
+		 * unless NR_FREE_PAGES is less than (low + min)/2.
+		 */
+		mark = (low_wmark_pages(zone) + min_wmark_pages(zone)) >> 1;
+		if (zone_watermark_ok_safe(zone, order, mark, highest_zoneidx))
+			return true;
+	}
+
+	return false;
+}
+
 /*
  * Prepare kswapd for sleeping. This verifies that there are no processes
  * waiting in throttle_direct_reclaim() and that watermarks have been met.
@@ -6662,7 +6700,8 @@ static bool prepare_kswapd_sleep(pg_data_t *pgdat, int order,
 		wake_up_all(&pgdat->pfmemalloc_wait);
 
 	/* Hopeless node, leave it to direct reclaim */
-	if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES)
+	if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES &&
+	    !may_reclaimable(pgdat, order, highest_zoneidx))
 		return true;
 
 	if (pgdat_balanced(pgdat, order, highest_zoneidx)) {
@@ -6940,8 +6979,10 @@ static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx)
 		goto restart;
 	}
 
-	if (!sc.nr_reclaimed)
+	if (!sc.nr_reclaimed) {
 		pgdat->kswapd_failures++;
+		pgdat->nr_may_reclaimable = 0;
+	}
 
 out:
 	clear_reclaim_active(pgdat, highest_zoneidx);
@@ -7204,7 +7245,8 @@ void wakeup_kswapd(struct zone *zone, gfp_t gfp_flags, int order,
 		return;
 
 	/* Hopeless node, leave it to direct reclaim if possible */
-	if (pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES ||
+	if ((pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES &&
+	     !may_reclaimable(pgdat, order, highest_zoneidx)) ||
 	    (pgdat_balanced(pgdat, order, highest_zoneidx) &&
 	     !pgdat_watermark_boosted(pgdat, highest_zoneidx))) {
 		/*
-- 
2.17.1