Received-SPF: pass (google.com: domain of linux-kernel+bounces-143504-linux.lists.archive=gmail.com@vger.kernel.org designates 147.75.80.249 as permitted sender) client-ip=147.75.80.249;
From: lipeifeng@oppo.com
To: lipeifeng@oppo.com,
	akpm@linux-foundation.org,
	david@fromorbit.com,
	zhengqi.arch@bytedance.com,
	roman.gushchin@linux.dev,
	muchun.song@linux.dev,
	21cnbao@gmail.com
Cc: linux-mm@kvack.org,
	linux-kernel@vger.kernel.org
Subject: [PATCH v2] mm/shrinker: add SHRINKER_NO_DIRECT_RECLAIM
Date: Sat, 13 Apr 2024 09:54:10 +0800
Message-Id: <20240413015410.30951-1-lipeifeng@oppo.com>
Content-Transfer-Encoding: 8bit
Content-Type: text/plain
Precedence: bulk
MIME-Version: 1.0
X-MS-Exchange-AntiSpam-MessageData-ChunkCount: 1
X-MS-Exchange-AntiSpam-MessageData-0:
	=?us-ascii?Q?uSToP8hThWN0HWkrxizNqRUKtvXbnnrOAbAaulDoYbfLGHqTcQcJoIayvzGZ?=
 =?us-ascii?Q?QGOqKy5brSYgfYb8c/JChGan/b8KJsQk5Ln5hL7O2DE8/CUlDepxOdDcaeiL?=
 =?us-ascii?Q?Xks8ZIi0c3j3z0EH+PVgidd0B4mzvziJOwNib7eW8og0IGEy6yE2zI5PDtDS?=
 =?us-ascii?Q?/Yh0R1s/wvEmr0X8vLbYMHidQ+SUl438aNC4yJ/mxcxK3FA2Hl0tcpT347qn?=
 =?us-ascii?Q?sHTrh9ChG0yiPcSI4bxoBFau+aF/y/6p9uMk/fqNCGXssYMJh0SEdDwrKI+E?=
 =?us-ascii?Q?j/geyRp0Tg36GH3Mb4KOYKZ9zKg/pdWXWCYTfLI2WNjeDv3815Y6Q0ZGq9eB?=
 =?us-ascii?Q?ILGEFFsDYzsYx+Bp6b67E6EvFTkiQdfRZOVmX5Hd2GYKUUyjldkedk9MKLEo?=
 =?us-ascii?Q?cjPKFRIns+ziU95hI2AAgUqCOvArflsARxZsLLcF1OW4661MqWgxRWoEI7Y3?=
 =?us-ascii?Q?Ix5CpuhD/n8j7DjhojntabA7i2R3lJRwOI2rW2GfK5n44JIMSQe+Ne/r/Wiy?=
 =?us-ascii?Q?bwE3oi+dXXqAmq9Wfh3A3hUdwd8nk+7T1QjqcxkPUP5d+922yEIgHfdyoMNb?=
 =?us-ascii?Q?/kodXqECvavnBMtC3t75NcduXEIiid5uVVD1QH8XuzcFy6ZM0E8PY3nWrIqH?=
 =?us-ascii?Q?sDTrW176z9rnlQc9woMcunBXFZMQ57Dxkhf34LAjW2q1D3ucAESSVrxIqTOQ?=
 =?us-ascii?Q?7ibWeNoHAclCk46L1k26nOuaB0t9uhpHYzzEhW/toOcfLXVPyxdASw6PvDN7?=
 =?us-ascii?Q?5QLcwz02zKmiLcPSzr7tcFY2VrJnzeJzenuOSQmtEaFgP2orLQrsM3dybnF8?=
 =?us-ascii?Q?Q7koSocf4W4MaM8H81T0oFn+WrvLJQagIIFmPIKJo2+RcgR2gaEMbG1v7blC?=
 =?us-ascii?Q?nyn7apv5LR0tqJR7MaR8aGEkDFX6b+15lvXFAKqPjsWHuKaUB31vXkc/W/2H?=
 =?us-ascii?Q?/zQCkvsZWprPymiB1PHgbgRiaZHkiQ9Q9yvyUjWuJMTbd3Qo/l9KKDF2Kzx0?=
 =?us-ascii?Q?Y+kcScNcpcT3MWvsowuETO/+D/MVw9CCNf+y3SzynNypbaSNyUKrIJ0Uj2h9?=
 =?us-ascii?Q?q9YTDqo/6qLBiAIU/ZxtwkAwIboM8o6vlhXE+0YpWLS2HQ2U2nyQDDPsN/Cs?=
 =?us-ascii?Q?xbZUGlx4RV6CiG60Ylp+YLi0MPomx8HELEtRzEgg2KOCunUv6iCSV3kfDGdV?=
 =?us-ascii?Q?rX/9eQU6jn1ABGQTrdUrBcleJwIbrjuOvY6nkHNOS4UiMRBX5Shu5PTJbSxZ?=
 =?us-ascii?Q?pjzZspLNiuW3KHvhVfHor7paj0S87pixfHiGZLopol81gt74j9OZFvSfqCsg?=
 =?us-ascii?Q?rgclfJgMARcInHBbCsq/uFY1xLDgYPhhkjkWVNRS9q4lh3WUyPSi+4mwhu9G?=
 =?us-ascii?Q?dZjOiAsxokuI6ZJYa77v6kxffCKXA0I3lO96K5l6gTDGysKkuxC55VS45il9?=
 =?us-ascii?Q?BfpUpvQNUmyfLlZkB0EgJ/cEQ7G6lpJ6yZ1MyWIxYGGcMUsM/UIakrcFuSj3?=
 =?us-ascii?Q?FyDieOkhpW0xuuxC4ghoCMXL+a+ewO7fSK53LHOPxcuRK18PZMXq4PCkSzsr?=
 =?us-ascii?Q?6cfC8yE3eYV8SZX4Z8oYG8Wngo4vkXXEMN6RsQRg?=
X-OriginatorOrg: oppo.com
X-MS-Exchange-CrossTenant-Network-Message-Id: 281291fe-d430-4854-243e-08dc5b5ca303
X-MS-Exchange-CrossTenant-AuthSource: TYZPR02MB5595.apcprd02.prod.outlook.com
X-MS-Exchange-CrossTenant-AuthAs: Internal
X-MS-Exchange-CrossTenant-OriginalArrivalTime: 13 Apr 2024 01:54:21.1246
 (UTC)
X-MS-Exchange-CrossTenant-FromEntityHeader: Hosted
X-MS-Exchange-CrossTenant-Id: f1905eb1-c353-41c5-9516-62b4a54b5ee6
X-MS-Exchange-CrossTenant-MailboxType: HOSTED
X-MS-Exchange-CrossTenant-UserPrincipalName: DjBzoSgEeSdZHlp5G+4Ks+koat92wl/IEnR+wnndGz2bEqUy6X/X52OOYXiFREqSG+mjqkFDz906r4c1MtbcEg==
X-MS-Exchange-Transport-CrossTenantHeadersStamped: SEYPR02MB5632

From: Peifeng Li <lipeifeng@oppo.com>

In the case of insufficient memory, threads will be in direct_reclaim to
reclaim memory, direct_reclaim will call shrink_slab to run sequentially
each shrinker callback. If there is a lock-contention in the shrinker
callback,such as spinlock,mutex_lock and so on, threads may be likely to
be stuck in direct_reclaim for a long time, even if the memfree has reached
the high watermarks of the zone, resulting in poor performance of threads.

Example 1: shrinker callback may wait for spinlock
static unsigned long mb_cache_shrink(struct mb_cache *cache,
                                     unsigned long nr_to_scan)
{
        struct mb_cache_entry *entry;
        unsigned long shrunk = 0;

        spin_lock(&cache->c_list_lock);
        while (nr_to_scan-- && !list_empty(&cache->c_list)) {
                entry = list_first_entry(&cache->c_list,
                                         struct mb_cache_entry, e_list);
                if (test_bit(MBE_REFERENCED_B, &entry->e_flags) ||
                    atomic_cmpxchg(&entry->e_refcnt, 1, 0) != 1) {
                        clear_bit(MBE_REFERENCED_B, &entry->e_flags);
                        list_move_tail(&entry->e_list, &cache->c_list);
                        continue;
                }
                list_del_init(&entry->e_list);
                cache->c_entry_count--;
                spin_unlock(&cache->c_list_lock);
                __mb_cache_entry_free(cache, entry);
                shrunk++;
                cond_resched();
                spin_lock(&cache->c_list_lock);
        }
        spin_unlock(&cache->c_list_lock);

        return shrunk;
}
Example 2: shrinker callback may wait for mutex lock
static
unsigned long kbase_mem_evictable_reclaim_scan_objects(struct shrinker *s,
		struct shrink_control *sc)
{
	struct kbase_context *kctx;
	struct kbase_mem_phy_alloc *alloc;
	struct kbase_mem_phy_alloc *tmp;
	unsigned long freed = 0;

	kctx = container_of(s, struct kbase_context, reclaim);

	// MTK add to prevent false alarm
	lockdep_off();

	mutex_lock(&kctx->jit_evict_lock);

	list_for_each_entry_safe(alloc, tmp, &kctx->evict_list, evict_node) {
		int err;

		err = kbase_mem_shrink_gpu_mapping(kctx, alloc->reg,
				0, alloc->nents);
		if (err != 0) {
			freed = -1;
			goto out_unlock;
		}

		alloc->evicted = alloc->nents;

		kbase_free_phy_pages_helper(alloc, alloc->evicted);
		freed += alloc->evicted;
		list_del_init(&alloc->evict_node);

		kbase_jit_backing_lost(alloc->reg);

		if (freed > sc->nr_to_scan)
			break;
	}
out_unlock:
	mutex_unlock(&kctx->jit_evict_lock);

	// MTK add to prevent false alarm
	lockdep_on();

	return freed;
}

In mobile-phone,threads are likely to be stuck in shrinker callback during
direct_reclaim, with example like the following:
<...>-2806    [004] ..... 866458.339840: mm_shrink_slab_start:
			dynamic_mem_shrink_scan+0x0/0xb8 ... priority 2
<...>-2806    [004] ..... 866459.339933: mm_shrink_slab_end:
			dynamic_mem_shrink_scan+0x0/0xb8 ...

For the above reason, the patch introduces SHRINKER_NO_DIRECT_RECLAIM that
allows driver to set shrinker callback not to be called in direct_reclaim
unless sc->priority is 0.

The reason why sc->priority=0 allows shrinker callback to be called in
direct_reclaim is for two reasons:
1.Always call all shrinker callback in drop_slab that priority is 0.
2.sc->priority is 0 during direct_reclaim, allow direct_reclaim to call
shrinker callback, to reclaim memory timely.

Note:
1.shrinker_register() default not to set SHRINKER_NO_DIRECT_RECLAIM, to
maintain the current behavior of the code.
2.Logic of kswapd and drop_slab to call shrinker callback isn't affected.

Signed-off-by: Peifeng Li <lipeifeng@oppo.com>
---
-v2: fix the commit message
 include/linux/shrinker.h |  5 +++++
 mm/shrinker.c            | 38 +++++++++++++++++++++++++++++++++++---
 2 files changed, 40 insertions(+), 3 deletions(-)

diff --git a/include/linux/shrinker.h b/include/linux/shrinker.h
index 1a00be90d93a..2d5a8b3a720b 100644
--- a/include/linux/shrinker.h
+++ b/include/linux/shrinker.h
@@ -130,6 +130,11 @@ struct shrinker {
  * non-MEMCG_AWARE shrinker should not have this flag set.
  */
 #define SHRINKER_NONSLAB	BIT(4)
+/*
+ * Can shrinker callback be called in direct_relcaim unless
+ * sc->priority is 0?
+ */
+#define SHRINKER_NO_DIRECT_RECLAIM	BIT(5)
 
 __printf(2, 3)
 struct shrinker *shrinker_alloc(unsigned int flags, const char *fmt, ...);
diff --git a/mm/shrinker.c b/mm/shrinker.c
index dc5d2a6fcfc4..7a5dffd166cd 100644
--- a/mm/shrinker.c
+++ b/mm/shrinker.c
@@ -4,7 +4,7 @@
 #include <linux/shrinker.h>
 #include <linux/rculist.h>
 #include <trace/events/vmscan.h>
-
+#include <linux/swap.h>
 #include "internal.h"
 
 LIST_HEAD(shrinker_list);
@@ -544,7 +544,23 @@ static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid,
 			if (!memcg_kmem_online() &&
 			    !(shrinker->flags & SHRINKER_NONSLAB))
 				continue;
-
+			/*
+			 * SHRINKER_NO_DIRECT_RECLAIM, mean that shrinker callback
+			 * should not be called in direct_reclaim unless priority
+			 * is 0.
+			 */
+			if ((shrinker->flags & SHRINKER_NO_DIRECT_RECLAIM) &&
+					!current_is_kswapd()) {
+				/*
+				 * 1.Always call shrinker callback in drop_slab that
+				 * priority is 0.
+				 * 2.sc->priority is 0 during direct_reclaim, allow
+				 * direct_reclaim to call shrinker callback, to reclaim
+				 * memory timely.
+				 */
+				if (priority)
+					continue;
+			}
 			ret = do_shrink_slab(&sc, shrinker, priority);
 			if (ret == SHRINK_EMPTY) {
 				clear_bit(offset, unit->map);
@@ -658,7 +674,23 @@ unsigned long shrink_slab(gfp_t gfp_mask, int nid, struct mem_cgroup *memcg,
 			continue;
 
 		rcu_read_unlock();
-
+		/*
+		 * SHRINKER_NO_DIRECT_RECLAIM, mean that shrinker callback
+		 * should not be called in direct_reclaim unless priority
+		 * is 0.
+		 */
+		if ((shrinker->flags & SHRINKER_NO_DIRECT_RECLAIM) &&
+				!current_is_kswapd()) {
+			/*
+			 * 1.Always call shrinker callback in drop_slab that
+			 * priority is 0.
+			 * 2.sc->priority is 0 during direct_reclaim, allow
+			 * direct_reclaim to call shrinker callback, to reclaim
+			 * memory timely.
+			 */
+			if (priority)
+				continue;
+		}
 		ret = do_shrink_slab(&sc, shrinker, priority);
 		if (ret == SHRINK_EMPTY)
 			ret = 0;
-- 
2.34.1