Received-SPF: pass (google.com: domain of linux-kernel+bounces-142209-linux.lists.archive=gmail.com@vger.kernel.org designates 139.178.88.99 as permitted sender) client-ip=139.178.88.99;
From: lipeifeng@oppo.com
To: lipeifeng@oppo.com,
	akpm@linux-foundation.org,
	david@fromorbit.com,
	zhengqi.arch@bytedance.com,
	roman.gushchin@linux.dev,
	muchun.song@linux.dev
Cc: linux-mm@kvack.org,
	linux-kernel@vger.kernel.org
Subject: [PATCH] mm/shrinker: add SHRINKER_NO_DIRECT_RECLAIM
Date: Fri, 12 Apr 2024 16:07:06 +0800
Message-Id: <20240412080706.744-1-lipeifeng@oppo.com>
Content-Transfer-Encoding: 8bit
Content-Type: text/plain
Precedence: bulk
MIME-Version: 1.0
X-MS-Exchange-AntiSpam-MessageData-ChunkCount: 1
X-MS-Exchange-AntiSpam-MessageData-0:
	=?us-ascii?Q?oGLf+IeVgpj+XI8CSi1Ax4kJCfFOPm1zPxQaAQJ7nlETaTOzputRPh2fx+4K?=
 =?us-ascii?Q?RRqFGnSnrUWLhvn5xO6eCDaWETr+XcwI9fBi85hx98tWw17EgIMp+vbENBkb?=
 =?us-ascii?Q?4Nre6EPTNgwiO90W84hq80A1t4/sOdGK7PPMDqvHzFG7i60RwG/dXbSBc4pg?=
 =?us-ascii?Q?hxOGE6I2MpOqRuc8arwutdgkSXUCraNN7y8qdDV/EYnQdSQeh4rPEKZnOTEt?=
 =?us-ascii?Q?n+Z5rpxhf5/N8W2GtUAZRsD9meaTZ2s8hz9K09H1KN8m5WPSHC4XlKTkb7HV?=
 =?us-ascii?Q?6bt40fvwLKHBgThUYIw4bqkaXg+81nBRRRyRZeUl9Rn4KF9qR5jvtx3hFyB3?=
 =?us-ascii?Q?EhLxQ9c3S9x/dMUgcWPwxUjB3lYs/Omh02tKDpfr+zBXXLBv0j++CPg/Fa0W?=
 =?us-ascii?Q?a6shsHY1tbMIJsU1fgnB58JFgXTtKU+1CZdYaLFxAa9Ek3g796Ry+/la1x1g?=
 =?us-ascii?Q?MqAKqgeLBPOEWC9oIQ/foCzslxqwt/2usQX4Jxk2XBDY0jPjpJx6sTYfE4hK?=
 =?us-ascii?Q?di5NVAvRgNx3tSff7nBoNxjSwhxU+AmVbqdKBLqZLIeIufsAJUf7J5wLynGE?=
 =?us-ascii?Q?Cr7AvYlZd3fuTeQPubZ7g6313UllNa83mVL9OzLbbURmnTfcsMSpgrzhfBow?=
 =?us-ascii?Q?nIhUeYu3M9mVgsQ4fWXmASk7klEcmWtmf+U6y3LNkFdiBeuUkz4HNk14eOd8?=
 =?us-ascii?Q?+GkxhkX1BXSmwNGcdmKDYdOkJ422OrmD8EYyC8krnT9ro+dviKe+yi83ujL1?=
 =?us-ascii?Q?hN7v9jgkAq5HlHqx8AwGSeWzuelZyRqzI3ky1yUqD+kmBbklNxGT/FhVinj+?=
 =?us-ascii?Q?Lvp0xqwa05sIIluSOHXR5r3Y8Mty7p2/d7FEQ+cENYTZBxiAzZJ2OfSN5YK+?=
 =?us-ascii?Q?1suSQritVElJwrTPeukzJTy9QQXvVsp1KP7T/GVl/otyF7X3oZS/Fa3pxWEi?=
 =?us-ascii?Q?lxe7jWq0HFtrEWcxNbNthgpRRe0umPo2n6aDVtspOlPOC9udnYwW+hlM/6JE?=
 =?us-ascii?Q?Z26QdZ1vbg4S9sCV4Lu1cKB3eX8lSnVYV0k2B3SBed+wY0NhWy/kSj4bkzL/?=
 =?us-ascii?Q?ofl1RLRMiVclqZcWJLqS1xnj7AY89rK0sxA7syFU4gAiga+IS3RpzOkb/Ntf?=
 =?us-ascii?Q?a6lCotV5GlL11cmv2k/YlYniBZ0XH2J1XHoZ0wtt8/tvC2UzSkcSNOURccQQ?=
 =?us-ascii?Q?i3wcxd1DLvLQjwX0VesHpA891/kWZtndB/zAnFk0BspL5DT+mrNdmX0o78gm?=
 =?us-ascii?Q?KeZlh3hvwH72b4DUWbDa6po7S5mOS1CaT+ls0BQfmtqwWfK7O+fXoQncTFdm?=
 =?us-ascii?Q?l8L6wYJ0yLVokEYiyzAylGLNbpNvHF+PAUBD+22ICiOp13DA1CjZ4+VOPjMH?=
 =?us-ascii?Q?cXip1YN4xzVKbAtOs9bVqSY0fxLEAd2ndDIOxxMbmo27LtbP68cNYdqZrEDe?=
 =?us-ascii?Q?tzYlm2LYNtcc82hk9uAmHc2OVoMxwCMiji0erxJzywo2d9jmPPQSBoR3OK3P?=
 =?us-ascii?Q?hgPFwAeUfxZ7rdYSX443eC2Cq/XQ6vdRVupC4ZzYoxZFmS4AB1M8zt/HSKCw?=
 =?us-ascii?Q?AXA29ZrVlUrB01qkfXXWKaSK5AVH/3M0JbFVJ346?=
X-OriginatorOrg: oppo.com
X-MS-Exchange-CrossTenant-Network-Message-Id: 934f5831-5776-4959-8a64-08dc5ac792f7
X-MS-Exchange-CrossTenant-AuthSource: TYZPR02MB5595.apcprd02.prod.outlook.com
X-MS-Exchange-CrossTenant-AuthAs: Internal
X-MS-Exchange-CrossTenant-OriginalArrivalTime: 12 Apr 2024 08:07:19.1962
 (UTC)
X-MS-Exchange-CrossTenant-FromEntityHeader: Hosted
X-MS-Exchange-CrossTenant-Id: f1905eb1-c353-41c5-9516-62b4a54b5ee6
X-MS-Exchange-CrossTenant-MailboxType: HOSTED
X-MS-Exchange-CrossTenant-UserPrincipalName: SJTNBmGBqienfADFx52klyxu5DfPj7SZqkc868B8gVx8XM5nqFxOSwb5d0czHGWcGxHK+yBLEFsprTfPVyBYsg==
X-MS-Exchange-Transport-CrossTenantHeadersStamped: JH0PR02MB7549

From: Peifeng Li <lipeifeng@oppo.com>

In the case of insufficient memory, threads will be in direct_reclaim to
reclaim memory, direct_reclaim will call shrink_slab to run sequentially
each shrinker callback. If there is a lock-contention in the shrinker
callback,such as spinlock,mutex_lock and so on, threads may be likely to
be stuck in direct_reclaim for a long time, even if the memfree has reached
the high watermarks of the zone, resulting in poor performance of threads.

Example 1: shrinker callback may wait for spinlock
static unsigned long mb_cache_shrink(struct mb_cache *cache,
                                     unsigned long nr_to_scan)
{
        struct mb_cache_entry *entry;
        unsigned long shrunk = 0;

        spin_lock(&cache->c_list_lock);
        while (nr_to_scan-- && !list_empty(&cache->c_list)) {
                entry = list_first_entry(&cache->c_list,
                                         struct mb_cache_entry, e_list);
                if (test_bit(MBE_REFERENCED_B, &entry->e_flags) ||
                    atomic_cmpxchg(&entry->e_refcnt, 1, 0) != 1) {
                        clear_bit(MBE_REFERENCED_B, &entry->e_flags);
                        list_move_tail(&entry->e_list, &cache->c_list);
                        continue;
                }
                list_del_init(&entry->e_list);
                cache->c_entry_count--;
                spin_unlock(&cache->c_list_lock);
                __mb_cache_entry_free(cache, entry);
                shrunk++;
                cond_resched();
                spin_lock(&cache->c_list_lock);
        }
        spin_unlock(&cache->c_list_lock);

        return shrunk;
}
Example 2: shrinker callback may wait for mutex lock
static
unsigned long kbase_mem_evictable_reclaim_scan_objects(struct shrinker *s,
		struct shrink_control *sc)
{
	struct kbase_context *kctx;
	struct kbase_mem_phy_alloc *alloc;
	struct kbase_mem_phy_alloc *tmp;
	unsigned long freed = 0;

	kctx = container_of(s, struct kbase_context, reclaim);

	// MTK add to prevent false alarm
	lockdep_off();

	mutex_lock(&kctx->jit_evict_lock);

	list_for_each_entry_safe(alloc, tmp, &kctx->evict_list, evict_node) {
		int err;

		err = kbase_mem_shrink_gpu_mapping(kctx, alloc->reg,
				0, alloc->nents);
		if (err != 0) {
			freed = -1;
			goto out_unlock;
		}

		alloc->evicted = alloc->nents;

		kbase_free_phy_pages_helper(alloc, alloc->evicted);
		freed += alloc->evicted;
		list_del_init(&alloc->evict_node);

		kbase_jit_backing_lost(alloc->reg);

		if (freed > sc->nr_to_scan)
			break;
	}
out_unlock:
	mutex_unlock(&kctx->jit_evict_lock);

	// MTK add to prevent false alarm
	lockdep_on();

	return freed;
}

In mobile-phone,threads are likely to be stuck in shrinker callback during
direct_reclaim, with example like the following:
<...>-2806    [004] ..... 866458.339840: mm_shrink_slab_start:
			dynamic_mem_shrink_scan+0x0/0xb8 ... priority 2
<...>-2806    [004] ..... 866459.339933: mm_shrink_slab_end:
			dynamic_mem_shrink_scan+0x0/0xb8 ...

For the above reason, the patch introduces SHRINKER_NO_DIRECT_RECLAIM that
allows driver to set shrinker callback not to be called in direct_reclaim
unless sc->priority is 0.

The reason why sc->priority=0 allows shrinker callback to be called in
direct_reclaim is for two reasons:
1.Always call all shrinker callback in drop_slab that priority is 0.
2.sc->priority is 0 during direct_reclaim, allow direct_reclaim to call
shrinker callback, to reclaim memory timely.

Note:
1.Register_shrinker_prepared() default not to set
SHRINKER_NO_DIRECT_RECLAIM, to maintain the current behavior of the code.
2.Logic of kswapd and drop_slab to call shrinker callback isn't affected.

Signed-off-by: Peifeng Li <lipeifeng@oppo.com>
---
 include/linux/shrinker.h |  5 +++++
 mm/shrinker.c            | 36 ++++++++++++++++++++++++++++++++++--
 2 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/include/linux/shrinker.h b/include/linux/shrinker.h
index 1a00be90d93a..2d5a8b3a720b 100644
--- a/include/linux/shrinker.h
+++ b/include/linux/shrinker.h
@@ -130,6 +130,11 @@ struct shrinker {
  * non-MEMCG_AWARE shrinker should not have this flag set.
  */
 #define SHRINKER_NONSLAB	BIT(4)
+/*
+ * Can shrinker callback be called in direct_relcaim unless
+ * sc->priority is 0?
+ */
+#define SHRINKER_NO_DIRECT_RECLAIM	BIT(5)
 
 __printf(2, 3)
 struct shrinker *shrinker_alloc(unsigned int flags, const char *fmt, ...);
diff --git a/mm/shrinker.c b/mm/shrinker.c
index dc5d2a6fcfc4..3ac50da72494 100644
--- a/mm/shrinker.c
+++ b/mm/shrinker.c
@@ -544,7 +544,23 @@ static unsigned long shrink_slab_memcg(gfp_t gfp_mask, int nid,
 			if (!memcg_kmem_online() &&
 			    !(shrinker->flags & SHRINKER_NONSLAB))
 				continue;
-
+			/*
+			 * SHRINKER_NO_DIRECT_RECLAIM, mean that shrinker callback
+			 * should not be called in direct_reclaim unless priority
+			 * is 0.
+			 */
+			if ((shrinker->flags & SHRINKER_NO_DIRECT_RECLAIM) &&
+					!current_is_kswapd()) {
+				/*
+				 * 1.Always call shrinker callback in drop_slab that
+				 * priority is 0.
+				 * 2.sc->priority is 0 during direct_reclaim, allow
+				 * direct_reclaim to call shrinker callback, to reclaim
+				 * memory timely.
+				 */
+				if (priority)
+					continue;
+			}
 			ret = do_shrink_slab(&sc, shrinker, priority);
 			if (ret == SHRINK_EMPTY) {
 				clear_bit(offset, unit->map);
@@ -658,7 +674,23 @@ unsigned long shrink_slab(gfp_t gfp_mask, int nid, struct mem_cgroup *memcg,
 			continue;
 
 		rcu_read_unlock();
-
+		/*
+		 * SHRINKER_NO_DIRECT_RECLAIM, mean that shrinker callback
+		 * should not be called in direct_reclaim unless priority
+		 * is 0.
+		 */
+		if ((shrinker->flags & SHRINKER_NO_DIRECT_RECLAIM) &&
+				!current_is_kswapd()) {
+			/*
+			 * 1.Always call shrinker callback in drop_slab that
+			 * priority is 0.
+			 * 2.sc->priority is 0 during direct_reclaim, allow
+			 * direct_reclaim to call shrinker callback, to reclaim
+			 * memory timely.
+			 */
+			if (priority)
+				continue;
+		}
 		ret = do_shrink_slab(&sc, shrinker, priority);
 		if (ret == SHRINK_EMPTY)
 			ret = 0;
-- 
2.34.1