Received-SPF: pass (google.com: domain of linux-kernel-owner@vger.kernel.org designates 2620:137:e000::1:20 as permitted sender) client-ip=2620:137:e000::1:20;
From:   "Liam R. Howlett" <Liam.Howlett@oracle.com>
To:     Andrew Morton <akpm@linux-foundation.org>
Cc:     linux-kernel@vger.kernel.org, linux-mm@kvack.org,
        maple-tree@lists.infradead.org,
        "Liam R. Howlett" <Liam.Howlett@oracle.com>
Subject: [PATCH 24/34] maple_tree: Try harder to keep active node with mas_prev()
Date:   Tue, 25 Apr 2023 10:09:45 -0400
Message-Id: <20230425140955.3834476-25-Liam.Howlett@oracle.com>
In-Reply-To: <20230425140955.3834476-1-Liam.Howlett@oracle.com>
References: <20230425140955.3834476-1-Liam.Howlett@oracle.com>
Content-Transfer-Encoding: 8bit
Content-Type: text/plain
MIME-Version: 1.0
X-MS-Exchange-AntiSpam-MessageData-ChunkCount: 1
X-MS-Exchange-AntiSpam-MessageData-0: =?us-ascii?Q?4kDY+B1M1yhelL/UXyb/5xBKfmDRw5Pn0dw5bQKWAoUHS4RHK1R1JYAPMzC1?=
 =?us-ascii?Q?brxnoUxZM9eXfNXBPPn7rYVmPrj7qVrt65GXtMYIJdWteR+kvqkMfbZDYa7h?=
 =?us-ascii?Q?6kFtbIOVBiulWs1WP16AtzCvuCSEgBct739mCr5RvIpMVcnn4NyO0Kn0tVxQ?=
 =?us-ascii?Q?CtZ6ALeFHJGdtRsWR9SlHgYP3muf/QLTBK9OXCBIoxdSxlBZix9CPLw1W3RB?=
 =?us-ascii?Q?CbgXLoVukyOd1glS7bbFTRDk4qlcWfdOTihYj5wEw8WjnCO3uhb/9XkXkhZz?=
 =?us-ascii?Q?ysBf0MswO/uqtOyrPpuvMDgbxTwuPd/+U/CF2HLCJC9D+hnu9ElctGMX7nGn?=
 =?us-ascii?Q?EldVsKJglawrya9RfQImK3RssDLRtZojVTdzrcgeycAbPXNaT2oy7/Xo8l0H?=
 =?us-ascii?Q?tDhIn3EtdzZk0p7cx2wKUFuCkONYpN6ssj+NmdQ0Q8szjoF60xRK5Dl0nqR3?=
 =?us-ascii?Q?Wv2BNaDGkn1OlSiibwFJzc5pS+nOjMlRJiRcmUe5YNIulNk5z9/a2ei+gDxN?=
 =?us-ascii?Q?CoZub/aZpAFRErBbfYyib/p2HJG3setIcOhASllIcQL589ASXxIFtoz0LrQC?=
 =?us-ascii?Q?aY9n/k5stWHz6ApevEV/Ln4pKsa5elT9wmld0rXsfW9ULI4zMyqhIHgG7c8r?=
 =?us-ascii?Q?N7Ex66o/S0OVbaFQOZhgF6ZWCAz7jdtOpvkGdNdloq1olkY3Eac9ZlEYwHcX?=
 =?us-ascii?Q?fnqDSJsLTSnJgsK6C43zM3dV0d5CQwIJQ4VmBjsNWZadInZNqs5x3txRbDX4?=
 =?us-ascii?Q?jUunwalzPRWSy8FMe0STnn4Mw60kx9zeTUOTEgMzATylcGQxP6cOOhwa0UEz?=
 =?us-ascii?Q?qDxZT556yafKN3i3cJFP7RMboZtP0oxixVSxyenB5Trakr3t562X5aSwKWIw?=
 =?us-ascii?Q?NbLbQTLZ8UDjCtT+EIeFnoSWM5cLb0R+v5edIv/6FUzzbZIlKJZVybGuC+DR?=
 =?us-ascii?Q?g7hiqYUd4vF/xEoTQCjCtqA/QkwmJAQtr++PuxEgrFHCqcQCrekT+Gt20Nqj?=
 =?us-ascii?Q?Vz7pmWEp5+9k5puV54Tk4oQ4wbtJAcBkari8E11QFKbULOzK37OpY3syfpSG?=
 =?us-ascii?Q?bb4l3vV92Z+gmfNVbCPHdKu1/VIbeIxxCUWL7n+hDSrOdvJLl5jinO8hBIEn?=
 =?us-ascii?Q?JG19v5pAmApt98qeQ4tivDvhWYfQkbKVFauuG+SJfeqw1+BkP/s/sPcjEWQL?=
 =?us-ascii?Q?zNlfv+cDOpFIbCpaDFEn+jM+4CR4CiYNlNVG2XeM9LTGIoqo94PZsG5YYf7G?=
 =?us-ascii?Q?SzrJI3tRCO4m3KMBy8F8xuY3GkV9iIukCeKOcM6x2ifyzMgaWpZKMsOsQbYn?=
 =?us-ascii?Q?SPoNzvyM8Kn2MlfBrPQ+a/9x00Z/8HCP+I5zYirqDMJeVyR3do+0mDVSngXs?=
 =?us-ascii?Q?joUoIA8rlHOToEBoNyqRnWT2fKt1CVfwG97TAc+z/aSrgXmFwOgUBoRcOz94?=
 =?us-ascii?Q?AtL4l0uuN8LKPHrUFIVLK1DZxtdjqe3GI16BGoSKOp8AtkLGdRpy08aeAvHl?=
 =?us-ascii?Q?EkyK8AuFNzNduV6AWXifHrjBhMzrSck67UUSgWi/NmooPXM1+6ufBqGdpFil?=
 =?us-ascii?Q?V5F4/Ym7XCw10xq7jIr8dThZKUWrHUJCgxIBHVkMfX9UWiQpVBhoJhHZUiqp?=
 =?us-ascii?Q?Gw=3D=3D?=
X-MS-Exchange-AntiSpam-ExternalHop-MessageData-ChunkCount: 1
X-MS-Exchange-AntiSpam-ExternalHop-MessageData-0: OtH3qyJNK5w6N/V9pbzP3H9Wr8DaiadfVB+bJEz+IpgGlkNTh0qVgULXt+ORS2nOXE7c2rqeAJSuGjPhv5wHzM1JpR70UPttk843i4+3RGfJvBEiFGmCbOd8v3dgYNvhdBMDCuQrrauEuHhN1w9xQRZrF9WWzqjcy2B6Agk7HWuUxrs7LB5wE0UhRv8vfpyEpMZzHHAMC46XLk9IVg47ifbCaIJwFOX02AtMedxG/djXHIVRk+Tede21MxRdTSGouiHV8PRTlAI/5DtAdkandMKJ5dVjEZZf5u69+M+c5KrcdKD0ymjc8sWjKWuGIHFlvX+mlVuru9m9cTvZ42jNFjXErdxGlkRELcIG/b0yuvUDfZhDOqW1Wv+XeLmywLMun08CnmtZA6oK7KWen/Ty86qPjTt3BjErVAHtHxmpbe+z051SLMnPpj+bvZHaXleqTiGgOBqR0elp41G8mRQG2/JH784Uf5I1KT1Bd0xVMHAxhJea1kyyMRry+BLnOM1SIbHlCaPbkfJhFFmzLPHkcTqbcZ1A1qIJYK30lrYqelplnIBALozRE6jxwOYrv82bKEwUhj9dGMyiCEdsIU10k+pI+rb71+jR/JNNj6tOkgqS0Ii/Eqd4N9H4aXHbwLLWhZl6wbRA8tBAIY8NfHYr4PACV/VJgsV4olhUGCF9SaaAfrbaJSEzOyJ/oz1ZG7mR5SpuZ41/rcLA1C/IfFq0Rr+SEZJBQrqmUbAKUqoCWFz7z3NMDNtsN2z1g3528yY52F8XBw4edyccqFXmD3YLk7W+w/pmhDAk03KsDdgC9eIxSasfGOo+Db+NGsM0c2XE/YpQ73Flj2hhN822kcs7BMkYMy5Y+ARa5lncrSYydcoWK9gnGdmUv1L58XX2DdTtd5RCuNiAHxSx9yxuDMfW/g==
X-OriginatorOrg: oracle.com
X-MS-Exchange-CrossTenant-Network-Message-Id: 82f8dd6e-ad7b-40dc-c5e3-08db4596efdd
X-MS-Exchange-CrossTenant-AuthSource: SN6PR10MB3022.namprd10.prod.outlook.com
X-MS-Exchange-CrossTenant-AuthAs: Internal
X-MS-Exchange-CrossTenant-OriginalArrivalTime: 25 Apr 2023 14:11:17.5993
 (UTC)
X-MS-Exchange-CrossTenant-FromEntityHeader: Hosted
X-MS-Exchange-CrossTenant-Id: 4e2c6054-71cb-48f1-bd6c-3a9705aca71b
X-MS-Exchange-CrossTenant-MailboxType: HOSTED
X-MS-Exchange-CrossTenant-UserPrincipalName: U6gb4qHGiIDF8j+QwmNBBQSS2x74LLYvJbJA9WafTILvyxSueuhlWAW9h+uO2dwVvOiyvMB3RWzRrnex9p2MLg==
X-MS-Exchange-Transport-CrossTenantHeadersStamped: CO6PR10MB5572
X-Proofpoint-Virus-Version: vendor=baseguard
 engine=ICAP:2.0.254,Aquarius:18.0.942,Hydra:6.0.573,FMLib:17.11.170.22
 definitions=2023-04-25_06,2023-04-25_01,2023-02-09_01
X-Proofpoint-Spam-Details: rule=notspam policy=default score=0 phishscore=0 mlxlogscore=999
 adultscore=0 malwarescore=0 mlxscore=0 spamscore=0 bulkscore=0
 suspectscore=0 classifier=spam adjust=0 reason=mlx scancount=1
 engine=8.12.0-2303200000 definitions=main-2304250128
X-Proofpoint-ORIG-GUID: SgGJfYOnvijv_m_87PRucyzLAeumF0po
X-Proofpoint-GUID: SgGJfYOnvijv_m_87PRucyzLAeumF0po
X-Spam-Status: No, score=-2.8 required=5.0 tests=BAYES_00,DKIM_SIGNED,
        DKIM_VALID,DKIM_VALID_AU,DKIM_VALID_EF,RCVD_IN_DNSWL_LOW,
        RCVD_IN_MSPIKE_H2,SPF_HELO_NONE,SPF_NONE,T_SCC_BODY_TEXT_LINE
        autolearn=ham autolearn_force=no version=3.4.6
X-Spam-Checker-Version: SpamAssassin 3.4.6 (2021-04-09) on
        lindbergh.monkeyblade.net
Precedence: bulk
List-ID: <linux-kernel.vger.kernel.org>
X-Mailing-List: linux-kernel@vger.kernel.org

Keep a reference to the node when possible with mas_prev().  This will
avoid re-walking the tree.  In keeping a reference to the node, keep the
last/index accurate to the range being referenced.  This means the limit
may be within the range, but the range may extend outside of the limit.

Also fix the single entry tree to respect the range (of 0), or set the
node to MAS_NONE in the case of shifting beyond 0.

Signed-off-by: Liam R. Howlett <Liam.Howlett@oracle.com>
---
 lib/maple_tree.c | 125 +++++++++++++++++++++++++++++++----------------
 1 file changed, 83 insertions(+), 42 deletions(-)

diff --git a/lib/maple_tree.c b/lib/maple_tree.c
index ef7a6ceca864c..20f0a10dc5608 100644
--- a/lib/maple_tree.c
+++ b/lib/maple_tree.c
@@ -4828,7 +4828,7 @@ static inline void *mas_prev_nentry(struct ma_state *mas, unsigned long limit,
 				    unsigned long index)
 {
 	unsigned long pivot, min;
-	unsigned char offset;
+	unsigned char offset, count;
 	struct maple_node *mn;
 	enum maple_type mt;
 	unsigned long *pivots;
@@ -4842,29 +4842,42 @@ static inline void *mas_prev_nentry(struct ma_state *mas, unsigned long limit,
 	mn = mas_mn(mas);
 	mt = mte_node_type(mas->node);
 	offset = mas->offset - 1;
-	if (offset >= mt_slots[mt])
-		offset = mt_slots[mt] - 1;
-
 	slots = ma_slots(mn, mt);
 	pivots = ma_pivots(mn, mt);
+	count = ma_data_end(mn, mt, pivots, mas->max);
 	if (unlikely(ma_dead_node(mn))) {
 		mas_rewalk(mas, index);
 		goto retry;
 	}
 
-	if (offset == mt_pivots[mt])
+	offset = mas->offset - 1;
+	if (offset >= mt_slots[mt])
+		offset = mt_slots[mt] - 1;
+
+	if (offset >= count) {
 		pivot = mas->max;
-	else
+		offset = count;
+	} else {
 		pivot = pivots[offset];
+	}
 
 	if (unlikely(ma_dead_node(mn))) {
 		mas_rewalk(mas, index);
 		goto retry;
 	}
 
-	while (offset && ((!mas_slot(mas, slots, offset) && pivot >= limit) ||
-	       !pivot))
+	while (offset && !mas_slot(mas, slots, offset)) {
 		pivot = pivots[--offset];
+		if (pivot >= limit)
+			break;
+	}
+
+	/*
+	 * If the slot was null but we've shifted outside the limits, then set
+	 * the range to the last NULL.
+	 */
+	if (unlikely((pivot < limit) && (offset < mas->offset)))
+		pivot = pivots[++offset];
 
 	min = mas_safe_min(mas, pivots, offset);
 	entry = mas_slot(mas, slots, offset);
@@ -4873,32 +4886,33 @@ static inline void *mas_prev_nentry(struct ma_state *mas, unsigned long limit,
 		goto retry;
 	}
 
-	if (likely(entry)) {
-		mas->offset = offset;
-		mas->last = pivot;
-		mas->index = min;
-	}
+	mas->offset = offset;
+	mas->last = pivot;
+	mas->index = min;
 	return entry;
 }
 
 static inline void *mas_prev_entry(struct ma_state *mas, unsigned long min)
 {
 	void *entry;
+	struct maple_enode *prev_enode;
+	unsigned char prev_offset;
 
-	if (mas->index < min) {
-		mas->index = mas->last = min;
-		mas->node = MAS_NONE;
+	if (mas->index < min)
 		return NULL;
-	}
+
 retry:
+	prev_enode = mas->node;
+	prev_offset = mas->offset;
 	while (likely(!mas_is_none(mas))) {
 		entry = mas_prev_nentry(mas, min, mas->index);
-		if (unlikely(mas->last < min))
-			goto not_found;
 
 		if (likely(entry))
 			return entry;
 
+		if (unlikely(mas->index <= min))
+			return NULL;
+
 		if (unlikely(mas_prev_node(mas, min))) {
 			mas_rewalk(mas, mas->index);
 			goto retry;
@@ -4907,9 +4921,8 @@ static inline void *mas_prev_entry(struct ma_state *mas, unsigned long min)
 		mas->offset++;
 	}
 
-	mas->offset--;
-not_found:
-	mas->index = mas->last = min;
+	mas->node = prev_enode;
+	mas->offset = prev_offset;
 	return NULL;
 }
 
@@ -5958,15 +5971,8 @@ EXPORT_SYMBOL_GPL(mt_next);
  */
 void *mas_prev(struct ma_state *mas, unsigned long min)
 {
-	if (!mas->index) {
-		/* Nothing comes before 0 */
-		mas->last = 0;
-		mas->node = MAS_NONE;
-		return NULL;
-	}
-
-	if (unlikely(mas_is_ptr(mas)))
-		return NULL;
+	if (mas->index <= min)
+		goto none;
 
 	if (mas_is_none(mas) || mas_is_paused(mas))
 		mas->node = MAS_START;
@@ -5974,19 +5980,30 @@ void *mas_prev(struct ma_state *mas, unsigned long min)
 	if (mas_is_start(mas)) {
 		mas_walk(mas);
 		if (!mas->index)
-			return NULL;
+			goto none;
 	}
 
-	if (mas_is_ptr(mas)) {
-		if (!mas->index) {
-			mas->last = 0;
-			return NULL;
-		}
-
+	if (unlikely(mas_is_ptr(mas))) {
+		if (!mas->index)
+			goto none;
 		mas->index = mas->last = 0;
-		return mas_root_locked(mas);
+		return mas_root(mas);
+	}
+
+	if (mas_is_none(mas)) {
+		if (mas->index) {
+			/* Walked to out-of-range pointer? */
+			mas->index = mas->last = 0;
+			mas->node = MAS_ROOT;
+			return mas_root(mas);
+		}
+		return NULL;
 	}
 	return mas_prev_entry(mas, min);
+
+none:
+	mas->node = MAS_NONE;
+	return NULL;
 }
 EXPORT_SYMBOL_GPL(mas_prev);
 
@@ -6112,8 +6129,16 @@ EXPORT_SYMBOL_GPL(mas_find);
  */
 void *mas_find_rev(struct ma_state *mas, unsigned long min)
 {
+	if (unlikely(mas_is_none(mas))) {
+		if (mas->index <= min)
+			goto none;
+
+		mas->last = mas->index;
+		mas->node = MAS_START;
+	}
+
 	if (unlikely(mas_is_paused(mas))) {
-		if (unlikely(mas->last == ULONG_MAX)) {
+		if (unlikely(mas->index <= min)) {
 			mas->node = MAS_NONE;
 			return NULL;
 		}
@@ -6133,14 +6158,30 @@ void *mas_find_rev(struct ma_state *mas, unsigned long min)
 			return entry;
 	}
 
-	if (unlikely(!mas_searchable(mas)))
-		return NULL;
+	if (unlikely(!mas_searchable(mas))) {
+		if (mas_is_ptr(mas))
+			goto none;
+
+		if (mas_is_none(mas)) {
+			/*
+			 * Walked to the location, and there was nothing so the
+			 * previous location is 0.
+			 */
+			mas->last = mas->index = 0;
+			mas->node = MAS_ROOT;
+			return mas_root(mas);
+		}
+	}
 
 	if (mas->index < min)
 		return NULL;
 
 	/* Retries on dead nodes handled by mas_prev_entry */
 	return mas_prev_entry(mas, min);
+
+none:
+	mas->node = MAS_NONE;
+	return NULL;
 }
 EXPORT_SYMBOL_GPL(mas_find_rev);
 
-- 
2.39.2