Received-SPF: pass (google.com: domain of linux-kernel-owner@vger.kernel.org designates 23.128.96.18 as permitted sender) client-ip=23.128.96.18;
From:   Liam Howlett <liam.howlett@oracle.com>
To:     "maple-tree@lists.infradead.org" <maple-tree@lists.infradead.org>,
        "linux-mm@kvack.org" <linux-mm@kvack.org>,
        "linux-kernel@vger.kernel.org" <linux-kernel@vger.kernel.org>,
        Andrew Morton <akpm@linux-foundation.org>
CC:     Song Liu <songliubraving@fb.com>,
        Davidlohr Bueso <dave@stgolabs.net>,
        "Paul E . McKenney" <paulmck@kernel.org>,
        Matthew Wilcox <willy@infradead.org>,
        Laurent Dufour <ldufour@linux.ibm.com>,
        David Rientjes <rientjes@google.com>,
        Axel Rasmussen <axelrasmussen@google.com>,
        Suren Baghdasaryan <surenb@google.com>,
        Vlastimil Babka <vbabka@suse.cz>,
        Rik van Riel <riel@surriel.com>,
        Peter Zijlstra <peterz@infradead.org>,
        Michel Lespinasse <walken.cr@gmail.com>,
        Liam Howlett <liam.howlett@oracle.com>
Subject: [PATCH 93/94] mm/mmap: Add mas_split_vma() and use it for munmap()
Thread-Topic: [PATCH 93/94] mm/mmap: Add mas_split_vma() and use it for
 munmap()
Thread-Index: AQHXPERD3sgzzL1RL02DDjtlqcDZNA==
Date:   Wed, 28 Apr 2021 15:36:32 +0000
Message-ID: <20210428153542.2814175-94-Liam.Howlett@Oracle.com>
References: <20210428153542.2814175-1-Liam.Howlett@Oracle.com>
In-Reply-To: <20210428153542.2814175-1-Liam.Howlett@Oracle.com>
Accept-Language: en-US
Content-Language: en-US
Content-Type: text/plain; charset="iso-8859-1"
Content-Transfer-Encoding: quoted-printable
MIME-Version: 1.0
X-MS-Exchange-CrossTenant-Network-Message-Id: 1a9267b0-ccbd-4e06-f3f0-08d90a5b9b4d
X-MS-Exchange-CrossTenant-originalarrivaltime: 28 Apr 2021 15:36:32.3218
 (UTC)
X-MS-Exchange-CrossTenant-fromentityheader: Hosted
X-MS-Exchange-CrossTenant-id: 4e2c6054-71cb-48f1-bd6c-3a9705aca71b
X-MS-Exchange-CrossTenant-mailboxtype: HOSTED
X-MS-Exchange-CrossTenant-userprincipalname: Q4aYUPOJw8l+rW0cfbLp0J4wLdkk/Us6dtYd6deMBE6PkJCx+/u3Di/IQBQXM4QrG4QKBi91vNuaFa7/nQ693Q==
X-MS-Exchange-Transport-CrossTenantHeadersStamped: CO1PR10MB4548
X-Proofpoint-Virus-Version: vendor=nai engine=6200 definitions=9968 signatures=668683
X-Proofpoint-Spam-Details: rule=notspam policy=default score=0 phishscore=0 spamscore=0 adultscore=0
 suspectscore=0 mlxscore=0 mlxlogscore=999 malwarescore=0 bulkscore=0
 classifier=spam adjust=0 reason=mlx scancount=1 engine=8.12.0-2104060000
 definitions=main-2104280101
X-Proofpoint-ORIG-GUID: 1v08-ndGKnGx4cwHLi2DfbKECeWRkOsE
X-Proofpoint-GUID: 1v08-ndGKnGx4cwHLi2DfbKECeWRkOsE
Precedence: bulk
List-ID: <linux-kernel.vger.kernel.org>
X-Mailing-List: linux-kernel@vger.kernel.org

Use the maple state when splitting a node to not have to rewalk/reset the s=
tate on splits.
This is also needed to clean the locks up

Signed-off-by: Liam R. Howlett <Liam.Howlett@Oracle.com>
---
 mm/mmap.c | 185 +++++++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 175 insertions(+), 10 deletions(-)

diff --git a/mm/mmap.c b/mm/mmap.c
index ae1ffe726405..5335bd72bda3 100644
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -2182,6 +2182,178 @@ static void unmap_region(struct mm_struct *mm,
 		      max);
 	tlb_finish_mmu(&tlb);
 }
+
+/*
+ *
+ * Does not support inserting a new vma and modifying the other side of th=
e vma
+ * mas will point to insert or the new zeroed area.
+ */
+static inline
+int vma_shrink(struct ma_state *mas, struct vm_area_struct *vma,
+	       unsigned long start, unsigned long end, pgoff_t pgoff,
+	       struct vm_area_struct *insert)
+{
+	struct mm_struct *mm =3D vma->vm_mm;
+	struct address_space *mapping =3D NULL;
+	struct rb_root_cached *root =3D NULL;
+	struct anon_vma *anon_vma =3D NULL;
+	struct file *file =3D vma->vm_file;
+	unsigned long old_end =3D vma->vm_end, old_start =3D vma->vm_start;
+
+	validate_mm(mm);
+	vma_adjust_trans_huge(vma, start, end, 0);
+	if (file) {
+		mapping =3D file->f_mapping;
+		root =3D &mapping->i_mmap;
+		uprobe_munmap(vma, vma->vm_start, vma->vm_end);
+
+		i_mmap_lock_write(mapping);
+		/*
+		 * Put into interval tree now, so instantiated pages are visible
+		 * to arm/parisc __flush_dcache_page throughout; but we cannot
+		 * insert into address space until vma start or end is updated.
+		 */
+
+		if (insert)
+			__vma_link_file(insert);
+	}
+
+	anon_vma =3D vma->anon_vma;
+	if (anon_vma) {
+		anon_vma_lock_write(anon_vma);
+		anon_vma_interval_tree_pre_update_vma(vma);
+	}
+
+	if (file) {
+		flush_dcache_mmap_lock(mapping);
+		vma_interval_tree_remove(vma, root);
+	}
+
+	vma->vm_start =3D start;
+	vma->vm_end =3D end;
+	vma->vm_pgoff =3D pgoff;
+	if (!insert) {
+
+		/* If vm_start changed, and the insert does not end at the old
+		 * start, then that area needs to be zeroed
+		 */
+		if (old_start !=3D vma->vm_start) {
+			mas->last =3D end;
+			mas_store_gfp(mas, NULL, GFP_KERNEL);
+		}
+
+		/* If vm_end changed, and the insert does not start at the new
+		 * end, then that area needs to be zeroed
+		 */
+		if (old_end !=3D vma->vm_end) {
+			mas->index =3D end;
+			mas->last =3D old_end;
+			mas_store_gfp(mas, NULL, GFP_KERNEL);
+		}
+	}
+
+	if (file) {
+		vma_interval_tree_insert(vma, root);
+		flush_dcache_mmap_unlock(mapping);
+	}
+
+	if (insert) {  // Insert.
+		vma_mas_store(insert, mas);
+		mm->map_count++;
+	}
+
+	if (anon_vma) {
+		anon_vma_interval_tree_post_update_vma(vma);
+		anon_vma_unlock_write(anon_vma);
+	}
+
+	if (file) {
+		i_mmap_unlock_write(mapping);
+		uprobe_mmap(vma);
+		if (insert)
+			uprobe_mmap(insert);
+	}
+
+	validate_mm(mm);
+	return 0;
+}
+
+/*
+ * mas_split_vma() - Split the VMA into two.
+ *
+ * @mm: The mm_struct
+ * @mas: The maple state - must point to the vma being altered
+ * @vma: The vma to split
+ * @addr: The address to split @vma
+ * @new_below: Add the new vma at the lower address (first part) of vma.
+ *
+ * Note: The @mas must point to the vma that is being split or MAS_START.
+ * Upon return, @mas points to the new VMA.  sysctl_max_map_count is not
+ * checked.
+ */
+int mas_split_vma(struct mm_struct *mm, struct ma_state *mas,
+		  struct vm_area_struct *vma, unsigned long addr, int new_below)
+{
+	struct vm_area_struct *new;
+	int err;
+
+	validate_mm(mm);
+	if (vma->vm_ops && vma->vm_ops->may_split) {
+		err =3D vma->vm_ops->may_split(vma, addr);
+		if (err)
+			return err;
+	}
+
+	new =3D vm_area_dup(vma);
+	if (!new)
+		return -ENOMEM;
+
+	if (new_below)
+		new->vm_end =3D addr;
+	else {
+		new->vm_start =3D addr;
+		new->vm_pgoff +=3D ((addr - vma->vm_start) >> PAGE_SHIFT);
+	}
+
+	err =3D vma_dup_policy(vma, new);
+	if (err)
+		goto out_free_vma;
+
+	err =3D anon_vma_clone(new, vma);
+	if (err)
+		goto out_free_mpol;
+
+	if (new->vm_file)
+		get_file(new->vm_file);
+
+	if (new->vm_ops && new->vm_ops->open)
+		new->vm_ops->open(new);
+
+	if (new_below)
+		err =3D vma_shrink(mas, vma, addr, vma->vm_end, vma->vm_pgoff +
+			((addr - new->vm_start) >> PAGE_SHIFT), new);
+	else
+		err =3D vma_shrink(mas, vma, vma->vm_start, addr, vma->vm_pgoff,
+				 new);
+
+	validate_mm(mm);
+	/* Success. */
+	if (!err)
+		return 0;
+
+	/* Clean everything up if vma_adjust failed. */
+	if (new->vm_ops && new->vm_ops->close)
+		new->vm_ops->close(new);
+	if (new->vm_file)
+		fput(new->vm_file);
+	unlink_anon_vmas(new);
+ out_free_mpol:
+	mpol_put(vma_policy(new));
+ out_free_vma:
+	vm_area_free(new);
+	return err;
+}
+
 /*
  * __split_vma() bypasses sysctl_max_map_count checking.  We use this wher=
e it
  * has already been checked or doesn't make sense to fail.
@@ -2330,12 +2502,11 @@ static int do_mas_align_munmap(struct ma_state *mas=
, struct vm_area_struct *vma,
 		if (end < vma->vm_end && mm->map_count >=3D sysctl_max_map_count)
 			return -ENOMEM;
=20
-		error =3D __split_vma(mm, vma, start, 0);
+		error =3D mas_split_vma(mm, mas, vma, start, 0);
 		if (error)
 			return error;
=20
 		prev =3D vma;
-		mas_set_range(mas, start, end - 1);
 		vma =3D mas_walk(mas);
=20
 	} else {
@@ -2353,11 +2524,10 @@ static int do_mas_align_munmap(struct ma_state *mas=
, struct vm_area_struct *vma,
 	/* Does it split the last one? */
 	if (last && end < last->vm_end) {
 		int error;
-		error =3D __split_vma(mm, last, end, 1);
+		error =3D mas_split_vma(mm, mas, last, end, 1);
 		if (error)
 			return error;
-		mas_set(mas, end - 1);
-		last =3D mas_walk(mas);
+		validate_mm(mm);
 	}
 	next =3D mas_next(mas, ULONG_MAX);
=20
@@ -2518,11 +2688,8 @@ unsigned long mmap_region(struct file *file, unsigne=
d long addr,
 		vm_flags |=3D VM_ACCOUNT;
 	}
=20
-	mas_set_range(&mas, addr, end - 1);
-	mas_walk(&mas);  // Walk to the empty area (munmapped above)
 	ma_prev =3D mas;
 	prev =3D mas_prev(&ma_prev, 0);
-
 	if (vm_flags & VM_SPECIAL)
 		goto cannot_expand;
=20
@@ -2694,10 +2861,8 @@ unsigned long mmap_region(struct file *file, unsigne=
d long addr,
 	 * a completely new data area).
 	 */
 	vma->vm_flags |=3D VM_SOFTDIRTY;
-
 	vma_set_page_prot(vma);
 	validate_mm(mm);
-
 	return addr;
=20
 unmap_and_free_vma:
--=20
2.30.2