Received-SPF: pass (google.com: domain of linux-kernel-owner@vger.kernel.org designates 2620:137:e000::1:20 as permitted sender) client-ip=2620:137:e000::1:20;
From:   Alexey Romanov <avromanov@sberdevices.ru>
To:     <minchan@kernel.org>, <senozhatsky@chromium.org>,
        <akpm@linux-foundation.org>
CC:     <linux-mm@kvack.org>, <linux-kernel@vger.kernel.org>,
        <kernel@sberdevices.ru>, Alexey Romanov <avromanov@sberdevices.ru>
Subject: [PATCH v1 1/2] zsmalloc: add allocated objects counter for subpage
Date:   Mon, 19 Jun 2023 17:35:05 +0300
Message-ID: <20230619143506.45253-2-avromanov@sberdevices.ru>
In-Reply-To: <20230619143506.45253-1-avromanov@sberdevices.ru>
References: <20230619143506.45253-1-avromanov@sberdevices.ru>
MIME-Version: 1.0
Content-Transfer-Encoding: 8bit
Content-Type: text/plain
Precedence: bulk

We use a variable of type unsigned int to store the offset
of the first object at the subpage In turn, the offset cannot
exceed the size of PAGE_SIZE, which is usually 4096. Thus,
12 bits are enough to store the offset.

We can use the remaining bytes to store, for example, the
count of allocated objects on a subpage. If the page size is
4Kb, then no more than 128 (4096 / 32) objects can be allocated
on the subpage, which means that one byte is enough to store
the counter of allocated objects.

This patch adds support for counting the number of allocated
objects on a subpage in the first byte of the page_type field.
The offset of the first object is now stored in the remaining
bytes of this field.

The sum of allocated counter for all subpages is zspage->inuse.
We only count objects that have been tagged (I'm talking about
OBJ_ALLOCATED_TAG) on a subpage.

So, for example, in the situation:

            subpage 1                   subpage 2
[obj1_s - obj1_e, obj2_s - ] -> [obj2_e, obj3_s - obj3_e, free space]

Allocated counter for subpage 1 will be 2, and 1 for subpage 2.

Signed-off-by: Alexey Romanov <avromanov@sberdevices.ru>
---
 mm/zsmalloc.c | 38 +++++++++++++++++++++++++++++++++++---
 1 file changed, 35 insertions(+), 3 deletions(-)

diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c
index c0d433541636..dd6e2c3429e0 100644
--- a/mm/zsmalloc.c
+++ b/mm/zsmalloc.c
@@ -20,7 +20,10 @@
  *	page->index: links together all component pages of a zspage
  *		For the huge page, this is always 0, so we use this field
  *		to store handle.
- *	page->page_type: first object offset in a subpage of zspage
+ *	page->page_type:
+ *		First byte: count of allocated objects (OBJ_ALLOCATED_TAG)
+ *		in a subpage of zspage.
+ *		Other bytes: first object offset in a subpage of zspage.
  *
  * Usage of struct page flags:
  *	PG_private: identifies the first component page
@@ -126,6 +129,9 @@
 #define OBJ_INDEX_BITS	(BITS_PER_LONG - _PFN_BITS - OBJ_TAG_BITS)
 #define OBJ_INDEX_MASK	((_AC(1, UL) << OBJ_INDEX_BITS) - 1)
 
+#define OBJ_ALLOCATED_BITS	(sizeof(u8) * BITS_PER_BYTE)
+#define OBJ_ALLOCATED_MASK	((1UL << OBJ_ALLOCATED_BITS) - 1)
+
 #define HUGE_BITS	1
 #define FULLNESS_BITS	4
 #define CLASS_BITS	8
@@ -520,14 +526,37 @@ static inline struct page *get_first_page(struct zspage *zspage)
 	return first_page;
 }
 
+static inline u8 get_obj_allocated(struct page *page)
+{
+	return page->page_type & OBJ_ALLOCATED_MASK;
+}
+
+static inline void set_obj_allocated(struct page *page, u8 value)
+{
+	page->page_type = (page->page_type & ~OBJ_ALLOCATED_MASK) | value;
+}
+
+static inline void mod_obj_allocated(struct page *page, s8 value)
+{
+	u8 inuse = get_obj_allocated(page);
+	/*
+	 * Overflow is not possible:
+	 * 1. Maximum number of objects allocated on a subpage is 128.
+	 * 2. We use this function only with value = 1 or -1.
+	 */
+	inuse += value;
+	set_obj_allocated(page, inuse);
+}
+
 static inline unsigned int get_first_obj_offset(struct page *page)
 {
-	return page->page_type;
+	return page->page_type >> OBJ_ALLOCATED_BITS;
 }
 
 static inline void set_first_obj_offset(struct page *page, unsigned int offset)
 {
-	page->page_type = offset;
+	page->page_type = (page->page_type & OBJ_ALLOCATED_MASK) |
+			  (offset << OBJ_ALLOCATED_BITS);
 }
 
 static inline unsigned int get_freeobj(struct zspage *zspage)
@@ -1126,6 +1155,7 @@ static struct zspage *alloc_zspage(struct zs_pool *pool,
 		}
 
 		inc_zone_page_state(page, NR_ZSPAGES);
+		set_obj_allocated(page, 0);
 		pages[i] = page;
 	}
 
@@ -1456,6 +1486,7 @@ static unsigned long obj_malloc(struct zs_pool *pool,
 
 	kunmap_atomic(vaddr);
 	mod_zspage_inuse(zspage, 1);
+	mod_obj_allocated(m_page, 1);
 
 	obj = location_to_obj(m_page, obj);
 
@@ -1576,6 +1607,7 @@ static void obj_free(int class_size, unsigned long obj, unsigned long *handle)
 
 	kunmap_atomic(vaddr);
 	mod_zspage_inuse(zspage, -1);
+	mod_obj_allocated(f_page, -1);
 }
 
 void zs_free(struct zs_pool *pool, unsigned long handle)
-- 
2.38.1