Received-SPF: pass (google.com: domain of linux-kernel-owner@vger.kernel.org designates 2620:137:e000::1:20 as permitted sender) client-ip=2620:137:e000::1:20;
Message-ID: <20220716230953.738288030@linutronix.de>
From:   Thomas Gleixner <tglx@linutronix.de>
To:     LKML <linux-kernel@vger.kernel.org>
Cc:     x86@kernel.org, Linus Torvalds <torvalds@linux-foundation.org>,
        Tim Chen <tim.c.chen@linux.intel.com>,
        Josh Poimboeuf <jpoimboe@kernel.org>,
        Andrew Cooper <Andrew.Cooper3@citrix.com>,
        Pawan Gupta <pawan.kumar.gupta@linux.intel.com>,
        Johannes Wikner <kwikner@ethz.ch>,
        Alyssa Milburn <alyssa.milburn@linux.intel.com>,
        Jann Horn <jannh@google.com>, "H.J. Lu" <hjl.tools@gmail.com>,
        Joao Moreira <joao.moreira@intel.com>,
        Joseph Nuzman <joseph.nuzman@intel.com>,
        Steven Rostedt <rostedt@goodmis.org>,
        "Peter Zijlstra (Intel)" <peterz@infradead.org>
Subject: [patch 18/38] mm/vmalloc: Provide huge page mappings
References: <20220716230344.239749011@linutronix.de>
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Date:   Sun, 17 Jul 2022 01:17:38 +0200 (CEST)
Precedence: bulk

From: Peter Zijlstra <peterz@infradead.org>

Provide VM_HUGE_VMAP, which unconditionally tries to use huge
mappings. Unlike VM_ALLOW_HUGE_VMAP it doesn't care about the number
of NUMA nodes or the size of the allocation.

If the page allocator fails to provide huge pages, it will silently
fall back.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
---
 include/linux/vmalloc.h |    3 ++-
 mm/vmalloc.c            |   33 +++++++++++++++++++--------------
 2 files changed, 21 insertions(+), 15 deletions(-)

--- a/include/linux/vmalloc.h
+++ b/include/linux/vmalloc.h
@@ -27,10 +27,11 @@ struct notifier_block;		/* in notifier.h
 #define VM_FLUSH_RESET_PERMS	0x00000100	/* reset direct map and flush TLB on unmap, can't be freed in atomic context */
 #define VM_MAP_PUT_PAGES	0x00000200	/* put pages and free array in vfree */
 #define VM_ALLOW_HUGE_VMAP	0x00000400      /* Allow for huge pages on archs with HAVE_ARCH_HUGE_VMALLOC */
+#define VM_HUGE_VMAP		0x00000800      /* Force for huge pages on archs with HAVE_ARCH_HUGE_VMALLOC */
 
 #if (defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)) && \
 	!defined(CONFIG_KASAN_VMALLOC)
-#define VM_DEFER_KMEMLEAK	0x00000800	/* defer kmemleak object creation */
+#define VM_DEFER_KMEMLEAK	0x00001000	/* defer kmemleak object creation */
 #else
 #define VM_DEFER_KMEMLEAK	0
 #endif
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -3099,23 +3099,28 @@ void *__vmalloc_node_range(unsigned long
 		return NULL;
 	}
 
-	if (vmap_allow_huge && (vm_flags & VM_ALLOW_HUGE_VMAP)) {
-		unsigned long size_per_node;
+	if (vmap_allow_huge && (vm_flags & (VM_HUGE_VMAP|VM_ALLOW_HUGE_VMAP))) {
 
-		/*
-		 * Try huge pages. Only try for PAGE_KERNEL allocations,
-		 * others like modules don't yet expect huge pages in
-		 * their allocations due to apply_to_page_range not
-		 * supporting them.
-		 */
+		if (vm_flags & VM_ALLOW_HUGE_VMAP) {
+			unsigned long size_per_node;
 
-		size_per_node = size;
-		if (node == NUMA_NO_NODE)
-			size_per_node /= num_online_nodes();
-		if (arch_vmap_pmd_supported(prot) && size_per_node >= PMD_SIZE)
+			/*
+			 * Try huge pages. Only try for PAGE_KERNEL allocations,
+			 * others like modules don't yet expect huge pages in
+			 * their allocations due to apply_to_page_range not
+			 * supporting them.
+			 */
+
+			size_per_node = size;
+			if (node == NUMA_NO_NODE)
+				size_per_node /= num_online_nodes();
+			if (arch_vmap_pmd_supported(prot) && size_per_node >= PMD_SIZE)
+				shift = PMD_SHIFT;
+			else
+				shift = arch_vmap_pte_supported_shift(size_per_node);
+		} else {
 			shift = PMD_SHIFT;
-		else
-			shift = arch_vmap_pte_supported_shift(size_per_node);
+		}
 
 		align = max(real_align, 1UL << shift);
 		size = ALIGN(real_size, 1UL << shift);