We must add "hugetlb_free_vmemmap=on" to boot cmdline and reboot the
server to enable the feature of freeing vmemmap pages of HugeTLB
pages. Rebooting usually taske a long time. Add a sysctl to enable
the feature at runtime and do not need to reboot.
Signed-off-by: Muchun Song <[email protected]>
---
Documentation/admin-guide/sysctl/vm.rst | 13 +++++++++++++
include/linux/hugetlb.h | 5 +++++
include/linux/memory_hotplug.h | 1 +
kernel/sysctl.c | 11 +++++++++++
mm/hugetlb_vmemmap.c | 23 +++++++++++++++++------
mm/hugetlb_vmemmap.h | 4 +++-
mm/memory_hotplug.c | 2 +-
7 files changed, 51 insertions(+), 8 deletions(-)
diff --git a/Documentation/admin-guide/sysctl/vm.rst b/Documentation/admin-guide/sysctl/vm.rst
index f4804ce37c58..01f18e6cc227 100644
--- a/Documentation/admin-guide/sysctl/vm.rst
+++ b/Documentation/admin-guide/sysctl/vm.rst
@@ -561,6 +561,19 @@ Change the minimum size of the hugepage pool.
See Documentation/admin-guide/mm/hugetlbpage.rst
+hugetlb_free_vmemmap
+====================
+
+A toggle value indicating if vmemmap pages are allowed to be optimized.
+If it is off (0), then it can be set true (1). Once true, the vmemmap
+pages associated with each HugeTLB page will be optimized, and the toggle
+cannot be set back to false. It only optimizes the subsequent allocation
+of HugeTLB pages from buddy system, while already allocated HugeTLB pages
+will not be optimized.
+
+See Documentation/admin-guide/mm/hugetlbpage.rst
+
+
nr_hugepages_mempolicy
======================
diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
index 53c1b6082a4c..cc4ab21892f5 100644
--- a/include/linux/hugetlb.h
+++ b/include/linux/hugetlb.h
@@ -1080,6 +1080,11 @@ static inline void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr
}
#endif /* CONFIG_HUGETLB_PAGE */
+#ifdef CONFIG_HUGETLB_PAGE_FREE_VMEMMAP
+int hugetlb_vmemmap_sysctl_handler(struct ctl_table *table, int write,
+ void *buffer, size_t *length, loff_t *ppos);
+#endif
+
static inline spinlock_t *huge_pte_lock(struct hstate *h,
struct mm_struct *mm, pte_t *pte)
{
diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h
index e0b2209ab71c..b30f9fdaed73 100644
--- a/include/linux/memory_hotplug.h
+++ b/include/linux/memory_hotplug.h
@@ -349,6 +349,7 @@ extern int arch_create_linear_mapping(int nid, u64 start, u64 size,
struct mhp_params *params);
void arch_remove_linear_mapping(u64 start, u64 size);
extern bool mhp_supports_memmap_on_memory(unsigned long size);
+extern bool memmap_on_memory;
#endif /* CONFIG_MEMORY_HOTPLUG */
#endif /* __LINUX_MEMORY_HOTPLUG_H */
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index ab3e9c937268..77f039849b2a 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -2223,6 +2223,17 @@ static struct ctl_table vm_table[] = {
.mode = 0644,
.proc_handler = hugetlb_sysctl_handler,
},
+#ifdef CONFIG_HUGETLB_PAGE_FREE_VMEMMAP
+ {
+ .procname = "hugetlb_free_vmemmap",
+ .data = &hugetlb_free_vmemmap_enabled_key.key,
+ .maxlen = sizeof(hugetlb_free_vmemmap_enabled_key.key),
+ .mode = 0644,
+ /* only handle a transition from default "0" to "1" */
+ .proc_handler = hugetlb_vmemmap_sysctl_handler,
+ .extra1 = SYSCTL_ONE,
+ },
+#endif
#ifdef CONFIG_NUMA
{
.procname = "nr_hugepages_mempolicy",
diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c
index 836d1117f08b..3167021055d6 100644
--- a/mm/hugetlb_vmemmap.c
+++ b/mm/hugetlb_vmemmap.c
@@ -10,6 +10,7 @@
#define pr_fmt(fmt) "HugeTLB: " fmt
+#include <linux/memory_hotplug.h>
#include "hugetlb_vmemmap.h"
/*
@@ -118,17 +119,14 @@ void __init hugetlb_vmemmap_init(struct hstate *h)
BUILD_BUG_ON(__NR_USED_SUBPAGE >=
RESERVE_VMEMMAP_SIZE / sizeof(struct page));
- if (!hugetlb_free_vmemmap_enabled())
- return;
-
- if (IS_ENABLED(CONFIG_HUGETLB_PAGE_FREE_VMEMMAP_DEFAULT_ON) &&
- !is_power_of_2(sizeof(struct page))) {
+ if (!is_power_of_2(sizeof(struct page))) {
/*
* The hugetlb_free_vmemmap_enabled_key can be enabled when
* CONFIG_HUGETLB_PAGE_FREE_VMEMMAP_DEFAULT_ON. It should
* be disabled if "struct page" crosses page boundaries.
*/
- static_branch_disable(&hugetlb_free_vmemmap_enabled_key);
+ if (IS_ENABLED(CONFIG_HUGETLB_PAGE_FREE_VMEMMAP_DEFAULT_ON))
+ static_branch_disable(&hugetlb_free_vmemmap_enabled_key);
return;
}
@@ -147,3 +145,16 @@ void __init hugetlb_vmemmap_init(struct hstate *h)
pr_info("can free %d vmemmap pages for %s\n", h->nr_free_vmemmap_pages,
h->name);
}
+
+int hugetlb_vmemmap_sysctl_handler(struct ctl_table *table, int write,
+ void *buffer, size_t *length, loff_t *ppos)
+{
+ /*
+ * The vmemmap pages cannot be optimized if a "struct page" crosses page
+ * boundaries or memory_hotplug.memmap_on_memory is enabled.
+ */
+ if (write && (!is_power_of_2(sizeof(struct page)) || memmap_on_memory))
+ return -EPERM;
+
+ return proc_do_static_key(table, write, buffer, length, ppos);
+}
diff --git a/mm/hugetlb_vmemmap.h b/mm/hugetlb_vmemmap.h
index cb2bef8f9e73..b67a159027f4 100644
--- a/mm/hugetlb_vmemmap.h
+++ b/mm/hugetlb_vmemmap.h
@@ -21,7 +21,9 @@ void hugetlb_vmemmap_init(struct hstate *h);
*/
static inline unsigned int free_vmemmap_pages_per_hpage(struct hstate *h)
{
- return h->nr_free_vmemmap_pages;
+ if (hugetlb_free_vmemmap_enabled())
+ return h->nr_free_vmemmap_pages;
+ return 0;
}
#else
static inline int alloc_huge_page_vmemmap(struct hstate *h, struct page *head)
diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index c226a337c1ef..b5cc5abde05a 100644
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -46,7 +46,7 @@
/*
* memory_hotplug.memmap_on_memory parameter
*/
-static bool memmap_on_memory __ro_after_init;
+bool memmap_on_memory __ro_after_init;
#ifdef CONFIG_MHP_MEMMAP_ON_MEMORY
module_param(memmap_on_memory, bool, 0444);
MODULE_PARM_DESC(memmap_on_memory, "Enable memmap on memory for memory hotplug");
--
2.11.0
On Mon, Feb 28, 2022 at 10:42 PM Luis Chamberlain <[email protected]> wrote:
>
> On Mon, Feb 28, 2022 at 03:10:22PM +0800, Muchun Song wrote:
> > We must add "hugetlb_free_vmemmap=on" to boot cmdline and reboot the
> > server to enable the feature of freeing vmemmap pages of HugeTLB
> > pages. Rebooting usually taske a long time. Add a sysctl to enable
> > the feature at runtime and do not need to reboot.
> >
> > Signed-off-by: Muchun Song <[email protected]>
> > ---
> > Documentation/admin-guide/sysctl/vm.rst | 13 +++++++++++++
> > include/linux/hugetlb.h | 5 +++++
> > include/linux/memory_hotplug.h | 1 +
> > kernel/sysctl.c | 11 +++++++++++
>
> kernel/sysctl.c is a hot mess with tons of knobs from all over the
> place. And so we've been moving these to their own place. For instance
> all the filesystem knobs are now properly in fs/. So Adding new ones
> to the file is undesirable. If this is going to be added, please add
> it somewhere in mm / hugetlb code.
>
I also have realized this when reading another patch [1]. I will
move this into its hugetlb related code. Anyway, thanks for
your suggestion.
[1] https://lore.kernel.org/linux-fsdevel/[email protected]/
On Mon, Feb 28, 2022 at 03:10:22PM +0800, Muchun Song wrote:
> We must add "hugetlb_free_vmemmap=on" to boot cmdline and reboot the
> server to enable the feature of freeing vmemmap pages of HugeTLB
> pages. Rebooting usually taske a long time. Add a sysctl to enable
> the feature at runtime and do not need to reboot.
>
> Signed-off-by: Muchun Song <[email protected]>
> ---
> Documentation/admin-guide/sysctl/vm.rst | 13 +++++++++++++
> include/linux/hugetlb.h | 5 +++++
> include/linux/memory_hotplug.h | 1 +
> kernel/sysctl.c | 11 +++++++++++
kernel/sysctl.c is a hot mess with tons of knobs from all over the
place. And so we've been moving these to their own place. For instance
all the filesystem knobs are now properly in fs/. So Adding new ones
to the file is undesirable. If this is going to be added, please add
it somewhere in mm / hugetlb code.
Luis
Hi Muchun,
Thank you for the patch! Yet something to improve:
[auto build test ERROR on hnaz-mm/master]
[also build test ERROR on next-20220225]
[cannot apply to mcgrof/sysctl-next linus/master v5.17-rc6]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]
url: https://github.com/0day-ci/linux/commits/Muchun-Song/add-hugetlb_free_vmemmap-sysctl/20220228-151303
base: https://github.com/hnaz/linux-mm master
config: x86_64-randconfig-c022 (https://download.01.org/0day-ci/archive/20220301/[email protected]/config)
compiler: gcc-9 (Debian 9.3.0-22) 9.3.0
reproduce (this is a W=1 build):
# https://github.com/0day-ci/linux/commit/a2a3193fd949ff24ed0c138ef495e67373839483
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Muchun-Song/add-hugetlb_free_vmemmap-sysctl/20220228-151303
git checkout a2a3193fd949ff24ed0c138ef495e67373839483
# save the config file to linux build tree
mkdir build_dir
make W=1 O=build_dir ARCH=x86_64 SHELL=/bin/bash
If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <[email protected]>
All errors (new ones prefixed by >>):
mm/hugetlb_vmemmap.c: In function 'hugetlb_vmemmap_sysctl_handler':
>> mm/hugetlb_vmemmap.c:156:55: error: 'memmap_on_memory' undeclared (first use in this function)
156 | if (write && (!is_power_of_2(sizeof(struct page)) || memmap_on_memory))
| ^~~~~~~~~~~~~~~~
mm/hugetlb_vmemmap.c:156:55: note: each undeclared identifier is reported only once for each function it appears in
vim +/memmap_on_memory +156 mm/hugetlb_vmemmap.c
148
149 int hugetlb_vmemmap_sysctl_handler(struct ctl_table *table, int write,
150 void *buffer, size_t *length, loff_t *ppos)
151 {
152 /*
153 * The vmemmap pages cannot be optimized if a "struct page" crosses page
154 * boundaries or memory_hotplug.memmap_on_memory is enabled.
155 */
> 156 if (write && (!is_power_of_2(sizeof(struct page)) || memmap_on_memory))
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/[email protected]
Hi Muchun,
Thank you for the patch! Yet something to improve:
[auto build test ERROR on hnaz-mm/master]
[also build test ERROR on next-20220225]
[cannot apply to mcgrof/sysctl-next linus/master v5.17-rc6]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]
url: https://github.com/0day-ci/linux/commits/Muchun-Song/add-hugetlb_free_vmemmap-sysctl/20220228-151303
base: https://github.com/hnaz/linux-mm master
config: x86_64-randconfig-a001 (https://download.01.org/0day-ci/archive/20220228/[email protected]/config)
compiler: clang version 15.0.0 (https://github.com/llvm/llvm-project d271fc04d5b97b12e6b797c6067d3c96a8d7470e)
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# https://github.com/0day-ci/linux/commit/a2a3193fd949ff24ed0c138ef495e67373839483
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Muchun-Song/add-hugetlb_free_vmemmap-sysctl/20220228-151303
git checkout a2a3193fd949ff24ed0c138ef495e67373839483
# save the config file to linux build tree
mkdir build_dir
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross W=1 O=build_dir ARCH=x86_64 SHELL=/bin/bash
If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <[email protected]>
All errors (new ones prefixed by >>):
>> mm/hugetlb_vmemmap.c:156:55: error: use of undeclared identifier 'memmap_on_memory'
if (write && (!is_power_of_2(sizeof(struct page)) || memmap_on_memory))
^
1 error generated.
vim +/memmap_on_memory +156 mm/hugetlb_vmemmap.c
148
149 int hugetlb_vmemmap_sysctl_handler(struct ctl_table *table, int write,
150 void *buffer, size_t *length, loff_t *ppos)
151 {
152 /*
153 * The vmemmap pages cannot be optimized if a "struct page" crosses page
154 * boundaries or memory_hotplug.memmap_on_memory is enabled.
155 */
> 156 if (write && (!is_power_of_2(sizeof(struct page)) || memmap_on_memory))
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/[email protected]