This series is based on next-20220407.
This series amis to add hugetlb_optimize_vmemmap sysctl to enable or disable
the feature of optimizing vmemmap pages associated with HugeTLB pages.
v8:
- Fix compilation (scripts/selinux/mdp/mdp.c) error when
CONFIG_SECURITY_SELINUX is selected.
v7:
- Fix circular dependency issue reported by kernel test robot.
- Introduce CONFIG_HUGETLB_PAGE_HAS_OPTIMIZE_VMEMMAP instead of
STRUCT_PAGE_SIZE_IS_POWER_OF_2.
- Add more comments into vm.rst to explain hugetlb_optimize_vmemmap (Andrew).
- Drop the patch "sysctl: allow to set extra1 to SYSCTL_ONE".
- Add a new patch "use kstrtobool for hugetlb_vmemmap param parsing".
- Reuse static_key's refcount to count the number of HugeTLB pages with
vmemmap pages optimized to simplify the lock scheme.
v6:
- Remove "make syncconfig" from Kbuild.
v5:
- Fix not working properly if one is workig off of a very clean build
reported by Luis Chamberlain.
- Add Suggested-by for Luis Chamberlain.
v4:
- Introduce STRUCT_PAGE_SIZE_IS_POWER_OF_2 inspired by Luis.
v3:
- Add pr_warn_once() (Mike).
- Handle the transition from enabling to disabling (Luis)
v2:
- Fix compilation when !CONFIG_MHP_MEMMAP_ON_MEMORY reported by kernel
test robot <[email protected]>.
- Move sysctl code from kernel/sysctl.c to mm/hugetlb_vmemmap.c.
Muchun Song (4):
mm: hugetlb_vmemmap: introduce
CONFIG_HUGETLB_PAGE_HAS_OPTIMIZE_VMEMMAP
mm: memory_hotplug: override memmap_on_memory when
hugetlb_free_vmemmap=on
mm: hugetlb_vmemmap: use kstrtobool for hugetlb_vmemmap param parsing
mm: hugetlb_vmemmap: add hugetlb_optimize_vmemmap sysctl
Documentation/admin-guide/kernel-parameters.txt | 6 +-
Documentation/admin-guide/sysctl/vm.rst | 27 +++++++
Kbuild | 19 +++++
arch/x86/mm/init_64.c | 2 +-
include/linux/hugetlb.h | 2 +-
include/linux/kconfig.h | 4 +
include/linux/memory_hotplug.h | 9 +++
include/linux/mm.h | 2 +-
include/linux/page-flags.h | 2 +-
kernel/autoconf_ext.c | 26 ++++++
mm/hugetlb_vmemmap.c | 102 ++++++++++++++++++++----
mm/hugetlb_vmemmap.h | 8 +-
mm/memory_hotplug.c | 27 +++++--
mm/sparse-vmemmap.c | 4 +-
scripts/mod/Makefile | 2 +
15 files changed, 207 insertions(+), 35 deletions(-)
create mode 100644 kernel/autoconf_ext.c
--
2.11.0
When "hugetlb_free_vmemmap=on" and "memory_hotplug.memmap_on_memory"
are both passed to boot cmdline, the variable of "memmap_on_memory"
will be set to 1 even if the vmemmap pages will not be allocated from
the hotadded memory since the former takes precedence over the latter.
In the next patch, we want to enable or disable the feature of freeing
vmemmap pages of HugeTLB via sysctl. We need a way to know if the
feature of memory_hotplug.memmap_on_memory is enabled when enabling
the feature of freeing vmemmap pages since those two features are not
compatible, however, the variable of "memmap_on_memory" cannot indicate
this nowadays. Do not set "memmap_on_memory" to 1 when both parameters
are passed to cmdline, in this case, "memmap_on_memory" could indicate
if this feature is enabled by the users.
Also introduce mhp_memmap_on_memory() helper to move the definition of
"memmap_on_memory" to the scope of CONFIG_MHP_MEMMAP_ON_MEMORY. In the
next patch, mhp_memmap_on_memory() will also be exported to be used in
hugetlb_vmemmap.c.
Signed-off-by: Muchun Song <[email protected]>
---
mm/memory_hotplug.c | 32 ++++++++++++++++++++++++++------
1 file changed, 26 insertions(+), 6 deletions(-)
diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index 74430f88853d..f6eab03397d3 100644
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -42,14 +42,36 @@
#include "internal.h"
#include "shuffle.h"
+#ifdef CONFIG_MHP_MEMMAP_ON_MEMORY
+static int memmap_on_memory_set(const char *val, const struct kernel_param *kp)
+{
+ if (hugetlb_optimize_vmemmap_enabled())
+ return 0;
+ return param_set_bool(val, kp);
+}
+
+static const struct kernel_param_ops memmap_on_memory_ops = {
+ .flags = KERNEL_PARAM_OPS_FL_NOARG,
+ .set = memmap_on_memory_set,
+ .get = param_get_bool,
+};
/*
* memory_hotplug.memmap_on_memory parameter
*/
static bool memmap_on_memory __ro_after_init;
-#ifdef CONFIG_MHP_MEMMAP_ON_MEMORY
-module_param(memmap_on_memory, bool, 0444);
+module_param_cb(memmap_on_memory, &memmap_on_memory_ops, &memmap_on_memory, 0444);
MODULE_PARM_DESC(memmap_on_memory, "Enable memmap on memory for memory hotplug");
+
+static inline bool mhp_memmap_on_memory(void)
+{
+ return memmap_on_memory;
+}
+#else
+static inline bool mhp_memmap_on_memory(void)
+{
+ return false;
+}
#endif
enum {
@@ -1272,9 +1294,7 @@ bool mhp_supports_memmap_on_memory(unsigned long size)
* altmap as an alternative source of memory, and we do not exactly
* populate a single PMD.
*/
- return memmap_on_memory &&
- !hugetlb_optimize_vmemmap_enabled() &&
- IS_ENABLED(CONFIG_MHP_MEMMAP_ON_MEMORY) &&
+ return mhp_memmap_on_memory() &&
size == memory_block_size_bytes() &&
IS_ALIGNED(vmemmap_size, PMD_SIZE) &&
IS_ALIGNED(remaining_size, (pageblock_nr_pages << PAGE_SHIFT));
@@ -2081,7 +2101,7 @@ static int __ref try_remove_memory(u64 start, u64 size)
* We only support removing memory added with MHP_MEMMAP_ON_MEMORY in
* the same granularity it was added - a single memory block.
*/
- if (memmap_on_memory) {
+ if (mhp_memmap_on_memory()) {
nr_vmemmap_pages = walk_memory_blocks(start, size, NULL,
get_nr_vmemmap_pages_cb);
if (nr_vmemmap_pages) {
--
2.11.0
We must add hugetlb_free_vmemmap=on (or "off") to the boot cmdline and
reboot the server to enable or disable the feature of optimizing vmemmap
pages associated with HugeTLB pages. However, rebooting usually takes a
long time. So add a sysctl to enable or disable the feature at runtime
without rebooting.
Once enabled, the vmemmap pages of subsequent allocation of HugeTLB pages
from buddy allocator will be optimized (7 pages per 2MB HugeTLB page and
4095 pages per 1GB HugeTLB page), whereas already allocated HugeTLB pages
will not be optimized. When those optimized HugeTLB pages are freed from
the HugeTLB pool to the buddy allocator, the vmemmap pages representing
that range needs to be remapped again and the vmemmap pages discarded
earlier need to be rellocated again. If your use case is that HugeTLB
pages are allocated 'on the fly' instead of being pulled from the HugeTLB
pool, you should weigh the benefits of memory savings against the more
overhead of allocation or freeing HugeTLB pages between the HugeTLB pool
and the buddy allocator. Another behavior to note is that if the system
is under heavy memory pressure, it could prevent the user from freeing
HugeTLB pages from the HugeTLB pool to the buddy allocator since the
allocation of vmemmap pages could be failed, you have to retry later if
your system encounter this situation.
Once disabled, the vmemmap pages of subsequent allocation of HugeTLB
pages from buddy allocator will not be optimized, whereas already
optimized HugeTLB pages will not be affected. If you want to make sure
there is no optimized HugeTLB pages, you can set "nr_hugepages" to 0
first and then disable this.
Signed-off-by: Muchun Song <[email protected]>
---
Documentation/admin-guide/sysctl/vm.rst | 27 ++++++++++
include/linux/memory_hotplug.h | 9 ++++
mm/hugetlb_vmemmap.c | 90 +++++++++++++++++++++++++++++----
mm/hugetlb_vmemmap.h | 4 +-
mm/memory_hotplug.c | 7 +--
5 files changed, 121 insertions(+), 16 deletions(-)
diff --git a/Documentation/admin-guide/sysctl/vm.rst b/Documentation/admin-guide/sysctl/vm.rst
index 747e325ebcd0..5c804ada85c5 100644
--- a/Documentation/admin-guide/sysctl/vm.rst
+++ b/Documentation/admin-guide/sysctl/vm.rst
@@ -562,6 +562,33 @@ Change the minimum size of the hugepage pool.
See Documentation/admin-guide/mm/hugetlbpage.rst
+hugetlb_optimize_vmemmap
+========================
+
+Enable (set to 1) or disable (set to 0) the feature of optimizing vmemmap pages
+associated with each HugeTLB page.
+
+Once enabled, the vmemmap pages of subsequent allocation of HugeTLB pages from
+buddy allocator will be optimized (7 pages per 2MB HugeTLB page and 4095 pages
+per 1GB HugeTLB page), whereas already allocated HugeTLB pages will not be
+optimized. When those optimized HugeTLB pages are freed from the HugeTLB pool
+to the buddy allocator, the vmemmap pages representing that range needs to be
+remapped again and the vmemmap pages discarded earlier need to be rellocated
+again. If your use case is that HugeTLB pages are allocated 'on the fly'
+instead of being pulled from the HugeTLB pool, you should weigh the benefits of
+memory savings against the more overhead of allocation or freeing HugeTLB pages
+between the HugeTLB pool and the buddy allocator. Another behavior to note is
+that if the system is under heavy memory pressure, it could prevent the user
+from freeing HugeTLB pages from the HugeTLB pool to the buddy allocator since
+the allocation of vmemmap pages could be failed, you have to retry later if
+your system encounter this situation.
+
+Once disabled, the vmemmap pages of subsequent allocation of HugeTLB pages from
+buddy allocator will not be optimized, whereas already optimized HugeTLB pages
+will not be affected. If you want to make sure there is no optimized HugeTLB
+pages, you can set "nr_hugepages" to 0 first and then disable this.
+
+
nr_hugepages_mempolicy
======================
diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h
index 7ab15d6fb227..93f2cbea0f9b 100644
--- a/include/linux/memory_hotplug.h
+++ b/include/linux/memory_hotplug.h
@@ -348,4 +348,13 @@ void arch_remove_linear_mapping(u64 start, u64 size);
extern bool mhp_supports_memmap_on_memory(unsigned long size);
#endif /* CONFIG_MEMORY_HOTPLUG */
+#ifdef CONFIG_MHP_MEMMAP_ON_MEMORY
+bool mhp_memmap_on_memory(void);
+#else
+static inline bool mhp_memmap_on_memory(void)
+{
+ return false;
+}
+#endif
+
#endif /* __LINUX_MEMORY_HOTPLUG_H */
diff --git a/mm/hugetlb_vmemmap.c b/mm/hugetlb_vmemmap.c
index 4b6a5cf16f11..94571d63916d 100644
--- a/mm/hugetlb_vmemmap.c
+++ b/mm/hugetlb_vmemmap.c
@@ -176,6 +176,7 @@
*/
#define pr_fmt(fmt) "HugeTLB: " fmt
+#include <linux/memory_hotplug.h>
#include "hugetlb_vmemmap.h"
#ifdef CONFIG_HUGETLB_PAGE_HAS_OPTIMIZE_VMEMMAP
@@ -189,21 +190,40 @@
#define RESERVE_VMEMMAP_NR 1U
#define RESERVE_VMEMMAP_SIZE (RESERVE_VMEMMAP_NR << PAGE_SHIFT)
+enum vmemmap_optimize_mode {
+ VMEMMAP_OPTIMIZE_OFF,
+ VMEMMAP_OPTIMIZE_ON,
+};
+
DEFINE_STATIC_KEY_MAYBE(CONFIG_HUGETLB_PAGE_OPTIMIZE_VMEMMAP_DEFAULT_ON,
hugetlb_optimize_vmemmap_key);
EXPORT_SYMBOL(hugetlb_optimize_vmemmap_key);
+static enum vmemmap_optimize_mode vmemmap_optimize_mode =
+ IS_ENABLED(CONFIG_HUGETLB_PAGE_FREE_VMEMMAP_DEFAULT_ON);
+
+static void vmemmap_optimize_mode_switch(enum vmemmap_optimize_mode to)
+{
+ if (vmemmap_optimize_mode == to)
+ return;
+
+ if (to == VMEMMAP_OPTIMIZE_OFF)
+ static_branch_dec(&hugetlb_optimize_vmemmap_key);
+ else
+ static_branch_inc(&hugetlb_optimize_vmemmap_key);
+ vmemmap_optimize_mode = to;
+}
+
static int __init hugetlb_vmemmap_early_param(char *buf)
{
bool enable;
+ enum vmemmap_optimize_mode mode;
if (kstrtobool(buf, &enable))
return -EINVAL;
- if (enable)
- static_branch_enable(&hugetlb_optimize_vmemmap_key);
- else
- static_branch_disable(&hugetlb_optimize_vmemmap_key);
+ mode = enable ? VMEMMAP_OPTIMIZE_ON : VMEMMAP_OPTIMIZE_OFF;
+ vmemmap_optimize_mode_switch(mode);
return 0;
}
@@ -236,8 +256,10 @@ int hugetlb_vmemmap_alloc(struct hstate *h, struct page *head)
*/
ret = vmemmap_remap_alloc(vmemmap_addr, vmemmap_end, vmemmap_reuse,
GFP_KERNEL | __GFP_NORETRY | __GFP_THISNODE);
- if (!ret)
+ if (!ret) {
ClearHPageVmemmapOptimized(head);
+ static_branch_dec(&hugetlb_optimize_vmemmap_key);
+ }
return ret;
}
@@ -251,6 +273,8 @@ void hugetlb_vmemmap_free(struct hstate *h, struct page *head)
if (!vmemmap_pages)
return;
+ static_branch_inc(&hugetlb_optimize_vmemmap_key);
+
vmemmap_addr += RESERVE_VMEMMAP_SIZE;
vmemmap_end = vmemmap_addr + (vmemmap_pages << PAGE_SHIFT);
vmemmap_reuse = vmemmap_addr - PAGE_SIZE;
@@ -260,7 +284,9 @@ void hugetlb_vmemmap_free(struct hstate *h, struct page *head)
* to the page which @vmemmap_reuse is mapped to, then free the pages
* which the range [@vmemmap_addr, @vmemmap_end] is mapped to.
*/
- if (!vmemmap_remap_free(vmemmap_addr, vmemmap_end, vmemmap_reuse))
+ if (vmemmap_remap_free(vmemmap_addr, vmemmap_end, vmemmap_reuse))
+ static_branch_dec(&hugetlb_optimize_vmemmap_key);
+ else
SetHPageVmemmapOptimized(head);
}
@@ -277,9 +303,6 @@ void __init hugetlb_vmemmap_init(struct hstate *h)
BUILD_BUG_ON(__NR_USED_SUBPAGE >=
RESERVE_VMEMMAP_SIZE / sizeof(struct page));
- if (!hugetlb_optimize_vmemmap_enabled())
- return;
-
vmemmap_pages = (nr_pages * sizeof(struct page)) >> PAGE_SHIFT;
/*
* The head page is not to be freed to buddy allocator, the other tail
@@ -295,4 +318,53 @@ void __init hugetlb_vmemmap_init(struct hstate *h)
pr_info("can optimize %d vmemmap pages for %s\n",
h->optimize_vmemmap_pages, h->name);
}
+
+#ifdef CONFIG_PROC_SYSCTL
+static int hugetlb_optimize_vmemmap_handler(struct ctl_table *table, int write,
+ void *buffer, size_t *length,
+ loff_t *ppos)
+{
+ int ret;
+ enum vmemmap_optimize_mode mode;
+ static DEFINE_MUTEX(sysctl_mutex);
+
+ if (write && !capable(CAP_SYS_ADMIN))
+ return -EPERM;
+
+ mutex_lock(&sysctl_mutex);
+ mode = vmemmap_optimize_mode;
+ table->data = &mode;
+ ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
+ if (write && !ret)
+ vmemmap_optimize_mode_switch(mode);
+ mutex_unlock(&sysctl_mutex);
+
+ return ret;
+}
+
+static struct ctl_table hugetlb_vmemmap_sysctls[] = {
+ {
+ .procname = "hugetlb_optimize_vmemmap",
+ .maxlen = sizeof(enum vmemmap_optimize_mode),
+ .mode = 0644,
+ .proc_handler = hugetlb_optimize_vmemmap_handler,
+ .extra1 = SYSCTL_ZERO,
+ .extra2 = SYSCTL_ONE,
+ },
+ { }
+};
+
+static __init int hugetlb_vmemmap_sysctls_init(void)
+{
+ /*
+ * The vmemmap pages cannot be optimized if
+ * "memory_hotplug.memmap_on_memory" is enabled.
+ */
+ if (!mhp_memmap_on_memory())
+ register_sysctl_init("vm", hugetlb_vmemmap_sysctls);
+
+ return 0;
+}
+late_initcall(hugetlb_vmemmap_sysctls_init);
+#endif /* CONFIG_PROC_SYSCTL */
#endif /* CONFIG_HUGETLB_PAGE_HAS_OPTIMIZE_VMEMMAP */
diff --git a/mm/hugetlb_vmemmap.h b/mm/hugetlb_vmemmap.h
index 3afae3ff37fa..63ae2766ffe0 100644
--- a/mm/hugetlb_vmemmap.h
+++ b/mm/hugetlb_vmemmap.h
@@ -21,7 +21,9 @@ void hugetlb_vmemmap_init(struct hstate *h);
*/
static inline unsigned int hugetlb_optimize_vmemmap_pages(struct hstate *h)
{
- return h->optimize_vmemmap_pages;
+ if (hugetlb_optimize_vmemmap_enabled())
+ return h->optimize_vmemmap_pages;
+ return 0;
}
#else
static inline int hugetlb_vmemmap_alloc(struct hstate *h, struct page *head)
diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index f6eab03397d3..af844a01e956 100644
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -63,15 +63,10 @@ static bool memmap_on_memory __ro_after_init;
module_param_cb(memmap_on_memory, &memmap_on_memory_ops, &memmap_on_memory, 0444);
MODULE_PARM_DESC(memmap_on_memory, "Enable memmap on memory for memory hotplug");
-static inline bool mhp_memmap_on_memory(void)
+bool mhp_memmap_on_memory(void)
{
return memmap_on_memory;
}
-#else
-static inline bool mhp_memmap_on_memory(void)
-{
- return false;
-}
#endif
enum {
--
2.11.0