Add a function to early map kernel memory using huge pages.
For 512k pages, just use standard page table and map in using 512k
pages.
For 8M pages, create a hugepd table and populate the two PGD
entries with it.
This function can only be used to create page tables at startup. Once
the regular SLAB allocation functions replace memblock functions,
this function cannot allocate new pages anymore. However it can still
update existing mappings with new protections.
hugepd_none() macro is moved into asm/hugetlb.h to be usable outside
of mm/hugetlbpage.c
early_pte_alloc_kernel() is made visible.
_PAGE_HUGE flag is now displayed by ptdump.
Signed-off-by: Christophe Leroy <[email protected]>
---
arch/powerpc/include/asm/hugetlb.h | 2 +
.../include/asm/nohash/32/hugetlb-8xx.h | 5 ++
arch/powerpc/include/asm/pgtable.h | 2 +
arch/powerpc/mm/hugetlbpage.c | 2 -
arch/powerpc/mm/nohash/8xx.c | 52 +++++++++++++++++++
arch/powerpc/mm/pgtable_32.c | 2 +-
arch/powerpc/mm/ptdump/8xx.c | 5 ++
arch/powerpc/platforms/Kconfig.cputype | 1 +
8 files changed, 68 insertions(+), 3 deletions(-)
diff --git a/arch/powerpc/include/asm/hugetlb.h b/arch/powerpc/include/asm/hugetlb.h
index e4276af034e9..0572c5cd12f2 100644
--- a/arch/powerpc/include/asm/hugetlb.h
+++ b/arch/powerpc/include/asm/hugetlb.h
@@ -13,6 +13,8 @@
#include <asm/nohash/32/hugetlb-8xx.h>
#endif /* CONFIG_PPC_BOOK3S_64 */
+#define hugepd_none(hpd) (hpd_val(hpd) == 0)
+
extern bool hugetlb_disabled;
void hugetlbpage_init_default(void);
diff --git a/arch/powerpc/include/asm/nohash/32/hugetlb-8xx.h b/arch/powerpc/include/asm/nohash/32/hugetlb-8xx.h
index 1c7d4693a78e..e752a5807a59 100644
--- a/arch/powerpc/include/asm/nohash/32/hugetlb-8xx.h
+++ b/arch/powerpc/include/asm/nohash/32/hugetlb-8xx.h
@@ -35,6 +35,11 @@ static inline void hugepd_populate(hugepd_t *hpdp, pte_t *new, unsigned int pshi
*hpdp = __hugepd(__pa(new) | _PMD_USER | _PMD_PRESENT | _PMD_PAGE_8M);
}
+static inline void hugepd_populate_kernel(hugepd_t *hpdp, pte_t *new, unsigned int pshift)
+{
+ *hpdp = __hugepd(__pa(new) | _PMD_PRESENT | _PMD_PAGE_8M);
+}
+
static inline int check_and_get_huge_psize(int shift)
{
return shift_to_mmu_psize(shift);
diff --git a/arch/powerpc/include/asm/pgtable.h b/arch/powerpc/include/asm/pgtable.h
index b80bfd41828d..ffddb052068c 100644
--- a/arch/powerpc/include/asm/pgtable.h
+++ b/arch/powerpc/include/asm/pgtable.h
@@ -105,6 +105,8 @@ unsigned long vmalloc_to_phys(void *vmalloc_addr);
void pgtable_cache_add(unsigned int shift);
+pte_t *early_pte_alloc_kernel(pmd_t *pmdp, unsigned long va);
+
#if defined(CONFIG_STRICT_KERNEL_RWX) || defined(CONFIG_PPC32)
void mark_initmem_nx(void);
#else
diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c
index 243e90db400c..30d2d05d681d 100644
--- a/arch/powerpc/mm/hugetlbpage.c
+++ b/arch/powerpc/mm/hugetlbpage.c
@@ -28,8 +28,6 @@
bool hugetlb_disabled = false;
-#define hugepd_none(hpd) (hpd_val(hpd) == 0)
-
#define PTE_T_ORDER (__builtin_ffs(sizeof(pte_basic_t)) - \
__builtin_ffs(sizeof(void *)))
diff --git a/arch/powerpc/mm/nohash/8xx.c b/arch/powerpc/mm/nohash/8xx.c
index d9f205d9a654..81ddcd9554e1 100644
--- a/arch/powerpc/mm/nohash/8xx.c
+++ b/arch/powerpc/mm/nohash/8xx.c
@@ -9,8 +9,10 @@
#include <linux/memblock.h>
#include <linux/mmu_context.h>
+#include <linux/hugetlb.h>
#include <asm/fixmap.h>
#include <asm/code-patching.h>
+#include <asm/pgalloc.h>
#include <mm/mmu_decl.h>
@@ -54,6 +56,56 @@ unsigned long p_block_mapped(phys_addr_t pa)
return 0;
}
+static pte_t __init *early_hugepd_alloc_kernel(hugepd_t *pmdp, unsigned long va)
+{
+ if (hugepd_none(*pmdp)) {
+ pte_t *ptep = memblock_alloc(sizeof(pte_basic_t), SZ_4K);
+
+ if (!ptep)
+ return NULL;
+
+ hugepd_populate_kernel((hugepd_t *)pmdp, ptep, PAGE_SHIFT_8M);
+ hugepd_populate_kernel((hugepd_t *)pmdp + 1, ptep, PAGE_SHIFT_8M);
+ }
+ return hugepte_offset(*(hugepd_t *)pmdp, va, PGDIR_SHIFT);
+}
+
+static int __ref __early_map_kernel_hugepage(unsigned long va, phys_addr_t pa,
+ pgprot_t prot, int psize, bool new)
+{
+ pmd_t *pmdp = pmd_ptr_k(va);
+ pte_t *ptep;
+
+ if (WARN_ON(psize != MMU_PAGE_512K && psize != MMU_PAGE_8M))
+ return -EINVAL;
+
+ if (new) {
+ if (WARN_ON(slab_is_available()))
+ return -EINVAL;
+
+ if (psize == MMU_PAGE_512K)
+ ptep = early_pte_alloc_kernel(pmdp, va);
+ else
+ ptep = early_hugepd_alloc_kernel((hugepd_t *)pmdp, va);
+ } else {
+ if (psize == MMU_PAGE_512K)
+ ptep = pte_offset_kernel(pmdp, va);
+ else
+ ptep = hugepte_offset(*(hugepd_t *)pmdp, va, PGDIR_SHIFT);
+ }
+
+ if (WARN_ON(!ptep))
+ return -ENOMEM;
+
+ /* The PTE should never be already present */
+ if (new && WARN_ON(pte_present(*ptep) && pgprot_val(prot)))
+ return -EINVAL;
+
+ set_huge_pte_at(&init_mm, va, ptep, pte_mkhuge(pfn_pte(pa >> PAGE_SHIFT, prot)));
+
+ return 0;
+}
+
/*
* MMU_init_hw does the chip-specific initialization of the MMU hardware.
*/
diff --git a/arch/powerpc/mm/pgtable_32.c b/arch/powerpc/mm/pgtable_32.c
index bd0cb6e3573e..05902bbff8d6 100644
--- a/arch/powerpc/mm/pgtable_32.c
+++ b/arch/powerpc/mm/pgtable_32.c
@@ -61,7 +61,7 @@ static void __init *early_alloc_pgtable(unsigned long size)
return ptr;
}
-static pte_t __init *early_pte_alloc_kernel(pmd_t *pmdp, unsigned long va)
+pte_t __init *early_pte_alloc_kernel(pmd_t *pmdp, unsigned long va)
{
if (pmd_none(*pmdp)) {
pte_t *ptep = early_alloc_pgtable(PTE_FRAG_SIZE);
diff --git a/arch/powerpc/mm/ptdump/8xx.c b/arch/powerpc/mm/ptdump/8xx.c
index a3169677dced..b3185b32793d 100644
--- a/arch/powerpc/mm/ptdump/8xx.c
+++ b/arch/powerpc/mm/ptdump/8xx.c
@@ -11,6 +11,11 @@
static const struct flag_info flag_array[] = {
{
+ .mask = _PAGE_HUGE,
+ .val = _PAGE_HUGE,
+ .set = "h",
+ .clear = " ",
+ }, {
.mask = _PAGE_RO | _PAGE_NA,
.val = 0,
.set = "rw",
diff --git a/arch/powerpc/platforms/Kconfig.cputype b/arch/powerpc/platforms/Kconfig.cputype
index 6a50392df7b5..f4dfaf43930f 100644
--- a/arch/powerpc/platforms/Kconfig.cputype
+++ b/arch/powerpc/platforms/Kconfig.cputype
@@ -56,6 +56,7 @@ config PPC_8xx
select PPC_HAVE_KUEP
select PPC_HAVE_KUAP
select HAVE_ARCH_VMAP_STACK
+ select HUGETLB_PAGE
config 40x
bool "AMCC 40x"
--
2.25.0
Hi Christophe,
Thank you for the patch! Yet something to improve:
[auto build test ERROR on next-20200316]
[cannot apply to powerpc/next v5.6-rc6 v5.6-rc5 v5.6-rc4 v5.6-rc6]
[if your patch is applied to the wrong git tree, please drop us a note to help
improve the system. BTW, we also suggest to use '--base' option to specify the
base tree in git format-patch, please see https://stackoverflow.com/a/37406982]
url: https://github.com/0day-ci/linux/commits/Christophe-Leroy/Use-hugepages-to-map-kernel-mem-on-8xx/20200317-065610
base: 8548fd2f20ed19b0e8c0585b71fdfde1ae00ae3c
config: powerpc-tqm8xx_defconfig (attached as .config)
compiler: powerpc-linux-gcc (GCC) 9.2.0
reproduce:
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# save the attached .config to linux build tree
GCC_VERSION=9.2.0 make.cross ARCH=powerpc
If you fix the issue, kindly add following tag
Reported-by: kbuild test robot <[email protected]>
All errors (new ones prefixed by >>):
In file included from arch/powerpc/mm/fault.c:33:
include/linux/hugetlb.h: In function 'hstate_inode':
>> include/linux/hugetlb.h:522:9: error: implicit declaration of function 'HUGETLBFS_SB'; did you mean 'HUGETLBFS_MAGIC'? [-Werror=implicit-function-declaration]
522 | return HUGETLBFS_SB(i->i_sb)->hstate;
| ^~~~~~~~~~~~
| HUGETLBFS_MAGIC
>> include/linux/hugetlb.h:522:30: error: invalid type argument of '->' (have 'int')
522 | return HUGETLBFS_SB(i->i_sb)->hstate;
| ^~
cc1: all warnings being treated as errors
--
In file included from arch/powerpc/mm/mem.c:30:
include/linux/hugetlb.h: In function 'hstate_inode':
>> include/linux/hugetlb.h:522:9: error: implicit declaration of function 'HUGETLBFS_SB' [-Werror=implicit-function-declaration]
522 | return HUGETLBFS_SB(i->i_sb)->hstate;
| ^~~~~~~~~~~~
>> include/linux/hugetlb.h:522:30: error: invalid type argument of '->' (have 'int')
522 | return HUGETLBFS_SB(i->i_sb)->hstate;
| ^~
cc1: all warnings being treated as errors
--
In file included from arch/powerpc/mm/nohash/8xx.c:12:
include/linux/hugetlb.h: In function 'hstate_inode':
>> include/linux/hugetlb.h:522:9: error: implicit declaration of function 'HUGETLBFS_SB' [-Werror=implicit-function-declaration]
522 | return HUGETLBFS_SB(i->i_sb)->hstate;
| ^~~~~~~~~~~~
>> include/linux/hugetlb.h:522:30: error: invalid type argument of '->' (have 'int')
522 | return HUGETLBFS_SB(i->i_sb)->hstate;
| ^~
At top level:
arch/powerpc/mm/nohash/8xx.c:73:18: error: '__early_map_kernel_hugepage' defined but not used [-Werror=unused-function]
73 | static int __ref __early_map_kernel_hugepage(unsigned long va, phys_addr_t pa,
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~
cc1: all warnings being treated as errors
--
In file included from arch/powerpc//mm/nohash/8xx.c:12:
include/linux/hugetlb.h: In function 'hstate_inode':
>> include/linux/hugetlb.h:522:9: error: implicit declaration of function 'HUGETLBFS_SB' [-Werror=implicit-function-declaration]
522 | return HUGETLBFS_SB(i->i_sb)->hstate;
| ^~~~~~~~~~~~
>> include/linux/hugetlb.h:522:30: error: invalid type argument of '->' (have 'int')
522 | return HUGETLBFS_SB(i->i_sb)->hstate;
| ^~
At top level:
arch/powerpc//mm/nohash/8xx.c:73:18: error: '__early_map_kernel_hugepage' defined but not used [-Werror=unused-function]
73 | static int __ref __early_map_kernel_hugepage(unsigned long va, phys_addr_t pa,
| ^~~~~~~~~~~~~~~~~~~~~~~~~~~
cc1: all warnings being treated as errors
--
In file included from include/linux/migrate.h:8,
from kernel///sched/sched.h:53,
from kernel///sched/loadavg.c:9:
include/linux/hugetlb.h: In function 'hstate_inode':
>> include/linux/hugetlb.h:522:9: error: implicit declaration of function 'HUGETLBFS_SB'; did you mean 'HUGETLBFS_MAGIC'? [-Werror=implicit-function-declaration]
522 | return HUGETLBFS_SB(i->i_sb)->hstate;
| ^~~~~~~~~~~~
| HUGETLBFS_MAGIC
>> include/linux/hugetlb.h:522:30: error: invalid type argument of '->' (have 'int')
522 | return HUGETLBFS_SB(i->i_sb)->hstate;
| ^~
cc1: some warnings being treated as errors
vim +522 include/linux/hugetlb.h
a5516438959d90 Andi Kleen 2008-07-23 519
a137e1cc6d6e7d Andi Kleen 2008-07-23 520 static inline struct hstate *hstate_inode(struct inode *i)
a5516438959d90 Andi Kleen 2008-07-23 521 {
7fab358d90e6ba Chen Gang 2016-05-20 @522 return HUGETLBFS_SB(i->i_sb)->hstate;
a5516438959d90 Andi Kleen 2008-07-23 523 }
a5516438959d90 Andi Kleen 2008-07-23 524
:::::: The code at line 522 was first introduced by commit
:::::: 7fab358d90e6ba9d9cb702bee0c8a5f5c13bb6df include/linux/hugetlb*.h: clean up code
:::::: TO: Chen Gang <[email protected]>
:::::: CC: Linus Torvalds <[email protected]>
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/[email protected]
Le 17/03/2020 ? 02:39, kbuild test robot a ?crit?:
> Hi Christophe,
>
> Thank you for the patch! Yet something to improve:
>
> [auto build test ERROR on next-20200316]
> [cannot apply to powerpc/next v5.6-rc6 v5.6-rc5 v5.6-rc4 v5.6-rc6]
> [if your patch is applied to the wrong git tree, please drop us a note to help
> improve the system. BTW, we also suggest to use '--base' option to specify the
> base tree in git format-patch, please see https://stackoverflow.com/a/37406982]
>
> url: https://github.com/0day-ci/linux/commits/Christophe-Leroy/Use-hugepages-to-map-kernel-mem-on-8xx/20200317-065610
> base: 8548fd2f20ed19b0e8c0585b71fdfde1ae00ae3c
> config: powerpc-tqm8xx_defconfig (attached as .config)
> compiler: powerpc-linux-gcc (GCC) 9.2.0
> reproduce:
> wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
> chmod +x ~/bin/make.cross
> # save the attached .config to linux build tree
> GCC_VERSION=9.2.0 make.cross ARCH=powerpc
>
> If you fix the issue, kindly add following tag
> Reported-by: kbuild test robot <[email protected]>
>
> All errors (new ones prefixed by >>):
>
> In file included from arch/powerpc/mm/fault.c:33:
> include/linux/hugetlb.h: In function 'hstate_inode':
>>> include/linux/hugetlb.h:522:9: error: implicit declaration of function 'HUGETLBFS_SB'; did you mean 'HUGETLBFS_MAGIC'? [-Werror=implicit-function-declaration]
> 522 | return HUGETLBFS_SB(i->i_sb)->hstate;
> | ^~~~~~~~~~~~
> | HUGETLBFS_MAGIC
>>> include/linux/hugetlb.h:522:30: error: invalid type argument of '->' (have 'int')
> 522 | return HUGETLBFS_SB(i->i_sb)->hstate;
> | ^~
> cc1: all warnings being treated as errors
hstate_inode() shouldn't use HUGETLBFS_SB() which CONFIG_HUGETLBFS is
not set.
Proposed fix at https://patchwork.ozlabs.org/patch/1256108/
[...]
>>> include/linux/hugetlb.h:522:30: error: invalid type argument of '->' (have 'int')
> 522 | return HUGETLBFS_SB(i->i_sb)->hstate;
> | ^~
> At top level:
> arch/powerpc//mm/nohash/8xx.c:73:18: error: '__early_map_kernel_hugepage' defined but not used [-Werror=unused-function]
> 73 | static int __ref __early_map_kernel_hugepage(unsigned long va, phys_addr_t pa,
> | ^~~~~~~~~~~~~~~~~~~~~~~~~~~
> cc1: all warnings being treated as errors
This patch is a preparation patch. The function is not used yet, that's
normal. Ok, it breaks bisectability. Should it be squashed with the
first user of the function ?
Christophe
Le 17/03/2020 ? 15:43, Christophe Leroy a ?crit?:
>
>
> Le 17/03/2020 ? 02:39, kbuild test robot a ?crit?:
>> Hi Christophe,
>>
>> Thank you for the patch! Yet something to improve:
>>
>> [auto build test ERROR on next-20200316]
>> [cannot apply to powerpc/next v5.6-rc6 v5.6-rc5 v5.6-rc4 v5.6-rc6]
>> [if your patch is applied to the wrong git tree, please drop us a note
>> to help
>> improve the system. BTW, we also suggest to use '--base' option to
>> specify the
>> base tree in git format-patch, please see
>> https://stackoverflow.com/a/37406982]
>>
>> url:
>> https://github.com/0day-ci/linux/commits/Christophe-Leroy/Use-hugepages-to-map-kernel-mem-on-8xx/20200317-065610
>>
>> base:??? 8548fd2f20ed19b0e8c0585b71fdfde1ae00ae3c
>> config: powerpc-tqm8xx_defconfig (attached as .config)
>> compiler: powerpc-linux-gcc (GCC) 9.2.0
>> reproduce:
>> ???????? wget
>> https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross
>> -O ~/bin/make.cross
>> ???????? chmod +x ~/bin/make.cross
>> ???????? # save the attached .config to linux build tree
>> ???????? GCC_VERSION=9.2.0 make.cross ARCH=powerpc
>>
>> If you fix the issue, kindly add following tag
>> Reported-by: kbuild test robot <[email protected]>
>>
>> All errors (new ones prefixed by >>):
>>
>> ??? In file included from arch/powerpc/mm/fault.c:33:
>> ??? include/linux/hugetlb.h: In function 'hstate_inode':
>>>> include/linux/hugetlb.h:522:9: error: implicit declaration of
>>>> function 'HUGETLBFS_SB'; did you mean 'HUGETLBFS_MAGIC'?
>>>> [-Werror=implicit-function-declaration]
>> ????? 522 |? return HUGETLBFS_SB(i->i_sb)->hstate;
>> ????????? |???????? ^~~~~~~~~~~~
>> ????????? |???????? HUGETLBFS_MAGIC
>>>> include/linux/hugetlb.h:522:30: error: invalid type argument of '->'
>>>> (have 'int')
>> ????? 522 |? return HUGETLBFS_SB(i->i_sb)->hstate;
>> ????????? |????????????????????????????? ^~
>> ??? cc1: all warnings being treated as errors
>
> hstate_inode() shouldn't use HUGETLBFS_SB() which CONFIG_HUGETLBFS is
> not set.
>
> Proposed fix at https://patchwork.ozlabs.org/patch/1256108/
The fix is going in 5.7 via mm tree it seems, see
https://patchwork.kernel.org/patch/11470105/
Christophe
>
> [...]
>
>
>>>> include/linux/hugetlb.h:522:30: error: invalid type argument of '->'
>>>> (have 'int')
>> ????? 522 |? return HUGETLBFS_SB(i->i_sb)->hstate;
>> ????????? |????????????????????????????? ^~
>> ??? At top level:
>> ??? arch/powerpc//mm/nohash/8xx.c:73:18: error:
>> '__early_map_kernel_hugepage' defined but not used
>> [-Werror=unused-function]
>> ?????? 73 | static int __ref __early_map_kernel_hugepage(unsigned long
>> va, phys_addr_t pa,
>> ????????? |????????????????? ^~~~~~~~~~~~~~~~~~~~~~~~~~~
>> ??? cc1: all warnings being treated as errors
>
> This patch is a preparation patch. The function is not used yet, that's
> normal. Ok, it breaks bisectability. Should it be squashed with the
> first user of the function ?
>
> Christophe