This series of patches are follow up work (and depends on)
Toshi Kani <[email protected]>'s patches "fix memory leak/
panic in ioremap huge pages".
This series of patches are tested on 4.9 kernel with Cortex-A75
based SoC.
These patches can also go into '-stable' branch (if accepted)
for 4.6 onwards.
From V6->V7:
- Fixed compilation issue in x86 case
- V6 patches were not properly enumarated
From V5->V6:
- Use __flush_tlb_kernel_pgtable() for both PUD and PMD. Remove
"bool tlb_inv" based variance as it is not need now
- Re-naming for consistency
From V4->V5:
- Add new API __flush_tlb_kernel_pgtable(unsigned long addr)
for kernel addresses
From V3->V4:
- Add header for 'addr' in x86 implementation
- Re-order pmd/pud clear and table free
- Avoid redundant TLB invalidatation in one perticular case
From V2->V3:
- Use the exisiting page table free interface to do arm64
specific things
From V1->V2:
- Rebased my patches on top of "[PATCH v2 1/2] mm/vmalloc:
Add interfaces to free unmapped page table"
- Honored BBM for ARM64
Chintan Pandya (4):
ioremap: Update pgtable free interfaces with addr
arm64: tlbflush: Introduce __flush_tlb_kernel_pgtable
arm64: Implement page table free interfaces
Revert "arm64: Enforce BBM for huge IO/VMAP mappings"
arch/arm64/include/asm/tlbflush.h | 6 ++++++
arch/arm64/mm/mmu.c | 37 +++++++++++++++++++++++++------------
arch/x86/mm/pgtable.c | 8 +++++---
include/asm-generic/pgtable.h | 8 ++++----
lib/ioremap.c | 4 ++--
5 files changed, 42 insertions(+), 21 deletions(-)
--
Qualcomm India Private Limited, on behalf of Qualcomm Innovation
Center, Inc., is a member of Code Aurora Forum, a Linux Foundation
Collaborative Project
Implement pud_free_pmd_page() and pmd_free_pte_page().
Implementation requires,
1) Clearing off the current pud/pmd entry
2) Invalidate TLB which could have previously
valid but not stale entry
3) Freeing of the un-used next level page tables
Signed-off-by: Chintan Pandya <[email protected]>
---
arch/arm64/mm/mmu.c | 29 +++++++++++++++++++++++++----
1 file changed, 25 insertions(+), 4 deletions(-)
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index da98828..0f651db 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -45,6 +45,7 @@
#include <asm/memblock.h>
#include <asm/mmu_context.h>
#include <asm/ptdump.h>
+#include <asm/tlbflush.h>
#define NO_BLOCK_MAPPINGS BIT(0)
#define NO_CONT_MAPPINGS BIT(1)
@@ -973,12 +974,32 @@ int pmd_clear_huge(pmd_t *pmdp)
return 1;
}
-int pud_free_pmd_page(pud_t *pud, unsigned long addr)
+int pmd_free_pte_page(pmd_t *pmdp, unsigned long addr)
{
- return pud_none(*pud);
+ pmd_t *table;
+
+ if (pmd_present(READ_ONCE(*pmdp))) {
+ table = __va(pmd_val(*pmdp));
+ pmd_clear(pmdp);
+ __flush_tlb_kernel_pgtable(addr);
+ free_page((unsigned long) table);
+ }
+ return 1;
}
-int pmd_free_pte_page(pmd_t *pmd, unsigned long addr)
+int pud_free_pmd_page(pud_t *pudp, unsigned long addr)
{
- return pmd_none(*pmd);
+ pmd_t *table;
+ int i;
+
+ if (pud_present(READ_ONCE(*pudp))) {
+ table = __va(pud_val(*pudp));
+ for (i = 0; i < PTRS_PER_PMD; i++)
+ pmd_free_pte_page(&table[i], addr + (i * PMD_SIZE));
+
+ pud_clear(pudp);
+ __flush_tlb_kernel_pgtable(addr);
+ free_page((unsigned long) table);
+ }
+ return 1;
}
--
Qualcomm India Private Limited, on behalf of Qualcomm Innovation
Center, Inc., is a member of Code Aurora Forum, a Linux Foundation
Collaborative Project
This commit 15122ee2c515a ("arm64: Enforce BBM for huge
IO/VMAP mappings") is a temporary work-around until the
issues with CONFIG_HAVE_ARCH_HUGE_VMAP gets fixed.
Revert this change as we have fixes for the issue.
Signed-off-by: Chintan Pandya <[email protected]>
---
arch/arm64/mm/mmu.c | 8 --------
1 file changed, 8 deletions(-)
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 0f651db..170009b 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -935,10 +935,6 @@ int pud_set_huge(pud_t *pudp, phys_addr_t phys, pgprot_t prot)
pgprot_t sect_prot = __pgprot(PUD_TYPE_SECT |
pgprot_val(mk_sect_prot(prot)));
- /* ioremap_page_range doesn't honour BBM */
- if (pud_present(READ_ONCE(*pudp)))
- return 0;
-
BUG_ON(phys & ~PUD_MASK);
set_pud(pudp, pfn_pud(__phys_to_pfn(phys), sect_prot));
return 1;
@@ -949,10 +945,6 @@ int pmd_set_huge(pmd_t *pmdp, phys_addr_t phys, pgprot_t prot)
pgprot_t sect_prot = __pgprot(PMD_TYPE_SECT |
pgprot_val(mk_sect_prot(prot)));
- /* ioremap_page_range doesn't honour BBM */
- if (pmd_present(READ_ONCE(*pmdp)))
- return 0;
-
BUG_ON(phys & ~PMD_MASK);
set_pmd(pmdp, pfn_pmd(__phys_to_pfn(phys), sect_prot));
return 1;
--
Qualcomm India Private Limited, on behalf of Qualcomm Innovation
Center, Inc., is a member of Code Aurora Forum, a Linux Foundation
Collaborative Project
This patch ("mm/vmalloc: Add interfaces to free unmapped
page table") adds following 2 interfaces to free the page
table in case we implement huge mapping.
pud_free_pmd_page() and pmd_free_pte_page()
Some architectures (like arm64) needs to do proper TLB
maintanance after updating pagetable entry even in map.
Why ? Read this,
https://patchwork.kernel.org/patch/10134581/
Pass 'addr' in these interfaces so that proper TLB ops
can be performed.
Signed-off-by: Chintan Pandya <[email protected]>
---
arch/arm64/mm/mmu.c | 4 ++--
arch/x86/mm/pgtable.c | 8 +++++---
include/asm-generic/pgtable.h | 8 ++++----
lib/ioremap.c | 4 ++--
4 files changed, 13 insertions(+), 11 deletions(-)
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 2dbb2c9..da98828 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -973,12 +973,12 @@ int pmd_clear_huge(pmd_t *pmdp)
return 1;
}
-int pud_free_pmd_page(pud_t *pud)
+int pud_free_pmd_page(pud_t *pud, unsigned long addr)
{
return pud_none(*pud);
}
-int pmd_free_pte_page(pmd_t *pmd)
+int pmd_free_pte_page(pmd_t *pmd, unsigned long addr)
{
return pmd_none(*pmd);
}
diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c
index 34cda7e..aa67de9 100644
--- a/arch/x86/mm/pgtable.c
+++ b/arch/x86/mm/pgtable.c
@@ -706,11 +706,12 @@ int pmd_clear_huge(pmd_t *pmd)
/**
* pud_free_pmd_page - Clear pud entry and free pmd page.
* @pud: Pointer to a PUD.
+ * @addr: Virtual address associated with pud.
*
* Context: The pud range has been unmaped and TLB purged.
* Return: 1 if clearing the entry succeeded. 0 otherwise.
*/
-int pud_free_pmd_page(pud_t *pud)
+int pud_free_pmd_page(pud_t *pud, unsigned long addr)
{
pmd_t *pmd;
int i;
@@ -721,7 +722,7 @@ int pud_free_pmd_page(pud_t *pud)
pmd = (pmd_t *)pud_page_vaddr(*pud);
for (i = 0; i < PTRS_PER_PMD; i++)
- if (!pmd_free_pte_page(&pmd[i]))
+ if (!pmd_free_pte_page(&pmd[i]), addr + (i * PMD_SIZE))
return 0;
pud_clear(pud);
@@ -733,11 +734,12 @@ int pud_free_pmd_page(pud_t *pud)
/**
* pmd_free_pte_page - Clear pmd entry and free pte page.
* @pmd: Pointer to a PMD.
+ * @addr: Virtual address associated with pmd.
*
* Context: The pmd range has been unmaped and TLB purged.
* Return: 1 if clearing the entry succeeded. 0 otherwise.
*/
-int pmd_free_pte_page(pmd_t *pmd)
+int pmd_free_pte_page(pmd_t *pmd, unsigned long addr)
{
pte_t *pte;
diff --git a/include/asm-generic/pgtable.h b/include/asm-generic/pgtable.h
index bfbb44a..e6310f4 100644
--- a/include/asm-generic/pgtable.h
+++ b/include/asm-generic/pgtable.h
@@ -983,8 +983,8 @@ static inline int p4d_clear_huge(p4d_t *p4d)
int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot);
int pud_clear_huge(pud_t *pud);
int pmd_clear_huge(pmd_t *pmd);
-int pud_free_pmd_page(pud_t *pud);
-int pmd_free_pte_page(pmd_t *pmd);
+int pud_free_pmd_page(pud_t *pud, unsigned long addr);
+int pmd_free_pte_page(pmd_t *pmd, unsigned long addr);
#else /* !CONFIG_HAVE_ARCH_HUGE_VMAP */
static inline int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot)
{
@@ -1010,11 +1010,11 @@ static inline int pmd_clear_huge(pmd_t *pmd)
{
return 0;
}
-static inline int pud_free_pmd_page(pud_t *pud)
+static inline int pud_free_pmd_page(pud_t *pud, unsigned long addr)
{
return 0;
}
-static inline int pmd_free_pte_page(pmd_t *pmd)
+static inline int pmd_free_pte_page(pmd_t *pmd, unsigned long addr)
{
return 0;
}
diff --git a/lib/ioremap.c b/lib/ioremap.c
index 54e5bba..517f585 100644
--- a/lib/ioremap.c
+++ b/lib/ioremap.c
@@ -92,7 +92,7 @@ static inline int ioremap_pmd_range(pud_t *pud, unsigned long addr,
if (ioremap_pmd_enabled() &&
((next - addr) == PMD_SIZE) &&
IS_ALIGNED(phys_addr + addr, PMD_SIZE) &&
- pmd_free_pte_page(pmd)) {
+ pmd_free_pte_page(pmd, addr)) {
if (pmd_set_huge(pmd, phys_addr + addr, prot))
continue;
}
@@ -119,7 +119,7 @@ static inline int ioremap_pud_range(p4d_t *p4d, unsigned long addr,
if (ioremap_pud_enabled() &&
((next - addr) == PUD_SIZE) &&
IS_ALIGNED(phys_addr + addr, PUD_SIZE) &&
- pud_free_pmd_page(pud)) {
+ pud_free_pmd_page(pud, addr)) {
if (pud_set_huge(pud, phys_addr + addr, prot))
continue;
}
--
Qualcomm India Private Limited, on behalf of Qualcomm Innovation
Center, Inc., is a member of Code Aurora Forum, a Linux Foundation
Collaborative Project
Add an interface to invalidate intermediate page tables
from TLB for kernel.
Signed-off-by: Chintan Pandya <[email protected]>
---
arch/arm64/include/asm/tlbflush.h | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/arch/arm64/include/asm/tlbflush.h b/arch/arm64/include/asm/tlbflush.h
index 9e82dd7..6a4816d 100644
--- a/arch/arm64/include/asm/tlbflush.h
+++ b/arch/arm64/include/asm/tlbflush.h
@@ -209,6 +209,12 @@ static inline void __flush_tlb_pgtable(struct mm_struct *mm,
dsb(ish);
}
+static inline void __flush_tlb_kernel_pgtable(unsigned long addr)
+{
+ addr >>= 12;
+ __tlbi(vaae1is, addr);
+ dsb(ish);
+}
#endif
#endif
--
Qualcomm India Private Limited, on behalf of Qualcomm Innovation
Center, Inc., is a member of Code Aurora Forum, a Linux Foundation
Collaborative Project
Hi Chintan,
Thank you for the patch! Yet something to improve:
[auto build test ERROR on v4.16-rc7]
[also build test ERROR on next-20180328]
[cannot apply to arm64/for-next/core tip/x86/core asm-generic/master]
[if your patch is applied to the wrong git tree, please drop us a note to help improve the system]
url: https://github.com/0day-ci/linux/commits/Chintan-Pandya/ioremap-Update-pgtable-free-interfaces-with-addr/20180329-133736
config: x86_64-rhel (attached as .config)
compiler: gcc-7 (Debian 7.3.0-1) 7.3.0
reproduce:
# save the attached .config to linux build tree
make ARCH=x86_64
All errors (new ones prefixed by >>):
arch/x86/mm/pgtable.c: In function 'pud_free_pmd_page':
>> arch/x86/mm/pgtable.c:725:8: error: too few arguments to function 'pmd_free_pte_page'
if (!pmd_free_pte_page(&pmd[i]), addr + (i * PMD_SIZE))
^~~~~~~~~~~~~~~~~
In file included from arch/x86/include/asm/pgtable.h:1292:0,
from include/linux/memremap.h:8,
from include/linux/mm.h:27,
from arch/x86/mm/pgtable.c:2:
include/asm-generic/pgtable.h:987:5: note: declared here
int pmd_free_pte_page(pmd_t *pmd, unsigned long addr);
^~~~~~~~~~~~~~~~~
arch/x86/mm/pgtable.c:725:34: warning: left-hand operand of comma expression has no effect [-Wunused-value]
if (!pmd_free_pte_page(&pmd[i]), addr + (i * PMD_SIZE))
^
vim +/pmd_free_pte_page +725 arch/x86/mm/pgtable.c
705
706 /**
707 * pud_free_pmd_page - Clear pud entry and free pmd page.
708 * @pud: Pointer to a PUD.
709 * @addr: Virtual address associated with pud.
710 *
711 * Context: The pud range has been unmaped and TLB purged.
712 * Return: 1 if clearing the entry succeeded. 0 otherwise.
713 */
714 int pud_free_pmd_page(pud_t *pud, unsigned long addr)
715 {
716 pmd_t *pmd;
717 int i;
718
719 if (pud_none(*pud))
720 return 1;
721
722 pmd = (pmd_t *)pud_page_vaddr(*pud);
723
724 for (i = 0; i < PTRS_PER_PMD; i++)
> 725 if (!pmd_free_pte_page(&pmd[i]), addr + (i * PMD_SIZE))
726 return 0;
727
728 pud_clear(pud);
729 free_page((unsigned long)pmd);
730
731 return 1;
732 }
733
---
0-DAY kernel test infrastructure Open Source Technology Center
https://lists.01.org/pipermail/kbuild-all Intel Corporation
On 3/29/2018 5:23 PM, kbuild test robot wrote:
> Hi Chintan,
>
> Thank you for the patch! Yet something to improve:
>
> [auto build test ERROR on v4.16-rc7]
> [also build test ERROR on next-20180328]
> [cannot apply to arm64/for-next/core tip/x86/core asm-generic/master]
> [if your patch is applied to the wrong git tree, please drop us a note to help improve the system]
>
> url: https://github.com/0day-ci/linux/commits/Chintan-Pandya/ioremap-Update-pgtable-free-interfaces-with-addr/20180329-133736
> config: x86_64-rhel (attached as .config)
> compiler: gcc-7 (Debian 7.3.0-1) 7.3.0
> reproduce:
> # save the attached .config to linux build tree
> make ARCH=x86_64
>
> All errors (new ones prefixed by >>):
>
> arch/x86/mm/pgtable.c: In function 'pud_free_pmd_page':
>>> arch/x86/mm/pgtable.c:725:8: error: too few arguments to function 'pmd_free_pte_page'
> if (!pmd_free_pte_page(&pmd[i]), addr + (i * PMD_SIZE))
> ^~~~~~~~~~~~~~~~~
> In file included from arch/x86/include/asm/pgtable.h:1292:0,
> from include/linux/memremap.h:8,
> from include/linux/mm.h:27,
> from arch/x86/mm/pgtable.c:2:
> include/asm-generic/pgtable.h:987:5: note: declared here
> int pmd_free_pte_page(pmd_t *pmd, unsigned long addr);
> ^~~~~~~~~~~~~~~~~
> arch/x86/mm/pgtable.c:725:34: warning: left-hand operand of comma expression has no effect [-Wunused-value]
> if (!pmd_free_pte_page(&pmd[i]), addr + (i * PMD_SIZE))
> ^
>
> vim +/pmd_free_pte_page +725 arch/x86/mm/pgtable.c
>
> 705
> 706 /**
> 707 * pud_free_pmd_page - Clear pud entry and free pmd page.
> 708 * @pud: Pointer to a PUD.
> 709 * @addr: Virtual address associated with pud.
> 710 *
> 711 * Context: The pud range has been unmaped and TLB purged.
> 712 * Return: 1 if clearing the entry succeeded. 0 otherwise.
> 713 */
> 714 int pud_free_pmd_page(pud_t *pud, unsigned long addr)
> 715 {
> 716 pmd_t *pmd;
> 717 int i;
> 718
> 719 if (pud_none(*pud))
> 720 return 1;
> 721
> 722 pmd = (pmd_t *)pud_page_vaddr(*pud);
> 723
> 724 for (i = 0; i < PTRS_PER_PMD; i++)
> > 725 if (!pmd_free_pte_page(&pmd[i]), addr + (i * PMD_SIZE))
Missed it. Will upload v8
> 726 return 0;
> 727
> 728 pud_clear(pud);
> 729 free_page((unsigned long)pmd);
> 730
> 731 return 1;
> 732 }
> 733
>
> ---
> 0-DAY kernel test infrastructure Open Source Technology Center
> https://lists.01.org/pipermail/kbuild-all Intel Corporation
>
Chintan
--
Qualcomm India Private Limited, on behalf of Qualcomm Innovation Center,
Inc. is a member of the Code Aurora Forum, a Linux Foundation
Collaborative Project