In split page table lock case, we embed spinlock_t into struct page. For
obvious reason, we don't want to increase size of struct page if
spinlock_t is too big, like with DEBUG_SPINLOCK or DEBUG_LOCK_ALLOC or on
-rt kernel. So we disble split page table lock, if spinlock_t is too big.
This patchset allows to allocate the lock dynamically if spinlock_t is
big. In this page->ptl is used to store pointer to spinlock instead of
spinlock itself. It costs additional cache line for indirect access, but
fix page fault scalability for multi-threaded applications.
LOCK_STAT depends on DEBUG_SPINLOCK, so on current kernel enabling
LOCK_STAT to analyse scalability issues breaks scalability. ;)
The patchset mostly fixes this. Results for ./thp_memscale -c 80 -b 512M
on 4-socket machine:
baseline, no CONFIG_LOCK_STAT: 9.115460703 seconds time elapsed
baseline, CONFIG_LOCK_STAT=y: 53.890567123 seconds time elapsed
patched, no CONFIG_LOCK_STAT: 8.852250368 seconds time elapsed
patched, CONFIG_LOCK_STAT=y: 11.069770759 seconds time elapsed
Patch count is scary, but most of them trivial. Overview:
Patches 1-4 Few bug fixes. No dependencies to other patches.
Probably should applied as soon as possible.
Patch 5 Changes signature of pgtable_page_ctor(). We will use it
for dynamic lock allocation, so it can fail.
Patches 6-8 Add missing constructor/destructor calls on few archs.
It's fixes NR_PAGETABLE accounting and prepare to use
split ptl.
Patches 9-33 Add pgtable_page_ctor() fail handling to all archs.
Patches 34 Finally adds support of dynamically-allocated page->pte.
Also contains documentation for split page table lock.
Any comments?
Kirill A. Shutemov (34):
x86: add missed pgtable_pmd_page_ctor/dtor calls for preallocated pmds
cris: fix potential NULL-pointer dereference
m32r: fix potential NULL-pointer dereference
xtensa: fix potential NULL-pointer dereference
mm: allow pgtable_page_ctor() to fail
microblaze: add missing pgtable_page_ctor/dtor calls
mn10300: add missing pgtable_page_ctor/dtor calls
openrisc: add missing pgtable_page_ctor/dtor calls
alpha: handle pgtable_page_ctor() fail
arc: handle pgtable_page_ctor() fail
arm: handle pgtable_page_ctor() fail
arm64: handle pgtable_page_ctor() fail
avr32: handle pgtable_page_ctor() fail
cris: handle pgtable_page_ctor() fail
frv: handle pgtable_page_ctor() fail
hexagon: handle pgtable_page_ctor() fail
ia64: handle pgtable_page_ctor() fail
m32r: handle pgtable_page_ctor() fail
m68k: handle pgtable_page_ctor() fail
metag: handle pgtable_page_ctor() fail
mips: handle pgtable_page_ctor() fail
parisc: handle pgtable_page_ctor() fail
powerpc: handle pgtable_page_ctor() fail
s390: handle pgtable_page_ctor() fail
score: handle pgtable_page_ctor() fail
sh: handle pgtable_page_ctor() fail
sparc: handle pgtable_page_ctor() fail
tile: handle pgtable_page_ctor() fail
um: handle pgtable_page_ctor() fail
unicore32: handle pgtable_page_ctor() fail
x86: handle pgtable_page_ctor() fail
xtensa: handle pgtable_page_ctor() fail
iommu/arm-smmu: handle pgtable_page_ctor() fail
mm: dynamically allocate page->ptl if it cannot be embedded to struct
page
Documentation/vm/split_page_table_lock | 90 ++++++++++++++++++++++++++++++++
arch/alpha/include/asm/pgalloc.h | 5 +-
arch/arc/include/asm/pgalloc.h | 11 ++--
arch/arm/include/asm/pgalloc.h | 12 +++--
arch/arm64/include/asm/pgalloc.h | 9 ++--
arch/avr32/include/asm/pgalloc.h | 5 +-
arch/cris/include/asm/pgalloc.h | 7 ++-
arch/frv/mm/pgalloc.c | 12 +++--
arch/hexagon/include/asm/pgalloc.h | 10 ++--
arch/ia64/include/asm/pgalloc.h | 5 +-
arch/m32r/include/asm/pgalloc.h | 7 ++-
arch/m68k/include/asm/motorola_pgalloc.h | 5 +-
arch/m68k/include/asm/sun3_pgalloc.h | 5 +-
arch/metag/include/asm/pgalloc.h | 8 ++-
arch/microblaze/include/asm/pgalloc.h | 12 +++--
arch/mips/include/asm/pgalloc.h | 9 ++--
arch/mn10300/include/asm/pgalloc.h | 1 +
arch/mn10300/mm/pgtable.c | 9 +++-
arch/openrisc/include/asm/pgalloc.h | 10 +++-
arch/parisc/include/asm/pgalloc.h | 8 ++-
arch/powerpc/include/asm/pgalloc-64.h | 5 +-
arch/powerpc/mm/pgtable_32.c | 5 +-
arch/powerpc/mm/pgtable_64.c | 7 +--
arch/s390/mm/pgtable.c | 11 +++-
arch/score/include/asm/pgalloc.h | 9 ++--
arch/sh/include/asm/pgalloc.h | 5 +-
arch/sparc/mm/init_64.c | 11 ++--
arch/sparc/mm/srmmu.c | 5 +-
arch/tile/mm/pgtable.c | 6 ++-
arch/um/kernel/mem.c | 8 ++-
arch/unicore32/include/asm/pgalloc.h | 14 ++---
arch/x86/mm/pgtable.c | 19 +++++--
arch/x86/xen/mmu.c | 2 +-
arch/xtensa/include/asm/pgalloc.h | 11 +++-
drivers/iommu/arm-smmu.c | 5 +-
include/linux/mm.h | 73 +++++++++++++++++++-------
include/linux/mm_types.h | 5 +-
mm/Kconfig | 2 -
mm/memory.c | 19 +++++++
39 files changed, 365 insertions(+), 97 deletions(-)
create mode 100644 Documentation/vm/split_page_table_lock
--
1.8.4.rc3
Signed-off-by: Kirill A. Shutemov <[email protected]>
Cc: Chris Metcalf <[email protected]>
---
arch/tile/mm/pgtable.c | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/arch/tile/mm/pgtable.c b/arch/tile/mm/pgtable.c
index 4fd9ec0b58..5e86eac4bf 100644
--- a/arch/tile/mm/pgtable.c
+++ b/arch/tile/mm/pgtable.c
@@ -241,6 +241,11 @@ struct page *pgtable_alloc_one(struct mm_struct *mm, unsigned long address,
if (p == NULL)
return NULL;
+ if (!pgtable_page_ctor(p)) {
+ __free_pages(p, L2_USER_PGTABLE_ORDER);
+ return NULL;
+ }
+
/*
* Make every page have a page_count() of one, not just the first.
* We don't use __GFP_COMP since it doesn't look like it works
@@ -251,7 +256,6 @@ struct page *pgtable_alloc_one(struct mm_struct *mm, unsigned long address,
inc_zone_page_state(p+i, NR_PAGETABLE);
}
- pgtable_page_ctor(p);
return p;
}
--
1.8.4.rc3
Signed-off-by: Kirill A. Shutemov <[email protected]>
Cc: Jeff Dike <[email protected]>
Cc: Richard Weinberger <[email protected]>
---
arch/um/kernel/mem.c | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/arch/um/kernel/mem.c b/arch/um/kernel/mem.c
index 7ddb64baf3..8636e90542 100644
--- a/arch/um/kernel/mem.c
+++ b/arch/um/kernel/mem.c
@@ -279,8 +279,12 @@ pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
struct page *pte;
pte = alloc_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO);
- if (pte)
- pgtable_page_ctor(pte);
+ if (!pte)
+ return NULL;
+ if (!pgtable_page_ctor(pte)) {
+ __free_page(pte);
+ return NULL;
+ }
return pte;
}
--
1.8.4.rc3
Signed-off-by: Kirill A. Shutemov <[email protected]>
Cc: Chris Zankel <[email protected]>
Cc: Max Filippov <[email protected]>
---
arch/xtensa/include/asm/pgalloc.h | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/arch/xtensa/include/asm/pgalloc.h b/arch/xtensa/include/asm/pgalloc.h
index 037671a655..b8774f1e21 100644
--- a/arch/xtensa/include/asm/pgalloc.h
+++ b/arch/xtensa/include/asm/pgalloc.h
@@ -58,7 +58,10 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
if (!pte)
return NULL;
page = virt_to_page(pte);
- pgtable_page_ctor(page);
+ if (!pgtable_page_ctor(page)) {
+ kmem_cache_free(pgtable_cache, pte);
+ return NULL;
+ }
return page;
}
--
1.8.4.rc3
Add missing check for memory allocation fail.
Signed-off-by: Kirill A. Shutemov <[email protected]>
Cc: Hirokazu Takata <[email protected]>
---
arch/m32r/include/asm/pgalloc.h | 2 ++
1 file changed, 2 insertions(+)
diff --git a/arch/m32r/include/asm/pgalloc.h b/arch/m32r/include/asm/pgalloc.h
index 0fc7361989..ac4208bcc5 100644
--- a/arch/m32r/include/asm/pgalloc.h
+++ b/arch/m32r/include/asm/pgalloc.h
@@ -43,6 +43,8 @@ static __inline__ pgtable_t pte_alloc_one(struct mm_struct *mm,
{
struct page *pte = alloc_page(GFP_KERNEL|__GFP_ZERO);
+ if (!pte)
+ return NULL;
pgtable_page_ctor(pte);
return pte;
}
--
1.8.4.rc3
Signed-off-by: Kirill A. Shutemov <[email protected]>
Cc: James Hogan <[email protected]>
---
arch/metag/include/asm/pgalloc.h | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/arch/metag/include/asm/pgalloc.h b/arch/metag/include/asm/pgalloc.h
index 275d928514..3104df0a48 100644
--- a/arch/metag/include/asm/pgalloc.h
+++ b/arch/metag/include/asm/pgalloc.h
@@ -52,8 +52,12 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
{
struct page *pte;
pte = alloc_pages(GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO, 0);
- if (pte)
- pgtable_page_ctor(pte);
+ if (!pte)
+ return NULL;
+ if (!pgtable_page_ctor(pte)) {
+ __free_page(pte);
+ return NULL;
+ }
return pte;
}
--
1.8.4.rc3
Signed-off-by: Kirill A. Shutemov <[email protected]>
Cc: Will Deacon <[email protected]>
Cc: Grant Likely <[email protected]>
Cc: Rob Herring <[email protected]>
---
drivers/iommu/arm-smmu.c | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c
index f417e89e1e..2b256a5075 100644
--- a/drivers/iommu/arm-smmu.c
+++ b/drivers/iommu/arm-smmu.c
@@ -1211,7 +1211,10 @@ static int arm_smmu_alloc_init_pte(struct arm_smmu_device *smmu, pmd_t *pmd,
arm_smmu_flush_pgtable(smmu, page_address(table),
ARM_SMMU_PTE_HWTABLE_SIZE);
- pgtable_page_ctor(table);
+ if (!pgtable_page_ctor(table)) {
+ __free_page(table);
+ return -ENOMEM;
+ }
pmd_populate(NULL, pmd, table);
arm_smmu_flush_pgtable(smmu, pmd, sizeof(*pmd));
}
--
1.8.4.rc3
If split page table lock is in use, we embed the lock into struct page
of table's page. We have to disable split lock, if spinlock_t is too big
be to be embedded, like when DEBUG_SPINLOCK or DEBUG_LOCK_ALLOC enabled.
This patch add support for dynamic allocation of split page table lock
if we can't embed it to struct page.
page->ptl is unsigned long now and we use it as spinlock_t if
sizeof(spinlock_t) <= sizeof(long), otherwise it's pointer to
spinlock_t.
The spinlock_t allocated in pgtable_page_ctor() for PTE table and in
pgtable_pmd_page_ctor() for PMD table. All other helpers converted to
support dynamically allocated page->ptl.
Signed-off-by: Kirill A. Shutemov <[email protected]>
---
Documentation/vm/split_page_table_lock | 90 ++++++++++++++++++++++++++++++++++
arch/x86/xen/mmu.c | 2 +-
include/linux/mm.h | 72 +++++++++++++++++++--------
include/linux/mm_types.h | 5 +-
mm/Kconfig | 2 -
mm/memory.c | 19 +++++++
6 files changed, 166 insertions(+), 24 deletions(-)
create mode 100644 Documentation/vm/split_page_table_lock
diff --git a/Documentation/vm/split_page_table_lock b/Documentation/vm/split_page_table_lock
new file mode 100644
index 0000000000..e2f617b732
--- /dev/null
+++ b/Documentation/vm/split_page_table_lock
@@ -0,0 +1,90 @@
+Split page table lock
+=====================
+
+Originally, mm->page_table_lock spinlock protected all page tables of the
+mm_struct. But this approach leads to poor page fault scalability of
+multi-threaded applications due high contention on the lock. To improve
+scalability, split page table lock was introduced.
+
+With split page table lock we have separate per-table lock to serialize
+access to the table. At the moment we use split lock for PTE and PMD
+tables. Access to higher level tables protected by mm->page_table_lock.
+
+There are helpers to lock/unlock a table and other accessor functions:
+ - pte_offset_map_lock()
+ maps pte and takes PTE table lock, returns pointer to the taken
+ lock;
+ - pte_unmap_unlock()
+ unlocks and unmaps PTE table;
+ - pte_alloc_map_lock()
+ allocates PTE table if needed and take the lock, returns pointer
+ to taken lock or NULL if allocation failed;
+ - pte_lockptr()
+ returns pointer to PTE table lock;
+ - pmd_lock()
+ takes PMD table lock, returns pointer to taken lock;
+ - pmd_lockptr()
+ returns pointer to PMD table lock;
+
+Split page table lock for PTE tables is enabled compile-time if
+CONFIG_SPLIT_PTLOCK_CPUS (usually 4) is less or equal to NR_CPUS.
+If split lock is disabled, all tables guaded by mm->page_table_lock.
+
+Split page table lock for PMD tables is enabled, if it's enabled for PTE
+tables and the architecture supports it (see below).
+
+Hugetlb and split page table lock
+---------------------------------
+
+Hugetlb can support several page sizes. We use split lock only for PMD
+level, but not for PUD.
+
+Hugetlb-specific helpers:
+ - huge_pte_lock()
+ takes pmd split lock for PMD_SIZE page, mm->page_table_lock
+ otherwise;
+ - huge_pte_lockptr()
+ returns pointer to table lock;
+
+Support of split page table lock by an architecture
+---------------------------------------------------
+
+There's no need in special enabling of PTE split page table lock:
+everything required is done by pgtable_page_ctor() and pgtable_page_dtor(),
+which must be called on PTE table allocation / freeing.
+
+PMD split lock only makes sense if you have more than two page table
+levels.
+
+PMD split lock enabling requires pgtable_pmd_page_ctor() call on PMD table
+allocation and pgtable_pmd_page_dtor() on freeing.
+
+Allocation usually happens in pmd_alloc_one(), freeing in pmd_free(), but
+make sure you cover all PMD table allocation / freeing paths: i.e X86_PAE
+preallocate few PMDs on pgd_alloc().
+
+With everything in place you can set CONFIG_ARCH_ENABLE_SPLIT_PMD_PTLOCK.
+
+NOTE: pgtable_page_ctor() and pgtable_pmd_page_ctor() can fail -- it must
+be handled properly.
+
+page->ptl
+---------
+
+page->ptl is used to access split page table lock, where 'page' is struct
+page of page containing the table. It shares storage with page->private
+(and few other fields in union).
+
+To avoid increasing size of struct page and have best performance, we use a
+trick:
+ - if spinlock_t fits into long, we use page->ptr as spinlock, so we
+ can avoid indirect access and save a cache line.
+ - if size of spinlock_t is bigger then size of long, we use page->ptl as
+ pointer to spinlock_t and allocate it dynamically. This allows to use
+ split lock with enabled DEBUG_SPINLOCK or DEBUG_LOCK_ALLOC, but costs
+ one more cache line for indirect access;
+
+The spinlock_t allocated in pgtable_page_ctor() for PTE table and in
+pgtable_pmd_page_ctor() for PMD table.
+
+Please, never access page->ptl directly -- use appropriate helper.
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 455c873ce0..49c962fe7e 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -797,7 +797,7 @@ static spinlock_t *xen_pte_lock(struct page *page, struct mm_struct *mm)
spinlock_t *ptl = NULL;
#if USE_SPLIT_PTE_PTLOCKS
- ptl = __pte_lockptr(page);
+ ptl = ptlock_ptr(page);
spin_lock_nest_lock(ptl, &mm->page_table_lock);
#endif
diff --git a/include/linux/mm.h b/include/linux/mm.h
index f6467032a9..658e8b317f 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -1233,32 +1233,64 @@ static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long a
#endif /* CONFIG_MMU && !__ARCH_HAS_4LEVEL_HACK */
#if USE_SPLIT_PTE_PTLOCKS
-/*
- * We tuck a spinlock to guard each pagetable page into its struct page,
- * at page->private, with BUILD_BUG_ON to make sure that this will not
- * overflow into the next struct page (as it might with DEBUG_SPINLOCK).
- * When freeing, reset page->mapping so free_pages_check won't complain.
- */
-#define __pte_lockptr(page) &((page)->ptl)
-#define pte_lock_init(_page) do { \
- spin_lock_init(__pte_lockptr(_page)); \
-} while (0)
-#define pte_lock_deinit(page) ((page)->mapping = NULL)
-#define pte_lockptr(mm, pmd) ({(void)(mm); __pte_lockptr(pmd_page(*(pmd)));})
+bool __ptlock_alloc(struct page *page);
+void __ptlock_free(struct page *page);
+static inline bool ptlock_alloc(struct page *page)
+{
+ if (sizeof(spinlock_t) > sizeof(page->ptl))
+ return __ptlock_alloc(page);
+ return true;
+}
+static inline void ptlock_free(struct page *page)
+{
+ if (sizeof(spinlock_t) > sizeof(page->ptl))
+ __ptlock_free(page);
+}
+
+static inline spinlock_t *ptlock_ptr(struct page *page)
+{
+ if (sizeof(spinlock_t) > sizeof(page->ptl))
+ return (spinlock_t *) page->ptl;
+ else
+ return (spinlock_t *) &page->ptl;
+}
+
+static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd)
+{
+ return ptlock_ptr(pmd_page(*pmd));
+}
+
+static inline bool ptlock_init(struct page *page)
+{
+ if (!ptlock_alloc(page))
+ return false;
+ spin_lock_init(ptlock_ptr(page));
+ return true;
+}
+
+/* Reset page->mapping so free_pages_check won't complain. */
+static inline void pte_lock_deinit(struct page *page)
+{
+ page->mapping = NULL;
+ ptlock_free(page);
+}
+
#else /* !USE_SPLIT_PTE_PTLOCKS */
/*
* We use mm->page_table_lock to guard all pagetable pages of the mm.
*/
-#define pte_lock_init(page) do {} while (0)
-#define pte_lock_deinit(page) do {} while (0)
-#define pte_lockptr(mm, pmd) ({(void)(pmd); &(mm)->page_table_lock;})
+static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd)
+{
+ return &mm->page_table_lock;
+}
+static inline bool ptlock_init(struct page *page) { return true; }
+static inline void pte_lock_deinit(struct page *page) {}
#endif /* USE_SPLIT_PTE_PTLOCKS */
static inline bool pgtable_page_ctor(struct page *page)
{
- pte_lock_init(page);
inc_zone_page_state(page, NR_PAGETABLE);
- return true;
+ return ptlock_init(page);
}
static inline void pgtable_page_dtor(struct page *page)
@@ -1299,16 +1331,15 @@ static inline void pgtable_page_dtor(struct page *page)
static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
{
- return &virt_to_page(pmd)->ptl;
+ return ptlock_ptr(virt_to_page(pmd));
}
static inline bool pgtable_pmd_page_ctor(struct page *page)
{
- spin_lock_init(&page->ptl);
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
page->pmd_huge_pte = NULL;
#endif
- return true;
+ return ptlock_init(page);
}
static inline void pgtable_pmd_page_dtor(struct page *page)
@@ -1316,6 +1347,7 @@ static inline void pgtable_pmd_page_dtor(struct page *page)
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
VM_BUG_ON(page->pmd_huge_pte);
#endif
+ ptlock_free(page);
}
#define pmd_huge_pte(mm, pmd) (virt_to_page(pmd)->pmd_huge_pte)
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index bacc15f078..257ac12fac 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -147,7 +147,10 @@ struct page {
* system if PG_buddy is set.
*/
#if USE_SPLIT_PTE_PTLOCKS
- spinlock_t ptl;
+ unsigned long ptl; /* It's spinlock_t if it fits to long,
+ * otherwise it's pointer to dynamicaly
+ * allocated spinlock_t.
+ */
#endif
struct kmem_cache *slab_cache; /* SL[AU]B: Pointer to slab */
struct page *first_page; /* Compound tail pages */
diff --git a/mm/Kconfig b/mm/Kconfig
index d19f7d380b..9e8c8ae3b6 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -211,8 +211,6 @@ config SPLIT_PTLOCK_CPUS
int
default "999999" if ARM && !CPU_CACHE_VIPT
default "999999" if PARISC && !PA20
- default "999999" if DEBUG_SPINLOCK || DEBUG_LOCK_ALLOC
- default "999999" if !64BIT && GENERIC_LOCKBREAK
default "4"
config ARCH_ENABLE_SPLIT_PMD_PTLOCK
diff --git a/mm/memory.c b/mm/memory.c
index 1200d6230c..7e11f745bc 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -4330,3 +4330,22 @@ void copy_user_huge_page(struct page *dst, struct page *src,
}
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
+
+#if USE_SPLIT_PTE_PTLOCKS
+bool __ptlock_alloc(struct page *page)
+{
+ spinlock_t *ptl;
+
+ ptl = kmalloc(sizeof(spinlock_t), GFP_KERNEL);
+ if (!ptl)
+ return false;
+ page->ptl = (unsigned long)ptl;
+ return true;
+}
+
+void __ptlock_free(struct page *page)
+{
+ if (sizeof(spinlock_t) > sizeof(page->ptl))
+ kfree((spinlock_t *)page->ptl);
+}
+#endif
--
1.8.4.rc3
Signed-off-by: Kirill A. Shutemov <[email protected]>
Cc: "David S. Miller" <[email protected]>
---
arch/sparc/mm/init_64.c | 11 ++++++-----
arch/sparc/mm/srmmu.c | 5 ++++-
2 files changed, 10 insertions(+), 6 deletions(-)
diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c
index ed82edad1a..d6de9353ee 100644
--- a/arch/sparc/mm/init_64.c
+++ b/arch/sparc/mm/init_64.c
@@ -2519,12 +2519,13 @@ pgtable_t pte_alloc_one(struct mm_struct *mm,
return pte;
page = __alloc_for_cache(mm);
- if (page) {
- pgtable_page_ctor(page);
- pte = (pte_t *) page_address(page);
+ if (!page)
+ return NULL;
+ if (!pgtable_page_ctor(page)) {
+ free_hot_cold_page(page, 0);
+ return NULL;
}
-
- return pte;
+ return (pte_t *) page_address(page);
}
void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
diff --git a/arch/sparc/mm/srmmu.c b/arch/sparc/mm/srmmu.c
index 5d721df48a..869023abe5 100644
--- a/arch/sparc/mm/srmmu.c
+++ b/arch/sparc/mm/srmmu.c
@@ -345,7 +345,10 @@ pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
if ((pte = (unsigned long)pte_alloc_one_kernel(mm, address)) == 0)
return NULL;
page = pfn_to_page(__nocache_pa(pte) >> PAGE_SHIFT);
- pgtable_page_ctor(page);
+ if (!pgtable_page_ctor(page)) {
+ __free_page(page);
+ return NULL;
+ }
return page;
}
--
1.8.4.rc3
Signed-off-by: Kirill A. Shutemov <[email protected]>
Cc: Thomas Gleixner <[email protected]>
Cc: Ingo Molnar <[email protected]>
Cc: "H. Peter Anvin" <[email protected]>
---
arch/x86/mm/pgtable.c | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c
index 1a7d21342e..a7cccb6d7f 100644
--- a/arch/x86/mm/pgtable.c
+++ b/arch/x86/mm/pgtable.c
@@ -25,8 +25,12 @@ pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
struct page *pte;
pte = alloc_pages(__userpte_alloc_gfp, 0);
- if (pte)
- pgtable_page_ctor(pte);
+ if (!pte)
+ return NULL;
+ if (!pgtable_page_ctor(pte)) {
+ __free_page(pte);
+ return NULL;
+ }
return pte;
}
--
1.8.4.rc3
Signed-off-by: Kirill A. Shutemov <[email protected]>
Cc: Martin Schwidefsky <[email protected]>
Cc: Heiko Carstens <[email protected]>
---
arch/s390/mm/pgtable.c | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git a/arch/s390/mm/pgtable.c b/arch/s390/mm/pgtable.c
index c463e5cd3b..b109edb90b 100644
--- a/arch/s390/mm/pgtable.c
+++ b/arch/s390/mm/pgtable.c
@@ -754,7 +754,11 @@ static inline unsigned long *page_table_alloc_pgste(struct mm_struct *mm,
__free_page(page);
return NULL;
}
- pgtable_page_ctor(page);
+ if (!pgtable_page_ctor(page)) {
+ kfree(mp);
+ __free_page(page);
+ return NULL;
+ }
mp->vmaddr = vmaddr & PMD_MASK;
INIT_LIST_HEAD(&mp->mapper);
page->index = (unsigned long) mp;
@@ -884,7 +888,10 @@ unsigned long *page_table_alloc(struct mm_struct *mm, unsigned long vmaddr)
page = alloc_page(GFP_KERNEL|__GFP_REPEAT);
if (!page)
return NULL;
- pgtable_page_ctor(page);
+ if (!pgtable_page_ctor(page)) {
+ __free_page(page);
+ return NULL;
+ }
atomic_set(&page->_mapcount, 1);
table = (unsigned long *) page_to_phys(page);
clear_table(table, _PAGE_INVALID, PAGE_SIZE);
--
1.8.4.rc3
Signed-off-by: Kirill A. Shutemov <[email protected]>
Cc: "James E.J. Bottomley" <[email protected]>
Cc: Helge Deller <[email protected]>
---
arch/parisc/include/asm/pgalloc.h | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/arch/parisc/include/asm/pgalloc.h b/arch/parisc/include/asm/pgalloc.h
index fc987a1c12..f213f5b4c4 100644
--- a/arch/parisc/include/asm/pgalloc.h
+++ b/arch/parisc/include/asm/pgalloc.h
@@ -121,8 +121,12 @@ static inline pgtable_t
pte_alloc_one(struct mm_struct *mm, unsigned long address)
{
struct page *page = alloc_page(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO);
- if (page)
- pgtable_page_ctor(page);
+ if (!page)
+ return NULL;
+ if (!pgtable_page_ctor(page)) {
+ __free_page(page);
+ return NULL;
+ }
return page;
}
--
1.8.4.rc3
Signed-off-by: Kirill A. Shutemov <[email protected]>
Cc: Chen Liqin <[email protected]>
Cc: Lennox Wu <[email protected]>
---
arch/score/include/asm/pgalloc.h | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/arch/score/include/asm/pgalloc.h b/arch/score/include/asm/pgalloc.h
index 059a61b707..2a861ffbd5 100644
--- a/arch/score/include/asm/pgalloc.h
+++ b/arch/score/include/asm/pgalloc.h
@@ -54,9 +54,12 @@ static inline struct page *pte_alloc_one(struct mm_struct *mm,
struct page *pte;
pte = alloc_pages(GFP_KERNEL | __GFP_REPEAT, PTE_ORDER);
- if (pte) {
- clear_highpage(pte);
- pgtable_page_ctor(pte);
+ if (!pte)
+ return NULL;
+ clear_highpage(pte);
+ if (!pgtable_page_ctor(pte)) {
+ __free_page(pte);
+ return NULL;
}
return pte;
}
--
1.8.4.rc3
Signed-off-by: Kirill A. Shutemov <[email protected]>
Cc: Benjamin Herrenschmidt <[email protected]>
Cc: Paul Mackerras <[email protected]>
---
arch/powerpc/include/asm/pgalloc-64.h | 5 ++++-
arch/powerpc/mm/pgtable_32.c | 5 ++++-
arch/powerpc/mm/pgtable_64.c | 7 ++++---
3 files changed, 12 insertions(+), 5 deletions(-)
diff --git a/arch/powerpc/include/asm/pgalloc-64.h b/arch/powerpc/include/asm/pgalloc-64.h
index f65e27b09b..16cb92d215 100644
--- a/arch/powerpc/include/asm/pgalloc-64.h
+++ b/arch/powerpc/include/asm/pgalloc-64.h
@@ -91,7 +91,10 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
if (!pte)
return NULL;
page = virt_to_page(pte);
- pgtable_page_ctor(page);
+ if (!pgtable_page_ctor(page)) {
+ __free_page(page);
+ return NULL;
+ }
return page;
}
diff --git a/arch/powerpc/mm/pgtable_32.c b/arch/powerpc/mm/pgtable_32.c
index 6c856fb8c1..5b96017152 100644
--- a/arch/powerpc/mm/pgtable_32.c
+++ b/arch/powerpc/mm/pgtable_32.c
@@ -121,7 +121,10 @@ pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
ptepage = alloc_pages(flags, 0);
if (!ptepage)
return NULL;
- pgtable_page_ctor(ptepage);
+ if (!pgtable_page_ctor(ptepage)) {
+ __free_page(ptepage);
+ return NULL;
+ }
return ptepage;
}
diff --git a/arch/powerpc/mm/pgtable_64.c b/arch/powerpc/mm/pgtable_64.c
index 536eec72c0..9d95786aa8 100644
--- a/arch/powerpc/mm/pgtable_64.c
+++ b/arch/powerpc/mm/pgtable_64.c
@@ -378,6 +378,10 @@ static pte_t *__alloc_for_cache(struct mm_struct *mm, int kernel)
__GFP_REPEAT | __GFP_ZERO);
if (!page)
return NULL;
+ if (!kernel && !pgtable_page_ctor(page)) {
+ __free_page(page);
+ return NULL;
+ }
ret = page_address(page);
spin_lock(&mm->page_table_lock);
@@ -392,9 +396,6 @@ static pte_t *__alloc_for_cache(struct mm_struct *mm, int kernel)
}
spin_unlock(&mm->page_table_lock);
- if (!kernel)
- pgtable_page_ctor(page);
-
return (pte_t *)ret;
}
--
1.8.4.rc3
Signed-off-by: Kirill A. Shutemov <[email protected]>
Cc: Paul Mundt <[email protected]>
---
arch/sh/include/asm/pgalloc.h | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/arch/sh/include/asm/pgalloc.h b/arch/sh/include/asm/pgalloc.h
index 8c00785c60..a33673b368 100644
--- a/arch/sh/include/asm/pgalloc.h
+++ b/arch/sh/include/asm/pgalloc.h
@@ -47,7 +47,10 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
if (!pg)
return NULL;
page = virt_to_page(pg);
- pgtable_page_ctor(page);
+ if (!pgtable_page_ctor(page)) {
+ quicklist_free(QUICK_PT, NULL, pg);
+ return NULL;
+ }
return page;
}
--
1.8.4.rc3
Add missing check for memory allocation fail.
Signed-off-by: Kirill A. Shutemov <[email protected]>
Cc: Chris Zankel <[email protected]>
Cc: Max Filippov <[email protected]>
---
arch/xtensa/include/asm/pgalloc.h | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/arch/xtensa/include/asm/pgalloc.h b/arch/xtensa/include/asm/pgalloc.h
index cf914c8c24..037671a655 100644
--- a/arch/xtensa/include/asm/pgalloc.h
+++ b/arch/xtensa/include/asm/pgalloc.h
@@ -51,9 +51,13 @@ static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
unsigned long addr)
{
+ pte_t *pte;
struct page *page;
- page = virt_to_page(pte_alloc_one_kernel(mm, addr));
+ pte = pte_alloc_one_kernel(mm, addr);
+ if (!pte)
+ return NULL;
+ page = virt_to_page(pte);
pgtable_page_ctor(page);
return page;
}
--
1.8.4.rc3
Signed-off-by: Kirill A. Shutemov <[email protected]>
Cc: Tony Luck <[email protected]>
Cc: Fenghua Yu <[email protected]>
---
arch/ia64/include/asm/pgalloc.h | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/arch/ia64/include/asm/pgalloc.h b/arch/ia64/include/asm/pgalloc.h
index 96a8d927db..5767cdfc08 100644
--- a/arch/ia64/include/asm/pgalloc.h
+++ b/arch/ia64/include/asm/pgalloc.h
@@ -91,7 +91,10 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long addr)
if (!pg)
return NULL;
page = virt_to_page(pg);
- pgtable_page_ctor(page);
+ if (!pgtable_page_ctor(page)) {
+ quicklist_free(0, NULL, pg);
+ return NULL;
+ }
return page;
}
--
1.8.4.rc3
Signed-off-by: Kirill A. Shutemov <[email protected]>
Cc: Guan Xuetao <[email protected]>
---
arch/unicore32/include/asm/pgalloc.h | 14 ++++++++------
1 file changed, 8 insertions(+), 6 deletions(-)
diff --git a/arch/unicore32/include/asm/pgalloc.h b/arch/unicore32/include/asm/pgalloc.h
index 0213e373a8..2e02d1356f 100644
--- a/arch/unicore32/include/asm/pgalloc.h
+++ b/arch/unicore32/include/asm/pgalloc.h
@@ -51,12 +51,14 @@ pte_alloc_one(struct mm_struct *mm, unsigned long addr)
struct page *pte;
pte = alloc_pages(PGALLOC_GFP, 0);
- if (pte) {
- if (!PageHighMem(pte)) {
- void *page = page_address(pte);
- clean_dcache_area(page, PTRS_PER_PTE * sizeof(pte_t));
- }
- pgtable_page_ctor(pte);
+ if (!pte)
+ return NULL;
+ if (!PageHighMem(pte)) {
+ void *page = page_address(pte);
+ clean_dcache_area(page, PTRS_PER_PTE * sizeof(pte_t));
+ }
+ if (!pgtable_page_ctor(pte)) {
+ __free_page(pte);
}
return pte;
--
1.8.4.rc3
Signed-off-by: Kirill A. Shutemov <[email protected]>
Cc: Hirokazu Takata <[email protected]>
---
arch/m32r/include/asm/pgalloc.h | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/arch/m32r/include/asm/pgalloc.h b/arch/m32r/include/asm/pgalloc.h
index ac4208bcc5..2d55a064cc 100644
--- a/arch/m32r/include/asm/pgalloc.h
+++ b/arch/m32r/include/asm/pgalloc.h
@@ -45,7 +45,10 @@ static __inline__ pgtable_t pte_alloc_one(struct mm_struct *mm,
if (!pte)
return NULL;
- pgtable_page_ctor(pte);
+ if (!pgtable_page_ctor(pte)) {
+ __free_page(pte);
+ return NULL;
+ }
return pte;
}
--
1.8.4.rc3
Signed-off-by: Kirill A. Shutemov <[email protected]>
Cc: Ralf Baechle <[email protected]>
---
arch/mips/include/asm/pgalloc.h | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/arch/mips/include/asm/pgalloc.h b/arch/mips/include/asm/pgalloc.h
index 881d18b4e2..b336037e87 100644
--- a/arch/mips/include/asm/pgalloc.h
+++ b/arch/mips/include/asm/pgalloc.h
@@ -80,9 +80,12 @@ static inline struct page *pte_alloc_one(struct mm_struct *mm,
struct page *pte;
pte = alloc_pages(GFP_KERNEL | __GFP_REPEAT, PTE_ORDER);
- if (pte) {
- clear_highpage(pte);
- pgtable_page_ctor(pte);
+ if (!pte)
+ return NULL;
+ clear_highpage(pte);
+ if (!pgtable_page_ctor(pte)) {
+ __free_page(pte);
+ return NULL;
}
return pte;
}
--
1.8.4.rc3
Signed-off-by: Kirill A. Shutemov <[email protected]>
Cc: Richard Kuo <[email protected]>
---
arch/hexagon/include/asm/pgalloc.h | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/arch/hexagon/include/asm/pgalloc.h b/arch/hexagon/include/asm/pgalloc.h
index 679bf6d664..4c9d382d77 100644
--- a/arch/hexagon/include/asm/pgalloc.h
+++ b/arch/hexagon/include/asm/pgalloc.h
@@ -65,10 +65,12 @@ static inline struct page *pte_alloc_one(struct mm_struct *mm,
struct page *pte;
pte = alloc_page(GFP_KERNEL | __GFP_REPEAT | __GFP_ZERO);
-
- if (pte)
- pgtable_page_ctor(pte);
-
+ if (!pte)
+ return NULL;
+ if (!pgtable_page_ctor(pte)) {
+ __free_page(pte);
+ return NULL;
+ }
return pte;
}
--
1.8.4.rc3
It will fix NR_PAGETABLE accounting. It's also required if the arch is
going ever support split ptl.
Signed-off-by: Kirill A. Shutemov <[email protected]>
Cc: David Howells <[email protected]>
Cc: Koichi Yasutake <[email protected]>
---
arch/mn10300/include/asm/pgalloc.h | 1 +
arch/mn10300/mm/pgtable.c | 9 +++++++--
2 files changed, 8 insertions(+), 2 deletions(-)
diff --git a/arch/mn10300/include/asm/pgalloc.h b/arch/mn10300/include/asm/pgalloc.h
index 146bacf193..0f25d5fa86 100644
--- a/arch/mn10300/include/asm/pgalloc.h
+++ b/arch/mn10300/include/asm/pgalloc.h
@@ -46,6 +46,7 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
static inline void pte_free(struct mm_struct *mm, struct page *pte)
{
+ pgtable_page_dtor(pte);
__free_page(pte);
}
diff --git a/arch/mn10300/mm/pgtable.c b/arch/mn10300/mm/pgtable.c
index bd9ada693f..e77a7c7280 100644
--- a/arch/mn10300/mm/pgtable.c
+++ b/arch/mn10300/mm/pgtable.c
@@ -78,8 +78,13 @@ struct page *pte_alloc_one(struct mm_struct *mm, unsigned long address)
#else
pte = alloc_pages(GFP_KERNEL|__GFP_REPEAT, 0);
#endif
- if (pte)
- clear_highpage(pte);
+ if (!pte)
+ return NULL;
+ clear_highpage(pte);
+ if (!pgtable_page_ctor(pte)) {
+ __free_page(pte);
+ return NULL;
+ }
return pte;
}
--
1.8.4.rc3
Signed-off-by: Kirill A. Shutemov <[email protected]>
Cc: Geert Uytterhoeven <[email protected]>
---
arch/m68k/include/asm/motorola_pgalloc.h | 5 ++++-
arch/m68k/include/asm/sun3_pgalloc.h | 5 ++++-
2 files changed, 8 insertions(+), 2 deletions(-)
diff --git a/arch/m68k/include/asm/motorola_pgalloc.h b/arch/m68k/include/asm/motorola_pgalloc.h
index 2f02f264e6..dd254eeb03 100644
--- a/arch/m68k/include/asm/motorola_pgalloc.h
+++ b/arch/m68k/include/asm/motorola_pgalloc.h
@@ -40,7 +40,10 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long addres
flush_tlb_kernel_page(pte);
nocache_page(pte);
kunmap(page);
- pgtable_page_ctor(page);
+ if (!pgtable_page_ctor(page)) {
+ __free_page(page);
+ return NULL;
+ }
return page;
}
diff --git a/arch/m68k/include/asm/sun3_pgalloc.h b/arch/m68k/include/asm/sun3_pgalloc.h
index 48d80d5a66..f868506e33 100644
--- a/arch/m68k/include/asm/sun3_pgalloc.h
+++ b/arch/m68k/include/asm/sun3_pgalloc.h
@@ -59,7 +59,10 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
return NULL;
clear_highpage(page);
- pgtable_page_ctor(page);
+ if (!pgtable_page_ctor(page)) {
+ __free_page(page);
+ return NULL;
+ }
return page;
}
--
1.8.4.rc3
Signed-off-by: Kirill A. Shutemov <[email protected]>
Cc: Vineet Gupta <[email protected]>
---
arch/arc/include/asm/pgalloc.h | 11 ++++++++---
1 file changed, 8 insertions(+), 3 deletions(-)
diff --git a/arch/arc/include/asm/pgalloc.h b/arch/arc/include/asm/pgalloc.h
index 36a9f20c21..81208bfd9d 100644
--- a/arch/arc/include/asm/pgalloc.h
+++ b/arch/arc/include/asm/pgalloc.h
@@ -105,11 +105,16 @@ static inline pgtable_t
pte_alloc_one(struct mm_struct *mm, unsigned long address)
{
pgtable_t pte_pg;
+ struct page *page;
pte_pg = __get_free_pages(GFP_KERNEL | __GFP_REPEAT, __get_order_pte());
- if (pte_pg) {
- memzero((void *)pte_pg, PTRS_PER_PTE * 4);
- pgtable_page_ctor(virt_to_page(pte_pg));
+ if (!pte_pg)
+ return 0;
+ memzero((void *)pte_pg, PTRS_PER_PTE * 4);
+ page = virt_to_page(pte_pg);
+ if (!pgtable_page_ctor(page)) {
+ __free_page(page);
+ return 0;
}
return pte_pg;
--
1.8.4.rc3
Signed-off-by: Kirill A. Shutemov <[email protected]>
Cc: David Howells <[email protected]>
---
arch/frv/mm/pgalloc.c | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
diff --git a/arch/frv/mm/pgalloc.c b/arch/frv/mm/pgalloc.c
index f6084bc524..41907d25ed 100644
--- a/arch/frv/mm/pgalloc.c
+++ b/arch/frv/mm/pgalloc.c
@@ -37,11 +37,15 @@ pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
#else
page = alloc_pages(GFP_KERNEL|__GFP_REPEAT, 0);
#endif
- if (page) {
- clear_highpage(page);
- pgtable_page_ctor(page);
- flush_dcache_page(page);
+ if (!page)
+ return NULL;
+
+ clear_highpage(page);
+ if (!pgtable_page_ctor(page)) {
+ __free_page(page);
+ return NULL;
}
+ flush_dcache_page(page);
return page;
}
--
1.8.4.rc3
Signed-off-by: Kirill A. Shutemov <[email protected]>
Cc: Mikael Starvik <[email protected]>
Cc: Jesper Nilsson <[email protected]>
---
arch/cris/include/asm/pgalloc.h | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/arch/cris/include/asm/pgalloc.h b/arch/cris/include/asm/pgalloc.h
index d9504d38c2..235ece437d 100644
--- a/arch/cris/include/asm/pgalloc.h
+++ b/arch/cris/include/asm/pgalloc.h
@@ -34,7 +34,10 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long addres
pte = alloc_pages(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO, 0);
if (!pte)
return NULL;
- pgtable_page_ctor(pte);
+ if (!pgtable_page_ctor(pte)) {
+ __free_page(pte);
+ return NULL;
+ }
return pte;
}
--
1.8.4.rc3
Add missing check for memory allocation fail.
Signed-off-by: Kirill A. Shutemov <[email protected]>
Cc: Mikael Starvik <[email protected]>
Cc: Jesper Nilsson <[email protected]>
---
arch/cris/include/asm/pgalloc.h | 2 ++
1 file changed, 2 insertions(+)
diff --git a/arch/cris/include/asm/pgalloc.h b/arch/cris/include/asm/pgalloc.h
index 6da975db11..d9504d38c2 100644
--- a/arch/cris/include/asm/pgalloc.h
+++ b/arch/cris/include/asm/pgalloc.h
@@ -32,6 +32,8 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long addres
{
struct page *pte;
pte = alloc_pages(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO, 0);
+ if (!pte)
+ return NULL;
pgtable_page_ctor(pte);
return pte;
}
--
1.8.4.rc3
Signed-off-by: Kirill A. Shutemov <[email protected]>
Cc: Richard Henderson <[email protected]>
Cc: Ivan Kokshaysky <[email protected]>
Cc: Matt Turner <[email protected]>
---
arch/alpha/include/asm/pgalloc.h | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/arch/alpha/include/asm/pgalloc.h b/arch/alpha/include/asm/pgalloc.h
index bc2a0daf2d..aab14a019c 100644
--- a/arch/alpha/include/asm/pgalloc.h
+++ b/arch/alpha/include/asm/pgalloc.h
@@ -72,7 +72,10 @@ pte_alloc_one(struct mm_struct *mm, unsigned long address)
if (!pte)
return NULL;
page = virt_to_page(pte);
- pgtable_page_ctor(page);
+ if (!pgtable_page_ctor(page)) {
+ __free_page(page);
+ return NULL;
+ }
return page;
}
--
1.8.4.rc3
Signed-off-by: Kirill A. Shutemov <[email protected]>
Cc: Haavard Skinnemoen <[email protected]>
Cc: Hans-Christian Egtvedt <[email protected]>
---
arch/avr32/include/asm/pgalloc.h | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/arch/avr32/include/asm/pgalloc.h b/arch/avr32/include/asm/pgalloc.h
index bc7e8ae479..1aba19d68c 100644
--- a/arch/avr32/include/asm/pgalloc.h
+++ b/arch/avr32/include/asm/pgalloc.h
@@ -68,7 +68,10 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
return NULL;
page = virt_to_page(pg);
- pgtable_page_ctor(page);
+ if (!pgtable_page_ctor(page)) {
+ quicklist_free(QUICK_PT, NULL, pg);
+ return NULL;
+ }
return page;
}
--
1.8.4.rc3
I've missed that we preallocate few pmds on pgd_alloc() if X86_PAE
enabled. Let's add missed constructor/destructor calls.
I haven't noticed it during testing since prep_new_page() clears
page->mapping and therefore page->ptl. It's effectively equal to
spin_lock_init(&page->ptl).
Signed-off-by: Kirill A. Shutemov <[email protected]>
---
arch/x86/mm/pgtable.c | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
diff --git a/arch/x86/mm/pgtable.c b/arch/x86/mm/pgtable.c
index dfa537a03b..1a7d21342e 100644
--- a/arch/x86/mm/pgtable.c
+++ b/arch/x86/mm/pgtable.c
@@ -189,8 +189,10 @@ static void free_pmds(pmd_t *pmds[])
int i;
for(i = 0; i < PREALLOCATED_PMDS; i++)
- if (pmds[i])
+ if (pmds[i]) {
+ pgtable_pmd_page_dtor(virt_to_page(pmds[i]));
free_page((unsigned long)pmds[i]);
+ }
}
static int preallocate_pmds(pmd_t *pmds[])
@@ -200,8 +202,13 @@ static int preallocate_pmds(pmd_t *pmds[])
for(i = 0; i < PREALLOCATED_PMDS; i++) {
pmd_t *pmd = (pmd_t *)__get_free_page(PGALLOC_GFP);
- if (pmd == NULL)
+ if (!pmd)
+ failed = true;
+ if (pmd && !pgtable_pmd_page_ctor(virt_to_page(pmd))) {
+ free_page((unsigned long)pmds[i]);
+ pmd = NULL;
failed = true;
+ }
pmds[i] = pmd;
}
--
1.8.4.rc3
Signed-off-by: Kirill A. Shutemov <[email protected]>
Cc: Catalin Marinas <[email protected]>
Cc: Will Deacon <[email protected]>
---
arch/arm64/include/asm/pgalloc.h | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/arch/arm64/include/asm/pgalloc.h b/arch/arm64/include/asm/pgalloc.h
index f214069ec5..9bea6e74a0 100644
--- a/arch/arm64/include/asm/pgalloc.h
+++ b/arch/arm64/include/asm/pgalloc.h
@@ -63,9 +63,12 @@ pte_alloc_one(struct mm_struct *mm, unsigned long addr)
struct page *pte;
pte = alloc_pages(PGALLOC_GFP, 0);
- if (pte)
- pgtable_page_ctor(pte);
-
+ if (!pte)
+ return NULL;
+ if (!pgtable_page_ctor(pte)) {
+ __free_page(pte);
+ return NULL;
+ }
return pte;
}
--
1.8.4.rc3
Signed-off-by: Kirill A. Shutemov <[email protected]>
Cc: Russell King <[email protected]>
---
arch/arm/include/asm/pgalloc.h | 12 +++++++-----
1 file changed, 7 insertions(+), 5 deletions(-)
diff --git a/arch/arm/include/asm/pgalloc.h b/arch/arm/include/asm/pgalloc.h
index 943504f53f..78a7793616 100644
--- a/arch/arm/include/asm/pgalloc.h
+++ b/arch/arm/include/asm/pgalloc.h
@@ -102,12 +102,14 @@ pte_alloc_one(struct mm_struct *mm, unsigned long addr)
#else
pte = alloc_pages(PGALLOC_GFP, 0);
#endif
- if (pte) {
- if (!PageHighMem(pte))
- clean_pte_table(page_address(pte));
- pgtable_page_ctor(pte);
+ if (!pte)
+ return NULL;
+ if (!PageHighMem(pte))
+ clean_pte_table(page_address(pte));
+ if (!pgtable_page_ctor(pte)) {
+ __free_page(pte);
+ return NULL;
}
-
return pte;
}
--
1.8.4.rc3
It will fix NR_PAGETABLE accounting. It's also required if the arch is
going ever support split ptl.
Signed-off-by: Kirill A. Shutemov <[email protected]>
Cc: Jonas Bonn <[email protected]>
---
arch/openrisc/include/asm/pgalloc.h | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
diff --git a/arch/openrisc/include/asm/pgalloc.h b/arch/openrisc/include/asm/pgalloc.h
index 05c39ecd2e..21484e5b9e 100644
--- a/arch/openrisc/include/asm/pgalloc.h
+++ b/arch/openrisc/include/asm/pgalloc.h
@@ -78,8 +78,13 @@ static inline struct page *pte_alloc_one(struct mm_struct *mm,
{
struct page *pte;
pte = alloc_pages(GFP_KERNEL|__GFP_REPEAT, 0);
- if (pte)
- clear_page(page_address(pte));
+ if (!pte)
+ return NULL;
+ clear_page(page_address(pte));
+ if (!pgtable_page_ctor(pte)) {
+ __free_page(pte);
+ return NULL;
+ }
return pte;
}
@@ -90,6 +95,7 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
static inline void pte_free(struct mm_struct *mm, struct page *pte)
{
+ pgtable_page_dtor(pte);
__free_page(pte);
}
--
1.8.4.rc3
It will fix NR_PAGETABLE accounting. It's also required if the arch is
going ever support split ptl.
Signed-off-by: Kirill A. Shutemov <[email protected]>
Cc: Michal Simek <[email protected]>
---
arch/microblaze/include/asm/pgalloc.h | 12 +++++++++---
1 file changed, 9 insertions(+), 3 deletions(-)
diff --git a/arch/microblaze/include/asm/pgalloc.h b/arch/microblaze/include/asm/pgalloc.h
index ebd3579248..7fdf7fabc7 100644
--- a/arch/microblaze/include/asm/pgalloc.h
+++ b/arch/microblaze/include/asm/pgalloc.h
@@ -122,8 +122,13 @@ static inline struct page *pte_alloc_one(struct mm_struct *mm,
#endif
ptepage = alloc_pages(flags, 0);
- if (ptepage)
- clear_highpage(ptepage);
+ if (!ptepage)
+ return NULL;
+ clear_highpage(ptepage);
+ if (!pgtable_page_ctor(ptepage)) {
+ __free_page(ptepage);
+ return NULL;
+ }
return ptepage;
}
@@ -158,8 +163,9 @@ extern inline void pte_free_slow(struct page *ptepage)
__free_page(ptepage);
}
-extern inline void pte_free(struct mm_struct *mm, struct page *ptepage)
+static inline void pte_free(struct mm_struct *mm, struct page *ptepage)
{
+ pgtable_page_dtor(ptepage);
__free_page(ptepage);
}
--
1.8.4.rc3
Change pgtable_page_ctor() return type from void to bool.
Returns true, if initialization is successful and false otherwise.
Current implementation never fails, but it will change later.
Signed-off-by: Kirill A. Shutemov <[email protected]>
---
include/linux/mm.h | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 75735f6171..f6467032a9 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -1254,10 +1254,11 @@ static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long a
#define pte_lockptr(mm, pmd) ({(void)(pmd); &(mm)->page_table_lock;})
#endif /* USE_SPLIT_PTE_PTLOCKS */
-static inline void pgtable_page_ctor(struct page *page)
+static inline bool pgtable_page_ctor(struct page *page)
{
pte_lock_init(page);
inc_zone_page_state(page, NR_PAGETABLE);
+ return true;
}
static inline void pgtable_page_dtor(struct page *page)
--
1.8.4.rc3
From: "Kirill A. Shutemov" <[email protected]>
Date: Thu, 10 Oct 2013 21:05:52 +0300
> Signed-off-by: Kirill A. Shutemov <[email protected]>
Acked-by: David S. Miller <[email protected]>
Around Thu 10 Oct 2013 21:05:38 +0300 or thereabout, Kirill A. Shutemov wrote:
> Signed-off-by: Kirill A. Shutemov <[email protected]>
Acked-by: Hans-Christian Egtvedt <[email protected]>
Given [1].
> Cc: Haavard Skinnemoen <[email protected]>
> Cc: Hans-Christian Egtvedt <[email protected]>
> ---
> arch/avr32/include/asm/pgalloc.h | 5 ++++-
> 1 file changed, 4 insertions(+), 1 deletion(-)
>
> diff --git a/arch/avr32/include/asm/pgalloc.h b/arch/avr32/include/asm/pgalloc.h
> index bc7e8ae479..1aba19d68c 100644
> --- a/arch/avr32/include/asm/pgalloc.h
> +++ b/arch/avr32/include/asm/pgalloc.h
> @@ -68,7 +68,10 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
> return NULL;
>
> page = virt_to_page(pg);
> - pgtable_page_ctor(page);
> + if (!pgtable_page_ctor(page)) {
> + quicklist_free(QUICK_PT, NULL, pg);
> + return NULL;
> + }
>
> return page;
> }
1: I'm assuming that pgtable_page_ctor() now returns success/error, but there
is a patch series that I have not seen.
--
mvh
Hans-Christian Egtvedt
On Thu, Oct 10, 2013 at 09:05:25PM +0300, Kirill A. Shutemov wrote:
> Any comments?
Reviewed-by: Peter Zijlstra <[email protected]>
On 10/10/2013 2:05 PM, Kirill A. Shutemov wrote:
> Signed-off-by: Kirill A. Shutemov <[email protected]>
> Cc: Chris Metcalf <[email protected]>
> ---
> arch/tile/mm/pgtable.c | 6 +++++-
> 1 file changed, 5 insertions(+), 1 deletion(-)
Acked-by: Chris Metcalf <[email protected]>
--
Chris Metcalf, Tilera Corp.
http://www.tilera.com
On Thu, 10 Oct 2013, Kirill A. Shutemov wrote:
> +static inline bool ptlock_alloc(struct page *page)
> +{
> + if (sizeof(spinlock_t) > sizeof(page->ptl))
> + return __ptlock_alloc(page);
> + return true;
> +}
Could you make the check a CONFIG option? CONFIG_PTLOCK_DOES_NOT_FIT_IN_PAGE_STRUCT or
so?
> --- a/include/linux/mm_types.h
> +++ b/include/linux/mm_types.h
> @@ -147,7 +147,10 @@ struct page {
> * system if PG_buddy is set.
> */
> #if USE_SPLIT_PTE_PTLOCKS
> - spinlock_t ptl;
> + unsigned long ptl; /* It's spinlock_t if it fits to long,
> + * otherwise it's pointer to dynamicaly
> + * allocated spinlock_t.
> + */
If you had such a CONFIG option then you could use the proper type here.
#ifdef CONFIG_PTLOCK_NOT_FITTING
spinlock_t *ptl;
#else
spinlock_t ptl;
#endif
Or some such thing?
Christoph Lameter wrote:
> On Thu, 10 Oct 2013, Kirill A. Shutemov wrote:
>
> > +static inline bool ptlock_alloc(struct page *page)
> > +{
> > + if (sizeof(spinlock_t) > sizeof(page->ptl))
> > + return __ptlock_alloc(page);
> > + return true;
> > +}
>
> Could you make the check a CONFIG option? CONFIG_PTLOCK_DOES_NOT_FIT_IN_PAGE_STRUCT or
> so?
No. We will have to track what affects sizeof(spinlock_t) manually.
Not a fun and error prune.
C sucks. ;)
--
Kirill A. Shutemov
So, all I see is this patch, with such a brilliant description which
describes what this change is about, why it is being made, and so
forth, and you're sending it to me, presumably because you want me to
do something with it. No, not really.
What context do I have to say whether this is correct or not? How can
I test it when the mainline version of pgtable_page_ctor returns void,
so if I were to apply this patch I'd get compile errors.
Oh, I guess you're changing pgtable_page_ctor() in some way. What is
the nature of that change?
Please, I'm not a mind reader. Please ensure that your "generic" patch
of your series reaches the appropriate recipients: if you don't want to
explicitly Cc: all the people individually, please at least copy all
relevant mailing lists found for the entire series.
(No, I am not on the excessively noisy linux-arch: I dropped off it
years ago because it just became yet another mailing list to endlessly
talk mainly about x86 rather than being a separate list to linux-kernel
which discussed problems relevant to many arch maintainers.)
Thanks.
On Thu, Oct 10, 2013 at 09:05:36PM +0300, Kirill A. Shutemov wrote:
> Signed-off-by: Kirill A. Shutemov <[email protected]>
> Cc: Russell King <[email protected]>
> ---
> arch/arm/include/asm/pgalloc.h | 12 +++++++-----
> 1 file changed, 7 insertions(+), 5 deletions(-)
>
> diff --git a/arch/arm/include/asm/pgalloc.h b/arch/arm/include/asm/pgalloc.h
> index 943504f53f..78a7793616 100644
> --- a/arch/arm/include/asm/pgalloc.h
> +++ b/arch/arm/include/asm/pgalloc.h
> @@ -102,12 +102,14 @@ pte_alloc_one(struct mm_struct *mm, unsigned long addr)
> #else
> pte = alloc_pages(PGALLOC_GFP, 0);
> #endif
> - if (pte) {
> - if (!PageHighMem(pte))
> - clean_pte_table(page_address(pte));
> - pgtable_page_ctor(pte);
> + if (!pte)
> + return NULL;
> + if (!PageHighMem(pte))
> + clean_pte_table(page_address(pte));
> + if (!pgtable_page_ctor(pte)) {
> + __free_page(pte);
> + return NULL;
> }
> -
> return pte;
> }
>
> --
> 1.8.4.rc3
>
On Thu, Oct 10, 2013 at 09:18:05PM +0100, Russell King - ARM Linux wrote:
> So, all I see is this patch, with such a brilliant description which
> describes what this change is about, why it is being made, and so
> forth, and you're sending it to me, presumably because you want me to
> do something with it. No, not really.
Fair enough. Description should be better.
> What context do I have to say whether this is correct or not? How can
> I test it when the mainline version of pgtable_page_ctor returns void,
> so if I were to apply this patch I'd get compile errors.
>
> Oh, I guess you're changing pgtable_page_ctor() in some way. What is
> the nature of that change?
>
> Please, I'm not a mind reader. Please ensure that your "generic" patch
> of your series reaches the appropriate recipients: if you don't want to
> explicitly Cc: all the people individually, please at least copy all
> relevant mailing lists found for the entire series.
The patchset touches every arch with MMU -- the list would be too long.
I hoped all maintainers has access to archive of linux-kernel/linux-arch
to get context.
mbox with cover letter and three relevant patches attached.
> (No, I am not on the excessively noisy linux-arch: I dropped off it
> years ago because it just became yet another mailing list to endlessly
> talk mainly about x86 rather than being a separate list to linux-kernel
> which discussed problems relevant to many arch maintainers.)
>
> Thanks.
>
> On Thu, Oct 10, 2013 at 09:05:36PM +0300, Kirill A. Shutemov wrote:
> > Signed-off-by: Kirill A. Shutemov <[email protected]>
> > Cc: Russell King <[email protected]>
> > ---
> > arch/arm/include/asm/pgalloc.h | 12 +++++++-----
> > 1 file changed, 7 insertions(+), 5 deletions(-)
> >
> > diff --git a/arch/arm/include/asm/pgalloc.h b/arch/arm/include/asm/pgalloc.h
> > index 943504f53f..78a7793616 100644
> > --- a/arch/arm/include/asm/pgalloc.h
> > +++ b/arch/arm/include/asm/pgalloc.h
> > @@ -102,12 +102,14 @@ pte_alloc_one(struct mm_struct *mm, unsigned long addr)
> > #else
> > pte = alloc_pages(PGALLOC_GFP, 0);
> > #endif
> > - if (pte) {
> > - if (!PageHighMem(pte))
> > - clean_pte_table(page_address(pte));
> > - pgtable_page_ctor(pte);
> > + if (!pte)
> > + return NULL;
> > + if (!PageHighMem(pte))
> > + clean_pte_table(page_address(pte));
> > + if (!pgtable_page_ctor(pte)) {
> > + __free_page(pte);
> > + return NULL;
> > }
> > -
> > return pte;
> > }
> >
> > --
> > 1.8.4.rc3
> >
--
Kirill A. Shutemov
* Kirill A. Shutemov <[email protected]> wrote:
> I've missed that we preallocate few pmds on pgd_alloc() if X86_PAE
> enabled. Let's add missed constructor/destructor calls.
>
> I haven't noticed it during testing since prep_new_page() clears
> page->mapping and therefore page->ptl. It's effectively equal to
> spin_lock_init(&page->ptl).
>
> Signed-off-by: Kirill A. Shutemov <[email protected]>
Acked-by: Ingo Molnar <[email protected]>
Thanks,
Ingo
On Thu, Oct 10, 2013 at 08:05:39PM +0200, Kirill A. Shutemov wrote:
> Signed-off-by: Kirill A. Shutemov <[email protected]>
> Cc: Mikael Starvik <[email protected]>
Acked-by: Jesper Nilsson <[email protected]>
> ---
> arch/cris/include/asm/pgalloc.h | 5 ++++-
> 1 file changed, 4 insertions(+), 1 deletion(-)
>
> diff --git a/arch/cris/include/asm/pgalloc.h b/arch/cris/include/asm/pgalloc.h
> index d9504d38c2..235ece437d 100644
> --- a/arch/cris/include/asm/pgalloc.h
> +++ b/arch/cris/include/asm/pgalloc.h
> @@ -34,7 +34,10 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long addres
> pte = alloc_pages(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO, 0);
> if (!pte)
> return NULL;
> - pgtable_page_ctor(pte);
> + if (!pgtable_page_ctor(pte)) {
> + __free_page(pte);
> + return NULL;
> + }
> return pte;
> }
>
> --
> 1.8.4.rc3
/^JN - Jesper Nilsson
--
Jesper Nilsson -- [email protected]
On Thu, Oct 10, 2013 at 08:05:27PM +0200, Kirill A. Shutemov wrote:
> Add missing check for memory allocation fail.
>
> Signed-off-by: Kirill A. Shutemov <[email protected]>
> Cc: Mikael Starvik <[email protected]>
Acked-by: Jesper Nilsson <[email protected]>
> ---
> arch/cris/include/asm/pgalloc.h | 2 ++
> 1 file changed, 2 insertions(+)
>
> diff --git a/arch/cris/include/asm/pgalloc.h b/arch/cris/include/asm/pgalloc.h
> index 6da975db11..d9504d38c2 100644
> --- a/arch/cris/include/asm/pgalloc.h
> +++ b/arch/cris/include/asm/pgalloc.h
> @@ -32,6 +32,8 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long addres
> {
> struct page *pte;
> pte = alloc_pages(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO, 0);
> + if (!pte)
> + return NULL;
> pgtable_page_ctor(pte);
> return pte;
> }
> --
> 1.8.4.rc3
/^JN - Jesper Nilsson
--
Jesper Nilsson -- [email protected]
On Thu, Oct 10, 2013 at 07:05:58PM +0100, Kirill A. Shutemov wrote:
> Signed-off-by: Kirill A. Shutemov <[email protected]>
> Cc: Will Deacon <[email protected]>
> Cc: Grant Likely <[email protected]>
> Cc: Rob Herring <[email protected]>
> ---
> drivers/iommu/arm-smmu.c | 5 ++++-
> 1 file changed, 4 insertions(+), 1 deletion(-)
>
> diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c
> index f417e89e1e..2b256a5075 100644
> --- a/drivers/iommu/arm-smmu.c
> +++ b/drivers/iommu/arm-smmu.c
> @@ -1211,7 +1211,10 @@ static int arm_smmu_alloc_init_pte(struct arm_smmu_device *smmu, pmd_t *pmd,
>
> arm_smmu_flush_pgtable(smmu, page_address(table),
> ARM_SMMU_PTE_HWTABLE_SIZE);
> - pgtable_page_ctor(table);
> + if (!pgtable_page_ctor(table)) {
> + __free_page(table);
> + return -ENOMEM;
> + }
> pmd_populate(NULL, pmd, table);
> arm_smmu_flush_pgtable(smmu, pmd, sizeof(*pmd));
> }
Acked-by: Will Deacon <[email protected]>
I have quite a few changes queued for this driver, but it doesn't look like
you'll get a conflict with the iommu tree.
Will
On Thu, 10 Oct 2013, Kirill A. Shutemov wrote:
> Christoph Lameter wrote:
> > On Thu, 10 Oct 2013, Kirill A. Shutemov wrote:
> >
> > > +static inline bool ptlock_alloc(struct page *page)
> > > +{
> > > + if (sizeof(spinlock_t) > sizeof(page->ptl))
> > > + return __ptlock_alloc(page);
> > > + return true;
> > > +}
> >
> > Could you make the check a CONFIG option? CONFIG_PTLOCK_DOES_NOT_FIT_IN_PAGE_STRUCT or
> > so?
>
> No. We will have to track what affects sizeof(spinlock_t) manually.
> Not a fun and error prune.
You can generate a config option depending on the size of the object via
Kbuild. Kbuild will determine the setting before building the kernel as a
whole by runing some small C program.
Acked-by: David Howells <[email protected]>
for the FRV and MN10300 patches.
Can you move pte_alloc_one() to common code, at least for some arches? I
think that the FRV and MN10300 ones should end up the same after this - and I
wouldn't be surprised if some of the other arches do too.
David
Thanks.
Acked-by: Lennox Wu <[email protected]>
?? 2013/10/11 ?W?? 02:05, Kirill A. Shutemov ????:
> Signed-off-by: Kirill A. Shutemov <[email protected]>
> Cc: Chen Liqin <[email protected]>
> Cc: Lennox Wu <[email protected]>
> ---
> arch/score/include/asm/pgalloc.h | 9 ++++++---
> 1 file changed, 6 insertions(+), 3 deletions(-)
>
> diff --git a/arch/score/include/asm/pgalloc.h b/arch/score/include/asm/pgalloc.h
> index 059a61b707..2a861ffbd5 100644
> --- a/arch/score/include/asm/pgalloc.h
> +++ b/arch/score/include/asm/pgalloc.h
> @@ -54,9 +54,12 @@ static inline struct page *pte_alloc_one(struct mm_struct *mm,
> struct page *pte;
>
> pte = alloc_pages(GFP_KERNEL | __GFP_REPEAT, PTE_ORDER);
> - if (pte) {
> - clear_highpage(pte);
> - pgtable_page_ctor(pte);
> + if (!pte)
> + return NULL;
> + clear_highpage(pte);
> + if (!pgtable_page_ctor(pte)) {
> + __free_page(pte);
> + return NULL;
> }
> return pte;
> }
Christoph Lameter wrote:
> On Thu, 10 Oct 2013, Kirill A. Shutemov wrote:
>
> > Christoph Lameter wrote:
> > > On Thu, 10 Oct 2013, Kirill A. Shutemov wrote:
> > >
> > > > +static inline bool ptlock_alloc(struct page *page)
> > > > +{
> > > > + if (sizeof(spinlock_t) > sizeof(page->ptl))
> > > > + return __ptlock_alloc(page);
> > > > + return true;
> > > > +}
> > >
> > > Could you make the check a CONFIG option? CONFIG_PTLOCK_DOES_NOT_FIT_IN_PAGE_STRUCT or
> > > so?
> >
> > No. We will have to track what affects sizeof(spinlock_t) manually.
> > Not a fun and error prune.
>
> You can generate a config option depending on the size of the object via
> Kbuild. Kbuild will determine the setting before building the kernel as a
> whole by runing some small C program.
I don't think it's any better than what we have there now.
--
Kirill A. Shutemov
David Howells wrote:
>
> Acked-by: David Howells <[email protected]>
>
> for the FRV and MN10300 patches.
>
> Can you move pte_alloc_one() to common code, at least for some arches? I
> think that the FRV and MN10300 ones should end up the same after this - and I
> wouldn't be surprised if some of the other arches do too.
There's no true approach for generic. It depends on what pgtable_t is:
pointer to struct page or virtual address of the allocated page table.
Some arches also use some sort of cache for page table allocator. Others
don't.
I don't see a sensible way generalize it.
--
Kirill A. Shutemov
On Mon, 14 Oct 2013, Kirill A. Shutemov wrote:
> > > > Could you make the check a CONFIG option? CONFIG_PTLOCK_DOES_NOT_FIT_IN_PAGE_STRUCT or
> > > > so?
> > >
> > > No. We will have to track what affects sizeof(spinlock_t) manually.
> > > Not a fun and error prune.
> >
> > You can generate a config option depending on the size of the object via
> > Kbuild. Kbuild will determine the setting before building the kernel as a
> > whole by runing some small C program.
>
> I don't think it's any better than what we have there now.
Well with the CONFIG options we can then create macros etc that handle
things differently depending on the ptl being in the page or not.
Christoph Lameter wrote:
> On Mon, 14 Oct 2013, Kirill A. Shutemov wrote:
>
> > > > > Could you make the check a CONFIG option? CONFIG_PTLOCK_DOES_NOT_FIT_IN_PAGE_STRUCT or
> > > > > so?
> > > >
> > > > No. We will have to track what affects sizeof(spinlock_t) manually.
> > > > Not a fun and error prune.
> > >
> > > You can generate a config option depending on the size of the object via
> > > Kbuild. Kbuild will determine the setting before building the kernel as a
> > > whole by runing some small C program.
> >
> > I don't think it's any better than what we have there now.
>
> Well with the CONFIG options we can then create macros etc that handle
> things differently depending on the ptl being in the page or not.
Feel free to propose a patch. I don't see much point.
--
Kirill A. Shutemov
On Tue, 15 Oct 2013, Kirill A. Shutemov wrote:
> Feel free to propose a patch. I don't see much point.
Right now you are using a long to stand in for a spinlock_t or a pointer
to a spinlock_t. An #ifdef would allow to define the proper type and
therefore the compiler to check that the ptl is correctly used.
Christoph Lameter wrote:
> On Tue, 15 Oct 2013, Kirill A. Shutemov wrote:
>
> > Feel free to propose a patch. I don't see much point.
>
> Right now you are using a long to stand in for a spinlock_t or a pointer
> to a spinlock_t. An #ifdef would allow to define the proper type and
> therefore the compiler to check that the ptl is correctly used.
You should not use it directly anyway: page->ptl is not there at all if
USE_SPLIT_PTE_PTLOCKS is 0. Compiler checks limited to few helpers and use
a kbuild hack is overkill to me.
--
Kirill A. Shutemov
On 10/10/2013 11:35 PM, Kirill A. Shutemov wrote:
> Signed-off-by: Kirill A. Shutemov <[email protected]>
Acked-by: Vineet Gupta <[email protected]> [for arch/arc bits]
Thx,
-Vineet
On Thu, Oct 10, 2013 at 8:05 PM, Kirill A. Shutemov
<[email protected]> wrote:
> Signed-off-by: Kirill A. Shutemov <[email protected]>
> Cc: Geert Uytterhoeven <[email protected]>
> ---
> arch/m68k/include/asm/motorola_pgalloc.h | 5 ++++-
> arch/m68k/include/asm/sun3_pgalloc.h | 5 ++++-
> 2 files changed, 8 insertions(+), 2 deletions(-)
>
> diff --git a/arch/m68k/include/asm/motorola_pgalloc.h b/arch/m68k/include/asm/motorola_pgalloc.h
> index 2f02f264e6..dd254eeb03 100644
> --- a/arch/m68k/include/asm/motorola_pgalloc.h
> +++ b/arch/m68k/include/asm/motorola_pgalloc.h
> @@ -40,7 +40,10 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long addres
> flush_tlb_kernel_page(pte);
> nocache_page(pte);
^^^^^^^^^^^^^^^^^^
> kunmap(page);
> - pgtable_page_ctor(page);
> + if (!pgtable_page_ctor(page)) {
> + __free_page(page);
Shouldn't you mark the page cacheable again, like is done in pte_free()?
> + return NULL;
> + }
> return page;
> }
>
> diff --git a/arch/m68k/include/asm/sun3_pgalloc.h b/arch/m68k/include/asm/sun3_pgalloc.h
> index 48d80d5a66..f868506e33 100644
> --- a/arch/m68k/include/asm/sun3_pgalloc.h
> +++ b/arch/m68k/include/asm/sun3_pgalloc.h
> @@ -59,7 +59,10 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
> return NULL;
>
> clear_highpage(page);
> - pgtable_page_ctor(page);
> + if (!pgtable_page_ctor(page)) {
> + __free_page(page);
> + return NULL;
> + }
> return page;
Gr{oetje,eeting}s,
Geert
--
Geert Uytterhoeven -- There's lots of Linux beyond ia32 -- [email protected]
In personal conversations with technical people, I call myself a hacker. But
when I'm talking to journalists I just say "programmer" or something like that.
-- Linus Torvalds
Geert Uytterhoeven wrote:
> On Thu, Oct 10, 2013 at 8:05 PM, Kirill A. Shutemov
> <[email protected]> wrote:
> > Signed-off-by: Kirill A. Shutemov <[email protected]>
> > Cc: Geert Uytterhoeven <[email protected]>
> > ---
> > arch/m68k/include/asm/motorola_pgalloc.h | 5 ++++-
> > arch/m68k/include/asm/sun3_pgalloc.h | 5 ++++-
> > 2 files changed, 8 insertions(+), 2 deletions(-)
> >
> > diff --git a/arch/m68k/include/asm/motorola_pgalloc.h b/arch/m68k/include/asm/motorola_pgalloc.h
> > index 2f02f264e6..dd254eeb03 100644
> > --- a/arch/m68k/include/asm/motorola_pgalloc.h
> > +++ b/arch/m68k/include/asm/motorola_pgalloc.h
> > @@ -40,7 +40,10 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long addres
> > flush_tlb_kernel_page(pte);
> > nocache_page(pte);
> ^^^^^^^^^^^^^^^^^^
> > kunmap(page);
> > - pgtable_page_ctor(page);
> > + if (!pgtable_page_ctor(page)) {
> > + __free_page(page);
>
> Shouldn't you mark the page cacheable again, like is done in pte_free()?
Hm. You're right. Updated patch below.
BTW, what's the point of playing with kmap()/kunmap() there? Looks like
m68k doesn't support highmem.
And even if it will support, code in pte_alloc_one() doesn't make any
sense: kmap() creates temporary mapping for highmem page and it
nocache_page() sets flags in pte which will be destroyed by kunmap().
It think kmap() should be replaced by page_address() and kunmap() should
be dropped.
>From 2f9dce7982e4459e91cd49e38256c24d69786f8b Mon Sep 17 00:00:00 2001
From: "Kirill A. Shutemov" <[email protected]>
Date: Wed, 9 Oct 2013 03:58:39 +0300
Subject: [PATCHv2] m68k: handle pgtable_page_ctor() fail
Signed-off-by: Kirill A. Shutemov <[email protected]>
Cc: Geert Uytterhoeven <[email protected]>
---
arch/m68k/include/asm/motorola_pgalloc.h | 8 ++++++--
arch/m68k/include/asm/sun3_pgalloc.h | 5 ++++-
2 files changed, 10 insertions(+), 3 deletions(-)
diff --git a/arch/m68k/include/asm/motorola_pgalloc.h b/arch/m68k/include/asm/motorola_pgalloc.h
index 2f02f264e6..24bcba496c 100644
--- a/arch/m68k/include/asm/motorola_pgalloc.h
+++ b/arch/m68k/include/asm/motorola_pgalloc.h
@@ -29,18 +29,22 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
static inline pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
{
- struct page *page = alloc_pages(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO, 0);
+ struct page *page;
pte_t *pte;
+ page = alloc_pages(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO, 0);
if(!page)
return NULL;
+ if (!pgtable_page_ctor(page)) {
+ __free_page(page);
+ return NULL;
+ }
pte = kmap(page);
__flush_page_to_ram(pte);
flush_tlb_kernel_page(pte);
nocache_page(pte);
kunmap(page);
- pgtable_page_ctor(page);
return page;
}
diff --git a/arch/m68k/include/asm/sun3_pgalloc.h b/arch/m68k/include/asm/sun3_pgalloc.h
index 48d80d5a66..f868506e33 100644
--- a/arch/m68k/include/asm/sun3_pgalloc.h
+++ b/arch/m68k/include/asm/sun3_pgalloc.h
@@ -59,7 +59,10 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
return NULL;
clear_highpage(page);
- pgtable_page_ctor(page);
+ if (!pgtable_page_ctor(page)) {
+ __free_page(page);
+ return NULL;
+ }
return page;
}
--
Kirill A. Shutemov
Kirill A. Shutemov wrote:
> Geert Uytterhoeven wrote:
> > On Thu, Oct 10, 2013 at 8:05 PM, Kirill A. Shutemov
> > <[email protected]> wrote:
> > > Signed-off-by: Kirill A. Shutemov <[email protected]>
> > > Cc: Geert Uytterhoeven <[email protected]>
> > > ---
> > > arch/m68k/include/asm/motorola_pgalloc.h | 5 ++++-
> > > arch/m68k/include/asm/sun3_pgalloc.h | 5 ++++-
> > > 2 files changed, 8 insertions(+), 2 deletions(-)
> > >
> > > diff --git a/arch/m68k/include/asm/motorola_pgalloc.h b/arch/m68k/include/asm/motorola_pgalloc.h
> > > index 2f02f264e6..dd254eeb03 100644
> > > --- a/arch/m68k/include/asm/motorola_pgalloc.h
> > > +++ b/arch/m68k/include/asm/motorola_pgalloc.h
> > > @@ -40,7 +40,10 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long addres
> > > flush_tlb_kernel_page(pte);
> > > nocache_page(pte);
> > ^^^^^^^^^^^^^^^^^^
> > > kunmap(page);
> > > - pgtable_page_ctor(page);
> > > + if (!pgtable_page_ctor(page)) {
> > > + __free_page(page);
> >
> > Shouldn't you mark the page cacheable again, like is done in pte_free()?
>
> Hm. You're right. Updated patch below.
I also noticed that one of pte_alloc_one didn't have the constructor at
all. New patch revision bellow.
>From 22109077ef31ea28bbfe6b72902854214285bc0c Mon Sep 17 00:00:00 2001
From: "Kirill A. Shutemov" <[email protected]>
Date: Wed, 9 Oct 2013 03:58:39 +0300
Subject: [PATCHv3] m68k: handle pgtable_page_ctor() fail
Signed-off-by: Kirill A. Shutemov <[email protected]>
Cc: Geert Uytterhoeven <[email protected]>
---
arch/m68k/include/asm/mcf_pgalloc.h | 4 ++++
arch/m68k/include/asm/motorola_pgalloc.h | 8 ++++++--
arch/m68k/include/asm/sun3_pgalloc.h | 5 ++++-
3 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/arch/m68k/include/asm/mcf_pgalloc.h b/arch/m68k/include/asm/mcf_pgalloc.h
index 313f3dd23c..f9924fbcfe 100644
--- a/arch/m68k/include/asm/mcf_pgalloc.h
+++ b/arch/m68k/include/asm/mcf_pgalloc.h
@@ -56,6 +56,10 @@ static inline struct page *pte_alloc_one(struct mm_struct *mm,
if (!page)
return NULL;
+ if (!pgtable_page_ctor(page)) {
+ __free_page(page);
+ return NULL;
+ }
pte = kmap(page);
if (pte) {
diff --git a/arch/m68k/include/asm/motorola_pgalloc.h b/arch/m68k/include/asm/motorola_pgalloc.h
index 2f02f264e6..24bcba496c 100644
--- a/arch/m68k/include/asm/motorola_pgalloc.h
+++ b/arch/m68k/include/asm/motorola_pgalloc.h
@@ -29,18 +29,22 @@ static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
static inline pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
{
- struct page *page = alloc_pages(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO, 0);
+ struct page *page;
pte_t *pte;
+ page = alloc_pages(GFP_KERNEL|__GFP_REPEAT|__GFP_ZERO, 0);
if(!page)
return NULL;
+ if (!pgtable_page_ctor(page)) {
+ __free_page(page);
+ return NULL;
+ }
pte = kmap(page);
__flush_page_to_ram(pte);
flush_tlb_kernel_page(pte);
nocache_page(pte);
kunmap(page);
- pgtable_page_ctor(page);
return page;
}
diff --git a/arch/m68k/include/asm/sun3_pgalloc.h b/arch/m68k/include/asm/sun3_pgalloc.h
index 48d80d5a66..f868506e33 100644
--- a/arch/m68k/include/asm/sun3_pgalloc.h
+++ b/arch/m68k/include/asm/sun3_pgalloc.h
@@ -59,7 +59,10 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm,
return NULL;
clear_highpage(page);
- pgtable_page_ctor(page);
+ if (!pgtable_page_ctor(page)) {
+ __free_page(page);
+ return NULL;
+ }
return page;
}
--
Kirill A. Shutemov