For git users patches are available in git:
git://github.com/aryabinin/linux.git kasan/arm64v3
Changes since v2:
- Rebase on top of v4.2-rc3
- Address feedback from Catalin.
- Print memory assignment fro Linus
- Add message about KASAN being initialized
Changes since v1:
- Address feedback from Catalin.
- Generalize some kasan init code from arch/x86/mm/kasan_init_64.c
and reuse it for arm64.
- Some bugfixes, including:
add missing arm64/include/asm/kasan.h
add tlb flush after changing ttbr1
- Add code comments.
Andrey Ryabinin (4):
mm: kasan: introduce generic kasan_populate_zero_shadow()
arm64: introduce VA_START macro - the first kernel virtual address.
arm64: move PGD_SIZE definition to pgalloc.h
arm64: add KASAN support
Linus Walleij (1):
ARM64: kasan: print memory assignment
arch/arm64/Kconfig | 17 ++++
arch/arm64/include/asm/kasan.h | 24 ++++++
arch/arm64/include/asm/memory.h | 2 +
arch/arm64/include/asm/pgalloc.h | 1 +
arch/arm64/include/asm/pgtable.h | 9 +-
arch/arm64/include/asm/string.h | 16 ++++
arch/arm64/kernel/arm64ksyms.c | 3 +
arch/arm64/kernel/head.S | 3 +
arch/arm64/kernel/module.c | 16 +++-
arch/arm64/kernel/setup.c | 2 +
arch/arm64/lib/memcpy.S | 3 +
arch/arm64/lib/memmove.S | 7 +-
arch/arm64/lib/memset.S | 3 +
arch/arm64/mm/Makefile | 3 +
arch/arm64/mm/init.c | 6 ++
arch/arm64/mm/kasan_init.c | 176 +++++++++++++++++++++++++++++++++++++++
arch/arm64/mm/pgd.c | 2 -
arch/x86/mm/kasan_init_64.c | 8 +-
include/linux/kasan.h | 8 ++
mm/kasan/Makefile | 2 +-
mm/kasan/kasan_init.c | 142 +++++++++++++++++++++++++++++++
21 files changed, 440 insertions(+), 13 deletions(-)
create mode 100644 arch/arm64/include/asm/kasan.h
create mode 100644 arch/arm64/mm/kasan_init.c
create mode 100644 mm/kasan/kasan_init.c
--
2.4.5
Introduce generic kasan_populate_zero_shadow(start, end).
This function maps kasan_zero_page to the [start, end] addresses.
In follow on patches it will be used for ARMv8 (and maybe other
architectures) and will replace x86_64 specific populate_zero_shadow().
Signed-off-by: Andrey Ryabinin <[email protected]>
---
arch/x86/mm/kasan_init_64.c | 8 +--
include/linux/kasan.h | 8 +++
mm/kasan/Makefile | 2 +-
mm/kasan/kasan_init.c | 142 ++++++++++++++++++++++++++++++++++++++++++++
4 files changed, 155 insertions(+), 5 deletions(-)
create mode 100644 mm/kasan/kasan_init.c
diff --git a/arch/x86/mm/kasan_init_64.c b/arch/x86/mm/kasan_init_64.c
index e1840f3..2390dba 100644
--- a/arch/x86/mm/kasan_init_64.c
+++ b/arch/x86/mm/kasan_init_64.c
@@ -12,9 +12,9 @@
extern pgd_t early_level4_pgt[PTRS_PER_PGD];
extern struct range pfn_mapped[E820_X_MAX];
-static pud_t kasan_zero_pud[PTRS_PER_PUD] __page_aligned_bss;
-static pmd_t kasan_zero_pmd[PTRS_PER_PMD] __page_aligned_bss;
-static pte_t kasan_zero_pte[PTRS_PER_PTE] __page_aligned_bss;
+pud_t kasan_zero_pud[PTRS_PER_PUD] __page_aligned_bss;
+pmd_t kasan_zero_pmd[PTRS_PER_PMD] __page_aligned_bss;
+pte_t kasan_zero_pte[PTRS_PER_PTE] __page_aligned_bss;
/*
* This page used as early shadow. We don't use empty_zero_page
@@ -24,7 +24,7 @@ static pte_t kasan_zero_pte[PTRS_PER_PTE] __page_aligned_bss;
* that allowed to access, but not instrumented by kasan
* (vmalloc/vmemmap ...).
*/
-static unsigned char kasan_zero_page[PAGE_SIZE] __page_aligned_bss;
+unsigned char kasan_zero_page[PAGE_SIZE] __page_aligned_bss;
static int __init map_range(struct range *range)
{
diff --git a/include/linux/kasan.h b/include/linux/kasan.h
index 5486d77..5ef3925 100644
--- a/include/linux/kasan.h
+++ b/include/linux/kasan.h
@@ -13,8 +13,16 @@ struct vm_struct;
#define KASAN_SHADOW_OFFSET _AC(CONFIG_KASAN_SHADOW_OFFSET, UL)
#include <asm/kasan.h>
+#include <asm/pgtable.h>
#include <linux/sched.h>
+extern unsigned char kasan_zero_page[PAGE_SIZE];
+extern pte_t kasan_zero_pte[PTRS_PER_PTE];
+extern pmd_t kasan_zero_pmd[PTRS_PER_PMD];
+extern pud_t kasan_zero_pud[PTRS_PER_PUD];
+
+void kasan_populate_zero_shadow(const void *start, const void *end);
+
static inline void *kasan_mem_to_shadow(const void *addr)
{
return (void *)((unsigned long)addr >> KASAN_SHADOW_SCALE_SHIFT)
diff --git a/mm/kasan/Makefile b/mm/kasan/Makefile
index bd837b8..6471014 100644
--- a/mm/kasan/Makefile
+++ b/mm/kasan/Makefile
@@ -5,4 +5,4 @@ CFLAGS_REMOVE_kasan.o = -pg
# see: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=63533
CFLAGS_kasan.o := $(call cc-option, -fno-conserve-stack -fno-stack-protector)
-obj-y := kasan.o report.o
+obj-y := kasan.o report.o kasan_init.o
diff --git a/mm/kasan/kasan_init.c b/mm/kasan/kasan_init.c
new file mode 100644
index 0000000..37fb46a
--- /dev/null
+++ b/mm/kasan/kasan_init.c
@@ -0,0 +1,142 @@
+#include <linux/bootmem.h>
+#include <linux/init.h>
+#include <linux/kasan.h>
+#include <linux/kernel.h>
+#include <linux/memblock.h>
+#include <linux/pfn.h>
+
+#include <asm/page.h>
+#include <asm/pgalloc.h>
+
+static __init void *early_alloc(size_t size, int node)
+{
+ return memblock_virt_alloc_try_nid(size, size, __pa(MAX_DMA_ADDRESS),
+ BOOTMEM_ALLOC_ACCESSIBLE, node);
+}
+
+static int __init zero_pte_populate(pmd_t *pmd, unsigned long addr,
+ unsigned long end)
+{
+ pte_t *pte = pte_offset_kernel(pmd, addr);
+ pte_t zero_pte;
+
+ zero_pte = pfn_pte(PFN_DOWN(__pa(kasan_zero_page)), PAGE_KERNEL);
+ zero_pte = pte_wrprotect(zero_pte);
+
+ while (addr + PAGE_SIZE <= end) {
+ set_pte_at(&init_mm, addr, pte, zero_pte);
+ addr += PAGE_SIZE;
+ pte = pte_offset_kernel(pmd, addr);
+ }
+ return 0;
+}
+
+static int __init zero_pmd_populate(pud_t *pud, unsigned long addr,
+ unsigned long end)
+{
+ int ret = 0;
+ pmd_t *pmd = pmd_offset(pud, addr);
+ unsigned long next;
+
+ do {
+ next = pmd_addr_end(addr, end);
+
+ if (IS_ALIGNED(addr, PMD_SIZE) && end - addr >= PMD_SIZE) {
+ pmd_populate_kernel(&init_mm, pmd, kasan_zero_pte);
+ continue;
+ }
+
+ if (pmd_none(*pmd)) {
+ void *p = early_alloc(PAGE_SIZE, NUMA_NO_NODE);
+ if (!p)
+ return -ENOMEM;
+ pmd_populate_kernel(&init_mm, pmd, p);
+ }
+ zero_pte_populate(pmd, addr, pmd_addr_end(addr, end));
+ } while (pmd++, addr = next, addr != end);
+
+ return ret;
+}
+
+static int __init zero_pud_populate(pgd_t *pgd, unsigned long addr,
+ unsigned long end)
+{
+ int ret = 0;
+ pud_t *pud = pud_offset(pgd, addr);
+ unsigned long next;
+
+ do {
+ next = pud_addr_end(addr, end);
+ if (IS_ALIGNED(addr, PUD_SIZE) && end - addr >= PUD_SIZE) {
+ pmd_t *pmd;
+
+ pud_populate(&init_mm, pud, kasan_zero_pmd);
+ pmd = pmd_offset(pud, addr);
+ pmd_populate_kernel(&init_mm, pmd, kasan_zero_pte);
+ continue;
+ }
+
+ if (pud_none(*pud)) {
+ void *p = early_alloc(PAGE_SIZE, NUMA_NO_NODE);
+ if (!p)
+ return -ENOMEM;
+ pud_populate(&init_mm, pud, p);
+ }
+ zero_pmd_populate(pud, addr, pud_addr_end(addr, end));
+ } while (pud++, addr = next, addr != end);
+
+ return ret;
+}
+
+static int __init zero_pgd_populate(unsigned long addr, unsigned long end)
+{
+ int ret = 0;
+ pgd_t *pgd = pgd_offset_k(addr);
+ unsigned long next;
+
+ do {
+ next = pgd_addr_end(addr, end);
+
+ if (IS_ALIGNED(addr, PGDIR_SIZE) && end - addr >= PGDIR_SIZE) {
+ pud_t *pud;
+ pmd_t *pmd;
+
+ /*
+ * kasan_zero_pud should be populated with pmds
+ * at this moment.
+ * [pud,pmd]_populate*() bellow needed only for
+ * 3,2 - level page tables where we don't have
+ * puds,pmds, so pgd_populate(), pud_populate()
+ * is noops.
+ */
+ pgd_populate(&init_mm, pgd, kasan_zero_pud);
+ pud = pud_offset(pgd, addr);
+ pud_populate(&init_mm, pud, kasan_zero_pmd);
+ pmd = pmd_offset(pud, addr);
+ pmd_populate_kernel(&init_mm, pmd, kasan_zero_pte);
+ continue;
+ }
+
+ if (pgd_none(*pgd)) {
+ void *p = early_alloc(PAGE_SIZE, NUMA_NO_NODE);
+ if (!p)
+ return -ENOMEM;
+ pgd_populate(&init_mm, pgd, p);
+ }
+ zero_pud_populate(pgd, addr, next);
+ } while (pgd++, addr = next, addr != end);
+
+ return ret;
+}
+
+/**
+ * kasan_populate_zero_shadow - populate shadow memory region with
+ * kasan_zero_page
+ * @start - start of the memory range to populate
+ * @end - end of the memory range to populate
+ */
+void __init kasan_populate_zero_shadow(const void *start, const void *end)
+{
+ if (zero_pgd_populate((unsigned long)start, (unsigned long)end))
+ panic("kasan: unable to map zero shadow!");
+}
--
2.4.5
In order to not use lengthy (UL(0xffffffffffffffff) << VA_BITS) everywhere,
replace it with VA_START.
Signed-off-by: Andrey Ryabinin <[email protected]>
---
arch/arm64/include/asm/memory.h | 2 ++
arch/arm64/include/asm/pgtable.h | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h
index f800d45..288c19d 100644
--- a/arch/arm64/include/asm/memory.h
+++ b/arch/arm64/include/asm/memory.h
@@ -42,12 +42,14 @@
* PAGE_OFFSET - the virtual address of the start of the kernel image (top
* (VA_BITS - 1))
* VA_BITS - the maximum number of bits for virtual addresses.
+ * VA_START - the first kernel virtual address.
* TASK_SIZE - the maximum size of a user space task.
* TASK_UNMAPPED_BASE - the lower boundary of the mmap VM area.
* The module space lives between the addresses given by TASK_SIZE
* and PAGE_OFFSET - it must be within 128MB of the kernel text.
*/
#define VA_BITS (CONFIG_ARM64_VA_BITS)
+#define VA_START (UL(0xffffffffffffffff) << VA_BITS)
#define PAGE_OFFSET (UL(0xffffffffffffffff) << (VA_BITS - 1))
#define MODULES_END (PAGE_OFFSET)
#define MODULES_VADDR (MODULES_END - SZ_64M)
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index 56283f8..a1f9f61 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -40,7 +40,7 @@
* fixed mappings and modules
*/
#define VMEMMAP_SIZE ALIGN((1UL << (VA_BITS - PAGE_SHIFT)) * sizeof(struct page), PUD_SIZE)
-#define VMALLOC_START (UL(0xffffffffffffffff) << VA_BITS)
+#define VMALLOC_START (VA_START)
#define VMALLOC_END (PAGE_OFFSET - PUD_SIZE - VMEMMAP_SIZE - SZ_64K)
#define vmemmap ((struct page *)(VMALLOC_END + SZ_64K))
--
2.4.5
This will be used by KASAN latter.
Signed-off-by: Andrey Ryabinin <[email protected]>
---
arch/arm64/include/asm/pgalloc.h | 1 +
arch/arm64/mm/pgd.c | 2 --
2 files changed, 1 insertion(+), 2 deletions(-)
diff --git a/arch/arm64/include/asm/pgalloc.h b/arch/arm64/include/asm/pgalloc.h
index 7642056..c150539 100644
--- a/arch/arm64/include/asm/pgalloc.h
+++ b/arch/arm64/include/asm/pgalloc.h
@@ -27,6 +27,7 @@
#define check_pgt_cache() do { } while (0)
#define PGALLOC_GFP (GFP_KERNEL | __GFP_NOTRACK | __GFP_REPEAT | __GFP_ZERO)
+#define PGD_SIZE (PTRS_PER_PGD * sizeof(pgd_t))
#if CONFIG_PGTABLE_LEVELS > 2
diff --git a/arch/arm64/mm/pgd.c b/arch/arm64/mm/pgd.c
index 71ca104..cb3ba1b 100644
--- a/arch/arm64/mm/pgd.c
+++ b/arch/arm64/mm/pgd.c
@@ -28,8 +28,6 @@
#include "mm.h"
-#define PGD_SIZE (PTRS_PER_PGD * sizeof(pgd_t))
-
static struct kmem_cache *pgd_cache;
pgd_t *pgd_alloc(struct mm_struct *mm)
--
2.4.5
This patch adds arch specific code for kernel address sanitizer
(see Documentation/kasan.txt).
1/8 of kernel addresses reserved for shadow memory. There was no
big enough hole for this, so virtual addresses for shadow were
stolen from vmalloc area.
At early boot stage the whole shadow region populated with just
one physical page (kasan_zero_page). Later, this page reused
as readonly zero shadow for some memory that KASan currently
don't track (vmalloc).
After mapping the physical memory, pages for shadow memory are
allocated and mapped.
Functions like memset/memmove/memcpy do a lot of memory accesses.
If bad pointer passed to one of these function it is important
to catch this. Compiler's instrumentation cannot do this since
these functions are written in assembly.
KASan replaces memory functions with manually instrumented variants.
Original functions declared as weak symbols so strong definitions
in mm/kasan/kasan.c could replace them. Original functions have aliases
with '__' prefix in name, so we could call non-instrumented variant
if needed.
Some files built without kasan instrumentation (e.g. mm/slub.c).
Original mem* function replaced (via #define) with prefixed variants
to disable memory access checks for such files.
Signed-off-by: Andrey Ryabinin <[email protected]>
Tested-by: Linus Walleij <[email protected]>
---
arch/arm64/Kconfig | 17 ++++
arch/arm64/include/asm/kasan.h | 24 ++++++
arch/arm64/include/asm/pgtable.h | 7 ++
arch/arm64/include/asm/string.h | 16 ++++
arch/arm64/kernel/arm64ksyms.c | 3 +
arch/arm64/kernel/head.S | 3 +
arch/arm64/kernel/module.c | 16 +++-
arch/arm64/kernel/setup.c | 2 +
arch/arm64/lib/memcpy.S | 3 +
arch/arm64/lib/memmove.S | 7 +-
arch/arm64/lib/memset.S | 3 +
arch/arm64/mm/Makefile | 3 +
arch/arm64/mm/kasan_init.c | 176 +++++++++++++++++++++++++++++++++++++++
13 files changed, 275 insertions(+), 5 deletions(-)
create mode 100644 arch/arm64/include/asm/kasan.h
create mode 100644 arch/arm64/mm/kasan_init.c
diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 318175f..61ebb7c 100644
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -46,6 +46,7 @@ config ARM64
select HAVE_ARCH_AUDITSYSCALL
select HAVE_ARCH_BITREVERSE
select HAVE_ARCH_JUMP_LABEL
+ select HAVE_ARCH_KASAN if SPARSEMEM_VMEMMAP
select HAVE_ARCH_KGDB
select HAVE_ARCH_SECCOMP_FILTER
select HAVE_ARCH_TRACEHOOK
@@ -122,6 +123,22 @@ config GENERIC_CSUM
config GENERIC_CALIBRATE_DELAY
def_bool y
+config KASAN_SHADOW_OFFSET
+ hex
+ depends on KASAN
+ default 0xdfff200000000000 if ARM64_VA_BITS_48
+ default 0xdffffc8000000000 if ARM64_VA_BITS_42
+ default 0xdfffff9000000000 if ARM64_VA_BITS_39
+ help
+ This value used to address to corresponding shadow address
+ by the following formula:
+ shadow_addr = (address >> 3) + KASAN_SHADOW_OFFSET;
+
+ (1 << 61) shadow addresses - [KASAN_SHADOW_OFFSET,KASAN_SHADOW_END]
+ cover all 64-bits of virtual addresses. So KASAN_SHADOW_OFFSET
+ should satisfy the following equation:
+ KASAN_SHADOW_OFFSET = KASAN_SHADOW_END - (1ULL << 61)
+
config ZONE_DMA
def_bool y
diff --git a/arch/arm64/include/asm/kasan.h b/arch/arm64/include/asm/kasan.h
new file mode 100644
index 0000000..914dc72
--- /dev/null
+++ b/arch/arm64/include/asm/kasan.h
@@ -0,0 +1,24 @@
+#ifndef __ASM_KASAN_H
+#define __ASM_KASAN_H
+
+#ifndef __ASSEMBLY__
+
+#ifdef CONFIG_KASAN
+
+#include <asm/memory.h>
+
+/*
+ * KASAN_SHADOW_START: beginning of the kernel virtual addresses.
+ * KASAN_SHADOW_END: KASAN_SHADOW_START + 1/8 of kernel virtual addresses.
+ */
+#define KASAN_SHADOW_START (VA_START)
+#define KASAN_SHADOW_END (KASAN_SHADOW_START + (1UL << (VA_BITS - 3)))
+
+void kasan_init(void);
+
+#else
+static inline void kasan_init(void) { }
+#endif
+
+#endif
+#endif
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index a1f9f61..a071da4 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -40,7 +40,14 @@
* fixed mappings and modules
*/
#define VMEMMAP_SIZE ALIGN((1UL << (VA_BITS - PAGE_SHIFT)) * sizeof(struct page), PUD_SIZE)
+
+#ifndef CONFIG_KASAN
#define VMALLOC_START (VA_START)
+#else
+#include <asm/kasan.h>
+#define VMALLOC_START (KASAN_SHADOW_END + SZ_64K)
+#endif
+
#define VMALLOC_END (PAGE_OFFSET - PUD_SIZE - VMEMMAP_SIZE - SZ_64K)
#define vmemmap ((struct page *)(VMALLOC_END + SZ_64K))
diff --git a/arch/arm64/include/asm/string.h b/arch/arm64/include/asm/string.h
index 64d2d48..2eb714c 100644
--- a/arch/arm64/include/asm/string.h
+++ b/arch/arm64/include/asm/string.h
@@ -36,17 +36,33 @@ extern __kernel_size_t strnlen(const char *, __kernel_size_t);
#define __HAVE_ARCH_MEMCPY
extern void *memcpy(void *, const void *, __kernel_size_t);
+extern void *__memcpy(void *, const void *, __kernel_size_t);
#define __HAVE_ARCH_MEMMOVE
extern void *memmove(void *, const void *, __kernel_size_t);
+extern void *__memmove(void *, const void *, __kernel_size_t);
#define __HAVE_ARCH_MEMCHR
extern void *memchr(const void *, int, __kernel_size_t);
#define __HAVE_ARCH_MEMSET
extern void *memset(void *, int, __kernel_size_t);
+extern void *__memset(void *, int, __kernel_size_t);
#define __HAVE_ARCH_MEMCMP
extern int memcmp(const void *, const void *, size_t);
+
+#if defined(CONFIG_KASAN) && !defined(__SANITIZE_ADDRESS__)
+
+/*
+ * For files that are not instrumented (e.g. mm/slub.c) we
+ * should use not instrumented version of mem* functions.
+ */
+
+#define memcpy(dst, src, len) __memcpy(dst, src, len)
+#define memmove(dst, src, len) __memmove(dst, src, len)
+#define memset(s, c, n) __memset(s, c, n)
+#endif
+
#endif
diff --git a/arch/arm64/kernel/arm64ksyms.c b/arch/arm64/kernel/arm64ksyms.c
index a85843d..3b6d8cc 100644
--- a/arch/arm64/kernel/arm64ksyms.c
+++ b/arch/arm64/kernel/arm64ksyms.c
@@ -51,6 +51,9 @@ EXPORT_SYMBOL(strnlen);
EXPORT_SYMBOL(memset);
EXPORT_SYMBOL(memcpy);
EXPORT_SYMBOL(memmove);
+EXPORT_SYMBOL(__memset);
+EXPORT_SYMBOL(__memcpy);
+EXPORT_SYMBOL(__memmove);
EXPORT_SYMBOL(memchr);
EXPORT_SYMBOL(memcmp);
diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index c0ff3ce..945bd6d 100644
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -449,6 +449,9 @@ __mmap_switched:
str_l x21, __fdt_pointer, x5 // Save FDT pointer
str_l x24, memstart_addr, x6 // Save PHYS_OFFSET
mov x29, #0
+#ifdef CONFIG_KASAN
+ bl kasan_early_init
+#endif
b start_kernel
ENDPROC(__mmap_switched)
diff --git a/arch/arm64/kernel/module.c b/arch/arm64/kernel/module.c
index 67bf410..7d90c0f 100644
--- a/arch/arm64/kernel/module.c
+++ b/arch/arm64/kernel/module.c
@@ -21,6 +21,7 @@
#include <linux/bitops.h>
#include <linux/elf.h>
#include <linux/gfp.h>
+#include <linux/kasan.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/moduleloader.h>
@@ -34,9 +35,18 @@
void *module_alloc(unsigned long size)
{
- return __vmalloc_node_range(size, 1, MODULES_VADDR, MODULES_END,
- GFP_KERNEL, PAGE_KERNEL_EXEC, 0,
- NUMA_NO_NODE, __builtin_return_address(0));
+ void *p;
+
+ p = __vmalloc_node_range(size, MODULE_ALIGN, MODULES_VADDR, MODULES_END,
+ GFP_KERNEL, PAGE_KERNEL_EXEC, 0,
+ NUMA_NO_NODE, __builtin_return_address(0));
+
+ if (p && (kasan_module_alloc(p, size) < 0)) {
+ vfree(p);
+ return NULL;
+ }
+
+ return p;
}
enum aarch64_reloc_op {
diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c
index f3067d4..bf82ab8 100644
--- a/arch/arm64/kernel/setup.c
+++ b/arch/arm64/kernel/setup.c
@@ -54,6 +54,7 @@
#include <asm/elf.h>
#include <asm/cpufeature.h>
#include <asm/cpu_ops.h>
+#include <asm/kasan.h>
#include <asm/sections.h>
#include <asm/setup.h>
#include <asm/smp_plat.h>
@@ -392,6 +393,7 @@ void __init setup_arch(char **cmdline_p)
acpi_boot_table_init();
paging_init();
+ kasan_init();
request_standard_resources();
early_ioremap_reset();
diff --git a/arch/arm64/lib/memcpy.S b/arch/arm64/lib/memcpy.S
index 8a9a96d..42cc4b7 100644
--- a/arch/arm64/lib/memcpy.S
+++ b/arch/arm64/lib/memcpy.S
@@ -56,6 +56,8 @@ C_h .req x12
D_l .req x13
D_h .req x14
+ .weak memcpy
+ENTRY(__memcpy)
ENTRY(memcpy)
mov dst, dstin
cmp count, #16
@@ -199,3 +201,4 @@ ENTRY(memcpy)
b.ne .Ltail63
ret
ENDPROC(memcpy)
+ENDPROC(__memcpy)
diff --git a/arch/arm64/lib/memmove.S b/arch/arm64/lib/memmove.S
index 57b19ea..8819433 100644
--- a/arch/arm64/lib/memmove.S
+++ b/arch/arm64/lib/memmove.S
@@ -57,12 +57,14 @@ C_h .req x12
D_l .req x13
D_h .req x14
+ .weak memmove
+ENTRY(__memmove)
ENTRY(memmove)
cmp dstin, src
- b.lo memcpy
+ b.lo __memcpy
add tmp1, src, count
cmp dstin, tmp1
- b.hs memcpy /* No overlap. */
+ b.hs __memcpy /* No overlap. */
add dst, dstin, count
add src, src, count
@@ -195,3 +197,4 @@ ENTRY(memmove)
b.ne .Ltail63
ret
ENDPROC(memmove)
+ENDPROC(__memmove)
diff --git a/arch/arm64/lib/memset.S b/arch/arm64/lib/memset.S
index 7c72dfd..edc0e7d 100644
--- a/arch/arm64/lib/memset.S
+++ b/arch/arm64/lib/memset.S
@@ -54,6 +54,8 @@ dst .req x8
tmp3w .req w9
tmp3 .req x9
+ .weak memset
+ENTRY(__memset)
ENTRY(memset)
mov dst, dstin /* Preserve return value. */
and A_lw, val, #255
@@ -214,3 +216,4 @@ ENTRY(memset)
b.ne .Ltail_maybe_long
ret
ENDPROC(memset)
+ENDPROC(__memset)
diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile
index 773d37a..57f57fd 100644
--- a/arch/arm64/mm/Makefile
+++ b/arch/arm64/mm/Makefile
@@ -4,3 +4,6 @@ obj-y := dma-mapping.o extable.o fault.o init.o \
context.o proc.o pageattr.o
obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o
obj-$(CONFIG_ARM64_PTDUMP) += dump.o
+
+obj-$(CONFIG_KASAN) += kasan_init.o
+KASAN_SANITIZE_kasan_init.o := n
diff --git a/arch/arm64/mm/kasan_init.c b/arch/arm64/mm/kasan_init.c
new file mode 100644
index 0000000..4e67453
--- /dev/null
+++ b/arch/arm64/mm/kasan_init.c
@@ -0,0 +1,176 @@
+#define pr_fmt(fmt) "kasan: " fmt
+#include <linux/kasan.h>
+#include <linux/kernel.h>
+#include <linux/memblock.h>
+#include <linux/start_kernel.h>
+
+#include <asm/page.h>
+#include <asm/pgalloc.h>
+#include <asm/pgtable.h>
+#include <asm/tlbflush.h>
+
+/*
+ * This page serves two purposes:
+ * - It used as early shadow memory. The entire shadow region populated
+ * with this page, before we will be able to setup normal shadow memory.
+ * - Latter it reused it as zero shadow to cover large ranges of memory
+ * that allowed to access, but not handled by kasan (vmalloc/vmemmap ...).
+ */
+unsigned char kasan_zero_page[PAGE_SIZE] __page_aligned_bss;
+
+static pgd_t tmp_pg_dir[PTRS_PER_PGD] __initdata __aligned(PGD_SIZE);
+
+#if CONFIG_PGTABLE_LEVELS > 3
+pud_t kasan_zero_pud[PTRS_PER_PUD] __page_aligned_bss;
+#endif
+#if CONFIG_PGTABLE_LEVELS > 2
+pmd_t kasan_zero_pmd[PTRS_PER_PMD] __page_aligned_bss;
+#endif
+pte_t kasan_zero_pte[PTRS_PER_PTE] __page_aligned_bss;
+
+static void __init kasan_early_pte_populate(pmd_t *pmd, unsigned long addr,
+ unsigned long end)
+{
+ pte_t *pte;
+ unsigned long next;
+
+ if (pmd_none(*pmd))
+ pmd_populate_kernel(&init_mm, pmd, kasan_zero_pte);
+
+ pte = pte_offset_kernel(pmd, addr);
+ do {
+ next = addr + PAGE_SIZE;
+ set_pte(pte, pfn_pte(virt_to_pfn(kasan_zero_page),
+ PAGE_KERNEL));
+ } while (pte++, addr = next, addr != end && pte_none(*pte));
+}
+
+static void __init kasan_early_pmd_populate(pud_t *pud,
+ unsigned long addr,
+ unsigned long end)
+{
+ pmd_t *pmd;
+ unsigned long next;
+
+ if (pud_none(*pud))
+ pud_populate(&init_mm, pud, kasan_zero_pmd);
+
+ pmd = pmd_offset(pud, addr);
+ do {
+ next = pmd_addr_end(addr, end);
+ kasan_early_pte_populate(pmd, addr, next);
+ } while (pmd++, addr = next, addr != end && pmd_none(*pmd));
+}
+
+static void __init kasan_early_pud_populate(pgd_t *pgd,
+ unsigned long addr,
+ unsigned long end)
+{
+ pud_t *pud;
+ unsigned long next;
+
+ if (pgd_none(*pgd))
+ pgd_populate(&init_mm, pgd, kasan_zero_pud);
+
+ pud = pud_offset(pgd, addr);
+ do {
+ next = pud_addr_end(addr, end);
+ kasan_early_pmd_populate(pud, addr, next);
+ } while (pud++, addr = next, addr != end && pud_none(*pud));
+}
+
+static void __init kasan_map_early_shadow(void)
+{
+ unsigned long addr = KASAN_SHADOW_START;
+ unsigned long end = KASAN_SHADOW_END;
+ unsigned long next;
+ pgd_t *pgd;
+
+ pgd = pgd_offset_k(addr);
+ do {
+ next = pgd_addr_end(addr, end);
+ kasan_early_pud_populate(pgd, addr, next);
+ } while (pgd++, addr = next, addr != end);
+}
+
+void __init kasan_early_init(void)
+{
+ BUILD_BUG_ON(KASAN_SHADOW_OFFSET != KASAN_SHADOW_END - (1UL << 61));
+ BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, PGDIR_SIZE));
+ BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, PGDIR_SIZE));
+ kasan_map_early_shadow();
+}
+
+static void __init clear_pgds(unsigned long start,
+ unsigned long end)
+{
+ /*
+ * Remove references to kasan page tables from
+ * swapper_pg_dir. pgd_clear() can't be used
+ * here because it's nop on 2,3-level pagetable setups
+ */
+ for (; start && start < end; start += PGDIR_SIZE)
+ set_pgd(pgd_offset_k(start), __pgd(0));
+}
+
+static void __init cpu_set_ttbr1(unsigned long ttbr1)
+{
+ asm(
+ " msr ttbr1_el1, %0\n"
+ " isb"
+ :
+ : "r" (ttbr1));
+}
+
+void __init kasan_init(void)
+{
+ struct memblock_region *reg;
+
+ /*
+ * We are going to perform proper setup of shadow memory.
+ * At first we should unmap early shadow (clear_pgds() call bellow).
+ * However, instrumented code couldn't execute without shadow memory.
+ * tmp_pg_dir used to keep early shadow mapped until full shadow
+ * setup will be finished.
+ */
+ memcpy(tmp_pg_dir, swapper_pg_dir, sizeof(tmp_pg_dir));
+ cpu_set_ttbr1(__pa(tmp_pg_dir));
+ flush_tlb_all();
+
+ clear_pgds(KASAN_SHADOW_START, KASAN_SHADOW_END);
+
+ kasan_populate_zero_shadow((void *)KASAN_SHADOW_START,
+ kasan_mem_to_shadow((void *)MODULES_VADDR));
+
+ for_each_memblock(memory, reg) {
+ void *start = (void *)__phys_to_virt(reg->base);
+ void *end = (void *)__phys_to_virt(reg->base + reg->size);
+
+ if (start >= end)
+ break;
+
+ /*
+ * end + 1 here is intentional. We check several shadow bytes in
+ * advance to slightly speed up fastpath. In some rare cases
+ * we could cross boundary of mapped shadow, so we just map
+ * some more here.
+ */
+ vmemmap_populate((unsigned long)kasan_mem_to_shadow(start),
+ (unsigned long)kasan_mem_to_shadow(end) + 1,
+ pfn_to_nid(virt_to_pfn(start)));
+ }
+
+ /*
+ * Stack instrumentation writes to shadow memory
+ * marking redzones around stack variables. So
+ * at this point kasan_zero_page will be poisoned
+ * with non-zero values. Wipe them out.
+ */
+ memset(kasan_zero_page, 0, PAGE_SIZE);
+ cpu_set_ttbr1(__pa(swapper_pg_dir));
+ flush_tlb_all();
+
+ /* At this point kasan is fully initialized. Enable error messages */
+ init_task.kasan_depth = 0;
+ pr_info("Kernel address sanitizer initialized\n");
+}
--
2.4.5
From: Linus Walleij <[email protected]>
This prints out the virtual memory assigned to KASan in the
boot crawl along with other memory assignments, if and only
if KASan is activated.
Example dmesg from the Juno Development board:
Memory: 1691156K/2080768K available (5465K kernel code, 444K rwdata,
2160K rodata, 340K init, 217K bss, 373228K reserved, 16384K cma-reserved)
Virtual kernel memory layout:
kasan : 0xffffff8000000000 - 0xffffff9000000000 ( 64 GB)
vmalloc : 0xffffff9000000000 - 0xffffffbdbfff0000 ( 182 GB)
vmemmap : 0xffffffbdc0000000 - 0xffffffbfc0000000 ( 8 GB maximum)
0xffffffbdc2000000 - 0xffffffbdc3fc0000 ( 31 MB actual)
fixed : 0xffffffbffabfd000 - 0xffffffbffac00000 ( 12 KB)
PCI I/O : 0xffffffbffae00000 - 0xffffffbffbe00000 ( 16 MB)
modules : 0xffffffbffc000000 - 0xffffffc000000000 ( 64 MB)
memory : 0xffffffc000000000 - 0xffffffc07f000000 ( 2032 MB)
.init : 0xffffffc0007f5000 - 0xffffffc00084a000 ( 340 KB)
.text : 0xffffffc000080000 - 0xffffffc0007f45b4 ( 7634 KB)
.data : 0xffffffc000850000 - 0xffffffc0008bf200 ( 445 KB)
Signed-off-by: Linus Walleij <[email protected]>
Signed-off-by: Andrey Ryabinin <[email protected]>
---
arch/arm64/mm/init.c | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index ad87ce8..3930692 100644
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -298,6 +298,9 @@ void __init mem_init(void)
#define MLK_ROUNDUP(b, t) b, t, DIV_ROUND_UP(((t) - (b)), SZ_1K)
pr_notice("Virtual kernel memory layout:\n"
+#ifdef CONFIG_KASAN
+ " kasan : 0x%16lx - 0x%16lx (%6ld GB)\n"
+#endif
" vmalloc : 0x%16lx - 0x%16lx (%6ld GB)\n"
#ifdef CONFIG_SPARSEMEM_VMEMMAP
" vmemmap : 0x%16lx - 0x%16lx (%6ld GB maximum)\n"
@@ -310,6 +313,9 @@ void __init mem_init(void)
" .init : 0x%p" " - 0x%p" " (%6ld KB)\n"
" .text : 0x%p" " - 0x%p" " (%6ld KB)\n"
" .data : 0x%p" " - 0x%p" " (%6ld KB)\n",
+#ifdef CONFIG_KASAN
+ MLG(KASAN_SHADOW_START, KASAN_SHADOW_END),
+#endif
MLG(VMALLOC_START, VMALLOC_END),
#ifdef CONFIG_SPARSEMEM_VMEMMAP
MLG((unsigned long)vmemmap,
--
2.4.5
On Wed, Jul 22, 2015 at 01:30:33PM +0300, Andrey Ryabinin wrote:
> diff --git a/arch/x86/mm/kasan_init_64.c b/arch/x86/mm/kasan_init_64.c
> index e1840f3..2390dba 100644
> --- a/arch/x86/mm/kasan_init_64.c
> +++ b/arch/x86/mm/kasan_init_64.c
> @@ -12,9 +12,9 @@
> extern pgd_t early_level4_pgt[PTRS_PER_PGD];
> extern struct range pfn_mapped[E820_X_MAX];
>
> -static pud_t kasan_zero_pud[PTRS_PER_PUD] __page_aligned_bss;
> -static pmd_t kasan_zero_pmd[PTRS_PER_PMD] __page_aligned_bss;
> -static pte_t kasan_zero_pte[PTRS_PER_PTE] __page_aligned_bss;
> +pud_t kasan_zero_pud[PTRS_PER_PUD] __page_aligned_bss;
> +pmd_t kasan_zero_pmd[PTRS_PER_PMD] __page_aligned_bss;
> +pte_t kasan_zero_pte[PTRS_PER_PTE] __page_aligned_bss;
>
> /*
> * This page used as early shadow. We don't use empty_zero_page
> @@ -24,7 +24,7 @@ static pte_t kasan_zero_pte[PTRS_PER_PTE] __page_aligned_bss;
> * that allowed to access, but not instrumented by kasan
> * (vmalloc/vmemmap ...).
> */
> -static unsigned char kasan_zero_page[PAGE_SIZE] __page_aligned_bss;
> +unsigned char kasan_zero_page[PAGE_SIZE] __page_aligned_bss;
Did you lose part of the patch when rebasing? I can see you copied
kasan_populate_zero_shadow() to the mm code but it's still present in
the x86 one and the above changes to remove static seem meaningless.
Or you plan to submit the rest of the x86 code separately?
BTW, you could even move kasan_zero_p[tme]d arrays to mm/.
> +static int __init zero_pmd_populate(pud_t *pud, unsigned long addr,
> + unsigned long end)
> +{
> + int ret = 0;
> + pmd_t *pmd = pmd_offset(pud, addr);
> + unsigned long next;
> +
> + do {
> + next = pmd_addr_end(addr, end);
> +
> + if (IS_ALIGNED(addr, PMD_SIZE) && end - addr >= PMD_SIZE) {
> + pmd_populate_kernel(&init_mm, pmd, kasan_zero_pte);
> + continue;
> + }
> +
> + if (pmd_none(*pmd)) {
> + void *p = early_alloc(PAGE_SIZE, NUMA_NO_NODE);
> + if (!p)
> + return -ENOMEM;
> + pmd_populate_kernel(&init_mm, pmd, p);
> + }
> + zero_pte_populate(pmd, addr, pmd_addr_end(addr, end));
You could use "next" directly has the last argument here.
> + } while (pmd++, addr = next, addr != end);
> +
> + return ret;
> +}
> +
> +static int __init zero_pud_populate(pgd_t *pgd, unsigned long addr,
> + unsigned long end)
> +{
> + int ret = 0;
> + pud_t *pud = pud_offset(pgd, addr);
> + unsigned long next;
> +
> + do {
> + next = pud_addr_end(addr, end);
> + if (IS_ALIGNED(addr, PUD_SIZE) && end - addr >= PUD_SIZE) {
> + pmd_t *pmd;
> +
> + pud_populate(&init_mm, pud, kasan_zero_pmd);
> + pmd = pmd_offset(pud, addr);
> + pmd_populate_kernel(&init_mm, pmd, kasan_zero_pte);
> + continue;
> + }
> +
> + if (pud_none(*pud)) {
> + void *p = early_alloc(PAGE_SIZE, NUMA_NO_NODE);
> + if (!p)
> + return -ENOMEM;
> + pud_populate(&init_mm, pud, p);
> + }
> + zero_pmd_populate(pud, addr, pud_addr_end(addr, end));
Same here.
> + } while (pud++, addr = next, addr != end);
> +
> + return ret;
> +}
> +
> +static int __init zero_pgd_populate(unsigned long addr, unsigned long end)
> +{
> + int ret = 0;
> + pgd_t *pgd = pgd_offset_k(addr);
> + unsigned long next;
> +
> + do {
> + next = pgd_addr_end(addr, end);
> +
> + if (IS_ALIGNED(addr, PGDIR_SIZE) && end - addr >= PGDIR_SIZE) {
> + pud_t *pud;
> + pmd_t *pmd;
> +
> + /*
> + * kasan_zero_pud should be populated with pmds
> + * at this moment.
> + * [pud,pmd]_populate*() bellow needed only for
s/bellow/below/
> + * 3,2 - level page tables where we don't have
> + * puds,pmds, so pgd_populate(), pud_populate()
> + * is noops.
> + */
--
Catalin
On Wed, Jul 22, 2015 at 01:30:34PM +0300, Andrey Ryabinin wrote:
> In order to not use lengthy (UL(0xffffffffffffffff) << VA_BITS) everywhere,
> replace it with VA_START.
>
> Signed-off-by: Andrey Ryabinin <[email protected]>
Reviewed-by: Catalin Marinas <[email protected]>
On Wed, Jul 22, 2015 at 01:30:35PM +0300, Andrey Ryabinin wrote:
> This will be used by KASAN latter.
>
> Signed-off-by: Andrey Ryabinin <[email protected]>
Acked-by: Catalin Marinas <[email protected]>
On Wed, Jul 22, 2015 at 01:30:37PM +0300, Andrey Ryabinin wrote:
> From: Linus Walleij <[email protected]>
>
> This prints out the virtual memory assigned to KASan in the
> boot crawl along with other memory assignments, if and only
> if KASan is activated.
>
> Example dmesg from the Juno Development board:
>
> Memory: 1691156K/2080768K available (5465K kernel code, 444K rwdata,
> 2160K rodata, 340K init, 217K bss, 373228K reserved, 16384K cma-reserved)
> Virtual kernel memory layout:
> kasan : 0xffffff8000000000 - 0xffffff9000000000 ( 64 GB)
> vmalloc : 0xffffff9000000000 - 0xffffffbdbfff0000 ( 182 GB)
> vmemmap : 0xffffffbdc0000000 - 0xffffffbfc0000000 ( 8 GB maximum)
> 0xffffffbdc2000000 - 0xffffffbdc3fc0000 ( 31 MB actual)
> fixed : 0xffffffbffabfd000 - 0xffffffbffac00000 ( 12 KB)
> PCI I/O : 0xffffffbffae00000 - 0xffffffbffbe00000 ( 16 MB)
> modules : 0xffffffbffc000000 - 0xffffffc000000000 ( 64 MB)
> memory : 0xffffffc000000000 - 0xffffffc07f000000 ( 2032 MB)
> .init : 0xffffffc0007f5000 - 0xffffffc00084a000 ( 340 KB)
> .text : 0xffffffc000080000 - 0xffffffc0007f45b4 ( 7634 KB)
> .data : 0xffffffc000850000 - 0xffffffc0008bf200 ( 445 KB)
>
> Signed-off-by: Linus Walleij <[email protected]>
> Signed-off-by: Andrey Ryabinin <[email protected]>
Acked-by: Catalin Marinas <[email protected]>
Hi Andrey,
Could you please check minor comments below?
On Wed, Jul 22, 2015 at 1:30 PM, Andrey Ryabinin <[email protected]> wrote:
> Introduce generic kasan_populate_zero_shadow(start, end).
> This function maps kasan_zero_page to the [start, end] addresses.
>
> In follow on patches it will be used for ARMv8 (and maybe other
> architectures) and will replace x86_64 specific populate_zero_shadow().
>
> Signed-off-by: Andrey Ryabinin <[email protected]>
> ---
> arch/x86/mm/kasan_init_64.c | 8 +--
> include/linux/kasan.h | 8 +++
> mm/kasan/Makefile | 2 +-
> mm/kasan/kasan_init.c | 142 ++++++++++++++++++++++++++++++++++++++++++++
> 4 files changed, 155 insertions(+), 5 deletions(-)
> create mode 100644 mm/kasan/kasan_init.c
>
[..]
> diff --git a/mm/kasan/kasan_init.c b/mm/kasan/kasan_init.c
> new file mode 100644
> index 0000000..37fb46a
> --- /dev/null
> +++ b/mm/kasan/kasan_init.c
> @@ -0,0 +1,142 @@
> +#include <linux/bootmem.h>
> +#include <linux/init.h>
> +#include <linux/kasan.h>
> +#include <linux/kernel.h>
> +#include <linux/memblock.h>
> +#include <linux/pfn.h>
> +
> +#include <asm/page.h>
> +#include <asm/pgalloc.h>
> +
Are you releasing code under GPL?
Shouldn't there be any license header in such new file?
> +static __init void *early_alloc(size_t size, int node)
> +{
> + return memblock_virt_alloc_try_nid(size, size, __pa(MAX_DMA_ADDRESS),
> + BOOTMEM_ALLOC_ACCESSIBLE, node);
> +}
> +
> +static int __init zero_pte_populate(pmd_t *pmd, unsigned long addr,
> + unsigned long end)
> +{
> + pte_t *pte = pte_offset_kernel(pmd, addr);
> + pte_t zero_pte;
> +
> + zero_pte = pfn_pte(PFN_DOWN(__pa(kasan_zero_page)), PAGE_KERNEL);
> + zero_pte = pte_wrprotect(zero_pte);
> +
> + while (addr + PAGE_SIZE <= end) {
> + set_pte_at(&init_mm, addr, pte, zero_pte);
> + addr += PAGE_SIZE;
> + pte = pte_offset_kernel(pmd, addr);
> + }
> + return 0;
> +}
> +
> +static int __init zero_pmd_populate(pud_t *pud, unsigned long addr,
> + unsigned long end)
> +{
> + int ret = 0;
> + pmd_t *pmd = pmd_offset(pud, addr);
> + unsigned long next;
> +
> + do {
> + next = pmd_addr_end(addr, end);
> +
> + if (IS_ALIGNED(addr, PMD_SIZE) && end - addr >= PMD_SIZE) {
> + pmd_populate_kernel(&init_mm, pmd, kasan_zero_pte);
> + continue;
> + }
> +
> + if (pmd_none(*pmd)) {
> + void *p = early_alloc(PAGE_SIZE, NUMA_NO_NODE);
> + if (!p)
> + return -ENOMEM;
> + pmd_populate_kernel(&init_mm, pmd, p);
> + }
> + zero_pte_populate(pmd, addr, pmd_addr_end(addr, end));
> + } while (pmd++, addr = next, addr != end);
> +
> + return ret;
In zero_{pgd, pud, pmd}_populate you're not resetting ret variable
used as return value inside function so maybe you don't need ret
variable at all. What about return 0 in the end and -ENOMEM in error
case?
> +}
> +
> +static int __init zero_pud_populate(pgd_t *pgd, unsigned long addr,
> + unsigned long end)
> +{
> + int ret = 0;
> + pud_t *pud = pud_offset(pgd, addr);
> + unsigned long next;
> +
> + do {
> + next = pud_addr_end(addr, end);
> + if (IS_ALIGNED(addr, PUD_SIZE) && end - addr >= PUD_SIZE) {
> + pmd_t *pmd;
> +
> + pud_populate(&init_mm, pud, kasan_zero_pmd);
> + pmd = pmd_offset(pud, addr);
> + pmd_populate_kernel(&init_mm, pmd, kasan_zero_pte);
> + continue;
> + }
> +
> + if (pud_none(*pud)) {
> + void *p = early_alloc(PAGE_SIZE, NUMA_NO_NODE);
> + if (!p)
> + return -ENOMEM;
> + pud_populate(&init_mm, pud, p);
> + }
> + zero_pmd_populate(pud, addr, pud_addr_end(addr, end));
> + } while (pud++, addr = next, addr != end);
> +
> + return ret;
> +}
> +
> +static int __init zero_pgd_populate(unsigned long addr, unsigned long end)
> +{
> + int ret = 0;
> + pgd_t *pgd = pgd_offset_k(addr);
> + unsigned long next;
> +
> + do {
> + next = pgd_addr_end(addr, end);
> +
> + if (IS_ALIGNED(addr, PGDIR_SIZE) && end - addr >= PGDIR_SIZE) {
> + pud_t *pud;
> + pmd_t *pmd;
> +
> + /*
> + * kasan_zero_pud should be populated with pmds
> + * at this moment.
> + * [pud,pmd]_populate*() bellow needed only for
> + * 3,2 - level page tables where we don't have
> + * puds,pmds, so pgd_populate(), pud_populate()
> + * is noops.
> + */
> + pgd_populate(&init_mm, pgd, kasan_zero_pud);
> + pud = pud_offset(pgd, addr);
> + pud_populate(&init_mm, pud, kasan_zero_pmd);
> + pmd = pmd_offset(pud, addr);
> + pmd_populate_kernel(&init_mm, pmd, kasan_zero_pte);
> + continue;
> + }
> +
> + if (pgd_none(*pgd)) {
> + void *p = early_alloc(PAGE_SIZE, NUMA_NO_NODE);
> + if (!p)
> + return -ENOMEM;
> + pgd_populate(&init_mm, pgd, p);
> + }
> + zero_pud_populate(pgd, addr, next);
But you're not checking return value after zero_pud_populate() and
zero_pmd_populate() that might fail with ENOMEM.
Is it critical here on init or can they be converted to return void?
> +/**
> + * kasan_populate_zero_shadow - populate shadow memory region with
> + * kasan_zero_page
> + * @start - start of the memory range to populate
> + * @end - end of the memory range to populate
> + */
> +void __init kasan_populate_zero_shadow(const void *start, const void *end)
> +{
> + if (zero_pgd_populate((unsigned long)start, (unsigned long)end))
> + panic("kasan: unable to map zero shadow!");
> +}
> --
> 2.4.5
>
>
> _______________________________________________
> linux-arm-kernel mailing list
> [email protected]
> http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
--
Best regards, Klimov Alexey
On 07/22/2015 05:17 PM, Catalin Marinas wrote:
> On Wed, Jul 22, 2015 at 01:30:33PM +0300, Andrey Ryabinin wrote:
>> diff --git a/arch/x86/mm/kasan_init_64.c b/arch/x86/mm/kasan_init_64.c
>> index e1840f3..2390dba 100644
>> --- a/arch/x86/mm/kasan_init_64.c
>> +++ b/arch/x86/mm/kasan_init_64.c
>> @@ -12,9 +12,9 @@
>> extern pgd_t early_level4_pgt[PTRS_PER_PGD];
>> extern struct range pfn_mapped[E820_X_MAX];
>>
>> -static pud_t kasan_zero_pud[PTRS_PER_PUD] __page_aligned_bss;
>> -static pmd_t kasan_zero_pmd[PTRS_PER_PMD] __page_aligned_bss;
>> -static pte_t kasan_zero_pte[PTRS_PER_PTE] __page_aligned_bss;
>> +pud_t kasan_zero_pud[PTRS_PER_PUD] __page_aligned_bss;
>> +pmd_t kasan_zero_pmd[PTRS_PER_PMD] __page_aligned_bss;
>> +pte_t kasan_zero_pte[PTRS_PER_PTE] __page_aligned_bss;
>>
>> /*
>> * This page used as early shadow. We don't use empty_zero_page
>> @@ -24,7 +24,7 @@ static pte_t kasan_zero_pte[PTRS_PER_PTE] __page_aligned_bss;
>> * that allowed to access, but not instrumented by kasan
>> * (vmalloc/vmemmap ...).
>> */
>> -static unsigned char kasan_zero_page[PAGE_SIZE] __page_aligned_bss;
>> +unsigned char kasan_zero_page[PAGE_SIZE] __page_aligned_bss;
>
> Did you lose part of the patch when rebasing? I can see you copied
> kasan_populate_zero_shadow() to the mm code but it's still present in
> the x86 one and the above changes to remove static seem meaningless.
>
> Or you plan to submit the rest of the x86 code separately?
>
Yes, I was going to send x86 patch later.
Static has to be removed because this conflicts with kasan_zero_p* declarations in include/linux/kasan.h.
> BTW, you could even move kasan_zero_p[tme]d arrays to mm/.
>
Makes sense.
On Wed, Jul 22, 2015 at 1:30 PM, Andrey Ryabinin <[email protected]> wrote:
> For git users patches are available in git:
> git://github.com/aryabinin/linux.git kasan/arm64v3
>
> Changes since v2:
> - Rebase on top of v4.2-rc3
> - Address feedback from Catalin.
> - Print memory assignment fro Linus
from?
> - Add message about KASAN being initialized
>
> Changes since v1:
> - Address feedback from Catalin.
> - Generalize some kasan init code from arch/x86/mm/kasan_init_64.c
> and reuse it for arm64.
> - Some bugfixes, including:
> add missing arm64/include/asm/kasan.h
> add tlb flush after changing ttbr1
> - Add code comments.
>
>
> Andrey Ryabinin (4):
> mm: kasan: introduce generic kasan_populate_zero_shadow()
> arm64: introduce VA_START macro - the first kernel virtual address.
> arm64: move PGD_SIZE definition to pgalloc.h
> arm64: add KASAN support
>
> Linus Walleij (1):
> ARM64: kasan: print memory assignment
>
> arch/arm64/Kconfig | 17 ++++
> arch/arm64/include/asm/kasan.h | 24 ++++++
> arch/arm64/include/asm/memory.h | 2 +
> arch/arm64/include/asm/pgalloc.h | 1 +
> arch/arm64/include/asm/pgtable.h | 9 +-
> arch/arm64/include/asm/string.h | 16 ++++
> arch/arm64/kernel/arm64ksyms.c | 3 +
> arch/arm64/kernel/head.S | 3 +
> arch/arm64/kernel/module.c | 16 +++-
> arch/arm64/kernel/setup.c | 2 +
> arch/arm64/lib/memcpy.S | 3 +
> arch/arm64/lib/memmove.S | 7 +-
> arch/arm64/lib/memset.S | 3 +
> arch/arm64/mm/Makefile | 3 +
> arch/arm64/mm/init.c | 6 ++
> arch/arm64/mm/kasan_init.c | 176 +++++++++++++++++++++++++++++++++++++++
> arch/arm64/mm/pgd.c | 2 -
> arch/x86/mm/kasan_init_64.c | 8 +-
> include/linux/kasan.h | 8 ++
> mm/kasan/Makefile | 2 +-
> mm/kasan/kasan_init.c | 142 +++++++++++++++++++++++++++++++
> 21 files changed, 440 insertions(+), 13 deletions(-)
> create mode 100644 arch/arm64/include/asm/kasan.h
> create mode 100644 arch/arm64/mm/kasan_init.c
> create mode 100644 mm/kasan/kasan_init.c
Could you please check license header in all new files?
By the way, i don't remember if checkpatch can detect missing GPL header.
Best regards,
Alexey Klimov
On 07/22/2015 05:25 PM, Alexey Klimov wrote:
> Hi Andrey,
>
> Could you please check minor comments below?
>
> On Wed, Jul 22, 2015 at 1:30 PM, Andrey Ryabinin <[email protected]> wrote:
>> Introduce generic kasan_populate_zero_shadow(start, end).
>> This function maps kasan_zero_page to the [start, end] addresses.
>>
>> In follow on patches it will be used for ARMv8 (and maybe other
>> architectures) and will replace x86_64 specific populate_zero_shadow().
>>
>> Signed-off-by: Andrey Ryabinin <[email protected]>
>> ---
>> arch/x86/mm/kasan_init_64.c | 8 +--
>> include/linux/kasan.h | 8 +++
>> mm/kasan/Makefile | 2 +-
>> mm/kasan/kasan_init.c | 142 ++++++++++++++++++++++++++++++++++++++++++++
>> 4 files changed, 155 insertions(+), 5 deletions(-)
>> create mode 100644 mm/kasan/kasan_init.c
>>
>
> [..]
>
>> diff --git a/mm/kasan/kasan_init.c b/mm/kasan/kasan_init.c
>> new file mode 100644
>> index 0000000..37fb46a
>> --- /dev/null
>> +++ b/mm/kasan/kasan_init.c
>> @@ -0,0 +1,142 @@
>> +#include <linux/bootmem.h>
>> +#include <linux/init.h>
>> +#include <linux/kasan.h>
>> +#include <linux/kernel.h>
>> +#include <linux/memblock.h>
>> +#include <linux/pfn.h>
>> +
>> +#include <asm/page.h>
>> +#include <asm/pgalloc.h>
>> +
>
> Are you releasing code under GPL?
> Shouldn't there be any license header in such new file?
>
Sure, will do.
...
>> +
>> + if (pgd_none(*pgd)) {
>> + void *p = early_alloc(PAGE_SIZE, NUMA_NO_NODE);
>> + if (!p)
>> + return -ENOMEM;
>> + pgd_populate(&init_mm, pgd, p);
>> + }
>> + zero_pud_populate(pgd, addr, next);
>
> But you're not checking return value after zero_pud_populate() and
> zero_pmd_populate() that might fail with ENOMEM.
> Is it critical here on init or can they be converted to return void?
>
I think it's better to convert these functions to void.
BTW, this check after early_alloc() is pointless because early_alloc() will panic
if allocation failed.
>
>> +/**
>> + * kasan_populate_zero_shadow - populate shadow memory region with
>> + * kasan_zero_page
>> + * @start - start of the memory range to populate
>> + * @end - end of the memory range to populate
>> + */
>> +void __init kasan_populate_zero_shadow(const void *start, const void *end)
>> +{
>> + if (zero_pgd_populate((unsigned long)start, (unsigned long)end))
>> + panic("kasan: unable to map zero shadow!");
>> +}
>> --
>> 2.4.5
>>
>>
>> _______________________________________________
>> linux-arm-kernel mailing list
>> [email protected]
>> http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
>
>
>
On Wed, Jul 22, 2015 at 01:30:36PM +0300, Andrey Ryabinin wrote:
> diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
> index 318175f..61ebb7c 100644
> --- a/arch/arm64/Kconfig
> +++ b/arch/arm64/Kconfig
> @@ -46,6 +46,7 @@ config ARM64
> select HAVE_ARCH_AUDITSYSCALL
> select HAVE_ARCH_BITREVERSE
> select HAVE_ARCH_JUMP_LABEL
> + select HAVE_ARCH_KASAN if SPARSEMEM_VMEMMAP
> select HAVE_ARCH_KGDB
> select HAVE_ARCH_SECCOMP_FILTER
> select HAVE_ARCH_TRACEHOOK
> @@ -122,6 +123,22 @@ config GENERIC_CSUM
> config GENERIC_CALIBRATE_DELAY
> def_bool y
>
> +config KASAN_SHADOW_OFFSET
> + hex
> + depends on KASAN
> + default 0xdfff200000000000 if ARM64_VA_BITS_48
> + default 0xdffffc8000000000 if ARM64_VA_BITS_42
> + default 0xdfffff9000000000 if ARM64_VA_BITS_39
> + help
> + This value used to address to corresponding shadow address
"This value is used to map an address to the corresponding shadow
address..." (unless you meant something else but the above doesn't read
well).
> + by the following formula:
> + shadow_addr = (address >> 3) + KASAN_SHADOW_OFFSET;
> +
> + (1 << 61) shadow addresses - [KASAN_SHADOW_OFFSET,KASAN_SHADOW_END]
> + cover all 64-bits of virtual addresses. So KASAN_SHADOW_OFFSET
> + should satisfy the following equation:
> + KASAN_SHADOW_OFFSET = KASAN_SHADOW_END - (1ULL << 61)
I think we should generate KASAN_SHADOW_OFFSET in the Makefile directly
using some awk snippet/script (we are going to get a 47-bit VA as well
with 16KB page configuration):
---------8<-----------------
#!/bin/awk -f
BEGIN {
# 32-bit arithmetics
va_bits = ARGV[1] - 32
va_start = and(0xffffffff, lshift(0xffffffff, va_bits))
shadow_end = va_start + lshift(1, va_bits - 3)
shadow_offset = shadow_end - lshift(1, 64 - 3 - 32)
printf("0x%x00000000\n", shadow_offset)
}
-------8<-----------------
Otherwise the code looks fine.
--
Catalin