Received: by 2002:a05:6a10:8c0a:0:0:0:0 with SMTP id go10csp4297672pxb; Mon, 8 Feb 2021 12:47:34 -0800 (PST) X-Google-Smtp-Source: ABdhPJxFKIEKsEtmUV/glY7tFB5pfOpEL8ucPHIFRnTbtFH+W1Woyy9L/rnQeAl3DuOJ3mLgt079 X-Received: by 2002:a50:fe02:: with SMTP id f2mr18599075edt.195.1612817254184; Mon, 08 Feb 2021 12:47:34 -0800 (PST) ARC-Seal: i=1; a=rsa-sha256; t=1612817254; cv=none; d=google.com; s=arc-20160816; b=iMyvyYpDiA5STqjN/IVCYqi+0ZAqa4KXsEKCe37FUvoEj3mPb+eZn8Bcz8JR9M3VMF LYLPu5mGtfuYjT33Y43PPIFq91vS7ekekO5VH88c/hSLb5gXOAE1PQX8gCiHYzElrzCd SvoNegtDqe8OdjTKE8wGJTbw8GV1x6S5kblv16hvbDNIikhiRRh1Hq6qHC6M0l4bjvoE UpTj920iJhUwMdZmyNP6Wqe1n3ngor1fgyysdeqy0zktcYuM1P14GkRvu8hSH/M7xiRK Q+NXzyljKGpIf9ML66FJPwmDou9LGFR2y93VrcchWRIclBRK05ZwJUwndB7xPAllnMcQ ITOQ== ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=google.com; s=arc-20160816; h=list-id:precedence:content-transfer-encoding:mime-version :references:in-reply-to:message-id:date:subject:cc:to:from; bh=7cuSSFVmzAI8j2LCy7Kkhe8lqAWlu8EgJFbOmPIOHNY=; b=NhKSB+FqGXqtN9tRfFF6dPwI9obljDUetYYACkTqPTZAEbHrk0CnilZguJ/1FmTLcf cLb/pSlGky3qRY7iPVjQJ204l3FfEj3389h/U4Ck2nZuv0H54Msdl6aPaFy4XO/iLwrM a5JZoVJfujquiil/QEd0OIB+IduPlovjJOPXqEmb8epqUJOHF41e/EitFewkFy5Jp5aR otnXRv9Y+6PxgyxnfqwHT76pb55HsOf0ulL3Acue6x2kcFOCd4NMibT2CmQIqO3cVElv oHEbUqZIBifQi+sdmZxpj+GmvIhlCRjNzIzZ6ek/YG7JmfYr9QXtCaP2X5U4DmJ62ogw Hgrw== ARC-Authentication-Results: i=1; mx.google.com; spf=pass (google.com: domain of linux-kernel-owner@vger.kernel.org designates 23.128.96.18 as permitted sender) smtp.mailfrom=linux-kernel-owner@vger.kernel.org Return-Path: Received: from vger.kernel.org (vger.kernel.org. [23.128.96.18]) by mx.google.com with ESMTP id ko21si12006382ejc.91.2021.02.08.12.47.10; Mon, 08 Feb 2021 12:47:34 -0800 (PST) Received-SPF: pass (google.com: domain of linux-kernel-owner@vger.kernel.org designates 23.128.96.18 as permitted sender) client-ip=23.128.96.18; Authentication-Results: mx.google.com; spf=pass (google.com: domain of linux-kernel-owner@vger.kernel.org designates 23.128.96.18 as permitted sender) smtp.mailfrom=linux-kernel-owner@vger.kernel.org Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S233528AbhBHUqY (ORCPT + 99 others); Mon, 8 Feb 2021 15:46:24 -0500 Received: from relay1-d.mail.gandi.net ([217.70.183.193]:39013 "EHLO relay1-d.mail.gandi.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S236055AbhBHTe0 (ORCPT ); Mon, 8 Feb 2021 14:34:26 -0500 X-Originating-IP: 2.7.49.219 Received: from debian.home (lfbn-lyo-1-457-219.w2-7.abo.wanadoo.fr [2.7.49.219]) (Authenticated sender: alex@ghiti.fr) by relay1-d.mail.gandi.net (Postfix) with ESMTPSA id D3BB0240005; Mon, 8 Feb 2021 19:33:31 +0000 (UTC) From: Alexandre Ghiti To: Andrey Ryabinin , Alexander Potapenko , Dmitry Vyukov , Paul Walmsley , Palmer Dabbelt , Albert Ou , kasan-dev@googlegroups.com, linux-riscv@lists.infradead.org, linux-kernel@vger.kernel.org Cc: Alexandre Ghiti Subject: [PATCH 3/4] riscv: Improve kasan population function Date: Mon, 8 Feb 2021 14:30:16 -0500 Message-Id: <20210208193017.30904-4-alex@ghiti.fr> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20210208193017.30904-1-alex@ghiti.fr> References: <20210208193017.30904-1-alex@ghiti.fr> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Current population code populates a whole page table without taking care of what could have been already allocated and without taking into account possible index in page table, assuming the virtual address to map is always aligned on the page table size, which, for example, won't be the case when the kernel will get pushed to the end of the address space. Address those problems by rewriting the kasan population function, splitting it into subfunctions for each different page table level. Signed-off-by: Alexandre Ghiti --- arch/riscv/mm/kasan_init.c | 91 ++++++++++++++++++++++++++------------ 1 file changed, 63 insertions(+), 28 deletions(-) diff --git a/arch/riscv/mm/kasan_init.c b/arch/riscv/mm/kasan_init.c index 7bbe09416a2e..b7d4d9abd144 100644 --- a/arch/riscv/mm/kasan_init.c +++ b/arch/riscv/mm/kasan_init.c @@ -47,37 +47,72 @@ asmlinkage void __init kasan_early_init(void) local_flush_tlb_all(); } -static void __init populate(void *start, void *end) +static void kasan_populate_pte(pmd_t *pmd, unsigned long vaddr, unsigned long end) +{ + phys_addr_t phys_addr; + pte_t *ptep, *base_pte; + + if (pmd_none(*pmd)) + base_pte = memblock_alloc(PTRS_PER_PTE * sizeof(pte_t), PAGE_SIZE); + else + base_pte = (pte_t *)pmd_page_vaddr(*pmd); + + ptep = base_pte + pte_index(vaddr); + + do { + if (pte_none(*ptep)) { + phys_addr = memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); + set_pte(ptep, pfn_pte(PFN_DOWN(phys_addr), PAGE_KERNEL)); + } + } while (ptep++, vaddr += PAGE_SIZE, vaddr != end); + + set_pmd(pmd, pfn_pmd(PFN_DOWN(__pa(base_pte)), PAGE_TABLE)); +} + +static void kasan_populate_pmd(pgd_t *pgd, unsigned long vaddr, unsigned long end) +{ + phys_addr_t phys_addr; + pmd_t *pmdp, *base_pmd; + unsigned long next; + + base_pmd = (pmd_t *)pgd_page_vaddr(*pgd); + if (base_pmd == lm_alias(kasan_early_shadow_pmd)) + base_pmd = memblock_alloc(PTRS_PER_PMD * sizeof(pmd_t), PAGE_SIZE); + + pmdp = base_pmd + pmd_index(vaddr); + + do { + next = pmd_addr_end(vaddr, end); + kasan_populate_pte(pmdp, vaddr, next); + } while (pmdp++, vaddr = next, vaddr != end); + + /* + * Wait for the whole PGD to be populated before setting the PGD in + * the page table, otherwise, if we did set the PGD before populating + * it entirely, memblock could allocate a page at a physical address + * where KASAN is not populated yet and then we'd get a page fault. + */ + set_pgd(pgd, pfn_pgd(PFN_DOWN(__pa(base_pmd)), PAGE_TABLE)); +} + +static void kasan_populate_pgd(unsigned long vaddr, unsigned long end) +{ + phys_addr_t phys_addr; + pgd_t *pgdp = pgd_offset_k(vaddr); + unsigned long next; + + do { + next = pgd_addr_end(vaddr, end); + kasan_populate_pmd(pgdp, vaddr, next); + } while (pgdp++, vaddr = next, vaddr != end); +} + +static void __init kasan_populate(void *start, void *end) { - unsigned long i, offset; unsigned long vaddr = (unsigned long)start & PAGE_MASK; unsigned long vend = PAGE_ALIGN((unsigned long)end); - unsigned long n_pages = (vend - vaddr) / PAGE_SIZE; - unsigned long n_ptes = - ((n_pages + PTRS_PER_PTE) & -PTRS_PER_PTE) / PTRS_PER_PTE; - unsigned long n_pmds = - ((n_ptes + PTRS_PER_PMD) & -PTRS_PER_PMD) / PTRS_PER_PMD; - - pte_t *pte = - memblock_alloc(n_ptes * PTRS_PER_PTE * sizeof(pte_t), PAGE_SIZE); - pmd_t *pmd = - memblock_alloc(n_pmds * PTRS_PER_PMD * sizeof(pmd_t), PAGE_SIZE); - pgd_t *pgd = pgd_offset_k(vaddr); - - for (i = 0; i < n_pages; i++) { - phys_addr_t phys = memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE); - set_pte(&pte[i], pfn_pte(PHYS_PFN(phys), PAGE_KERNEL)); - } - - for (i = 0, offset = 0; i < n_ptes; i++, offset += PTRS_PER_PTE) - set_pmd(&pmd[i], - pfn_pmd(PFN_DOWN(__pa(&pte[offset])), - __pgprot(_PAGE_TABLE))); - for (i = 0, offset = 0; i < n_pmds; i++, offset += PTRS_PER_PMD) - set_pgd(&pgd[i], - pfn_pgd(PFN_DOWN(__pa(&pmd[offset])), - __pgprot(_PAGE_TABLE))); + kasan_populate_pgd(vaddr, vend); local_flush_tlb_all(); memset(start, KASAN_SHADOW_INIT, end - start); @@ -99,7 +134,7 @@ void __init kasan_init(void) if (start >= end) break; - populate(kasan_mem_to_shadow(start), kasan_mem_to_shadow(end)); + kasan_populate(kasan_mem_to_shadow(start), kasan_mem_to_shadow(end)); }; for (i = 0; i < PTRS_PER_PTE; i++) -- 2.20.1