From: Lan Tianyu <[email protected]>
This patchset is to introduce hv ept tlb range list flush function
support in the KVM MMU component. Flushing ept tlbs of several address
range can be done via single hypercall and new list flush function is
used in the kvm_mmu_commit_zap_page() and FNAME(sync_page). This patchset
also adds more hv ept tlb range flush support in more KVM MMU function.
This patchset is based on the fix patch "x86/Hyper-V: Fix definition HV_MAX_FLUSH_REP_COUNT".
(https://www.mail-archive.com/[email protected]/msg1939455.html)
Change since v2:
1) Fix calculation of flush pages in the kvm_fill_hv_flush_list_func()
2) Change the logic of setting/clearing last_level flag
Change since v1:
1) Make flush list as a hlist instead of list in order to
keep struct kvm_mmu_page size.
2) Add last_level flag in the struct kvm_mmu_page instead
of spte pointer
3) Move tlb flush from kvm_mmu_notifier_clear_flush_young() to kvm_age_hva()
4) Use range flush in the kvm_vm_ioctl_get/clear_dirty_log()
Lan Tianyu (10):
X86/Hyper-V: Add parameter offset for
hyperv_fill_flush_guest_mapping_list()
KVM/VMX: Fill range list in kvm_fill_hv_flush_list_func()
KVM/MMU: Introduce tlb flush with range list
KVM/MMU: Use range flush in sync_page()
KVM/MMU: Flush tlb directly in the kvm_mmu_slot_gfn_write_protect()
KVM: Add kvm_get_memslot() to get memslot via slot id
KVM: Use tlb range flush in the kvm_vm_ioctl_get/clear_dirty_log()
KVM: Add flush parameter for kvm_age_hva()
KVM/MMU: Use tlb range flush in the kvm_age_hva()
KVM/MMU: Add last_level flag in the struct mmu_spte_page
arch/arm/include/asm/kvm_host.h | 3 +-
arch/arm64/include/asm/kvm_host.h | 3 +-
arch/mips/include/asm/kvm_host.h | 3 +-
arch/mips/kvm/mmu.c | 11 ++++++--
arch/powerpc/include/asm/kvm_host.h | 3 +-
arch/powerpc/kvm/book3s.c | 10 +++++--
arch/powerpc/kvm/e500_mmu_host.c | 3 +-
arch/x86/hyperv/nested.c | 4 +--
arch/x86/include/asm/kvm_host.h | 11 +++++++-
arch/x86/include/asm/mshyperv.h | 2 +-
arch/x86/kvm/mmu.c | 55 ++++++++++++++++++++++++++++++-------
arch/x86/kvm/mmu.h | 7 +++++
arch/x86/kvm/paging_tmpl.h | 5 ++--
arch/x86/kvm/vmx/vmx.c | 18 ++++++++++--
arch/x86/kvm/x86.c | 16 ++++++++---
include/linux/kvm_host.h | 1 +
virt/kvm/arm/mmu.c | 13 +++++++--
virt/kvm/kvm_main.c | 51 ++++++++++++++--------------------
18 files changed, 156 insertions(+), 63 deletions(-)
--
2.14.4
From: Lan Tianyu <[email protected]>
This patch is to introduce tlb flush with range list interface and use
struct kvm_mmu_page as list entry. Use flush list function in the
kvm_mmu_commit_zap_page().
Signed-off-by: Lan Tianyu <[email protected]>
---
arch/x86/kvm/mmu.c | 24 +++++++++++++++++++++++-
1 file changed, 23 insertions(+), 1 deletion(-)
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index 8d43b7c0f56f..7a862c56b954 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -291,6 +291,20 @@ static void kvm_flush_remote_tlbs_with_address(struct kvm *kvm,
range.start_gfn = start_gfn;
range.pages = pages;
+ range.flush_list = NULL;
+
+ kvm_flush_remote_tlbs_with_range(kvm, &range);
+}
+
+static void kvm_flush_remote_tlbs_with_list(struct kvm *kvm,
+ struct hlist_head *flush_list)
+{
+ struct kvm_tlb_range range;
+
+ if (hlist_empty(flush_list))
+ return;
+
+ range.flush_list = flush_list;
kvm_flush_remote_tlbs_with_range(kvm, &range);
}
@@ -2719,6 +2733,7 @@ static void kvm_mmu_commit_zap_page(struct kvm *kvm,
struct list_head *invalid_list)
{
struct kvm_mmu_page *sp, *nsp;
+ HLIST_HEAD(flush_list);
if (list_empty(invalid_list))
return;
@@ -2732,7 +2747,14 @@ static void kvm_mmu_commit_zap_page(struct kvm *kvm,
* In addition, kvm_flush_remote_tlbs waits for all vcpus to exit
* guest mode and/or lockless shadow page table walks.
*/
- kvm_flush_remote_tlbs(kvm);
+ if (kvm_available_flush_tlb_with_range()) {
+ list_for_each_entry(sp, invalid_list, link)
+ hlist_add_head(&sp->flush_link, &flush_list);
+
+ kvm_flush_remote_tlbs_with_list(kvm, &flush_list);
+ } else {
+ kvm_flush_remote_tlbs(kvm);
+ }
list_for_each_entry_safe(sp, nsp, invalid_list, link) {
WARN_ON(!sp->role.invalid || sp->root_count);
--
2.14.4
From: Lan Tianyu <[email protected]>
This patch is to use range flush to flush tlbs of input struct
kvm_mmu_page in the sync_page(). If range flush is not available,
kvm_flush_remote_tlbs_with_address() will call kvm_flush_remote_tlbs().
Signed-off-by: Lan Tianyu <[email protected]>
---
arch/x86/kvm/paging_tmpl.h | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/arch/x86/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h
index 6bdca39829bc..768c5c64e3f8 100644
--- a/arch/x86/kvm/paging_tmpl.h
+++ b/arch/x86/kvm/paging_tmpl.h
@@ -1033,8 +1033,9 @@ static int FNAME(sync_page)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp)
true, false, host_writable);
}
- if (set_spte_ret & SET_SPTE_NEED_REMOTE_TLB_FLUSH)
- kvm_flush_remote_tlbs(vcpu->kvm);
+
+ kvm_flush_remote_tlbs_with_address(vcpu->kvm, sp->gfn,
+ KVM_PAGES_PER_HPAGE(sp->role.level + 1));
return nr_present;
}
--
2.14.4
From: Lan Tianyu <[email protected]>
This patch is to flush tlb directly in the kvm_mmu_slot_gfn_write_protect()
when range flush is available.
Signed-off-by: Lan Tianyu <[email protected]>
---
arch/x86/kvm/mmu.c | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index 7a862c56b954..60b1771e400e 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -1729,6 +1729,11 @@ bool kvm_mmu_slot_gfn_write_protect(struct kvm *kvm,
write_protected |= __rmap_write_protect(kvm, rmap_head, true);
}
+ if (write_protected && kvm_available_flush_tlb_with_range()) {
+ kvm_flush_remote_tlbs_with_address(kvm, gfn, 1);
+ write_protected = false;
+ }
+
return write_protected;
}
--
2.14.4
From: Lan Tianyu <[email protected]>
This patch is to use tlb range flush to flush memslot's in the
kvm_vm_ioctl_get/clear_dirty_log() instead of flushing tlbs
of entire ept page table when range flush is available.
Signed-off-by: Lan Tianyu <[email protected]>
---
arch/x86/kvm/mmu.c | 8 +-------
arch/x86/kvm/mmu.h | 7 +++++++
arch/x86/kvm/x86.c | 16 ++++++++++++----
3 files changed, 20 insertions(+), 11 deletions(-)
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index 60b1771e400e..e9a727aad603 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -266,12 +266,6 @@ static void mmu_spte_set(u64 *sptep, u64 spte);
static union kvm_mmu_page_role
kvm_mmu_calc_root_page_role(struct kvm_vcpu *vcpu);
-
-static inline bool kvm_available_flush_tlb_with_range(void)
-{
- return kvm_x86_ops->tlb_remote_flush_with_range;
-}
-
static void kvm_flush_remote_tlbs_with_range(struct kvm *kvm,
struct kvm_tlb_range *range)
{
@@ -284,7 +278,7 @@ static void kvm_flush_remote_tlbs_with_range(struct kvm *kvm,
kvm_flush_remote_tlbs(kvm);
}
-static void kvm_flush_remote_tlbs_with_address(struct kvm *kvm,
+void kvm_flush_remote_tlbs_with_address(struct kvm *kvm,
u64 start_gfn, u64 pages)
{
struct kvm_tlb_range range;
diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h
index bbdc60f2fae8..5e0d9418b912 100644
--- a/arch/x86/kvm/mmu.h
+++ b/arch/x86/kvm/mmu.h
@@ -63,6 +63,13 @@ void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly,
bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu);
int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code,
u64 fault_address, char *insn, int insn_len);
+void kvm_flush_remote_tlbs_with_address(struct kvm *kvm,
+ u64 start_gfn, u64 pages);
+
+static inline bool kvm_available_flush_tlb_with_range(void)
+{
+ return kvm_x86_ops->tlb_remote_flush_with_range;
+}
static inline unsigned int kvm_mmu_available_pages(struct kvm *kvm)
{
diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 40d8272bee96..35738a7256f4 100644
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -4448,9 +4448,13 @@ int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, struct kvm_dirty_log *log)
* kvm_mmu_slot_remove_write_access().
*/
lockdep_assert_held(&kvm->slots_lock);
- if (flush)
- kvm_flush_remote_tlbs(kvm);
+ if (flush) {
+ struct kvm_memory_slot *memslot = kvm_get_memslot(kvm,
+ log->slot);
+ kvm_flush_remote_tlbs_with_address(kvm, memslot->base_gfn,
+ memslot->npages);
+ }
mutex_unlock(&kvm->slots_lock);
return r;
}
@@ -4475,9 +4479,13 @@ int kvm_vm_ioctl_clear_dirty_log(struct kvm *kvm, struct kvm_clear_dirty_log *lo
* kvm_mmu_slot_remove_write_access().
*/
lockdep_assert_held(&kvm->slots_lock);
- if (flush)
- kvm_flush_remote_tlbs(kvm);
+ if (flush) {
+ struct kvm_memory_slot *memslot = kvm_get_memslot(kvm,
+ log->slot);
+ kvm_flush_remote_tlbs_with_address(kvm, memslot->base_gfn,
+ memslot->npages);
+ }
mutex_unlock(&kvm->slots_lock);
return r;
}
--
2.14.4
From: Lan Tianyu <[email protected]>
Populate ranges on the flush list into struct hv_guest_mapping_flush_list
when flush list is available in the struct kvm_tlb_range.
Signed-off-by: Lan Tianyu <[email protected]>
---
Change since v2:
- Fix calculation of flush pages in the kvm_fill_hv_flush_list_func()
---
arch/x86/include/asm/kvm_host.h | 7 +++++++
arch/x86/kvm/vmx/vmx.c | 18 ++++++++++++++++--
2 files changed, 23 insertions(+), 2 deletions(-)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 875ae7256608..9fc9dd0c92cb 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -318,6 +318,12 @@ struct kvm_rmap_head {
struct kvm_mmu_page {
struct list_head link;
+
+ /*
+ * Tlb flush with range list uses struct kvm_mmu_page as list entry
+ * and all list operations should be under protection of mmu_lock.
+ */
+ struct hlist_node flush_link;
struct hlist_node hash_link;
bool unsync;
bool mmio_cached;
@@ -441,6 +447,7 @@ struct kvm_mmu {
struct kvm_tlb_range {
u64 start_gfn;
u64 pages;
+ struct hlist_head *flush_list;
};
enum pmc_type {
diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index 77b5379e3655..85139d318490 100644
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -432,9 +432,23 @@ static int kvm_fill_hv_flush_list_func(struct hv_guest_mapping_flush_list *flush
void *data)
{
struct kvm_tlb_range *range = data;
+ struct kvm_mmu_page *sp;
- return hyperv_fill_flush_guest_mapping_list(flush, 0, range->start_gfn,
- range->pages);
+ if (!range->flush_list) {
+ return hyperv_fill_flush_guest_mapping_list(flush,
+ 0, range->start_gfn, range->pages);
+ } else {
+ int offset = 0;
+
+ hlist_for_each_entry(sp, range->flush_list, flush_link) {
+ int pages = KVM_PAGES_PER_HPAGE(sp->role.level + 1);
+
+ offset = hyperv_fill_flush_guest_mapping_list(flush,
+ offset, sp->gfn, pages);
+ }
+
+ return offset;
+ }
}
static inline int __hv_remote_flush_tlb_with_range(struct kvm *kvm,
--
2.14.4
From: Lan Tianyu <[email protected]>
Add parameter offset to specify start position to add flush ranges in
guest address list of struct hv_guest_mapping_flush_list.
Signed-off-by: Lan Tianyu <[email protected]>
---
arch/x86/hyperv/nested.c | 4 ++--
arch/x86/include/asm/mshyperv.h | 2 +-
arch/x86/kvm/vmx/vmx.c | 2 +-
3 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/arch/x86/hyperv/nested.c b/arch/x86/hyperv/nested.c
index dd0a843f766d..96f8bac7476d 100644
--- a/arch/x86/hyperv/nested.c
+++ b/arch/x86/hyperv/nested.c
@@ -58,11 +58,11 @@ EXPORT_SYMBOL_GPL(hyperv_flush_guest_mapping);
int hyperv_fill_flush_guest_mapping_list(
struct hv_guest_mapping_flush_list *flush,
- u64 start_gfn, u64 pages)
+ int offset, u64 start_gfn, u64 pages)
{
u64 cur = start_gfn;
u64 additional_pages;
- int gpa_n = 0;
+ int gpa_n = offset;
do {
/*
diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h
index cc60e617931c..d6be685ab6b0 100644
--- a/arch/x86/include/asm/mshyperv.h
+++ b/arch/x86/include/asm/mshyperv.h
@@ -357,7 +357,7 @@ int hyperv_flush_guest_mapping_range(u64 as,
hyperv_fill_flush_list_func fill_func, void *data);
int hyperv_fill_flush_guest_mapping_list(
struct hv_guest_mapping_flush_list *flush,
- u64 start_gfn, u64 end_gfn);
+ int offset, u64 start_gfn, u64 end_gfn);
#ifdef CONFIG_X86_64
void hv_apic_init(void);
diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index 4950bb20e06a..77b5379e3655 100644
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -433,7 +433,7 @@ static int kvm_fill_hv_flush_list_func(struct hv_guest_mapping_flush_list *flush
{
struct kvm_tlb_range *range = data;
- return hyperv_fill_flush_guest_mapping_list(flush, range->start_gfn,
+ return hyperv_fill_flush_guest_mapping_list(flush, 0, range->start_gfn,
range->pages);
}
--
2.14.4
From: Lan Tianyu <[email protected]>
When build tlb range flush list, just add leaf node into flush list in order
to avoid overlap of address range in the list. If parent node and leaf node
are added into flush list, parent node's address range will cover leaf node's.
Otherwise, not all leaf nodes of the parent are actually allocated when flush
tlb. The side affect is that flush list would be overflow and go back to
non-range tlb flush if redundant address ranges was too many. This patch is
to add last_level flag in the struct kvm_mmu_page and set the flag to be true
in the set_spte() and clear the flag when the child node is allocated.
Signed-off-by: Lan Tianyu <[email protected]>
---
Change since v2:
- Always set last_level flag to be true in the set_spte().
- Clear last_level flag when assign child node.
---
arch/x86/include/asm/kvm_host.h | 1 +
arch/x86/kvm/mmu.c | 8 ++++++++
2 files changed, 9 insertions(+)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 3e8bd78940c4..1a0a381c442d 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -327,6 +327,7 @@ struct kvm_mmu_page {
struct hlist_node hash_link;
bool unsync;
bool mmio_cached;
+ bool last_level;
/*
* The following two entries are used to key the shadow page in the
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index 2e13aac28293..f5a33cf71d73 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -2754,6 +2754,7 @@ static void kvm_mmu_commit_zap_page(struct kvm *kvm,
*/
if (kvm_available_flush_tlb_with_range()) {
list_for_each_entry(sp, invalid_list, link)
+ if (sp->last_level)
hlist_add_head(&sp->flush_link, &flush_list);
kvm_flush_remote_tlbs_with_list(kvm, &flush_list);
@@ -2956,6 +2957,7 @@ static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
if (level > PT_PAGE_TABLE_LEVEL)
spte |= PT_PAGE_SIZE_MASK;
+
if (tdp_enabled)
spte |= kvm_x86_ops->get_mt_mask(vcpu, gfn,
kvm_is_mmio_pfn(pfn));
@@ -3010,6 +3012,8 @@ static int set_spte(struct kvm_vcpu *vcpu, u64 *sptep,
if (speculative)
spte = mark_spte_for_access_track(spte);
+ sp->last_level = true;
+
set_pte:
if (mmu_spte_update(sptep, spte))
ret |= SET_SPTE_NEED_REMOTE_TLB_FLUSH;
@@ -3200,6 +3204,10 @@ static int __direct_map(struct kvm_vcpu *vcpu, int write, int map_writable,
iterator.level - 1, 1, ACC_ALL);
link_shadow_page(vcpu, iterator.sptep, sp);
+
+ sp = page_header(__pa(iterator.sptep));
+ if (sp->last_level)
+ sp->last_level = false;
}
}
return emulate;
--
2.14.4
From: Lan Tianyu <[email protected]>
This patch is to add flush parameter for kvm_aga_hva() and move tlb
flush from kvm_mmu_notifier_clear_flush_young() to kvm_age_hva().
kvm_age_hva() can check whether tlb flush is necessary when
return value young is more than 0. Flush tlb if both conditions
are met.
Signed-off-by: Lan Tianyu <[email protected]>
---
arch/arm/include/asm/kvm_host.h | 3 ++-
arch/arm64/include/asm/kvm_host.h | 3 ++-
arch/mips/include/asm/kvm_host.h | 3 ++-
arch/mips/kvm/mmu.c | 11 +++++++++--
arch/powerpc/include/asm/kvm_host.h | 3 ++-
arch/powerpc/kvm/book3s.c | 10 ++++++++--
arch/powerpc/kvm/e500_mmu_host.c | 3 ++-
arch/x86/include/asm/kvm_host.h | 3 ++-
arch/x86/kvm/mmu.c | 10 ++++++++--
virt/kvm/arm/mmu.c | 13 +++++++++++--
virt/kvm/kvm_main.c | 6 ++----
11 files changed, 50 insertions(+), 18 deletions(-)
diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h
index ca56537b61bc..b3c6a6db8173 100644
--- a/arch/arm/include/asm/kvm_host.h
+++ b/arch/arm/include/asm/kvm_host.h
@@ -229,7 +229,8 @@ int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte);
unsigned long kvm_arm_num_regs(struct kvm_vcpu *vcpu);
int kvm_arm_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices);
-int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end);
+int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end,
+ bool flush);
int kvm_test_age_hva(struct kvm *kvm, unsigned long hva);
struct kvm_vcpu *kvm_arm_get_running_vcpu(void);
diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 7732d0ba4e60..182bbb2de60a 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -361,7 +361,8 @@ int __kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu,
int kvm_unmap_hva_range(struct kvm *kvm,
unsigned long start, unsigned long end);
int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte);
-int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end);
+int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end,
+ bool flush);
int kvm_test_age_hva(struct kvm *kvm, unsigned long hva);
struct kvm_vcpu *kvm_arm_get_running_vcpu(void);
diff --git a/arch/mips/include/asm/kvm_host.h b/arch/mips/include/asm/kvm_host.h
index 41204a49cf95..4eff221853d6 100644
--- a/arch/mips/include/asm/kvm_host.h
+++ b/arch/mips/include/asm/kvm_host.h
@@ -937,7 +937,8 @@ enum kvm_mips_fault_result kvm_trap_emul_gva_fault(struct kvm_vcpu *vcpu,
int kvm_unmap_hva_range(struct kvm *kvm,
unsigned long start, unsigned long end);
int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte);
-int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end);
+int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end,
+ bool flush);
int kvm_test_age_hva(struct kvm *kvm, unsigned long hva);
/* Emulation */
diff --git a/arch/mips/kvm/mmu.c b/arch/mips/kvm/mmu.c
index 97e538a8c1be..288a22d70cf8 100644
--- a/arch/mips/kvm/mmu.c
+++ b/arch/mips/kvm/mmu.c
@@ -579,9 +579,16 @@ static int kvm_test_age_hva_handler(struct kvm *kvm, gfn_t gfn, gfn_t gfn_end,
return pte_young(*gpa_pte);
}
-int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end)
+int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end,
+ bool flush)
{
- return handle_hva_to_gpa(kvm, start, end, kvm_age_hva_handler, NULL);
+ int young = handle_hva_to_gpa(kvm, start, end,
+ kvm_age_hva_handler, NULL);
+
+ if (flush && young > 0)
+ kvm_flush_remote_tlbs(kvm);
+
+ return young;
}
int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h
index 19693b8add93..c8043f1ac672 100644
--- a/arch/powerpc/include/asm/kvm_host.h
+++ b/arch/powerpc/include/asm/kvm_host.h
@@ -70,7 +70,8 @@
extern int kvm_unmap_hva_range(struct kvm *kvm,
unsigned long start, unsigned long end);
-extern int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end);
+extern int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end,
+ bool flush);
extern int kvm_test_age_hva(struct kvm *kvm, unsigned long hva);
extern int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte);
diff --git a/arch/powerpc/kvm/book3s.c b/arch/powerpc/kvm/book3s.c
index bd1a677dd9e4..09a67ebbde8a 100644
--- a/arch/powerpc/kvm/book3s.c
+++ b/arch/powerpc/kvm/book3s.c
@@ -841,9 +841,15 @@ int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end)
return kvm->arch.kvm_ops->unmap_hva_range(kvm, start, end);
}
-int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end)
+int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end,
+ bool flush)
{
- return kvm->arch.kvm_ops->age_hva(kvm, start, end);
+ int young = kvm->arch.kvm_ops->age_hva(kvm, start, end);
+
+ if (flush && young > 0)
+ kvm_flush_remote_tlbs(kvm);
+
+ return young;
}
int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
diff --git a/arch/powerpc/kvm/e500_mmu_host.c b/arch/powerpc/kvm/e500_mmu_host.c
index c3f312b2bcb3..e2f6c23ec39a 100644
--- a/arch/powerpc/kvm/e500_mmu_host.c
+++ b/arch/powerpc/kvm/e500_mmu_host.c
@@ -745,7 +745,8 @@ int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end)
return 0;
}
-int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end)
+int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end,
+ bool flush)
{
/* XXX could be more clever ;) */
return 0;
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 9fc9dd0c92cb..3e8bd78940c4 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -1514,7 +1514,8 @@ asmlinkage void kvm_spurious_fault(void);
#define KVM_ARCH_WANT_MMU_NOTIFIER
int kvm_unmap_hva_range(struct kvm *kvm, unsigned long start, unsigned long end);
-int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end);
+int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end,
+ bool flush);
int kvm_test_age_hva(struct kvm *kvm, unsigned long hva);
int kvm_set_spte_hva(struct kvm *kvm, unsigned long hva, pte_t pte);
int kvm_cpu_has_injectable_intr(struct kvm_vcpu *v);
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index e9a727aad603..295833dafc59 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -1988,9 +1988,15 @@ static void rmap_recycle(struct kvm_vcpu *vcpu, u64 *spte, gfn_t gfn)
KVM_PAGES_PER_HPAGE(sp->role.level));
}
-int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end)
+int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end,
+ bool flush)
{
- return kvm_handle_hva_range(kvm, start, end, 0, kvm_age_rmapp);
+ int young = kvm_handle_hva_range(kvm, start, end, 0, kvm_age_rmapp);
+
+ if (flush && young > 0)
+ kvm_flush_remote_tlbs(kvm);
+
+ return young;
}
int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
diff --git a/virt/kvm/arm/mmu.c b/virt/kvm/arm/mmu.c
index e0355e0f8712..e6ccaa5d58b8 100644
--- a/virt/kvm/arm/mmu.c
+++ b/virt/kvm/arm/mmu.c
@@ -2107,12 +2107,21 @@ static int kvm_test_age_hva_handler(struct kvm *kvm, gpa_t gpa, u64 size, void *
return pte_young(*pte);
}
-int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end)
+int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end,
+ bool flush)
{
+ int young;
+
if (!kvm->arch.pgd)
return 0;
trace_kvm_age_hva(start, end);
- return handle_hva_to_gpa(kvm, start, end, kvm_age_hva_handler, NULL);
+
+ young = handle_hva_to_gpa(kvm, start, end, kvm_age_hva_handler, NULL);
+
+ if (flush && young > 0)
+ kvm_flush_remote_tlbs(kvm);
+
+ return young;
}
int kvm_test_age_hva(struct kvm *kvm, unsigned long hva)
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index 61ddc3f0425c..90ef8191cb4c 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -433,9 +433,7 @@ static int kvm_mmu_notifier_clear_flush_young(struct mmu_notifier *mn,
idx = srcu_read_lock(&kvm->srcu);
spin_lock(&kvm->mmu_lock);
- young = kvm_age_hva(kvm, start, end);
- if (young)
- kvm_flush_remote_tlbs(kvm);
+ young = kvm_age_hva(kvm, start, end, true);
spin_unlock(&kvm->mmu_lock);
srcu_read_unlock(&kvm->srcu, idx);
@@ -466,7 +464,7 @@ static int kvm_mmu_notifier_clear_young(struct mmu_notifier *mn,
* cadence. If we find this inaccurate, we might come up with a
* more sophisticated heuristic later.
*/
- young = kvm_age_hva(kvm, start, end);
+ young = kvm_age_hva(kvm, start, end, false);
spin_unlock(&kvm->mmu_lock);
srcu_read_unlock(&kvm->srcu, idx);
--
2.14.4
From: Lan Tianyu <[email protected]>
This patch is to add kvm_get_memslot() to get struct kvm_memory_slot
via slot it and remove redundant codes. The function will also be used
in the following changes.
Signed-off-by: Lan Tianyu <[email protected]>
---
include/linux/kvm_host.h | 1 +
virt/kvm/kvm_main.c | 45 +++++++++++++++++++--------------------------
2 files changed, 20 insertions(+), 26 deletions(-)
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index 9d55c63db09b..87e665069f6d 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -779,6 +779,7 @@ int kvm_get_dirty_log_protect(struct kvm *kvm,
struct kvm_dirty_log *log, bool *flush);
int kvm_clear_dirty_log_protect(struct kvm *kvm,
struct kvm_clear_dirty_log *log, bool *flush);
+struct kvm_memory_slot *kvm_get_memslot(struct kvm *kvm, u32 slot);
void kvm_arch_mmu_enable_log_dirty_pt_masked(struct kvm *kvm,
struct kvm_memory_slot *slot,
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index 276af92ace6c..61ddc3f0425c 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -1099,22 +1099,30 @@ static int kvm_vm_ioctl_set_memory_region(struct kvm *kvm,
return kvm_set_memory_region(kvm, mem);
}
+struct kvm_memory_slot *kvm_get_memslot(struct kvm *kvm, u32 slot)
+{
+ struct kvm_memslots *slots;
+ int as_id, id;
+
+ as_id = slot >> 16;
+ id = (u16)slot;
+ if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
+ return NULL;
+
+ slots = __kvm_memslots(kvm, as_id);
+ return id_to_memslot(slots, id);
+}
+
int kvm_get_dirty_log(struct kvm *kvm,
struct kvm_dirty_log *log, int *is_dirty)
{
- struct kvm_memslots *slots;
struct kvm_memory_slot *memslot;
- int i, as_id, id;
unsigned long n;
unsigned long any = 0;
+ int i;
- as_id = log->slot >> 16;
- id = (u16)log->slot;
- if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
- return -EINVAL;
+ memslot = kvm_get_memslot(kvm, log->slot);
- slots = __kvm_memslots(kvm, as_id);
- memslot = id_to_memslot(slots, id);
if (!memslot->dirty_bitmap)
return -ENOENT;
@@ -1158,20 +1166,13 @@ EXPORT_SYMBOL_GPL(kvm_get_dirty_log);
int kvm_get_dirty_log_protect(struct kvm *kvm,
struct kvm_dirty_log *log, bool *flush)
{
- struct kvm_memslots *slots;
struct kvm_memory_slot *memslot;
- int i, as_id, id;
unsigned long n;
unsigned long *dirty_bitmap;
unsigned long *dirty_bitmap_buffer;
+ int i;
- as_id = log->slot >> 16;
- id = (u16)log->slot;
- if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
- return -EINVAL;
-
- slots = __kvm_memslots(kvm, as_id);
- memslot = id_to_memslot(slots, id);
+ memslot = kvm_get_memslot(kvm, log->slot);
dirty_bitmap = memslot->dirty_bitmap;
if (!dirty_bitmap)
@@ -1227,24 +1228,16 @@ EXPORT_SYMBOL_GPL(kvm_get_dirty_log_protect);
int kvm_clear_dirty_log_protect(struct kvm *kvm,
struct kvm_clear_dirty_log *log, bool *flush)
{
- struct kvm_memslots *slots;
struct kvm_memory_slot *memslot;
- int as_id, id;
gfn_t offset;
unsigned long i, n;
unsigned long *dirty_bitmap;
unsigned long *dirty_bitmap_buffer;
- as_id = log->slot >> 16;
- id = (u16)log->slot;
- if (as_id >= KVM_ADDRESS_SPACE_NUM || id >= KVM_USER_MEM_SLOTS)
- return -EINVAL;
-
if ((log->first_page & 63) || (log->num_pages & 63))
return -EINVAL;
- slots = __kvm_memslots(kvm, as_id);
- memslot = id_to_memslot(slots, id);
+ memslot = kvm_get_memslot(kvm, log->slot);
dirty_bitmap = memslot->dirty_bitmap;
if (!dirty_bitmap)
--
2.14.4
From: Lan Tianyu <[email protected]>
This patch is to use tlb range flush in the kvm_age_hva().
If range flush is not available, kvm_flush_remote_tlbs_with_address()
will call kvm_flush_remote_tlbs().
Signed-off-by: Lan Tianyu <[email protected]>
---
arch/x86/kvm/mmu.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index 295833dafc59..2e13aac28293 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -1994,7 +1994,7 @@ int kvm_age_hva(struct kvm *kvm, unsigned long start, unsigned long end,
int young = kvm_handle_hva_range(kvm, start, end, 0, kvm_age_rmapp);
if (flush && young > 0)
- kvm_flush_remote_tlbs(kvm);
+ kvm_flush_remote_tlbs_with_address(kvm, start, end - start + 1);
return young;
}
--
2.14.4
int hyperv_fill_flush_guest_mapping_list(
struct hv_guest_mapping_flush_list *flush,
- u64 start_gfn, u64 pages)
+ int offset, u64 start_gfn, u64 pages)
{
u64 cur = start_gfn;
u64 additional_pages;
- int gpa_n = 0;
+ int gpa_n = offset;
do {
/*
Do you mean to support negative offsets here? Maybe unsigned would be better?
On 22/02/19 16:06, [email protected] wrote:
> From: Lan Tianyu <[email protected]>
>
> This patchset is to introduce hv ept tlb range list flush function
> support in the KVM MMU component. Flushing ept tlbs of several address
> range can be done via single hypercall and new list flush function is
> used in the kvm_mmu_commit_zap_page() and FNAME(sync_page). This patchset
> also adds more hv ept tlb range flush support in more KVM MMU function.
>
> This patchset is based on the fix patch "x86/Hyper-V: Fix definition HV_MAX_FLUSH_REP_COUNT".
> (https://www.mail-archive.com/[email protected]/msg1939455.html)
Note that this won't make it in 5.1 unless Linus releases an -rc8.
Otherwise, I'll get to it next week.
Thanks,
Paolo
On Sat, Feb 23, 2019 at 2:26 AM Paolo Bonzini <[email protected]> wrote:
>
> On 22/02/19 16:06, [email protected] wrote:
> > From: Lan Tianyu <[email protected]>
> >
> > This patchset is to introduce hv ept tlb range list flush function
> > support in the KVM MMU component. Flushing ept tlbs of several address
> > range can be done via single hypercall and new list flush function is
> > used in the kvm_mmu_commit_zap_page() and FNAME(sync_page). This patchset
> > also adds more hv ept tlb range flush support in more KVM MMU function.
> >
> > This patchset is based on the fix patch "x86/Hyper-V: Fix definition HV_MAX_FLUSH_REP_COUNT".
> > (https://www.mail-archive.com/[email protected]/msg1939455.html)
>
> Note that this won't make it in 5.1 unless Linus releases an -rc8.
> Otherwise, I'll get to it next week.
Hi Paolo:
Sure. Thanks for your review.
--
Best regards
Tianyu Lan
Hi Stephen:
Thanks for your review.
On Sat, Feb 23, 2019 at 1:08 AM Stephen Hemminger
<[email protected]> wrote:
>
> int hyperv_fill_flush_guest_mapping_list(
> struct hv_guest_mapping_flush_list *flush,
> - u64 start_gfn, u64 pages)
> + int offset, u64 start_gfn, u64 pages)
> {
> u64 cur = start_gfn;
> u64 additional_pages;
> - int gpa_n = 0;
> + int gpa_n = offset;
>
> do {
> /*
>
> Do you mean to support negative offsets here? Maybe unsigned would be better?
Yes, this makes sense. Will update. Thanks.
--
Best regards
Tianyu Lan
From: Lan Tianyu <[email protected]>
Add parameter offset to specify start position to add flush ranges in
guest address list of struct hv_guest_mapping_flush_list.
Signed-off-by: Lan Tianyu <[email protected]>
---
Update "offset" parameter's type of hyperv_fill_flush_guest_mapping_list()
arch/x86/hyperv/nested.c | 4 ++--
arch/x86/include/asm/mshyperv.h | 2 +-
arch/x86/kvm/vmx/vmx.c | 2 +-
3 files changed, 4 insertions(+), 4 deletions(-)
diff --git a/arch/x86/hyperv/nested.c b/arch/x86/hyperv/nested.c
index dd0a843f766d..d54c2276c922 100644
--- a/arch/x86/hyperv/nested.c
+++ b/arch/x86/hyperv/nested.c
@@ -58,11 +58,11 @@ EXPORT_SYMBOL_GPL(hyperv_flush_guest_mapping);
int hyperv_fill_flush_guest_mapping_list(
struct hv_guest_mapping_flush_list *flush,
- u64 start_gfn, u64 pages)
+ u32 offset, u64 start_gfn, u64 pages)
{
u64 cur = start_gfn;
u64 additional_pages;
- int gpa_n = 0;
+ int gpa_n = offset;
do {
/*
diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h
index cc60e617931c..8b63ed95780e 100644
--- a/arch/x86/include/asm/mshyperv.h
+++ b/arch/x86/include/asm/mshyperv.h
@@ -357,7 +357,7 @@ int hyperv_flush_guest_mapping_range(u64 as,
hyperv_fill_flush_list_func fill_func, void *data);
int hyperv_fill_flush_guest_mapping_list(
struct hv_guest_mapping_flush_list *flush,
- u64 start_gfn, u64 end_gfn);
+ u32 offset, u64 start_gfn, u64 end_gfn);
#ifdef CONFIG_X86_64
void hv_apic_init(void);
diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index 4950bb20e06a..77b5379e3655 100644
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -433,7 +433,7 @@ static int kvm_fill_hv_flush_list_func(struct hv_guest_mapping_flush_list *flush
{
struct kvm_tlb_range *range = data;
- return hyperv_fill_flush_guest_mapping_list(flush, range->start_gfn,
+ return hyperv_fill_flush_guest_mapping_list(flush, 0, range->start_gfn,
range->pages);
}
--
2.14.4
From: Lan Tianyu <[email protected]>
Populate ranges on the flush list into struct hv_guest_mapping_flush_list
when flush list is available in the struct kvm_tlb_range.
Signed-off-by: Lan Tianyu <[email protected]>
---
Update:
- Add check of return value "offset" in the kvm_fill_hv_flush_list_func()
Change since v2:
- Fix calculation of flush pages in the kvm_fill_hv_flush_list_func()
---
arch/x86/include/asm/kvm_host.h | 7 +++++++
arch/x86/kvm/vmx/vmx.c | 21 +++++++++++++++++++--
2 files changed, 26 insertions(+), 2 deletions(-)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 875ae7256608..9fc9dd0c92cb 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -318,6 +318,12 @@ struct kvm_rmap_head {
struct kvm_mmu_page {
struct list_head link;
+
+ /*
+ * Tlb flush with range list uses struct kvm_mmu_page as list entry
+ * and all list operations should be under protection of mmu_lock.
+ */
+ struct hlist_node flush_link;
struct hlist_node hash_link;
bool unsync;
bool mmio_cached;
@@ -441,6 +447,7 @@ struct kvm_mmu {
struct kvm_tlb_range {
u64 start_gfn;
u64 pages;
+ struct hlist_head *flush_list;
};
enum pmc_type {
diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index 77b5379e3655..197545121355 100644
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -432,9 +432,26 @@ static int kvm_fill_hv_flush_list_func(struct hv_guest_mapping_flush_list *flush
void *data)
{
struct kvm_tlb_range *range = data;
+ struct kvm_mmu_page *sp;
- return hyperv_fill_flush_guest_mapping_list(flush, 0, range->start_gfn,
- range->pages);
+ if (!range->flush_list) {
+ return hyperv_fill_flush_guest_mapping_list(flush,
+ 0, range->start_gfn, range->pages);
+ } else {
+ int offset = 0;
+
+ hlist_for_each_entry(sp, range->flush_list, flush_link) {
+ int pages = KVM_PAGES_PER_HPAGE(sp->role.level + 1);
+
+ offset = hyperv_fill_flush_guest_mapping_list(flush,
+ offset, sp->gfn, pages);
+ if (offset < 0)
+ return offset;
+
+ }
+
+ return offset;
+ }
}
static inline int __hv_remote_flush_tlb_with_range(struct kvm *kvm,
--
2.14.4