Use try_cmpxchg64 instead of cmpxchg64 (*ptr, old, new) != old in
tdp_mmu_set_spte_atomic. cmpxchg returns success in ZF flag, so this
change saves a compare after cmpxchg (and related move instruction
in front of cmpxchg). Also, remove explicit assignment to iter->old_spte
when cmpxchg fails, this is what try_cmpxchg does implicitly.
Cc: Paolo Bonzini <[email protected]>
Cc: Sean Christopherson <[email protected]>
Signed-off-by: Uros Bizjak <[email protected]>
---
Patch requires commits 0aa7be05d83cc584da0782405e8007e351dfb6cc
and c2df0a6af177b6c06a859806a876f92b072dc624 from tip.git
---
arch/x86/kvm/mmu/tdp_mmu.c | 12 +-----------
1 file changed, 1 insertion(+), 11 deletions(-)
diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c
index 922b06bf4b94..1ccc1a0f8123 100644
--- a/arch/x86/kvm/mmu/tdp_mmu.c
+++ b/arch/x86/kvm/mmu/tdp_mmu.c
@@ -633,7 +633,6 @@ static inline int tdp_mmu_set_spte_atomic(struct kvm *kvm,
u64 new_spte)
{
u64 *sptep = rcu_dereference(iter->sptep);
- u64 old_spte;
/*
* The caller is responsible for ensuring the old SPTE is not a REMOVED
@@ -649,17 +648,8 @@ static inline int tdp_mmu_set_spte_atomic(struct kvm *kvm,
* Note, fast_pf_fix_direct_spte() can also modify TDP MMU SPTEs and
* does not hold the mmu_lock.
*/
- old_spte = cmpxchg64(sptep, iter->old_spte, new_spte);
- if (old_spte != iter->old_spte) {
- /*
- * The page table entry was modified by a different logical
- * CPU. Refresh iter->old_spte with the current value so the
- * caller operates on fresh data, e.g. if it retries
- * tdp_mmu_set_spte_atomic().
- */
- iter->old_spte = old_spte;
+ if (!try_cmpxchg64(sptep, &iter->old_spte, new_spte))
return -EBUSY;
- }
__handle_changed_spte(kvm, iter->as_id, iter->gfn, iter->old_spte,
new_spte, iter->level, true);
--
2.35.1
On Wed, May 18, 2022 at 7:08 AM Uros Bizjak <[email protected]> wrote:
>
> Use try_cmpxchg64 instead of cmpxchg64 (*ptr, old, new) != old in
> tdp_mmu_set_spte_atomic. cmpxchg returns success in ZF flag, so this
> change saves a compare after cmpxchg (and related move instruction
> in front of cmpxchg). Also, remove explicit assignment to iter->old_spte
> when cmpxchg fails, this is what try_cmpxchg does implicitly.
>
> Cc: Paolo Bonzini <[email protected]>
> Cc: Sean Christopherson <[email protected]>
> Signed-off-by: Uros Bizjak <[email protected]>
Nice cleanup.
Reviewed-by: David Matlack <[email protected]>
> ---
> Patch requires commits 0aa7be05d83cc584da0782405e8007e351dfb6cc
> and c2df0a6af177b6c06a859806a876f92b072dc624 from tip.git
> ---
> arch/x86/kvm/mmu/tdp_mmu.c | 12 +-----------
> 1 file changed, 1 insertion(+), 11 deletions(-)
>
> diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c
> index 922b06bf4b94..1ccc1a0f8123 100644
> --- a/arch/x86/kvm/mmu/tdp_mmu.c
> +++ b/arch/x86/kvm/mmu/tdp_mmu.c
> @@ -633,7 +633,6 @@ static inline int tdp_mmu_set_spte_atomic(struct kvm *kvm,
> u64 new_spte)
> {
> u64 *sptep = rcu_dereference(iter->sptep);
> - u64 old_spte;
>
> /*
> * The caller is responsible for ensuring the old SPTE is not a REMOVED
> @@ -649,17 +648,8 @@ static inline int tdp_mmu_set_spte_atomic(struct kvm *kvm,
> * Note, fast_pf_fix_direct_spte() can also modify TDP MMU SPTEs and
> * does not hold the mmu_lock.
> */
> - old_spte = cmpxchg64(sptep, iter->old_spte, new_spte);
> - if (old_spte != iter->old_spte) {
> - /*
> - * The page table entry was modified by a different logical
> - * CPU. Refresh iter->old_spte with the current value so the
> - * caller operates on fresh data, e.g. if it retries
> - * tdp_mmu_set_spte_atomic().
> - */
> - iter->old_spte = old_spte;
> + if (!try_cmpxchg64(sptep, &iter->old_spte, new_spte))
> return -EBUSY;
> - }
>
> __handle_changed_spte(kvm, iter->as_id, iter->gfn, iter->old_spte,
> new_spte, iter->level, true);
> --
> 2.35.1
>
Queued, thanks.
Paolo