From: Paul Durrant <[email protected]>
Some pfncache pages may actually be overlays on guest memory that have a
fixed HVA within the VMM. It's pointless to invalidate such cached
mappings if the overlay is moved so allow a cache to be activated directly
with the HVA to cater for such cases. A subsequent patch will make use
of this facility.
Signed-off-by: Paul Durrant <[email protected]>
Reviewed-by: David Woodhouse <[email protected]>
---
Cc: Sean Christopherson <[email protected]>
Cc: Paolo Bonzini <[email protected]>
Cc: David Woodhouse <[email protected]>
v13:
- Define kvm_is_error_gpa() to check GPA validity.
- Add an either/or address check to __kvm_gpc_refresh() as requested.
- Make sure memslot is NULL if the cache is activated with an HVA.
v11:
- Fixed kvm_gpc_check() to ignore memslot generation if the cache is not
activated with a GPA. (This breakage occured during the re-work for v8).
v9:
- Pass both GPA and HVA into __kvm_gpc_refresh() rather than overloading
the address paraneter and using a bool flag to indicated what it is.
v8:
- Re-worked to avoid messing with struct gfn_to_pfn_cache.
---
include/linux/kvm_host.h | 21 +++++++++
virt/kvm/pfncache.c | 94 +++++++++++++++++++++++++++++-----------
2 files changed, 89 insertions(+), 26 deletions(-)
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index 41ee515b304e..043cc824d55a 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -148,6 +148,11 @@ static inline bool kvm_is_error_hva(unsigned long addr)
#endif
+static inline bool kvm_is_error_gpa(gpa_t gpa)
+{
+ return gpa == INVALID_GPA;
+}
+
#define KVM_ERR_PTR_BAD_PAGE (ERR_PTR(-ENOENT))
static inline bool is_error_page(struct page *page)
@@ -1343,6 +1348,22 @@ void kvm_gpc_init(struct gfn_to_pfn_cache *gpc, struct kvm *kvm);
*/
int kvm_gpc_activate(struct gfn_to_pfn_cache *gpc, gpa_t gpa, unsigned long len);
+/**
+ * kvm_gpc_activate_hva - prepare a cached kernel mapping and HPA for a given HVA.
+ *
+ * @gpc: struct gfn_to_pfn_cache object.
+ * @hva: userspace virtual address to map.
+ * @len: sanity check; the range being access must fit a single page.
+ *
+ * @return: 0 for success.
+ * -EINVAL for a mapping which would cross a page boundary.
+ * -EFAULT for an untranslatable guest physical address.
+ *
+ * The semantics of this function are the same as those of kvm_gpc_activate(). It
+ * merely bypasses a layer of address translation.
+ */
+int kvm_gpc_activate_hva(struct gfn_to_pfn_cache *gpc, unsigned long hva, unsigned long len);
+
/**
* kvm_gpc_check - check validity of a gfn_to_pfn_cache.
*
diff --git a/virt/kvm/pfncache.c b/virt/kvm/pfncache.c
index 97eec8ee3449..4e64d349b2f7 100644
--- a/virt/kvm/pfncache.c
+++ b/virt/kvm/pfncache.c
@@ -48,7 +48,14 @@ bool kvm_gpc_check(struct gfn_to_pfn_cache *gpc, unsigned long len)
if (!gpc->active)
return false;
- if (gpc->generation != slots->generation || kvm_is_error_hva(gpc->uhva))
+ /*
+ * If the page was cached from a memslot, make sure the memslots have
+ * not been re-configured.
+ */
+ if (!kvm_is_error_gpa(gpc->gpa) && gpc->generation != slots->generation)
+ return false;
+
+ if (kvm_is_error_hva(gpc->uhva))
return false;
if (offset_in_page(gpc->uhva) + len > PAGE_SIZE)
@@ -209,11 +216,11 @@ static kvm_pfn_t hva_to_pfn_retry(struct gfn_to_pfn_cache *gpc)
return -EFAULT;
}
-static int __kvm_gpc_refresh(struct gfn_to_pfn_cache *gpc, gpa_t gpa,
+static int __kvm_gpc_refresh(struct gfn_to_pfn_cache *gpc, gpa_t gpa, unsigned long uhva,
unsigned long len)
{
- struct kvm_memslots *slots = kvm_memslots(gpc->kvm);
- unsigned long page_offset = offset_in_page(gpa);
+ unsigned long page_offset = kvm_is_error_gpa(gpa) ?
+ offset_in_page(uhva) : offset_in_page(gpa);
bool unmap_old = false;
unsigned long old_uhva;
kvm_pfn_t old_pfn;
@@ -221,6 +228,10 @@ static int __kvm_gpc_refresh(struct gfn_to_pfn_cache *gpc, gpa_t gpa,
void *old_khva;
int ret;
+ /* Either gpa or uhva must be valid, but not both */
+ if (WARN_ON_ONCE(kvm_is_error_gpa(gpa) == kvm_is_error_hva(uhva)))
+ return -EINVAL;
+
/*
* If must fit within a single page. The 'len' argument is
* only to enforce that.
@@ -246,29 +257,39 @@ static int __kvm_gpc_refresh(struct gfn_to_pfn_cache *gpc, gpa_t gpa,
old_khva = (void *)PAGE_ALIGN_DOWN((uintptr_t)gpc->khva);
old_uhva = PAGE_ALIGN_DOWN(gpc->uhva);
- /* Refresh the userspace HVA if necessary */
- if (gpc->gpa != gpa || gpc->generation != slots->generation ||
- kvm_is_error_hva(gpc->uhva)) {
- gfn_t gfn = gpa_to_gfn(gpa);
-
- gpc->gpa = gpa;
- gpc->generation = slots->generation;
- gpc->memslot = __gfn_to_memslot(slots, gfn);
- gpc->uhva = gfn_to_hva_memslot(gpc->memslot, gfn);
+ if (kvm_is_error_gpa(gpa)) {
+ gpc->gpa = INVALID_GPA;
+ gpc->memslot = NULL;
+ gpc->uhva = PAGE_ALIGN_DOWN(uhva);
- if (kvm_is_error_hva(gpc->uhva)) {
- ret = -EFAULT;
- goto out;
- }
-
- /*
- * Even if the GPA and/or the memslot generation changed, the
- * HVA may still be the same.
- */
if (gpc->uhva != old_uhva)
hva_change = true;
} else {
- gpc->uhva = old_uhva;
+ struct kvm_memslots *slots = kvm_memslots(gpc->kvm);
+
+ if (gpc->gpa != gpa || gpc->generation != slots->generation ||
+ kvm_is_error_hva(gpc->uhva)) {
+ gfn_t gfn = gpa_to_gfn(gpa);
+
+ gpc->gpa = gpa;
+ gpc->generation = slots->generation;
+ gpc->memslot = __gfn_to_memslot(slots, gfn);
+ gpc->uhva = gfn_to_hva_memslot(gpc->memslot, gfn);
+
+ if (kvm_is_error_hva(gpc->uhva)) {
+ ret = -EFAULT;
+ goto out;
+ }
+
+ /*
+ * Even if the GPA and/or the memslot generation changed, the
+ * HVA may still be the same.
+ */
+ if (gpc->uhva != old_uhva)
+ hva_change = true;
+ } else {
+ gpc->uhva = old_uhva;
+ }
}
/* Note: the offset must be correct before calling hva_to_pfn_retry() */
@@ -319,7 +340,16 @@ static int __kvm_gpc_refresh(struct gfn_to_pfn_cache *gpc, gpa_t gpa,
int kvm_gpc_refresh(struct gfn_to_pfn_cache *gpc, unsigned long len)
{
- return __kvm_gpc_refresh(gpc, gpc->gpa, len);
+ unsigned long uhva = gpc->uhva;
+
+ /*
+ * If the GPA is valid then invalidate the HVA, otherwise
+ * __kvm_gpc_refresh() will fail its strict either/or address check.
+ */
+ if (!kvm_is_error_gpa(gpc->gpa))
+ uhva = KVM_HVA_ERR_BAD;
+
+ return __kvm_gpc_refresh(gpc, gpc->gpa, uhva, len);
}
void kvm_gpc_init(struct gfn_to_pfn_cache *gpc, struct kvm *kvm)
@@ -329,10 +359,12 @@ void kvm_gpc_init(struct gfn_to_pfn_cache *gpc, struct kvm *kvm)
gpc->kvm = kvm;
gpc->pfn = KVM_PFN_ERR_FAULT;
+ gpc->gpa = INVALID_GPA;
gpc->uhva = KVM_HVA_ERR_BAD;
}
-int kvm_gpc_activate(struct gfn_to_pfn_cache *gpc, gpa_t gpa, unsigned long len)
+static int __kvm_gpc_activate(struct gfn_to_pfn_cache *gpc, gpa_t gpa, unsigned long uhva,
+ unsigned long len)
{
struct kvm *kvm = gpc->kvm;
@@ -353,7 +385,17 @@ int kvm_gpc_activate(struct gfn_to_pfn_cache *gpc, gpa_t gpa, unsigned long len)
gpc->active = true;
write_unlock_irq(&gpc->lock);
}
- return __kvm_gpc_refresh(gpc, gpa, len);
+ return __kvm_gpc_refresh(gpc, gpa, uhva, len);
+}
+
+int kvm_gpc_activate(struct gfn_to_pfn_cache *gpc, gpa_t gpa, unsigned long len)
+{
+ return __kvm_gpc_activate(gpc, gpa, KVM_HVA_ERR_BAD, len);
+}
+
+int kvm_gpc_activate_hva(struct gfn_to_pfn_cache *gpc, unsigned long uhva, unsigned long len)
+{
+ return __kvm_gpc_activate(gpc, INVALID_GPA, uhva, len);
}
void kvm_gpc_deactivate(struct gfn_to_pfn_cache *gpc)
--
2.39.2
On Thu, Feb 15, 2024, Paul Durrant wrote:
> @@ -319,7 +340,16 @@ static int __kvm_gpc_refresh(struct gfn_to_pfn_cache *gpc, gpa_t gpa,
>
> int kvm_gpc_refresh(struct gfn_to_pfn_cache *gpc, unsigned long len)
> {
> - return __kvm_gpc_refresh(gpc, gpc->gpa, len);
> + unsigned long uhva = gpc->uhva;
> +
> + /*
> + * If the GPA is valid then invalidate the HVA, otherwise
> + * __kvm_gpc_refresh() will fail its strict either/or address check.
> + */
It's not just to make the strict check happy, though that's obviously the direct
motivation, it's so that there's one root of truth. The strict check is there to
enforce that behavior and to make it more clear to readers that it's an either/or
situation.
> + if (!kvm_is_error_gpa(gpc->gpa))
> + uhva = KVM_HVA_ERR_BAD;
This would be a good time to use a ternary operator.
/*
* If the GPA is valid then ignore the HVA, as a cache can be GPA-based
* or HVA-based, not both. For GPA-based caches, the HVA will be
* recomputed during refresh if necessary.
*/
unsigned long uhva = kvm_is_error_gpa(gpc->gpa) ? gpc->uhva :
KVM_HVA_ERR_BAD;
On 19/02/2024 21:49, Sean Christopherson wrote:
> On Thu, Feb 15, 2024, Paul Durrant wrote:
>> @@ -319,7 +340,16 @@ static int __kvm_gpc_refresh(struct gfn_to_pfn_cache *gpc, gpa_t gpa,
>>
>> int kvm_gpc_refresh(struct gfn_to_pfn_cache *gpc, unsigned long len)
>> {
>> - return __kvm_gpc_refresh(gpc, gpc->gpa, len);
>> + unsigned long uhva = gpc->uhva;
>> +
>> + /*
>> + * If the GPA is valid then invalidate the HVA, otherwise
>> + * __kvm_gpc_refresh() will fail its strict either/or address check.
>> + */
>
> It's not just to make the strict check happy, though that's obviously the direct
> motivation, it's so that there's one root of truth. The strict check is there to
> enforce that behavior and to make it more clear to readers that it's an either/or
> situation.
>
>> + if (!kvm_is_error_gpa(gpc->gpa))
>> + uhva = KVM_HVA_ERR_BAD;
>
> This would be a good time to use a ternary operator.
>
> /*
> * If the GPA is valid then ignore the HVA, as a cache can be GPA-based
> * or HVA-based, not both. For GPA-based caches, the HVA will be
> * recomputed during refresh if necessary.
> */
> unsigned long uhva = kvm_is_error_gpa(gpc->gpa) ? gpc->uhva :
> KVM_HVA_ERR_BAD;
Ok. I thought that actually looked a little less neat in this case, but
I'll change.