Doorbell emulation for KVM on PAPR guests is broken as support for DPDES
was not added in initial patch series [1].
Add DPDES support and doorbell handling support for V2 API.
[1] lore.kernel.org/linuxppc-dev/[email protected]
Changes in v2:
1. Split DPDES support into its own patch
Gautam Menghani (2):
arch/powerpc/kvm: Add DPDES support in helper library for Guest state
buffer
arch/powerpc/kvm: Fix doorbell emulation for v2 API
Documentation/arch/powerpc/kvm-nested.rst | 4 +++-
arch/powerpc/include/asm/guest-state-buffer.h | 3 ++-
arch/powerpc/include/asm/kvm_book3s.h | 1 +
arch/powerpc/kvm/book3s_hv.c | 5 +++++
arch/powerpc/kvm/book3s_hv_nestedv2.c | 7 +++++++
arch/powerpc/kvm/test-guest-state-buffer.c | 2 +-
6 files changed, 19 insertions(+), 3 deletions(-)
--
2.45.1
Doorbell emulation is broken for KVM on PAPR guests as support for
DPDES was not added in the initial patch series. Due to this, a KVM on
PAPR guest with SMT > 1 cannot be booted with the XICS interrupt
controller as doorbells are setup in the initial probe path when using XICS
(pSeries_smp_probe()).
Command to replicate the above bug:
qemu-system-ppc64 \
-drive file=rhel.qcow2,format=qcow2 \
-m 20G \
-smp 8,cores=1,threads=8 \
-cpu host \
-nographic \
-machine pseries,ic-mode=xics -accel kvm
Add doorbell state handling support in the host
KVM code to fix doorbell emulation.
Fixes: 19d31c5f1157 ("KVM: PPC: Add support for nestedv2 guests")
Cc: [email protected] # v6.7
Signed-off-by: Gautam Menghani <[email protected]>
---
arch/powerpc/kvm/book3s_hv.c | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c
index 35cb014a0c51..21c69647d27c 100644
--- a/arch/powerpc/kvm/book3s_hv.c
+++ b/arch/powerpc/kvm/book3s_hv.c
@@ -4116,6 +4116,11 @@ static int kvmhv_vcpu_entry_nestedv2(struct kvm_vcpu *vcpu, u64 time_limit,
int trap;
long rc;
+ if (vcpu->arch.doorbell_request) {
+ vcpu->arch.doorbell_request = 0;
+ kvmppc_set_dpdes(vcpu, 1);
+ }
+
io = &vcpu->arch.nestedv2_io;
msr = mfmsr();
--
2.45.1
Add support for using DPDES in the library for using guest state
buffers. DPDES support is needed for enabling usage of doorbells in a
L2 KVM on PAPR guest.
Fixes: 6ccbbc33f06a ("KVM: PPC: Add helper library for Guest State Buffers")
Cc: [email protected] # v6.7
Signed-off-by: Gautam Menghani <[email protected]>
---
Documentation/arch/powerpc/kvm-nested.rst | 4 +++-
arch/powerpc/include/asm/guest-state-buffer.h | 3 ++-
arch/powerpc/include/asm/kvm_book3s.h | 1 +
arch/powerpc/kvm/book3s_hv_nestedv2.c | 7 +++++++
arch/powerpc/kvm/test-guest-state-buffer.c | 2 +-
5 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/Documentation/arch/powerpc/kvm-nested.rst b/Documentation/arch/powerpc/kvm-nested.rst
index 630602a8aa00..5defd13cc6c1 100644
--- a/Documentation/arch/powerpc/kvm-nested.rst
+++ b/Documentation/arch/powerpc/kvm-nested.rst
@@ -546,7 +546,9 @@ table information.
+--------+-------+----+--------+----------------------------------+
| 0x1052 | 0x08 | RW | T | CTRL |
+--------+-------+----+--------+----------------------------------+
-| 0x1053-| | | | Reserved |
+| 0x1053 | 0x08 | RW | T | DPDES |
++--------+-------+----+--------+----------------------------------+
+| 0x1054-| | | | Reserved |
| 0x1FFF | | | | |
+--------+-------+----+--------+----------------------------------+
| 0x2000 | 0x04 | RW | T | CR |
diff --git a/arch/powerpc/include/asm/guest-state-buffer.h b/arch/powerpc/include/asm/guest-state-buffer.h
index 808149f31576..d107abe1468f 100644
--- a/arch/powerpc/include/asm/guest-state-buffer.h
+++ b/arch/powerpc/include/asm/guest-state-buffer.h
@@ -81,6 +81,7 @@
#define KVMPPC_GSID_HASHKEYR 0x1050
#define KVMPPC_GSID_HASHPKEYR 0x1051
#define KVMPPC_GSID_CTRL 0x1052
+#define KVMPPC_GSID_DPDES 0x1053
#define KVMPPC_GSID_CR 0x2000
#define KVMPPC_GSID_PIDR 0x2001
@@ -110,7 +111,7 @@
#define KVMPPC_GSE_META_COUNT (KVMPPC_GSE_META_END - KVMPPC_GSE_META_START + 1)
#define KVMPPC_GSE_DW_REGS_START KVMPPC_GSID_GPR(0)
-#define KVMPPC_GSE_DW_REGS_END KVMPPC_GSID_CTRL
+#define KVMPPC_GSE_DW_REGS_END KVMPPC_GSID_DPDES
#define KVMPPC_GSE_DW_REGS_COUNT \
(KVMPPC_GSE_DW_REGS_END - KVMPPC_GSE_DW_REGS_START + 1)
diff --git a/arch/powerpc/include/asm/kvm_book3s.h b/arch/powerpc/include/asm/kvm_book3s.h
index 3e1e2a698c9e..10618622d7ef 100644
--- a/arch/powerpc/include/asm/kvm_book3s.h
+++ b/arch/powerpc/include/asm/kvm_book3s.h
@@ -594,6 +594,7 @@ static inline u##size kvmppc_get_##reg(struct kvm_vcpu *vcpu) \
KVMPPC_BOOK3S_VCORE_ACCESSOR(vtb, 64, KVMPPC_GSID_VTB)
+KVMPPC_BOOK3S_VCORE_ACCESSOR(dpdes, 64, KVMPPC_GSID_DPDES)
KVMPPC_BOOK3S_VCORE_ACCESSOR_GET(arch_compat, 32, KVMPPC_GSID_LOGICAL_PVR)
KVMPPC_BOOK3S_VCORE_ACCESSOR_GET(lpcr, 64, KVMPPC_GSID_LPCR)
KVMPPC_BOOK3S_VCORE_ACCESSOR_SET(tb_offset, 64, KVMPPC_GSID_TB_OFFSET)
diff --git a/arch/powerpc/kvm/book3s_hv_nestedv2.c b/arch/powerpc/kvm/book3s_hv_nestedv2.c
index 8e6f5355f08b..36863fff2a99 100644
--- a/arch/powerpc/kvm/book3s_hv_nestedv2.c
+++ b/arch/powerpc/kvm/book3s_hv_nestedv2.c
@@ -311,6 +311,10 @@ static int gs_msg_ops_vcpu_fill_info(struct kvmppc_gs_buff *gsb,
rc = kvmppc_gse_put_u64(gsb, iden,
vcpu->arch.vcore->vtb);
break;
+ case KVMPPC_GSID_DPDES:
+ rc = kvmppc_gse_put_u64(gsb, iden,
+ vcpu->arch.vcore->dpdes);
+ break;
case KVMPPC_GSID_LPCR:
rc = kvmppc_gse_put_u64(gsb, iden,
vcpu->arch.vcore->lpcr);
@@ -543,6 +547,9 @@ static int gs_msg_ops_vcpu_refresh_info(struct kvmppc_gs_msg *gsm,
case KVMPPC_GSID_VTB:
vcpu->arch.vcore->vtb = kvmppc_gse_get_u64(gse);
break;
+ case KVMPPC_GSID_DPDES:
+ vcpu->arch.vcore->dpdes = kvmppc_gse_get_u64(gse);
+ break;
case KVMPPC_GSID_LPCR:
vcpu->arch.vcore->lpcr = kvmppc_gse_get_u64(gse);
break;
diff --git a/arch/powerpc/kvm/test-guest-state-buffer.c b/arch/powerpc/kvm/test-guest-state-buffer.c
index 4720b8dc8837..2571ccc618c9 100644
--- a/arch/powerpc/kvm/test-guest-state-buffer.c
+++ b/arch/powerpc/kvm/test-guest-state-buffer.c
@@ -151,7 +151,7 @@ static void test_gs_bitmap(struct kunit *test)
i++;
}
- for (u16 iden = KVMPPC_GSID_GPR(0); iden <= KVMPPC_GSID_CTRL; iden++) {
+ for (u16 iden = KVMPPC_GSID_GPR(0); iden <= KVMPPC_GSE_DW_REGS_END; iden++) {
kvmppc_gsbm_set(&gsbm, iden);
kvmppc_gsbm_set(&gsbm1, iden);
KUNIT_EXPECT_TRUE(test, kvmppc_gsbm_test(&gsbm, iden));
--
2.45.1
On Wed Jun 5, 2024 at 9:39 PM AEST, Gautam Menghani wrote:
> Add support for using DPDES in the library for using guest state
> buffers. DPDES support is needed for enabling usage of doorbells in a
> L2 KVM on PAPR guest.
>
Reviewed-by: Nicholas Piggin <[email protected]>
> Fixes: 6ccbbc33f06a ("KVM: PPC: Add helper library for Guest State Buffers")
> Cc: [email protected] # v6.7
> Signed-off-by: Gautam Menghani <[email protected]>
> ---
> Documentation/arch/powerpc/kvm-nested.rst | 4 +++-
> arch/powerpc/include/asm/guest-state-buffer.h | 3 ++-
> arch/powerpc/include/asm/kvm_book3s.h | 1 +
> arch/powerpc/kvm/book3s_hv_nestedv2.c | 7 +++++++
> arch/powerpc/kvm/test-guest-state-buffer.c | 2 +-
> 5 files changed, 14 insertions(+), 3 deletions(-)
>
> diff --git a/Documentation/arch/powerpc/kvm-nested.rst b/Documentation/arch/powerpc/kvm-nested.rst
> index 630602a8aa00..5defd13cc6c1 100644
> --- a/Documentation/arch/powerpc/kvm-nested.rst
> +++ b/Documentation/arch/powerpc/kvm-nested.rst
> @@ -546,7 +546,9 @@ table information.
> +--------+-------+----+--------+----------------------------------+
> | 0x1052 | 0x08 | RW | T | CTRL |
> +--------+-------+----+--------+----------------------------------+
> -| 0x1053-| | | | Reserved |
> +| 0x1053 | 0x08 | RW | T | DPDES |
> ++--------+-------+----+--------+----------------------------------+
> +| 0x1054-| | | | Reserved |
> | 0x1FFF | | | | |
> +--------+-------+----+--------+----------------------------------+
> | 0x2000 | 0x04 | RW | T | CR |
> diff --git a/arch/powerpc/include/asm/guest-state-buffer.h b/arch/powerpc/include/asm/guest-state-buffer.h
> index 808149f31576..d107abe1468f 100644
> --- a/arch/powerpc/include/asm/guest-state-buffer.h
> +++ b/arch/powerpc/include/asm/guest-state-buffer.h
> @@ -81,6 +81,7 @@
> #define KVMPPC_GSID_HASHKEYR 0x1050
> #define KVMPPC_GSID_HASHPKEYR 0x1051
> #define KVMPPC_GSID_CTRL 0x1052
> +#define KVMPPC_GSID_DPDES 0x1053
>
> #define KVMPPC_GSID_CR 0x2000
> #define KVMPPC_GSID_PIDR 0x2001
> @@ -110,7 +111,7 @@
> #define KVMPPC_GSE_META_COUNT (KVMPPC_GSE_META_END - KVMPPC_GSE_META_START + 1)
>
> #define KVMPPC_GSE_DW_REGS_START KVMPPC_GSID_GPR(0)
> -#define KVMPPC_GSE_DW_REGS_END KVMPPC_GSID_CTRL
> +#define KVMPPC_GSE_DW_REGS_END KVMPPC_GSID_DPDES
> #define KVMPPC_GSE_DW_REGS_COUNT \
> (KVMPPC_GSE_DW_REGS_END - KVMPPC_GSE_DW_REGS_START + 1)
>
> diff --git a/arch/powerpc/include/asm/kvm_book3s.h b/arch/powerpc/include/asm/kvm_book3s.h
> index 3e1e2a698c9e..10618622d7ef 100644
> --- a/arch/powerpc/include/asm/kvm_book3s.h
> +++ b/arch/powerpc/include/asm/kvm_book3s.h
> @@ -594,6 +594,7 @@ static inline u##size kvmppc_get_##reg(struct kvm_vcpu *vcpu) \
>
>
> KVMPPC_BOOK3S_VCORE_ACCESSOR(vtb, 64, KVMPPC_GSID_VTB)
> +KVMPPC_BOOK3S_VCORE_ACCESSOR(dpdes, 64, KVMPPC_GSID_DPDES)
> KVMPPC_BOOK3S_VCORE_ACCESSOR_GET(arch_compat, 32, KVMPPC_GSID_LOGICAL_PVR)
> KVMPPC_BOOK3S_VCORE_ACCESSOR_GET(lpcr, 64, KVMPPC_GSID_LPCR)
> KVMPPC_BOOK3S_VCORE_ACCESSOR_SET(tb_offset, 64, KVMPPC_GSID_TB_OFFSET)
> diff --git a/arch/powerpc/kvm/book3s_hv_nestedv2.c b/arch/powerpc/kvm/book3s_hv_nestedv2.c
> index 8e6f5355f08b..36863fff2a99 100644
> --- a/arch/powerpc/kvm/book3s_hv_nestedv2.c
> +++ b/arch/powerpc/kvm/book3s_hv_nestedv2.c
> @@ -311,6 +311,10 @@ static int gs_msg_ops_vcpu_fill_info(struct kvmppc_gs_buff *gsb,
> rc = kvmppc_gse_put_u64(gsb, iden,
> vcpu->arch.vcore->vtb);
> break;
> + case KVMPPC_GSID_DPDES:
> + rc = kvmppc_gse_put_u64(gsb, iden,
> + vcpu->arch.vcore->dpdes);
> + break;
> case KVMPPC_GSID_LPCR:
> rc = kvmppc_gse_put_u64(gsb, iden,
> vcpu->arch.vcore->lpcr);
> @@ -543,6 +547,9 @@ static int gs_msg_ops_vcpu_refresh_info(struct kvmppc_gs_msg *gsm,
> case KVMPPC_GSID_VTB:
> vcpu->arch.vcore->vtb = kvmppc_gse_get_u64(gse);
> break;
> + case KVMPPC_GSID_DPDES:
> + vcpu->arch.vcore->dpdes = kvmppc_gse_get_u64(gse);
> + break;
> case KVMPPC_GSID_LPCR:
> vcpu->arch.vcore->lpcr = kvmppc_gse_get_u64(gse);
> break;
> diff --git a/arch/powerpc/kvm/test-guest-state-buffer.c b/arch/powerpc/kvm/test-guest-state-buffer.c
> index 4720b8dc8837..2571ccc618c9 100644
> --- a/arch/powerpc/kvm/test-guest-state-buffer.c
> +++ b/arch/powerpc/kvm/test-guest-state-buffer.c
> @@ -151,7 +151,7 @@ static void test_gs_bitmap(struct kunit *test)
> i++;
> }
>
> - for (u16 iden = KVMPPC_GSID_GPR(0); iden <= KVMPPC_GSID_CTRL; iden++) {
> + for (u16 iden = KVMPPC_GSID_GPR(0); iden <= KVMPPC_GSE_DW_REGS_END; iden++) {
> kvmppc_gsbm_set(&gsbm, iden);
> kvmppc_gsbm_set(&gsbm1, iden);
> KUNIT_EXPECT_TRUE(test, kvmppc_gsbm_test(&gsbm, iden));
On Wed Jun 5, 2024 at 9:39 PM AEST, Gautam Menghani wrote:
> Doorbell emulation is broken for KVM on PAPR guests as support for
> DPDES was not added in the initial patch series. Due to this, a KVM on
> PAPR guest with SMT > 1 cannot be booted with the XICS interrupt
> controller as doorbells are setup in the initial probe path when using XICS
> (pSeries_smp_probe()).
>
> Command to replicate the above bug:
>
> qemu-system-ppc64 \
> -drive file=rhel.qcow2,format=qcow2 \
> -m 20G \
> -smp 8,cores=1,threads=8 \
> -cpu host \
> -nographic \
> -machine pseries,ic-mode=xics -accel kvm
>
> Add doorbell state handling support in the host
> KVM code to fix doorbell emulation.
Reviewed-by: Nicholas Piggin <[email protected]>
>
> Fixes: 19d31c5f1157 ("KVM: PPC: Add support for nestedv2 guests")
> Cc: [email protected] # v6.7
> Signed-off-by: Gautam Menghani <[email protected]>
> ---
> arch/powerpc/kvm/book3s_hv.c | 5 +++++
> 1 file changed, 5 insertions(+)
>
> diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c
> index 35cb014a0c51..21c69647d27c 100644
> --- a/arch/powerpc/kvm/book3s_hv.c
> +++ b/arch/powerpc/kvm/book3s_hv.c
> @@ -4116,6 +4116,11 @@ static int kvmhv_vcpu_entry_nestedv2(struct kvm_vcpu *vcpu, u64 time_limit,
> int trap;
> long rc;
>
> + if (vcpu->arch.doorbell_request) {
> + vcpu->arch.doorbell_request = 0;
> + kvmppc_set_dpdes(vcpu, 1);
> + }
> +
> io = &vcpu->arch.nestedv2_io;
>
> msr = mfmsr();
On Wed Jun 5, 2024 at 9:39 PM AEST, Gautam Menghani wrote:
> Doorbell emulation for KVM on PAPR guests is broken as support for DPDES
> was not added in initial patch series [1].
> Add DPDES support and doorbell handling support for V2 API.
Looks good, thanks. So fix for v1 doorbells is coming?
Thanks,
Nick
>
> [1] lore.kernel.org/linuxppc-dev/[email protected]
>
> Changes in v2:
> 1. Split DPDES support into its own patch
>
> Gautam Menghani (2):
> arch/powerpc/kvm: Add DPDES support in helper library for Guest state
> buffer
> arch/powerpc/kvm: Fix doorbell emulation for v2 API
>
> Documentation/arch/powerpc/kvm-nested.rst | 4 +++-
> arch/powerpc/include/asm/guest-state-buffer.h | 3 ++-
> arch/powerpc/include/asm/kvm_book3s.h | 1 +
> arch/powerpc/kvm/book3s_hv.c | 5 +++++
> arch/powerpc/kvm/book3s_hv_nestedv2.c | 7 +++++++
> arch/powerpc/kvm/test-guest-state-buffer.c | 2 +-
> 6 files changed, 19 insertions(+), 3 deletions(-)
On Thu, Jun 06, 2024 at 01:00:19PM GMT, Nicholas Piggin wrote:
> On Wed Jun 5, 2024 at 9:39 PM AEST, Gautam Menghani wrote:
> > Doorbell emulation for KVM on PAPR guests is broken as support for DPDES
> > was not added in initial patch series [1].
> > Add DPDES support and doorbell handling support for V2 API.
>
> Looks good, thanks. So fix for v1 doorbells is coming?
Yes I've root caused the doorbell breakage in V1 API to
commit 7c3ded5735141ff4d049747c9f76672a8b737c49. I'll send out a fix
soon.
Thanks,
Gautam
On Wed, Jun 05, 2024 at 05:09:08PM GMT, Gautam Menghani wrote:
> Doorbell emulation for KVM on PAPR guests is broken as support for DPDES
> was not added in initial patch series [1].
> Add DPDES support and doorbell handling support for V2 API.
>
> [1] lore.kernel.org/linuxppc-dev/[email protected]
>
> Changes in v2:
> 1. Split DPDES support into its own patch
>
> Gautam Menghani (2):
> arch/powerpc/kvm: Add DPDES support in helper library for Guest state
> buffer
> arch/powerpc/kvm: Fix doorbell emulation for v2 API
>
> Documentation/arch/powerpc/kvm-nested.rst | 4 +++-
> arch/powerpc/include/asm/guest-state-buffer.h | 3 ++-
> arch/powerpc/include/asm/kvm_book3s.h | 1 +
> arch/powerpc/kvm/book3s_hv.c | 5 +++++
> arch/powerpc/kvm/book3s_hv_nestedv2.c | 7 +++++++
> arch/powerpc/kvm/test-guest-state-buffer.c | 2 +-
> 6 files changed, 19 insertions(+), 3 deletions(-)
>
> --
> 2.45.1
>
Hi Michael,
This patch series is to be backported for all kernels >= 6.7. So the tag
should be
Cc: [email protected] # v6.7+
and not
Cc: [email protected] # v6.7
Should I send a new version of this series or can you please make this
change when pulling in your tree?
Thanks,
Gautam
Gautam Menghani <[email protected]> writes:
> On Wed, Jun 05, 2024 at 05:09:08PM GMT, Gautam Menghani wrote:
>> Doorbell emulation for KVM on PAPR guests is broken as support for DPDES
>> was not added in initial patch series [1].
>> Add DPDES support and doorbell handling support for V2 API.
>>
>> [1] lore.kernel.org/linuxppc-dev/[email protected]
>>
>> Changes in v2:
>> 1. Split DPDES support into its own patch
>>
>> Gautam Menghani (2):
>> arch/powerpc/kvm: Add DPDES support in helper library for Guest state
>> buffer
>> arch/powerpc/kvm: Fix doorbell emulation for v2 API
>>
>> Documentation/arch/powerpc/kvm-nested.rst | 4 +++-
>> arch/powerpc/include/asm/guest-state-buffer.h | 3 ++-
>> arch/powerpc/include/asm/kvm_book3s.h | 1 +
>> arch/powerpc/kvm/book3s_hv.c | 5 +++++
>> arch/powerpc/kvm/book3s_hv_nestedv2.c | 7 +++++++
>> arch/powerpc/kvm/test-guest-state-buffer.c | 2 +-
>> 6 files changed, 19 insertions(+), 3 deletions(-)
>>
>> --
>> 2.45.1
>>
>
>
> Hi Michael,
>
> This patch series is to be backported for all kernels >= 6.7. So the tag
> should be
> Cc: [email protected] # v6.7+
>
> and not
> Cc: [email protected] # v6.7
>
> Should I send a new version of this series or can you please make this
> change when pulling in your tree?
I can make the change when applying.
cheers