tree: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git master
head: d93a0d43e3d0ba9e19387be4dae4a8d5b175a8d7
commit: 97e4910232fa1f81e806aa60c25a0450276d99a2 linux/compiler-clang.h: define HAVE_BUILTIN_BSWAP*
date: 3 weeks ago
config: powerpc64-randconfig-r006-20210403 (attached as .config)
compiler: clang version 13.0.0 (https://github.com/llvm/llvm-project 0fe8af94688aa03c01913c2001d6a1a911f42ce6)
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# install powerpc64 cross compiling tool for clang build
# apt-get install binutils-powerpc64-linux-gnu
# https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=97e4910232fa1f81e806aa60c25a0450276d99a2
git remote add linus https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
git fetch --no-tags linus master
git checkout 97e4910232fa1f81e806aa60c25a0450276d99a2
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross ARCH=powerpc64
If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <[email protected]>
All errors (new ones prefixed by >>):
>> arch/powerpc/kvm/book3s_hv_nested.c:264:6: error: stack frame size of 2304 bytes in function 'kvmhv_enter_nested_guest' [-Werror,-Wframe-larger-than=]
long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu)
^
1 error generated.
vim +/kvmhv_enter_nested_guest +264 arch/powerpc/kvm/book3s_hv_nested.c
afe75049303f75 Ravi Bangoria 2020-12-16 263
360cae313702cd Paul Mackerras 2018-10-08 @264 long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu)
360cae313702cd Paul Mackerras 2018-10-08 265 {
360cae313702cd Paul Mackerras 2018-10-08 266 long int err, r;
360cae313702cd Paul Mackerras 2018-10-08 267 struct kvm_nested_guest *l2;
360cae313702cd Paul Mackerras 2018-10-08 268 struct pt_regs l2_regs, saved_l1_regs;
afe75049303f75 Ravi Bangoria 2020-12-16 269 struct hv_guest_state l2_hv = {0}, saved_l1_hv;
360cae313702cd Paul Mackerras 2018-10-08 270 struct kvmppc_vcore *vc = vcpu->arch.vcore;
360cae313702cd Paul Mackerras 2018-10-08 271 u64 hv_ptr, regs_ptr;
360cae313702cd Paul Mackerras 2018-10-08 272 u64 hdec_exp;
360cae313702cd Paul Mackerras 2018-10-08 273 s64 delta_purr, delta_spurr, delta_ic, delta_vtb;
360cae313702cd Paul Mackerras 2018-10-08 274 u64 mask;
360cae313702cd Paul Mackerras 2018-10-08 275 unsigned long lpcr;
360cae313702cd Paul Mackerras 2018-10-08 276
360cae313702cd Paul Mackerras 2018-10-08 277 if (vcpu->kvm->arch.l1_ptcr == 0)
360cae313702cd Paul Mackerras 2018-10-08 278 return H_NOT_AVAILABLE;
360cae313702cd Paul Mackerras 2018-10-08 279
360cae313702cd Paul Mackerras 2018-10-08 280 /* copy parameters in */
360cae313702cd Paul Mackerras 2018-10-08 281 hv_ptr = kvmppc_get_gpr(vcpu, 4);
1508c22f112ce1 Alexey Kardashevskiy 2020-06-09 282 regs_ptr = kvmppc_get_gpr(vcpu, 5);
1508c22f112ce1 Alexey Kardashevskiy 2020-06-09 283 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
afe75049303f75 Ravi Bangoria 2020-12-16 284 err = kvmhv_read_guest_state_and_regs(vcpu, &l2_hv, &l2_regs,
afe75049303f75 Ravi Bangoria 2020-12-16 285 hv_ptr, regs_ptr);
1508c22f112ce1 Alexey Kardashevskiy 2020-06-09 286 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
360cae313702cd Paul Mackerras 2018-10-08 287 if (err)
360cae313702cd Paul Mackerras 2018-10-08 288 return H_PARAMETER;
1508c22f112ce1 Alexey Kardashevskiy 2020-06-09 289
10b5022db7861a Suraj Jitindar Singh 2018-10-08 290 if (kvmppc_need_byteswap(vcpu))
10b5022db7861a Suraj Jitindar Singh 2018-10-08 291 byteswap_hv_regs(&l2_hv);
afe75049303f75 Ravi Bangoria 2020-12-16 292 if (l2_hv.version > HV_GUEST_STATE_VERSION)
360cae313702cd Paul Mackerras 2018-10-08 293 return H_P2;
360cae313702cd Paul Mackerras 2018-10-08 294
10b5022db7861a Suraj Jitindar Singh 2018-10-08 295 if (kvmppc_need_byteswap(vcpu))
10b5022db7861a Suraj Jitindar Singh 2018-10-08 296 byteswap_pt_regs(&l2_regs);
9d0b048da788c1 Suraj Jitindar Singh 2018-10-08 297 if (l2_hv.vcpu_token >= NR_CPUS)
9d0b048da788c1 Suraj Jitindar Singh 2018-10-08 298 return H_PARAMETER;
9d0b048da788c1 Suraj Jitindar Singh 2018-10-08 299
360cae313702cd Paul Mackerras 2018-10-08 300 /* translate lpid */
360cae313702cd Paul Mackerras 2018-10-08 301 l2 = kvmhv_get_nested(vcpu->kvm, l2_hv.lpid, true);
360cae313702cd Paul Mackerras 2018-10-08 302 if (!l2)
360cae313702cd Paul Mackerras 2018-10-08 303 return H_PARAMETER;
360cae313702cd Paul Mackerras 2018-10-08 304 if (!l2->l1_gr_to_hr) {
360cae313702cd Paul Mackerras 2018-10-08 305 mutex_lock(&l2->tlb_lock);
360cae313702cd Paul Mackerras 2018-10-08 306 kvmhv_update_ptbl_cache(l2);
360cae313702cd Paul Mackerras 2018-10-08 307 mutex_unlock(&l2->tlb_lock);
360cae313702cd Paul Mackerras 2018-10-08 308 }
360cae313702cd Paul Mackerras 2018-10-08 309
360cae313702cd Paul Mackerras 2018-10-08 310 /* save l1 values of things */
360cae313702cd Paul Mackerras 2018-10-08 311 vcpu->arch.regs.msr = vcpu->arch.shregs.msr;
360cae313702cd Paul Mackerras 2018-10-08 312 saved_l1_regs = vcpu->arch.regs;
360cae313702cd Paul Mackerras 2018-10-08 313 kvmhv_save_hv_regs(vcpu, &saved_l1_hv);
360cae313702cd Paul Mackerras 2018-10-08 314
360cae313702cd Paul Mackerras 2018-10-08 315 /* convert TB values/offsets to host (L0) values */
360cae313702cd Paul Mackerras 2018-10-08 316 hdec_exp = l2_hv.hdec_expiry - vc->tb_offset;
360cae313702cd Paul Mackerras 2018-10-08 317 vc->tb_offset += l2_hv.tb_offset;
360cae313702cd Paul Mackerras 2018-10-08 318
360cae313702cd Paul Mackerras 2018-10-08 319 /* set L1 state to L2 state */
360cae313702cd Paul Mackerras 2018-10-08 320 vcpu->arch.nested = l2;
360cae313702cd Paul Mackerras 2018-10-08 321 vcpu->arch.nested_vcpu_id = l2_hv.vcpu_token;
360cae313702cd Paul Mackerras 2018-10-08 322 vcpu->arch.regs = l2_regs;
360cae313702cd Paul Mackerras 2018-10-08 323 vcpu->arch.shregs.msr = vcpu->arch.regs.msr;
360cae313702cd Paul Mackerras 2018-10-08 324 mask = LPCR_DPFD | LPCR_ILE | LPCR_TC | LPCR_AIL | LPCR_LD |
360cae313702cd Paul Mackerras 2018-10-08 325 LPCR_LPES | LPCR_MER;
360cae313702cd Paul Mackerras 2018-10-08 326 lpcr = (vc->lpcr & ~mask) | (l2_hv.lpcr & mask);
73937deb4b2d7f Suraj Jitindar Singh 2018-10-08 327 sanitise_hv_regs(vcpu, &l2_hv);
360cae313702cd Paul Mackerras 2018-10-08 328 restore_hv_regs(vcpu, &l2_hv);
360cae313702cd Paul Mackerras 2018-10-08 329
360cae313702cd Paul Mackerras 2018-10-08 330 vcpu->arch.ret = RESUME_GUEST;
360cae313702cd Paul Mackerras 2018-10-08 331 vcpu->arch.trap = 0;
360cae313702cd Paul Mackerras 2018-10-08 332 do {
360cae313702cd Paul Mackerras 2018-10-08 333 if (mftb() >= hdec_exp) {
360cae313702cd Paul Mackerras 2018-10-08 334 vcpu->arch.trap = BOOK3S_INTERRUPT_HV_DECREMENTER;
360cae313702cd Paul Mackerras 2018-10-08 335 r = RESUME_HOST;
360cae313702cd Paul Mackerras 2018-10-08 336 break;
360cae313702cd Paul Mackerras 2018-10-08 337 }
8c99d34578628b Tianjia Zhang 2020-04-27 338 r = kvmhv_run_single_vcpu(vcpu, hdec_exp, lpcr);
360cae313702cd Paul Mackerras 2018-10-08 339 } while (is_kvmppc_resume_guest(r));
360cae313702cd Paul Mackerras 2018-10-08 340
360cae313702cd Paul Mackerras 2018-10-08 341 /* save L2 state for return */
360cae313702cd Paul Mackerras 2018-10-08 342 l2_regs = vcpu->arch.regs;
360cae313702cd Paul Mackerras 2018-10-08 343 l2_regs.msr = vcpu->arch.shregs.msr;
360cae313702cd Paul Mackerras 2018-10-08 344 delta_purr = vcpu->arch.purr - l2_hv.purr;
360cae313702cd Paul Mackerras 2018-10-08 345 delta_spurr = vcpu->arch.spurr - l2_hv.spurr;
360cae313702cd Paul Mackerras 2018-10-08 346 delta_ic = vcpu->arch.ic - l2_hv.ic;
360cae313702cd Paul Mackerras 2018-10-08 347 delta_vtb = vc->vtb - l2_hv.vtb;
360cae313702cd Paul Mackerras 2018-10-08 348 save_hv_return_state(vcpu, vcpu->arch.trap, &l2_hv);
360cae313702cd Paul Mackerras 2018-10-08 349
360cae313702cd Paul Mackerras 2018-10-08 350 /* restore L1 state */
360cae313702cd Paul Mackerras 2018-10-08 351 vcpu->arch.nested = NULL;
360cae313702cd Paul Mackerras 2018-10-08 352 vcpu->arch.regs = saved_l1_regs;
360cae313702cd Paul Mackerras 2018-10-08 353 vcpu->arch.shregs.msr = saved_l1_regs.msr & ~MSR_TS_MASK;
360cae313702cd Paul Mackerras 2018-10-08 354 /* set L1 MSR TS field according to L2 transaction state */
360cae313702cd Paul Mackerras 2018-10-08 355 if (l2_regs.msr & MSR_TS_MASK)
360cae313702cd Paul Mackerras 2018-10-08 356 vcpu->arch.shregs.msr |= MSR_TS_S;
360cae313702cd Paul Mackerras 2018-10-08 357 vc->tb_offset = saved_l1_hv.tb_offset;
360cae313702cd Paul Mackerras 2018-10-08 358 restore_hv_regs(vcpu, &saved_l1_hv);
360cae313702cd Paul Mackerras 2018-10-08 359 vcpu->arch.purr += delta_purr;
360cae313702cd Paul Mackerras 2018-10-08 360 vcpu->arch.spurr += delta_spurr;
360cae313702cd Paul Mackerras 2018-10-08 361 vcpu->arch.ic += delta_ic;
360cae313702cd Paul Mackerras 2018-10-08 362 vc->vtb += delta_vtb;
360cae313702cd Paul Mackerras 2018-10-08 363
360cae313702cd Paul Mackerras 2018-10-08 364 kvmhv_put_nested(l2);
360cae313702cd Paul Mackerras 2018-10-08 365
360cae313702cd Paul Mackerras 2018-10-08 366 /* copy l2_hv_state and regs back to guest */
10b5022db7861a Suraj Jitindar Singh 2018-10-08 367 if (kvmppc_need_byteswap(vcpu)) {
10b5022db7861a Suraj Jitindar Singh 2018-10-08 368 byteswap_hv_regs(&l2_hv);
10b5022db7861a Suraj Jitindar Singh 2018-10-08 369 byteswap_pt_regs(&l2_regs);
10b5022db7861a Suraj Jitindar Singh 2018-10-08 370 }
1508c22f112ce1 Alexey Kardashevskiy 2020-06-09 371 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
afe75049303f75 Ravi Bangoria 2020-12-16 372 err = kvmhv_write_guest_state_and_regs(vcpu, &l2_hv, &l2_regs,
afe75049303f75 Ravi Bangoria 2020-12-16 373 hv_ptr, regs_ptr);
1508c22f112ce1 Alexey Kardashevskiy 2020-06-09 374 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
360cae313702cd Paul Mackerras 2018-10-08 375 if (err)
360cae313702cd Paul Mackerras 2018-10-08 376 return H_AUTHORITY;
360cae313702cd Paul Mackerras 2018-10-08 377
360cae313702cd Paul Mackerras 2018-10-08 378 if (r == -EINTR)
360cae313702cd Paul Mackerras 2018-10-08 379 return H_INTERRUPT;
360cae313702cd Paul Mackerras 2018-10-08 380
873db2cd9a6d7f Suraj Jitindar Singh 2018-12-14 381 if (vcpu->mmio_needed) {
873db2cd9a6d7f Suraj Jitindar Singh 2018-12-14 382 kvmhv_nested_mmio_needed(vcpu, regs_ptr);
873db2cd9a6d7f Suraj Jitindar Singh 2018-12-14 383 return H_TOO_HARD;
873db2cd9a6d7f Suraj Jitindar Singh 2018-12-14 384 }
873db2cd9a6d7f Suraj Jitindar Singh 2018-12-14 385
360cae313702cd Paul Mackerras 2018-10-08 386 return vcpu->arch.trap;
360cae313702cd Paul Mackerras 2018-10-08 387 }
360cae313702cd Paul Mackerras 2018-10-08 388
:::::: The code at line 264 was first introduced by commit
:::::: 360cae313702cdd0b90f82c261a8302fecef030a KVM: PPC: Book3S HV: Nested guest entry via hypercall
:::::: TO: Paul Mackerras <[email protected]>
:::::: CC: Michael Ellerman <[email protected]>
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/[email protected]
Excerpts from kernel test robot's message of April 3, 2021 8:47 pm:
> tree: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git master
> head: d93a0d43e3d0ba9e19387be4dae4a8d5b175a8d7
> commit: 97e4910232fa1f81e806aa60c25a0450276d99a2 linux/compiler-clang.h: define HAVE_BUILTIN_BSWAP*
> date: 3 weeks ago
> config: powerpc64-randconfig-r006-20210403 (attached as .config)
> compiler: clang version 13.0.0 (https://github.com/llvm/llvm-project 0fe8af94688aa03c01913c2001d6a1a911f42ce6)
> reproduce (this is a W=1 build):
> wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
> chmod +x ~/bin/make.cross
> # install powerpc64 cross compiling tool for clang build
> # apt-get install binutils-powerpc64-linux-gnu
> # https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=97e4910232fa1f81e806aa60c25a0450276d99a2
> git remote add linus https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
> git fetch --no-tags linus master
> git checkout 97e4910232fa1f81e806aa60c25a0450276d99a2
> # save the attached .config to linux build tree
> COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross ARCH=powerpc64
>
> If you fix the issue, kindly add following tag as appropriate
> Reported-by: kernel test robot <[email protected]>
>
> All errors (new ones prefixed by >>):
>
>>> arch/powerpc/kvm/book3s_hv_nested.c:264:6: error: stack frame size of 2304 bytes in function 'kvmhv_enter_nested_guest' [-Werror,-Wframe-larger-than=]
> long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu)
> ^
> 1 error generated.
>
>
> vim +/kvmhv_enter_nested_guest +264 arch/powerpc/kvm/book3s_hv_nested.c
Not much changed here recently. It's not that big a concern because it's
only called in the KVM ioctl path, not in any deep IO paths or anything,
and doesn't recurse. Might be a bit of inlining or stack spilling put it
over the edge.
powerpc does make it an error though, would be good to avoid that so the
robot doesn't keep tripping over.
Thanks,
Nick
>
> afe75049303f75 Ravi Bangoria 2020-12-16 263
> 360cae313702cd Paul Mackerras 2018-10-08 @264 long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu)
> 360cae313702cd Paul Mackerras 2018-10-08 265 {
> 360cae313702cd Paul Mackerras 2018-10-08 266 long int err, r;
> 360cae313702cd Paul Mackerras 2018-10-08 267 struct kvm_nested_guest *l2;
> 360cae313702cd Paul Mackerras 2018-10-08 268 struct pt_regs l2_regs, saved_l1_regs;
> afe75049303f75 Ravi Bangoria 2020-12-16 269 struct hv_guest_state l2_hv = {0}, saved_l1_hv;
> 360cae313702cd Paul Mackerras 2018-10-08 270 struct kvmppc_vcore *vc = vcpu->arch.vcore;
> 360cae313702cd Paul Mackerras 2018-10-08 271 u64 hv_ptr, regs_ptr;
> 360cae313702cd Paul Mackerras 2018-10-08 272 u64 hdec_exp;
> 360cae313702cd Paul Mackerras 2018-10-08 273 s64 delta_purr, delta_spurr, delta_ic, delta_vtb;
> 360cae313702cd Paul Mackerras 2018-10-08 274 u64 mask;
> 360cae313702cd Paul Mackerras 2018-10-08 275 unsigned long lpcr;
> 360cae313702cd Paul Mackerras 2018-10-08 276
> 360cae313702cd Paul Mackerras 2018-10-08 277 if (vcpu->kvm->arch.l1_ptcr == 0)
> 360cae313702cd Paul Mackerras 2018-10-08 278 return H_NOT_AVAILABLE;
> 360cae313702cd Paul Mackerras 2018-10-08 279
> 360cae313702cd Paul Mackerras 2018-10-08 280 /* copy parameters in */
> 360cae313702cd Paul Mackerras 2018-10-08 281 hv_ptr = kvmppc_get_gpr(vcpu, 4);
> 1508c22f112ce1 Alexey Kardashevskiy 2020-06-09 282 regs_ptr = kvmppc_get_gpr(vcpu, 5);
> 1508c22f112ce1 Alexey Kardashevskiy 2020-06-09 283 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
> afe75049303f75 Ravi Bangoria 2020-12-16 284 err = kvmhv_read_guest_state_and_regs(vcpu, &l2_hv, &l2_regs,
> afe75049303f75 Ravi Bangoria 2020-12-16 285 hv_ptr, regs_ptr);
> 1508c22f112ce1 Alexey Kardashevskiy 2020-06-09 286 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
> 360cae313702cd Paul Mackerras 2018-10-08 287 if (err)
> 360cae313702cd Paul Mackerras 2018-10-08 288 return H_PARAMETER;
> 1508c22f112ce1 Alexey Kardashevskiy 2020-06-09 289
> 10b5022db7861a Suraj Jitindar Singh 2018-10-08 290 if (kvmppc_need_byteswap(vcpu))
> 10b5022db7861a Suraj Jitindar Singh 2018-10-08 291 byteswap_hv_regs(&l2_hv);
> afe75049303f75 Ravi Bangoria 2020-12-16 292 if (l2_hv.version > HV_GUEST_STATE_VERSION)
> 360cae313702cd Paul Mackerras 2018-10-08 293 return H_P2;
> 360cae313702cd Paul Mackerras 2018-10-08 294
> 10b5022db7861a Suraj Jitindar Singh 2018-10-08 295 if (kvmppc_need_byteswap(vcpu))
> 10b5022db7861a Suraj Jitindar Singh 2018-10-08 296 byteswap_pt_regs(&l2_regs);
> 9d0b048da788c1 Suraj Jitindar Singh 2018-10-08 297 if (l2_hv.vcpu_token >= NR_CPUS)
> 9d0b048da788c1 Suraj Jitindar Singh 2018-10-08 298 return H_PARAMETER;
> 9d0b048da788c1 Suraj Jitindar Singh 2018-10-08 299
> 360cae313702cd Paul Mackerras 2018-10-08 300 /* translate lpid */
> 360cae313702cd Paul Mackerras 2018-10-08 301 l2 = kvmhv_get_nested(vcpu->kvm, l2_hv.lpid, true);
> 360cae313702cd Paul Mackerras 2018-10-08 302 if (!l2)
> 360cae313702cd Paul Mackerras 2018-10-08 303 return H_PARAMETER;
> 360cae313702cd Paul Mackerras 2018-10-08 304 if (!l2->l1_gr_to_hr) {
> 360cae313702cd Paul Mackerras 2018-10-08 305 mutex_lock(&l2->tlb_lock);
> 360cae313702cd Paul Mackerras 2018-10-08 306 kvmhv_update_ptbl_cache(l2);
> 360cae313702cd Paul Mackerras 2018-10-08 307 mutex_unlock(&l2->tlb_lock);
> 360cae313702cd Paul Mackerras 2018-10-08 308 }
> 360cae313702cd Paul Mackerras 2018-10-08 309
> 360cae313702cd Paul Mackerras 2018-10-08 310 /* save l1 values of things */
> 360cae313702cd Paul Mackerras 2018-10-08 311 vcpu->arch.regs.msr = vcpu->arch.shregs.msr;
> 360cae313702cd Paul Mackerras 2018-10-08 312 saved_l1_regs = vcpu->arch.regs;
> 360cae313702cd Paul Mackerras 2018-10-08 313 kvmhv_save_hv_regs(vcpu, &saved_l1_hv);
> 360cae313702cd Paul Mackerras 2018-10-08 314
> 360cae313702cd Paul Mackerras 2018-10-08 315 /* convert TB values/offsets to host (L0) values */
> 360cae313702cd Paul Mackerras 2018-10-08 316 hdec_exp = l2_hv.hdec_expiry - vc->tb_offset;
> 360cae313702cd Paul Mackerras 2018-10-08 317 vc->tb_offset += l2_hv.tb_offset;
> 360cae313702cd Paul Mackerras 2018-10-08 318
> 360cae313702cd Paul Mackerras 2018-10-08 319 /* set L1 state to L2 state */
> 360cae313702cd Paul Mackerras 2018-10-08 320 vcpu->arch.nested = l2;
> 360cae313702cd Paul Mackerras 2018-10-08 321 vcpu->arch.nested_vcpu_id = l2_hv.vcpu_token;
> 360cae313702cd Paul Mackerras 2018-10-08 322 vcpu->arch.regs = l2_regs;
> 360cae313702cd Paul Mackerras 2018-10-08 323 vcpu->arch.shregs.msr = vcpu->arch.regs.msr;
> 360cae313702cd Paul Mackerras 2018-10-08 324 mask = LPCR_DPFD | LPCR_ILE | LPCR_TC | LPCR_AIL | LPCR_LD |
> 360cae313702cd Paul Mackerras 2018-10-08 325 LPCR_LPES | LPCR_MER;
> 360cae313702cd Paul Mackerras 2018-10-08 326 lpcr = (vc->lpcr & ~mask) | (l2_hv.lpcr & mask);
> 73937deb4b2d7f Suraj Jitindar Singh 2018-10-08 327 sanitise_hv_regs(vcpu, &l2_hv);
> 360cae313702cd Paul Mackerras 2018-10-08 328 restore_hv_regs(vcpu, &l2_hv);
> 360cae313702cd Paul Mackerras 2018-10-08 329
> 360cae313702cd Paul Mackerras 2018-10-08 330 vcpu->arch.ret = RESUME_GUEST;
> 360cae313702cd Paul Mackerras 2018-10-08 331 vcpu->arch.trap = 0;
> 360cae313702cd Paul Mackerras 2018-10-08 332 do {
> 360cae313702cd Paul Mackerras 2018-10-08 333 if (mftb() >= hdec_exp) {
> 360cae313702cd Paul Mackerras 2018-10-08 334 vcpu->arch.trap = BOOK3S_INTERRUPT_HV_DECREMENTER;
> 360cae313702cd Paul Mackerras 2018-10-08 335 r = RESUME_HOST;
> 360cae313702cd Paul Mackerras 2018-10-08 336 break;
> 360cae313702cd Paul Mackerras 2018-10-08 337 }
> 8c99d34578628b Tianjia Zhang 2020-04-27 338 r = kvmhv_run_single_vcpu(vcpu, hdec_exp, lpcr);
> 360cae313702cd Paul Mackerras 2018-10-08 339 } while (is_kvmppc_resume_guest(r));
> 360cae313702cd Paul Mackerras 2018-10-08 340
> 360cae313702cd Paul Mackerras 2018-10-08 341 /* save L2 state for return */
> 360cae313702cd Paul Mackerras 2018-10-08 342 l2_regs = vcpu->arch.regs;
> 360cae313702cd Paul Mackerras 2018-10-08 343 l2_regs.msr = vcpu->arch.shregs.msr;
> 360cae313702cd Paul Mackerras 2018-10-08 344 delta_purr = vcpu->arch.purr - l2_hv.purr;
> 360cae313702cd Paul Mackerras 2018-10-08 345 delta_spurr = vcpu->arch.spurr - l2_hv.spurr;
> 360cae313702cd Paul Mackerras 2018-10-08 346 delta_ic = vcpu->arch.ic - l2_hv.ic;
> 360cae313702cd Paul Mackerras 2018-10-08 347 delta_vtb = vc->vtb - l2_hv.vtb;
> 360cae313702cd Paul Mackerras 2018-10-08 348 save_hv_return_state(vcpu, vcpu->arch.trap, &l2_hv);
> 360cae313702cd Paul Mackerras 2018-10-08 349
> 360cae313702cd Paul Mackerras 2018-10-08 350 /* restore L1 state */
> 360cae313702cd Paul Mackerras 2018-10-08 351 vcpu->arch.nested = NULL;
> 360cae313702cd Paul Mackerras 2018-10-08 352 vcpu->arch.regs = saved_l1_regs;
> 360cae313702cd Paul Mackerras 2018-10-08 353 vcpu->arch.shregs.msr = saved_l1_regs.msr & ~MSR_TS_MASK;
> 360cae313702cd Paul Mackerras 2018-10-08 354 /* set L1 MSR TS field according to L2 transaction state */
> 360cae313702cd Paul Mackerras 2018-10-08 355 if (l2_regs.msr & MSR_TS_MASK)
> 360cae313702cd Paul Mackerras 2018-10-08 356 vcpu->arch.shregs.msr |= MSR_TS_S;
> 360cae313702cd Paul Mackerras 2018-10-08 357 vc->tb_offset = saved_l1_hv.tb_offset;
> 360cae313702cd Paul Mackerras 2018-10-08 358 restore_hv_regs(vcpu, &saved_l1_hv);
> 360cae313702cd Paul Mackerras 2018-10-08 359 vcpu->arch.purr += delta_purr;
> 360cae313702cd Paul Mackerras 2018-10-08 360 vcpu->arch.spurr += delta_spurr;
> 360cae313702cd Paul Mackerras 2018-10-08 361 vcpu->arch.ic += delta_ic;
> 360cae313702cd Paul Mackerras 2018-10-08 362 vc->vtb += delta_vtb;
> 360cae313702cd Paul Mackerras 2018-10-08 363
> 360cae313702cd Paul Mackerras 2018-10-08 364 kvmhv_put_nested(l2);
> 360cae313702cd Paul Mackerras 2018-10-08 365
> 360cae313702cd Paul Mackerras 2018-10-08 366 /* copy l2_hv_state and regs back to guest */
> 10b5022db7861a Suraj Jitindar Singh 2018-10-08 367 if (kvmppc_need_byteswap(vcpu)) {
> 10b5022db7861a Suraj Jitindar Singh 2018-10-08 368 byteswap_hv_regs(&l2_hv);
> 10b5022db7861a Suraj Jitindar Singh 2018-10-08 369 byteswap_pt_regs(&l2_regs);
> 10b5022db7861a Suraj Jitindar Singh 2018-10-08 370 }
> 1508c22f112ce1 Alexey Kardashevskiy 2020-06-09 371 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
> afe75049303f75 Ravi Bangoria 2020-12-16 372 err = kvmhv_write_guest_state_and_regs(vcpu, &l2_hv, &l2_regs,
> afe75049303f75 Ravi Bangoria 2020-12-16 373 hv_ptr, regs_ptr);
> 1508c22f112ce1 Alexey Kardashevskiy 2020-06-09 374 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
> 360cae313702cd Paul Mackerras 2018-10-08 375 if (err)
> 360cae313702cd Paul Mackerras 2018-10-08 376 return H_AUTHORITY;
> 360cae313702cd Paul Mackerras 2018-10-08 377
> 360cae313702cd Paul Mackerras 2018-10-08 378 if (r == -EINTR)
> 360cae313702cd Paul Mackerras 2018-10-08 379 return H_INTERRUPT;
> 360cae313702cd Paul Mackerras 2018-10-08 380
> 873db2cd9a6d7f Suraj Jitindar Singh 2018-12-14 381 if (vcpu->mmio_needed) {
> 873db2cd9a6d7f Suraj Jitindar Singh 2018-12-14 382 kvmhv_nested_mmio_needed(vcpu, regs_ptr);
> 873db2cd9a6d7f Suraj Jitindar Singh 2018-12-14 383 return H_TOO_HARD;
> 873db2cd9a6d7f Suraj Jitindar Singh 2018-12-14 384 }
> 873db2cd9a6d7f Suraj Jitindar Singh 2018-12-14 385
> 360cae313702cd Paul Mackerras 2018-10-08 386 return vcpu->arch.trap;
> 360cae313702cd Paul Mackerras 2018-10-08 387 }
> 360cae313702cd Paul Mackerras 2018-10-08 388
>
> :::::: The code at line 264 was first introduced by commit
> :::::: 360cae313702cdd0b90f82c261a8302fecef030a KVM: PPC: Book3S HV: Nested guest entry via hypercall
>
> :::::: TO: Paul Mackerras <[email protected]>
> :::::: CC: Michael Ellerman <[email protected]>
>
> ---
> 0-DAY CI Kernel Test Service, Intel Corporation
> https://lists.01.org/hyperkitty/list/[email protected]
>
On 6/20/2021 4:59 PM, Nicholas Piggin wrote:
> Excerpts from kernel test robot's message of April 3, 2021 8:47 pm:
>> tree: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git master
>> head: d93a0d43e3d0ba9e19387be4dae4a8d5b175a8d7
>> commit: 97e4910232fa1f81e806aa60c25a0450276d99a2 linux/compiler-clang.h: define HAVE_BUILTIN_BSWAP*
>> date: 3 weeks ago
>> config: powerpc64-randconfig-r006-20210403 (attached as .config)
>> compiler: clang version 13.0.0 (https://github.com/llvm/llvm-project 0fe8af94688aa03c01913c2001d6a1a911f42ce6)
>> reproduce (this is a W=1 build):
>> wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
>> chmod +x ~/bin/make.cross
>> # install powerpc64 cross compiling tool for clang build
>> # apt-get install binutils-powerpc64-linux-gnu
>> # https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=97e4910232fa1f81e806aa60c25a0450276d99a2
>> git remote add linus https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
>> git fetch --no-tags linus master
>> git checkout 97e4910232fa1f81e806aa60c25a0450276d99a2
>> # save the attached .config to linux build tree
>> COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross ARCH=powerpc64
>>
>> If you fix the issue, kindly add following tag as appropriate
>> Reported-by: kernel test robot <[email protected]>
>>
>> All errors (new ones prefixed by >>):
>>
>>>> arch/powerpc/kvm/book3s_hv_nested.c:264:6: error: stack frame size of 2304 bytes in function 'kvmhv_enter_nested_guest' [-Werror,-Wframe-larger-than=]
>> long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu)
>> ^
>> 1 error generated.
>>
>>
>> vim +/kvmhv_enter_nested_guest +264 arch/powerpc/kvm/book3s_hv_nested.c
>
> Not much changed here recently. It's not that big a concern because it's
> only called in the KVM ioctl path, not in any deep IO paths or anything,
> and doesn't recurse. Might be a bit of inlining or stack spilling put it
> over the edge.
It appears to be the fact that LLVM's PowerPC backend does not emit
efficient byteswap assembly:
https://github.com/ClangBuiltLinux/linux/issues/1292
https://bugs.llvm.org/show_bug.cgi?id=49610
> powerpc does make it an error though, would be good to avoid that so the
> robot doesn't keep tripping over.
Marking byteswap_pt_regs as 'noinline_for_stack' drastically reduces the
stack usage. If that is an acceptable solution, I can send it along
tomorrow.
Cheers,
Nathan
> Thanks,
> Nick
>
>
>>
>> afe75049303f75 Ravi Bangoria 2020-12-16 263
>> 360cae313702cd Paul Mackerras 2018-10-08 @264 long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu)
>> 360cae313702cd Paul Mackerras 2018-10-08 265 {
>> 360cae313702cd Paul Mackerras 2018-10-08 266 long int err, r;
>> 360cae313702cd Paul Mackerras 2018-10-08 267 struct kvm_nested_guest *l2;
>> 360cae313702cd Paul Mackerras 2018-10-08 268 struct pt_regs l2_regs, saved_l1_regs;
>> afe75049303f75 Ravi Bangoria 2020-12-16 269 struct hv_guest_state l2_hv = {0}, saved_l1_hv;
>> 360cae313702cd Paul Mackerras 2018-10-08 270 struct kvmppc_vcore *vc = vcpu->arch.vcore;
>> 360cae313702cd Paul Mackerras 2018-10-08 271 u64 hv_ptr, regs_ptr;
>> 360cae313702cd Paul Mackerras 2018-10-08 272 u64 hdec_exp;
>> 360cae313702cd Paul Mackerras 2018-10-08 273 s64 delta_purr, delta_spurr, delta_ic, delta_vtb;
>> 360cae313702cd Paul Mackerras 2018-10-08 274 u64 mask;
>> 360cae313702cd Paul Mackerras 2018-10-08 275 unsigned long lpcr;
>> 360cae313702cd Paul Mackerras 2018-10-08 276
>> 360cae313702cd Paul Mackerras 2018-10-08 277 if (vcpu->kvm->arch.l1_ptcr == 0)
>> 360cae313702cd Paul Mackerras 2018-10-08 278 return H_NOT_AVAILABLE;
>> 360cae313702cd Paul Mackerras 2018-10-08 279
>> 360cae313702cd Paul Mackerras 2018-10-08 280 /* copy parameters in */
>> 360cae313702cd Paul Mackerras 2018-10-08 281 hv_ptr = kvmppc_get_gpr(vcpu, 4);
>> 1508c22f112ce1 Alexey Kardashevskiy 2020-06-09 282 regs_ptr = kvmppc_get_gpr(vcpu, 5);
>> 1508c22f112ce1 Alexey Kardashevskiy 2020-06-09 283 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
>> afe75049303f75 Ravi Bangoria 2020-12-16 284 err = kvmhv_read_guest_state_and_regs(vcpu, &l2_hv, &l2_regs,
>> afe75049303f75 Ravi Bangoria 2020-12-16 285 hv_ptr, regs_ptr);
>> 1508c22f112ce1 Alexey Kardashevskiy 2020-06-09 286 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
>> 360cae313702cd Paul Mackerras 2018-10-08 287 if (err)
>> 360cae313702cd Paul Mackerras 2018-10-08 288 return H_PARAMETER;
>> 1508c22f112ce1 Alexey Kardashevskiy 2020-06-09 289
>> 10b5022db7861a Suraj Jitindar Singh 2018-10-08 290 if (kvmppc_need_byteswap(vcpu))
>> 10b5022db7861a Suraj Jitindar Singh 2018-10-08 291 byteswap_hv_regs(&l2_hv);
>> afe75049303f75 Ravi Bangoria 2020-12-16 292 if (l2_hv.version > HV_GUEST_STATE_VERSION)
>> 360cae313702cd Paul Mackerras 2018-10-08 293 return H_P2;
>> 360cae313702cd Paul Mackerras 2018-10-08 294
>> 10b5022db7861a Suraj Jitindar Singh 2018-10-08 295 if (kvmppc_need_byteswap(vcpu))
>> 10b5022db7861a Suraj Jitindar Singh 2018-10-08 296 byteswap_pt_regs(&l2_regs);
>> 9d0b048da788c1 Suraj Jitindar Singh 2018-10-08 297 if (l2_hv.vcpu_token >= NR_CPUS)
>> 9d0b048da788c1 Suraj Jitindar Singh 2018-10-08 298 return H_PARAMETER;
>> 9d0b048da788c1 Suraj Jitindar Singh 2018-10-08 299
>> 360cae313702cd Paul Mackerras 2018-10-08 300 /* translate lpid */
>> 360cae313702cd Paul Mackerras 2018-10-08 301 l2 = kvmhv_get_nested(vcpu->kvm, l2_hv.lpid, true);
>> 360cae313702cd Paul Mackerras 2018-10-08 302 if (!l2)
>> 360cae313702cd Paul Mackerras 2018-10-08 303 return H_PARAMETER;
>> 360cae313702cd Paul Mackerras 2018-10-08 304 if (!l2->l1_gr_to_hr) {
>> 360cae313702cd Paul Mackerras 2018-10-08 305 mutex_lock(&l2->tlb_lock);
>> 360cae313702cd Paul Mackerras 2018-10-08 306 kvmhv_update_ptbl_cache(l2);
>> 360cae313702cd Paul Mackerras 2018-10-08 307 mutex_unlock(&l2->tlb_lock);
>> 360cae313702cd Paul Mackerras 2018-10-08 308 }
>> 360cae313702cd Paul Mackerras 2018-10-08 309
>> 360cae313702cd Paul Mackerras 2018-10-08 310 /* save l1 values of things */
>> 360cae313702cd Paul Mackerras 2018-10-08 311 vcpu->arch.regs.msr = vcpu->arch.shregs.msr;
>> 360cae313702cd Paul Mackerras 2018-10-08 312 saved_l1_regs = vcpu->arch.regs;
>> 360cae313702cd Paul Mackerras 2018-10-08 313 kvmhv_save_hv_regs(vcpu, &saved_l1_hv);
>> 360cae313702cd Paul Mackerras 2018-10-08 314
>> 360cae313702cd Paul Mackerras 2018-10-08 315 /* convert TB values/offsets to host (L0) values */
>> 360cae313702cd Paul Mackerras 2018-10-08 316 hdec_exp = l2_hv.hdec_expiry - vc->tb_offset;
>> 360cae313702cd Paul Mackerras 2018-10-08 317 vc->tb_offset += l2_hv.tb_offset;
>> 360cae313702cd Paul Mackerras 2018-10-08 318
>> 360cae313702cd Paul Mackerras 2018-10-08 319 /* set L1 state to L2 state */
>> 360cae313702cd Paul Mackerras 2018-10-08 320 vcpu->arch.nested = l2;
>> 360cae313702cd Paul Mackerras 2018-10-08 321 vcpu->arch.nested_vcpu_id = l2_hv.vcpu_token;
>> 360cae313702cd Paul Mackerras 2018-10-08 322 vcpu->arch.regs = l2_regs;
>> 360cae313702cd Paul Mackerras 2018-10-08 323 vcpu->arch.shregs.msr = vcpu->arch.regs.msr;
>> 360cae313702cd Paul Mackerras 2018-10-08 324 mask = LPCR_DPFD | LPCR_ILE | LPCR_TC | LPCR_AIL | LPCR_LD |
>> 360cae313702cd Paul Mackerras 2018-10-08 325 LPCR_LPES | LPCR_MER;
>> 360cae313702cd Paul Mackerras 2018-10-08 326 lpcr = (vc->lpcr & ~mask) | (l2_hv.lpcr & mask);
>> 73937deb4b2d7f Suraj Jitindar Singh 2018-10-08 327 sanitise_hv_regs(vcpu, &l2_hv);
>> 360cae313702cd Paul Mackerras 2018-10-08 328 restore_hv_regs(vcpu, &l2_hv);
>> 360cae313702cd Paul Mackerras 2018-10-08 329
>> 360cae313702cd Paul Mackerras 2018-10-08 330 vcpu->arch.ret = RESUME_GUEST;
>> 360cae313702cd Paul Mackerras 2018-10-08 331 vcpu->arch.trap = 0;
>> 360cae313702cd Paul Mackerras 2018-10-08 332 do {
>> 360cae313702cd Paul Mackerras 2018-10-08 333 if (mftb() >= hdec_exp) {
>> 360cae313702cd Paul Mackerras 2018-10-08 334 vcpu->arch.trap = BOOK3S_INTERRUPT_HV_DECREMENTER;
>> 360cae313702cd Paul Mackerras 2018-10-08 335 r = RESUME_HOST;
>> 360cae313702cd Paul Mackerras 2018-10-08 336 break;
>> 360cae313702cd Paul Mackerras 2018-10-08 337 }
>> 8c99d34578628b Tianjia Zhang 2020-04-27 338 r = kvmhv_run_single_vcpu(vcpu, hdec_exp, lpcr);
>> 360cae313702cd Paul Mackerras 2018-10-08 339 } while (is_kvmppc_resume_guest(r));
>> 360cae313702cd Paul Mackerras 2018-10-08 340
>> 360cae313702cd Paul Mackerras 2018-10-08 341 /* save L2 state for return */
>> 360cae313702cd Paul Mackerras 2018-10-08 342 l2_regs = vcpu->arch.regs;
>> 360cae313702cd Paul Mackerras 2018-10-08 343 l2_regs.msr = vcpu->arch.shregs.msr;
>> 360cae313702cd Paul Mackerras 2018-10-08 344 delta_purr = vcpu->arch.purr - l2_hv.purr;
>> 360cae313702cd Paul Mackerras 2018-10-08 345 delta_spurr = vcpu->arch.spurr - l2_hv.spurr;
>> 360cae313702cd Paul Mackerras 2018-10-08 346 delta_ic = vcpu->arch.ic - l2_hv.ic;
>> 360cae313702cd Paul Mackerras 2018-10-08 347 delta_vtb = vc->vtb - l2_hv.vtb;
>> 360cae313702cd Paul Mackerras 2018-10-08 348 save_hv_return_state(vcpu, vcpu->arch.trap, &l2_hv);
>> 360cae313702cd Paul Mackerras 2018-10-08 349
>> 360cae313702cd Paul Mackerras 2018-10-08 350 /* restore L1 state */
>> 360cae313702cd Paul Mackerras 2018-10-08 351 vcpu->arch.nested = NULL;
>> 360cae313702cd Paul Mackerras 2018-10-08 352 vcpu->arch.regs = saved_l1_regs;
>> 360cae313702cd Paul Mackerras 2018-10-08 353 vcpu->arch.shregs.msr = saved_l1_regs.msr & ~MSR_TS_MASK;
>> 360cae313702cd Paul Mackerras 2018-10-08 354 /* set L1 MSR TS field according to L2 transaction state */
>> 360cae313702cd Paul Mackerras 2018-10-08 355 if (l2_regs.msr & MSR_TS_MASK)
>> 360cae313702cd Paul Mackerras 2018-10-08 356 vcpu->arch.shregs.msr |= MSR_TS_S;
>> 360cae313702cd Paul Mackerras 2018-10-08 357 vc->tb_offset = saved_l1_hv.tb_offset;
>> 360cae313702cd Paul Mackerras 2018-10-08 358 restore_hv_regs(vcpu, &saved_l1_hv);
>> 360cae313702cd Paul Mackerras 2018-10-08 359 vcpu->arch.purr += delta_purr;
>> 360cae313702cd Paul Mackerras 2018-10-08 360 vcpu->arch.spurr += delta_spurr;
>> 360cae313702cd Paul Mackerras 2018-10-08 361 vcpu->arch.ic += delta_ic;
>> 360cae313702cd Paul Mackerras 2018-10-08 362 vc->vtb += delta_vtb;
>> 360cae313702cd Paul Mackerras 2018-10-08 363
>> 360cae313702cd Paul Mackerras 2018-10-08 364 kvmhv_put_nested(l2);
>> 360cae313702cd Paul Mackerras 2018-10-08 365
>> 360cae313702cd Paul Mackerras 2018-10-08 366 /* copy l2_hv_state and regs back to guest */
>> 10b5022db7861a Suraj Jitindar Singh 2018-10-08 367 if (kvmppc_need_byteswap(vcpu)) {
>> 10b5022db7861a Suraj Jitindar Singh 2018-10-08 368 byteswap_hv_regs(&l2_hv);
>> 10b5022db7861a Suraj Jitindar Singh 2018-10-08 369 byteswap_pt_regs(&l2_regs);
>> 10b5022db7861a Suraj Jitindar Singh 2018-10-08 370 }
>> 1508c22f112ce1 Alexey Kardashevskiy 2020-06-09 371 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
>> afe75049303f75 Ravi Bangoria 2020-12-16 372 err = kvmhv_write_guest_state_and_regs(vcpu, &l2_hv, &l2_regs,
>> afe75049303f75 Ravi Bangoria 2020-12-16 373 hv_ptr, regs_ptr);
>> 1508c22f112ce1 Alexey Kardashevskiy 2020-06-09 374 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx);
>> 360cae313702cd Paul Mackerras 2018-10-08 375 if (err)
>> 360cae313702cd Paul Mackerras 2018-10-08 376 return H_AUTHORITY;
>> 360cae313702cd Paul Mackerras 2018-10-08 377
>> 360cae313702cd Paul Mackerras 2018-10-08 378 if (r == -EINTR)
>> 360cae313702cd Paul Mackerras 2018-10-08 379 return H_INTERRUPT;
>> 360cae313702cd Paul Mackerras 2018-10-08 380
>> 873db2cd9a6d7f Suraj Jitindar Singh 2018-12-14 381 if (vcpu->mmio_needed) {
>> 873db2cd9a6d7f Suraj Jitindar Singh 2018-12-14 382 kvmhv_nested_mmio_needed(vcpu, regs_ptr);
>> 873db2cd9a6d7f Suraj Jitindar Singh 2018-12-14 383 return H_TOO_HARD;
>> 873db2cd9a6d7f Suraj Jitindar Singh 2018-12-14 384 }
>> 873db2cd9a6d7f Suraj Jitindar Singh 2018-12-14 385
>> 360cae313702cd Paul Mackerras 2018-10-08 386 return vcpu->arch.trap;
>> 360cae313702cd Paul Mackerras 2018-10-08 387 }
>> 360cae313702cd Paul Mackerras 2018-10-08 388
>>
>> :::::: The code at line 264 was first introduced by commit
>> :::::: 360cae313702cdd0b90f82c261a8302fecef030a KVM: PPC: Book3S HV: Nested guest entry via hypercall
>>
>> :::::: TO: Paul Mackerras <[email protected]>
>> :::::: CC: Michael Ellerman <[email protected]>
>>
>> ---
>> 0-DAY CI Kernel Test Service, Intel Corporation
>> https://lists.01.org/hyperkitty/list/[email protected]
>>
Nathan Chancellor <[email protected]> writes:
> On 6/20/2021 4:59 PM, Nicholas Piggin wrote:
>> Excerpts from kernel test robot's message of April 3, 2021 8:47 pm:
>>> tree: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git master
>>> head: d93a0d43e3d0ba9e19387be4dae4a8d5b175a8d7
>>> commit: 97e4910232fa1f81e806aa60c25a0450276d99a2 linux/compiler-clang.h: define HAVE_BUILTIN_BSWAP*
>>> date: 3 weeks ago
>>> config: powerpc64-randconfig-r006-20210403 (attached as .config)
>>> compiler: clang version 13.0.0 (https://github.com/llvm/llvm-project 0fe8af94688aa03c01913c2001d6a1a911f42ce6)
>>> reproduce (this is a W=1 build):
>>> wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
>>> chmod +x ~/bin/make.cross
>>> # install powerpc64 cross compiling tool for clang build
>>> # apt-get install binutils-powerpc64-linux-gnu
>>> # https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=97e4910232fa1f81e806aa60c25a0450276d99a2
>>> git remote add linus https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
>>> git fetch --no-tags linus master
>>> git checkout 97e4910232fa1f81e806aa60c25a0450276d99a2
>>> # save the attached .config to linux build tree
>>> COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross ARCH=powerpc64
>>>
>>> If you fix the issue, kindly add following tag as appropriate
>>> Reported-by: kernel test robot <[email protected]>
>>>
>>> All errors (new ones prefixed by >>):
>>>
>>>>> arch/powerpc/kvm/book3s_hv_nested.c:264:6: error: stack frame size of 2304 bytes in function 'kvmhv_enter_nested_guest' [-Werror,-Wframe-larger-than=]
>>> long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu)
>>> ^
>>> 1 error generated.
>>>
>>>
>>> vim +/kvmhv_enter_nested_guest +264 arch/powerpc/kvm/book3s_hv_nested.c
>>
>> Not much changed here recently. It's not that big a concern because it's
>> only called in the KVM ioctl path, not in any deep IO paths or anything,
>> and doesn't recurse. Might be a bit of inlining or stack spilling put it
>> over the edge.
>
> It appears to be the fact that LLVM's PowerPC backend does not emit
> efficient byteswap assembly:
>
> https://github.com/ClangBuiltLinux/linux/issues/1292
>
> https://bugs.llvm.org/show_bug.cgi?id=49610
>
>> powerpc does make it an error though, would be good to avoid that so the
>> robot doesn't keep tripping over.
>
> Marking byteswap_pt_regs as 'noinline_for_stack' drastically reduces the
> stack usage. If that is an acceptable solution, I can send it along
> tomorrow.
Yeah that should be OK. Can you post the before/after disassembly when
you post the patch?
It should just be two extra function calls, which shouldn't be enough
overhead to be measurable.
cheers
On Mon, Jun 21, 2021 at 11:46 AM Michael Ellerman <[email protected]> wrote:
> Nathan Chancellor <[email protected]> writes:
> > On 6/20/2021 4:59 PM, Nicholas Piggin wrote:
> >> Excerpts from kernel test robot's message of April 3, 2021 8:47 pm:
> >>>
> >>> vim +/kvmhv_enter_nested_guest +264 arch/powerpc/kvm/book3s_hv_nested.c
> >>
> >> Not much changed here recently. It's not that big a concern because it's
> >> only called in the KVM ioctl path, not in any deep IO paths or anything,
> >> and doesn't recurse. Might be a bit of inlining or stack spilling put it
> >> over the edge.
> >
> > It appears to be the fact that LLVM's PowerPC backend does not emit
> > efficient byteswap assembly:
> >
> > https://github.com/ClangBuiltLinux/linux/issues/1292
> >
> > https://bugs.llvm.org/show_bug.cgi?id=49610
> >
> >> powerpc does make it an error though, would be good to avoid that so the
> >> robot doesn't keep tripping over.
> >
> > Marking byteswap_pt_regs as 'noinline_for_stack' drastically reduces the
> > stack usage. If that is an acceptable solution, I can send it along
> > tomorrow.
>
> Yeah that should be OK.
That's fine with me as well.
> Can you post the before/after disassembly when
> you post the patch?
>
> It should just be two extra function calls, which shouldn't be enough
> overhead to be measurable.
The thing I remember is that the 'before' code here is some seriously bad
output from llvm, and it would be helpful to have someone get the compiler
to emit the correct powerpc byteswap instructions and avoid the excessive
stack spilling.
The warning here is just a symptom of a missed optimization and the
same thing probably happens elsewhere on powerpc, even if it doesn't
exceed the stack warning limit.
Arnd
On Mon, Jun 21, 2021 at 07:46:03PM +1000, Michael Ellerman wrote:
> Nathan Chancellor <[email protected]> writes:
> > On 6/20/2021 4:59 PM, Nicholas Piggin wrote:
> >> Excerpts from kernel test robot's message of April 3, 2021 8:47 pm:
> >>> tree: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git master
> >>> head: d93a0d43e3d0ba9e19387be4dae4a8d5b175a8d7
> >>> commit: 97e4910232fa1f81e806aa60c25a0450276d99a2 linux/compiler-clang.h: define HAVE_BUILTIN_BSWAP*
> >>> date: 3 weeks ago
> >>> config: powerpc64-randconfig-r006-20210403 (attached as .config)
> >>> compiler: clang version 13.0.0 (https://github.com/llvm/llvm-project 0fe8af94688aa03c01913c2001d6a1a911f42ce6)
> >>> reproduce (this is a W=1 build):
> >>> wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
> >>> chmod +x ~/bin/make.cross
> >>> # install powerpc64 cross compiling tool for clang build
> >>> # apt-get install binutils-powerpc64-linux-gnu
> >>> # https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=97e4910232fa1f81e806aa60c25a0450276d99a2
> >>> git remote add linus https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
> >>> git fetch --no-tags linus master
> >>> git checkout 97e4910232fa1f81e806aa60c25a0450276d99a2
> >>> # save the attached .config to linux build tree
> >>> COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross ARCH=powerpc64
> >>>
> >>> If you fix the issue, kindly add following tag as appropriate
> >>> Reported-by: kernel test robot <[email protected]>
> >>>
> >>> All errors (new ones prefixed by >>):
> >>>
> >>>>> arch/powerpc/kvm/book3s_hv_nested.c:264:6: error: stack frame size of 2304 bytes in function 'kvmhv_enter_nested_guest' [-Werror,-Wframe-larger-than=]
> >>> long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu)
> >>> ^
> >>> 1 error generated.
> >>>
> >>>
> >>> vim +/kvmhv_enter_nested_guest +264 arch/powerpc/kvm/book3s_hv_nested.c
> >>
> >> Not much changed here recently. It's not that big a concern because it's
> >> only called in the KVM ioctl path, not in any deep IO paths or anything,
> >> and doesn't recurse. Might be a bit of inlining or stack spilling put it
> >> over the edge.
> >
> > It appears to be the fact that LLVM's PowerPC backend does not emit
> > efficient byteswap assembly:
> >
> > https://github.com/ClangBuiltLinux/linux/issues/1292
> >
> > https://bugs.llvm.org/show_bug.cgi?id=49610
> >
> >> powerpc does make it an error though, would be good to avoid that so the
> >> robot doesn't keep tripping over.
> >
> > Marking byteswap_pt_regs as 'noinline_for_stack' drastically reduces the
> > stack usage. If that is an acceptable solution, I can send it along
> > tomorrow.
>
> Yeah that should be OK. Can you post the before/after disassembly when
> you post the patch?
>
> It should just be two extra function calls, which shouldn't be enough
> overhead to be measurable.
The diff is pretty large so I have attached it here along with the full
disassembly of the files before and after the patch I am about to send.
I will reply to this message so the history is there.
Cheers,
Nathan
On Mon, Jun 21, 2021 at 11:01:54AM -0700, Nathan Chancellor wrote:
> On Mon, Jun 21, 2021 at 07:46:03PM +1000, Michael Ellerman wrote:
> > Nathan Chancellor <[email protected]> writes:
> > > On 6/20/2021 4:59 PM, Nicholas Piggin wrote:
> > >> Excerpts from kernel test robot's message of April 3, 2021 8:47 pm:
> > >>> tree: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git master
> > >>> head: d93a0d43e3d0ba9e19387be4dae4a8d5b175a8d7
> > >>> commit: 97e4910232fa1f81e806aa60c25a0450276d99a2 linux/compiler-clang.h: define HAVE_BUILTIN_BSWAP*
> > >>> date: 3 weeks ago
> > >>> config: powerpc64-randconfig-r006-20210403 (attached as .config)
> > >>> compiler: clang version 13.0.0 (https://github.com/llvm/llvm-project 0fe8af94688aa03c01913c2001d6a1a911f42ce6)
> > >>> reproduce (this is a W=1 build):
> > >>> wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
> > >>> chmod +x ~/bin/make.cross
> > >>> # install powerpc64 cross compiling tool for clang build
> > >>> # apt-get install binutils-powerpc64-linux-gnu
> > >>> # https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=97e4910232fa1f81e806aa60c25a0450276d99a2
> > >>> git remote add linus https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
> > >>> git fetch --no-tags linus master
> > >>> git checkout 97e4910232fa1f81e806aa60c25a0450276d99a2
> > >>> # save the attached .config to linux build tree
> > >>> COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross ARCH=powerpc64
> > >>>
> > >>> If you fix the issue, kindly add following tag as appropriate
> > >>> Reported-by: kernel test robot <[email protected]>
> > >>>
> > >>> All errors (new ones prefixed by >>):
> > >>>
> > >>>>> arch/powerpc/kvm/book3s_hv_nested.c:264:6: error: stack frame size of 2304 bytes in function 'kvmhv_enter_nested_guest' [-Werror,-Wframe-larger-than=]
> > >>> long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu)
> > >>> ^
> > >>> 1 error generated.
> > >>>
> > >>>
> > >>> vim +/kvmhv_enter_nested_guest +264 arch/powerpc/kvm/book3s_hv_nested.c
> > >>
> > >> Not much changed here recently. It's not that big a concern because it's
> > >> only called in the KVM ioctl path, not in any deep IO paths or anything,
> > >> and doesn't recurse. Might be a bit of inlining or stack spilling put it
> > >> over the edge.
> > >
> > > It appears to be the fact that LLVM's PowerPC backend does not emit
> > > efficient byteswap assembly:
> > >
> > > https://github.com/ClangBuiltLinux/linux/issues/1292
> > >
> > > https://bugs.llvm.org/show_bug.cgi?id=49610
> > >
> > >> powerpc does make it an error though, would be good to avoid that so the
> > >> robot doesn't keep tripping over.
> > >
> > > Marking byteswap_pt_regs as 'noinline_for_stack' drastically reduces the
> > > stack usage. If that is an acceptable solution, I can send it along
> > > tomorrow.
> >
> > Yeah that should be OK. Can you post the before/after disassembly when
> > you post the patch?
> >
> > It should just be two extra function calls, which shouldn't be enough
> > overhead to be measurable.
>
> The diff is pretty large so I have attached it here along with the full
> disassembly of the files before and after the patch I am about to send.
> I will reply to this message so the history is there.
It would help if I actually attached them :^)
Cheers,
Nathan
LLVM does not emit optimal byteswap assembly, which results in high
stack usage in kvmhv_enter_nested_guest() due to the inlining of
byteswap_pt_regs(). With LLVM 12.0.0:
arch/powerpc/kvm/book3s_hv_nested.c:289:6: error: stack frame size of
2512 bytes in function 'kvmhv_enter_nested_guest' [-Werror,-Wframe-larger-than=]
long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu)
^
1 error generated.
While this gets fixed in LLVM, mark byteswap_pt_regs() as
noinline_for_stack so that it does not get inlined and break the build
due to -Werror by default in arch/powerpc/. Not inlining saves
approximately 800 bytes with LLVM 12.0.0:
arch/powerpc/kvm/book3s_hv_nested.c:290:6: warning: stack frame size of
1728 bytes in function 'kvmhv_enter_nested_guest' [-Wframe-larger-than=]
long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu)
^
1 warning generated.
Link: https://github.com/ClangBuiltLinux/linux/issues/1292
Link: https://bugs.llvm.org/show_bug.cgi?id=49610
Link: https://lore.kernel.org/r/[email protected]/
Link: https://gist.github.com/ba710e3703bf45043a31e2806c843ffd
Reported-by: kernel test robot <[email protected]>
Signed-off-by: Nathan Chancellor <[email protected]>
---
arch/powerpc/kvm/book3s_hv_nested.c | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/arch/powerpc/kvm/book3s_hv_nested.c b/arch/powerpc/kvm/book3s_hv_nested.c
index 60724f674421..1b3ff0af1264 100644
--- a/arch/powerpc/kvm/book3s_hv_nested.c
+++ b/arch/powerpc/kvm/book3s_hv_nested.c
@@ -53,7 +53,8 @@ void kvmhv_save_hv_regs(struct kvm_vcpu *vcpu, struct hv_guest_state *hr)
hr->dawrx1 = vcpu->arch.dawrx1;
}
-static void byteswap_pt_regs(struct pt_regs *regs)
+/* Use noinline_for_stack due to https://bugs.llvm.org/show_bug.cgi?id=49610 */
+static noinline_for_stack void byteswap_pt_regs(struct pt_regs *regs)
{
unsigned long *addr = (unsigned long *) regs;
base-commit: 4a21192e2796c3338c4b0083b494a84a61311aaf
--
2.32.0.93.g670b81a890
Excerpts from Nathan Chancellor's message of June 22, 2021 4:24 am:
> LLVM does not emit optimal byteswap assembly, which results in high
> stack usage in kvmhv_enter_nested_guest() due to the inlining of
> byteswap_pt_regs(). With LLVM 12.0.0:
>
> arch/powerpc/kvm/book3s_hv_nested.c:289:6: error: stack frame size of
> 2512 bytes in function 'kvmhv_enter_nested_guest' [-Werror,-Wframe-larger-than=]
> long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu)
> ^
> 1 error generated.
>
> While this gets fixed in LLVM, mark byteswap_pt_regs() as
> noinline_for_stack so that it does not get inlined and break the build
> due to -Werror by default in arch/powerpc/. Not inlining saves
> approximately 800 bytes with LLVM 12.0.0:
>
> arch/powerpc/kvm/book3s_hv_nested.c:290:6: warning: stack frame size of
> 1728 bytes in function 'kvmhv_enter_nested_guest' [-Wframe-larger-than=]
> long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu)
> ^
> 1 warning generated.
>
> Link: https://github.com/ClangBuiltLinux/linux/issues/1292
> Link: https://bugs.llvm.org/show_bug.cgi?id=49610
> Link: https://lore.kernel.org/r/[email protected]/
> Link: https://gist.github.com/ba710e3703bf45043a31e2806c843ffd
> Reported-by: kernel test robot <[email protected]>
> Signed-off-by: Nathan Chancellor <[email protected]>
Seems okay to me. If it was something where performance might be
signficiant I guess you could ifdef on CC_IS_CLANG, but for this
it shouldn't matter.
Acked-by: Nicholas Piggin <[email protected]>
Thanks,
Nick
> ---
> arch/powerpc/kvm/book3s_hv_nested.c | 3 ++-
> 1 file changed, 2 insertions(+), 1 deletion(-)
>
> diff --git a/arch/powerpc/kvm/book3s_hv_nested.c b/arch/powerpc/kvm/book3s_hv_nested.c
> index 60724f674421..1b3ff0af1264 100644
> --- a/arch/powerpc/kvm/book3s_hv_nested.c
> +++ b/arch/powerpc/kvm/book3s_hv_nested.c
> @@ -53,7 +53,8 @@ void kvmhv_save_hv_regs(struct kvm_vcpu *vcpu, struct hv_guest_state *hr)
> hr->dawrx1 = vcpu->arch.dawrx1;
> }
>
> -static void byteswap_pt_regs(struct pt_regs *regs)
> +/* Use noinline_for_stack due to https://bugs.llvm.org/show_bug.cgi?id=49610 */
> +static noinline_for_stack void byteswap_pt_regs(struct pt_regs *regs)
> {
> unsigned long *addr = (unsigned long *) regs;
>
>
> base-commit: 4a21192e2796c3338c4b0083b494a84a61311aaf
> --
> 2.32.0.93.g670b81a890
>
>
On Mon, 21 Jun 2021 11:24:40 -0700, Nathan Chancellor wrote:
> LLVM does not emit optimal byteswap assembly, which results in high
> stack usage in kvmhv_enter_nested_guest() due to the inlining of
> byteswap_pt_regs(). With LLVM 12.0.0:
>
> arch/powerpc/kvm/book3s_hv_nested.c:289:6: error: stack frame size of
> 2512 bytes in function 'kvmhv_enter_nested_guest' [-Werror,-Wframe-larger-than=]
> long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu)
> ^
> 1 error generated.
>
> [...]
Applied to powerpc/topic/ppc-kvm.
[1/1] KVM: PPC: Book3S HV: Workaround high stack usage with clang
https://git.kernel.org/powerpc/c/51696f39cbee5bb684e7959c0c98b5f54548aa34
cheers