Return-Path: Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1754041AbdCPVPT (ORCPT ); Thu, 16 Mar 2017 17:15:19 -0400 Received: from mail-qt0-f194.google.com ([209.85.216.194]:36629 "EHLO mail-qt0-f194.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1752397AbdCPVPF (ORCPT ); Thu, 16 Mar 2017 17:15:05 -0400 Date: Thu, 16 Mar 2017 17:14:15 -0400 From: "Gabriel L. Somlo" To: "Michael S. Tsirkin" Cc: Radim =?utf-8?B?S3LEjW3DocWZ?= , linux-kernel@vger.kernel.org, Paolo Bonzini , Jonathan Corbet , Thomas Gleixner , Ingo Molnar , "H. Peter Anvin" , x86@kernel.org, Joerg Roedel , kvm@vger.kernel.org, linux-doc@vger.kernel.org Subject: Re: [PATCH v5 untested] kvm: better MWAIT emulation for guests Message-ID: <20170316211414.GO4085@HEDWIG.INI.CMU.EDU> References: <20170316145819.GC4085@HEDWIG.INI.CMU.EDU> <20170316153517.GL14081@potion> <20170316160157.GN14081@potion> <20170316164749.GG4085@HEDWIG.INI.CMU.EDU> <20170316192622-mutt-send-email-mst@kernel.org> <20170316174149.GK4085@HEDWIG.INI.CMU.EDU> <20170316202024-mutt-send-email-mst@kernel.org> <20170316192440.GL4085@HEDWIG.INI.CMU.EDU> <20170316212635-mutt-send-email-mst@kernel.org> <20170316201710.GN4085@HEDWIG.INI.CMU.EDU> MIME-Version: 1.0 Content-Type: text/plain; charset=us-ascii Content-Disposition: inline In-Reply-To: <20170316201710.GN4085@HEDWIG.INI.CMU.EDU> X-Clacks-Overhead: GNU Terry Pratchett User-Agent: Mutt/1.7.1 (2016-10-04) Sender: linux-kernel-owner@vger.kernel.org List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Length: 7547 Lines: 187 On Thu, Mar 16, 2017 at 04:17:11PM -0400, Gabriel L. Somlo wrote: > On Thu, Mar 16, 2017 at 09:27:56PM +0200, Michael S. Tsirkin wrote: > > On Thu, Mar 16, 2017 at 03:24:41PM -0400, Gabriel L. Somlo wrote: > > > On Thu, Mar 16, 2017 at 08:29:32PM +0200, Michael S. Tsirkin wrote: > > > > Let's take a step back and try to figure out how is > > > > mwait called. How about dumping code of VCPUs > > > > around mwait? gdb disa command will do this. > > > > > > Started guest with '-s', tried to attach from gdb with > > > "target remote localhost:1234", got > > > "remote 'g' packet reply is too long: " > > > > Try > > > > set arch x86-64:x86-64 > > 'set architecture i386:x86-64:intel' is what worked for me; > > Been rooting around for a while, can't find mwait or monitor :( > > Guess I'll have to recompile KVM to actually issue an invalid opcode, > so OS X will print a panic message with the exact address :) > > Stay tuned... OK, so I found a few instances. The one closest to where a random interrupt from gdb landed, was this one: ... 0xffffff7f813ff379: mov 0x90(%r15),%rax 0xffffff7f813ff380: mov 0x18(%rax),%rsi 0xffffff7f813ff384: xor %ecx,%ecx 0xffffff7f813ff386: mov %rsi,%rax 0xffffff7f813ff389: xor %edx,%edx 0xffffff7f813ff38b: monitor %rax,%rcx,%rdx 0xffffff7f813ff38e: test %r14,%r14 0xffffff7f813ff391: je 0xffffff7f813ff3ad 0xffffff7f813ff393: movq $0x0,0x8(%r14) 0xffffff7f813ff39b: movl $0x0,(%r14) 0xffffff7f813ff3a2: test %ebx,%ebx 0xffffff7f813ff3a4: je 0xffffff7f813ff3b2 0xffffff7f813ff3a6: mfence 0xffffff7f813ff3a9: wbinvd 0xffffff7f813ff3ab: jmp 0xffffff7f813ff3b2 0xffffff7f813ff3ad: cmpl $0x0,(%rsi) 0xffffff7f813ff3b0: jne 0xffffff7f813ff3d6 0xffffff7f813ff3b2: mov %r12d,%eax 0xffffff7f813ff3b5: imul $0x148,%rax,%rax 0xffffff7f813ff3bc: lea 0x153bd(%rip),%rcx # 0xffffff7f81414780 0xffffff7f813ff3c3: mov (%rcx),%rcx 0xffffff7f813ff3c6: mov 0x20(%rcx),%rcx 0xffffff7f813ff3ca: mov 0xc(%rcx,%rax,1),%eax 0xffffff7f813ff3ce: mov $0x1,%ecx 0xffffff7f813ff3d3: mwait %rax,%rcx => 0xffffff7f813ff3d6: lfence 0xffffff7f813ff3d9: rdtsc 0xffffff7f813ff3db: lfence 0xffffff7f813ff3de: mov %rax,%rbx 0xffffff7f813ff3e1: mov %rdx,%r15 ... Also, there were a few more within the range occupied by AppleIntelCPUPowerManagement.kext (which provides is the "smart" idle loop used by OS X): ... 0xffffff7f813f799a: mov 0x90(%r15),%rax 0xffffff7f813f79a1: mov 0x18(%rax),%r15 0xffffff7f813f79a5: xor %ecx,%ecx 0xffffff7f813f79a7: mov %r15,%rax 0xffffff7f813f79aa: xor %edx,%edx 0xffffff7f813f79ac: monitor %rax,%rcx,%rdx 0xffffff7f813f79af: mov %r12d,%r12d 0xffffff7f813f79b2: imul $0x148,%r12,%r13 0xffffff7f813f79b9: lea 0x1cdc0(%rip),%rax # 0xffffff7f81414780 0xffffff7f813f79c0: mov (%rax),%rax 0xffffff7f813f79c3: mov 0x20(%rax),%rcx 0xffffff7f813f79c7: testb $0x10,0x2(%rcx,%r13,1) 0xffffff7f813f79cd: je 0xffffff7f813f79d5 0xffffff7f813f79cf: callq *0x80(%rax) 0xffffff7f813f79d5: test %r14,%r14 0xffffff7f813f79d8: je 0xffffff7f813f79f4 0xffffff7f813f79da: movq $0x0,0x8(%r14) 0xffffff7f813f79e2: movl $0x0,(%r14) 0xffffff7f813f79e9: test %ebx,%ebx 0xffffff7f813f79eb: je 0xffffff7f813f79fa 0xffffff7f813f79ed: mfence 0xffffff7f813f79f0: wbinvd 0xffffff7f813f79f2: jmp 0xffffff7f813f79fa 0xffffff7f813f79f4: cmpl $0x0,(%r15) 0xffffff7f813f79f8: jne 0xffffff7f813f7a15 0xffffff7f813f79fa: lea 0x1cd7f(%rip),%rax # 0xffffff7f81414780 0xffffff7f813f7a01: mov (%rax),%rax 0xffffff7f813f7a04: mov 0x20(%rax),%rax 0xffffff7f813f7a08: mov 0xc(%rax,%r13,1),%eax 0xffffff7f813f7a0d: mov $0x1,%ecx 0xffffff7f813f7a12: mwait %rax,%rcx 0xffffff7f813f7a15: lfence 0xffffff7f813f7a18: rdtsc 0xffffff7f813f7a1a: lfence 0xffffff7f813f7a1d: mov %rax,%rbx 0xffffff7f813f7a20: mov %rdx,%r15 ... ... 0xffffff7f813f89c9: xor %ecx,%ecx 0xffffff7f813f89cb: mov %r13,%rax 0xffffff7f813f89ce: xor %edx,%edx 0xffffff7f813f89d0: monitor %rax,%rcx,%rdx 0xffffff7f813f89d3: mov %r12d,%r15d 0xffffff7f813f89d6: imul $0x148,%r15,%r12 0xffffff7f813f89dd: lea 0x1bd9c(%rip),%rax # 0xffffff7f81414780 0xffffff7f813f89e4: mov (%rax),%rax 0xffffff7f813f89e7: mov 0x20(%rax),%rcx 0xffffff7f813f89eb: testb $0x10,0x2(%rcx,%r12,1) 0xffffff7f813f89f1: je 0xffffff7f813f89f9 0xffffff7f813f89f3: callq *0x80(%rax) 0xffffff7f813f89f9: test %r14,%r14 0xffffff7f813f89fc: je 0xffffff7f813f8a18 0xffffff7f813f89fe: movq $0x0,0x8(%r14) 0xffffff7f813f8a06: movl $0x0,(%r14) 0xffffff7f813f8a0d: test %ebx,%ebx 0xffffff7f813f8a0f: je 0xffffff7f813f8a1f 0xffffff7f813f8a11: mfence 0xffffff7f813f8a14: wbinvd 0xffffff7f813f8a16: jmp 0xffffff7f813f8a1f 0xffffff7f813f8a18: cmpl $0x0,0x0(%r13) 0xffffff7f813f8a1d: jne 0xffffff7f813f8a3a 0xffffff7f813f8a1f: lea 0x1bd5a(%rip),%rax # 0xffffff7f81414780 0xffffff7f813f8a26: mov (%rax),%rax 0xffffff7f813f8a29: mov 0x20(%rax),%rax 0xffffff7f813f8a2d: mov 0xc(%rax,%r12,1),%eax 0xffffff7f813f8a32: mov $0x1,%ecx 0xffffff7f813f8a37: mwait %rax,%rcx 0xffffff7f813f8a3a: lfence 0xffffff7f813f8a3d: rdtsc 0xffffff7f813f8a3f: lfence 0xffffff7f813f8a42: mov %rax,%rbx 0xffffff7f813f8a45: mov %rdx,%r12 0xffffff7f813f8a48: shl $0x20,%r12 ... ... 0xffffff7f81401c10: mov %r13,%rax 0xffffff7f81401c13: xor %edx,%edx 0xffffff7f81401c15: monitor %rax,%rcx,%rdx 0xffffff7f81401c18: mov %r12d,%r15d 0xffffff7f81401c1b: imul $0x148,%r15,%r12 0xffffff7f81401c22: lea 0x12b57(%rip),%rax # 0xffffff7f81414780 0xffffff7f81401c29: mov (%rax),%rax 0xffffff7f81401c2c: mov 0x20(%rax),%rcx 0xffffff7f81401c30: testb $0x10,0x2(%rcx,%r12,1) 0xffffff7f81401c36: je 0xffffff7f81401c3e 0xffffff7f81401c38: callq *0x80(%rax) 0xffffff7f81401c3e: test %r14,%r14 0xffffff7f81401c41: je 0xffffff7f81401c5d 0xffffff7f81401c43: movq $0x0,0x8(%r14) 0xffffff7f81401c4b: movl $0x0,(%r14) 0xffffff7f81401c52: test %ebx,%ebx 0xffffff7f81401c54: je 0xffffff7f81401c64 0xffffff7f81401c56: mfence 0xffffff7f81401c59: wbinvd 0xffffff7f81401c5b: jmp 0xffffff7f81401c64 0xffffff7f81401c5d: cmpl $0x0,0x0(%r13) 0xffffff7f81401c62: jne 0xffffff7f81401c7f 0xffffff7f81401c64: lea 0x12b15(%rip),%rax # 0xffffff7f81414780 0xffffff7f81401c6b: mov (%rax),%rax 0xffffff7f81401c6e: mov 0x20(%rax),%rax 0xffffff7f81401c72: mov 0xc(%rax,%r12,1),%eax 0xffffff7f81401c77: mov $0x1,%ecx 0xffffff7f81401c7c: mwait %rax,%rcx 0xffffff7f81401c7f: lfence 0xffffff7f81401c82: rdtsc 0xffffff7f81401c84: lfence 0xffffff7f81401c87: mov %rax,%rbx 0xffffff7f81401c8a: mov %rdx,%r12 0xffffff7f81401c8d: shl $0x20,%r12 0xffffff7f81401c91: lea 0xaf1c(%rip),%rax # 0xffffff7f8140cbb4 0xffffff7f81401c98: testb $0x1,(%rax) ... If that's not enough context, I can email you the whole 'script' output I collected... HTH, --Gabriel