Received-SPF: pass (google.com: best guess record for domain of linux-kernel-owner@vger.kernel.org designates 209.132.180.67 as permitted sender) client-ip=209.132.180.67;
Date:   Mon, 26 Nov 2018 17:02:17 +0100
From:   Peter Zijlstra <peterz@infradead.org>
To:     Josh Poimboeuf <jpoimboe@redhat.com>
Cc:     x86@kernel.org, linux-kernel@vger.kernel.org,
        Ard Biesheuvel <ard.biesheuvel@linaro.org>,
        Andy Lutomirski <luto@kernel.org>,
        Steven Rostedt <rostedt@goodmis.org>,
        Ingo Molnar <mingo@kernel.org>,
        Thomas Gleixner <tglx@linutronix.de>,
        Linus Torvalds <torvalds@linux-foundation.org>,
        Masami Hiramatsu <mhiramat@kernel.org>,
        Jason Baron <jbaron@akamai.com>, Jiri Kosina <jkosina@suse.cz>,
        David Laight <David.Laight@ACULAB.COM>,
        Borislav Petkov <bp@alien8.de>,
        Julia Cartwright <julia@ni.com>, Jessica Yu <jeyu@kernel.org>,
        "H. Peter Anvin" <hpa@zytor.com>
Subject: Re: [PATCH v2 4/4] x86/static_call: Add inline static call
 implementation for x86-64
Message-ID: <20181126160217.GR2113@hirez.programming.kicks-ass.net>
References: <cover.1543200841.git.jpoimboe@redhat.com>
 <62188c62f6dda49ca2e20629ee8e5a62a6c0b500.1543200841.git.jpoimboe@redhat.com>
MIME-Version: 1.0
Content-Type: text/plain; charset=us-ascii
Content-Disposition: inline
In-Reply-To: <62188c62f6dda49ca2e20629ee8e5a62a6c0b500.1543200841.git.jpoimboe@redhat.com>
User-Agent: Mutt/1.10.1 (2018-07-13)
Sender: linux-kernel-owner@vger.kernel.org
Precedence: bulk

On Mon, Nov 26, 2018 at 07:55:00AM -0600, Josh Poimboeuf wrote:
> diff --git a/arch/x86/kernel/static_call.c b/arch/x86/kernel/static_call.c
> index 8026d176f25c..d3869295b88d 100644
> --- a/arch/x86/kernel/static_call.c
> +++ b/arch/x86/kernel/static_call.c
> @@ -9,13 +9,21 @@
>  
>  void static_call_bp_handler(void);
>  void *bp_handler_dest;
> +void *bp_handler_continue;
>  
>  asm(".pushsection .text, \"ax\"						\n"
>      ".globl static_call_bp_handler					\n"
>      ".type static_call_bp_handler, @function				\n"
>      "static_call_bp_handler:						\n"
> -    "ANNOTATE_RETPOLINE_SAFE						\n"
> +#ifdef CONFIG_HAVE_STATIC_CALL_INLINE
> +    ANNOTATE_RETPOLINE_SAFE
> +    "call *bp_handler_dest						\n"
> +    ANNOTATE_RETPOLINE_SAFE
> +    "jmp *bp_handler_continue						\n"
> +#else /* !CONFIG_HAVE_STATIC_CALL_INLINE */
> +    ANNOTATE_RETPOLINE_SAFE
>      "jmp *bp_handler_dest						\n"
> +#endif
>      ".popsection							\n");
>  
>  void arch_static_call_transform(void *site, void *tramp, void *func)
> @@ -25,7 +33,10 @@ void arch_static_call_transform(void *site, void *tramp, void *func)
>  	unsigned char insn_opcode;
>  	unsigned char opcodes[CALL_INSN_SIZE];
>  
> -	insn = (unsigned long)tramp;
> +	if (IS_ENABLED(CONFIG_HAVE_STATIC_CALL_INLINE))
> +		insn = (unsigned long)site;
> +	else
> +		insn = (unsigned long)tramp;
>  
>  	mutex_lock(&text_mutex);
>  
> @@ -41,8 +52,10 @@ void arch_static_call_transform(void *site, void *tramp, void *func)
>  	opcodes[0] = insn_opcode;
>  	memcpy(&opcodes[1], &dest_relative, CALL_INSN_SIZE - 1);
>  
> -	/* Set up the variable for the breakpoint handler: */
> +	/* Set up the variables for the breakpoint handler: */
>  	bp_handler_dest = func;
> +	if (IS_ENABLED(CONFIG_HAVE_STATIC_CALL_INLINE))
> +		bp_handler_continue = (void *)(insn + CALL_INSN_SIZE);
>  
>  	/* Patch the call site: */
>  	text_poke_bp((void *)insn, opcodes, CALL_INSN_SIZE,

OK, so this is where that static_call_bp_handler comes from; you need
that CALL to frob the stack.

But I still think it is broken; consider:

	CPU0				CPU1

	bp_handler = ponies;

	text_poke_bp(, &static_call_bp_handler)
	  text_poke(&int3);
	  on_each_cpu(sync)
	  				<IPI>
					  ...
					</IPI>

	  text_poke(/* all but first bytes */)
	  on_each_cpu(sync)
	  				<IPI>
					  ...
					</IPI>

	  				<int3>
					  pt_regs->ip = &static_call_bp_handler
					</int3>

					// VCPU takes a nap...
	  text_poke(/* first byte */)
	  on_each_cpu(sync)
	  				<IPI>
					  ...
					</IPI>

					// VCPU sleeps more
	bp_handler = unicorn;

					CALL unicorn

*whoops*

Now, granted, that is all rather 'unlikely', but that never stopped
Murphy.