Return-Path: <linux-kernel-owner@vger.kernel.org>
Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand
        id S1752476AbdGEJEr (ORCPT <rfc822;w@1wt.eu>);
        Wed, 5 Jul 2017 05:04:47 -0400
Received: from bombadil.infradead.org ([65.50.211.133]:58074 "EHLO
        bombadil.infradead.org" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
        with ESMTP id S1751925AbdGEJEq (ORCPT
        <rfc822;linux-kernel@vger.kernel.org>);
        Wed, 5 Jul 2017 05:04:46 -0400
Date: Wed, 5 Jul 2017 11:04:40 +0200
From: Peter Zijlstra <peterz@infradead.org>
To: Thomas Gleixner <tglx@linutronix.de>
Cc: Vikram Mulukutla <markivx@codeaurora.org>,
        Rusty Russell <rusty@rustcorp.com.au>, Tejun Heo <tj@kernel.org>,
        Andrew Morton <akpm@linux-foundation.org>,
        LKML <linux-kernel@vger.kernel.org>,
        Sebastian Sewior <bigeasy@linutronix.de>
Subject: Re: [PATCH] smp/hotplug: Move unparking of percpu threads to the
 control CPU
Message-ID: <20170705090440.hxnpsj2pgxykg3t6@hirez.programming.kicks-ass.net>
References: <alpine.DEB.2.20.1707042218020.2131@nanos>
MIME-Version: 1.0
Content-Type: text/plain; charset=us-ascii
Content-Disposition: inline
In-Reply-To: <alpine.DEB.2.20.1707042218020.2131@nanos>
User-Agent: NeoMutt/20170113 (1.7.2)
Sender: linux-kernel-owner@vger.kernel.org
List-ID: <linux-kernel.vger.kernel.org>
X-Mailing-List: linux-kernel@vger.kernel.org
Content-Length: 2294
Lines: 88

On Tue, Jul 04, 2017 at 10:20:23PM +0200, Thomas Gleixner wrote:
> --- a/kernel/cpu.c
> +++ b/kernel/cpu.c
> @@ -271,11 +271,25 @@ void cpu_hotplug_enable(void)
>  EXPORT_SYMBOL_GPL(cpu_hotplug_enable);
>  #endif	/* CONFIG_HOTPLUG_CPU */
>  
> +static void __cpuhp_kick_ap_work(struct cpuhp_cpu_state *st);
> +
>  static int bringup_wait_for_ap(unsigned int cpu)
>  {
>  	struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, cpu);
>  
> +	/* Wait for the CPU to reach IDLE_ONLINE */
>  	wait_for_completion(&st->done);
> +	BUG_ON(!cpu_online(cpu));
> +
> +	/* Unpark the stopper thread and the hotplug thread of the target cpu */
> +	stop_machine_unpark(cpu);
> +	kthread_unpark(st->thread);
> +
> +	/* Should we go further up ? */
> +	if (st->target > CPUHP_AP_ONLINE_IDLE) {
> +		__cpuhp_kick_ap_work(st);
> +		wait_for_completion(&st->done);
> +	}
>  	return st->result;
>  }
>  
> @@ -296,9 +310,7 @@ static int bringup_cpu(unsigned int cpu)
>  	irq_unlock_sparse();
>  	if (ret)
>  		return ret;
> -	ret = bringup_wait_for_ap(cpu);
> -	BUG_ON(!cpu_online(cpu));
> -	return ret;
> +	return bringup_wait_for_ap(cpu);
>  }
>  
>  /*
> @@ -775,23 +787,13 @@ void notify_cpu_starting(unsigned int cp

The comment right above this function now seems stale..

>  void cpuhp_online_idle(enum cpuhp_state state)
>  {
>  	struct cpuhp_cpu_state *st = this_cpu_ptr(&cpuhp_state);
> -	unsigned int cpu = smp_processor_id();
>  
>  	/* Happens for the boot cpu */
>  	if (state != CPUHP_AP_ONLINE_IDLE)
>  		return;
>  
>  	st->state = CPUHP_AP_ONLINE_IDLE;
> -
> -	/* Unpark the stopper thread and the hotplug thread of this cpu */
> -	stop_machine_unpark(cpu);
> -	kthread_unpark(st->thread);
> -
> -	/* Should we go further up ? */
> -	if (st->target > CPUHP_AP_ONLINE_IDLE)
> -		__cpuhp_kick_ap_work(st);
> -	else
> -		complete(&st->done);
> +	complete(&st->done);
>  }


OK, so if I get this right we do something like:


BP				AP

bringup_cpu();
  __cpu_up()  ------------>     /* stuff */
  bringup_wait_for_ap()
    wait_for_completion();
				cpuhp_online_idle();
		<------------    complete(&st->done);
    unpark()
				while(1)
				  do_idle();


Where you moved the unpark() from the AP's idle thread to the BP's
context and thus allow scheduling etc..

Yes that should work fine I think.