Received-SPF: pass (google.com: domain of linux-kernel-owner@vger.kernel.org designates 23.128.96.18 as permitted sender) client-ip=23.128.96.18;
From:   qiang.zhang@windriver.com
To:     pmladek@suse.com, tj@kernel.org
Cc:     akpm@linux-foundation.org, tglx@linutronix.de, linux-mm@kvack.org,
        linux-kernel@vger.kernel.org
Subject: [PATCH v2] kthread_worker: re-set CPU affinities if CPU come online
Date:   Wed, 28 Oct 2020 15:30:31 +0800
Message-Id: <20201028073031.4536-1-qiang.zhang@windriver.com>
Content-Type: text/plain
MIME-Version: 1.0
X-MS-Exchange-MessageSentRepresentingType: 1
Received: from pek-qzhang2-d1.wrs.com (60.247.85.82) by BY5PR20CA0020.namprd20.prod.outlook.com (2603:10b6:a03:1f4::33) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.3499.18 via Frontend Transport; Wed, 28 Oct 2020 07:30:40 +0000
X-MS-PublicTrafficType: Email
X-MS-Office365-Filtering-Correlation-Id: 10b00318-b6cf-4d4a-a03d-08d87b135fe5
X-MS-TrafficTypeDiagnostic: BYAPR11MB3352:
X-Microsoft-Antispam-PRVS: <BYAPR11MB335277C59075C8279ADD25A6FF170@BYAPR11MB3352.namprd11.prod.outlook.com>
X-MS-Oob-TLC-OOBClassifiers: OLM:751;
X-MS-Exchange-SenderADCheck: 1
X-Microsoft-Antispam: BCL:0;
X-Microsoft-Antispam-Message-Info: amuaVcqakWUaldVvVtXwVJRycrTnKMU6GScL5Ou7JnQSYzjCbXU8SlHnf/LdiG8tXuprWvo1XY2EZbzJJD6If1MKp8S9/VzLehGaLtuiGuLGiQqA9ZbjGvGKB4w8+2Vs3I3z1xHTJ5Rio3rnXm89XI3+ub/XmZTAW2YQ2dOjfZ3PTA17MEUuB+zftuiST+M4SgEp+Q8NLq3EXMNaXg5iSXap8bGUn1zzB8Ag1Ztf2spiIc+CuKQWhsmipgTaSVpZPkJGl9aUBIhnWoUJkTzu8gPk/WXHRazyuSMzW0enTvMKCg9HvDmTHR76hsT/Ymkh
X-Forefront-Antispam-Report: CIP:255.255.255.255;CTRY:;LANG:en;SCL:1;SRV:;IPV:NLI;SFV:NSPM;H:BYAPR11MB2632.namprd11.prod.outlook.com;PTR:;CAT:NONE;SFS:(4636009)(39840400004)(366004)(376002)(346002)(136003)(396003)(52116002)(4326008)(36756003)(86362001)(16526019)(8676002)(956004)(66556008)(66476007)(186003)(2616005)(2906002)(66946007)(9686003)(6512007)(478600001)(26005)(6486002)(83380400001)(6506007)(1076003)(6666004)(5660300002)(8936002)(316002);DIR:OUT;SFP:1101;
X-MS-Exchange-AntiSpam-MessageData: lgnPW0Yj6r668kCQFL7g63ugzBE5cLsX8EoNvzGk7fR+Y4S44MR2ephSVunS/95L4HkNIuTBmDVB5aUh2GKHoGm2aijwPw4yfuDdM1hCFWaEOKUmmk+o6i6kpaKA6OcHgvMuWTjK+9ycet6sz487OejVVhD7Jxat6FJrfXz0rEnHe2JkyORaMFWDORaPQlrUfBC7msNcBY8qaud/vSOjRETl8iKXXtUMIVN8/V/8T6XYY/gOaFyw+Yaf9jcEasW8u327Ae9Y7ZCfpZq6HRbwIDKDUufiNM3oZeWKJupV2ySh0xpw4yLXJC8ew3XxgRA9iJX7PW0T0cXseq53ooDL55i2nOqd4h0vG3nMXzMLe356D8yQStWYHgUOWgNQ/baCSwc/muPKRg6CG0EdQ52aoEUfmhgXdekW/1X5lVFxZsOEeOmu17DTEj3VPjT1lwKJlijMCtX1jhTCAi1ixt7oJVKlw+VLUnt+Kc1zgmBk/xivQybh6J/ruaja+wqa3UMEtLhxnTFKteMy79T+DHfsxu7Z+5DuEij2nKE+gYvaCEvg5rtn521ZXrgbKNbiqjaYH9ota+IwXBcqOQrhpG2IPgiROmaTR60E3BGbKW4JZfklPh+N5vkq9guCjlt9LpZ3l4DvAzHuThljW+8Vs1T4UQ==
X-OriginatorOrg: windriver.com
X-MS-Exchange-CrossTenant-Network-Message-Id: 10b00318-b6cf-4d4a-a03d-08d87b135fe5
X-MS-Exchange-CrossTenant-AuthSource: BYAPR11MB2632.namprd11.prod.outlook.com
X-MS-Exchange-CrossTenant-AuthAs: Internal
X-MS-Exchange-CrossTenant-OriginalArrivalTime: 28 Oct 2020 07:30:42.2930
 (UTC)
X-MS-Exchange-CrossTenant-FromEntityHeader: Hosted
X-MS-Exchange-CrossTenant-Id: 8ddb2873-a1ad-4a18-ae4e-4644631433be
X-MS-Exchange-CrossTenant-MailboxType: HOSTED
X-MS-Exchange-CrossTenant-UserPrincipalName: BzJEB6YpHiTeTKLcwi0DmJaoj+wFCX2LAG//gkzB2h8XmPhJdRoxpL7Fxr8ujSArPXGpZRyENyuYFk9TwA22FEerIK92JuZXvkB6Gl31h0w=
X-MS-Exchange-Transport-CrossTenantHeadersStamped: BYAPR11MB3352
Precedence: bulk
List-ID: <linux-kernel.vger.kernel.org>
X-Mailing-List: linux-kernel@vger.kernel.org

From: Zqiang <qiang.zhang@windriver.com>

When someone CPU offlined, the 'kthread_worker' which bind this CPU,
will run anywhere, if this CPU online, recovery of 'kthread_worker'
affinity by cpuhp notifiers.

Signed-off-by: Zqiang <qiang.zhang@windriver.com>
---
 v1->v2:
 rename variable kworker_online to kthread_worker_online.
 add 'cpuhp_node' and 'bind_cpu' init in KTHREAD_WORKER_INIT.
 add a comment explaining for WARN_ON_ONCE.

 include/linux/kthread.h |  4 ++++
 kernel/kthread.c        | 36 +++++++++++++++++++++++++++++++++++-
 2 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/include/linux/kthread.h b/include/linux/kthread.h
index 65b81e0c494d..c28963e87b18 100644
--- a/include/linux/kthread.h
+++ b/include/linux/kthread.h
@@ -93,6 +93,8 @@ struct kthread_worker {
 	struct list_head	delayed_work_list;
 	struct task_struct	*task;
 	struct kthread_work	*current_work;
+	struct hlist_node       cpuhp_node;
+	int                     bind_cpu;
 };
 
 struct kthread_work {
@@ -112,6 +114,8 @@ struct kthread_delayed_work {
 	.lock = __RAW_SPIN_LOCK_UNLOCKED((worker).lock),		\
 	.work_list = LIST_HEAD_INIT((worker).work_list),		\
 	.delayed_work_list = LIST_HEAD_INIT((worker).delayed_work_list),\
+	.cpuhp_node = {.next = NULL, .pprev = NULL},			\
+	.bind_cpu = -1,							\
 	}
 
 #define KTHREAD_WORK_INIT(work, fn)	{				\
diff --git a/kernel/kthread.c b/kernel/kthread.c
index 34516b0a6eb7..6c66df585225 100644
--- a/kernel/kthread.c
+++ b/kernel/kthread.c
@@ -28,8 +28,10 @@
 #include <linux/uaccess.h>
 #include <linux/numa.h>
 #include <linux/sched/isolation.h>
+#include <linux/cpu.h>
 #include <trace/events/sched.h>
 
+static enum cpuhp_state kthread_worker_online;
 
 static DEFINE_SPINLOCK(kthread_create_lock);
 static LIST_HEAD(kthread_create_list);
@@ -649,6 +651,8 @@ void __kthread_init_worker(struct kthread_worker *worker,
 	lockdep_set_class_and_name(&worker->lock, key, name);
 	INIT_LIST_HEAD(&worker->work_list);
 	INIT_LIST_HEAD(&worker->delayed_work_list);
+	worker->bind_cpu = -1;
+	INIT_HLIST_NODE(&worker->cpuhp_node);
 }
 EXPORT_SYMBOL_GPL(__kthread_init_worker);
 
@@ -744,8 +748,11 @@ __kthread_create_worker(int cpu, unsigned int flags,
 	if (IS_ERR(task))
 		goto fail_task;
 
-	if (cpu >= 0)
+	if (cpu >= 0) {
+		cpuhp_state_add_instance_nocalls(kthread_worker_online, &worker->cpuhp_node);
 		kthread_bind(task, cpu);
+		worker->bind_cpu = cpu;
+	}
 
 	worker->flags = flags;
 	worker->task = task;
@@ -1230,6 +1237,9 @@ void kthread_destroy_worker(struct kthread_worker *worker)
 	if (WARN_ON(!task))
 		return;
 
+	if (worker->bind_cpu >= 0)
+		cpuhp_state_remove_instance_nocalls(kthread_worker_online, &worker->cpuhp_node);
+
 	kthread_flush_worker(worker);
 	kthread_stop(task);
 	WARN_ON(!list_empty(&worker->work_list));
@@ -1237,6 +1247,30 @@ void kthread_destroy_worker(struct kthread_worker *worker)
 }
 EXPORT_SYMBOL(kthread_destroy_worker);
 
+static int kthread_worker_cpu_online(unsigned int cpu, struct hlist_node *node)
+{
+	struct kthread_worker *worker = hlist_entry(node, struct kthread_worker, cpuhp_node);
+	struct task_struct *task = worker->task;
+
+	/* as we're called from CPU_ONLINE, the following shouldn't fail */
+	if (cpu == worker->bind_cpu)
+		WARN_ON_ONCE(set_cpus_allowed_ptr(task, cpumask_of(cpu)) < 0);
+	return 0;
+}
+
+static __init int kthread_worker_hotplug_init(void)
+{
+	int ret;
+
+	ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "kthread-worker/online",
+					kthread_worker_cpu_online, NULL);
+	if (ret < 0)
+		return ret;
+	kthread_worker_online = ret;
+	return 0;
+}
+core_initcall(kthread_worker_hotplug_init);
+
 /**
  * kthread_use_mm - make the calling kthread operate on an address space
  * @mm: address space to operate on
-- 
2.17.1