As __put_task_struct() and copy_process() are hot path functions,
the call of sched_core_fork/free() will bring overhead when core
sched is disabled, and we skip them when core sched is disabled().
Signed-off-by: Cruz Zhao <[email protected]>
---
include/linux/sched.h | 10 ++++++++++
kernel/fork.c | 9 ++++++---
kernel/sched/sched.h | 10 ----------
3 files changed, 16 insertions(+), 13 deletions(-)
diff --git a/include/linux/sched.h b/include/linux/sched.h
index f64f8f2..a2266df 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -2406,9 +2406,19 @@ static inline void rseq_syscall(struct pt_regs *regs)
extern void sched_core_fork(struct task_struct *p);
extern int sched_core_share_pid(unsigned int cmd, pid_t pid, enum pid_type type,
unsigned long uaddr);
+DECLARE_STATIC_KEY_FALSE(__sched_core_enabled);
+static inline bool sched_core_disabled(void)
+{
+ return !static_branch_unlikely(&__sched_core_enabled);
+}
+
#else
static inline void sched_core_free(struct task_struct *tsk) { }
static inline void sched_core_fork(struct task_struct *p) { }
+static inline bool sched_core_disabled(void)
+{
+ return true;
+}
#endif
#endif
diff --git a/kernel/fork.c b/kernel/fork.c
index 0d13baf..611f80b 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -843,7 +843,8 @@ void __put_task_struct(struct task_struct *tsk)
exit_creds(tsk);
delayacct_tsk_free(tsk);
put_signal_struct(tsk->signal);
- sched_core_free(tsk);
+ if (!sched_core_disabled())
+ sched_core_free(tsk);
free_task(tsk);
}
EXPORT_SYMBOL_GPL(__put_task_struct);
@@ -2381,7 +2382,8 @@ static __latent_entropy struct task_struct *copy_process(
klp_copy_process(p);
- sched_core_fork(p);
+ if (!sched_core_disabled())
+ sched_core_fork(p);
spin_lock(¤t->sighand->siglock);
@@ -2469,7 +2471,8 @@ static __latent_entropy struct task_struct *copy_process(
return p;
bad_fork_cancel_cgroup:
- sched_core_free(p);
+ if (!sched_core_disabled())
+ sched_core_free(p);
spin_unlock(¤t->sighand->siglock);
write_unlock_irq(&tasklist_lock);
cgroup_cancel_fork(p, args);
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 5b21448..c6aeeda 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -1157,11 +1157,6 @@ static inline bool sched_core_enabled(struct rq *rq)
return static_branch_unlikely(&__sched_core_enabled) && rq->core_enabled;
}
-static inline bool sched_core_disabled(void)
-{
- return !static_branch_unlikely(&__sched_core_enabled);
-}
-
/*
* Be careful with this function; not for general use. The return value isn't
* stable unless you actually hold a relevant rq->__lock.
@@ -1257,11 +1252,6 @@ static inline bool sched_core_enabled(struct rq *rq)
return false;
}
-static inline bool sched_core_disabled(void)
-{
- return true;
-}
-
static inline raw_spinlock_t *rq_lockp(struct rq *rq)
{
return &rq->__lock;
--
1.8.3.1
Hi Cruz,
Thank you for the patch! Yet something to improve:
[auto build test ERROR on tip/sched/core]
[also build test ERROR on hnaz-mm/master linus/master v5.18-rc3 next-20220422]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch]
url: https://github.com/intel-lab-lkp/linux/commits/Cruz-Zhao/sched-core-Skip-sched_core_fork-free-when-core-sched-is-disabled/20220425-003934
base: https://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git a658353167bf2ea6052cee071dbcc13e0f229dc9
config: ia64-allyesconfig (https://download.01.org/0day-ci/archive/20220425/[email protected]/config)
compiler: ia64-linux-gcc (GCC) 11.2.0
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# https://github.com/intel-lab-lkp/linux/commit/211b98683083c1f71c1821b89877743bd1a4cc99
git remote add linux-review https://github.com/intel-lab-lkp/linux
git fetch --no-tags linux-review Cruz-Zhao/sched-core-Skip-sched_core_fork-free-when-core-sched-is-disabled/20220425-003934
git checkout 211b98683083c1f71c1821b89877743bd1a4cc99
# save the config file
mkdir build_dir && cp config build_dir/.config
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-11.2.0 make.cross W=1 O=build_dir ARCH=ia64 prepare
If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <[email protected]>
All error/warnings (new ones prefixed by >>):
scripts/genksyms/parse.y: warning: 9 shift/reduce conflicts [-Wconflicts-sr]
scripts/genksyms/parse.y: warning: 5 reduce/reduce conflicts [-Wconflicts-rr]
scripts/genksyms/parse.y: note: rerun with option '-Wcounterexamples' to generate conflict counterexamples
In file included from include/linux/sched/signal.h:7,
from arch/ia64/kernel/asm-offsets.c:10:
>> include/linux/sched.h:2400:1: warning: data definition has no type or storage class
2400 | DECLARE_STATIC_KEY_FALSE(__sched_core_enabled);
| ^~~~~~~~~~~~~~~~~~~~~~~~
>> include/linux/sched.h:2400:1: error: type defaults to 'int' in declaration of 'DECLARE_STATIC_KEY_FALSE' [-Werror=implicit-int]
>> include/linux/sched.h:2400:1: warning: parameter names (without types) in function declaration
include/linux/sched.h: In function 'sched_core_disabled':
>> include/linux/sched.h:2403:17: error: implicit declaration of function 'static_branch_unlikely' [-Werror=implicit-function-declaration]
2403 | return !static_branch_unlikely(&__sched_core_enabled);
| ^~~~~~~~~~~~~~~~~~~~~~
>> include/linux/sched.h:2403:41: error: '__sched_core_enabled' undeclared (first use in this function); did you mean 'sched_core_disabled'?
2403 | return !static_branch_unlikely(&__sched_core_enabled);
| ^~~~~~~~~~~~~~~~~~~~
| sched_core_disabled
include/linux/sched.h:2403:41: note: each undeclared identifier is reported only once for each function it appears in
arch/ia64/kernel/asm-offsets.c: At top level:
arch/ia64/kernel/asm-offsets.c:23:6: warning: no previous prototype for 'foo' [-Wmissing-prototypes]
23 | void foo(void)
| ^~~
cc1: some warnings being treated as errors
make[2]: *** [scripts/Makefile.build:120: arch/ia64/kernel/asm-offsets.s] Error 1
make[2]: Target '__build' not remade because of errors.
make[1]: *** [Makefile:1194: prepare0] Error 2
make[1]: Target 'prepare' not remade because of errors.
make: *** [Makefile:219: __sub-make] Error 2
make: Target 'prepare' not remade because of errors.
vim +2400 include/linux/sched.h
2394
2395 #ifdef CONFIG_SCHED_CORE
2396 extern void sched_core_free(struct task_struct *tsk);
2397 extern void sched_core_fork(struct task_struct *p);
2398 extern int sched_core_share_pid(unsigned int cmd, pid_t pid, enum pid_type type,
2399 unsigned long uaddr);
> 2400 DECLARE_STATIC_KEY_FALSE(__sched_core_enabled);
2401 static inline bool sched_core_disabled(void)
2402 {
> 2403 return !static_branch_unlikely(&__sched_core_enabled);
2404 }
2405
--
0-DAY CI Kernel Test Service
https://01.org/lkp