Provide a new command allowing processes to register their intent to use
the private expedited command.
This allows PowerPC to skip the full memory barrier in switch_mm(), and
only issue the barrier when scheduling into a task belonging to a
process that has registered to use expedited private.
Processes are now required to register before using
MEMBARRIER_CMD_PRIVATE_EXPEDITED, otherwise that command returns EPERM.
[ Runtime testing on the PowerPC architecture would be welcome. ]
Signed-off-by: Mathieu Desnoyers <[email protected]>
CC: Peter Zijlstra <[email protected]>
CC: Paul E. McKenney <[email protected]>
CC: Boqun Feng <[email protected]>
CC: Andrew Hunter <[email protected]>
CC: Maged Michael <[email protected]>
CC: [email protected]
CC: Avi Kivity <[email protected]>
CC: Benjamin Herrenschmidt <[email protected]>
CC: Paul Mackerras <[email protected]>
CC: Michael Ellerman <[email protected]>
CC: Dave Watson <[email protected]>
CC: Alan Stern <[email protected]>
CC: Will Deacon <[email protected]>
CC: Andy Lutomirski <[email protected]>
CC: [email protected]
---
MAINTAINERS | 2 ++
arch/powerpc/Kconfig | 1 +
arch/powerpc/include/asm/membarrier.h | 37 ++++++++++++++++++++++++++++
arch/powerpc/include/asm/thread_info.h | 3 +++
arch/powerpc/kernel/Makefile | 2 ++
arch/powerpc/kernel/membarrier.c | 44 +++++++++++++++++++++++++++++++++
fs/exec.c | 1 +
include/linux/mm_types.h | 3 +++
include/linux/sched/mm.h | 45 ++++++++++++++++++++++++++++++++++
include/uapi/linux/membarrier.h | 23 +++++++++++------
kernel/fork.c | 2 ++
kernel/sched/core.c | 16 +++---------
kernel/sched/membarrier.c | 25 ++++++++++++++++---
13 files changed, 180 insertions(+), 24 deletions(-)
create mode 100644 arch/powerpc/include/asm/membarrier.h
create mode 100644 arch/powerpc/kernel/membarrier.c
diff --git a/MAINTAINERS b/MAINTAINERS
index ef65785cdff2..ac2e2623c34f 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -8805,6 +8805,8 @@ L: [email protected]
S: Supported
F: kernel/sched/membarrier.c
F: include/uapi/linux/membarrier.h
+F: arch/powerpc/kernel/membarrier.c
+F: arch/powerpc/include/asm/membarrier.h
MEMORY MANAGEMENT
L: [email protected]
diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
index 809c468edab1..6f44c5f74f71 100644
--- a/arch/powerpc/Kconfig
+++ b/arch/powerpc/Kconfig
@@ -138,6 +138,7 @@ config PPC
select ARCH_HAS_ELF_RANDOMIZE
select ARCH_HAS_FORTIFY_SOURCE
select ARCH_HAS_GCOV_PROFILE_ALL
+ select ARCH_HAS_MEMBARRIER_HOOKS
select ARCH_HAS_SCALED_CPUTIME if VIRT_CPU_ACCOUNTING_NATIVE
select ARCH_HAS_SG_CHAIN
select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST
diff --git a/arch/powerpc/include/asm/membarrier.h b/arch/powerpc/include/asm/membarrier.h
new file mode 100644
index 000000000000..1342dc6c6872
--- /dev/null
+++ b/arch/powerpc/include/asm/membarrier.h
@@ -0,0 +1,37 @@
+#ifndef _ASM_POWERPC_MEMBARRIER_H
+#define _ASM_POWERPC_MEMBARRIER_H
+
+static inline void membarrier_arch_sched_in(struct task_struct *prev,
+ struct task_struct *next)
+{
+ /*
+ * Only need the full barrier when switching between processes.
+ */
+ if (likely(!test_thread_flag(TIF_MEMBARRIER_PRIVATE_EXPEDITED)
+ || prev->mm == next->mm))
+ return;
+
+ /*
+ * The membarrier system call requires a full memory barrier
+ * after storing to rq->curr, before going back to user-space.
+ */
+ smp_mb();
+}
+static inline void membarrier_arch_fork(struct task_struct *t,
+ unsigned long clone_flags)
+{
+ /*
+ * Coherence of TIF_MEMBARRIER_PRIVATE_EXPEDITED against thread
+ * fork is protected by siglock. membarrier_arch_fork is called
+ * with siglock held.
+ */
+ if (t->mm->membarrier_private_expedited)
+ set_ti_thread_flag(t, TIF_MEMBARRIER_PRIVATE_EXPEDITED);
+}
+static inline void membarrier_arch_execve(struct task_struct *t)
+{
+ clear_ti_thread_flag(t, TIF_MEMBARRIER_PRIVATE_EXPEDITED);
+}
+void membarrier_arch_register_private_expedited(struct task_struct *t);
+
+#endif /* _ASM_POWERPC_MEMBARRIER_H */
diff --git a/arch/powerpc/include/asm/thread_info.h b/arch/powerpc/include/asm/thread_info.h
index a941cc6fc3e9..2a208487724b 100644
--- a/arch/powerpc/include/asm/thread_info.h
+++ b/arch/powerpc/include/asm/thread_info.h
@@ -100,6 +100,7 @@ static inline struct thread_info *current_thread_info(void)
#if defined(CONFIG_PPC64)
#define TIF_ELF2ABI 18 /* function descriptors must die! */
#endif
+#define TIF_MEMBARRIER_PRIVATE_EXPEDITED 19 /* membarrier */
/* as above, but as bit values */
#define _TIF_SYSCALL_TRACE (1<<TIF_SYSCALL_TRACE)
@@ -119,6 +120,8 @@ static inline struct thread_info *current_thread_info(void)
#define _TIF_SYSCALL_TRACEPOINT (1<<TIF_SYSCALL_TRACEPOINT)
#define _TIF_EMULATE_STACK_STORE (1<<TIF_EMULATE_STACK_STORE)
#define _TIF_NOHZ (1<<TIF_NOHZ)
+#define _TIF_MEMBARRIER_PRIVATE_EXPEDITED \
+ (1<<TIF_MEMBARRIER_PRIVATE_EXPEDITED)
#define _TIF_SYSCALL_DOTRACE (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | \
_TIF_SECCOMP | _TIF_SYSCALL_TRACEPOINT | \
_TIF_NOHZ)
diff --git a/arch/powerpc/kernel/Makefile b/arch/powerpc/kernel/Makefile
index 91960f83039c..2dd4b9e3313a 100644
--- a/arch/powerpc/kernel/Makefile
+++ b/arch/powerpc/kernel/Makefile
@@ -135,6 +135,8 @@ endif
obj-$(CONFIG_EPAPR_PARAVIRT) += epapr_paravirt.o epapr_hcalls.o
obj-$(CONFIG_KVM_GUEST) += kvm.o kvm_emul.o
+obj-$(CONFIG_MEMBARRIER) += membarrier.o
+
# Disable GCOV & sanitizers in odd or sensitive code
GCOV_PROFILE_prom_init.o := n
UBSAN_SANITIZE_prom_init.o := n
diff --git a/arch/powerpc/kernel/membarrier.c b/arch/powerpc/kernel/membarrier.c
new file mode 100644
index 000000000000..6f47e190a409
--- /dev/null
+++ b/arch/powerpc/kernel/membarrier.c
@@ -0,0 +1,44 @@
+/*
+ * Copyright (C) 2010-2017 Mathieu Desnoyers <[email protected]>
+ *
+ * membarrier system call - PowerPC architecture code
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/sched/mm.h>
+#include <linux/sched/signal.h>
+#include <linux/thread_info.h>
+#include <linux/spinlock.h>
+#include <linux/rcupdate.h>
+
+void membarrier_arch_register_private_expedited(struct task_struct *p)
+{
+ struct task_struct *t;
+
+ if (get_nr_threads(p) == 1) {
+ set_thread_flag(TIF_MEMBARRIER_PRIVATE_EXPEDITED);
+ return;
+ }
+ /*
+ * Coherence of TIF_MEMBARRIER_PRIVATE_EXPEDITED against thread
+ * fork is protected by siglock.
+ */
+ spin_lock(&p->sighand->siglock);
+ for_each_thread(p, t)
+ set_ti_thread_flag(t, TIF_MEMBARRIER_PRIVATE_EXPEDITED);
+ spin_unlock(&p->sighand->siglock);
+ /*
+ * Ensure all future scheduler executions will observe the new
+ * thread flag state for this process.
+ */
+ synchronize_sched();
+}
diff --git a/fs/exec.c b/fs/exec.c
index ac34d9724684..b2448f2731b3 100644
--- a/fs/exec.c
+++ b/fs/exec.c
@@ -1802,6 +1802,7 @@ static int do_execveat_common(int fd, struct filename *filename,
/* execve succeeded */
current->fs->in_exec = 0;
current->in_execve = 0;
+ membarrier_execve(current);
acct_update_integrals(current);
task_numa_free(current);
free_bprm(bprm);
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 46f4ecf5479a..5e0fe8ce053b 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -445,6 +445,9 @@ struct mm_struct {
unsigned long flags; /* Must use atomic bitops to access the bits */
struct core_state *core_state; /* coredumping support */
+#ifdef CONFIG_MEMBARRIER
+ int membarrier_private_expedited;
+#endif
#ifdef CONFIG_AIO
spinlock_t ioctx_lock;
struct kioctx_table __rcu *ioctx_table;
diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h
index 3a19c253bdb1..df4005e2c4cf 100644
--- a/include/linux/sched/mm.h
+++ b/include/linux/sched/mm.h
@@ -205,4 +205,49 @@ static inline void memalloc_noreclaim_restore(unsigned int flags)
current->flags = (current->flags & ~PF_MEMALLOC) | flags;
}
+#ifdef CONFIG_ARCH_HAS_MEMBARRIER_HOOKS
+#include <asm/membarrier.h>
+#else
+static inline void membarrier_arch_sched_in(struct task_struct *prev,
+ struct task_struct *next)
+{
+}
+static inline void membarrier_arch_register_private_expedited(
+ struct task_struct *p)
+{
+}
+static inline void membarrier_arch_fork(struct task_struct *t,
+ unsigned long clone_flags)
+{
+}
+static inline void membarrier_arch_execve(struct task_struct *t)
+{
+}
+#endif
+
+#ifdef CONFIG_MEMBARRIER
+static inline void membarrier_fork(struct task_struct *t,
+ unsigned long clone_flags)
+{
+ if (!current->mm || !t->mm)
+ return;
+ t->mm->membarrier_private_expedited =
+ current->mm->membarrier_private_expedited;
+ membarrier_arch_fork(t, clone_flags);
+}
+static inline void membarrier_execve(struct task_struct *t)
+{
+ t->mm->membarrier_private_expedited = 0;
+ membarrier_arch_execve(t);
+}
+#else
+static inline void membarrier_fork(struct task_struct *t,
+ unsigned long clone_flags)
+{
+}
+static inline void membarrier_execve(struct task_struct *t)
+{
+}
+#endif
+
#endif /* _LINUX_SCHED_MM_H */
diff --git a/include/uapi/linux/membarrier.h b/include/uapi/linux/membarrier.h
index 6d47b3249d8a..4e01ad7ffe98 100644
--- a/include/uapi/linux/membarrier.h
+++ b/include/uapi/linux/membarrier.h
@@ -52,21 +52,30 @@
* (non-running threads are de facto in such a
* state). This only covers threads from the
* same processes as the caller thread. This
- * command returns 0. The "expedited" commands
- * complete faster than the non-expedited ones,
- * they never block, but have the downside of
- * causing extra overhead.
+ * command returns 0 on success. The
+ * "expedited" commands complete faster than
+ * the non-expedited ones, they never block,
+ * but have the downside of causing extra
+ * overhead. A process needs to register its
+ * intent to use the private expedited command
+ * prior to using it, otherwise this command
+ * returns -EPERM.
+ * @MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED:
+ * Register the process intent to use
+ * MEMBARRIER_CMD_PRIVATE_EXPEDITED. Always
+ * returns 0.
*
* Command to be passed to the membarrier system call. The commands need to
* be a single bit each, except for MEMBARRIER_CMD_QUERY which is assigned to
* the value 0.
*/
enum membarrier_cmd {
- MEMBARRIER_CMD_QUERY = 0,
- MEMBARRIER_CMD_SHARED = (1 << 0),
+ MEMBARRIER_CMD_QUERY = 0,
+ MEMBARRIER_CMD_SHARED = (1 << 0),
/* reserved for MEMBARRIER_CMD_SHARED_EXPEDITED (1 << 1) */
/* reserved for MEMBARRIER_CMD_PRIVATE (1 << 2) */
- MEMBARRIER_CMD_PRIVATE_EXPEDITED = (1 << 3),
+ MEMBARRIER_CMD_PRIVATE_EXPEDITED = (1 << 3),
+ MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED = (1 << 4),
};
#endif /* _UAPI_LINUX_MEMBARRIER_H */
diff --git a/kernel/fork.c b/kernel/fork.c
index 10646182440f..bd4a93915e08 100644
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -1840,6 +1840,8 @@ static __latent_entropy struct task_struct *copy_process(
*/
copy_seccomp(p);
+ membarrier_fork(p, clone_flags);
+
/*
* Process group and session signals need to be delivered to just the
* parent before the fork or both the parent and the child after the
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 18a6966567da..c5c1b2c51807 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -2644,16 +2644,6 @@ static struct rq *finish_task_switch(struct task_struct *prev)
prev_state = prev->state;
vtime_task_switch(prev);
perf_event_task_sched_in(prev, current);
- /*
- * The membarrier system call requires a full memory barrier
- * after storing to rq->curr, before going back to user-space.
- *
- * TODO: This smp_mb__after_unlock_lock can go away if PPC end
- * up adding a full barrier to switch_mm(), or we should figure
- * out if a smp_mb__after_unlock_lock is really the proper API
- * to use.
- */
- smp_mb__after_unlock_lock();
finish_lock_switch(rq, prev);
finish_arch_post_lock_switch();
@@ -3354,9 +3344,8 @@ static void __sched notrace __schedule(bool preempt)
* care of this barrier. For weakly ordered machines for
* which spin_unlock() acts as a RELEASE barrier (only
* arm64 and PowerPC), arm64 has a full barrier in
- * switch_to(), and PowerPC has
- * smp_mb__after_unlock_lock() before
- * finish_lock_switch().
+ * switch_to(), and PowerPC has a full barrier in
+ * membarrier_arch_sched_in().
*/
++*switch_count;
@@ -3364,6 +3353,7 @@ static void __sched notrace __schedule(bool preempt)
/* Also unlocks the rq: */
rq = context_switch(rq, prev, next, &rf);
+ membarrier_arch_sched_in(prev, next);
} else {
rq->clock_update_flags &= ~(RQCF_ACT_SKIP|RQCF_REQ_SKIP);
rq_unlock_irq(rq, &rf);
diff --git a/kernel/sched/membarrier.c b/kernel/sched/membarrier.c
index a92fddc22747..00a2618a36ba 100644
--- a/kernel/sched/membarrier.c
+++ b/kernel/sched/membarrier.c
@@ -26,21 +26,25 @@
* except MEMBARRIER_CMD_QUERY.
*/
#define MEMBARRIER_CMD_BITMASK \
- (MEMBARRIER_CMD_SHARED | MEMBARRIER_CMD_PRIVATE_EXPEDITED)
+ (MEMBARRIER_CMD_SHARED | MEMBARRIER_CMD_PRIVATE_EXPEDITED \
+ | MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED)
static void ipi_mb(void *info)
{
smp_mb(); /* IPIs should be serializing but paranoid. */
}
-static void membarrier_private_expedited(void)
+static int membarrier_private_expedited(void)
{
int cpu;
bool fallback = false;
cpumask_var_t tmpmask;
+ if (!current->mm->membarrier_private_expedited)
+ return -EPERM;
+
if (num_online_cpus() == 1)
- return;
+ return 0;
/*
* Matches memory barriers around rq->curr modification in
@@ -94,6 +98,17 @@ static void membarrier_private_expedited(void)
* rq->curr modification in scheduler.
*/
smp_mb(); /* exit from system call is not a mb */
+ return 0;
+}
+
+static void membarrier_register_private_expedited(void)
+{
+ struct task_struct *p = current;
+
+ if (READ_ONCE(p->mm->membarrier_private_expedited))
+ return;
+ WRITE_ONCE(p->mm->membarrier_private_expedited, 1);
+ membarrier_arch_register_private_expedited(p);
}
/**
@@ -144,7 +159,9 @@ SYSCALL_DEFINE2(membarrier, int, cmd, int, flags)
synchronize_sched();
return 0;
case MEMBARRIER_CMD_PRIVATE_EXPEDITED:
- membarrier_private_expedited();
+ return membarrier_private_expedited();
+ case MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED:
+ membarrier_register_private_expedited();
return 0;
default:
return -EINVAL;
--
2.11.0
Test the new MEMBARRIER_CMD_PRIVATE_EXPEDITED and
MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED commands.
Add checks expecting specific error values on system calls expected to
fail.
Signed-off-by: Mathieu Desnoyers <[email protected]>
CC: Peter Zijlstra <[email protected]>
CC: Paul E. McKenney <[email protected]>
CC: Boqun Feng <[email protected]>
CC: Andrew Hunter <[email protected]>
CC: Maged Michael <[email protected]>
CC: [email protected]
CC: Avi Kivity <[email protected]>
CC: Benjamin Herrenschmidt <[email protected]>
CC: Paul Mackerras <[email protected]>
CC: Michael Ellerman <[email protected]>
CC: Dave Watson <[email protected]>
CC: Alan Stern <[email protected]>
CC: Will Deacon <[email protected]>
CC: Andy Lutomirski <[email protected]>
CC: [email protected]
---
.../testing/selftests/membarrier/membarrier_test.c | 109 ++++++++++++++++++---
1 file changed, 94 insertions(+), 15 deletions(-)
diff --git a/tools/testing/selftests/membarrier/membarrier_test.c b/tools/testing/selftests/membarrier/membarrier_test.c
index 21399fcf1a59..f85657374b59 100644
--- a/tools/testing/selftests/membarrier/membarrier_test.c
+++ b/tools/testing/selftests/membarrier/membarrier_test.c
@@ -15,49 +15,119 @@ static int sys_membarrier(int cmd, int flags)
static int test_membarrier_cmd_fail(void)
{
int cmd = -1, flags = 0;
+ const char *test_name = "sys membarrier invalid command";
if (sys_membarrier(cmd, flags) != -1) {
ksft_exit_fail_msg(
- "sys membarrier invalid command test: command = %d, flags = %d. Should fail, but passed\n",
- cmd, flags);
+ "%s test: command = %d, flags = %d. Should fail, but passed\n",
+ test_name, cmd, flags);
+ }
+ if (errno != EINVAL) {
+ ksft_exit_fail_msg(
+ "%s test: flags = %d. Should return (%d: \"%s\"), but returned (%d: \"%s\").\n",
+ test_name, flags, EINVAL, strerror(EINVAL),
+ errno, strerror(errno));
}
ksft_test_result_pass(
- "sys membarrier invalid command test: command = %d, flags = %d. Failed as expected\n",
- cmd, flags);
+ "%s test: command = %d, flags = %d, errno = %d. Failed as expected\n",
+ test_name, cmd, flags, errno);
return 0;
}
static int test_membarrier_flags_fail(void)
{
int cmd = MEMBARRIER_CMD_QUERY, flags = 1;
+ const char *test_name = "sys membarrier MEMBARRIER_CMD_QUERY invalid flags";
if (sys_membarrier(cmd, flags) != -1) {
ksft_exit_fail_msg(
- "sys membarrier MEMBARRIER_CMD_QUERY invalid flags test: flags = %d. Should fail, but passed\n",
- flags);
+ "%s test: flags = %d. Should fail, but passed\n",
+ test_name, flags);
+ }
+ if (errno != EINVAL) {
+ ksft_exit_fail_msg(
+ "%s test: flags = %d. Should return (%d: \"%s\"), but returned (%d: \"%s\").\n",
+ test_name, flags, EINVAL, strerror(EINVAL),
+ errno, strerror(errno));
}
ksft_test_result_pass(
- "sys membarrier MEMBARRIER_CMD_QUERY invalid flags test: flags = %d. Failed as expected\n",
- flags);
+ "%s test: flags = %d, errno = %d. Failed as expected\n",
+ test_name, flags, errno);
return 0;
}
-static int test_membarrier_success(void)
+static int test_membarrier_shared_success(void)
{
int cmd = MEMBARRIER_CMD_SHARED, flags = 0;
- const char *test_name = "sys membarrier MEMBARRIER_CMD_SHARED\n";
+ const char *test_name = "sys membarrier MEMBARRIER_CMD_SHARED";
+
+ if (sys_membarrier(cmd, flags) != 0) {
+ ksft_exit_fail_msg(
+ "%s test: flags = %d, errno = %d\n",
+ test_name, flags, errno);
+ }
+
+ ksft_test_result_pass(
+ "%s test: flags = %d\n", test_name, flags);
+ return 0;
+}
+
+static int test_membarrier_private_expedited_fail(void)
+{
+ int cmd = MEMBARRIER_CMD_PRIVATE_EXPEDITED, flags = 0;
+ const char *test_name = "sys membarrier MEMBARRIER_CMD_PRIVATE_EXPEDITED not registered failure";
+
+ if (sys_membarrier(cmd, flags) != -1) {
+ ksft_exit_fail_msg(
+ "%s test: flags = %d. Should fail, but passed\n",
+ test_name, flags);
+ }
+ if (errno != EPERM) {
+ ksft_exit_fail_msg(
+ "%s test: flags = %d. Should return (%d: \"%s\"), but returned (%d: \"%s\").\n",
+ test_name, flags, EPERM, strerror(EPERM),
+ errno, strerror(errno));
+ }
+
+ ksft_test_result_pass(
+ "%s test: flags = %d, errno = %d\n",
+ test_name, flags, errno);
+ return 0;
+}
+
+static int test_membarrier_register_private_expedited_success(void)
+{
+ int cmd = MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED, flags = 0;
+ const char *test_name = "sys membarrier MEMBARRIER_CMD_REGISTER_PRIVATE_EXPEDITED";
if (sys_membarrier(cmd, flags) != 0) {
ksft_exit_fail_msg(
- "sys membarrier MEMBARRIER_CMD_SHARED test: flags = %d\n",
- flags);
+ "%s test: flags = %d, errno = %d\n",
+ test_name, flags, errno);
}
ksft_test_result_pass(
- "sys membarrier MEMBARRIER_CMD_SHARED test: flags = %d\n",
- flags);
+ "%s test: flags = %d\n",
+ test_name, flags);
+ return 0;
+}
+
+static int test_membarrier_private_expedited_success(void)
+{
+ int cmd = MEMBARRIER_CMD_PRIVATE_EXPEDITED, flags = 0;
+ const char *test_name = "sys membarrier MEMBARRIER_CMD_PRIVATE_EXPEDITED";
+
+ if (sys_membarrier(cmd, flags) != 0) {
+ ksft_exit_fail_msg(
+ "%s test: flags = %d, errno = %d\n",
+ test_name, flags, errno);
+ }
+
+ ksft_test_result_pass(
+ "%s test: flags = %d\n",
+ test_name, flags);
return 0;
}
@@ -71,7 +141,16 @@ static int test_membarrier(void)
status = test_membarrier_flags_fail();
if (status)
return status;
- status = test_membarrier_success();
+ status = test_membarrier_shared_success();
+ if (status)
+ return status;
+ status = test_membarrier_private_expedited_fail();
+ if (status)
+ return status;
+ status = test_membarrier_register_private_expedited_success();
+ if (status)
+ return status;
+ status = test_membarrier_private_expedited_success();
if (status)
return status;
return 0;
--
2.11.0
On Mon, 18 Sep 2017, Mathieu Desnoyers wrote:
> Provide a new command allowing processes to register their intent to use
> the private expedited command.
>
> This allows PowerPC to skip the full memory barrier in switch_mm(), and
> only issue the barrier when scheduling into a task belonging to a
> process that has registered to use expedited private.
>
> Processes are now required to register before using
> MEMBARRIER_CMD_PRIVATE_EXPEDITED, otherwise that command returns EPERM.
>
> [ Runtime testing on the PowerPC architecture would be welcome. ]
>
> Signed-off-by: Mathieu Desnoyers <[email protected]>
> +++ b/arch/powerpc/include/asm/membarrier.h
> @@ -0,0 +1,37 @@
> +#ifndef _ASM_POWERPC_MEMBARRIER_H
> +#define _ASM_POWERPC_MEMBARRIER_H
> +
> +static inline void membarrier_arch_sched_in(struct task_struct *prev,
> + struct task_struct *next)
> +{
> + /*
> + * Only need the full barrier when switching between processes.
> + */
> + if (likely(!test_thread_flag(TIF_MEMBARRIER_PRIVATE_EXPEDITED)
> + || prev->mm == next->mm))
> + return;
> +
> + /*
> + * The membarrier system call requires a full memory barrier
> + * after storing to rq->curr, before going back to user-space.
> + */
> + smp_mb();
> +}
> +static inline void membarrier_arch_fork(struct task_struct *t,
> + unsigned long clone_flags)
> +{
> + /*
> + * Coherence of TIF_MEMBARRIER_PRIVATE_EXPEDITED against thread
> + * fork is protected by siglock. membarrier_arch_fork is called
> + * with siglock held.
> + */
> + if (t->mm->membarrier_private_expedited)
> + set_ti_thread_flag(t, TIF_MEMBARRIER_PRIVATE_EXPEDITED);
> +}
Why have two separate bitflags for the same thing? Can't you just use
the mm->membarrier_private_expedited flag everywhere and forget about
TIF_MEMBARRIER_PRIVATE_EXPEDITED?
Alan Stern
----- On Sep 18, 2017, at 2:46 PM, Alan Stern [email protected] wrote:
> On Mon, 18 Sep 2017, Mathieu Desnoyers wrote:
>
>> Provide a new command allowing processes to register their intent to use
>> the private expedited command.
>>
>> This allows PowerPC to skip the full memory barrier in switch_mm(), and
>> only issue the barrier when scheduling into a task belonging to a
>> process that has registered to use expedited private.
>>
>> Processes are now required to register before using
>> MEMBARRIER_CMD_PRIVATE_EXPEDITED, otherwise that command returns EPERM.
>>
>> [ Runtime testing on the PowerPC architecture would be welcome. ]
>>
>> Signed-off-by: Mathieu Desnoyers <[email protected]>
>
>> +++ b/arch/powerpc/include/asm/membarrier.h
>> @@ -0,0 +1,37 @@
>> +#ifndef _ASM_POWERPC_MEMBARRIER_H
>> +#define _ASM_POWERPC_MEMBARRIER_H
>> +
>> +static inline void membarrier_arch_sched_in(struct task_struct *prev,
>> + struct task_struct *next)
>> +{
>> + /*
>> + * Only need the full barrier when switching between processes.
>> + */
>> + if (likely(!test_thread_flag(TIF_MEMBARRIER_PRIVATE_EXPEDITED)
>> + || prev->mm == next->mm))
I think I should use:
test_ti_thread_flag(next, TIF_MEMBARRIER_PRIVATE_EXPEDITED)
instead of test_thread_flag above.
>> + return;
>> +
>> + /*
>> + * The membarrier system call requires a full memory barrier
>> + * after storing to rq->curr, before going back to user-space.
>> + */
>> + smp_mb();
>> +}
>> +static inline void membarrier_arch_fork(struct task_struct *t,
>> + unsigned long clone_flags)
>> +{
>> + /*
>> + * Coherence of TIF_MEMBARRIER_PRIVATE_EXPEDITED against thread
>> + * fork is protected by siglock. membarrier_arch_fork is called
>> + * with siglock held.
>> + */
>> + if (t->mm->membarrier_private_expedited)
>> + set_ti_thread_flag(t, TIF_MEMBARRIER_PRIVATE_EXPEDITED);
>> +}
>
> Why have two separate bitflags for the same thing? Can't you just use
> the mm->membarrier_private_expedited flag everywhere and forget about
> TIF_MEMBARRIER_PRIVATE_EXPEDITED?
The goal here is to save chasing through the task->mm-> pointer in sched_in(),
and directly access the process registration state through a thread flag.
Thanks,
Mathieu
>
> Alan Stern
--
Mathieu Desnoyers
EfficiOS Inc.
http://www.efficios.com