by Donghai Qiao

[permalink] [raw]

Subject: [PATCH 03/11] smp: eliminate SCF_WAIT and SCF_RUN_LOCAL

The commit a32a4d8a815c ("smp: Run functions concurrently in
smp_call_function_many_cond()") was to improve the concurrent
of the cross call execution between local and remote. The change
in smp_call_function_many_cond() did what was intended, but the
new macro SCF_WAIT and SCF_RUN_LOCAL and the code around them
to handle local call were not unnecessary because the modified
function smp_call_function_many() was able to handle the local
cross call. So these two macros can be eliminated and the code
implemented around that can be removed as well.

Also this patch fixed a issue of a comparison between an integer
and a unsigned integer in smp_call_function_many_cond().

The changes with this patch and the changes made in subsequent
patches will eventually help eliminate the set of on_each_cpu*
functions.

Signed-off-by: Donghai Qiao <[email protected]>
---
kernel/smp.c | 32 +++++++-------------------------
1 file changed, 7 insertions(+), 25 deletions(-)

diff --git a/kernel/smp.c b/kernel/smp.c
index 6183a3586329..3f9bc5ae7180 100644
--- a/kernel/smp.c
+++ b/kernel/smp.c
@@ -787,23 +787,13 @@ int smp_call_function_any(const struct cpumask *mask,
}
EXPORT_SYMBOL_GPL(smp_call_function_any);

-/*
- * Flags to be used as scf_flags argument of smp_call_function_many_cond().
- *
- * %SCF_WAIT: Wait until function execution is completed
- * %SCF_RUN_LOCAL: Run also locally if local cpu is set in cpumask
- */
-#define SCF_WAIT (1U << 0)
-#define SCF_RUN_LOCAL (1U << 1)
-
static void smp_call_function_many_cond(const struct cpumask *mask,
smp_call_func_t func, void *info,
- unsigned int scf_flags,
+ bool wait,
smp_cond_func_t cond_func)
{
int cpu, last_cpu, this_cpu = smp_processor_id();
struct call_function_data *cfd;
- bool wait = scf_flags & SCF_WAIT;
bool run_remote = false;
bool run_local = false;
int nr_cpus = 0;
@@ -829,14 +819,14 @@ static void smp_call_function_many_cond(const struct cpumask *mask,
WARN_ON_ONCE(!in_task());

/* Check if we need local execution. */
- if ((scf_flags & SCF_RUN_LOCAL) && cpumask_test_cpu(this_cpu, mask))
+ if (cpumask_test_cpu(this_cpu, mask))
run_local = true;

/* Check if we need remote execution, i.e., any CPU excluding this one. */
cpu = cpumask_first_and(mask, cpu_online_mask);
if (cpu == this_cpu)
cpu = cpumask_next_and(cpu, mask, cpu_online_mask);
- if (cpu < nr_cpu_ids)
+ if ((unsigned int)cpu < nr_cpu_ids)
run_remote = true;

if (run_remote) {
@@ -911,12 +901,8 @@ static void smp_call_function_many_cond(const struct cpumask *mask,
* @mask: The set of cpus to run on (only runs on online subset).
* @func: The function to run. This must be fast and non-blocking.
* @info: An arbitrary pointer to pass to the function.
- * @wait: Bitmask that controls the operation. If %SCF_WAIT is set, wait
- * (atomically) until function has completed on other CPUs. If
- * %SCF_RUN_LOCAL is set, the function will also be run locally
- * if the local CPU is set in the @cpumask.
- *
- * If @wait is true, then returns once @func has returned.
+ * @wait: If wait is true, the call will not return until func()
+ * has completed on other CPUs.
*
* You must not call this function with disabled interrupts or from a
* hardware interrupt handler or from a bottom half handler. Preemption
@@ -925,7 +911,7 @@ static void smp_call_function_many_cond(const struct cpumask *mask,
void smp_call_function_many(const struct cpumask *mask,
smp_call_func_t func, void *info, bool wait)
{
- smp_call_function_many_cond(mask, func, info, wait * SCF_WAIT, NULL);
+ smp_call_function_many_cond(mask, func, info, wait, NULL);
}
EXPORT_SYMBOL(smp_call_function_many);

@@ -1061,13 +1047,9 @@ void __init smp_init(void)
void on_each_cpu_cond_mask(smp_cond_func_t cond_func, smp_call_func_t func,
void *info, bool wait, const struct cpumask *mask)
{
- unsigned int scf_flags = SCF_RUN_LOCAL;
-
- if (wait)
- scf_flags |= SCF_WAIT;

preempt_disable();
- smp_call_function_many_cond(mask, func, info, scf_flags, cond_func);
+ smp_call_function_many_cond(mask, func, info, wait, cond_func);
preempt_enable();
}
EXPORT_SYMBOL(on_each_cpu_cond_mask);
--
2.27.0

2022-04-16 02:40:09

by Donghai Qiao

[permalink] [raw]

Subject: [PATCH 02/11] smp: cross call interface

The functions of cross CPU call interface are defined below :

int smp_xcall(int cpu, smp_call_func_t func, void *info,
unsigned int flags)

int smp_xcall_cond(int cpu, smp_call_func_t func, void *info,
smp_cond_func_t condf, unsigned int flags)

void smp_xcall_mask(const struct cpumask *mask, smp_call_func_t func,
void *info, unsigned int flags)

void smp_xcall_mask_cond(const struct cpumask *mask, smp_call_func_t func,
void *info, smp_cond_func_t condf, unsigned int flags)

int smp_xcall_private(int cpu, call_single_data_t *csd, unsigned int flags)

int smp_xcall_any(const struct cpumask *mask, smp_call_func_t func,
void *info, unsigned int flags)

The motivation of submitting this patch set is intended to make the
existing cross CPU call mechanism become a bit more formal interface
and more friendly to the kernel developers.

Basically the minimum set of functions below can satisfy any demand
for cross CPU call from kernel consumers. For the sack of simplicity
self-explanatory and less code redundancy no ambiguity, the functions
in this interface are renamed, simplified, or eliminated. But they
are still inheriting the same semantics and parameter lists from their
previous version.

Signed-off-by: Donghai Qiao <[email protected]>
---
include/linux/smp.h | 30 +++++++++
kernel/smp.c | 156 ++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 186 insertions(+)

diff --git a/include/linux/smp.h b/include/linux/smp.h
index 31811da856a3..12d6efef34f7 100644
--- a/include/linux/smp.h
+++ b/include/linux/smp.h
@@ -161,6 +161,36 @@ do { \
*(_csd) = CSD_INIT((_func), (_info)); \
} while (0)

+
+/*
+ * smp_xcall Interface.
+ *
+ * Also see kernel/smp.c for the details.
+ */
+#define XCALL_TYPE_SYNC CSD_TYPE_SYNC
+#define XCALL_TYPE_ASYNC CSD_TYPE_ASYNC
+#define XCALL_TYPE_IRQ_WORK CSD_TYPE_IRQ_WORK
+#define XCALL_TYPE_TTWU CSD_TYPE_TTWU
+#define XCALL_TYPE_MASK CSD_FLAG_TYPE_MASK
+
+#define XCALL_ALL -1
+
+extern int smp_xcall(int cpu, smp_call_func_t func, void *info, unsigned int flags);
+
+extern int smp_xcall_cond(int cpu, smp_call_func_t func, void *info,
+ smp_cond_func_t condf, unsigned int flags);
+
+extern void smp_xcall_mask(const struct cpumask *mask, smp_call_func_t func,
+ void *info, unsigned int flags);
+
+extern void smp_xcall_mask_cond(const struct cpumask *mask, smp_call_func_t func,
+ void *info, smp_cond_func_t condf, unsigned int flags);
+
+extern int smp_xcall_private(int cpu, call_single_data_t *csd, unsigned int flags);
+
+extern int smp_xcall_any(const struct cpumask *mask, smp_call_func_t func,
+ void *info, unsigned int flags);
+
/*
* Enqueue a llist_node on the call_single_queue; be very careful, read
* flush_smp_call_function_queue() in detail.
diff --git a/kernel/smp.c b/kernel/smp.c
index b2b3878f0330..6183a3586329 100644
--- a/kernel/smp.c
+++ b/kernel/smp.c
@@ -1170,3 +1170,159 @@ int smp_call_on_cpu(unsigned int cpu, int (*func)(void *), void *par, bool phys)
return sscs.ret;
}
EXPORT_SYMBOL_GPL(smp_call_on_cpu);
+
+
+void __smp_call_mask_cond(const struct cpumask *mask,
+ smp_call_func_t func, void *info,
+ smp_cond_func_t cond_func,
+ unsigned int flags)
+{
+}
+
+/*
+ * smp_xcall Interface
+ *
+ * Consolidate the cross CPU call usage from the history below:
+ *
+ * Normally this interface cannot be used with interrupts disabled or
+ * from a hardware interrupt handler or from a bottom half handler.
+ * But there are two exceptions:
+ * 1) It can be used during early boot while early_boot_irqs_disabled
+ * is. In this scenario, you should use local_irq_save/restore()
+ * instead of local_irq_disable/enable()
+ * 2) Because smp_xcall_private(cpu, csd, XCALL_TYPE_ASYNC) is an asynchonous
+ * call with a preallocated csd structure, thus it can be called from
+ * the context where interrupts are disabled.
+ */
+
+/*
+ * Parameters:
+ *
+ * cpu: If cpu >=0 && cpu < nr_cpu_ids, the cross call is for that cpu.
+ * If cpu == -1, the cross call is for all the online CPUs
+ *
+ * func: It is the cross function that the destination CPUs need to execute.
+ * This function must be fast and non-blocking.
+ *
+ * info: It is the parameter to func().
+ *
+ * flags: The flags specify the manner the cross call is performaned in terms
+ * of synchronous or asynchronous.
+ *
+ * A synchronous cross call will not return immediately until all
+ * the destination CPUs have executed func() and responded the call.
+ *
+ * An asynchrouse cross call will return immediately as soon as it
+ * has fired all the cross calls and run func() locally if needed
+ * regardless the status of the target CPUs.
+ *
+ * Return: %0 on success or negative errno value on error.
+ */
+int smp_xcall(int cpu, smp_call_func_t func, void *info, unsigned int flags)
+{
+ return smp_xcall_cond(cpu, func, info, NULL, flags);
+}
+EXPORT_SYMBOL(smp_xcall);
+
+/*
+ * Parameters:
+ *
+ * cond_func: This is a condition function cond_func(cpu, info) invoked by
+ * the underlying cross call mechanism only. If the return value
+ * from cond_func(cpu, info) is true, the cross call will be sent
+ * to that cpu, otherwise the call will not be sent.
+ *
+ * Others: see smp_xcall().
+ *
+ * Return: %0 on success or negative errno value on error.
+ */
+int smp_xcall_cond(int cpu, smp_call_func_t func, void *info,
+ smp_cond_func_t cond_func, unsigned int flags)
+{
+ preempt_disable();
+ if (cpu == XCALL_ALL) {
+ __smp_call_mask_cond(cpu_online_mask, func, info, cond_func, flags);
+ } else if ((unsigned int)cpu < nr_cpu_ids)
+ __smp_call_mask_cond(cpumask_of(cpu), func, info, cond_func, flags);
+ else {
+ preempt_enable();
+ pr_warn("Invalid cpu ID = %d\n", cpu);
+ return -ENXIO;
+ }
+ preempt_enable();
+ return 0;
+}
+EXPORT_SYMBOL(smp_xcall_cond);
+
+/*
+ * Parameters:
+ *
+ * mask: This is the bitmap of CPUs to which the cross call will be sent.
+ *
+ * Others: see smp_xcall().
+ */
+void smp_xcall_mask(const struct cpumask *mask, smp_call_func_t func,
+ void *info, unsigned int flags)
+{
+ preempt_disable();
+ __smp_call_mask_cond(mask, func, info, NULL, flags);
+ preempt_enable();
+}
+EXPORT_SYMBOL(smp_xcall_mask);
+
+/*
+ * The combination of smp_xcall_cond() and smp_xcall_mask()
+ */
+void smp_xcall_mask_cond(const struct cpumask *mask,
+ smp_call_func_t func, void *info,
+ smp_cond_func_t cond_func,
+ unsigned int flags)
+{
+ preempt_disable();
+ __smp_call_mask_cond(mask, func, info, cond_func, flags);
+ preempt_enable();
+}
+EXPORT_SYMBOL(smp_xcall_mask_cond);
+
+/*
+ * This function provides an alternative way of sending a xcall call to
+ * only one CPU with a private csd instead of using the csd resource of
+ * the xcall. But it is the callers' responsibity to setup and maintain
+ * its private call_single_data_t struture.
+ *
+ * Because the call is asynchonous with a preallocated csd structure, thus
+ * it can be called from contexts with disabled interrupts.
+ *
+ * Parameters
+ *
+ * cpu: Must be a positive value less than nr_cpu_id.
+ * csd: The private csd provided by the caller.
+ *
+ * Others: see smp_xcall().
+ */
+int smp_xcall_private(int cpu, call_single_data_t *csd, unsigned int flags)
+{
+ return 0;
+}
+EXPORT_SYMBOL(smp_xcall_private);
+
+/*
+ * Parameters:
+ *
+ * mask: Run func() on one of the given CPUs in mask if it is oneline.
+ * CPU selection preference (from the original comments for
+ * smp_call_function_any()) :
+ * 1) current cpu if in @mask
+ * 2) any cpu of current node if in @mask
+ * 3) any other online cpu in @mask
+ *
+ * Others, see smp_xcall().
+ *
+ * Returns 0 on success, else a negative status code (if no cpus were online).
+ */
+int smp_xcall_any(const struct cpumask *mask, smp_call_func_t func,
+ void *info, unsigned int flags)
+{
+ return 0;
+}
+EXPORT_SYMBOL(smp_xcall_any);
--
2.27.0