Here are a few changes to minimize calls to stop_machine() and
flush_icache_*() in the various text patching functions, as well as
to simplify the code.
Samuel Holland (7):
riscv: jump_label: Batch icache maintenance
riscv: jump_label: Simplify assembly syntax
riscv: kprobes: Use patch_text_nosync() for insn slots
riscv: Simplify text patching loops
riscv: Pass patch_text() the length in bytes
riscv: Use offset_in_page() in text patching functions
riscv: Remove extra variable in patch_text_nosync()
arch/riscv/include/asm/jump_label.h | 4 ++-
arch/riscv/include/asm/patch.h | 3 +-
arch/riscv/kernel/jump_label.c | 16 ++++++---
arch/riscv/kernel/patch.c | 56 +++++++++++++----------------
arch/riscv/kernel/probes/kprobes.c | 20 ++++++-----
arch/riscv/net/bpf_jit_comp64.c | 7 ++--
6 files changed, 56 insertions(+), 50 deletions(-)
--
2.43.0
patch_text_nosync() already handles an arbitrary length of code, so this
removes a superfluous loop and reduces the number of icache flushes.
Signed-off-by: Samuel Holland <[email protected]>
---
arch/riscv/include/asm/patch.h | 2 +-
arch/riscv/kernel/patch.c | 15 +++++----------
arch/riscv/kernel/probes/kprobes.c | 20 +++++++++++---------
arch/riscv/net/bpf_jit_comp64.c | 7 ++++---
4 files changed, 21 insertions(+), 23 deletions(-)
diff --git a/arch/riscv/include/asm/patch.h b/arch/riscv/include/asm/patch.h
index 9f5d6e14c405..7228e266b9a1 100644
--- a/arch/riscv/include/asm/patch.h
+++ b/arch/riscv/include/asm/patch.h
@@ -9,7 +9,7 @@
int patch_insn_write(void *addr, const void *insn, size_t len);
int patch_text_nosync(void *addr, const void *insns, size_t len);
int patch_text_set_nosync(void *addr, u8 c, size_t len);
-int patch_text(void *addr, u32 *insns, int ninsns);
+int patch_text(void *addr, u32 *insns, size_t len);
extern int riscv_patch_in_stop_machine;
diff --git a/arch/riscv/kernel/patch.c b/arch/riscv/kernel/patch.c
index 7f030b46eae5..9aa0050225c0 100644
--- a/arch/riscv/kernel/patch.c
+++ b/arch/riscv/kernel/patch.c
@@ -19,7 +19,7 @@
struct patch_insn {
void *addr;
u32 *insns;
- int ninsns;
+ size_t len;
atomic_t cpu_count;
};
@@ -227,15 +227,10 @@ NOKPROBE_SYMBOL(patch_text_nosync);
static int patch_text_cb(void *data)
{
struct patch_insn *patch = data;
- unsigned long len;
- int i, ret = 0;
+ int ret = 0;
if (atomic_inc_return(&patch->cpu_count) == num_online_cpus()) {
- for (i = 0; ret == 0 && i < patch->ninsns; i++) {
- len = GET_INSN_LENGTH(patch->insns[i]);
- ret = patch_text_nosync(patch->addr + i * len,
- &patch->insns[i], len);
- }
+ ret = patch_text_nosync(patch->addr, patch->insns, patch->len);
atomic_inc(&patch->cpu_count);
} else {
while (atomic_read(&patch->cpu_count) <= num_online_cpus())
@@ -247,13 +242,13 @@ static int patch_text_cb(void *data)
}
NOKPROBE_SYMBOL(patch_text_cb);
-int patch_text(void *addr, u32 *insns, int ninsns)
+int patch_text(void *addr, u32 *insns, size_t len)
{
int ret;
struct patch_insn patch = {
.addr = addr,
.insns = insns,
- .ninsns = ninsns,
+ .len = len,
.cpu_count = ATOMIC_INIT(0),
};
diff --git a/arch/riscv/kernel/probes/kprobes.c b/arch/riscv/kernel/probes/kprobes.c
index cbf8197072bf..a64461fa715c 100644
--- a/arch/riscv/kernel/probes/kprobes.c
+++ b/arch/riscv/kernel/probes/kprobes.c
@@ -23,14 +23,14 @@ post_kprobe_handler(struct kprobe *, struct kprobe_ctlblk *, struct pt_regs *);
static void __kprobes arch_prepare_ss_slot(struct kprobe *p)
{
+ size_t len = GET_INSN_LENGTH(p->opcode);
u32 insn = __BUG_INSN_32;
- unsigned long offset = GET_INSN_LENGTH(p->opcode);
- p->ainsn.api.restore = (unsigned long)p->addr + offset;
+ p->ainsn.api.restore = (unsigned long)p->addr + len;
- patch_text_nosync(p->ainsn.api.insn, &p->opcode, 1);
- patch_text_nosync(p->ainsn.api.insn + offset,
- &insn, 1);
+ patch_text_nosync(p->ainsn.api.insn, &p->opcode, len);
+ patch_text_nosync(p->ainsn.api.insn + len,
+ &insn, GET_INSN_LENGTH(insn));
}
static void __kprobes arch_prepare_simulate(struct kprobe *p)
@@ -117,16 +117,18 @@ void *alloc_insn_page(void)
/* install breakpoint in text */
void __kprobes arch_arm_kprobe(struct kprobe *p)
{
- u32 insn = (p->opcode & __INSN_LENGTH_MASK) == __INSN_LENGTH_32 ?
- __BUG_INSN_32 : __BUG_INSN_16;
+ size_t len = GET_INSN_LENGTH(p->opcode);
+ u32 insn = len == 4 ? __BUG_INSN_32 : __BUG_INSN_16;
- patch_text(p->addr, &insn, 1);
+ patch_text(p->addr, &insn, len);
}
/* remove breakpoint from text */
void __kprobes arch_disarm_kprobe(struct kprobe *p)
{
- patch_text(p->addr, &p->opcode, 1);
+ size_t len = GET_INSN_LENGTH(p->opcode);
+
+ patch_text(p->addr, &p->opcode, len);
}
void __kprobes arch_remove_kprobe(struct kprobe *p)
diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c
index 719a97e7edb2..43be2585f0d4 100644
--- a/arch/riscv/net/bpf_jit_comp64.c
+++ b/arch/riscv/net/bpf_jit_comp64.c
@@ -14,6 +14,7 @@
#include "bpf_jit.h"
#define RV_FENTRY_NINSNS 2
+#define RV_FENTRY_NBYTES (RV_FENTRY_NINSNS * 4)
#define RV_REG_TCC RV_REG_A6
#define RV_REG_TCC_SAVED RV_REG_S6 /* Store A6 in S6 if program do calls */
@@ -681,7 +682,7 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type,
if (ret)
return ret;
- if (memcmp(ip, old_insns, RV_FENTRY_NINSNS * 4))
+ if (memcmp(ip, old_insns, RV_FENTRY_NBYTES))
return -EFAULT;
ret = gen_jump_or_nops(new_addr, ip, new_insns, is_call);
@@ -690,8 +691,8 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type,
cpus_read_lock();
mutex_lock(&text_mutex);
- if (memcmp(ip, new_insns, RV_FENTRY_NINSNS * 4))
- ret = patch_text(ip, new_insns, RV_FENTRY_NINSNS);
+ if (memcmp(ip, new_insns, RV_FENTRY_NBYTES))
+ ret = patch_text(ip, new_insns, RV_FENTRY_NBYTES);
mutex_unlock(&text_mutex);
cpus_read_unlock();
--
2.43.0
Switch to the batched version of the jump label update functions so
instruction cache maintenance is deferred until the end of the update.
Signed-off-by: Samuel Holland <[email protected]>
---
arch/riscv/include/asm/jump_label.h | 2 ++
arch/riscv/include/asm/patch.h | 1 +
arch/riscv/kernel/jump_label.c | 16 ++++++++++++----
arch/riscv/kernel/patch.c | 2 +-
4 files changed, 16 insertions(+), 5 deletions(-)
diff --git a/arch/riscv/include/asm/jump_label.h b/arch/riscv/include/asm/jump_label.h
index 4a35d787c019..6290b26f4a14 100644
--- a/arch/riscv/include/asm/jump_label.h
+++ b/arch/riscv/include/asm/jump_label.h
@@ -12,6 +12,8 @@
#include <linux/types.h>
#include <asm/asm.h>
+#define HAVE_JUMP_LABEL_BATCH
+
#define JUMP_LABEL_NOP_SIZE 4
static __always_inline bool arch_static_branch(struct static_key * const key,
diff --git a/arch/riscv/include/asm/patch.h b/arch/riscv/include/asm/patch.h
index e88b52d39eac..9f5d6e14c405 100644
--- a/arch/riscv/include/asm/patch.h
+++ b/arch/riscv/include/asm/patch.h
@@ -6,6 +6,7 @@
#ifndef _ASM_RISCV_PATCH_H
#define _ASM_RISCV_PATCH_H
+int patch_insn_write(void *addr, const void *insn, size_t len);
int patch_text_nosync(void *addr, const void *insns, size_t len);
int patch_text_set_nosync(void *addr, u8 c, size_t len);
int patch_text(void *addr, u32 *insns, int ninsns);
diff --git a/arch/riscv/kernel/jump_label.c b/arch/riscv/kernel/jump_label.c
index e6694759dbd0..11ad789c60c6 100644
--- a/arch/riscv/kernel/jump_label.c
+++ b/arch/riscv/kernel/jump_label.c
@@ -9,13 +9,14 @@
#include <linux/memory.h>
#include <linux/mutex.h>
#include <asm/bug.h>
+#include <asm/cacheflush.h>
#include <asm/patch.h>
#define RISCV_INSN_NOP 0x00000013U
#define RISCV_INSN_JAL 0x0000006fU
-void arch_jump_label_transform(struct jump_entry *entry,
- enum jump_label_type type)
+bool arch_jump_label_transform_queue(struct jump_entry *entry,
+ enum jump_label_type type)
{
void *addr = (void *)jump_entry_code(entry);
u32 insn;
@@ -24,7 +25,7 @@ void arch_jump_label_transform(struct jump_entry *entry,
long offset = jump_entry_target(entry) - jump_entry_code(entry);
if (WARN_ON(offset & 1 || offset < -524288 || offset >= 524288))
- return;
+ return true;
insn = RISCV_INSN_JAL |
(((u32)offset & GENMASK(19, 12)) << (12 - 12)) |
@@ -36,6 +37,13 @@ void arch_jump_label_transform(struct jump_entry *entry,
}
mutex_lock(&text_mutex);
- patch_text_nosync(addr, &insn, sizeof(insn));
+ patch_insn_write(addr, &insn, sizeof(insn));
mutex_unlock(&text_mutex);
+
+ return true;
+}
+
+void arch_jump_label_transform_apply(void)
+{
+ flush_icache_all();
}
diff --git a/arch/riscv/kernel/patch.c b/arch/riscv/kernel/patch.c
index 37e87fdcf6a0..bccd9ed04a05 100644
--- a/arch/riscv/kernel/patch.c
+++ b/arch/riscv/kernel/patch.c
@@ -188,7 +188,7 @@ int patch_text_set_nosync(void *addr, u8 c, size_t len)
}
NOKPROBE_SYMBOL(patch_text_set_nosync);
-static int patch_insn_write(void *addr, const void *insn, size_t len)
+int patch_insn_write(void *addr, const void *insn, size_t len)
{
size_t patched = 0;
size_t size;
--
2.43.0
This reduces the number of variables and makes the code easier to parse.
Signed-off-by: Samuel Holland <[email protected]>
---
arch/riscv/kernel/patch.c | 21 +++++++++++----------
1 file changed, 11 insertions(+), 10 deletions(-)
diff --git a/arch/riscv/kernel/patch.c b/arch/riscv/kernel/patch.c
index bccd9ed04a05..7f030b46eae5 100644
--- a/arch/riscv/kernel/patch.c
+++ b/arch/riscv/kernel/patch.c
@@ -155,7 +155,6 @@ NOKPROBE_SYMBOL(__patch_insn_write);
static int patch_insn_set(void *addr, u8 c, size_t len)
{
- size_t patched = 0;
size_t size;
int ret = 0;
@@ -163,11 +162,12 @@ static int patch_insn_set(void *addr, u8 c, size_t len)
* __patch_insn_set() can only work on 2 pages at a time so call it in a
* loop with len <= 2 * PAGE_SIZE.
*/
- while (patched < len && !ret) {
- size = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(addr + patched), len - patched);
- ret = __patch_insn_set(addr + patched, c, size);
+ while (len && !ret) {
+ size = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(addr), len);
+ ret = __patch_insn_set(addr, c, size);
- patched += size;
+ addr += size;
+ len -= size;
}
return ret;
@@ -190,7 +190,6 @@ NOKPROBE_SYMBOL(patch_text_set_nosync);
int patch_insn_write(void *addr, const void *insn, size_t len)
{
- size_t patched = 0;
size_t size;
int ret = 0;
@@ -198,11 +197,13 @@ int patch_insn_write(void *addr, const void *insn, size_t len)
* Copy the instructions to the destination address, two pages at a time
* because __patch_insn_write() can only handle len <= 2 * PAGE_SIZE.
*/
- while (patched < len && !ret) {
- size = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(addr + patched), len - patched);
- ret = __patch_insn_write(addr + patched, insn + patched, size);
+ while (len && !ret) {
+ size = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(addr), len);
+ ret = __patch_insn_write(addr, insn, size);
- patched += size;
+ addr += size;
+ insn += size;
+ len -= size;
}
return ret;
--
2.43.0
The idiomatic way to write "jal zero" is "j".
Signed-off-by: Samuel Holland <[email protected]>
---
arch/riscv/include/asm/jump_label.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/arch/riscv/include/asm/jump_label.h b/arch/riscv/include/asm/jump_label.h
index 6290b26f4a14..1c768d02bd0c 100644
--- a/arch/riscv/include/asm/jump_label.h
+++ b/arch/riscv/include/asm/jump_label.h
@@ -46,7 +46,7 @@ static __always_inline bool arch_static_branch_jump(struct static_key * const ke
" .option push \n\t"
" .option norelax \n\t"
" .option norvc \n\t"
- "1: jal zero, %l[label] \n\t"
+ "1: j %l[label] \n\t"
" .option pop \n\t"
" .pushsection __jump_table, \"aw\" \n\t"
" .align " RISCV_LGPTR " \n\t"
--
2.43.0
This cast is superfluous, and is incorrect anyway if compressed
instructions may be present.
Signed-off-by: Samuel Holland <[email protected]>
---
arch/riscv/kernel/patch.c | 12 ++++--------
1 file changed, 4 insertions(+), 8 deletions(-)
diff --git a/arch/riscv/kernel/patch.c b/arch/riscv/kernel/patch.c
index b0cf050738aa..80755aa627d2 100644
--- a/arch/riscv/kernel/patch.c
+++ b/arch/riscv/kernel/patch.c
@@ -176,13 +176,11 @@ NOKPROBE_SYMBOL(patch_insn_set);
int patch_text_set_nosync(void *addr, u8 c, size_t len)
{
- u32 *tp = addr;
int ret;
- ret = patch_insn_set(tp, c, len);
-
+ ret = patch_insn_set(addr, c, len);
if (!ret)
- flush_icache_range((uintptr_t)tp, (uintptr_t)tp + len);
+ flush_icache_range((uintptr_t)addr, (uintptr_t)addr + len);
return ret;
}
@@ -212,13 +210,11 @@ NOKPROBE_SYMBOL(patch_insn_write);
int patch_text_nosync(void *addr, const void *insns, size_t len)
{
- u32 *tp = addr;
int ret;
- ret = patch_insn_write(tp, insns, len);
-
+ ret = patch_insn_write(addr, insns, len);
if (!ret)
- flush_icache_range((uintptr_t) tp, (uintptr_t) tp + len);
+ flush_icache_range((uintptr_t)addr, (uintptr_t)addr + len);
return ret;
}
--
2.43.0
These instructions are not yet visible to the rest of the system,
so there is no need to do the whole stop_machine() dance.
Signed-off-by: Samuel Holland <[email protected]>
---
arch/riscv/kernel/probes/kprobes.c | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/arch/riscv/kernel/probes/kprobes.c b/arch/riscv/kernel/probes/kprobes.c
index 2f08c14a933d..cbf8197072bf 100644
--- a/arch/riscv/kernel/probes/kprobes.c
+++ b/arch/riscv/kernel/probes/kprobes.c
@@ -28,9 +28,9 @@ static void __kprobes arch_prepare_ss_slot(struct kprobe *p)
p->ainsn.api.restore = (unsigned long)p->addr + offset;
- patch_text(p->ainsn.api.insn, &p->opcode, 1);
- patch_text((void *)((unsigned long)(p->ainsn.api.insn) + offset),
- &insn, 1);
+ patch_text_nosync(p->ainsn.api.insn, &p->opcode, 1);
+ patch_text_nosync(p->ainsn.api.insn + offset,
+ &insn, 1);
}
static void __kprobes arch_prepare_simulate(struct kprobe *p)
--
2.43.0
This is a bit easier to parse than the equivalent bit manipulation.
Signed-off-by: Samuel Holland <[email protected]>
---
arch/riscv/kernel/patch.c | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/arch/riscv/kernel/patch.c b/arch/riscv/kernel/patch.c
index 9aa0050225c0..b0cf050738aa 100644
--- a/arch/riscv/kernel/patch.c
+++ b/arch/riscv/kernel/patch.c
@@ -54,7 +54,7 @@ static __always_inline void *patch_map(void *addr, const unsigned int fixmap)
BUG_ON(!page);
return (void *)set_fixmap_offset(fixmap, page_to_phys(page) +
- (uintaddr & ~PAGE_MASK));
+ offset_in_page(addr));
}
static void patch_unmap(int fixmap)
@@ -65,8 +65,8 @@ NOKPROBE_SYMBOL(patch_unmap);
static int __patch_insn_set(void *addr, u8 c, size_t len)
{
+ bool across_pages = (offset_in_page(addr) + len) > PAGE_SIZE;
void *waddr = addr;
- bool across_pages = (((uintptr_t)addr & ~PAGE_MASK) + len) > PAGE_SIZE;
/*
* Only two pages can be mapped at a time for writing.
@@ -98,8 +98,8 @@ NOKPROBE_SYMBOL(__patch_insn_set);
static int __patch_insn_write(void *addr, const void *insn, size_t len)
{
+ bool across_pages = (offset_in_page(addr) + len) > PAGE_SIZE;
void *waddr = addr;
- bool across_pages = (((uintptr_t) addr & ~PAGE_MASK) + len) > PAGE_SIZE;
int ret;
/*
--
2.43.0
Hi Samuel,
On Sun, Feb 11, 2024 at 06:55:11PM -0800, Samuel Holland wrote:
> Here are a few changes to minimize calls to stop_machine() and
> flush_icache_*() in the various text patching functions, as well as
> to simplify the code.
>
>
> Samuel Holland (7):
> riscv: jump_label: Batch icache maintenance
> riscv: jump_label: Simplify assembly syntax
> riscv: kprobes: Use patch_text_nosync() for insn slots
> riscv: Simplify text patching loops
> riscv: Pass patch_text() the length in bytes
> riscv: Use offset_in_page() in text patching functions
> riscv: Remove extra variable in patch_text_nosync()
This does look like a nice clean-up. Just curious (a "teach me"-like question),
how did you test these changes? kselftests, micro-benchmarks, other?
BTW, I recall a parallel work from Alex and Bjorn [1] that might have some minor
conflict with these changes; + both of them to Cc: for further sync.
Andrea
[1] https://lore.kernel.org/lkml/[email protected]/
Andrea Parri <[email protected]> writes:
> Hi Samuel,
>
> On Sun, Feb 11, 2024 at 06:55:11PM -0800, Samuel Holland wrote:
>> Here are a few changes to minimize calls to stop_machine() and
>> flush_icache_*() in the various text patching functions, as well as
>> to simplify the code.
>>
>>
>> Samuel Holland (7):
>> riscv: jump_label: Batch icache maintenance
>> riscv: jump_label: Simplify assembly syntax
>> riscv: kprobes: Use patch_text_nosync() for insn slots
>> riscv: Simplify text patching loops
>> riscv: Pass patch_text() the length in bytes
>> riscv: Use offset_in_page() in text patching functions
>> riscv: Remove extra variable in patch_text_nosync()
>
> This does look like a nice clean-up. Just curious (a "teach me"-like question),
> how did you test these changes? kselftests, micro-benchmarks, other?
>
> BTW, I recall a parallel work from Alex and Bjorn [1] that might have some minor
> conflict with these changes; + both of them to Cc: for further sync.
Indeed! I think Alex is still working on the v2.
Samuel Holland <[email protected]> writes:
> Switch to the batched version of the jump label update functions so
> instruction cache maintenance is deferred until the end of the update.
>
> Signed-off-by: Samuel Holland <[email protected]>
Reviewed-by: Björn Töpel <[email protected]>
Samuel Holland <[email protected]> writes:
> The idiomatic way to write "jal zero" is "j".
>
> Signed-off-by: Samuel Holland <[email protected]>
Reviewed-by: Björn Töpel <[email protected]>
Samuel Holland <[email protected]> writes:
> This reduces the number of variables and makes the code easier to parse.
>
> Signed-off-by: Samuel Holland <[email protected]>
> ---
>
> arch/riscv/kernel/patch.c | 21 +++++++++++----------
> 1 file changed, 11 insertions(+), 10 deletions(-)
>
> diff --git a/arch/riscv/kernel/patch.c b/arch/riscv/kernel/patch.c
> index bccd9ed04a05..7f030b46eae5 100644
> --- a/arch/riscv/kernel/patch.c
> +++ b/arch/riscv/kernel/patch.c
> @@ -155,7 +155,6 @@ NOKPROBE_SYMBOL(__patch_insn_write);
>
> static int patch_insn_set(void *addr, u8 c, size_t len)
> {
> - size_t patched = 0;
> size_t size;
> int ret = 0;
>
> @@ -163,11 +162,12 @@ static int patch_insn_set(void *addr, u8 c, size_t len)
> * __patch_insn_set() can only work on 2 pages at a time so call it in a
> * loop with len <= 2 * PAGE_SIZE.
> */
> - while (patched < len && !ret) {
> - size = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(addr + patched), len - patched);
> - ret = __patch_insn_set(addr + patched, c, size);
> + while (len && !ret) {
> + size = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(addr), len);
> + ret = __patch_insn_set(addr, c, size);
While you're at it, do:
ret = __patch_insn_set(addr, c, size);
if (ret)
return ret;
...
return 0;
}
and simplify the while-loop predicate?
>
> - patched += size;
> + addr += size;
> + len -= size;
> }
>
> return ret;
> @@ -190,7 +190,6 @@ NOKPROBE_SYMBOL(patch_text_set_nosync);
>
> int patch_insn_write(void *addr, const void *insn, size_t len)
> {
> - size_t patched = 0;
> size_t size;
> int ret = 0;
>
> @@ -198,11 +197,13 @@ int patch_insn_write(void *addr, const void *insn, size_t len)
> * Copy the instructions to the destination address, two pages at a time
> * because __patch_insn_write() can only handle len <= 2 * PAGE_SIZE.
> */
> - while (patched < len && !ret) {
> - size = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(addr + patched), len - patched);
> - ret = __patch_insn_write(addr + patched, insn + patched, size);
> + while (len && !ret) {
> + size = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(addr), len);
> + ret = __patch_insn_write(addr, insn, size);
Same comment as above.
Björn
Samuel Holland <[email protected]> writes:
> patch_text_nosync() already handles an arbitrary length of code, so this
> removes a superfluous loop and reduces the number of icache flushes.
>
> Signed-off-by: Samuel Holland <[email protected]>
Nice!
Nit: 100 chars lines, please.
Reviewed-by: Björn Töpel <[email protected]>
Samuel Holland <[email protected]> writes:
> These instructions are not yet visible to the rest of the system,
> so there is no need to do the whole stop_machine() dance.
>
> Signed-off-by: Samuel Holland <[email protected]>
> ---
>
> arch/riscv/kernel/probes/kprobes.c | 6 +++---
> 1 file changed, 3 insertions(+), 3 deletions(-)
>
> diff --git a/arch/riscv/kernel/probes/kprobes.c b/arch/riscv/kernel/probes/kprobes.c
> index 2f08c14a933d..cbf8197072bf 100644
> --- a/arch/riscv/kernel/probes/kprobes.c
> +++ b/arch/riscv/kernel/probes/kprobes.c
> @@ -28,9 +28,9 @@ static void __kprobes arch_prepare_ss_slot(struct kprobe *p)
>
> p->ainsn.api.restore = (unsigned long)p->addr + offset;
>
> - patch_text(p->ainsn.api.insn, &p->opcode, 1);
> - patch_text((void *)((unsigned long)(p->ainsn.api.insn) + offset),
> - &insn, 1);
> + patch_text_nosync(p->ainsn.api.insn, &p->opcode, 1);
> + patch_text_nosync(p->ainsn.api.insn + offset,
> + &insn, 1);
Nit: 100 chars lines!
Nice find!
Reviewed-by: Björn Töpel <[email protected]>
Samuel Holland <[email protected]> writes:
> This cast is superfluous, and is incorrect anyway if compressed
> instructions may be present.
>
> Signed-off-by: Samuel Holland <[email protected]>
Reviewed-by: Björn Töpel <[email protected]>
Samuel Holland <[email protected]> writes:
> This is a bit easier to parse than the equivalent bit manipulation.
>
> Signed-off-by: Samuel Holland <[email protected]>
Reviewed-by: Björn Töpel <[email protected]>
...
> > - while (patched < len && !ret) {
> > - size = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(addr + patched), len - patched);
> > - ret = __patch_insn_set(addr + patched, c, size);
> > + while (len && !ret) {
> > + size = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(addr), len);
Does that need to be min_t()?
Both arguments seem to be unsigned.
(Did it even ever need to be?)
David
-
Registered Address Lakeside, Bramley Road, Mount Farm, Milton Keynes, MK1 1PT, UK
Registration No: 1397386 (Wales)
On Mon, Feb 19, 2024 at 1:25 PM Björn Töpel <bjorn@kernelorg> wrote:
>
> Andrea Parri <[email protected]> writes:
>
> > Hi Samuel,
> >
> > On Sun, Feb 11, 2024 at 06:55:11PM -0800, Samuel Holland wrote:
> >> Here are a few changes to minimize calls to stop_machine() and
> >> flush_icache_*() in the various text patching functions, as well as
> >> to simplify the code.
> >>
> >>
> >> Samuel Holland (7):
> >> riscv: jump_label: Batch icache maintenance
> >> riscv: jump_label: Simplify assembly syntax
> >> riscv: kprobes: Use patch_text_nosync() for insn slots
> >> riscv: Simplify text patching loops
> >> riscv: Pass patch_text() the length in bytes
> >> riscv: Use offset_in_page() in text patching functions
> >> riscv: Remove extra variable in patch_text_nosync()
> >
> > This does look like a nice clean-up. Just curious (a "teach me"-like question),
> > how did you test these changes? kselftests, micro-benchmarks, other?
> >
> > BTW, I recall a parallel work from Alex and Bjorn [1] that might have some minor
> > conflict with these changes; + both of them to Cc: for further sync.
>
> Indeed! I think Alex is still working on the v2.
Actually I was blocked by Andrea's comment about patch_map(), but it's
not related so I can spin another version soon. I'd say mine should
land first because AIA support may get into 6.9 (?) and then this
patch would be needed. In case you re-spin another version, can you
rebase on top of it? Unless you have another solution of course.
Thanks,
Alex
Hi David,
On 2024-02-19 4:13 PM, David Laight wrote:
> ...
>>> - while (patched < len && !ret) {
>>> - size = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(addr + patched), len - patched);
>>> - ret = __patch_insn_set(addr + patched, c, size);
>>> + while (len && !ret) {
>>> + size = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(addr), len);
>
> Does that need to be min_t()?
> Both arguments seem to be unsigned.
> (Did it even ever need to be?)
You're right, this never needed min_t(). I'll update this for v2.
Regards,
Samuel
Hi Björn,
On 2024-02-19 7:03 AM, Björn Töpel wrote:
> Samuel Holland <[email protected]> writes:
>
>> This reduces the number of variables and makes the code easier to parse.
>>
>> Signed-off-by: Samuel Holland <[email protected]>
>> ---
>>
>> arch/riscv/kernel/patch.c | 21 +++++++++++----------
>> 1 file changed, 11 insertions(+), 10 deletions(-)
>>
>> diff --git a/arch/riscv/kernel/patch.c b/arch/riscv/kernel/patch.c
>> index bccd9ed04a05..7f030b46eae5 100644
>> --- a/arch/riscv/kernel/patch.c
>> +++ b/arch/riscv/kernel/patch.c
>> @@ -155,7 +155,6 @@ NOKPROBE_SYMBOL(__patch_insn_write);
>>
>> static int patch_insn_set(void *addr, u8 c, size_t len)
>> {
>> - size_t patched = 0;
>> size_t size;
>> int ret = 0;
>>
>> @@ -163,11 +162,12 @@ static int patch_insn_set(void *addr, u8 c, size_t len)
>> * __patch_insn_set() can only work on 2 pages at a time so call it in a
>> * loop with len <= 2 * PAGE_SIZE.
>> */
>> - while (patched < len && !ret) {
>> - size = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(addr + patched), len - patched);
>> - ret = __patch_insn_set(addr + patched, c, size);
>> + while (len && !ret) {
>> + size = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(addr), len);
>> + ret = __patch_insn_set(addr, c, size);
>
> While you're at it, do:
> ret = __patch_insn_set(addr, c, size);
> if (ret)
> return ret;
> ...
> return 0;
> }
>
> and simplify the while-loop predicate?
Yes, this looks better and reduces the code size as well. I'll make this change
in v2.
Regards,
Samuel
>>
>> - patched += size;
>> + addr += size;
>> + len -= size;
>> }
>>
>> return ret;
>> @@ -190,7 +190,6 @@ NOKPROBE_SYMBOL(patch_text_set_nosync);
>>
>> int patch_insn_write(void *addr, const void *insn, size_t len)
>> {
>> - size_t patched = 0;
>> size_t size;
>> int ret = 0;
>>
>> @@ -198,11 +197,13 @@ int patch_insn_write(void *addr, const void *insn, size_t len)
>> * Copy the instructions to the destination address, two pages at a time
>> * because __patch_insn_write() can only handle len <= 2 * PAGE_SIZE.
>> */
>> - while (patched < len && !ret) {
>> - size = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(addr + patched), len - patched);
>> - ret = __patch_insn_write(addr + patched, insn + patched, size);
>> + while (len && !ret) {
>> + size = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(addr), len);
>> + ret = __patch_insn_write(addr, insn, size);
>
> Same comment as above.
>
>
> Björn
Hi Andrea,
On 2024-02-13 6:58 AM, Andrea Parri wrote:
> On Sun, Feb 11, 2024 at 06:55:11PM -0800, Samuel Holland wrote:
>> Here are a few changes to minimize calls to stop_machine() and
>> flush_icache_*() in the various text patching functions, as well as
>> to simplify the code.
>>
>>
>> Samuel Holland (7):
>> riscv: jump_label: Batch icache maintenance
>> riscv: jump_label: Simplify assembly syntax
>> riscv: kprobes: Use patch_text_nosync() for insn slots
>> riscv: Simplify text patching loops
>> riscv: Pass patch_text() the length in bytes
>> riscv: Use offset_in_page() in text patching functions
>> riscv: Remove extra variable in patch_text_nosync()
>
> This does look like a nice clean-up. Just curious (a "teach me"-like question),
> how did you test these changes? kselftests, micro-benchmarks, other?
For all my patches, I do boot testing on various physical boards (Unmatched, D1,
some internal hardware), plus some standard internal workloads. For
performance-related patches, I run microbenchmarks or whole-system benchmarks
(e.g. UnixBench). For this series specifically, I didn't do any extra
benchmarking, as all of the functional changes should be as fast or faster by
virtue of simply doing less work.
> BTW, I recall a parallel work from Alex and Bjorn [1] that might have some minor
> conflict with these changes; + both of them to Cc: for further sync.
As suggested by Alex, v2 of this series will be based on the latest version of
that patch.
Regards,
Samuel
>
> Andrea
>
> [1] https://lore.kernel.org/lkml/[email protected]/