The bpf_seq_printf, bpf_trace_printk and bpf_snprintf helpers share one
per-cpu buffer that they use to store temporary data (arguments to
bprintf). They "get" that buffer with try_get_fmt_tmp_buf and "put" it
by the end of their scope with bpf_bprintf_cleanup.
If one of these helpers gets called within the scope of one of these
helpers, for example: a first bpf program gets called, uses
bpf_trace_printk which calls raw_spin_lock_irqsave which is traced by
another bpf program that calls bpf_snprintf, then the second "get"
fails. Essentially, these helpers are not re-entrant. They would return
-EBUSY and print a warning message once.
This patch triples the number of bprintf buffers to allow three levels
of nesting. This is very similar to what was done for tracepoints in
"9594dc3c7e7 bpf: fix nested bpf tracepoints with per-cpu data"
Fixes: d9c9e4db186a ("bpf: Factorize bpf_trace_printk and bpf_seq_printf")
Reported-by: [email protected]
Signed-off-by: Florent Revest <[email protected]>
---
kernel/bpf/helpers.c | 27 ++++++++++++++-------------
1 file changed, 14 insertions(+), 13 deletions(-)
diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c
index 544773970dbc..302410ebbea9 100644
--- a/kernel/bpf/helpers.c
+++ b/kernel/bpf/helpers.c
@@ -696,34 +696,35 @@ static int bpf_trace_copy_string(char *buf, void *unsafe_ptr, char fmt_ptype,
*/
#define MAX_PRINTF_BUF_LEN 512
-struct bpf_printf_buf {
- char tmp_buf[MAX_PRINTF_BUF_LEN];
+/* Support executing three nested bprintf helper calls on a given CPU */
+struct bpf_bprintf_buffers {
+ char tmp_bufs[3][MAX_PRINTF_BUF_LEN];
};
-static DEFINE_PER_CPU(struct bpf_printf_buf, bpf_printf_buf);
-static DEFINE_PER_CPU(int, bpf_printf_buf_used);
+static DEFINE_PER_CPU(struct bpf_bprintf_bufs, bpf_bprintf_bufs);
+static DEFINE_PER_CPU(int, bpf_bprintf_nest_level);
static int try_get_fmt_tmp_buf(char **tmp_buf)
{
- struct bpf_printf_buf *bufs;
- int used;
+ struct bpf_bprintf_buffers *bufs;
+ int nest_level;
preempt_disable();
- used = this_cpu_inc_return(bpf_printf_buf_used);
- if (WARN_ON_ONCE(used > 1)) {
- this_cpu_dec(bpf_printf_buf_used);
+ nest_level = this_cpu_inc_return(bpf_bprintf_nest_level);
+ if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(bufs->tmp_bufs))) {
+ this_cpu_dec(bpf_bprintf_nest_level);
preempt_enable();
return -EBUSY;
}
- bufs = this_cpu_ptr(&bpf_printf_buf);
- *tmp_buf = bufs->tmp_buf;
+ bufs = this_cpu_ptr(&bpf_bprintf_buf);
+ *tmp_buf = bufs->tmp_bufs[nest_level - 1];
return 0;
}
void bpf_bprintf_cleanup(void)
{
- if (this_cpu_read(bpf_printf_buf_used)) {
- this_cpu_dec(bpf_printf_buf_used);
+ if (this_cpu_read(bpf_bprintf_nest_level)) {
+ this_cpu_dec(bpf_bprintf_nest_level);
preempt_enable();
}
}
--
2.31.1.607.g51e8a6a459-goog
Hi Florent,
I love your patch! Yet something to improve:
[auto build test ERROR on bpf/master]
url: https://github.com/0day-ci/linux/commits/Florent-Revest/bpf-Fix-nested-bpf_bprintf_prepare-with-more-per-cpu-buffers/20210511-053835
base: https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf.git master
config: powerpc-randconfig-r021-20210510 (attached as .config)
compiler: clang version 13.0.0 (https://github.com/llvm/llvm-project a0fed635fe1701470062495a6ffee1c608f3f1bc)
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# install powerpc cross compiling tool for clang build
# apt-get install binutils-powerpc-linux-gnu
# https://github.com/0day-ci/linux/commit/1f5d45d9243d8ca8ece81e778579fc46a1946887
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Florent-Revest/bpf-Fix-nested-bpf_bprintf_prepare-with-more-per-cpu-buffers/20210511-053835
git checkout 1f5d45d9243d8ca8ece81e778579fc46a1946887
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross W=1 ARCH=powerpc
If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <[email protected]>
All errors (new ones prefixed by >>):
>> kernel/bpf/helpers.c:718:23: error: use of undeclared identifier 'bpf_bprintf_buf'; did you mean 'bpf_bprintf_bufs'?
bufs = this_cpu_ptr(&bpf_bprintf_buf);
^~~~~~~~~~~~~~~
bpf_bprintf_bufs
include/linux/percpu-defs.h:265:39: note: expanded from macro 'this_cpu_ptr'
#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
^
include/linux/percpu-defs.h:264:38: note: expanded from macro 'raw_cpu_ptr'
#define raw_cpu_ptr(ptr) per_cpu_ptr(ptr, 0)
^
include/linux/percpu-defs.h:263:65: note: expanded from macro 'per_cpu_ptr'
#define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); VERIFY_PERCPU_PTR(ptr); })
^
include/linux/percpu-defs.h:259:20: note: expanded from macro 'VERIFY_PERCPU_PTR'
__verify_pcpu_ptr(__p); \
^
include/linux/percpu-defs.h:219:47: note: expanded from macro '__verify_pcpu_ptr'
const void __percpu *__vpp_verify = (typeof((ptr) + 0))NULL; \
^
kernel/bpf/helpers.c:703:48: note: 'bpf_bprintf_bufs' declared here
static DEFINE_PER_CPU(struct bpf_bprintf_bufs, bpf_bprintf_bufs);
^
>> kernel/bpf/helpers.c:718:9: error: arithmetic on a pointer to an incomplete type 'typeof(struct bpf_bprintf_bufs)' (aka 'struct bpf_bprintf_bufs')
bufs = this_cpu_ptr(&bpf_bprintf_buf);
^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
include/linux/percpu-defs.h:265:27: note: expanded from macro 'this_cpu_ptr'
#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
^~~~~~~~~~~~~~~~
include/linux/percpu-defs.h:264:26: note: expanded from macro 'raw_cpu_ptr'
#define raw_cpu_ptr(ptr) per_cpu_ptr(ptr, 0)
^~~~~~~~~~~~~~~~~~~
include/linux/percpu-defs.h:263:47: note: expanded from macro 'per_cpu_ptr'
#define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); VERIFY_PERCPU_PTR(ptr); })
^~~~~~~~~~~~~~~~~~~~~~
include/linux/percpu-defs.h:259:2: note: expanded from macro 'VERIFY_PERCPU_PTR'
__verify_pcpu_ptr(__p); \
^~~~~~~~~~~~~~~~~~~~~~
include/linux/percpu-defs.h:219:52: note: expanded from macro '__verify_pcpu_ptr'
const void __percpu *__vpp_verify = (typeof((ptr) + 0))NULL; \
~~~~~ ^
kernel/bpf/helpers.c:703:30: note: forward declaration of 'struct bpf_bprintf_bufs'
static DEFINE_PER_CPU(struct bpf_bprintf_bufs, bpf_bprintf_bufs);
^
>> kernel/bpf/helpers.c:718:23: error: use of undeclared identifier 'bpf_bprintf_buf'; did you mean 'bpf_bprintf_bufs'?
bufs = this_cpu_ptr(&bpf_bprintf_buf);
^~~~~~~~~~~~~~~
bpf_bprintf_bufs
include/linux/percpu-defs.h:265:39: note: expanded from macro 'this_cpu_ptr'
#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
^
include/linux/percpu-defs.h:264:38: note: expanded from macro 'raw_cpu_ptr'
#define raw_cpu_ptr(ptr) per_cpu_ptr(ptr, 0)
^
include/linux/percpu-defs.h:263:65: note: expanded from macro 'per_cpu_ptr'
#define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); VERIFY_PERCPU_PTR(ptr); })
^
include/linux/percpu-defs.h:260:12: note: expanded from macro 'VERIFY_PERCPU_PTR'
(typeof(*(__p)) __kernel __force *)(__p); \
^
kernel/bpf/helpers.c:703:48: note: 'bpf_bprintf_bufs' declared here
static DEFINE_PER_CPU(struct bpf_bprintf_bufs, bpf_bprintf_bufs);
^
>> kernel/bpf/helpers.c:718:23: error: use of undeclared identifier 'bpf_bprintf_buf'; did you mean 'bpf_bprintf_bufs'?
bufs = this_cpu_ptr(&bpf_bprintf_buf);
^~~~~~~~~~~~~~~
bpf_bprintf_bufs
include/linux/percpu-defs.h:265:39: note: expanded from macro 'this_cpu_ptr'
#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
^
include/linux/percpu-defs.h:264:38: note: expanded from macro 'raw_cpu_ptr'
#define raw_cpu_ptr(ptr) per_cpu_ptr(ptr, 0)
^
include/linux/percpu-defs.h:263:65: note: expanded from macro 'per_cpu_ptr'
#define per_cpu_ptr(ptr, cpu) ({ (void)(cpu); VERIFY_PERCPU_PTR(ptr); })
^
include/linux/percpu-defs.h:260:38: note: expanded from macro 'VERIFY_PERCPU_PTR'
(typeof(*(__p)) __kernel __force *)(__p); \
^
kernel/bpf/helpers.c:703:48: note: 'bpf_bprintf_bufs' declared here
static DEFINE_PER_CPU(struct bpf_bprintf_bufs, bpf_bprintf_bufs);
^
>> kernel/bpf/helpers.c:718:7: error: incompatible pointer types assigning to 'struct bpf_bprintf_buffers *' from 'typeof (*(&bpf_bprintf_bufs)) *' (aka 'struct bpf_bprintf_bufs *') [-Werror,-Wincompatible-pointer-types]
bufs = this_cpu_ptr(&bpf_bprintf_buf);
^ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
>> kernel/bpf/helpers.c:703:48: error: tentative definition has type 'typeof(struct bpf_bprintf_bufs)' (aka 'struct bpf_bprintf_bufs') that is never completed
static DEFINE_PER_CPU(struct bpf_bprintf_bufs, bpf_bprintf_bufs);
^
kernel/bpf/helpers.c:703:30: note: forward declaration of 'struct bpf_bprintf_bufs'
static DEFINE_PER_CPU(struct bpf_bprintf_bufs, bpf_bprintf_bufs);
^
6 errors generated.
vim +718 kernel/bpf/helpers.c
698
699 /* Support executing three nested bprintf helper calls on a given CPU */
700 struct bpf_bprintf_buffers {
701 char tmp_bufs[3][MAX_PRINTF_BUF_LEN];
702 };
> 703 static DEFINE_PER_CPU(struct bpf_bprintf_bufs, bpf_bprintf_bufs);
704 static DEFINE_PER_CPU(int, bpf_bprintf_nest_level);
705
706 static int try_get_fmt_tmp_buf(char **tmp_buf)
707 {
708 struct bpf_bprintf_buffers *bufs;
709 int nest_level;
710
711 preempt_disable();
712 nest_level = this_cpu_inc_return(bpf_bprintf_nest_level);
713 if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(bufs->tmp_bufs))) {
714 this_cpu_dec(bpf_bprintf_nest_level);
715 preempt_enable();
716 return -EBUSY;
717 }
> 718 bufs = this_cpu_ptr(&bpf_bprintf_buf);
719 *tmp_buf = bufs->tmp_bufs[nest_level - 1];
720
721 return 0;
722 }
723
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/[email protected]
Hi Florent,
I love your patch! Yet something to improve:
[auto build test ERROR on bpf/master]
url: https://github.com/0day-ci/linux/commits/Florent-Revest/bpf-Fix-nested-bpf_bprintf_prepare-with-more-per-cpu-buffers/20210511-053835
base: https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf.git master
config: s390-randconfig-r013-20210510 (attached as .config)
compiler: clang version 13.0.0 (https://github.com/llvm/llvm-project a0fed635fe1701470062495a6ffee1c608f3f1bc)
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# install s390 cross compiling tool for clang build
# apt-get install binutils-s390x-linux-gnu
# https://github.com/0day-ci/linux/commit/1f5d45d9243d8ca8ece81e778579fc46a1946887
git remote add linux-review https://github.com/0day-ci/linux
git fetch --no-tags linux-review Florent-Revest/bpf-Fix-nested-bpf_bprintf_prepare-with-more-per-cpu-buffers/20210511-053835
git checkout 1f5d45d9243d8ca8ece81e778579fc46a1946887
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross W=1 ARCH=s390
If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <[email protected]>
All errors (new ones prefixed by >>):
include/linux/percpu-defs.h:252:27: note: expanded from macro 'this_cpu_ptr'
#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
^~~~~~~~~~~~~~~~
include/linux/percpu-defs.h:241:2: note: expanded from macro 'raw_cpu_ptr'
__verify_pcpu_ptr(ptr); \
^~~~~~~~~~~~~~~~~~~~~~
include/linux/percpu-defs.h:219:52: note: expanded from macro '__verify_pcpu_ptr'
const void __percpu *__vpp_verify = (typeof((ptr) + 0))NULL; \
~~~~~ ^
kernel/bpf/helpers.c:703:30: note: forward declaration of 'struct bpf_bprintf_bufs'
static DEFINE_PER_CPU(struct bpf_bprintf_bufs, bpf_bprintf_bufs);
^
kernel/bpf/helpers.c:718:23: error: use of undeclared identifier 'bpf_bprintf_buf'; did you mean 'bpf_bprintf_bufs'?
bufs = this_cpu_ptr(&bpf_bprintf_buf);
^~~~~~~~~~~~~~~
bpf_bprintf_bufs
include/linux/percpu-defs.h:252:39: note: expanded from macro 'this_cpu_ptr'
#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
^
include/linux/percpu-defs.h:242:19: note: expanded from macro 'raw_cpu_ptr'
arch_raw_cpu_ptr(ptr); \
^
include/asm-generic/percpu.h:44:48: note: expanded from macro 'arch_raw_cpu_ptr'
#define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
^
include/linux/percpu-defs.h:231:23: note: expanded from macro 'SHIFT_PERCPU_PTR'
RELOC_HIDE((typeof(*(__p)) __kernel __force *)(__p), (__offset))
^
include/linux/compiler.h:181:31: note: expanded from macro 'RELOC_HIDE'
__ptr = (unsigned long) (ptr); \
^
kernel/bpf/helpers.c:703:48: note: 'bpf_bprintf_bufs' declared here
static DEFINE_PER_CPU(struct bpf_bprintf_bufs, bpf_bprintf_bufs);
^
kernel/bpf/helpers.c:718:23: error: use of undeclared identifier 'bpf_bprintf_buf'; did you mean 'bpf_bprintf_bufs'?
bufs = this_cpu_ptr(&bpf_bprintf_buf);
^~~~~~~~~~~~~~~
bpf_bprintf_bufs
include/linux/percpu-defs.h:252:39: note: expanded from macro 'this_cpu_ptr'
#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
^
include/linux/percpu-defs.h:242:19: note: expanded from macro 'raw_cpu_ptr'
arch_raw_cpu_ptr(ptr); \
^
include/asm-generic/percpu.h:44:48: note: expanded from macro 'arch_raw_cpu_ptr'
#define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
^
include/linux/percpu-defs.h:231:49: note: expanded from macro 'SHIFT_PERCPU_PTR'
RELOC_HIDE((typeof(*(__p)) __kernel __force *)(__p), (__offset))
^
include/linux/compiler.h:181:31: note: expanded from macro 'RELOC_HIDE'
__ptr = (unsigned long) (ptr); \
^
kernel/bpf/helpers.c:703:48: note: 'bpf_bprintf_bufs' declared here
static DEFINE_PER_CPU(struct bpf_bprintf_bufs, bpf_bprintf_bufs);
^
kernel/bpf/helpers.c:718:23: error: use of undeclared identifier 'bpf_bprintf_buf'; did you mean 'bpf_bprintf_bufs'?
bufs = this_cpu_ptr(&bpf_bprintf_buf);
^~~~~~~~~~~~~~~
bpf_bprintf_bufs
include/linux/percpu-defs.h:252:39: note: expanded from macro 'this_cpu_ptr'
#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
^
include/linux/percpu-defs.h:242:19: note: expanded from macro 'raw_cpu_ptr'
arch_raw_cpu_ptr(ptr); \
^
include/asm-generic/percpu.h:44:48: note: expanded from macro 'arch_raw_cpu_ptr'
#define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
^
include/linux/percpu-defs.h:231:23: note: expanded from macro 'SHIFT_PERCPU_PTR'
RELOC_HIDE((typeof(*(__p)) __kernel __force *)(__p), (__offset))
^
include/linux/compiler.h:182:13: note: expanded from macro 'RELOC_HIDE'
(typeof(ptr)) (__ptr + (off)); })
^
kernel/bpf/helpers.c:703:48: note: 'bpf_bprintf_bufs' declared here
static DEFINE_PER_CPU(struct bpf_bprintf_bufs, bpf_bprintf_bufs);
^
kernel/bpf/helpers.c:718:23: error: use of undeclared identifier 'bpf_bprintf_buf'; did you mean 'bpf_bprintf_bufs'?
bufs = this_cpu_ptr(&bpf_bprintf_buf);
^~~~~~~~~~~~~~~
bpf_bprintf_bufs
include/linux/percpu-defs.h:252:39: note: expanded from macro 'this_cpu_ptr'
#define this_cpu_ptr(ptr) raw_cpu_ptr(ptr)
^
include/linux/percpu-defs.h:242:19: note: expanded from macro 'raw_cpu_ptr'
arch_raw_cpu_ptr(ptr); \
^
include/asm-generic/percpu.h:44:48: note: expanded from macro 'arch_raw_cpu_ptr'
#define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
^
include/linux/percpu-defs.h:231:49: note: expanded from macro 'SHIFT_PERCPU_PTR'
RELOC_HIDE((typeof(*(__p)) __kernel __force *)(__p), (__offset))
^
include/linux/compiler.h:182:13: note: expanded from macro 'RELOC_HIDE'
(typeof(ptr)) (__ptr + (off)); })
^
kernel/bpf/helpers.c:703:48: note: 'bpf_bprintf_bufs' declared here
static DEFINE_PER_CPU(struct bpf_bprintf_bufs, bpf_bprintf_bufs);
^
>> kernel/bpf/helpers.c:718:7: error: incompatible pointer types assigning to 'struct bpf_bprintf_buffers *' from 'typeof ((typeof (*(&bpf_bprintf_bufs)) *)(&bpf_bprintf_bufs))' (aka 'struct bpf_bprintf_bufs *') [-Werror,-Wincompatible-pointer-types]
bufs = this_cpu_ptr(&bpf_bprintf_buf);
^ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
kernel/bpf/helpers.c:703:48: error: tentative definition has type 'typeof(struct bpf_bprintf_bufs)' (aka 'struct bpf_bprintf_bufs') that is never completed
static DEFINE_PER_CPU(struct bpf_bprintf_bufs, bpf_bprintf_bufs);
^
kernel/bpf/helpers.c:703:30: note: forward declaration of 'struct bpf_bprintf_bufs'
static DEFINE_PER_CPU(struct bpf_bprintf_bufs, bpf_bprintf_bufs);
^
12 warnings and 8 errors generated.
vim +718 kernel/bpf/helpers.c
705
706 static int try_get_fmt_tmp_buf(char **tmp_buf)
707 {
708 struct bpf_bprintf_buffers *bufs;
709 int nest_level;
710
711 preempt_disable();
712 nest_level = this_cpu_inc_return(bpf_bprintf_nest_level);
713 if (WARN_ON_ONCE(nest_level > ARRAY_SIZE(bufs->tmp_bufs))) {
714 this_cpu_dec(bpf_bprintf_nest_level);
715 preempt_enable();
716 return -EBUSY;
717 }
> 718 bufs = this_cpu_ptr(&bpf_bprintf_buf);
719 *tmp_buf = bufs->tmp_bufs[nest_level - 1];
720
721 return 0;
722 }
723
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/[email protected]
On Tue, May 11, 2021 at 3:58 AM kernel test robot <[email protected]> wrote:
> kernel/bpf/helpers.c:718:23: error: use of undeclared identifier 'bpf_bprintf_buf'; did you mean 'bpf_bprintf_bufs'?
Ugh, reminder to self: don't zealously rename variables just before
sending a patch out.
I'm sending a v2, sorry for the noise.