This avoids another inderect call per RX packet which save us around
20-40 ns.
Signed-off-by: Brian Vazquez <[email protected]>
---
net/core/fib_rules.c | 32 ++++++++++++++++++++++++++++----
net/ipv4/fib_rules.c | 12 ++++++++----
net/ipv6/fib6_rules.c | 12 ++++++++----
3 files changed, 44 insertions(+), 12 deletions(-)
diff --git a/net/core/fib_rules.c b/net/core/fib_rules.c
index bd7eba9066f8d..ceef012dd0e65 100644
--- a/net/core/fib_rules.c
+++ b/net/core/fib_rules.c
@@ -14,6 +14,7 @@
#include <net/sock.h>
#include <net/fib_rules.h>
#include <net/ip_tunnels.h>
+#include <linux/indirect_call_wrapper.h>
static const struct fib_kuid_range fib_kuid_range_unset = {
KUIDT_INIT(0),
@@ -242,6 +243,10 @@ static int nla_put_port_range(struct sk_buff *skb, int attrtype,
return nla_put(skb, attrtype, sizeof(*range), range);
}
+INDIRECT_CALLABLE_DECLARE(int fib6_rule_match(struct fib_rule *rule,
+ struct flowi *fl, int flags));
+INDIRECT_CALLABLE_DECLARE(int fib4_rule_match(struct fib_rule *rule,
+ struct flowi *fl, int flags));
static int fib_rule_match(struct fib_rule *rule, struct fib_rules_ops *ops,
struct flowi *fl, int flags,
struct fib_lookup_arg *arg)
@@ -267,11 +272,24 @@ static int fib_rule_match(struct fib_rule *rule, struct fib_rules_ops *ops,
uid_gt(fl->flowi_uid, rule->uid_range.end))
goto out;
- ret = ops->match(rule, fl, flags);
+ ret = INDIRECT_CALL_INET(ops->match,
+ fib6_rule_match,
+ fib4_rule_match,
+ rule, fl, flags);
out:
return (rule->flags & FIB_RULE_INVERT) ? !ret : ret;
}
+INDIRECT_CALLABLE_DECLARE(int fib6_rule_action(struct fib_rule *rule,
+ struct flowi *flp, int flags,
+ struct fib_lookup_arg *arg));
+INDIRECT_CALLABLE_DECLARE(int fib4_rule_action(struct fib_rule *rule,
+ struct flowi *flp, int flags,
+ struct fib_lookup_arg *arg));
+INDIRECT_CALLABLE_DECLARE(bool fib6_rule_suppress(struct fib_rule *rule,
+ struct fib_lookup_arg *arg));
+INDIRECT_CALLABLE_DECLARE(bool fib4_rule_suppress(struct fib_rule *rule,
+ struct fib_lookup_arg *arg));
int fib_rules_lookup(struct fib_rules_ops *ops, struct flowi *fl,
int flags, struct fib_lookup_arg *arg)
{
@@ -298,9 +316,15 @@ int fib_rules_lookup(struct fib_rules_ops *ops, struct flowi *fl,
} else if (rule->action == FR_ACT_NOP)
continue;
else
- err = ops->action(rule, fl, flags, arg);
-
- if (!err && ops->suppress && ops->suppress(rule, arg))
+ err = INDIRECT_CALL_INET(ops->action,
+ fib6_rule_action,
+ fib4_rule_action,
+ rule, fl, flags, arg);
+
+ if (!err && ops->suppress && INDIRECT_CALL_INET(ops->suppress,
+ fib6_rule_suppress,
+ fib4_rule_suppress,
+ rule, arg))
continue;
if (err != -EAGAIN) {
diff --git a/net/ipv4/fib_rules.c b/net/ipv4/fib_rules.c
index f99e3bac5cab2..fd3def3ffa6df 100644
--- a/net/ipv4/fib_rules.c
+++ b/net/ipv4/fib_rules.c
@@ -29,6 +29,7 @@
#include <net/ip_fib.h>
#include <net/nexthop.h>
#include <net/fib_rules.h>
+#include <linux/indirect_call_wrapper.h>
struct fib4_rule {
struct fib_rule common;
@@ -103,8 +104,9 @@ int __fib_lookup(struct net *net, struct flowi4 *flp,
}
EXPORT_SYMBOL_GPL(__fib_lookup);
-static int fib4_rule_action(struct fib_rule *rule, struct flowi *flp,
- int flags, struct fib_lookup_arg *arg)
+INDIRECT_CALLABLE_SCOPE int fib4_rule_action(struct fib_rule *rule,
+ struct flowi *flp, int flags,
+ struct fib_lookup_arg *arg)
{
int err = -EAGAIN;
struct fib_table *tbl;
@@ -138,7 +140,8 @@ static int fib4_rule_action(struct fib_rule *rule, struct flowi *flp,
return err;
}
-static bool fib4_rule_suppress(struct fib_rule *rule, struct fib_lookup_arg *arg)
+INDIRECT_CALLABLE_SCOPE bool fib4_rule_suppress(struct fib_rule *rule,
+ struct fib_lookup_arg *arg)
{
struct fib_result *result = (struct fib_result *) arg->result;
struct net_device *dev = NULL;
@@ -169,7 +172,8 @@ static bool fib4_rule_suppress(struct fib_rule *rule, struct fib_lookup_arg *arg
return true;
}
-static int fib4_rule_match(struct fib_rule *rule, struct flowi *fl, int flags)
+INDIRECT_CALLABLE_SCOPE int fib4_rule_match(struct fib_rule *rule,
+ struct flowi *fl, int flags)
{
struct fib4_rule *r = (struct fib4_rule *) rule;
struct flowi4 *fl4 = &fl->u.ip4;
diff --git a/net/ipv6/fib6_rules.c b/net/ipv6/fib6_rules.c
index 6053ef8515555..fb4db803a2531 100644
--- a/net/ipv6/fib6_rules.c
+++ b/net/ipv6/fib6_rules.c
@@ -13,6 +13,7 @@
#include <linux/netdevice.h>
#include <linux/notifier.h>
#include <linux/export.h>
+#include <linux/indirect_call_wrapper.h>
#include <net/fib_rules.h>
#include <net/ipv6.h>
@@ -255,8 +256,9 @@ static int __fib6_rule_action(struct fib_rule *rule, struct flowi *flp,
return err;
}
-static int fib6_rule_action(struct fib_rule *rule, struct flowi *flp,
- int flags, struct fib_lookup_arg *arg)
+INDIRECT_CALLABLE_SCOPE int fib6_rule_action(struct fib_rule *rule,
+ struct flowi *flp, int flags,
+ struct fib_lookup_arg *arg)
{
if (arg->lookup_ptr == fib6_table_lookup)
return fib6_rule_action_alt(rule, flp, flags, arg);
@@ -264,7 +266,8 @@ static int fib6_rule_action(struct fib_rule *rule, struct flowi *flp,
return __fib6_rule_action(rule, flp, flags, arg);
}
-static bool fib6_rule_suppress(struct fib_rule *rule, struct fib_lookup_arg *arg)
+INDIRECT_CALLABLE_SCOPE bool fib6_rule_suppress(struct fib_rule *rule,
+ struct fib_lookup_arg *arg)
{
struct fib6_result *res = arg->result;
struct rt6_info *rt = res->rt6;
@@ -296,7 +299,8 @@ static bool fib6_rule_suppress(struct fib_rule *rule, struct fib_lookup_arg *arg
return true;
}
-static int fib6_rule_match(struct fib_rule *rule, struct flowi *fl, int flags)
+INDIRECT_CALLABLE_SCOPE int fib6_rule_match(struct fib_rule *rule,
+ struct flowi *fl, int flags)
{
struct fib6_rule *r = (struct fib6_rule *) rule;
struct flowi6 *fl6 = &fl->u.ip6;
--
2.28.0.rc0.142.g3c755180ce-goog
Hi Brian,
Thank you for the patch! Yet something to improve:
[auto build test ERROR on net-next/master]
url: https://github.com/0day-ci/linux/commits/Brian-Vazquez/fib-use-indirect-call-wrappers-in-the-most-common-fib_rules_ops/20200725-095008
base: https://git.kernel.org/pub/scm/linux/kernel/git/davem/net-next.git dfd3d5266dc1d9a2b06e5a09bbff4cee547eeb5f
config: i386-randconfig-a011-20200724 (attached as .config)
compiler: gcc-9 (Debian 9.3.0-14) 9.3.0
reproduce (this is a W=1 build):
# save the attached .config to linux build tree
make W=1 ARCH=i386
If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <[email protected]>
All errors (new ones prefixed by >>):
ld: net/core/fib_rules.o: in function `fib_rule_match':
>> net/core/fib_rules.c:275: undefined reference to `fib6_rule_match'
>> ld: net/core/fib_rules.c:275: undefined reference to `fib6_rule_match'
ld: net/core/fib_rules.c:275: undefined reference to `fib4_rule_match'
ld: net/core/fib_rules.c:275: undefined reference to `fib4_rule_match'
ld: net/core/fib_rules.o: in function `fib_rules_lookup':
>> net/core/fib_rules.c:319: undefined reference to `fib6_rule_action'
>> ld: net/core/fib_rules.c:319: undefined reference to `fib6_rule_action'
ld: net/core/fib_rules.c:319: undefined reference to `fib4_rule_action'
ld: net/core/fib_rules.c:319: undefined reference to `fib4_rule_action'
>> ld: net/core/fib_rules.c:324: undefined reference to `fib6_rule_suppress'
>> ld: net/core/fib_rules.c:324: undefined reference to `fib6_rule_suppress'
ld: net/core/fib_rules.c:324: undefined reference to `fib4_rule_suppress'
ld: net/core/fib_rules.c:324: undefined reference to `fib4_rule_suppress'
vim +275 net/core/fib_rules.c
245
246 INDIRECT_CALLABLE_DECLARE(int fib6_rule_match(struct fib_rule *rule,
247 struct flowi *fl, int flags));
248 INDIRECT_CALLABLE_DECLARE(int fib4_rule_match(struct fib_rule *rule,
249 struct flowi *fl, int flags));
250 static int fib_rule_match(struct fib_rule *rule, struct fib_rules_ops *ops,
251 struct flowi *fl, int flags,
252 struct fib_lookup_arg *arg)
253 {
254 int ret = 0;
255
256 if (rule->iifindex && (rule->iifindex != fl->flowi_iif))
257 goto out;
258
259 if (rule->oifindex && (rule->oifindex != fl->flowi_oif))
260 goto out;
261
262 if ((rule->mark ^ fl->flowi_mark) & rule->mark_mask)
263 goto out;
264
265 if (rule->tun_id && (rule->tun_id != fl->flowi_tun_key.tun_id))
266 goto out;
267
268 if (rule->l3mdev && !l3mdev_fib_rule_match(rule->fr_net, fl, arg))
269 goto out;
270
271 if (uid_lt(fl->flowi_uid, rule->uid_range.start) ||
272 uid_gt(fl->flowi_uid, rule->uid_range.end))
273 goto out;
274
> 275 ret = INDIRECT_CALL_INET(ops->match,
276 fib6_rule_match,
277 fib4_rule_match,
278 rule, fl, flags);
279 out:
280 return (rule->flags & FIB_RULE_INVERT) ? !ret : ret;
281 }
282
283 INDIRECT_CALLABLE_DECLARE(int fib6_rule_action(struct fib_rule *rule,
284 struct flowi *flp, int flags,
285 struct fib_lookup_arg *arg));
286 INDIRECT_CALLABLE_DECLARE(int fib4_rule_action(struct fib_rule *rule,
287 struct flowi *flp, int flags,
288 struct fib_lookup_arg *arg));
289 INDIRECT_CALLABLE_DECLARE(bool fib6_rule_suppress(struct fib_rule *rule,
290 struct fib_lookup_arg *arg));
291 INDIRECT_CALLABLE_DECLARE(bool fib4_rule_suppress(struct fib_rule *rule,
292 struct fib_lookup_arg *arg));
293 int fib_rules_lookup(struct fib_rules_ops *ops, struct flowi *fl,
294 int flags, struct fib_lookup_arg *arg)
295 {
296 struct fib_rule *rule;
297 int err;
298
299 rcu_read_lock();
300
301 list_for_each_entry_rcu(rule, &ops->rules_list, list) {
302 jumped:
303 if (!fib_rule_match(rule, ops, fl, flags, arg))
304 continue;
305
306 if (rule->action == FR_ACT_GOTO) {
307 struct fib_rule *target;
308
309 target = rcu_dereference(rule->ctarget);
310 if (target == NULL) {
311 continue;
312 } else {
313 rule = target;
314 goto jumped;
315 }
316 } else if (rule->action == FR_ACT_NOP)
317 continue;
318 else
> 319 err = INDIRECT_CALL_INET(ops->action,
320 fib6_rule_action,
321 fib4_rule_action,
322 rule, fl, flags, arg);
323
> 324 if (!err && ops->suppress && INDIRECT_CALL_INET(ops->suppress,
325 fib6_rule_suppress,
326 fib4_rule_suppress,
327 rule, arg))
328 continue;
329
330 if (err != -EAGAIN) {
331 if ((arg->flags & FIB_LOOKUP_NOREF) ||
332 likely(refcount_inc_not_zero(&rule->refcnt))) {
333 arg->rule = rule;
334 goto out;
335 }
336 break;
337 }
338 }
339
340 err = -ESRCH;
341 out:
342 rcu_read_unlock();
343
344 return err;
345 }
346 EXPORT_SYMBOL_GPL(fib_rules_lookup);
347
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/[email protected]
Hi Brian,
Thank you for the patch! Perhaps something to improve:
[auto build test WARNING on net-next/master]
url: https://github.com/0day-ci/linux/commits/Brian-Vazquez/fib-use-indirect-call-wrappers-in-the-most-common-fib_rules_ops/20200725-095008
base: https://git.kernel.org/pub/scm/linux/kernel/git/davem/net-next.git dfd3d5266dc1d9a2b06e5a09bbff4cee547eeb5f
config: x86_64-allyesconfig (attached as .config)
compiler: clang version 12.0.0 (https://github.com/llvm/llvm-project 8bf4c1f4fb257774f66c8cda07adc6c5e8668326)
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# install x86_64 cross compiling tool for clang build
# apt-get install binutils-x86-64-linux-gnu
# save the attached .config to linux build tree
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=clang make.cross ARCH=x86_64
If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <[email protected]>
All warnings (new ones prefixed by >>):
>> net/ipv4/fib_rules.c:107:29: warning: no previous prototype for function 'fib4_rule_action' [-Wmissing-prototypes]
INDIRECT_CALLABLE_SCOPE int fib4_rule_action(struct fib_rule *rule,
^
net/ipv4/fib_rules.c:107:25: note: declare 'static' if the function is not intended to be used outside of this translation unit
INDIRECT_CALLABLE_SCOPE int fib4_rule_action(struct fib_rule *rule,
^
static
>> net/ipv4/fib_rules.c:143:30: warning: no previous prototype for function 'fib4_rule_suppress' [-Wmissing-prototypes]
INDIRECT_CALLABLE_SCOPE bool fib4_rule_suppress(struct fib_rule *rule,
^
net/ipv4/fib_rules.c:143:25: note: declare 'static' if the function is not intended to be used outside of this translation unit
INDIRECT_CALLABLE_SCOPE bool fib4_rule_suppress(struct fib_rule *rule,
^
static
>> net/ipv4/fib_rules.c:175:29: warning: no previous prototype for function 'fib4_rule_match' [-Wmissing-prototypes]
INDIRECT_CALLABLE_SCOPE int fib4_rule_match(struct fib_rule *rule,
^
net/ipv4/fib_rules.c:175:25: note: declare 'static' if the function is not intended to be used outside of this translation unit
INDIRECT_CALLABLE_SCOPE int fib4_rule_match(struct fib_rule *rule,
^
static
3 warnings generated.
--
>> net/ipv6/fib6_rules.c:259:29: warning: no previous prototype for function 'fib6_rule_action' [-Wmissing-prototypes]
INDIRECT_CALLABLE_SCOPE int fib6_rule_action(struct fib_rule *rule,
^
net/ipv6/fib6_rules.c:259:25: note: declare 'static' if the function is not intended to be used outside of this translation unit
INDIRECT_CALLABLE_SCOPE int fib6_rule_action(struct fib_rule *rule,
^
static
>> net/ipv6/fib6_rules.c:269:30: warning: no previous prototype for function 'fib6_rule_suppress' [-Wmissing-prototypes]
INDIRECT_CALLABLE_SCOPE bool fib6_rule_suppress(struct fib_rule *rule,
^
net/ipv6/fib6_rules.c:269:25: note: declare 'static' if the function is not intended to be used outside of this translation unit
INDIRECT_CALLABLE_SCOPE bool fib6_rule_suppress(struct fib_rule *rule,
^
static
>> net/ipv6/fib6_rules.c:302:29: warning: no previous prototype for function 'fib6_rule_match' [-Wmissing-prototypes]
INDIRECT_CALLABLE_SCOPE int fib6_rule_match(struct fib_rule *rule,
^
net/ipv6/fib6_rules.c:302:25: note: declare 'static' if the function is not intended to be used outside of this translation unit
INDIRECT_CALLABLE_SCOPE int fib6_rule_match(struct fib_rule *rule,
^
static
3 warnings generated.
vim +/fib4_rule_action +107 net/ipv4/fib_rules.c
106
> 107 INDIRECT_CALLABLE_SCOPE int fib4_rule_action(struct fib_rule *rule,
108 struct flowi *flp, int flags,
109 struct fib_lookup_arg *arg)
110 {
111 int err = -EAGAIN;
112 struct fib_table *tbl;
113 u32 tb_id;
114
115 switch (rule->action) {
116 case FR_ACT_TO_TBL:
117 break;
118
119 case FR_ACT_UNREACHABLE:
120 return -ENETUNREACH;
121
122 case FR_ACT_PROHIBIT:
123 return -EACCES;
124
125 case FR_ACT_BLACKHOLE:
126 default:
127 return -EINVAL;
128 }
129
130 rcu_read_lock();
131
132 tb_id = fib_rule_get_table(rule, arg);
133 tbl = fib_get_table(rule->fr_net, tb_id);
134 if (tbl)
135 err = fib_table_lookup(tbl, &flp->u.ip4,
136 (struct fib_result *)arg->result,
137 arg->flags);
138
139 rcu_read_unlock();
140 return err;
141 }
142
> 143 INDIRECT_CALLABLE_SCOPE bool fib4_rule_suppress(struct fib_rule *rule,
144 struct fib_lookup_arg *arg)
145 {
146 struct fib_result *result = (struct fib_result *) arg->result;
147 struct net_device *dev = NULL;
148
149 if (result->fi) {
150 struct fib_nh_common *nhc = fib_info_nhc(result->fi, 0);
151
152 dev = nhc->nhc_dev;
153 }
154
155 /* do not accept result if the route does
156 * not meet the required prefix length
157 */
158 if (result->prefixlen <= rule->suppress_prefixlen)
159 goto suppress_route;
160
161 /* do not accept result if the route uses a device
162 * belonging to a forbidden interface group
163 */
164 if (rule->suppress_ifgroup != -1 && dev && dev->group == rule->suppress_ifgroup)
165 goto suppress_route;
166
167 return false;
168
169 suppress_route:
170 if (!(arg->flags & FIB_LOOKUP_NOREF))
171 fib_info_put(result->fi);
172 return true;
173 }
174
> 175 INDIRECT_CALLABLE_SCOPE int fib4_rule_match(struct fib_rule *rule,
176 struct flowi *fl, int flags)
177 {
178 struct fib4_rule *r = (struct fib4_rule *) rule;
179 struct flowi4 *fl4 = &fl->u.ip4;
180 __be32 daddr = fl4->daddr;
181 __be32 saddr = fl4->saddr;
182
183 if (((saddr ^ r->src) & r->srcmask) ||
184 ((daddr ^ r->dst) & r->dstmask))
185 return 0;
186
187 if (r->tos && (r->tos != fl4->flowi4_tos))
188 return 0;
189
190 if (rule->ip_proto && (rule->ip_proto != fl4->flowi4_proto))
191 return 0;
192
193 if (fib_rule_port_range_set(&rule->sport_range) &&
194 !fib_rule_port_inrange(&rule->sport_range, fl4->fl4_sport))
195 return 0;
196
197 if (fib_rule_port_range_set(&rule->dport_range) &&
198 !fib_rule_port_inrange(&rule->dport_range, fl4->fl4_dport))
199 return 0;
200
201 return 1;
202 }
203
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/[email protected]