v2: Correct syntax error in net/ipv6/route.c
In ip6_dst_gc() replace:
if (entries > gc_thresh)
With:
if (entries > ops->gc_thresh)
Sending Ipv6 packets in a loop via a raw socket triggers an issue where a
route is cloned by ip6_rt_cache_alloc() for each packet sent. This quickly
consumes the Ipv6 max_size threshold which defaults to 4096 resulting in
these warnings:
[1] 99.187805] dst_alloc: 7728 callbacks suppressed
[2] Route cache is full: consider increasing sysctl net.ipv6.route.max_size.
.
.
[300] Route cache is full: consider increasing sysctl net.ipv6.route.max_size.
When this happens the packet is dropped and sendto() gets a network is
unreachable error:
# ./a.out -s
remaining pkt 200557 errno 101
remaining pkt 196462 errno 101
.
.
remaining pkt 126821 errno 101
Implement David Aherns suggestion to remove max_size check seeing that Ipv6
has a GC to manage memory usage. Ipv4 already does not check max_size.
Here are some memory comparisons for Ipv4 vs Ipv6 with the patch:
Test by running 5 instances of a program that sends UDP packets to a raw
socket 5000000 times. Compare Ipv4 and Ipv6 performance with a similar
program.
Ipv4:
Before test:
# grep -e Slab -e Free /proc/meminfo
MemFree: 29427108 kB
Slab: 237612 kB
# grep dst_cache /proc/slabinfo
ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0
xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0
ip_dst_cache 2881 3990 192 42 2 : tunables 0 0 0
During test:
# grep -e Slab -e Free /proc/meminfo
MemFree: 29417608 kB
Slab: 247712 kB
# grep dst_cache /proc/slabinfo
ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0
xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0
ip_dst_cache 44394 44394 192 42 2 : tunables 0 0 0
After test:
# grep -e Slab -e Free /proc/meminfo
MemFree: 29422308 kB
Slab: 238104 kB
# grep dst_cache /proc/slabinfo
ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0
xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0
ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0
Ipv6 with patch:
Errno 101 errors are not observed anymore with the patch.
Before test:
# grep -e Slab -e Free /proc/meminfo
MemFree: 29422308 kB
Slab: 238104 kB
# grep dst_cache /proc/slabinfo
ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0
xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0
ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0
During Test:
# grep -e Slab -e Free /proc/meminfo
MemFree: 29431516 kB
Slab: 240940 kB
# grep dst_cache /proc/slabinfo
ip6_dst_cache 11980 12064 256 32 2 : tunables 0 0 0
xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0
ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0
After Test:
# grep -e Slab -e Free /proc/meminfo
MemFree: 29441816 kB
Slab: 238132 kB
# grep dst_cache /proc/slabinfo
ip6_dst_cache 1902 2432 256 32 2 : tunables 0 0 0
xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0
ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0
Tested-by: Andrea Mayer <[email protected]>
Signed-off-by: Jon Maxwell <[email protected]>
---
include/net/dst_ops.h | 2 +-
net/core/dst.c | 8 ++------
net/ipv6/route.c | 13 +++++--------
3 files changed, 8 insertions(+), 15 deletions(-)
diff --git a/include/net/dst_ops.h b/include/net/dst_ops.h
index 88ff7bb2bb9b..632086b2f644 100644
--- a/include/net/dst_ops.h
+++ b/include/net/dst_ops.h
@@ -16,7 +16,7 @@ struct dst_ops {
unsigned short family;
unsigned int gc_thresh;
- int (*gc)(struct dst_ops *ops);
+ void (*gc)(struct dst_ops *ops);
struct dst_entry * (*check)(struct dst_entry *, __u32 cookie);
unsigned int (*default_advmss)(const struct dst_entry *);
unsigned int (*mtu)(const struct dst_entry *);
diff --git a/net/core/dst.c b/net/core/dst.c
index 6d2dd03dafa8..31c08a3386d3 100644
--- a/net/core/dst.c
+++ b/net/core/dst.c
@@ -82,12 +82,8 @@ void *dst_alloc(struct dst_ops *ops, struct net_device *dev,
if (ops->gc &&
!(flags & DST_NOCOUNT) &&
- dst_entries_get_fast(ops) > ops->gc_thresh) {
- if (ops->gc(ops)) {
- pr_notice_ratelimited("Route cache is full: consider increasing sysctl net.ipv6.route.max_size.\n");
- return NULL;
- }
- }
+ dst_entries_get_fast(ops) > ops->gc_thresh)
+ ops->gc(ops);
dst = kmem_cache_alloc(ops->kmem_cachep, GFP_ATOMIC);
if (!dst)
diff --git a/net/ipv6/route.c b/net/ipv6/route.c
index e74e0361fd92..b643dda68d31 100644
--- a/net/ipv6/route.c
+++ b/net/ipv6/route.c
@@ -91,7 +91,7 @@ static struct dst_entry *ip6_negative_advice(struct dst_entry *);
static void ip6_dst_destroy(struct dst_entry *);
static void ip6_dst_ifdown(struct dst_entry *,
struct net_device *dev, int how);
-static int ip6_dst_gc(struct dst_ops *ops);
+static void ip6_dst_gc(struct dst_ops *ops);
static int ip6_pkt_discard(struct sk_buff *skb);
static int ip6_pkt_discard_out(struct net *net, struct sock *sk, struct sk_buff *skb);
@@ -3284,11 +3284,10 @@ struct dst_entry *icmp6_dst_alloc(struct net_device *dev,
return dst;
}
-static int ip6_dst_gc(struct dst_ops *ops)
+static void ip6_dst_gc(struct dst_ops *ops)
{
struct net *net = container_of(ops, struct net, ipv6.ip6_dst_ops);
int rt_min_interval = net->ipv6.sysctl.ip6_rt_gc_min_interval;
- int rt_max_size = net->ipv6.sysctl.ip6_rt_max_size;
int rt_elasticity = net->ipv6.sysctl.ip6_rt_gc_elasticity;
int rt_gc_timeout = net->ipv6.sysctl.ip6_rt_gc_timeout;
unsigned long rt_last_gc = net->ipv6.ip6_rt_last_gc;
@@ -3296,11 +3295,10 @@ static int ip6_dst_gc(struct dst_ops *ops)
int entries;
entries = dst_entries_get_fast(ops);
- if (entries > rt_max_size)
+ if (entries > ops->gc_thresh)
entries = dst_entries_get_slow(ops);
- if (time_after(rt_last_gc + rt_min_interval, jiffies) &&
- entries <= rt_max_size)
+ if (time_after(rt_last_gc + rt_min_interval, jiffies))
goto out;
fib6_run_gc(atomic_inc_return(&net->ipv6.ip6_rt_gc_expire), net, true);
@@ -3310,7 +3308,6 @@ static int ip6_dst_gc(struct dst_ops *ops)
out:
val = atomic_read(&net->ipv6.ip6_rt_gc_expire);
atomic_set(&net->ipv6.ip6_rt_gc_expire, val - (val >> rt_elasticity));
- return entries > rt_max_size;
}
static int ip6_nh_lookup_table(struct net *net, struct fib6_config *cfg,
@@ -6512,7 +6509,7 @@ static int __net_init ip6_route_net_init(struct net *net)
#endif
net->ipv6.sysctl.flush_delay = 0;
- net->ipv6.sysctl.ip6_rt_max_size = 4096;
+ net->ipv6.sysctl.ip6_rt_max_size = INT_MAX;
net->ipv6.sysctl.ip6_rt_gc_min_interval = HZ / 2;
net->ipv6.sysctl.ip6_rt_gc_timeout = 60*HZ;
net->ipv6.sysctl.ip6_rt_gc_interval = 30*HZ;
--
2.31.1
Hi Jon,
On Thu, 12 Jan 2023 12:25:32 +1100
Jon Maxwell <[email protected]> wrote:
> v2: Correct syntax error in net/ipv6/route.c
>
> In ip6_dst_gc() replace:
>
> if (entries > gc_thresh)
>
> With:
>
> if (entries > ops->gc_thresh)
>
> Sending Ipv6 packets in a loop via a raw socket triggers an issue where a
> route is cloned by ip6_rt_cache_alloc() for each packet sent. This quickly
> consumes the Ipv6 max_size threshold which defaults to 4096 resulting in
> these warnings:
>
> [1] 99.187805] dst_alloc: 7728 callbacks suppressed
> [2] Route cache is full: consider increasing sysctl net.ipv6.route.max_size.
> .
> .
> [300] Route cache is full: consider increasing sysctl net.ipv6.route.max_size.
>
> When this happens the packet is dropped and sendto() gets a network is
> unreachable error:
>
> # ./a.out -s
>
> remaining pkt 200557 errno 101
> remaining pkt 196462 errno 101
> .
> .
> remaining pkt 126821 errno 101
>
> Implement David Aherns suggestion to remove max_size check seeing that Ipv6
> has a GC to manage memory usage. Ipv4 already does not check max_size.
>
> Here are some memory comparisons for Ipv4 vs Ipv6 with the patch:
>
> Test by running 5 instances of a program that sends UDP packets to a raw
> socket 5000000 times. Compare Ipv4 and Ipv6 performance with a similar
> program.
>
> Ipv4:
>
is it supposed to be Ipv6, right?
Ciao,
Andrea
> Before test:
>
> # grep -e Slab -e Free /proc/meminfo
> MemFree: 29427108 kB
> Slab: 237612 kB
>
> # grep dst_cache /proc/slabinfo
> ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0
> xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0
> ip_dst_cache 2881 3990 192 42 2 : tunables 0 0 0
>
> During test:
>
> # grep -e Slab -e Free /proc/meminfo
> MemFree: 29417608 kB
> Slab: 247712 kB
>
> # grep dst_cache /proc/slabinfo
> ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0
> xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0
> ip_dst_cache 44394 44394 192 42 2 : tunables 0 0 0
>
> After test:
>
> # grep -e Slab -e Free /proc/meminfo
> MemFree: 29422308 kB
> Slab: 238104 kB
>
> # grep dst_cache /proc/slabinfo
> ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0
> xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0
> ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0
>
> Ipv6 with patch:
>
> Errno 101 errors are not observed anymore with the patch.
>
> Before test:
>
> # grep -e Slab -e Free /proc/meminfo
> MemFree: 29422308 kB
> Slab: 238104 kB
>
> # grep dst_cache /proc/slabinfo
> ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0
> xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0
> ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0
>
> During Test:
>
> # grep -e Slab -e Free /proc/meminfo
> MemFree: 29431516 kB
> Slab: 240940 kB
>
> # grep dst_cache /proc/slabinfo
> ip6_dst_cache 11980 12064 256 32 2 : tunables 0 0 0
> xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0
> ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0
>
> After Test:
>
> # grep -e Slab -e Free /proc/meminfo
> MemFree: 29441816 kB
> Slab: 238132 kB
>
> # grep dst_cache /proc/slabinfo
> ip6_dst_cache 1902 2432 256 32 2 : tunables 0 0 0
> xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0
> ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0
>
> Tested-by: Andrea Mayer <[email protected]>
> Signed-off-by: Jon Maxwell <[email protected]>
> ---
> include/net/dst_ops.h | 2 +-
> net/core/dst.c | 8 ++------
> net/ipv6/route.c | 13 +++++--------
> 3 files changed, 8 insertions(+), 15 deletions(-)
>
> diff --git a/include/net/dst_ops.h b/include/net/dst_ops.h
> index 88ff7bb2bb9b..632086b2f644 100644
> --- a/include/net/dst_ops.h
> +++ b/include/net/dst_ops.h
> @@ -16,7 +16,7 @@ struct dst_ops {
> unsigned short family;
> unsigned int gc_thresh;
>
> - int (*gc)(struct dst_ops *ops);
> + void (*gc)(struct dst_ops *ops);
> struct dst_entry * (*check)(struct dst_entry *, __u32 cookie);
> unsigned int (*default_advmss)(const struct dst_entry *);
> unsigned int (*mtu)(const struct dst_entry *);
> diff --git a/net/core/dst.c b/net/core/dst.c
> index 6d2dd03dafa8..31c08a3386d3 100644
> --- a/net/core/dst.c
> +++ b/net/core/dst.c
> @@ -82,12 +82,8 @@ void *dst_alloc(struct dst_ops *ops, struct net_device *dev,
>
> if (ops->gc &&
> !(flags & DST_NOCOUNT) &&
> - dst_entries_get_fast(ops) > ops->gc_thresh) {
> - if (ops->gc(ops)) {
> - pr_notice_ratelimited("Route cache is full: consider increasing sysctl net.ipv6.route.max_size.\n");
> - return NULL;
> - }
> - }
> + dst_entries_get_fast(ops) > ops->gc_thresh)
> + ops->gc(ops);
>
> dst = kmem_cache_alloc(ops->kmem_cachep, GFP_ATOMIC);
> if (!dst)
> diff --git a/net/ipv6/route.c b/net/ipv6/route.c
> index e74e0361fd92..b643dda68d31 100644
> --- a/net/ipv6/route.c
> +++ b/net/ipv6/route.c
> @@ -91,7 +91,7 @@ static struct dst_entry *ip6_negative_advice(struct dst_entry *);
> static void ip6_dst_destroy(struct dst_entry *);
> static void ip6_dst_ifdown(struct dst_entry *,
> struct net_device *dev, int how);
> -static int ip6_dst_gc(struct dst_ops *ops);
> +static void ip6_dst_gc(struct dst_ops *ops);
>
> static int ip6_pkt_discard(struct sk_buff *skb);
> static int ip6_pkt_discard_out(struct net *net, struct sock *sk, struct sk_buff *skb);
> @@ -3284,11 +3284,10 @@ struct dst_entry *icmp6_dst_alloc(struct net_device *dev,
> return dst;
> }
>
> -static int ip6_dst_gc(struct dst_ops *ops)
> +static void ip6_dst_gc(struct dst_ops *ops)
> {
> struct net *net = container_of(ops, struct net, ipv6.ip6_dst_ops);
> int rt_min_interval = net->ipv6.sysctl.ip6_rt_gc_min_interval;
> - int rt_max_size = net->ipv6.sysctl.ip6_rt_max_size;
> int rt_elasticity = net->ipv6.sysctl.ip6_rt_gc_elasticity;
> int rt_gc_timeout = net->ipv6.sysctl.ip6_rt_gc_timeout;
> unsigned long rt_last_gc = net->ipv6.ip6_rt_last_gc;
> @@ -3296,11 +3295,10 @@ static int ip6_dst_gc(struct dst_ops *ops)
> int entries;
>
> entries = dst_entries_get_fast(ops);
> - if (entries > rt_max_size)
> + if (entries > ops->gc_thresh)
> entries = dst_entries_get_slow(ops);
>
> - if (time_after(rt_last_gc + rt_min_interval, jiffies) &&
> - entries <= rt_max_size)
> + if (time_after(rt_last_gc + rt_min_interval, jiffies))
> goto out;
>
> fib6_run_gc(atomic_inc_return(&net->ipv6.ip6_rt_gc_expire), net, true);
> @@ -3310,7 +3308,6 @@ static int ip6_dst_gc(struct dst_ops *ops)
> out:
> val = atomic_read(&net->ipv6.ip6_rt_gc_expire);
> atomic_set(&net->ipv6.ip6_rt_gc_expire, val - (val >> rt_elasticity));
> - return entries > rt_max_size;
> }
>
> static int ip6_nh_lookup_table(struct net *net, struct fib6_config *cfg,
> @@ -6512,7 +6509,7 @@ static int __net_init ip6_route_net_init(struct net *net)
> #endif
>
> net->ipv6.sysctl.flush_delay = 0;
> - net->ipv6.sysctl.ip6_rt_max_size = 4096;
> + net->ipv6.sysctl.ip6_rt_max_size = INT_MAX;
> net->ipv6.sysctl.ip6_rt_gc_min_interval = HZ / 2;
> net->ipv6.sysctl.ip6_rt_gc_timeout = 60*HZ;
> net->ipv6.sysctl.ip6_rt_gc_interval = 30*HZ;
> --
> 2.31.1
Hi Andrea,
On Fri, Jan 13, 2023 at 7:41 AM Andrea Mayer <[email protected]> wrote:
>
> Hi Jon,
>
> On Thu, 12 Jan 2023 12:25:32 +1100
> Jon Maxwell <[email protected]> wrote:
>
> > v2: Correct syntax error in net/ipv6/route.c
> >
> > In ip6_dst_gc() replace:
> >
> > if (entries > gc_thresh)
> >
> > With:
> >
> > if (entries > ops->gc_thresh)
> >
> > Sending Ipv6 packets in a loop via a raw socket triggers an issue where a
> > route is cloned by ip6_rt_cache_alloc() for each packet sent. This quickly
> > consumes the Ipv6 max_size threshold which defaults to 4096 resulting in
> > these warnings:
> >
> > [1] 99.187805] dst_alloc: 7728 callbacks suppressed
> > [2] Route cache is full: consider increasing sysctl net.ipv6.route.max_size.
> > .
> > .
> > [300] Route cache is full: consider increasing sysctl net.ipv6.route.max_size.
> >
> > When this happens the packet is dropped and sendto() gets a network is
> > unreachable error:
> >
> > # ./a.out -s
> >
> > remaining pkt 200557 errno 101
> > remaining pkt 196462 errno 101
> > .
> > .
> > remaining pkt 126821 errno 101
> >
> > Implement David Aherns suggestion to remove max_size check seeing that Ipv6
> > has a GC to manage memory usage. Ipv4 already does not check max_size.
> >
> > Here are some memory comparisons for Ipv4 vs Ipv6 with the patch:
> >
> > Test by running 5 instances of a program that sends UDP packets to a raw
> > socket 5000000 times. Compare Ipv4 and Ipv6 performance with a similar
> > program.
> >
> > Ipv4:
> >
>
> is it supposed to be Ipv6, right?
>
No, That is for Ipv4, DavidA asked me to include a comparison between Ipv4
and Ipv6. So that is taken for a test running 5 processes that sends 5000000
packets on an ipv4 raw socket. If you scroll down, Ipv6 memory statistics
are under "Ipv6 with patch:".
Regards
Jon
> Ciao,
> Andrea
>
> > Before test:
> >
> > # grep -e Slab -e Free /proc/meminfo
> > MemFree: 29427108 kB
> > Slab: 237612 kB
> >
> > # grep dst_cache /proc/slabinfo
> > ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0
> > xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0
> > ip_dst_cache 2881 3990 192 42 2 : tunables 0 0 0
> >
> > During test:
> >
> > # grep -e Slab -e Free /proc/meminfo
> > MemFree: 29417608 kB
> > Slab: 247712 kB
> >
> > # grep dst_cache /proc/slabinfo
> > ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0
> > xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0
> > ip_dst_cache 44394 44394 192 42 2 : tunables 0 0 0
> >
> > After test:
> >
> > # grep -e Slab -e Free /proc/meminfo
> > MemFree: 29422308 kB
> > Slab: 238104 kB
> >
> > # grep dst_cache /proc/slabinfo
> > ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0
> > xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0
> > ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0
> >
> > Ipv6 with patch:
> >
> > Errno 101 errors are not observed anymore with the patch.
> >
> > Before test:
> >
> > # grep -e Slab -e Free /proc/meminfo
> > MemFree: 29422308 kB
> > Slab: 238104 kB
> >
> > # grep dst_cache /proc/slabinfo
> > ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0
> > xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0
> > ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0
> >
> > During Test:
> >
> > # grep -e Slab -e Free /proc/meminfo
> > MemFree: 29431516 kB
> > Slab: 240940 kB
> >
> > # grep dst_cache /proc/slabinfo
> > ip6_dst_cache 11980 12064 256 32 2 : tunables 0 0 0
> > xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0
> > ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0
> >
> > After Test:
> >
> > # grep -e Slab -e Free /proc/meminfo
> > MemFree: 29441816 kB
> > Slab: 238132 kB
> >
> > # grep dst_cache /proc/slabinfo
> > ip6_dst_cache 1902 2432 256 32 2 : tunables 0 0 0
> > xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0
> > ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0
> >
> > Tested-by: Andrea Mayer <[email protected]>
> > Signed-off-by: Jon Maxwell <[email protected]>
> > ---
> > include/net/dst_ops.h | 2 +-
> > net/core/dst.c | 8 ++------
> > net/ipv6/route.c | 13 +++++--------
> > 3 files changed, 8 insertions(+), 15 deletions(-)
> >
> > diff --git a/include/net/dst_ops.h b/include/net/dst_ops.h
> > index 88ff7bb2bb9b..632086b2f644 100644
> > --- a/include/net/dst_ops.h
> > +++ b/include/net/dst_ops.h
> > @@ -16,7 +16,7 @@ struct dst_ops {
> > unsigned short family;
> > unsigned int gc_thresh;
> >
> > - int (*gc)(struct dst_ops *ops);
> > + void (*gc)(struct dst_ops *ops);
> > struct dst_entry * (*check)(struct dst_entry *, __u32 cookie);
> > unsigned int (*default_advmss)(const struct dst_entry *);
> > unsigned int (*mtu)(const struct dst_entry *);
> > diff --git a/net/core/dst.c b/net/core/dst.c
> > index 6d2dd03dafa8..31c08a3386d3 100644
> > --- a/net/core/dst.c
> > +++ b/net/core/dst.c
> > @@ -82,12 +82,8 @@ void *dst_alloc(struct dst_ops *ops, struct net_device *dev,
> >
> > if (ops->gc &&
> > !(flags & DST_NOCOUNT) &&
> > - dst_entries_get_fast(ops) > ops->gc_thresh) {
> > - if (ops->gc(ops)) {
> > - pr_notice_ratelimited("Route cache is full: consider increasing sysctl net.ipv6.route.max_size.\n");
> > - return NULL;
> > - }
> > - }
> > + dst_entries_get_fast(ops) > ops->gc_thresh)
> > + ops->gc(ops);
> >
> > dst = kmem_cache_alloc(ops->kmem_cachep, GFP_ATOMIC);
> > if (!dst)
> > diff --git a/net/ipv6/route.c b/net/ipv6/route.c
> > index e74e0361fd92..b643dda68d31 100644
> > --- a/net/ipv6/route.c
> > +++ b/net/ipv6/route.c
> > @@ -91,7 +91,7 @@ static struct dst_entry *ip6_negative_advice(struct dst_entry *);
> > static void ip6_dst_destroy(struct dst_entry *);
> > static void ip6_dst_ifdown(struct dst_entry *,
> > struct net_device *dev, int how);
> > -static int ip6_dst_gc(struct dst_ops *ops);
> > +static void ip6_dst_gc(struct dst_ops *ops);
> >
> > static int ip6_pkt_discard(struct sk_buff *skb);
> > static int ip6_pkt_discard_out(struct net *net, struct sock *sk, struct sk_buff *skb);
> > @@ -3284,11 +3284,10 @@ struct dst_entry *icmp6_dst_alloc(struct net_device *dev,
> > return dst;
> > }
> >
> > -static int ip6_dst_gc(struct dst_ops *ops)
> > +static void ip6_dst_gc(struct dst_ops *ops)
> > {
> > struct net *net = container_of(ops, struct net, ipv6.ip6_dst_ops);
> > int rt_min_interval = net->ipv6.sysctl.ip6_rt_gc_min_interval;
> > - int rt_max_size = net->ipv6.sysctl.ip6_rt_max_size;
> > int rt_elasticity = net->ipv6.sysctl.ip6_rt_gc_elasticity;
> > int rt_gc_timeout = net->ipv6.sysctl.ip6_rt_gc_timeout;
> > unsigned long rt_last_gc = net->ipv6.ip6_rt_last_gc;
> > @@ -3296,11 +3295,10 @@ static int ip6_dst_gc(struct dst_ops *ops)
> > int entries;
> >
> > entries = dst_entries_get_fast(ops);
> > - if (entries > rt_max_size)
> > + if (entries > ops->gc_thresh)
> > entries = dst_entries_get_slow(ops);
> >
> > - if (time_after(rt_last_gc + rt_min_interval, jiffies) &&
> > - entries <= rt_max_size)
> > + if (time_after(rt_last_gc + rt_min_interval, jiffies))
> > goto out;
> >
> > fib6_run_gc(atomic_inc_return(&net->ipv6.ip6_rt_gc_expire), net, true);
> > @@ -3310,7 +3308,6 @@ static int ip6_dst_gc(struct dst_ops *ops)
> > out:
> > val = atomic_read(&net->ipv6.ip6_rt_gc_expire);
> > atomic_set(&net->ipv6.ip6_rt_gc_expire, val - (val >> rt_elasticity));
> > - return entries > rt_max_size;
> > }
> >
> > static int ip6_nh_lookup_table(struct net *net, struct fib6_config *cfg,
> > @@ -6512,7 +6509,7 @@ static int __net_init ip6_route_net_init(struct net *net)
> > #endif
> >
> > net->ipv6.sysctl.flush_delay = 0;
> > - net->ipv6.sysctl.ip6_rt_max_size = 4096;
> > + net->ipv6.sysctl.ip6_rt_max_size = INT_MAX;
> > net->ipv6.sysctl.ip6_rt_gc_min_interval = HZ / 2;
> > net->ipv6.sysctl.ip6_rt_gc_timeout = 60*HZ;
> > net->ipv6.sysctl.ip6_rt_gc_interval = 30*HZ;
> > --
> > 2.31.1
On Fri, 13 Jan 2023 08:48:24 +1100
Jonathan Maxwell <[email protected]> wrote:
> Hi Andrea,
>
> On Fri, Jan 13, 2023 at 7:41 AM Andrea Mayer <[email protected]> wrote:
> >
> > Hi Jon,
> >
> > On Thu, 12 Jan 2023 12:25:32 +1100
> > Jon Maxwell <[email protected]> wrote:
> >
> > > v2: Correct syntax error in net/ipv6/route.c
> > >
> > > In ip6_dst_gc() replace:
> > >
> > > if (entries > gc_thresh)
> > >
> > > With:
> > >
> > > if (entries > ops->gc_thresh)
> > >
> > > Sending Ipv6 packets in a loop via a raw socket triggers an issue where a
> > > route is cloned by ip6_rt_cache_alloc() for each packet sent. This quickly
> > > consumes the Ipv6 max_size threshold which defaults to 4096 resulting in
> > > these warnings:
> > >
> > > [1] 99.187805] dst_alloc: 7728 callbacks suppressed
> > > [2] Route cache is full: consider increasing sysctl net.ipv6.route.max_size.
> > > .
> > > .
> > > [300] Route cache is full: consider increasing sysctl net.ipv6.route.max_size.
> > >
> > > When this happens the packet is dropped and sendto() gets a network is
> > > unreachable error:
> > >
> > > # ./a.out -s
> > >
> > > remaining pkt 200557 errno 101
> > > remaining pkt 196462 errno 101
> > > .
> > > .
> > > remaining pkt 126821 errno 101
> > >
> > > Implement David Aherns suggestion to remove max_size check seeing that Ipv6
> > > has a GC to manage memory usage. Ipv4 already does not check max_size.
> > >
> > > Here are some memory comparisons for Ipv4 vs Ipv6 with the patch:
> > >
> > > Test by running 5 instances of a program that sends UDP packets to a raw
> > > socket 5000000 times. Compare Ipv4 and Ipv6 performance with a similar
> > > program.
> > >
> > > Ipv4:
> > >
> >
> > is it supposed to be Ipv6, right?
> >
>
> No, That is for Ipv4, DavidA asked me to include a comparison between Ipv4
> and Ipv6. So that is taken for a test running 5 processes that sends 5000000
> packets on an ipv4 raw socket. If you scroll down, Ipv6 memory statistics
> are under "Ipv6 with patch:".
>
I asked because these tests also report ip6_dst_cache counters that I would not
have expected to find in IPv4 cases (and viceversa).
Thanks for the clarification.
Ciao,
Andrea
> Regards
>
> Jon
>
> > Ciao,
> > Andrea
> >
> > > Before test:
> > >
> > > # grep -e Slab -e Free /proc/meminfo
> > > MemFree: 29427108 kB
> > > Slab: 237612 kB
> > >
> > > # grep dst_cache /proc/slabinfo
> > > ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0
> > > xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0
> > > ip_dst_cache 2881 3990 192 42 2 : tunables 0 0 0
> > >
> > > During test:
> > >
> > > # grep -e Slab -e Free /proc/meminfo
> > > MemFree: 29417608 kB
> > > Slab: 247712 kB
> > >
> > > # grep dst_cache /proc/slabinfo
> > > ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0
> > > xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0
> > > ip_dst_cache 44394 44394 192 42 2 : tunables 0 0 0
> > >
> > > After test:
> > >
> > > # grep -e Slab -e Free /proc/meminfo
> > > MemFree: 29422308 kB
> > > Slab: 238104 kB
> > >
> > > # grep dst_cache /proc/slabinfo
> > > ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0
> > > xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0
> > > ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0
> > >
> > > Ipv6 with patch:
> > >
> > > Errno 101 errors are not observed anymore with the patch.
> > >
> > > Before test:
> > >
> > > # grep -e Slab -e Free /proc/meminfo
> > > MemFree: 29422308 kB
> > > Slab: 238104 kB
> > >
> > > # grep dst_cache /proc/slabinfo
> > > ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0
> > > xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0
> > > ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0
> > >
> > > During Test:
> > >
> > > # grep -e Slab -e Free /proc/meminfo
> > > MemFree: 29431516 kB
> > > Slab: 240940 kB
> > >
> > > # grep dst_cache /proc/slabinfo
> > > ip6_dst_cache 11980 12064 256 32 2 : tunables 0 0 0
> > > xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0
> > > ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0
> > >
> > > After Test:
> > >
> > > # grep -e Slab -e Free /proc/meminfo
> > > MemFree: 29441816 kB
> > > Slab: 238132 kB
> > >
> > > # grep dst_cache /proc/slabinfo
> > > ip6_dst_cache 1902 2432 256 32 2 : tunables 0 0 0
> > > xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0
> > > ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0
> > >
> > > Tested-by: Andrea Mayer <[email protected]>
> > > Signed-off-by: Jon Maxwell <[email protected]>
> > > ---
> > > include/net/dst_ops.h | 2 +-
> > > net/core/dst.c | 8 ++------
> > > net/ipv6/route.c | 13 +++++--------
> > > 3 files changed, 8 insertions(+), 15 deletions(-)
> > >
> > > diff --git a/include/net/dst_ops.h b/include/net/dst_ops.h
> > > index 88ff7bb2bb9b..632086b2f644 100644
> > > --- a/include/net/dst_ops.h
> > > +++ b/include/net/dst_ops.h
> > > @@ -16,7 +16,7 @@ struct dst_ops {
> > > unsigned short family;
> > > unsigned int gc_thresh;
> > >
> > > - int (*gc)(struct dst_ops *ops);
> > > + void (*gc)(struct dst_ops *ops);
> > > struct dst_entry * (*check)(struct dst_entry *, __u32 cookie);
> > > unsigned int (*default_advmss)(const struct dst_entry *);
> > > unsigned int (*mtu)(const struct dst_entry *);
> > > diff --git a/net/core/dst.c b/net/core/dst.c
> > > index 6d2dd03dafa8..31c08a3386d3 100644
> > > --- a/net/core/dst.c
> > > +++ b/net/core/dst.c
> > > @@ -82,12 +82,8 @@ void *dst_alloc(struct dst_ops *ops, struct net_device *dev,
> > >
> > > if (ops->gc &&
> > > !(flags & DST_NOCOUNT) &&
> > > - dst_entries_get_fast(ops) > ops->gc_thresh) {
> > > - if (ops->gc(ops)) {
> > > - pr_notice_ratelimited("Route cache is full: consider increasing sysctl net.ipv6.route.max_size.\n");
> > > - return NULL;
> > > - }
> > > - }
> > > + dst_entries_get_fast(ops) > ops->gc_thresh)
> > > + ops->gc(ops);
> > >
> > > dst = kmem_cache_alloc(ops->kmem_cachep, GFP_ATOMIC);
> > > if (!dst)
> > > diff --git a/net/ipv6/route.c b/net/ipv6/route.c
> > > index e74e0361fd92..b643dda68d31 100644
> > > --- a/net/ipv6/route.c
> > > +++ b/net/ipv6/route.c
> > > @@ -91,7 +91,7 @@ static struct dst_entry *ip6_negative_advice(struct dst_entry *);
> > > static void ip6_dst_destroy(struct dst_entry *);
> > > static void ip6_dst_ifdown(struct dst_entry *,
> > > struct net_device *dev, int how);
> > > -static int ip6_dst_gc(struct dst_ops *ops);
> > > +static void ip6_dst_gc(struct dst_ops *ops);
> > >
> > > static int ip6_pkt_discard(struct sk_buff *skb);
> > > static int ip6_pkt_discard_out(struct net *net, struct sock *sk, struct sk_buff *skb);
> > > @@ -3284,11 +3284,10 @@ struct dst_entry *icmp6_dst_alloc(struct net_device *dev,
> > > return dst;
> > > }
> > >
> > > -static int ip6_dst_gc(struct dst_ops *ops)
> > > +static void ip6_dst_gc(struct dst_ops *ops)
> > > {
> > > struct net *net = container_of(ops, struct net, ipv6.ip6_dst_ops);
> > > int rt_min_interval = net->ipv6.sysctl.ip6_rt_gc_min_interval;
> > > - int rt_max_size = net->ipv6.sysctl.ip6_rt_max_size;
> > > int rt_elasticity = net->ipv6.sysctl.ip6_rt_gc_elasticity;
> > > int rt_gc_timeout = net->ipv6.sysctl.ip6_rt_gc_timeout;
> > > unsigned long rt_last_gc = net->ipv6.ip6_rt_last_gc;
> > > @@ -3296,11 +3295,10 @@ static int ip6_dst_gc(struct dst_ops *ops)
> > > int entries;
> > >
> > > entries = dst_entries_get_fast(ops);
> > > - if (entries > rt_max_size)
> > > + if (entries > ops->gc_thresh)
> > > entries = dst_entries_get_slow(ops);
> > >
> > > - if (time_after(rt_last_gc + rt_min_interval, jiffies) &&
> > > - entries <= rt_max_size)
> > > + if (time_after(rt_last_gc + rt_min_interval, jiffies))
> > > goto out;
> > >
> > > fib6_run_gc(atomic_inc_return(&net->ipv6.ip6_rt_gc_expire), net, true);
> > > @@ -3310,7 +3308,6 @@ static int ip6_dst_gc(struct dst_ops *ops)
> > > out:
> > > val = atomic_read(&net->ipv6.ip6_rt_gc_expire);
> > > atomic_set(&net->ipv6.ip6_rt_gc_expire, val - (val >> rt_elasticity));
> > > - return entries > rt_max_size;
> > > }
> > >
> > > static int ip6_nh_lookup_table(struct net *net, struct fib6_config *cfg,
> > > @@ -6512,7 +6509,7 @@ static int __net_init ip6_route_net_init(struct net *net)
> > > #endif
> > >
> > > net->ipv6.sysctl.flush_delay = 0;
> > > - net->ipv6.sysctl.ip6_rt_max_size = 4096;
> > > + net->ipv6.sysctl.ip6_rt_max_size = INT_MAX;
> > > net->ipv6.sysctl.ip6_rt_gc_min_interval = HZ / 2;
> > > net->ipv6.sysctl.ip6_rt_gc_timeout = 60*HZ;
> > > net->ipv6.sysctl.ip6_rt_gc_interval = 30*HZ;
> > > --
> > > 2.31.1
--
Andrea Mayer <[email protected]>
On 1/11/23 6:25 PM, Jon Maxwell wrote:
> v2: Correct syntax error in net/ipv6/route.c
>
> In ip6_dst_gc() replace:
>
> if (entries > gc_thresh)
>
> With:
>
> if (entries > ops->gc_thresh)
>
> Sending Ipv6 packets in a loop via a raw socket triggers an issue where a
> route is cloned by ip6_rt_cache_alloc() for each packet sent. This quickly
> consumes the Ipv6 max_size threshold which defaults to 4096 resulting in
> these warnings:
>
> [1] 99.187805] dst_alloc: 7728 callbacks suppressed
> [2] Route cache is full: consider increasing sysctl net.ipv6.route.max_size.
> .
> .
> [300] Route cache is full: consider increasing sysctl net.ipv6.route.max_size.
>
> When this happens the packet is dropped and sendto() gets a network is
> unreachable error:
>
> # ./a.out -s
>
> remaining pkt 200557 errno 101
> remaining pkt 196462 errno 101
> .
> .
> remaining pkt 126821 errno 101
>
> Implement David Aherns suggestion to remove max_size check seeing that Ipv6
> has a GC to manage memory usage. Ipv4 already does not check max_size.
>
> Here are some memory comparisons for Ipv4 vs Ipv6 with the patch:
>
> Test by running 5 instances of a program that sends UDP packets to a raw
> socket 5000000 times. Compare Ipv4 and Ipv6 performance with a similar
> program.
>
> Ipv4:
>
> Before test:
>
> # grep -e Slab -e Free /proc/meminfo
> MemFree: 29427108 kB
> Slab: 237612 kB
>
> # grep dst_cache /proc/slabinfo
> ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0
> xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0
> ip_dst_cache 2881 3990 192 42 2 : tunables 0 0 0
>
> During test:
>
> # grep -e Slab -e Free /proc/meminfo
> MemFree: 29417608 kB
> Slab: 247712 kB
>
> # grep dst_cache /proc/slabinfo
> ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0
> xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0
> ip_dst_cache 44394 44394 192 42 2 : tunables 0 0 0
>
> After test:
>
> # grep -e Slab -e Free /proc/meminfo
> MemFree: 29422308 kB
> Slab: 238104 kB
>
> # grep dst_cache /proc/slabinfo
> ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0
> xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0
> ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0
>
> Ipv6 with patch:
>
> Errno 101 errors are not observed anymore with the patch.
>
> Before test:
>
> # grep -e Slab -e Free /proc/meminfo
> MemFree: 29422308 kB
> Slab: 238104 kB
>
> # grep dst_cache /proc/slabinfo
> ip6_dst_cache 1912 2528 256 32 2 : tunables 0 0 0
> xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0
> ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0
>
> During Test:
>
> # grep -e Slab -e Free /proc/meminfo
> MemFree: 29431516 kB
> Slab: 240940 kB
>
> # grep dst_cache /proc/slabinfo
> ip6_dst_cache 11980 12064 256 32 2 : tunables 0 0 0
> xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0
> ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0
>
> After Test:
>
> # grep -e Slab -e Free /proc/meminfo
> MemFree: 29441816 kB
> Slab: 238132 kB
>
> # grep dst_cache /proc/slabinfo
> ip6_dst_cache 1902 2432 256 32 2 : tunables 0 0 0
> xfrm_dst_cache 0 0 320 25 2 : tunables 0 0 0
> ip_dst_cache 3048 4116 192 42 2 : tunables 0 0 0
>
> Tested-by: Andrea Mayer <[email protected]>
> Signed-off-by: Jon Maxwell <[email protected]>
> ---
> include/net/dst_ops.h | 2 +-
> net/core/dst.c | 8 ++------
> net/ipv6/route.c | 13 +++++--------
> 3 files changed, 8 insertions(+), 15 deletions(-)
>
Thanks for the data in the commit message.
Reviewed-by: David Ahern <[email protected]>
Hello:
This patch was applied to netdev/net-next.git (master)
by Jakub Kicinski <[email protected]>:
On Thu, 12 Jan 2023 12:25:32 +1100 you wrote:
> v2: Correct syntax error in net/ipv6/route.c
>
> In ip6_dst_gc() replace:
>
> if (entries > gc_thresh)
>
> With:
>
> [...]
Here is the summary with links:
- [net-next,v2] ipv6: remove max_size check inline with ipv4
https://git.kernel.org/netdev/net-next/c/af6d10345ca7
You are awesome, thank you!
--
Deet-doot-dot, I am a bot.
https://korg.docs.kernel.org/patchwork/pwbot.html