Hi:
It seems that all the list_*_rcu primitives are missing a memory barrier
on the very first dereference. For example,
#define list_for_each_rcu(pos, head) \
for (pos = (head)->next; prefetch(pos->next), pos != (head); \
pos = rcu_dereference(pos->next))
It will go something like:
pos = (head)->next
prefetch(pos->next)
pos != (head)
do stuff
We're missing a barrier here.
pos = rcu_dereference(pos->next)
fetch pos->next
barrier given by rcu_dereference(pos->next)
store pos
Without the missing barrier, the pos->next value may turn out to be
stale. In fact, if "do stuff" were also dereferencing pos and relying
on list_for_each_rcu to provide the barrier then it may also break.
So here is a patch to make sure that we have a barrier for the first
element in the list.
Signed-off-by: Herbert Xu <[email protected]>
Cheers,
--
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <[email protected]>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
On Sat, Oct 15, 2005 at 10:26:49AM +1000, Herbert Xu wrote:
> Hi:
>
> It seems that all the list_*_rcu primitives are missing a memory barrier
> on the very first dereference. For example,
>
> #define list_for_each_rcu(pos, head) \
> for (pos = (head)->next; prefetch(pos->next), pos != (head); \
> pos = rcu_dereference(pos->next))
>
> It will go something like:
>
> pos = (head)->next
>
> prefetch(pos->next)
>
> pos != (head)
>
> do stuff
>
> We're missing a barrier here.
>
> pos = rcu_dereference(pos->next)
>
> fetch pos->next
>
> barrier given by rcu_dereference(pos->next)
>
> store pos
>
> Without the missing barrier, the pos->next value may turn out to be
> stale. In fact, if "do stuff" were also dereferencing pos and relying
> on list_for_each_rcu to provide the barrier then it may also break.
>
> So here is a patch to make sure that we have a barrier for the first
> element in the list.
Good catch!!! I wonder if Alpha SMP Linux machines have noticed...
Some comments interspersed.
> Signed-off-by: Herbert Xu <[email protected]>
>
> Cheers,
> --
> Visit Openswan at http://www.openswan.org/
> Email: Herbert Xu ~{PmV>HI~} <[email protected]>
> Home Page: http://gondor.apana.org.au/~herbert/
> PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
> diff --git a/include/linux/list.h b/include/linux/list.h
> --- a/include/linux/list.h
> +++ b/include/linux/list.h
> @@ -442,12 +442,15 @@ static inline void list_splice_init(stru
> * as long as the traversal is guarded by rcu_read_lock().
> */
> #define list_for_each_rcu(pos, head) \
> - for (pos = (head)->next; prefetch(pos->next), pos != (head); \
> - pos = rcu_dereference(pos->next))
> + for (pos = (head)->next; \
> + pos = rcu_dereference(pos), \
> + prefetch(pos->next), pos != (head); \
> + pos = pos->next)
Why not something like the following? Seems a bit simpler to me.
#define list_for_each_rcu(pos, head) \
for (pos = rcu_dereference((head)->next); \
prefetch(pos->next), pos != (head); \
pos = rcu_dereference(pos->next))
> #define __list_for_each_rcu(pos, head) \
> - for (pos = (head)->next; pos != (head); \
> - pos = rcu_dereference(pos->next))
> + for (pos = (head)->next; \
> + rcu_dereference(pos) != (head); \
> + pos = pos->next)
>
> /**
> * list_for_each_safe_rcu - iterate over an rcu-protected list safe
> @@ -461,8 +464,9 @@ static inline void list_splice_init(stru
> * as long as the traversal is guarded by rcu_read_lock().
> */
> #define list_for_each_safe_rcu(pos, n, head) \
> - for (pos = (head)->next, n = pos->next; pos != (head); \
> - pos = rcu_dereference(n), n = pos->next)
> + for (pos = (head)->next; \
> + n = rcu_dereference(pos)->next, pos != (head); \
> + pos = n)
>
> /**
> * list_for_each_entry_rcu - iterate over rcu list of given type
> @@ -474,11 +478,11 @@ static inline void list_splice_init(stru
> * the _rcu list-mutation primitives such as list_add_rcu()
> * as long as the traversal is guarded by rcu_read_lock().
> */
> -#define list_for_each_entry_rcu(pos, head, member) \
> - for (pos = list_entry((head)->next, typeof(*pos), member); \
> - prefetch(pos->member.next), &pos->member != (head); \
> - pos = rcu_dereference(list_entry(pos->member.next, \
> - typeof(*pos), member)))
> +#define list_for_each_entry_rcu(pos, head, member) \
> + for (pos = list_entry((head)->next, typeof(*pos), member); \
> + pos = rcu_dereference(pos), \
> + prefetch(pos->member.next), &pos->member != (head); \
> + pos = list_entry(pos->member.next, typeof(*pos), member))
>
>
> /**
> @@ -492,8 +496,10 @@ static inline void list_splice_init(stru
> * as long as the traversal is guarded by rcu_read_lock().
> */
> #define list_for_each_continue_rcu(pos, head) \
> - for ((pos) = (pos)->next; prefetch((pos)->next), (pos) != (head); \
> - (pos) = rcu_dereference((pos)->next))
> + for ((pos) = (pos)->next; \
> + (pos) = rcu_dereference((pos)), \
> + prefetch((pos)->next), (pos) != (head); \
> + (pos) = (pos)->next)
The above hurts my head -- childhood trauma due to having to use a
FORTRAN compiler that required "I=I" at odd intervals in order to
generate correct code... How about the following?
#define list_for_each_continue_rcu(pos, head) \
for ((pos) = (pos)->next; \
prefetch(rcu_dereference(pos)->next), (pos) != (head); \
(pos) = (pos)->next)
> /*
> * Double linked lists with a single pointer list head.
> @@ -696,8 +702,9 @@ static inline void hlist_add_after_rcu(s
> pos = n)
>
> #define hlist_for_each_rcu(pos, head) \
> - for ((pos) = (head)->first; pos && ({ prefetch((pos)->next); 1; }); \
> - (pos) = rcu_dereference((pos)->next))
> + for ((pos) = (head)->first; \
> + rcu_dereference((pos)) && ({ prefetch((pos)->next); 1; }); \
> + (pos) = (pos)->next)
>
> /**
> * hlist_for_each_entry - iterate over list of given type
> @@ -762,9 +769,9 @@ static inline void hlist_add_after_rcu(s
> */
> #define hlist_for_each_entry_rcu(tpos, pos, head, member) \
> for (pos = (head)->first; \
> - pos && ({ prefetch(pos->next); 1;}) && \
> + rcu_dereference(pos) && ({ prefetch(pos->next); 1;}) && \
> ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \
> - pos = rcu_dereference(pos->next))
> + pos = pos->next)
>
> #else
> #warning "don't include kernel headers in userspace"
On Fri, Oct 14, 2005 at 07:03:25PM -0700, Paul E. McKenney wrote:
>
> > diff --git a/include/linux/list.h b/include/linux/list.h
> > --- a/include/linux/list.h
> > +++ b/include/linux/list.h
> > @@ -442,12 +442,15 @@ static inline void list_splice_init(stru
> > * as long as the traversal is guarded by rcu_read_lock().
> > */
> > #define list_for_each_rcu(pos, head) \
> > - for (pos = (head)->next; prefetch(pos->next), pos != (head); \
> > - pos = rcu_dereference(pos->next))
> > + for (pos = (head)->next; \
> > + pos = rcu_dereference(pos), \
> > + prefetch(pos->next), pos != (head); \
> > + pos = pos->next)
>
> Why not something like the following? Seems a bit simpler to me.
>
> #define list_for_each_rcu(pos, head) \
> for (pos = rcu_dereference((head)->next); \
> prefetch(pos->next), pos != (head); \
> pos = rcu_dereference(pos->next))
In this case your version is indeed more concise. However, in most of
the other for_each macros having it in the loop conditional looks more
natural.
So in order to be consistent throughout list.h, I'd like to keep the
rcu_dereference in the loop conditional.
> > @@ -492,8 +496,10 @@ static inline void list_splice_init(stru
> > * as long as the traversal is guarded by rcu_read_lock().
> > */
> > #define list_for_each_continue_rcu(pos, head) \
> > - for ((pos) = (pos)->next; prefetch((pos)->next), (pos) != (head); \
> > - (pos) = rcu_dereference((pos)->next))
> > + for ((pos) = (pos)->next; \
> > + (pos) = rcu_dereference((pos)), \
> > + prefetch((pos)->next), (pos) != (head); \
> > + (pos) = (pos)->next)
>
> The above hurts my head -- childhood trauma due to having to use a
> FORTRAN compiler that required "I=I" at odd intervals in order to
> generate correct code... How about the following?
>
> #define list_for_each_continue_rcu(pos, head) \
> for ((pos) = (pos)->next; \
> prefetch(rcu_dereference(pos)->next), (pos) != (head); \
> (pos) = (pos)->next)
I chose to keep it out of prefetch because normally the argument to
prefetch does not have any side-effects. Even though today's prefetch
is an inline function which does respect side-effects, there is always
a possibility that someone somewhere might decide to implement prefetch
as a macro.
Besides, the expression
i = foo(i)
where foo has side-effects is pretty normal.
Cheers,
--
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <[email protected]>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
Hi Paul:
On Sat, Oct 15, 2005 at 12:39:18PM +1000, herbert wrote:
>
> Besides, the expression
>
> i = foo(i)
>
> where foo has side-effects is pretty normal.
Actually I've changed my mind on this. I think your version is
better because the side-effect of rcu_dereference will cause the
above assignment to occur twice when i refers to a memory-backed
variable.
Since all current prefetch implementations are safe as far as
side-effects are concerned, here is an updated version that
doesn't do i = foo(i).
Andrew, please replace the previous version with this.
Thanks,
--
Visit Openswan at http://www.openswan.org/
Email: Herbert Xu ~{PmV>HI~} <[email protected]>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
On Sat, Oct 15, 2005 at 01:22:41PM +1000, Herbert Xu wrote:
> Hi Paul:
>
> On Sat, Oct 15, 2005 at 12:39:18PM +1000, herbert wrote:
> >
> > Besides, the expression
> >
> > i = foo(i)
> >
> > where foo has side-effects is pretty normal.
>
> Actually I've changed my mind on this. I think your version is
> better because the side-effect of rcu_dereference will cause the
> above assignment to occur twice when i refers to a memory-backed
> variable.
>
> Since all current prefetch implementations are safe as far as
> side-effects are concerned, here is an updated version that
> doesn't do i = foo(i).
>
> Andrew, please replace the previous version with this.
Looks great to me!
Thanx, Paul
Acked-by: <[email protected]>
> Thanks,
> --
> Visit Openswan at http://www.openswan.org/
> Email: Herbert Xu ~{PmV>HI~} <[email protected]>
> Home Page: http://gondor.apana.org.au/~herbert/
> PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
> diff --git a/include/linux/list.h b/include/linux/list.h
> --- a/include/linux/list.h
> +++ b/include/linux/list.h
> @@ -442,12 +442,14 @@ static inline void list_splice_init(stru
> * as long as the traversal is guarded by rcu_read_lock().
> */
> #define list_for_each_rcu(pos, head) \
> - for (pos = (head)->next; prefetch(pos->next), pos != (head); \
> - pos = rcu_dereference(pos->next))
> + for (pos = (head)->next; \
> + prefetch(rcu_dereference(pos)->next), pos != (head); \
> + pos = pos->next)
>
> #define __list_for_each_rcu(pos, head) \
> - for (pos = (head)->next; pos != (head); \
> - pos = rcu_dereference(pos->next))
> + for (pos = (head)->next; \
> + rcu_dereference(pos) != (head); \
> + pos = pos->next)
>
> /**
> * list_for_each_safe_rcu - iterate over an rcu-protected list safe
> @@ -461,8 +463,9 @@ static inline void list_splice_init(stru
> * as long as the traversal is guarded by rcu_read_lock().
> */
> #define list_for_each_safe_rcu(pos, n, head) \
> - for (pos = (head)->next, n = pos->next; pos != (head); \
> - pos = rcu_dereference(n), n = pos->next)
> + for (pos = (head)->next; \
> + n = rcu_dereference(pos)->next, pos != (head); \
> + pos = n)
>
> /**
> * list_for_each_entry_rcu - iterate over rcu list of given type
> @@ -474,11 +477,11 @@ static inline void list_splice_init(stru
> * the _rcu list-mutation primitives such as list_add_rcu()
> * as long as the traversal is guarded by rcu_read_lock().
> */
> -#define list_for_each_entry_rcu(pos, head, member) \
> - for (pos = list_entry((head)->next, typeof(*pos), member); \
> - prefetch(pos->member.next), &pos->member != (head); \
> - pos = rcu_dereference(list_entry(pos->member.next, \
> - typeof(*pos), member)))
> +#define list_for_each_entry_rcu(pos, head, member) \
> + for (pos = list_entry((head)->next, typeof(*pos), member); \
> + prefetch(rcu_dereference(pos)->member.next), \
> + &pos->member != (head); \
> + pos = list_entry(pos->member.next, typeof(*pos), member))
>
>
> /**
> @@ -492,8 +495,9 @@ static inline void list_splice_init(stru
> * as long as the traversal is guarded by rcu_read_lock().
> */
> #define list_for_each_continue_rcu(pos, head) \
> - for ((pos) = (pos)->next; prefetch((pos)->next), (pos) != (head); \
> - (pos) = rcu_dereference((pos)->next))
> + for ((pos) = (pos)->next; \
> + prefetch(rcu_dereference((pos))->next), (pos) != (head); \
> + (pos) = (pos)->next)
>
> /*
> * Double linked lists with a single pointer list head.
> @@ -696,8 +700,9 @@ static inline void hlist_add_after_rcu(s
> pos = n)
>
> #define hlist_for_each_rcu(pos, head) \
> - for ((pos) = (head)->first; pos && ({ prefetch((pos)->next); 1; }); \
> - (pos) = rcu_dereference((pos)->next))
> + for ((pos) = (head)->first; \
> + rcu_dereference((pos)) && ({ prefetch((pos)->next); 1; }); \
> + (pos) = (pos)->next)
>
> /**
> * hlist_for_each_entry - iterate over list of given type
> @@ -762,9 +767,9 @@ static inline void hlist_add_after_rcu(s
> */
> #define hlist_for_each_entry_rcu(tpos, pos, head, member) \
> for (pos = (head)->first; \
> - pos && ({ prefetch(pos->next); 1;}) && \
> + rcu_dereference(pos) && ({ prefetch(pos->next); 1;}) && \
> ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \
> - pos = rcu_dereference(pos->next))
> + pos = pos->next)
>
> #else
> #warning "don't include kernel headers in userspace"