Received-SPF: pass (google.com: domain of linux-kernel-owner@vger.kernel.org designates 23.128.96.37 as permitted sender) client-ip=23.128.96.37;
MIME-Version: 1.0
References: <20231102032330.1036151-1-chengming.zhou@linux.dev> <20231102032330.1036151-8-chengming.zhou@linux.dev>
In-Reply-To: <20231102032330.1036151-8-chengming.zhou@linux.dev>
From:   Hyeonggon Yoo <42.hyeyoo@gmail.com>
Date:   Sun, 3 Dec 2023 18:23:57 +0900
Message-ID: <CAB=+i9RR-n4q5NU6LFqmhM8ys4kM0SPqwj0zYtr4twu=yYmPPA@mail.gmail.com>
Subject: Re: [PATCH v5 7/9] slub: Optimize deactivate_slab()
To:     chengming.zhou@linux.dev
Cc:     vbabka@suse.cz, cl@linux.com, penberg@kernel.org,
        rientjes@google.com, iamjoonsoo.kim@lge.com,
        akpm@linux-foundation.org, roman.gushchin@linux.dev,
        linux-mm@kvack.org, linux-kernel@vger.kernel.org,
        Chengming Zhou <zhouchengming@bytedance.com>
Content-Type: text/plain; charset="UTF-8"
Content-Transfer-Encoding: quoted-printable
Precedence: bulk

On Thu, Nov 2, 2023 at 12:25=E2=80=AFPM <chengming.zhou@linux.dev> wrote:
>
> From: Chengming Zhou <zhouchengming@bytedance.com>
>
> Since the introduce of unfrozen slabs on cpu partial list, we don't
> need to synchronize the slab frozen state under the node list_lock.
>
> The caller of deactivate_slab() and the caller of __slab_free() won't
> manipulate the slab list concurrently.
>
> So we can get node list_lock in the last stage if we really need to
> manipulate the slab list in this path.
>
> Signed-off-by: Chengming Zhou <zhouchengming@bytedance.com>
> Reviewed-by: Vlastimil Babka <vbabka@suse.cz>
> Tested-by: Hyeonggon Yoo <42.hyeyoo@gmail.com>
> ---
>  mm/slub.c | 79 ++++++++++++++++++-------------------------------------
>  1 file changed, 26 insertions(+), 53 deletions(-)
>
> diff --git a/mm/slub.c b/mm/slub.c
> index bcb5b2c4e213..d137468fe4b9 100644
> --- a/mm/slub.c
> +++ b/mm/slub.c
> @@ -2468,10 +2468,8 @@ static void init_kmem_cache_cpus(struct kmem_cache=
 *s)
>  static void deactivate_slab(struct kmem_cache *s, struct slab *slab,
>                             void *freelist)
>  {
> -       enum slab_modes { M_NONE, M_PARTIAL, M_FREE, M_FULL_NOLIST };
>         struct kmem_cache_node *n =3D get_node(s, slab_nid(slab));
>         int free_delta =3D 0;
> -       enum slab_modes mode =3D M_NONE;
>         void *nextfree, *freelist_iter, *freelist_tail;
>         int tail =3D DEACTIVATE_TO_HEAD;
>         unsigned long flags =3D 0;
> @@ -2509,65 +2507,40 @@ static void deactivate_slab(struct kmem_cache *s,=
 struct slab *slab,
>         /*
>          * Stage two: Unfreeze the slab while splicing the per-cpu
>          * freelist to the head of slab's freelist.
> -        *
> -        * Ensure that the slab is unfrozen while the list presence
> -        * reflects the actual number of objects during unfreeze.
> -        *
> -        * We first perform cmpxchg holding lock and insert to list
> -        * when it succeed. If there is mismatch then the slab is not
> -        * unfrozen and number of objects in the slab may have changed.
> -        * Then release lock and retry cmpxchg again.
>          */
> -redo:
> -
> -       old.freelist =3D READ_ONCE(slab->freelist);
> -       old.counters =3D READ_ONCE(slab->counters);
> -       VM_BUG_ON(!old.frozen);
> -
> -       /* Determine target state of the slab */
> -       new.counters =3D old.counters;
> -       if (freelist_tail) {
> -               new.inuse -=3D free_delta;
> -               set_freepointer(s, freelist_tail, old.freelist);
> -               new.freelist =3D freelist;
> -       } else
> -               new.freelist =3D old.freelist;
> -
> -       new.frozen =3D 0;
> +       do {
> +               old.freelist =3D READ_ONCE(slab->freelist);
> +               old.counters =3D READ_ONCE(slab->counters);
> +               VM_BUG_ON(!old.frozen);
> +
> +               /* Determine target state of the slab */
> +               new.counters =3D old.counters;
> +               new.frozen =3D 0;
> +               if (freelist_tail) {
> +                       new.inuse -=3D free_delta;
> +                       set_freepointer(s, freelist_tail, old.freelist);
> +                       new.freelist =3D freelist;
> +               } else {
> +                       new.freelist =3D old.freelist;
> +               }
> +       } while (!slab_update_freelist(s, slab,
> +               old.freelist, old.counters,
> +               new.freelist, new.counters,
> +               "unfreezing slab"));
>
> +       /*
> +        * Stage three: Manipulate the slab list based on the updated sta=
te.
> +        */

deactivate_slab() might unconsciously put empty slabs into partial list, li=
ke:

deactivate_slab()                    __slab_free()
cmpxchg(), slab's not empty
                                               cmpxchg(), slab's empty
and unfrozen
                                               spin_lock(&n->list_lock)
                                               (slab's empty but not
on partial list,

spin_unlock(&n->list_lock) and return)
spin_lock(&n->list_lock)
put slab into partial list
spin_unlock(&n->list_lock)

IMHO it should be fine in the real world, but just wanted to
mention as it doesn't seem to be intentional.

Otherwise it looks good to me!

>         if (!new.inuse && n->nr_partial >=3D s->min_partial) {
> -               mode =3D M_FREE;
> +               stat(s, DEACTIVATE_EMPTY);
> +               discard_slab(s, slab);
> +               stat(s, FREE_SLAB);
>         } else if (new.freelist) {
> -               mode =3D M_PARTIAL;
> -               /*
> -                * Taking the spinlock removes the possibility that
> -                * acquire_slab() will see a slab that is frozen
> -                */
>                 spin_lock_irqsave(&n->list_lock, flags);
> -       } else {
> -               mode =3D M_FULL_NOLIST;
> -       }
> -
> -
> -       if (!slab_update_freelist(s, slab,
> -                               old.freelist, old.counters,
> -                               new.freelist, new.counters,
> -                               "unfreezing slab")) {
> -               if (mode =3D=3D M_PARTIAL)
> -                       spin_unlock_irqrestore(&n->list_lock, flags);
> -               goto redo;
> -       }
> -
> -
> -       if (mode =3D=3D M_PARTIAL) {
>                 add_partial(n, slab, tail);
>                 spin_unlock_irqrestore(&n->list_lock, flags);
>                 stat(s, tail);
> -       } else if (mode =3D=3D M_FREE) {
> -               stat(s, DEACTIVATE_EMPTY);
> -               discard_slab(s, slab);
> -               stat(s, FREE_SLAB);
> -       } else if (mode =3D=3D M_FULL_NOLIST) {
> +       } else {
>                 stat(s, DEACTIVATE_FULL);
>         }
>  }
> --
> 2.20.1
>