Received-SPF: pass (google.com: domain of linux-kernel-owner@vger.kernel.org designates 23.128.96.33 as permitted sender) client-ip=23.128.96.33;
Received-SPF: Pass (protection.outlook.com: domain of micron.com designates
 137.201.242.130 as permitted sender) receiver=protection.outlook.com;
 client-ip=137.201.242.130; helo=mail.micron.com; pr=C
Message-ID: <d1c30b1f-1628-4092-acee-2b0980a3d888@micron.com>
Date:   Tue, 31 Oct 2023 23:53:15 +0530
MIME-Version: 1.0
User-Agent: Mozilla Thunderbird
Subject: Re: [EXT] [RFC PATCH v3 4/4] mm/mempolicy: modify interleave
 mempolicy to use node weights
Content-Language: en-US
To:     Gregory Price <gourry.memverge@gmail.com>,
        <linux-kernel@vger.kernel.org>
CC:     <linux-cxl@vger.kernel.org>, <linux-mm@kvack.org>,
        <ying.huang@intel.com>, <akpm@linux-foundation.org>,
        <aneesh.kumar@linux.ibm.com>, <weixugc@google.com>,
        <apopple@nvidia.com>, <hannes@cmpxchg.org>, <tim.c.chen@intel.com>,
        <dave.hansen@intel.com>, <mhocko@kernel.org>,
        <shy828301@gmail.com>, <gregkh@linuxfoundation.org>,
        <rafael@kernel.org>, Gregory Price <gregory.price@memverge.com>
References: <20231031003810.4532-1-gregory.price@memverge.com>
 <20231031003810.4532-5-gregory.price@memverge.com>
From:   Srinivasulu Thanneeru <sthanneeru.opensrc@micron.com>
In-Reply-To: <20231031003810.4532-5-gregory.price@memverge.com>
Content-Type: text/plain; charset="UTF-8"; format=flowed
Content-Transfer-Encoding: 7bit
X-MS-Exchange-CrossTenant-OriginalArrivalTime: 31 Oct 2023 18:23:22.7618
 (UTC)
X-MS-Exchange-CrossTenant-Network-Message-Id: b3fb78cf-2e2c-4de1-55f6-08dbda3e775e
X-MS-Exchange-CrossTenant-Id: f38a5ecd-2813-4862-b11b-ac1d563c806f
X-MS-Exchange-CrossTenant-OriginalAttributedTenantConnectingIp: TenantId=f38a5ecd-2813-4862-b11b-ac1d563c806f;Ip=[137.201.242.130];Helo=[mail.micron.com]
X-MS-Exchange-CrossTenant-AuthSource: MWH0EPF000971E3.namprd02.prod.outlook.com
X-MS-Exchange-CrossTenant-AuthAs: Anonymous
X-MS-Exchange-CrossTenant-FromEntityHeader: HybridOnPrem
X-MS-Exchange-Transport-CrossTenantHeadersStamped: PH0PR08MB8228
X-Spam-Status: No, score=-0.9 required=5.0 tests=DKIM_SIGNED,DKIM_VALID,
	DKIM_VALID_AU,HEADER_FROM_DIFFERENT_DOMAINS,MAILING_LIST_MULTI,
	SPF_HELO_NONE,SPF_PASS,T_SCC_BODY_TEXT_LINE autolearn=unavailable
	autolearn_force=no version=3.4.6
X-Spam-Checker-Version: SpamAssassin 3.4.6 (2021-04-09) on lipwig.vger.email
Precedence: bulk
List-ID: <linux-kernel.vger.kernel.org>
X-Mailing-List: linux-kernel@vger.kernel.org
X-Greylist: Sender passed SPF test, not delayed by milter-greylist-4.6.4 (lipwig.vger.email [0.0.0.0]); Tue, 31 Oct 2023 11:23:37 -0700 (PDT)


On 10/31/2023 6:08 AM, Gregory Price wrote:
>
>
> The node subsystem implements interleave weighting for the purpose
> of bandwidth optimization.  Each node may have different weights in
> relation to each compute node ("access node").
>
> The mempolicy MPOL_INTERLEAVE utilizes the node weights to implement
> weighted interleave.  By default, since all nodes default to a weight
> of 1, the original interleave behavior is retained.
>
> Examples
>
> Weight settings:
> echo 4 > node0/access0/il_weight
> echo 1 > node0/access1/il_weight
>
> echo 3 > node1/access0/il_weight
> echo 2 > node1/access1/il_weight
>
> Results:
>
> Task A:
>     cpunode:  0
>     nodemask: [0,1]
>     weights:  [4,3]
>     allocation result: [0,0,0,0,1,1,1 repeat]
>
> Task B:
>     cpunode:  1
>     nodemask: [0,1]
>     weights:  [1,2]
>     allocation result: [0,1,1 repeat]
>     Weights are relative to access node
>
> Signed-off-by: Gregory Price <gregory.price@memverge.com>
Thank you Gregory for the collaboration.
Signed-off-by: Srinivasulu Thanneeru <sthanneeru.opensrc@micron.com>
> ---
>   include/linux/mempolicy.h |   4 ++
>   mm/mempolicy.c            | 138 +++++++++++++++++++++++++++++---------
>   2 files changed, 112 insertions(+), 30 deletions(-)
>
> diff --git a/include/linux/mempolicy.h b/include/linux/mempolicy.h
> index d232de7cdc56..240468b669fd 100644
> --- a/include/linux/mempolicy.h
> +++ b/include/linux/mempolicy.h
> @@ -48,6 +48,10 @@ struct mempolicy {
>          nodemask_t nodes;       /* interleave/bind/perfer */
>          int home_node;          /* Home node to use for MPOL_BIND and MPOL_PREFERRED_MANY */
>
> +       /* weighted interleave settings */
> +       unsigned char cur_weight;
> +       unsigned char il_weights[MAX_NUMNODES];
> +
>          union {
>                  nodemask_t cpuset_mems_allowed; /* relative to these nodes */
>                  nodemask_t user_nodemask;       /* nodemask passed by user */
> diff --git a/mm/mempolicy.c b/mm/mempolicy.c
> index 29ebf1e7898c..d62e942a13bd 100644
> --- a/mm/mempolicy.c
> +++ b/mm/mempolicy.c
> @@ -102,6 +102,7 @@
>   #include <linux/mmu_notifier.h>
>   #include <linux/printk.h>
>   #include <linux/swapops.h>
> +#include <linux/memory-tiers.h>
>
>   #include <asm/tlbflush.h>
>   #include <asm/tlb.h>
> @@ -300,6 +301,7 @@ static struct mempolicy *mpol_new(unsigned short mode, unsigned short flags,
>          policy->mode = mode;
>          policy->flags = flags;
>          policy->home_node = NUMA_NO_NODE;
> +       policy->cur_weight = 0;
>
>          return policy;
>   }
> @@ -334,6 +336,7 @@ static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes)
>                  tmp = *nodes;
>
>          pol->nodes = tmp;
> +       pol->cur_weight = 0;
>   }
>
>   static void mpol_rebind_preferred(struct mempolicy *pol,
> @@ -881,8 +884,11 @@ static long do_set_mempolicy(unsigned short mode, unsigned short flags,
>
>          old = current->mempolicy;
>          current->mempolicy = new;
> -       if (new && new->mode == MPOL_INTERLEAVE)
> +       if (new && new->mode == MPOL_INTERLEAVE) {
>                  current->il_prev = MAX_NUMNODES-1;
> +               new->cur_weight = 0;
> +       }
> +
>          task_unlock(current);
>          mpol_put(old);
>          ret = 0;
> @@ -1903,12 +1909,21 @@ static int policy_node(gfp_t gfp, struct mempolicy *policy, int nd)
>   /* Do dynamic interleaving for a process */
>   static unsigned interleave_nodes(struct mempolicy *policy)
>   {
> -       unsigned next;
> +       unsigned int next;
> +       unsigned char next_weight;
>          struct task_struct *me = current;
>
>          next = next_node_in(me->il_prev, policy->nodes);
> -       if (next < MAX_NUMNODES)
> +       if (!policy->cur_weight) {
> +               /* If the node is set, at least 1 allocation is required */
> +               next_weight = node_get_il_weight(next, numa_node_id());
> +               policy->cur_weight = next_weight ? next_weight : 1;
> +       }
> +
> +       policy->cur_weight--;
> +       if (next < MAX_NUMNODES && !policy->cur_weight)
>                  me->il_prev = next;
> +
>          return next;
>   }
>
> @@ -1967,25 +1982,37 @@ unsigned int mempolicy_slab_node(void)
>   static unsigned offset_il_node(struct mempolicy *pol, unsigned long n)
>   {
>          nodemask_t nodemask = pol->nodes;
> -       unsigned int target, nnodes;
> -       int i;
> +       unsigned int target, nnodes, il_weight;
> +       unsigned char weight;
>          int nid;
> +       int cur_node = numa_node_id();
> +
>          /*
>           * The barrier will stabilize the nodemask in a register or on
>           * the stack so that it will stop changing under the code.
>           *
>           * Between first_node() and next_node(), pol->nodes could be changed
>           * by other threads. So we put pol->nodes in a local stack.
> +        *
> +        * Additionally, place the cur_node on the stack in case of a migration
>           */
>          barrier();
>
>          nnodes = nodes_weight(nodemask);
>          if (!nnodes)
> -               return numa_node_id();
> -       target = (unsigned int)n % nnodes;
> +               return cur_node;
> +
> +       il_weight = nodes_get_il_weights(cur_node, &nodemask, pol->il_weights);
> +       target = (unsigned int)n % il_weight;
>          nid = first_node(nodemask);
> -       for (i = 0; i < target; i++)
> -               nid = next_node(nid, nodemask);
> +       while (target) {
> +               weight = pol->il_weights[nid];
> +               if (target < weight)
> +                       break;
> +               target -= weight;
> +               nid = next_node_in(nid, nodemask);
> +       }
> +
>          return nid;
>   }
>
> @@ -2319,32 +2346,83 @@ static unsigned long alloc_pages_bulk_array_interleave(gfp_t gfp,
>                  struct mempolicy *pol, unsigned long nr_pages,
>                  struct page **page_array)
>   {
> -       int nodes;
> -       unsigned long nr_pages_per_node;
> -       int delta;
> -       int i;
> -       unsigned long nr_allocated;
> +       struct task_struct *me = current;
>          unsigned long total_allocated = 0;
> +       unsigned long nr_allocated;
> +       unsigned long rounds;
> +       unsigned long node_pages, delta;
> +       unsigned char weight;
> +       unsigned long il_weight;
> +       unsigned long req_pages = nr_pages;
> +       int nnodes, node, prev_node;
> +       int cur_node = numa_node_id();
> +       int i;
>
> -       nodes = nodes_weight(pol->nodes);
> -       nr_pages_per_node = nr_pages / nodes;
> -       delta = nr_pages - nodes * nr_pages_per_node;
> -
> -       for (i = 0; i < nodes; i++) {
> -               if (delta) {
> -                       nr_allocated = __alloc_pages_bulk(gfp,
> -                                       interleave_nodes(pol), NULL,
> -                                       nr_pages_per_node + 1, NULL,
> -                                       page_array);
> -                       delta--;
> -               } else {
> -                       nr_allocated = __alloc_pages_bulk(gfp,
> -                                       interleave_nodes(pol), NULL,
> -                                       nr_pages_per_node, NULL, page_array);
> +       prev_node = me->il_prev;
> +       nnodes = nodes_weight(pol->nodes);
> +       /* Continue allocating from most recent node */
> +       if (pol->cur_weight) {
> +               node = next_node_in(prev_node, pol->nodes);
> +               node_pages = pol->cur_weight;
> +               if (node_pages > nr_pages)
> +                       node_pages = nr_pages;
> +               nr_allocated = __alloc_pages_bulk(gfp, node, NULL, node_pages,
> +                                                 NULL, page_array);
> +               page_array += nr_allocated;
> +               total_allocated += nr_allocated;
> +               /* if that's all the pages, no need to interleave */
> +               if (req_pages <= pol->cur_weight) {
> +                       pol->cur_weight -= req_pages;
> +                       return total_allocated;
>                  }
> -
> +               /* Otherwise we adjust req_pages down, and continue from there */
> +               req_pages -= pol->cur_weight;
> +               pol->cur_weight = 0;
> +               prev_node = node;
> +       }
> +
> +       il_weight = nodes_get_il_weights(cur_node, &pol->nodes,
> +                                        pol->il_weights);
> +       rounds = req_pages / il_weight;
> +       delta = req_pages % il_weight;
> +       for (i = 0; i < nnodes; i++) {
> +               node = next_node_in(prev_node, pol->nodes);
> +               weight = pol->il_weights[node];
> +               node_pages = weight * rounds;
> +               if (delta > weight) {
> +                       node_pages += weight;
> +                       delta -= weight;
> +               } else if (delta) {
> +                       node_pages += delta;
> +                       delta = 0;
> +               }
> +               /* The number of requested pages may not hit every node */
> +               if (!node_pages)
> +                       break;
> +               /* If an over-allocation would occur, floor it */
> +               if (node_pages + total_allocated > nr_pages) {
> +                       node_pages = nr_pages - total_allocated;
> +                       delta = 0;
> +               }
> +               nr_allocated = __alloc_pages_bulk(gfp, node, NULL, node_pages,
> +                                                 NULL, page_array);
>                  page_array += nr_allocated;
>                  total_allocated += nr_allocated;
> +               prev_node = node;
> +       }
> +
> +       /*
> +        * Finally, we need to update me->il_prev and pol->cur_weight
> +        * If the last node allocated on has un-used weight, apply
> +        * the remainder as the cur_weight, otherwise proceed to next node
> +        */
> +       if (node_pages) {
> +               me->il_prev = prev_node;
> +               node_pages %= weight;
> +               pol->cur_weight = weight - node_pages;
> +       } else {
> +               me->il_prev = node;
> +               pol->cur_weight = 0;
>          }
>
>          return total_allocated;
> --
> 2.39.1
>
>