Received-SPF: pass (google.com: domain of linux-kernel+bounces-130786-linux.lists.archive=gmail.com@vger.kernel.org designates 2604:1380:4601:e00::3 as permitted sender) client-ip=2604:1380:4601:e00::3;
Precedence: bulk
MIME-Version: 1.0
References: <cover.1711674410.git.babu.moger@amd.com> <0db75c94886da62b8da498ef159d8fe27b0b3811.1711674410.git.babu.moger@amd.com>
In-Reply-To: <0db75c94886da62b8da498ef159d8fe27b0b3811.1711674410.git.babu.moger@amd.com>
From: Peter Newman <peternewman@google.com>
Date: Wed, 3 Apr 2024 17:30:53 -0700
Message-ID: <CALPaoCgRtYLnzpkL6uVjb+LvBbxD8ANRuzACP1UQ=rkzk8TCqg@mail.gmail.com>
Subject: Re: [RFC PATCH v3 07/17] x86/resctrl: Add support to enable/disable
 ABMC feature
To: Babu Moger <babu.moger@amd.com>
Cc: corbet@lwn.net, fenghua.yu@intel.com, reinette.chatre@intel.com, 
	tglx@linutronix.de, mingo@redhat.com, bp@alien8.de, 
	dave.hansen@linux.intel.com, x86@kernel.org, hpa@zytor.com, 
	paulmck@kernel.org, rdunlap@infradead.org, tj@kernel.org, 
	peterz@infradead.org, yanjiewtw@gmail.com, kim.phillips@amd.com, 
	lukas.bulwahn@gmail.com, seanjc@google.com, jmattson@google.com, 
	leitao@debian.org, jpoimboe@kernel.org, rick.p.edgecombe@intel.com, 
	kirill.shutemov@linux.intel.com, jithu.joseph@intel.com, kai.huang@intel.com, 
	kan.liang@linux.intel.com, daniel.sneddon@linux.intel.com, 
	pbonzini@redhat.com, sandipan.das@amd.com, ilpo.jarvinen@linux.intel.com, 
	maciej.wieczor-retman@intel.com, linux-doc@vger.kernel.org, 
	linux-kernel@vger.kernel.org, eranian@google.com, james.morse@arm.com
Content-Type: text/plain; charset="UTF-8"
Content-Transfer-Encoding: quoted-printable

Hi Babu,

On Thu, Mar 28, 2024 at 6:07=E2=80=AFPM Babu Moger <babu.moger@amd.com> wro=
te:
>  struct rdt_fs_context {
>         struct kernfs_fs_context        kfc;
>         bool                            enable_cdpl2;
> @@ -433,6 +436,7 @@ struct rdt_parse_data {
>   * @mbm_cfg_mask:      Bandwidth sources that can be tracked when Bandwi=
dth
>   *                     Monitoring Event Configuration (BMEC) is supporte=
d.
>   * @cdp_enabled:       CDP state of this resource
> + * @abmc_enabled:      ABMC feature is enabled
>   *
>   * Members of this structure are either private to the architecture
>   * e.g. mbm_width, or accessed via helpers that provide abstraction. e.g=
.
> @@ -448,6 +452,7 @@ struct rdt_hw_resource {
>         unsigned int            mbm_width;
>         unsigned int            mbm_cfg_mask;
>         bool                    cdp_enabled;
> +       bool                    abmc_enabled;
>  };
>
>  static inline struct rdt_hw_resource *resctrl_to_arch_res(struct rdt_res=
ource *r)
> @@ -491,6 +496,13 @@ static inline bool resctrl_arch_get_cdp_enabled(enum=
 resctrl_res_level l)
>
>  int resctrl_arch_set_cdp_enabled(enum resctrl_res_level l, bool enable);
>
> +static inline bool resctrl_arch_get_abmc_enabled(enum resctrl_res_level =
l)
> +{
> +       return rdt_resources_all[l].abmc_enabled;
> +}

This inline definition will not work in either this file or
fs/resctrl/internal.h, following James's change[1] moving the code.

resctrl_arch-definitions are either declared in linux/resctrl.h or
defined inline in a file like asm/resctrl.h.


> +
> +int resctrl_arch_set_abmc_enabled(enum resctrl_res_level l, bool enable)=
;
> +
>  /*
>   * To return the common struct rdt_resource, which is contained in struc=
t
>   * rdt_hw_resource, walk the resctrl member of struct rdt_hw_resource.
> diff --git a/arch/x86/kernel/cpu/resctrl/rdtgroup.c b/arch/x86/kernel/cpu=
/resctrl/rdtgroup.c
> index 05f551bc316e..f49073c86884 100644
> --- a/arch/x86/kernel/cpu/resctrl/rdtgroup.c
> +++ b/arch/x86/kernel/cpu/resctrl/rdtgroup.c
> @@ -850,9 +850,15 @@ static int rdtgroup_mbm_assign_show(struct kernfs_op=
en_file *of,
>                                     struct seq_file *s, void *v)
>  {
>         struct rdt_resource *r =3D of->kn->parent->priv;
> +       struct rdt_hw_resource *hw_res =3D resctrl_to_arch_res(r);
>
> -       if (r->mbm_assign_capable)
> +       if (r->mbm_assign_capable && hw_res->abmc_enabled) {
> +               seq_puts(s, "[abmc]\n");
> +               seq_puts(s, "legacy_mbm\n");
> +       } else if (r->mbm_assign_capable) {
>                 seq_puts(s, "abmc\n");
> +               seq_puts(s, "[legacy_mbm]\n");
> +       }

This looks like it would move to fs/resctrl/rdtgroup.c where it's not
possible to dereference an rdt_hw_resource struct.

It might be helpful to try building your changes on top of James's
change[1] to get an idea of how this would fit in post-refactoring.
I'll stop pointing out inconsistencies with his portability scheme
now.

>
>         return 0;
>  }
> @@ -2433,6 +2439,74 @@ int resctrl_arch_set_cdp_enabled(enum resctrl_res_=
level l, bool enable)
>         return 0;
>  }
>
> +static void resctrl_abmc_msrwrite(void *arg)
> +{
> +       bool *enable =3D arg;
> +       u64 msrval;
> +
> +       rdmsrl(MSR_IA32_L3_QOS_EXT_CFG, msrval);
> +
> +       if (*enable)
> +               msrval |=3D ABMC_ENABLE;
> +       else
> +               msrval &=3D ~ABMC_ENABLE;
> +
> +       wrmsrl(MSR_IA32_L3_QOS_EXT_CFG, msrval);
> +}
> +
> +static int resctrl_abmc_setup(enum resctrl_res_level l, bool enable)
> +{
> +       struct rdt_resource *r =3D &rdt_resources_all[l].r_resctrl;
> +       struct rdt_domain *d;
> +
> +       /* Update QOS_CFG MSR on all the CPUs in cpu_mask */
> +       list_for_each_entry(d, &r->domains, list) {
> +               on_each_cpu_mask(&d->cpu_mask, resctrl_abmc_msrwrite, &en=
able, 1);
> +               resctrl_arch_reset_rmid_all(r, d);
> +       }
> +
> +       return 0;
> +}
> +
> +static int resctrl_abmc_enable(enum resctrl_res_level l)
> +{
> +       struct rdt_hw_resource *hw_res =3D &rdt_resources_all[l];
> +       int ret =3D 0;
> +
> +       if (!hw_res->abmc_enabled) {
> +               ret =3D resctrl_abmc_setup(l, true);
> +               if (!ret)
> +                       hw_res->abmc_enabled =3D true;

Presumably this would be called holding the rdtgroup_mutex? Perhaps a
lockdep assertion somewhere would be appropriate?

Thanks!
-Peter

[1] https://lore.kernel.org/lkml/20240321165106.31602-32-james.morse@arm.co=
m/