2020-08-19 08:10:10

by liuqi (BA)

Subject: [RFC PATCH v2] coresight: etm4x: Modify core-commit of cpu to avoid the overflow of HiSilicon ETM

When too much trace information is generated on-chip, the ETM will
overflow, and cause data loss. This is a common phenomenon on ETM
devices.

But sometimes we do not want to lose performance trace data, so we
suppress the speed of instructions sent from CPU core to ETM to
avoid the overflow of ETM.

Signed-off-by: Qi Liu <[email protected]>
---

Changes since v1:
- ETM on HiSilicon Hip09 platform supports backpressure, so does
not need to modify core commit.

drivers/hwtracing/coresight/coresight-etm4x.c | 43 +++++++++++++++++++++++++++
1 file changed, 43 insertions(+)

diff --git a/drivers/hwtracing/coresight/coresight-etm4x.c b/drivers/hwtracing/coresight/coresight-etm4x.c
index 7797a57..7641f89 100644
--- a/drivers/hwtracing/coresight/coresight-etm4x.c
+++ b/drivers/hwtracing/coresight/coresight-etm4x.c
@@ -43,6 +43,10 @@ MODULE_PARM_DESC(boot_enable, "Enable tracing on boot");
#define PARAM_PM_SAVE_NEVER 1 /* never save any state */
#define PARAM_PM_SAVE_SELF_HOSTED 2 /* save self-hosted state only */

+#define CORE_COMMIT_CLEAR 0x3000
+#define CORE_COMMIT_SHIFT 12
+#define HISI_ETM_AMBA_ID_V1 0x000b6d01
+
static int pm_save_enable = PARAM_PM_SAVE_FIRMWARE;
module_param(pm_save_enable, int, 0444);
MODULE_PARM_DESC(pm_save_enable,
@@ -104,11 +108,40 @@ struct etm4_enable_arg {
int rc;
};

+static void etm4_cpu_actlr1_cfg(void *info)
+{
+ struct etm4_enable_arg *arg = (struct etm4_enable_arg *)info;
+ u64 val;
+
+ asm volatile("mrs %0,s3_1_c15_c2_5" : "=r"(val));
+ val &= ~CORE_COMMIT_CLEAR;
+ val |= arg->rc << CORE_COMMIT_SHIFT;
+ asm volatile("msr s3_1_c15_c2_5,%0" : : "r"(val));
+}
+
+static void etm4_config_core_commit(int cpu, int val)
+{
+ struct etm4_enable_arg arg = {0};
+
+ arg.rc = val;
+ smp_call_function_single(cpu, etm4_cpu_actlr1_cfg, &arg, 1);
+}
+
static int etm4_enable_hw(struct etmv4_drvdata *drvdata)
{
int i, rc;
+ struct amba_device *adev;
struct etmv4_config *config = &drvdata->config;
struct device *etm_dev = &drvdata->csdev->dev;
+ struct device *dev = drvdata->csdev->dev.parent;
+
+ adev = container_of(dev, struct amba_device, dev);
+ /*
+ * If ETM device is HiSilicon ETM device, reduce the
+ * core-commit to avoid ETM overflow.
+ */
+ if (adev->periphid == HISI_ETM_AMBA_ID_V1)
+ etm4_config_core_commit(drvdata->cpu, 1);

CS_UNLOCK(drvdata->base);

@@ -472,10 +505,20 @@ static void etm4_disable_hw(void *info)
{
u32 control;
struct etmv4_drvdata *drvdata = info;
+ struct device *dev = drvdata->csdev->dev.parent;
struct etmv4_config *config = &drvdata->config;
struct device *etm_dev = &drvdata->csdev->dev;
+ struct amba_device *adev;
int i;

+ adev = container_of(dev, struct amba_device, dev);
+ /*
+ * If ETM device is HiSilicon ETM device, resume the
+ * core-commit after ETM trace is complete.
+ */
+ if (adev->periphid == HISI_ETM_AMBA_ID_V1)
+ etm4_config_core_commit(drvdata->cpu, 0);
+
CS_UNLOCK(drvdata->base);

if (!drvdata->skip_power_up) {
--
2.8.1

2020-08-27 20:47:49

by Mathieu Poirier

Subject: Re: [RFC PATCH v2] coresight: etm4x: Modify core-commit of cpu to avoid the overflow of HiSilicon ETM

Hi Liu,

On Wed, Aug 19, 2020 at 04:06:37PM +0800, Qi Liu wrote:
> When too much trace information is generated on-chip, the ETM will
> overflow, and cause data loss. This is a common phenomenon on ETM
> devices.
>
> But sometimes we do not want to lose performance trace data, so we
> suppress the speed of instructions sent from CPU core to ETM to
> avoid the overflow of ETM.
>
> Signed-off-by: Qi Liu <[email protected]>
> ---
>
> Changes since v1:
> - ETM on HiSilicon Hip09 platform supports backpressure, so does
> not need to modify core commit.
>
> drivers/hwtracing/coresight/coresight-etm4x.c | 43 +++++++++++++++++++++++++++
> 1 file changed, 43 insertions(+)
>
> diff --git a/drivers/hwtracing/coresight/coresight-etm4x.c b/drivers/hwtracing/coresight/coresight-etm4x.c
> index 7797a57..7641f89 100644
> --- a/drivers/hwtracing/coresight/coresight-etm4x.c
> +++ b/drivers/hwtracing/coresight/coresight-etm4x.c
> @@ -43,6 +43,10 @@ MODULE_PARM_DESC(boot_enable, "Enable tracing on boot");
> #define PARAM_PM_SAVE_NEVER 1 /* never save any state */
> #define PARAM_PM_SAVE_SELF_HOSTED 2 /* save self-hosted state only */
>
> +#define CORE_COMMIT_CLEAR 0x3000
> +#define CORE_COMMIT_SHIFT 12
> +#define HISI_ETM_AMBA_ID_V1 0x000b6d01
> +
> static int pm_save_enable = PARAM_PM_SAVE_FIRMWARE;
> module_param(pm_save_enable, int, 0444);
> MODULE_PARM_DESC(pm_save_enable,
> @@ -104,11 +108,40 @@ struct etm4_enable_arg {
> int rc;
> };
>
> +static void etm4_cpu_actlr1_cfg(void *info)
> +{
> + struct etm4_enable_arg *arg = (struct etm4_enable_arg *)info;
> + u64 val;
> +
> + asm volatile("mrs %0,s3_1_c15_c2_5" : "=r"(val));
> + val &= ~CORE_COMMIT_CLEAR;
> + val |= arg->rc << CORE_COMMIT_SHIFT;
> + asm volatile("msr s3_1_c15_c2_5,%0" : : "r"(val));
> +}
> +
> +static void etm4_config_core_commit(int cpu, int val)
> +{
> + struct etm4_enable_arg arg = {0};
> +
> + arg.rc = val;
> + smp_call_function_single(cpu, etm4_cpu_actlr1_cfg, &arg, 1);

Function etm4_enable/disable_hw() are already running on the CPU they are
supposed to so no need to call smp_call_function_single().

> +}
> +
> static int etm4_enable_hw(struct etmv4_drvdata *drvdata)
> {
> int i, rc;
> + struct amba_device *adev;
> struct etmv4_config *config = &drvdata->config;
> struct device *etm_dev = &drvdata->csdev->dev;
> + struct device *dev = drvdata->csdev->dev.parent;
> +
> + adev = container_of(dev, struct amba_device, dev);
> + /*
> + * If ETM device is HiSilicon ETM device, reduce the
> + * core-commit to avoid ETM overflow.
> + */
> + if (adev->periphid == HISI_ETM_AMBA_ID_V1)

Do you have any documentation on this back pressure feature? I doubt this is
specific to Hip09 platform and as such would prefer to have a more generic
approach that works on any platform that supports it.

Anyone on the CS mailing list that knows what this is about?

Thanks,
Mathieu

> + etm4_config_core_commit(drvdata->cpu, 1);
>
> CS_UNLOCK(drvdata->base);
>
> @@ -472,10 +505,20 @@ static void etm4_disable_hw(void *info)
> {
> u32 control;
> struct etmv4_drvdata *drvdata = info;
> + struct device *dev = drvdata->csdev->dev.parent;
> struct etmv4_config *config = &drvdata->config;
> struct device *etm_dev = &drvdata->csdev->dev;
> + struct amba_device *adev;
> int i;
>
> + adev = container_of(dev, struct amba_device, dev);
> + /*
> + * If ETM device is HiSilicon ETM device, resume the
> + * core-commit after ETM trace is complete.
> + */
> + if (adev->periphid == HISI_ETM_AMBA_ID_V1)
> + etm4_config_core_commit(drvdata->cpu, 0);
> +
> CS_UNLOCK(drvdata->base);
>
> if (!drvdata->skip_power_up) {
> --
> 2.8.1
>

2020-08-28 09:01:39

by Al Grant

Subject: RE: [RFC PATCH v2] coresight: etm4x: Modify core-commit of cpu to avoid the overflow of HiSilicon ETM

Hi Mathieu and CS maintainers,

> Hi Liu,
>
> On Wed, Aug 19, 2020 at 04:06:37PM +0800, Qi Liu wrote:
> > When too much trace information is generated on-chip, the ETM will
> > overflow, and cause data loss. This is a common phenomenon on ETM
> > devices.
> >
> > But sometimes we do not want to lose performance trace data, so we
> > suppress the speed of instructions sent from CPU core to ETM to avoid
> > the overflow of ETM.
> >
> > Signed-off-by: Qi Liu <[email protected]>
> > ---
> >
> > Changes since v1:
> > - ETM on HiSilicon Hip09 platform supports backpressure, so does not
> > need to modify core commit.
> >
> > drivers/hwtracing/coresight/coresight-etm4x.c | 43
> > +++++++++++++++++++++++++++
> > 1 file changed, 43 insertions(+)
> >
> > diff --git a/drivers/hwtracing/coresight/coresight-etm4x.c
> > b/drivers/hwtracing/coresight/coresight-etm4x.c
> > index 7797a57..7641f89 100644
> > --- a/drivers/hwtracing/coresight/coresight-etm4x.c
> > +++ b/drivers/hwtracing/coresight/coresight-etm4x.c
> > @@ -43,6 +43,10 @@ MODULE_PARM_DESC(boot_enable, "Enable tracing
> on boot");
> > #define PARAM_PM_SAVE_NEVER 1 /* never save any state */
> > #define PARAM_PM_SAVE_SELF_HOSTED 2 /* save self-hosted state only */
> >
> > +#define CORE_COMMIT_CLEAR 0x3000
> > +#define CORE_COMMIT_SHIFT 12
> > +#define HISI_ETM_AMBA_ID_V1 0x000b6d01
> > +
> > static int pm_save_enable = PARAM_PM_SAVE_FIRMWARE;
> > module_param(pm_save_enable, int, 0444);
> > MODULE_PARM_DESC(pm_save_enable, @@ -104,11 +108,40 @@ struct
> > etm4_enable_arg {
> > int rc;
> > };
> >
> > +static void etm4_cpu_actlr1_cfg(void *info) {
> > + struct etm4_enable_arg *arg = (struct etm4_enable_arg *)info;
> > + u64 val;
> > +
> > + asm volatile("mrs %0,s3_1_c15_c2_5" : "=r"(val));
> > + val &= ~CORE_COMMIT_CLEAR;
> > + val |= arg->rc << CORE_COMMIT_SHIFT;
> > + asm volatile("msr s3_1_c15_c2_5,%0" : : "r"(val)); }
> > +
> > +static void etm4_config_core_commit(int cpu, int val) {
> > + struct etm4_enable_arg arg = {0};
> > +
> > + arg.rc = val;
> > + smp_call_function_single(cpu, etm4_cpu_actlr1_cfg, &arg, 1);
>
> Function etm4_enable/disable_hw() are already running on the CPU they are
> supposed to so no need to call smp_call_function_single().
>
> > +}
> > +
> > static int etm4_enable_hw(struct etmv4_drvdata *drvdata) {
> > int i, rc;
> > + struct amba_device *adev;
> > struct etmv4_config *config = &drvdata->config;
> > struct device *etm_dev = &drvdata->csdev->dev;
> > + struct device *dev = drvdata->csdev->dev.parent;
> > +
> > + adev = container_of(dev, struct amba_device, dev);
> > + /*
> > + * If ETM device is HiSilicon ETM device, reduce the
> > + * core-commit to avoid ETM overflow.
> > + */
> > + if (adev->periphid == HISI_ETM_AMBA_ID_V1)
>
> Do you have any documentation on this back pressure feature? I doubt this is
> specific to Hip09 platform and as such would prefer to have a more generic
> approach that works on any platform that supports it.

It's not a standard Arm architecture feature, this is a model-specific register.
Some cores may be able to throttle throughput under user control,
but this isn't standardized. It may (as in this case) be something that you
want to enable whenever you enable ETM - and, I guess, disable whenever
you disable ETM. It's a bit unclean to have model-specific code in the main
ETM driver, and names like CORE_COMMIT_CLEAR really ought to be more
specific, but I don't see that it's more ugly than the model-specific code in
e.g. arch/arm64/kernel/perf_event.c or its equivalent on other architectures.

Ideally, a core that has an inherent difficulty generating ETM at full speed,
i.e. where the ETM can't keep up with the core pipeline, would reduce
commit rate automatically, and some already do. So if this core needs it
to be done manually via a system register, we might allow that in the
same way as we might allow other core-specific actions that need to be
done to enable ETM.

There are of course issues with trace overflow at all stages up to and
including harvesting trace from buffers into perf.data (for which solutions
might involve DVFS, trace strobing, scheduling etc.), and I am assuming
this patch is not addressing those but dealing with a very specific concern
about overflow within the core and its ETM.

Al


>
> Anyone on the CS mailing list that knows what this is about?
>
> Thanks,
> Mathieu
>
> > + etm4_config_core_commit(drvdata->cpu, 1);
> >
> > CS_UNLOCK(drvdata->base);
> >
> > @@ -472,10 +505,20 @@ static void etm4_disable_hw(void *info) {
> > u32 control;
> > struct etmv4_drvdata *drvdata = info;
> > + struct device *dev = drvdata->csdev->dev.parent;
> > struct etmv4_config *config = &drvdata->config;
> > struct device *etm_dev = &drvdata->csdev->dev;
> > + struct amba_device *adev;
> > int i;
> >
> > + adev = container_of(dev, struct amba_device, dev);
> > + /*
> > + * If ETM device is HiSilicon ETM device, resume the
> > + * core-commit after ETM trace is complete.
> > + */
> > + if (adev->periphid == HISI_ETM_AMBA_ID_V1)
> > + etm4_config_core_commit(drvdata->cpu, 0);
> > +
> > CS_UNLOCK(drvdata->base);
> >
> > if (!drvdata->skip_power_up) {
> > --
> > 2.8.1
> >
> _______________________________________________
> CoreSight mailing list
> [email protected]
> https://lists.linaro.org/mailman/listinfo/coresight

2020-08-31 23:01:31

by Mathieu Poirier

Subject: Re: [RFC PATCH v2] coresight: etm4x: Modify core-commit of cpu to avoid the overflow of HiSilicon ETM

Following Al's comment I have the following recommendations...

On Wed, Aug 19, 2020 at 04:06:37PM +0800, Qi Liu wrote:
> When too much trace information is generated on-chip, the ETM will
> overflow, and cause data loss. This is a common phenomenon on ETM
> devices.
>
> But sometimes we do not want to lose performance trace data, so we
> suppress the speed of instructions sent from CPU core to ETM to
> avoid the overflow of ETM.
>
> Signed-off-by: Qi Liu <[email protected]>
> ---
>
> Changes since v1:
> - ETM on HiSilicon Hip09 platform supports backpressure, so does
> not need to modify core commit.
>
> drivers/hwtracing/coresight/coresight-etm4x.c | 43 +++++++++++++++++++++++++++
> 1 file changed, 43 insertions(+)
>
> diff --git a/drivers/hwtracing/coresight/coresight-etm4x.c b/drivers/hwtracing/coresight/coresight-etm4x.c
> index 7797a57..7641f89 100644
> --- a/drivers/hwtracing/coresight/coresight-etm4x.c
> +++ b/drivers/hwtracing/coresight/coresight-etm4x.c
> @@ -43,6 +43,10 @@ MODULE_PARM_DESC(boot_enable, "Enable tracing on boot");
> #define PARAM_PM_SAVE_NEVER 1 /* never save any state */
> #define PARAM_PM_SAVE_SELF_HOSTED 2 /* save self-hosted state only */
>
> +#define CORE_COMMIT_CLEAR 0x3000
> +#define CORE_COMMIT_SHIFT 12
> +#define HISI_ETM_AMBA_ID_V1 0x000b6d01

Do you have a name for the SoC that can be added so that other HiSilicon SoC can
be added? I suggest something like:

#define HISI_NAME_CORE_COMMIT_CLEAR
#define HISI_NAME_CORE_COMMIT_SHIFT
#define HISI_NAME_ETM_ID

Moreover I don't see an entry for 0x000b6d01 in the amba id table - is your
devices upstream? Needless to day that is mandatory in order to move forward
with this work.

> +
> static int pm_save_enable = PARAM_PM_SAVE_FIRMWARE;
> module_param(pm_save_enable, int, 0444);
> MODULE_PARM_DESC(pm_save_enable,
> @@ -104,11 +108,40 @@ struct etm4_enable_arg {
> int rc;
> };
>
> +static void etm4_cpu_actlr1_cfg(void *info)
> +{
> + struct etm4_enable_arg *arg = (struct etm4_enable_arg *)info;
> + u64 val;
> +
> + asm volatile("mrs %0,s3_1_c15_c2_5" : "=r"(val));
> + val &= ~CORE_COMMIT_CLEAR;
> + val |= arg->rc << CORE_COMMIT_SHIFT;
> + asm volatile("msr s3_1_c15_c2_5,%0" : : "r"(val));
> +}
> +
> +static void etm4_config_core_commit(int cpu, int val)
> +{
> + struct etm4_enable_arg arg = {0};
> +
> + arg.rc = val;
> + smp_call_function_single(cpu, etm4_cpu_actlr1_cfg, &arg, 1);
> +}
> +
> static int etm4_enable_hw(struct etmv4_drvdata *drvdata)
> {
> int i, rc;
> + struct amba_device *adev;
> struct etmv4_config *config = &drvdata->config;
> struct device *etm_dev = &drvdata->csdev->dev;
> + struct device *dev = drvdata->csdev->dev.parent;
> +
> + adev = container_of(dev, struct amba_device, dev);
> + /*
> + * If ETM device is HiSilicon ETM device, reduce the
> + * core-commit to avoid ETM overflow.
> + */
> + if (adev->periphid == HISI_ETM_AMBA_ID_V1)
> + etm4_config_core_commit(drvdata->cpu, 1);

I susggest to add a function like etm4_enable_arch_specific() and do the above
in there. The same goes for the disable path.

Thanks,
Mathieu

>
> CS_UNLOCK(drvdata->base);
>
> @@ -472,10 +505,20 @@ static void etm4_disable_hw(void *info)
> {
> u32 control;
> struct etmv4_drvdata *drvdata = info;
> + struct device *dev = drvdata->csdev->dev.parent;
> struct etmv4_config *config = &drvdata->config;
> struct device *etm_dev = &drvdata->csdev->dev;
> + struct amba_device *adev;
> int i;
>
> + adev = container_of(dev, struct amba_device, dev);
> + /*
> + * If ETM device is HiSilicon ETM device, resume the
> + * core-commit after ETM trace is complete.
> + */
> + if (adev->periphid == HISI_ETM_AMBA_ID_V1)
> + etm4_config_core_commit(drvdata->cpu, 0);
> +
> CS_UNLOCK(drvdata->base);
>
> if (!drvdata->skip_power_up) {
> --
> 2.8.1
>

2020-08-31 23:02:47

by Mathieu Poirier

Subject: Re: [RFC PATCH v2] coresight: etm4x: Modify core-commit of cpu to avoid the overflow of HiSilicon ETM

On Fri, Aug 28, 2020 at 09:00:16AM +0000, Al Grant wrote:
> Hi Mathieu and CS maintainers,
>
> > Hi Liu,
> >
> > On Wed, Aug 19, 2020 at 04:06:37PM +0800, Qi Liu wrote:
> > > When too much trace information is generated on-chip, the ETM will
> > > overflow, and cause data loss. This is a common phenomenon on ETM
> > > devices.
> > >
> > > But sometimes we do not want to lose performance trace data, so we
> > > suppress the speed of instructions sent from CPU core to ETM to avoid
> > > the overflow of ETM.
> > >
> > > Signed-off-by: Qi Liu <[email protected]>
> > > ---
> > >
> > > Changes since v1:
> > > - ETM on HiSilicon Hip09 platform supports backpressure, so does not
> > > need to modify core commit.
> > >
> > > drivers/hwtracing/coresight/coresight-etm4x.c | 43
> > > +++++++++++++++++++++++++++
> > > 1 file changed, 43 insertions(+)
> > >
> > > diff --git a/drivers/hwtracing/coresight/coresight-etm4x.c
> > > b/drivers/hwtracing/coresight/coresight-etm4x.c
> > > index 7797a57..7641f89 100644
> > > --- a/drivers/hwtracing/coresight/coresight-etm4x.c
> > > +++ b/drivers/hwtracing/coresight/coresight-etm4x.c
> > > @@ -43,6 +43,10 @@ MODULE_PARM_DESC(boot_enable, "Enable tracing
> > on boot");
> > > #define PARAM_PM_SAVE_NEVER 1 /* never save any state */
> > > #define PARAM_PM_SAVE_SELF_HOSTED 2 /* save self-hosted state only */
> > >
> > > +#define CORE_COMMIT_CLEAR 0x3000
> > > +#define CORE_COMMIT_SHIFT 12
> > > +#define HISI_ETM_AMBA_ID_V1 0x000b6d01
> > > +
> > > static int pm_save_enable = PARAM_PM_SAVE_FIRMWARE;
> > > module_param(pm_save_enable, int, 0444);
> > > MODULE_PARM_DESC(pm_save_enable, @@ -104,11 +108,40 @@ struct
> > > etm4_enable_arg {
> > > int rc;
> > > };
> > >
> > > +static void etm4_cpu_actlr1_cfg(void *info) {
> > > + struct etm4_enable_arg *arg = (struct etm4_enable_arg *)info;
> > > + u64 val;
> > > +
> > > + asm volatile("mrs %0,s3_1_c15_c2_5" : "=r"(val));
> > > + val &= ~CORE_COMMIT_CLEAR;
> > > + val |= arg->rc << CORE_COMMIT_SHIFT;
> > > + asm volatile("msr s3_1_c15_c2_5,%0" : : "r"(val)); }
> > > +
> > > +static void etm4_config_core_commit(int cpu, int val) {
> > > + struct etm4_enable_arg arg = {0};
> > > +
> > > + arg.rc = val;
> > > + smp_call_function_single(cpu, etm4_cpu_actlr1_cfg, &arg, 1);
> >
> > Function etm4_enable/disable_hw() are already running on the CPU they are
> > supposed to so no need to call smp_call_function_single().
> >
> > > +}
> > > +
> > > static int etm4_enable_hw(struct etmv4_drvdata *drvdata) {
> > > int i, rc;
> > > + struct amba_device *adev;
> > > struct etmv4_config *config = &drvdata->config;
> > > struct device *etm_dev = &drvdata->csdev->dev;
> > > + struct device *dev = drvdata->csdev->dev.parent;
> > > +
> > > + adev = container_of(dev, struct amba_device, dev);
> > > + /*
> > > + * If ETM device is HiSilicon ETM device, reduce the
> > > + * core-commit to avoid ETM overflow.
> > > + */
> > > + if (adev->periphid == HISI_ETM_AMBA_ID_V1)
> >
> > Do you have any documentation on this back pressure feature? I doubt this is
> > specific to Hip09 platform and as such would prefer to have a more generic
> > approach that works on any platform that supports it.
>
> It's not a standard Arm architecture feature, this is a model-specific register.
> Some cores may be able to throttle throughput under user control,
> but this isn't standardized. It may (as in this case) be something that you
> want to enable whenever you enable ETM - and, I guess, disable whenever
> you disable ETM. It's a bit unclean to have model-specific code in the main
> ETM driver, and names like CORE_COMMIT_CLEAR really ought to be more
> specific, but I don't see that it's more ugly than the model-specific code in
> e.g. arch/arm64/kernel/perf_event.c or its equivalent on other architectures.
>
> Ideally, a core that has an inherent difficulty generating ETM at full speed,
> i.e. where the ETM can't keep up with the core pipeline, would reduce
> commit rate automatically, and some already do. So if this core needs it
> to be done manually via a system register, we might allow that in the
> same way as we might allow other core-specific actions that need to be
> done to enable ETM.
>
> There are of course issues with trace overflow at all stages up to and
> including harvesting trace from buffers into perf.data (for which solutions
> might involve DVFS, trace strobing, scheduling etc.), and I am assuming
> this patch is not addressing those but dealing with a very specific concern
> about overflow within the core and its ETM.

Thanks for chiming in on this - very valuable input.

Mathieu

>
> Al
>
>
> >
> > Anyone on the CS mailing list that knows what this is about?
> >
> > Thanks,
> > Mathieu
> >
> > > + etm4_config_core_commit(drvdata->cpu, 1);
> > >
> > > CS_UNLOCK(drvdata->base);
> > >
> > > @@ -472,10 +505,20 @@ static void etm4_disable_hw(void *info) {
> > > u32 control;
> > > struct etmv4_drvdata *drvdata = info;
> > > + struct device *dev = drvdata->csdev->dev.parent;
> > > struct etmv4_config *config = &drvdata->config;
> > > struct device *etm_dev = &drvdata->csdev->dev;
> > > + struct amba_device *adev;
> > > int i;
> > >
> > > + adev = container_of(dev, struct amba_device, dev);
> > > + /*
> > > + * If ETM device is HiSilicon ETM device, resume the
> > > + * core-commit after ETM trace is complete.
> > > + */
> > > + if (adev->periphid == HISI_ETM_AMBA_ID_V1)
> > > + etm4_config_core_commit(drvdata->cpu, 0);
> > > +
> > > CS_UNLOCK(drvdata->base);
> > >
> > > if (!drvdata->skip_power_up) {
> > > --
> > > 2.8.1
> > >
> > _______________________________________________
> > CoreSight mailing list
> > [email protected]
> > https://lists.linaro.org/mailman/listinfo/coresight

2020-09-01 01:30:45

by liuqi (BA)

Subject: Re: [RFC PATCH v2] coresight: etm4x: Modify core-commit of cpu to avoid the overflow of HiSilicon ETM

Hi Al,

On 2020/8/28 17:00, Al Grant wrote:
> Hi Mathieu and CS maintainers,
>
>> Hi Liu,
>>
>> On Wed, Aug 19, 2020 at 04:06:37PM +0800, Qi Liu wrote:
>>> When too much trace information is generated on-chip, the ETM will
>>> overflow, and cause data loss. This is a common phenomenon on ETM
>>> devices.
>>>
>>> But sometimes we do not want to lose performance trace data, so we
>>> suppress the speed of instructions sent from CPU core to ETM to avoid
>>> the overflow of ETM.
>>>
>>> Signed-off-by: Qi Liu <[email protected]>
>>> ---
>>>
>>> Changes since v1:
>>> - ETM on HiSilicon Hip09 platform supports backpressure, so does not
>>> need to modify core commit.
>>>
>>> drivers/hwtracing/coresight/coresight-etm4x.c | 43
>>> +++++++++++++++++++++++++++
>>> 1 file changed, 43 insertions(+)
>>>
>>> diff --git a/drivers/hwtracing/coresight/coresight-etm4x.c
>>> b/drivers/hwtracing/coresight/coresight-etm4x.c
>>> index 7797a57..7641f89 100644
>>> --- a/drivers/hwtracing/coresight/coresight-etm4x.c
>>> +++ b/drivers/hwtracing/coresight/coresight-etm4x.c
>>> @@ -43,6 +43,10 @@ MODULE_PARM_DESC(boot_enable, "Enable tracing
>> on boot");
>>> #define PARAM_PM_SAVE_NEVER 1 /* never save any state */
>>> #define PARAM_PM_SAVE_SELF_HOSTED 2 /* save self-hosted state only */
>>>
>>> +#define CORE_COMMIT_CLEAR 0x3000
>>> +#define CORE_COMMIT_SHIFT 12
>>> +#define HISI_ETM_AMBA_ID_V1 0x000b6d01
>>> +
>>> static int pm_save_enable = PARAM_PM_SAVE_FIRMWARE;
>>> module_param(pm_save_enable, int, 0444);
>>> MODULE_PARM_DESC(pm_save_enable, @@ -104,11 +108,40 @@ struct
>>> etm4_enable_arg {
>>> int rc;
>>> };
>>>
>>> +static void etm4_cpu_actlr1_cfg(void *info) {
>>> + struct etm4_enable_arg *arg = (struct etm4_enable_arg *)info;
>>> + u64 val;
>>> +
>>> + asm volatile("mrs %0,s3_1_c15_c2_5" : "=r"(val));
>>> + val &= ~CORE_COMMIT_CLEAR;
>>> + val |= arg->rc << CORE_COMMIT_SHIFT;
>>> + asm volatile("msr s3_1_c15_c2_5,%0" : : "r"(val)); }
>>> +
>>> +static void etm4_config_core_commit(int cpu, int val) {
>>> + struct etm4_enable_arg arg = {0};
>>> +
>>> + arg.rc = val;
>>> + smp_call_function_single(cpu, etm4_cpu_actlr1_cfg, &arg, 1);
>> Function etm4_enable/disable_hw() are already running on the CPU they are
>> supposed to so no need to call smp_call_function_single().
>>
>>> +}
>>> +
>>> static int etm4_enable_hw(struct etmv4_drvdata *drvdata) {
>>> int i, rc;
>>> + struct amba_device *adev;
>>> struct etmv4_config *config = &drvdata->config;
>>> struct device *etm_dev = &drvdata->csdev->dev;
>>> + struct device *dev = drvdata->csdev->dev.parent;
>>> +
>>> + adev = container_of(dev, struct amba_device, dev);
>>> + /*
>>> + * If ETM device is HiSilicon ETM device, reduce the
>>> + * core-commit to avoid ETM overflow.
>>> + */
>>> + if (adev->periphid == HISI_ETM_AMBA_ID_V1)
>> Do you have any documentation on this back pressure feature? I doubt this is
>> specific to Hip09 platform and as such would prefer to have a more generic
>> approach that works on any platform that supports it.
> It's not a standard Arm architecture feature, this is a model-specific register.
> Some cores may be able to throttle throughput under user control,
> but this isn't standardized. It may (as in this case) be something that you
> want to enable whenever you enable ETM - and, I guess, disable whenever
> you disable ETM. It's a bit unclean to have model-specific code in the main
> ETM driver, and names like CORE_COMMIT_CLEAR really ought to be more
> specific, but I don't see that it's more ugly than the model-specific code in
> e.g. arch/arm64/kernel/perf_event.c or its equivalent on other architectures.
Thanks for the review.
Yes, this core commit is a specific feature to reduce commit rate and let ETM keep up
with core pipeline. Considering this, I'll change a more specific name for the macro
definition and send a new version.

Qi
> Ideally, a core that has an inherent difficulty generating ETM at full speed,
> i.e. where the ETM can't keep up with the core pipeline, would reduce
> commit rate automatically, and some already do. So if this core needs it
> to be done manually via a system register, we might allow that in the
> same way as we might allow other core-specific actions that need to be
> done to enable ETM.
>
> There are of course issues with trace overflow at all stages up to and
> including harvesting trace from buffers into perf.data (for which solutions
> might involve DVFS, trace strobing, scheduling etc.), and I am assuming
> this patch is not addressing those but dealing with a very specific concern
> about overflow within the core and its ETM.
>
> Al
>
>
>> Anyone on the CS mailing list that knows what this is about?
>>
>> Thanks,
>> Mathieu
>>
>>> + etm4_config_core_commit(drvdata->cpu, 1);
>>>
>>> CS_UNLOCK(drvdata->base);
>>>
>>> @@ -472,10 +505,20 @@ static void etm4_disable_hw(void *info) {
>>> u32 control;
>>> struct etmv4_drvdata *drvdata = info;
>>> + struct device *dev = drvdata->csdev->dev.parent;
>>> struct etmv4_config *config = &drvdata->config;
>>> struct device *etm_dev = &drvdata->csdev->dev;
>>> + struct amba_device *adev;
>>> int i;
>>>
>>> + adev = container_of(dev, struct amba_device, dev);
>>> + /*
>>> + * If ETM device is HiSilicon ETM device, resume the
>>> + * core-commit after ETM trace is complete.
>>> + */
>>> + if (adev->periphid == HISI_ETM_AMBA_ID_V1)
>>> + etm4_config_core_commit(drvdata->cpu, 0);
>>> +
>>> CS_UNLOCK(drvdata->base);
>>>
>>> if (!drvdata->skip_power_up) {
>>> --
>>> 2.8.1
>>>
>> _______________________________________________
>> CoreSight mailing list
>> [email protected]
>> https://lists.linaro.org/mailman/listinfo/coresight

2020-09-01 01:59:09

by liuqi (BA)

Subject: Re: [RFC PATCH v2] coresight: etm4x: Modify core-commit of cpu to avoid the overflow of HiSilicon ETM

Hi Mathieu,

Thanks for your review.

On 2020/9/1 6:13, Mathieu Poirier wrote:
> Following Al's comment I have the following recommendations...
>
> On Wed, Aug 19, 2020 at 04:06:37PM +0800, Qi Liu wrote:
>> When too much trace information is generated on-chip, the ETM will
>> overflow, and cause data loss. This is a common phenomenon on ETM
>> devices.
>>
>> But sometimes we do not want to lose performance trace data, so we
>> suppress the speed of instructions sent from CPU core to ETM to
>> avoid the overflow of ETM.
>>
>> Signed-off-by: Qi Liu <[email protected]>
>> ---
>>
>> Changes since v1:
>> - ETM on HiSilicon Hip09 platform supports backpressure, so does
>> not need to modify core commit.
>>
>> drivers/hwtracing/coresight/coresight-etm4x.c | 43 +++++++++++++++++++++++++++
>> 1 file changed, 43 insertions(+)
>>
>> diff --git a/drivers/hwtracing/coresight/coresight-etm4x.c b/drivers/hwtracing/coresight/coresight-etm4x.c
>> index 7797a57..7641f89 100644
>> --- a/drivers/hwtracing/coresight/coresight-etm4x.c
>> +++ b/drivers/hwtracing/coresight/coresight-etm4x.c
>> @@ -43,6 +43,10 @@ MODULE_PARM_DESC(boot_enable, "Enable tracing on boot");
>> #define PARAM_PM_SAVE_NEVER 1 /* never save any state */
>> #define PARAM_PM_SAVE_SELF_HOSTED 2 /* save self-hosted state only */
>>
>> +#define CORE_COMMIT_CLEAR 0x3000
>> +#define CORE_COMMIT_SHIFT 12
>> +#define HISI_ETM_AMBA_ID_V1 0x000b6d01
> Do you have a name for the SoC that can be added so that other HiSilicon SoC can
> be added? I suggest something like:
>
> #define HISI_NAME_CORE_COMMIT_CLEAR
> #define HISI_NAME_CORE_COMMIT_SHIFT
> #define HISI_NAME_ETM_ID
Will fix this next version.
> Moreover I don't see an entry for 0x000b6d01 in the amba id table - is your
> devices upstream? Needless to day that is mandatory in order to move forward
> with this work.
A patch has been applied to add this ETM id and here is the link:
https://lore.kernel.org/linux-arm-kernel/20200813210000.GO3393195@xps15/

>> +
>> static int pm_save_enable = PARAM_PM_SAVE_FIRMWARE;
>> module_param(pm_save_enable, int, 0444);
>> MODULE_PARM_DESC(pm_save_enable,
>> @@ -104,11 +108,40 @@ struct etm4_enable_arg {
>> int rc;
>> };
>>
>> +static void etm4_cpu_actlr1_cfg(void *info)
>> +{
>> + struct etm4_enable_arg *arg = (struct etm4_enable_arg *)info;
>> + u64 val;
>> +
>> + asm volatile("mrs %0,s3_1_c15_c2_5" : "=r"(val));
>> + val &= ~CORE_COMMIT_CLEAR;
>> + val |= arg->rc << CORE_COMMIT_SHIFT;
>> + asm volatile("msr s3_1_c15_c2_5,%0" : : "r"(val));
>> +}
>> +
>> +static void etm4_config_core_commit(int cpu, int val)
>> +{
>> + struct etm4_enable_arg arg = {0};
>> +
>> + arg.rc = val;
>> + smp_call_function_single(cpu, etm4_cpu_actlr1_cfg, &arg, 1);
>> +}
>> +
>> static int etm4_enable_hw(struct etmv4_drvdata *drvdata)
>> {
>> int i, rc;
>> + struct amba_device *adev;
>> struct etmv4_config *config = &drvdata->config;
>> struct device *etm_dev = &drvdata->csdev->dev;
>> + struct device *dev = drvdata->csdev->dev.parent;
>> +
>> + adev = container_of(dev, struct amba_device, dev);
>> + /*
>> + * If ETM device is HiSilicon ETM device, reduce the
>> + * core-commit to avoid ETM overflow.
>> + */
>> + if (adev->periphid == HISI_ETM_AMBA_ID_V1)
>> + etm4_config_core_commit(drvdata->cpu, 1);
> I susggest to add a function like etm4_enable_arch_specific() and do the above
> in there. The same goes for the disable path.
>
> Thanks,
> Mathieu
Thanks, I'll fix this next version : )

Qi
>
>> CS_UNLOCK(drvdata->base);
>>
>> @@ -472,10 +505,20 @@ static void etm4_disable_hw(void *info)
>> {
>> u32 control;
>> struct etmv4_drvdata *drvdata = info;
>> + struct device *dev = drvdata->csdev->dev.parent;
>> struct etmv4_config *config = &drvdata->config;
>> struct device *etm_dev = &drvdata->csdev->dev;
>> + struct amba_device *adev;
>> int i;
>>
>> + adev = container_of(dev, struct amba_device, dev);
>> + /*
>> + * If ETM device is HiSilicon ETM device, resume the
>> + * core-commit after ETM trace is complete.
>> + */
>> + if (adev->periphid == HISI_ETM_AMBA_ID_V1)
>> + etm4_config_core_commit(drvdata->cpu, 0);
>> +
>> CS_UNLOCK(drvdata->base);
>>
>> if (!drvdata->skip_power_up) {
>> --
>> 2.8.1
>>
> .
>

2020-09-01 15:59:34

by Mathieu Poirier

Subject: Re: [RFC PATCH v2] coresight: etm4x: Modify core-commit of cpu to avoid the overflow of HiSilicon ETM

On Mon, 31 Aug 2020 at 19:57, Qi Liu <[email protected]> wrote:
>
> Hi Mathieu,
>
> Thanks for your review.
>
> On 2020/9/1 6:13, Mathieu Poirier wrote:
> > Following Al's comment I have the following recommendations...
> >
> > On Wed, Aug 19, 2020 at 04:06:37PM +0800, Qi Liu wrote:
> >> When too much trace information is generated on-chip, the ETM will
> >> overflow, and cause data loss. This is a common phenomenon on ETM
> >> devices.
> >>
> >> But sometimes we do not want to lose performance trace data, so we
> >> suppress the speed of instructions sent from CPU core to ETM to
> >> avoid the overflow of ETM.
> >>
> >> Signed-off-by: Qi Liu <[email protected]>
> >> ---
> >>
> >> Changes since v1:
> >> - ETM on HiSilicon Hip09 platform supports backpressure, so does
> >> not need to modify core commit.
> >>
> >> drivers/hwtracing/coresight/coresight-etm4x.c | 43 +++++++++++++++++++++++++++
> >> 1 file changed, 43 insertions(+)
> >>
> >> diff --git a/drivers/hwtracing/coresight/coresight-etm4x.c b/drivers/hwtracing/coresight/coresight-etm4x.c
> >> index 7797a57..7641f89 100644
> >> --- a/drivers/hwtracing/coresight/coresight-etm4x.c
> >> +++ b/drivers/hwtracing/coresight/coresight-etm4x.c
> >> @@ -43,6 +43,10 @@ MODULE_PARM_DESC(boot_enable, "Enable tracing on boot");
> >> #define PARAM_PM_SAVE_NEVER 1 /* never save any state */
> >> #define PARAM_PM_SAVE_SELF_HOSTED 2 /* save self-hosted state only */
> >>
> >> +#define CORE_COMMIT_CLEAR 0x3000
> >> +#define CORE_COMMIT_SHIFT 12
> >> +#define HISI_ETM_AMBA_ID_V1 0x000b6d01
> > Do you have a name for the SoC that can be added so that other HiSilicon SoC can
> > be added? I suggest something like:
> >
> > #define HISI_NAME_CORE_COMMIT_CLEAR
> > #define HISI_NAME_CORE_COMMIT_SHIFT
> > #define HISI_NAME_ETM_ID
> Will fix this next version.
> > Moreover I don't see an entry for 0x000b6d01 in the amba id table - is your
> > devices upstream? Needless to day that is mandatory in order to move forward
> > with this work.
> A patch has been applied to add this ETM id and here is the link:
> https://lore.kernel.org/linux-arm-kernel/20200813210000.GO3393195@xps15/
>

Ah yes - my tree was on the wrong baseline when I reviewed your patch.
You can forget about this comment.

Thanks,
Mathieu

> >> +
> >> static int pm_save_enable = PARAM_PM_SAVE_FIRMWARE;
> >> module_param(pm_save_enable, int, 0444);
> >> MODULE_PARM_DESC(pm_save_enable,
> >> @@ -104,11 +108,40 @@ struct etm4_enable_arg {
> >> int rc;
> >> };
> >>
> >> +static void etm4_cpu_actlr1_cfg(void *info)
> >> +{
> >> + struct etm4_enable_arg *arg = (struct etm4_enable_arg *)info;
> >> + u64 val;
> >> +
> >> + asm volatile("mrs %0,s3_1_c15_c2_5" : "=r"(val));
> >> + val &= ~CORE_COMMIT_CLEAR;
> >> + val |= arg->rc << CORE_COMMIT_SHIFT;
> >> + asm volatile("msr s3_1_c15_c2_5,%0" : : "r"(val));
> >> +}
> >> +
> >> +static void etm4_config_core_commit(int cpu, int val)
> >> +{
> >> + struct etm4_enable_arg arg = {0};
> >> +
> >> + arg.rc = val;
> >> + smp_call_function_single(cpu, etm4_cpu_actlr1_cfg, &arg, 1);
> >> +}
> >> +
> >> static int etm4_enable_hw(struct etmv4_drvdata *drvdata)
> >> {
> >> int i, rc;
> >> + struct amba_device *adev;
> >> struct etmv4_config *config = &drvdata->config;
> >> struct device *etm_dev = &drvdata->csdev->dev;
> >> + struct device *dev = drvdata->csdev->dev.parent;
> >> +
> >> + adev = container_of(dev, struct amba_device, dev);
> >> + /*
> >> + * If ETM device is HiSilicon ETM device, reduce the
> >> + * core-commit to avoid ETM overflow.
> >> + */
> >> + if (adev->periphid == HISI_ETM_AMBA_ID_V1)
> >> + etm4_config_core_commit(drvdata->cpu, 1);
> > I susggest to add a function like etm4_enable_arch_specific() and do the above
> > in there. The same goes for the disable path.
> >
> > Thanks,
> > Mathieu
> Thanks, I'll fix this next version : )
>
> Qi
> >
> >> CS_UNLOCK(drvdata->base);
> >>
> >> @@ -472,10 +505,20 @@ static void etm4_disable_hw(void *info)
> >> {
> >> u32 control;
> >> struct etmv4_drvdata *drvdata = info;
> >> + struct device *dev = drvdata->csdev->dev.parent;
> >> struct etmv4_config *config = &drvdata->config;
> >> struct device *etm_dev = &drvdata->csdev->dev;
> >> + struct amba_device *adev;
> >> int i;
> >>
> >> + adev = container_of(dev, struct amba_device, dev);
> >> + /*
> >> + * If ETM device is HiSilicon ETM device, resume the
> >> + * core-commit after ETM trace is complete.
> >> + */
> >> + if (adev->periphid == HISI_ETM_AMBA_ID_V1)
> >> + etm4_config_core_commit(drvdata->cpu, 0);
> >> +
> >> CS_UNLOCK(drvdata->base);
> >>
> >> if (!drvdata->skip_power_up) {
> >> --
> >> 2.8.1
> >>
> > .
> >
>
>

2020-09-02 10:38:41

by Suzuki K Poulose

Subject: Re: [RFC PATCH v2] coresight: etm4x: Modify core-commit of cpu to avoid the overflow of HiSilicon ETM

On 08/27/2020 09:44 PM, Mathieu Poirier wrote:
> Hi Liu,
>
> On Wed, Aug 19, 2020 at 04:06:37PM +0800, Qi Liu wrote:
>> When too much trace information is generated on-chip, the ETM will
>> overflow, and cause data loss. This is a common phenomenon on ETM
>> devices.
>>
>> But sometimes we do not want to lose performance trace data, so we
>> suppress the speed of instructions sent from CPU core to ETM to
>> avoid the overflow of ETM.
>>
>> Signed-off-by: Qi Liu <[email protected]>
>> ---
>>
>> Changes since v1:
>> - ETM on HiSilicon Hip09 platform supports backpressure, so does
>> not need to modify core commit.
>>
>> drivers/hwtracing/coresight/coresight-etm4x.c | 43 +++++++++++++++++++++++++++
>> 1 file changed, 43 insertions(+)
>>
>> diff --git a/drivers/hwtracing/coresight/coresight-etm4x.c b/drivers/hwtracing/coresight/coresight-etm4x.c
>> index 7797a57..7641f89 100644
>> --- a/drivers/hwtracing/coresight/coresight-etm4x.c
>> +++ b/drivers/hwtracing/coresight/coresight-etm4x.c
>> @@ -43,6 +43,10 @@ MODULE_PARM_DESC(boot_enable, "Enable tracing on boot");
>> #define PARAM_PM_SAVE_NEVER 1 /* never save any state */
>> #define PARAM_PM_SAVE_SELF_HOSTED 2 /* save self-hosted state only */
>>
>> +#define CORE_COMMIT_CLEAR 0x3000
>> +#define CORE_COMMIT_SHIFT 12
>> +#define HISI_ETM_AMBA_ID_V1 0x000b6d01
>> +
>> static int pm_save_enable = PARAM_PM_SAVE_FIRMWARE;
>> module_param(pm_save_enable, int, 0444);
>> MODULE_PARM_DESC(pm_save_enable,
>> @@ -104,11 +108,40 @@ struct etm4_enable_arg {
>> int rc;
>> };
>>
>> +static void etm4_cpu_actlr1_cfg(void *info)
>> +{
>> + struct etm4_enable_arg *arg = (struct etm4_enable_arg *)info;
>> + u64 val;
>> +
>> + asm volatile("mrs %0,s3_1_c15_c2_5" : "=r"(val));


The ID register (S3_1_C15_c2_5) falls into implementation defined space.
See Arm ARM DDI 0487F.a, section "D12.3.2 Reserved encodings for
IMPLEMENTATION DEFINED registers".

So, please stop calling this "etm4_cpu_actlr1_cfg". Since,
1) actlr1 is not an architected name for the said encoding
2) The id register could mean something else on another CPU.

Rather this should indicate platform/CPU specific. e.g,

etm4_cpu_hisilicon_config_core_commit()


>> + val &= ~CORE_COMMIT_CLEAR;
>> + val |= arg->rc << CORE_COMMIT_SHIFT;
>> + asm volatile("msr s3_1_c15_c2_5,%0" : : "r"(val));
>> +}
>> +
>> +static void etm4_config_core_commit(int cpu, int val)
>> +{
>> + struct etm4_enable_arg arg = {0};
>> +
>> + arg.rc = val;
>> + smp_call_function_single(cpu, etm4_cpu_actlr1_cfg, &arg, 1);
>
> Function etm4_enable/disable_hw() are already running on the CPU they are
> supposed to so no need to call smp_call_function_single().
>
>> +}
>> +
>> static int etm4_enable_hw(struct etmv4_drvdata *drvdata)
>> {
>> int i, rc;
>> + struct amba_device *adev;
>> struct etmv4_config *config = &drvdata->config;
>> struct device *etm_dev = &drvdata->csdev->dev;
>> + struct device *dev = drvdata->csdev->dev.parent;
>> +
>> + adev = container_of(dev, struct amba_device, dev);
>> + /*
>> + * If ETM device is HiSilicon ETM device, reduce the
>> + * core-commit to avoid ETM overflow.
>> + */
>> + if (adev->periphid == HISI_ETM_AMBA_ID_V1)

Please could you move this check to the function above ?
Ideally, it would be good to have something like :

etm4_config_impdef_features();

And :

etm4_config_impdef_features(struct etm4_drvdata *drvdata)
{
etm4_cpu_hisilicon_config_core_commit(drvdata);
}

>
> Do you have any documentation on this back pressure feature? I doubt this is
> specific to Hip09 platform and as such would prefer to have a more generic
> approach that works on any platform that supports it.
>
> Anyone on the CS mailing list that knows what this is about?

I believe this is hisilicon specific. May be same across their CPUs, may
be a specific one. There is no architectural guarantee.


Cheers
Suzuki

2020-09-09 11:36:19

by Mike Leach

Subject: Re: [RFC PATCH v2] coresight: etm4x: Modify core-commit of cpu to avoid the overflow of HiSilicon ETM

Hi,

On Wed, 2 Sep 2020 at 11:36, Suzuki K Poulose <[email protected]> wrote:
>
> On 08/27/2020 09:44 PM, Mathieu Poirier wrote:
> > Hi Liu,
> >
> > On Wed, Aug 19, 2020 at 04:06:37PM +0800, Qi Liu wrote:
> >> When too much trace information is generated on-chip, the ETM will
> >> overflow, and cause data loss. This is a common phenomenon on ETM
> >> devices.
> >>
> >> But sometimes we do not want to lose performance trace data, so we
> >> suppress the speed of instructions sent from CPU core to ETM to
> >> avoid the overflow of ETM.
> >>
> >> Signed-off-by: Qi Liu <[email protected]>
> >> ---
> >>
> >> Changes since v1:
> >> - ETM on HiSilicon Hip09 platform supports backpressure, so does
> >> not need to modify core commit.
> >>
> >> drivers/hwtracing/coresight/coresight-etm4x.c | 43 +++++++++++++++++++++++++++
> >> 1 file changed, 43 insertions(+)
> >>
> >> diff --git a/drivers/hwtracing/coresight/coresight-etm4x.c b/drivers/hwtracing/coresight/coresight-etm4x.c
> >> index 7797a57..7641f89 100644
> >> --- a/drivers/hwtracing/coresight/coresight-etm4x.c
> >> +++ b/drivers/hwtracing/coresight/coresight-etm4x.c
> >> @@ -43,6 +43,10 @@ MODULE_PARM_DESC(boot_enable, "Enable tracing on boot");
> >> #define PARAM_PM_SAVE_NEVER 1 /* never save any state */
> >> #define PARAM_PM_SAVE_SELF_HOSTED 2 /* save self-hosted state only */
> >>
> >> +#define CORE_COMMIT_CLEAR 0x3000
> >> +#define CORE_COMMIT_SHIFT 12
> >> +#define HISI_ETM_AMBA_ID_V1 0x000b6d01
> >> +
> >> static int pm_save_enable = PARAM_PM_SAVE_FIRMWARE;
> >> module_param(pm_save_enable, int, 0444);
> >> MODULE_PARM_DESC(pm_save_enable,
> >> @@ -104,11 +108,40 @@ struct etm4_enable_arg {
> >> int rc;
> >> };
> >>
> >> +static void etm4_cpu_actlr1_cfg(void *info)
> >> +{
> >> + struct etm4_enable_arg *arg = (struct etm4_enable_arg *)info;
> >> + u64 val;
> >> +
> >> + asm volatile("mrs %0,s3_1_c15_c2_5" : "=r"(val));
>
>
> The ID register (S3_1_C15_c2_5) falls into implementation defined space.
> See Arm ARM DDI 0487F.a, section "D12.3.2 Reserved encodings for
> IMPLEMENTATION DEFINED registers".
>
> So, please stop calling this "etm4_cpu_actlr1_cfg". Since,
> 1) actlr1 is not an architected name for the said encoding
> 2) The id register could mean something else on another CPU.
>
> Rather this should indicate platform/CPU specific. e.g,
>
> etm4_cpu_hisilicon_config_core_commit()
>
>
> >> + val &= ~CORE_COMMIT_CLEAR;
> >> + val |= arg->rc << CORE_COMMIT_SHIFT;
> >> + asm volatile("msr s3_1_c15_c2_5,%0" : : "r"(val));
> >> +}
> >> +
> >> +static void etm4_config_core_commit(int cpu, int val)
> >> +{
> >> + struct etm4_enable_arg arg = {0};
> >> +
> >> + arg.rc = val;
> >> + smp_call_function_single(cpu, etm4_cpu_actlr1_cfg, &arg, 1);
> >
> > Function etm4_enable/disable_hw() are already running on the CPU they are
> > supposed to so no need to call smp_call_function_single().
> >
> >> +}
> >> +
> >> static int etm4_enable_hw(struct etmv4_drvdata *drvdata)
> >> {
> >> int i, rc;
> >> + struct amba_device *adev;
> >> struct etmv4_config *config = &drvdata->config;
> >> struct device *etm_dev = &drvdata->csdev->dev;
> >> + struct device *dev = drvdata->csdev->dev.parent;
> >> +
> >> + adev = container_of(dev, struct amba_device, dev);
> >> + /*
> >> + * If ETM device is HiSilicon ETM device, reduce the
> >> + * core-commit to avoid ETM overflow.
> >> + */
> >> + if (adev->periphid == HISI_ETM_AMBA_ID_V1)
>
> Please could you move this check to the function above ?
> Ideally, it would be good to have something like :
>
> etm4_config_impdef_features();
>
> And :
>
> etm4_config_impdef_features(struct etm4_drvdata *drvdata)
> {
> etm4_cpu_hisilicon_config_core_commit(drvdata);
> }
>

In addition to the above, Is it worth having this implementation
defined code gated in the kernel configuration - like we do for core
features sometimes?
i,.e.
CONFIG_ETM4X_IMPDEF_FEATURE (controls overall impdef support in the driver)
and
CONFIG_ETM4X_IMPDEF_HISILICON (depends on CONFIG_ETM4X_IMPDEF_FEATURE )

This way we keep non ETM architectural code off platforms that cannot
use it / test it.


> >
> > Do you have any documentation on this back pressure feature? I doubt this is
> > specific to Hip09 platform and as such would prefer to have a more generic
> > approach that works on any platform that supports it.
> >
> > Anyone on the CS mailing list that knows what this is about?
>
> I believe this is hisilicon specific. May be same across their CPUs, may
> be a specific one. There is no architectural guarantee.
>

This could be an implementation of the feature provided by the
TRCSTALLCTRL register - which allows PE to be stalled in response to
the ETM fifos approaching overflow.
At present we do nothing with this feature as we have yet to see a
target with it implemented, but if this is the case then it is an
ETMv4 architectural feature that could be added into the main driver
code, with use/access gated by the relevent TRCIDR bit.

Regards

Mike


>
> Cheers
> Suzuki



--
Mike Leach
Principal Engineer, ARM Ltd.
Manchester Design Centre. UK

2020-09-09 16:39:19

by Mathieu Poirier

Subject: Re: [RFC PATCH v2] coresight: etm4x: Modify core-commit of cpu to avoid the overflow of HiSilicon ETM

On Wed, Sep 09, 2020 at 12:30:02PM +0100, Mike Leach wrote:
> Hi,
>
> On Wed, 2 Sep 2020 at 11:36, Suzuki K Poulose <[email protected]> wrote:
> >
> > On 08/27/2020 09:44 PM, Mathieu Poirier wrote:
> > > Hi Liu,
> > >
> > > On Wed, Aug 19, 2020 at 04:06:37PM +0800, Qi Liu wrote:
> > >> When too much trace information is generated on-chip, the ETM will
> > >> overflow, and cause data loss. This is a common phenomenon on ETM
> > >> devices.
> > >>
> > >> But sometimes we do not want to lose performance trace data, so we
> > >> suppress the speed of instructions sent from CPU core to ETM to
> > >> avoid the overflow of ETM.
> > >>
> > >> Signed-off-by: Qi Liu <[email protected]>
> > >> ---
> > >>
> > >> Changes since v1:
> > >> - ETM on HiSilicon Hip09 platform supports backpressure, so does
> > >> not need to modify core commit.
> > >>
> > >> drivers/hwtracing/coresight/coresight-etm4x.c | 43 +++++++++++++++++++++++++++
> > >> 1 file changed, 43 insertions(+)
> > >>
> > >> diff --git a/drivers/hwtracing/coresight/coresight-etm4x.c b/drivers/hwtracing/coresight/coresight-etm4x.c
> > >> index 7797a57..7641f89 100644
> > >> --- a/drivers/hwtracing/coresight/coresight-etm4x.c
> > >> +++ b/drivers/hwtracing/coresight/coresight-etm4x.c
> > >> @@ -43,6 +43,10 @@ MODULE_PARM_DESC(boot_enable, "Enable tracing on boot");
> > >> #define PARAM_PM_SAVE_NEVER 1 /* never save any state */
> > >> #define PARAM_PM_SAVE_SELF_HOSTED 2 /* save self-hosted state only */
> > >>
> > >> +#define CORE_COMMIT_CLEAR 0x3000
> > >> +#define CORE_COMMIT_SHIFT 12
> > >> +#define HISI_ETM_AMBA_ID_V1 0x000b6d01
> > >> +
> > >> static int pm_save_enable = PARAM_PM_SAVE_FIRMWARE;
> > >> module_param(pm_save_enable, int, 0444);
> > >> MODULE_PARM_DESC(pm_save_enable,
> > >> @@ -104,11 +108,40 @@ struct etm4_enable_arg {
> > >> int rc;
> > >> };
> > >>
> > >> +static void etm4_cpu_actlr1_cfg(void *info)
> > >> +{
> > >> + struct etm4_enable_arg *arg = (struct etm4_enable_arg *)info;
> > >> + u64 val;
> > >> +
> > >> + asm volatile("mrs %0,s3_1_c15_c2_5" : "=r"(val));
> >
> >
> > The ID register (S3_1_C15_c2_5) falls into implementation defined space.
> > See Arm ARM DDI 0487F.a, section "D12.3.2 Reserved encodings for
> > IMPLEMENTATION DEFINED registers".
> >
> > So, please stop calling this "etm4_cpu_actlr1_cfg". Since,
> > 1) actlr1 is not an architected name for the said encoding
> > 2) The id register could mean something else on another CPU.
> >
> > Rather this should indicate platform/CPU specific. e.g,
> >
> > etm4_cpu_hisilicon_config_core_commit()
> >
> >
> > >> + val &= ~CORE_COMMIT_CLEAR;
> > >> + val |= arg->rc << CORE_COMMIT_SHIFT;
> > >> + asm volatile("msr s3_1_c15_c2_5,%0" : : "r"(val));
> > >> +}
> > >> +
> > >> +static void etm4_config_core_commit(int cpu, int val)
> > >> +{
> > >> + struct etm4_enable_arg arg = {0};
> > >> +
> > >> + arg.rc = val;
> > >> + smp_call_function_single(cpu, etm4_cpu_actlr1_cfg, &arg, 1);
> > >
> > > Function etm4_enable/disable_hw() are already running on the CPU they are
> > > supposed to so no need to call smp_call_function_single().
> > >
> > >> +}
> > >> +
> > >> static int etm4_enable_hw(struct etmv4_drvdata *drvdata)
> > >> {
> > >> int i, rc;
> > >> + struct amba_device *adev;
> > >> struct etmv4_config *config = &drvdata->config;
> > >> struct device *etm_dev = &drvdata->csdev->dev;
> > >> + struct device *dev = drvdata->csdev->dev.parent;
> > >> +
> > >> + adev = container_of(dev, struct amba_device, dev);
> > >> + /*
> > >> + * If ETM device is HiSilicon ETM device, reduce the
> > >> + * core-commit to avoid ETM overflow.
> > >> + */
> > >> + if (adev->periphid == HISI_ETM_AMBA_ID_V1)
> >
> > Please could you move this check to the function above ?
> > Ideally, it would be good to have something like :
> >
> > etm4_config_impdef_features();
> >
> > And :
> >
> > etm4_config_impdef_features(struct etm4_drvdata *drvdata)
> > {
> > etm4_cpu_hisilicon_config_core_commit(drvdata);
> > }
> >
>
> In addition to the above, Is it worth having this implementation
> defined code gated in the kernel configuration - like we do for core
> features sometimes?
> i,.e.
> CONFIG_ETM4X_IMPDEF_FEATURE (controls overall impdef support in the driver)
> and
> CONFIG_ETM4X_IMPDEF_HISILICON (depends on CONFIG_ETM4X_IMPDEF_FEATURE )
>
> This way we keep non ETM architectural code off platforms that cannot
> use it / test it.
>

That's a good idea - they do the same for CPU erratas.

>
> > >
> > > Do you have any documentation on this back pressure feature? I doubt this is
> > > specific to Hip09 platform and as such would prefer to have a more generic
> > > approach that works on any platform that supports it.
> > >
> > > Anyone on the CS mailing list that knows what this is about?
> >
> > I believe this is hisilicon specific. May be same across their CPUs, may
> > be a specific one. There is no architectural guarantee.
> >
>
> This could be an implementation of the feature provided by the
> TRCSTALLCTRL register - which allows PE to be stalled in response to
> the ETM fifos approaching overflow.
> At present we do nothing with this feature as we have yet to see a
> target with it implemented, but if this is the case then it is an
> ETMv4 architectural feature that could be added into the main driver
> code, with use/access gated by the relevent TRCIDR bit.
>
> Regards
>
> Mike
>
>
> >
> > Cheers
> > Suzuki
>
>
>
> --
> Mike Leach
> Principal Engineer, ARM Ltd.
> Manchester Design Centre. UK

2020-11-11 17:08:25

by Mathieu Poirier

Subject: Re: [RFC PATCH v2] coresight: etm4x: Modify core-commit of cpu to avoid the overflow of HiSilicon ETM

On Wed, Nov 11, 2020 at 04:58:23PM +0800, Qi Liu wrote:
> Hi Mathieu,
>
> On 2020/9/10 0:26, Mathieu Poirier wrote:
> > On Wed, Sep 09, 2020 at 12:30:02PM +0100, Mike Leach wrote:
> >> Hi,
> >>
> >> On Wed, 2 Sep 2020 at 11:36, Suzuki K Poulose <[email protected]> wrote:
> >>> On 08/27/2020 09:44 PM, Mathieu Poirier wrote:
> >>>> Hi Liu,
> >>>>
> >>>> On Wed, Aug 19, 2020 at 04:06:37PM +0800, Qi Liu wrote:
> >>>>> When too much trace information is generated on-chip, the ETM will
> >>>>> overflow, and cause data loss. This is a common phenomenon on ETM
> >>>>> devices.
> >>>>>
> >>>>> But sometimes we do not want to lose performance trace data, so we
> >>>>> suppress the speed of instructions sent from CPU core to ETM to
> >>>>> avoid the overflow of ETM.
> >>>>>
> >>>>> Signed-off-by: Qi Liu <[email protected]>
> >>>>> ---
> >>>>>
> >>>>> Changes since v1:
> >>>>> - ETM on HiSilicon Hip09 platform supports backpressure, so does
> >>>>> not need to modify core commit.
> >>>>>
> >>>>> drivers/hwtracing/coresight/coresight-etm4x.c | 43 +++++++++++++++++++++++++++
> >>>>> 1 file changed, 43 insertions(+)
> >>>>>
> >>>>> diff --git a/drivers/hwtracing/coresight/coresight-etm4x.c b/drivers/hwtracing/coresight/coresight-etm4x.c
> >>>>> index 7797a57..7641f89 100644
> >>>>> --- a/drivers/hwtracing/coresight/coresight-etm4x.c
> >>>>> +++ b/drivers/hwtracing/coresight/coresight-etm4x.c
> >>>>> @@ -43,6 +43,10 @@ MODULE_PARM_DESC(boot_enable, "Enable tracing on boot");
> >>>>> #define PARAM_PM_SAVE_NEVER 1 /* never save any state */
> >>>>> #define PARAM_PM_SAVE_SELF_HOSTED 2 /* save self-hosted state only */
> >>>>>
> >>>>> +#define CORE_COMMIT_CLEAR 0x3000
> >>>>> +#define CORE_COMMIT_SHIFT 12
> >>>>> +#define HISI_ETM_AMBA_ID_V1 0x000b6d01
> >>>>> +
> >>>>> static int pm_save_enable = PARAM_PM_SAVE_FIRMWARE;
> >>>>> module_param(pm_save_enable, int, 0444);
> >>>>> MODULE_PARM_DESC(pm_save_enable,
> >>>>> @@ -104,11 +108,40 @@ struct etm4_enable_arg {
> >>>>> int rc;
> >>>>> };
> >>>>>
> >>>>> +static void etm4_cpu_actlr1_cfg(void *info)
> >>>>> +{
> >>>>> + struct etm4_enable_arg *arg = (struct etm4_enable_arg *)info;
> >>>>> + u64 val;
> >>>>> +
> >>>>> + asm volatile("mrs %0,s3_1_c15_c2_5" : "=r"(val));
> >>>
> >>> The ID register (S3_1_C15_c2_5) falls into implementation defined space.
> >>> See Arm ARM DDI 0487F.a, section "D12.3.2 Reserved encodings for
> >>> IMPLEMENTATION DEFINED registers".
> >>>
> >>> So, please stop calling this "etm4_cpu_actlr1_cfg". Since,
> >>> 1) actlr1 is not an architected name for the said encoding
> >>> 2) The id register could mean something else on another CPU.
> >>>
> >>> Rather this should indicate platform/CPU specific. e.g,
> >>>
> >>> etm4_cpu_hisilicon_config_core_commit()
> >>>
> >>>
> >>>>> + val &= ~CORE_COMMIT_CLEAR;
> >>>>> + val |= arg->rc << CORE_COMMIT_SHIFT;
> >>>>> + asm volatile("msr s3_1_c15_c2_5,%0" : : "r"(val));
> >>>>> +}
> >>>>> +
> >>>>> +static void etm4_config_core_commit(int cpu, int val)
> >>>>> +{
> >>>>> + struct etm4_enable_arg arg = {0};
> >>>>> +
> >>>>> + arg.rc = val;
> >>>>> + smp_call_function_single(cpu, etm4_cpu_actlr1_cfg, &arg, 1);
> >>>> Function etm4_enable/disable_hw() are already running on the CPU they are
> >>>> supposed to so no need to call smp_call_function_single().
> >>>>
> >>>>> +}
> >>>>> +
> >>>>> static int etm4_enable_hw(struct etmv4_drvdata *drvdata)
> >>>>> {
> >>>>> int i, rc;
> >>>>> + struct amba_device *adev;
> >>>>> struct etmv4_config *config = &drvdata->config;
> >>>>> struct device *etm_dev = &drvdata->csdev->dev;
> >>>>> + struct device *dev = drvdata->csdev->dev.parent;
> >>>>> +
> >>>>> + adev = container_of(dev, struct amba_device, dev);
> >>>>> + /*
> >>>>> + * If ETM device is HiSilicon ETM device, reduce the
> >>>>> + * core-commit to avoid ETM overflow.
> >>>>> + */
> >>>>> + if (adev->periphid == HISI_ETM_AMBA_ID_V1)
> >>> Please could you move this check to the function above ?
> >>> Ideally, it would be good to have something like :
> >>>
> >>> etm4_config_impdef_features();
> >>>
> >>> And :
> >>>
> >>> etm4_config_impdef_features(struct etm4_drvdata *drvdata)
> >>> {
> >>> etm4_cpu_hisilicon_config_core_commit(drvdata);
> >>> }
> >>>
> >> In addition to the above, Is it worth having this implementation
> >> defined code gated in the kernel configuration - like we do for core
> >> features sometimes?
> >> i,.e.
> >> CONFIG_ETM4X_IMPDEF_FEATURE (controls overall impdef support in the driver)
> >> and
> >> CONFIG_ETM4X_IMPDEF_HISILICON (depends on CONFIG_ETM4X_IMPDEF_FEATURE )
> >>
> >> This way we keep non ETM architectural code off platforms that cannot
> >> use it / test it.
> >>
> > That's a good idea - they do the same for CPU erratas.
> >
> Considering that users sometimes use the same set of code on different platforms, how about
> using both CONFIG andperiphid to keep core-commit code off the platforms that do not
> need it?
> i, .e.
> CONFIG_ETM4X_IMPDEF_FEATURE ( controls overall impdef support in the driver )
> periphid ( match impdef code with the target platform )
>
> This way we could keep the same set of code working on different platforms, and it could help to
> ensure compatibility.

I'm not 100% sure of what you mean by "same set of code working on different
platforms"... Up to know the way we have been handling peripheral IDs has
worked quite well and I don't intend on changing it unless there is a really
good reason.

> I'll update this patch if this solution is ok : )
>
> Thanks!
> Qi
> >>>> Do you have any documentation on this back pressure feature? I doubt this is
> >>>> specific to Hip09 platform and as such would prefer to have a more generic
> >>>> approach that works on any platform that supports it.
> >>>>
> >>>> Anyone on the CS mailing list that knows what this is about?
> >>> I believe this is hisilicon specific. May be same across their CPUs, may
> >>> be a specific one. There is no architectural guarantee.
> >>>
> >> This could be an implementation of the feature provided by the
> >> TRCSTALLCTRL register - which allows PE to be stalled in response to
> >> the ETM fifos approaching overflow.
> >> At present we do nothing with this feature as we have yet to see a
> >> target with it implemented, but if this is the case then it is an
> >> ETMv4 architectural feature that could be added into the main driver
> >> code, with use/access gated by the relevent TRCIDR bit.
> >>
> >> Regards
> >>
> >> Mike
> >>
> >>
> >>> Cheers
> >>> Suzuki
> >>
> >>
> >> --
> >> Mike Leach
> >> Principal Engineer, ARM Ltd.
> >> Manchester Design Centre. UK
> > .
> >
>

2020-11-12 13:11:54

by liuqi (BA)

Subject: Re: [RFC PATCH v2] coresight: etm4x: Modify core-commit of cpu to avoid the overflow of HiSilicon ETM

On 2020/11/12 1:03, Mathieu Poirier wrote:
> On Wed, Nov 11, 2020 at 04:58:23PM +0800, Qi Liu wrote:
>> Hi Mathieu,
>>
>> On 2020/9/10 0:26, Mathieu Poirier wrote:
>>> On Wed, Sep 09, 2020 at 12:30:02PM +0100, Mike Leach wrote:
>>>> Hi,
>>>>
>>>> On Wed, 2 Sep 2020 at 11:36, Suzuki K Poulose <[email protected]> wrote:
>>>>> On 08/27/2020 09:44 PM, Mathieu Poirier wrote:
>>>>>> Hi Liu,
>>>>>>
>>>>>> On Wed, Aug 19, 2020 at 04:06:37PM +0800, Qi Liu wrote:
>>>>>>> When too much trace information is generated on-chip, the ETM will
>>>>>>> overflow, and cause data loss. This is a common phenomenon on ETM
>>>>>>> devices.
>>>>>>>
>>>>>>> But sometimes we do not want to lose performance trace data, so we
>>>>>>> suppress the speed of instructions sent from CPU core to ETM to
>>>>>>> avoid the overflow of ETM.
>>>>>>>
>>>>>>> Signed-off-by: Qi Liu <[email protected]>
>>>>>>> ---
>>>>>>>
>>>>>>> Changes since v1:
>>>>>>> - ETM on HiSilicon Hip09 platform supports backpressure, so does
>>>>>>> not need to modify core commit.
>>>>>>>
>>>>>>> drivers/hwtracing/coresight/coresight-etm4x.c | 43 +++++++++++++++++++++++++++
>>>>>>> 1 file changed, 43 insertions(+)
>>>>>>>
>>>>>>> diff --git a/drivers/hwtracing/coresight/coresight-etm4x.c b/drivers/hwtracing/coresight/coresight-etm4x.c
>>>>>>> index 7797a57..7641f89 100644
>>>>>>> --- a/drivers/hwtracing/coresight/coresight-etm4x.c
>>>>>>> +++ b/drivers/hwtracing/coresight/coresight-etm4x.c
>>>>>>> @@ -43,6 +43,10 @@ MODULE_PARM_DESC(boot_enable, "Enable tracing on boot");
>>>>>>> #define PARAM_PM_SAVE_NEVER 1 /* never save any state */
>>>>>>> #define PARAM_PM_SAVE_SELF_HOSTED 2 /* save self-hosted state only */
>>>>>>>
>>>>>>> +#define CORE_COMMIT_CLEAR 0x3000
>>>>>>> +#define CORE_COMMIT_SHIFT 12
>>>>>>> +#define HISI_ETM_AMBA_ID_V1 0x000b6d01
>>>>>>> +
>>>>>>> static int pm_save_enable = PARAM_PM_SAVE_FIRMWARE;
>>>>>>> module_param(pm_save_enable, int, 0444);
>>>>>>> MODULE_PARM_DESC(pm_save_enable,
>>>>>>> @@ -104,11 +108,40 @@ struct etm4_enable_arg {
>>>>>>> int rc;
>>>>>>> };
>>>>>>>
>>>>>>> +static void etm4_cpu_actlr1_cfg(void *info)
>>>>>>> +{
>>>>>>> + struct etm4_enable_arg *arg = (struct etm4_enable_arg *)info;
>>>>>>> + u64 val;
>>>>>>> +
>>>>>>> + asm volatile("mrs %0,s3_1_c15_c2_5" : "=r"(val));
>>>>>
>>>>> The ID register (S3_1_C15_c2_5) falls into implementation defined space.
>>>>> See Arm ARM DDI 0487F.a, section "D12.3.2 Reserved encodings for
>>>>> IMPLEMENTATION DEFINED registers".
>>>>>
>>>>> So, please stop calling this "etm4_cpu_actlr1_cfg". Since,
>>>>> 1) actlr1 is not an architected name for the said encoding
>>>>> 2) The id register could mean something else on another CPU.
>>>>>
>>>>> Rather this should indicate platform/CPU specific. e.g,
>>>>>
>>>>> etm4_cpu_hisilicon_config_core_commit()
>>>>>
>>>>>
>>>>>>> + val &= ~CORE_COMMIT_CLEAR;
>>>>>>> + val |= arg->rc << CORE_COMMIT_SHIFT;
>>>>>>> + asm volatile("msr s3_1_c15_c2_5,%0" : : "r"(val));
>>>>>>> +}
>>>>>>> +
>>>>>>> +static void etm4_config_core_commit(int cpu, int val)
>>>>>>> +{
>>>>>>> + struct etm4_enable_arg arg = {0};
>>>>>>> +
>>>>>>> + arg.rc = val;
>>>>>>> + smp_call_function_single(cpu, etm4_cpu_actlr1_cfg, &arg, 1);
>>>>>> Function etm4_enable/disable_hw() are already running on the CPU they are
>>>>>> supposed to so no need to call smp_call_function_single().
>>>>>>
>>>>>>> +}
>>>>>>> +
>>>>>>> static int etm4_enable_hw(struct etmv4_drvdata *drvdata)
>>>>>>> {
>>>>>>> int i, rc;
>>>>>>> + struct amba_device *adev;
>>>>>>> struct etmv4_config *config = &drvdata->config;
>>>>>>> struct device *etm_dev = &drvdata->csdev->dev;
>>>>>>> + struct device *dev = drvdata->csdev->dev.parent;
>>>>>>> +
>>>>>>> + adev = container_of(dev, struct amba_device, dev);
>>>>>>> + /*
>>>>>>> + * If ETM device is HiSilicon ETM device, reduce the
>>>>>>> + * core-commit to avoid ETM overflow.
>>>>>>> + */
>>>>>>> + if (adev->periphid == HISI_ETM_AMBA_ID_V1)
>>>>> Please could you move this check to the function above ?
>>>>> Ideally, it would be good to have something like :
>>>>>
>>>>> etm4_config_impdef_features();
>>>>>
>>>>> And :
>>>>>
>>>>> etm4_config_impdef_features(struct etm4_drvdata *drvdata)
>>>>> {
>>>>> etm4_cpu_hisilicon_config_core_commit(drvdata);
>>>>> }
>>>>>
>>>> In addition to the above, Is it worth having this implementation
>>>> defined code gated in the kernel configuration - like we do for core
>>>> features sometimes?
>>>> i,.e.
>>>> CONFIG_ETM4X_IMPDEF_FEATURE (controls overall impdef support in the driver)
>>>> and
>>>> CONFIG_ETM4X_IMPDEF_HISILICON (depends on CONFIG_ETM4X_IMPDEF_FEATURE )
>>>>
>>>> This way we keep non ETM architectural code off platforms that cannot
>>>> use it / test it.
>>>>
>>> That's a good idea - they do the same for CPU erratas.
>>>
>> Considering that users sometimes use the same set of code on different platforms, how about
>> using both CONFIG andperiphid to keep core-commit code off the platforms that do not
>> need it?
>> i, .e.
>> CONFIG_ETM4X_IMPDEF_FEATURE ( controls overall impdef support in the driver )
>> periphid ( match impdef code with the target platform )
>>
>> This way we could keep the same set of code working on different platforms, and it could help to
>> ensure compatibility.
>
> I'm not 100% sure of what you mean by "same set of code working on different
> platforms"... Up to know the way we have been handling peripheral IDs has
> worked quite well and I don't intend on changing it unless there is a really
> good reason.
>
Hi Mathieu,

Perhaps I didn't make it clear and here is the code to express what I mean:

#ifdef CONFIG_ETM4X_IMPDEF_FEATURE

#define HISI_HIP08_AMBA_ID 0x000b6d01
#define HISI_HIP08_AMBA_MASK 0xfffff
static void etm4_enable_arch_specific(struct etmv4_drvdata *drvdata)
{
struct device *dev = drvdata->csdev->dev.parent;
struct amba_device *adev = container_of(dev, struct amba_device, dev);

/*
* If ETM device is HiSilicon ETM device, reduce the
* core-commit to avoid ETM overflow.
*/
if (adev->periphid & HISI_HIP08_AMBA_MASK == HISI_HIP08_AMBA_ID)
etm4_hisi_config_core_commit(1);
}

#else
static void etm4_enable_arch_specific(struct etmv4_drvdata *drvdata)
{
}

#endif /* CONFIG_ETM4X_IMPDEF_FEATURE */

Thanks
Qi

>> I'll update this patch if this solution is ok : )
>>
>> Thanks!
>> Qi
>>>>>> Do you have any documentation on this back pressure feature? I doubt this is
>>>>>> specific to Hip09 platform and as such would prefer to have a more generic
>>>>>> approach that works on any platform that supports it.
>>>>>>
>>>>>> Anyone on the CS mailing list that knows what this is about?
>>>>> I believe this is hisilicon specific. May be same across their CPUs, may
>>>>> be a specific one. There is no architectural guarantee.
>>>>>
>>>> This could be an implementation of the feature provided by the
>>>> TRCSTALLCTRL register - which allows PE to be stalled in response to
>>>> the ETM fifos approaching overflow.
>>>> At present we do nothing with this feature as we have yet to see a
>>>> target with it implemented, but if this is the case then it is an
>>>> ETMv4 architectural feature that could be added into the main driver
>>>> code, with use/access gated by the relevent TRCIDR bit.
>>>>
>>>> Regards
>>>>
>>>> Mike
>>>>
>>>>
>>>>> Cheers
>>>>> Suzuki
>>>>
>>>>
>>>> --
>>>> Mike Leach
>>>> Principal Engineer, ARM Ltd.
>>>> Manchester Design Centre. UK
>>> .
>>>
>>
>
> .
>

2020-11-12 14:08:28

by Suzuki K Poulose

Subject: Re: [RFC PATCH v2] coresight: etm4x: Modify core-commit of cpu to avoid the overflow of HiSilicon ETM

On 11/12/20 1:09 PM, Qi Liu wrote:
>
>
> On 2020/11/12 1:03, Mathieu Poirier wrote:
>> On Wed, Nov 11, 2020 at 04:58:23PM +0800, Qi Liu wrote:
>>> Hi Mathieu,
>>>
>>> On 2020/9/10 0:26, Mathieu Poirier wrote:
>>>> On Wed, Sep 09, 2020 at 12:30:02PM +0100, Mike Leach wrote:
>>>>> Hi,
>>>>>
>>>>> On Wed, 2 Sep 2020 at 11:36, Suzuki K Poulose <[email protected]> wrote:
>>>>>> On 08/27/2020 09:44 PM, Mathieu Poirier wrote:
>>>>>>> Hi Liu,
>>>>>>>
>>>>>>> On Wed, Aug 19, 2020 at 04:06:37PM +0800, Qi Liu wrote:
>>>>>>>> When too much trace information is generated on-chip, the ETM will
>>>>>>>> overflow, and cause data loss. This is a common phenomenon on ETM
>>>>>>>> devices.
>>>>>>>>
>>>>>>>> But sometimes we do not want to lose performance trace data, so we
>>>>>>>> suppress the speed of instructions sent from CPU core to ETM to
>>>>>>>> avoid the overflow of ETM.
>>>>>>>>
>>>>>>>> Signed-off-by: Qi Liu <[email protected]>
>>>>>>>> ---
>>>>>>>>
>>>>>>>> Changes since v1:
>>>>>>>> - ETM on HiSilicon Hip09 platform supports backpressure, so does
>>>>>>>> not need to modify core commit.
>>>>>>>>
>>>>>>>> drivers/hwtracing/coresight/coresight-etm4x.c | 43 +++++++++++++++++++++++++++
>>>>>>>> 1 file changed, 43 insertions(+)
>>>>>>>>
>>>>>>>> diff --git a/drivers/hwtracing/coresight/coresight-etm4x.c b/drivers/hwtracing/coresight/coresight-etm4x.c
>>>>>>>> index 7797a57..7641f89 100644
>>>>>>>> --- a/drivers/hwtracing/coresight/coresight-etm4x.c
>>>>>>>> +++ b/drivers/hwtracing/coresight/coresight-etm4x.c
>>>>>>>> @@ -43,6 +43,10 @@ MODULE_PARM_DESC(boot_enable, "Enable tracing on boot");
>>>>>>>> #define PARAM_PM_SAVE_NEVER 1 /* never save any state */
>>>>>>>> #define PARAM_PM_SAVE_SELF_HOSTED 2 /* save self-hosted state only */
>>>>>>>>
>>>>>>>> +#define CORE_COMMIT_CLEAR 0x3000
>>>>>>>> +#define CORE_COMMIT_SHIFT 12
>>>>>>>> +#define HISI_ETM_AMBA_ID_V1 0x000b6d01
>>>>>>>> +
>>>>>>>> static int pm_save_enable = PARAM_PM_SAVE_FIRMWARE;
>>>>>>>> module_param(pm_save_enable, int, 0444);
>>>>>>>> MODULE_PARM_DESC(pm_save_enable,
>>>>>>>> @@ -104,11 +108,40 @@ struct etm4_enable_arg {
>>>>>>>> int rc;
>>>>>>>> };
>>>>>>>>
>>>>>>>> +static void etm4_cpu_actlr1_cfg(void *info)
>>>>>>>> +{
>>>>>>>> + struct etm4_enable_arg *arg = (struct etm4_enable_arg *)info;
>>>>>>>> + u64 val;
>>>>>>>> +
>>>>>>>> + asm volatile("mrs %0,s3_1_c15_c2_5" : "=r"(val));
>>>>>>
>>>>>> The ID register (S3_1_C15_c2_5) falls into implementation defined space.
>>>>>> See Arm ARM DDI 0487F.a, section "D12.3.2 Reserved encodings for
>>>>>> IMPLEMENTATION DEFINED registers".
>>>>>>
>>>>>> So, please stop calling this "etm4_cpu_actlr1_cfg". Since,
>>>>>> 1) actlr1 is not an architected name for the said encoding
>>>>>> 2) The id register could mean something else on another CPU.
>>>>>>
>>>>>> Rather this should indicate platform/CPU specific. e.g,
>>>>>>
>>>>>> etm4_cpu_hisilicon_config_core_commit()
>>>>>>
>>>>>>
>>>>>>>> + val &= ~CORE_COMMIT_CLEAR;
>>>>>>>> + val |= arg->rc << CORE_COMMIT_SHIFT;
>>>>>>>> + asm volatile("msr s3_1_c15_c2_5,%0" : : "r"(val));
>>>>>>>> +}
>>>>>>>> +
>>>>>>>> +static void etm4_config_core_commit(int cpu, int val)
>>>>>>>> +{
>>>>>>>> + struct etm4_enable_arg arg = {0};
>>>>>>>> +
>>>>>>>> + arg.rc = val;
>>>>>>>> + smp_call_function_single(cpu, etm4_cpu_actlr1_cfg, &arg, 1);
>>>>>>> Function etm4_enable/disable_hw() are already running on the CPU they are
>>>>>>> supposed to so no need to call smp_call_function_single().
>>>>>>>
>>>>>>>> +}
>>>>>>>> +
>>>>>>>> static int etm4_enable_hw(struct etmv4_drvdata *drvdata)
>>>>>>>> {
>>>>>>>> int i, rc;
>>>>>>>> + struct amba_device *adev;
>>>>>>>> struct etmv4_config *config = &drvdata->config;
>>>>>>>> struct device *etm_dev = &drvdata->csdev->dev;
>>>>>>>> + struct device *dev = drvdata->csdev->dev.parent;
>>>>>>>> +
>>>>>>>> + adev = container_of(dev, struct amba_device, dev);
>>>>>>>> + /*
>>>>>>>> + * If ETM device is HiSilicon ETM device, reduce the
>>>>>>>> + * core-commit to avoid ETM overflow.
>>>>>>>> + */
>>>>>>>> + if (adev->periphid == HISI_ETM_AMBA_ID_V1)
>>>>>> Please could you move this check to the function above ?
>>>>>> Ideally, it would be good to have something like :
>>>>>>
>>>>>> etm4_config_impdef_features();
>>>>>>
>>>>>> And :
>>>>>>
>>>>>> etm4_config_impdef_features(struct etm4_drvdata *drvdata)
>>>>>> {
>>>>>> etm4_cpu_hisilicon_config_core_commit(drvdata);
>>>>>> }
>>>>>>
>>>>> In addition to the above, Is it worth having this implementation
>>>>> defined code gated in the kernel configuration - like we do for core
>>>>> features sometimes?
>>>>> i,.e.
>>>>> CONFIG_ETM4X_IMPDEF_FEATURE (controls overall impdef support in the driver)
>>>>> and
>>>>> CONFIG_ETM4X_IMPDEF_HISILICON (depends on CONFIG_ETM4X_IMPDEF_FEATURE )
>>>>>
>>>>> This way we keep non ETM architectural code off platforms that cannot
>>>>> use it / test it.
>>>>>
>>>> That's a good idea - they do the same for CPU erratas.
>>>>
>>> Considering that users sometimes use the same set of code on different platforms, how about
>>> using both CONFIG andperiphid to keep core-commit code off the platforms that do not
>>> need it?
>>> i, .e.
>>> CONFIG_ETM4X_IMPDEF_FEATURE ( controls overall impdef support in the driver )
>>> periphid ( match impdef code with the target platform )
>>>
>>> This way we could keep the same set of code working on different platforms, and it could help to
>>> ensure compatibility.
>>
>> I'm not 100% sure of what you mean by "same set of code working on different
>> platforms"... Up to know the way we have been handling peripheral IDs has
>> worked quite well and I don't intend on changing it unless there is a really
>> good reason.
>>
> Hi Mathieu,
>
> Perhaps I didn't make it clear and here is the code to express what I mean:
>
> #ifdef CONFIG_ETM4X_IMPDEF_FEATURE
>
> #define HISI_HIP08_AMBA_ID 0x000b6d01
> #define HISI_HIP08_AMBA_MASK 0xfffff
> static void etm4_enable_arch_specific(struct etmv4_drvdata *drvdata)
> {
> struct device *dev = drvdata->csdev->dev.parent;
> struct amba_device *adev = container_of(dev, struct amba_device, dev);

There is no guarantee that this is always an "AMBA" device, with this
patchset (which is still under review). Also, doing this check at
every time we enable/disable the ETM is not idea..

May be we should add a concept of "features" and use a bit mask instead,
which can be set at probe time, where we do have this information.

#define ETM4x_IMPDEF_HISILICON_CORE_COMMIT 0
#define ETM4x_IMPDEF_ARCH_N_FEATS 1

struct etmv4_drvdata {

...
DECALRE_BITMAP(arch_features, ETM4x_IMPDEF_ARCH_FEAT_MAX);
}

struct etm4x_arch_feature {
void (*callback)(struct etmv4_drdvata *, bool enable);
};

static struct etm4x_arch_features[] = {
[ETM4x_IMPDEF_HISILICON_CORE_COMMIT] = {
.callback = etm4x_hisilicon_core_commit,
},
{}
};

static void etm4_enable_arch_specific(struct etmv4_drvdata *drvdata)
{
for_each_set_bit(i, &drvdata->arch_features) {
struct etm4x_arch_feature *ftr = &etm4x_arch_features[i];

if (ftr->callback)
ftr->callback(drvdata, true);
}
}

etm4x_check_arch_features() {
if (hisilicon_etm4x_match_pid)
set_bit(drvdata->arch_features, ETM4x_IMPDEF_HISILICON_CORE_COMMIT);
}

etm4x_probe() {

etm4x_check_arch_features();
}

Suzuki

2020-11-13 10:23:10

by liuqi (BA)

Subject: Re: [RFC PATCH v2] coresight: etm4x: Modify core-commit of cpu to avoid the overflow of HiSilicon ETM

On 2020/11/12 22:03, Suzuki K Poulose wrote:
> On 11/12/20 1:09 PM, Qi Liu wrote:
>>
>>
>> On 2020/11/12 1:03, Mathieu Poirier wrote:
>>> On Wed, Nov 11, 2020 at 04:58:23PM +0800, Qi Liu wrote:
>>>> Hi Mathieu,
>>>>
>>>> On 2020/9/10 0:26, Mathieu Poirier wrote:
>>>>> On Wed, Sep 09, 2020 at 12:30:02PM +0100, Mike Leach wrote:
>>>>>> Hi,
>>>>>>
>>>>>> On Wed, 2 Sep 2020 at 11:36, Suzuki K Poulose <[email protected]> wrote:
>>>>>>> On 08/27/2020 09:44 PM, Mathieu Poirier wrote:
>>>>>>>> Hi Liu,
>>>>>>>>
>>>>>>>> On Wed, Aug 19, 2020 at 04:06:37PM +0800, Qi Liu wrote:
>>>>>>>>> When too much trace information is generated on-chip, the ETM will
>>>>>>>>> overflow, and cause data loss. This is a common phenomenon on ETM
>>>>>>>>> devices.
>>>>>>>>>
>>>>>>>>> But sometimes we do not want to lose performance trace data, so we
>>>>>>>>> suppress the speed of instructions sent from CPU core to ETM to
>>>>>>>>> avoid the overflow of ETM.
>>>>>>>>>
>>>>>>>>> Signed-off-by: Qi Liu <[email protected]>
>>>>>>>>> ---
>>>>>>>>>
>>>>>>>>> Changes since v1:
>>>>>>>>> - ETM on HiSilicon Hip09 platform supports backpressure, so does
>>>>>>>>> not need to modify core commit.
>>>>>>>>>
>>>>>>>>> drivers/hwtracing/coresight/coresight-etm4x.c | 43 +++++++++++++++++++++++++++
>>>>>>>>> 1 file changed, 43 insertions(+)
>>>>>>>>>
>>>>>>>>> diff --git a/drivers/hwtracing/coresight/coresight-etm4x.c b/drivers/hwtracing/coresight/coresight-etm4x.c
>>>>>>>>> index 7797a57..7641f89 100644
>>>>>>>>> --- a/drivers/hwtracing/coresight/coresight-etm4x.c
>>>>>>>>> +++ b/drivers/hwtracing/coresight/coresight-etm4x.c
>>>>>>>>> @@ -43,6 +43,10 @@ MODULE_PARM_DESC(boot_enable, "Enable tracing on boot");
>>>>>>>>> #define PARAM_PM_SAVE_NEVER 1 /* never save any state */
>>>>>>>>> #define PARAM_PM_SAVE_SELF_HOSTED 2 /* save self-hosted state only */
>>>>>>>>>
>>>>>>>>> +#define CORE_COMMIT_CLEAR 0x3000
>>>>>>>>> +#define CORE_COMMIT_SHIFT 12
>>>>>>>>> +#define HISI_ETM_AMBA_ID_V1 0x000b6d01
>>>>>>>>> +
>>>>>>>>> static int pm_save_enable = PARAM_PM_SAVE_FIRMWARE;
>>>>>>>>> module_param(pm_save_enable, int, 0444);
>>>>>>>>> MODULE_PARM_DESC(pm_save_enable,
>>>>>>>>> @@ -104,11 +108,40 @@ struct etm4_enable_arg {
>>>>>>>>> int rc;
>>>>>>>>> };
>>>>>>>>>
>>>>>>>>> +static void etm4_cpu_actlr1_cfg(void *info)
>>>>>>>>> +{
>>>>>>>>> + struct etm4_enable_arg *arg = (struct etm4_enable_arg *)info;
>>>>>>>>> + u64 val;
>>>>>>>>> +
>>>>>>>>> + asm volatile("mrs %0,s3_1_c15_c2_5" : "=r"(val));
>>>>>>>
>>>>>>> The ID register (S3_1_C15_c2_5) falls into implementation defined space.
>>>>>>> See Arm ARM DDI 0487F.a, section "D12.3.2 Reserved encodings for
>>>>>>> IMPLEMENTATION DEFINED registers".
>>>>>>>
>>>>>>> So, please stop calling this "etm4_cpu_actlr1_cfg". Since,
>>>>>>> 1) actlr1 is not an architected name for the said encoding
>>>>>>> 2) The id register could mean something else on another CPU.
>>>>>>>
>>>>>>> Rather this should indicate platform/CPU specific. e.g,
>>>>>>>
>>>>>>> etm4_cpu_hisilicon_config_core_commit()
>>>>>>>
>>>>>>>
>>>>>>>>> + val &= ~CORE_COMMIT_CLEAR;
>>>>>>>>> + val |= arg->rc << CORE_COMMIT_SHIFT;
>>>>>>>>> + asm volatile("msr s3_1_c15_c2_5,%0" : : "r"(val));
>>>>>>>>> +}
>>>>>>>>> +
>>>>>>>>> +static void etm4_config_core_commit(int cpu, int val)
>>>>>>>>> +{
>>>>>>>>> + struct etm4_enable_arg arg = {0};
>>>>>>>>> +
>>>>>>>>> + arg.rc = val;
>>>>>>>>> + smp_call_function_single(cpu, etm4_cpu_actlr1_cfg, &arg, 1);
>>>>>>>> Function etm4_enable/disable_hw() are already running on the CPU they are
>>>>>>>> supposed to so no need to call smp_call_function_single().
>>>>>>>>
>>>>>>>>> +}
>>>>>>>>> +
>>>>>>>>> static int etm4_enable_hw(struct etmv4_drvdata *drvdata)
>>>>>>>>> {
>>>>>>>>> int i, rc;
>>>>>>>>> + struct amba_device *adev;
>>>>>>>>> struct etmv4_config *config = &drvdata->config;
>>>>>>>>> struct device *etm_dev = &drvdata->csdev->dev;
>>>>>>>>> + struct device *dev = drvdata->csdev->dev.parent;
>>>>>>>>> +
>>>>>>>>> + adev = container_of(dev, struct amba_device, dev);
>>>>>>>>> + /*
>>>>>>>>> + * If ETM device is HiSilicon ETM device, reduce the
>>>>>>>>> + * core-commit to avoid ETM overflow.
>>>>>>>>> + */
>>>>>>>>> + if (adev->periphid == HISI_ETM_AMBA_ID_V1)
>>>>>>> Please could you move this check to the function above ?
>>>>>>> Ideally, it would be good to have something like :
>>>>>>>
>>>>>>> etm4_config_impdef_features();
>>>>>>>
>>>>>>> And :
>>>>>>>
>>>>>>> etm4_config_impdef_features(struct etm4_drvdata *drvdata)
>>>>>>> {
>>>>>>> etm4_cpu_hisilicon_config_core_commit(drvdata);
>>>>>>> }
>>>>>>>
>>>>>> In addition to the above, Is it worth having this implementation
>>>>>> defined code gated in the kernel configuration - like we do for core
>>>>>> features sometimes?
>>>>>> i,.e.
>>>>>> CONFIG_ETM4X_IMPDEF_FEATURE (controls overall impdef support in the driver)
>>>>>> and
>>>>>> CONFIG_ETM4X_IMPDEF_HISILICON (depends on CONFIG_ETM4X_IMPDEF_FEATURE )
>>>>>>
>>>>>> This way we keep non ETM architectural code off platforms that cannot
>>>>>> use it / test it.
>>>>>>
>>>>> That's a good idea - they do the same for CPU erratas.
>>>>>
>>>> Considering that users sometimes use the same set of code on different platforms, how about
>>>> using both CONFIG andperiphid to keep core-commit code off the platforms that do not
>>>> need it?
>>>> i, .e.
>>>> CONFIG_ETM4X_IMPDEF_FEATURE ( controls overall impdef support in the driver )
>>>> periphid ( match impdef code with the target platform )
>>>>
>>>> This way we could keep the same set of code working on different platforms, and it could help to
>>>> ensure compatibility.
>>>
>>> I'm not 100% sure of what you mean by "same set of code working on different
>>> platforms"... Up to know the way we have been handling peripheral IDs has
>>> worked quite well and I don't intend on changing it unless there is a really
>>> good reason.
>>>
>> Hi Mathieu,
>>
>> Perhaps I didn't make it clear and here is the code to express what I mean:
>>
>> #ifdef CONFIG_ETM4X_IMPDEF_FEATURE
>>
>> #define HISI_HIP08_AMBA_ID 0x000b6d01
>> #define HISI_HIP08_AMBA_MASK 0xfffff
>> static void etm4_enable_arch_specific(struct etmv4_drvdata *drvdata)
>> {
>> struct device *dev = drvdata->csdev->dev.parent;
>> struct amba_device *adev = container_of(dev, struct amba_device, dev);
>
> There is no guarantee that this is always an "AMBA" device, with this
> patchset (which is still under review). Also, doing this check at
> every time we enable/disable the ETM is not idea..
>
> May be we should add a concept of "features" and use a bit mask instead,
> which can be set at probe time, where we do have this information.
>
> #define ETM4x_IMPDEF_HISILICON_CORE_COMMIT 0
> #define ETM4x_IMPDEF_ARCH_N_FEATS 1
>
> struct etmv4_drvdata {
>
> ...
> DECALRE_BITMAP(arch_features, ETM4x_IMPDEF_ARCH_FEAT_MAX);
> }
>
> struct etm4x_arch_feature {
> void (*callback)(struct etmv4_drdvata *, bool enable);
> };
>
> static struct etm4x_arch_features[] = {
> [ETM4x_IMPDEF_HISILICON_CORE_COMMIT] = {
> .callback = etm4x_hisilicon_core_commit,
> },
> {}
> };
>
> static void etm4_enable_arch_specific(struct etmv4_drvdata *drvdata)
> {
> for_each_set_bit(i, &drvdata->arch_features) {
> struct etm4x_arch_feature *ftr = &etm4x_arch_features[i];
>
> if (ftr->callback)
> ftr->callback(drvdata, true);
> }
> }
>
> etm4x_check_arch_features() {
> if (hisilicon_etm4x_match_pid)
> set_bit(drvdata->arch_features, ETM4x_IMPDEF_HISILICON_CORE_COMMIT);
> }
>
> etm4x_probe() {
>
> etm4x_check_arch_features();
> }
>
> Suzuki

Hi Suzuki,

Add a check in probe is a good idea, and perhaps we could also add CONFIG_ETM4X_IMPDEF_FEATURE
here, like:
etm4x_probe() {
...
#ifdef CONFIG_ETM4X_IMPDEF_FEATURE
etm4x_check_arch_features();
#endif
}

Thanks
Qi
>
> .
>

2020-11-13 10:26:57

by Suzuki K Poulose

Subject: Re: [RFC PATCH v2] coresight: etm4x: Modify core-commit of cpu to avoid the overflow of HiSilicon ETM

On 11/13/20 10:18 AM, Qi Liu wrote:
>
>
> On 2020/11/12 22:03, Suzuki K Poulose wrote:
>> On 11/12/20 1:09 PM, Qi Liu wrote:
>>>
>>>
>>> On 2020/11/12 1:03, Mathieu Poirier wrote:
>>>> On Wed, Nov 11, 2020 at 04:58:23PM +0800, Qi Liu wrote:
>>>>> Hi Mathieu,
>>>>>
>>>>> On 2020/9/10 0:26, Mathieu Poirier wrote:
>>>>>> On Wed, Sep 09, 2020 at 12:30:02PM +0100, Mike Leach wrote:
>>>>>>> Hi,
>>>>>>>
>>>>>>> On Wed, 2 Sep 2020 at 11:36, Suzuki K Poulose <[email protected]> wrote:
>>>>>>>> On 08/27/2020 09:44 PM, Mathieu Poirier wrote:
>>>>>>>>> Hi Liu,
>>>>>>>>>
>>>>>>>>> On Wed, Aug 19, 2020 at 04:06:37PM +0800, Qi Liu wrote:
>>>>>>>>>> When too much trace information is generated on-chip, the ETM will
>>>>>>>>>> overflow, and cause data loss. This is a common phenomenon on ETM
>>>>>>>>>> devices.
>>>>>>>>>>
>>>>>>>>>> But sometimes we do not want to lose performance trace data, so we
>>>>>>>>>> suppress the speed of instructions sent from CPU core to ETM to
>>>>>>>>>> avoid the overflow of ETM.
>>>>>>>>>>
>>>>>>>>>> Signed-off-by: Qi Liu <[email protected]>
>>>>>>>>>> ---
>>>>>>>>>>
>>>>>>>>>> Changes since v1:
>>>>>>>>>> - ETM on HiSilicon Hip09 platform supports backpressure, so does
>>>>>>>>>> not need to modify core commit.
>>>>>>>>>>
>>>>>>>>>> drivers/hwtracing/coresight/coresight-etm4x.c | 43 +++++++++++++++++++++++++++
>>>>>>>>>> 1 file changed, 43 insertions(+)
>>>>>>>>>>
>>>>>>>>>> diff --git a/drivers/hwtracing/coresight/coresight-etm4x.c b/drivers/hwtracing/coresight/coresight-etm4x.c
>>>>>>>>>> index 7797a57..7641f89 100644
>>>>>>>>>> --- a/drivers/hwtracing/coresight/coresight-etm4x.c
>>>>>>>>>> +++ b/drivers/hwtracing/coresight/coresight-etm4x.c
>>>>>>>>>> @@ -43,6 +43,10 @@ MODULE_PARM_DESC(boot_enable, "Enable tracing on boot");
>>>>>>>>>> #define PARAM_PM_SAVE_NEVER 1 /* never save any state */
>>>>>>>>>> #define PARAM_PM_SAVE_SELF_HOSTED 2 /* save self-hosted state only */
>>>>>>>>>>
>>>>>>>>>> +#define CORE_COMMIT_CLEAR 0x3000
>>>>>>>>>> +#define CORE_COMMIT_SHIFT 12
>>>>>>>>>> +#define HISI_ETM_AMBA_ID_V1 0x000b6d01
>>>>>>>>>> +
>>>>>>>>>> static int pm_save_enable = PARAM_PM_SAVE_FIRMWARE;
>>>>>>>>>> module_param(pm_save_enable, int, 0444);
>>>>>>>>>> MODULE_PARM_DESC(pm_save_enable,
>>>>>>>>>> @@ -104,11 +108,40 @@ struct etm4_enable_arg {
>>>>>>>>>> int rc;
>>>>>>>>>> };
>>>>>>>>>>
>>>>>>>>>> +static void etm4_cpu_actlr1_cfg(void *info)
>>>>>>>>>> +{
>>>>>>>>>> + struct etm4_enable_arg *arg = (struct etm4_enable_arg *)info;
>>>>>>>>>> + u64 val;
>>>>>>>>>> +
>>>>>>>>>> + asm volatile("mrs %0,s3_1_c15_c2_5" : "=r"(val));
>>>>>>>>
>>>>>>>> The ID register (S3_1_C15_c2_5) falls into implementation defined space.
>>>>>>>> See Arm ARM DDI 0487F.a, section "D12.3.2 Reserved encodings for
>>>>>>>> IMPLEMENTATION DEFINED registers".
>>>>>>>>
>>>>>>>> So, please stop calling this "etm4_cpu_actlr1_cfg". Since,
>>>>>>>> 1) actlr1 is not an architected name for the said encoding
>>>>>>>> 2) The id register could mean something else on another CPU.
>>>>>>>>
>>>>>>>> Rather this should indicate platform/CPU specific. e.g,
>>>>>>>>
>>>>>>>> etm4_cpu_hisilicon_config_core_commit()
>>>>>>>>
>>>>>>>>
>>>>>>>>>> + val &= ~CORE_COMMIT_CLEAR;
>>>>>>>>>> + val |= arg->rc << CORE_COMMIT_SHIFT;
>>>>>>>>>> + asm volatile("msr s3_1_c15_c2_5,%0" : : "r"(val));
>>>>>>>>>> +}
>>>>>>>>>> +
>>>>>>>>>> +static void etm4_config_core_commit(int cpu, int val)
>>>>>>>>>> +{
>>>>>>>>>> + struct etm4_enable_arg arg = {0};
>>>>>>>>>> +
>>>>>>>>>> + arg.rc = val;
>>>>>>>>>> + smp_call_function_single(cpu, etm4_cpu_actlr1_cfg, &arg, 1);
>>>>>>>>> Function etm4_enable/disable_hw() are already running on the CPU they are
>>>>>>>>> supposed to so no need to call smp_call_function_single().
>>>>>>>>>
>>>>>>>>>> +}
>>>>>>>>>> +
>>>>>>>>>> static int etm4_enable_hw(struct etmv4_drvdata *drvdata)
>>>>>>>>>> {
>>>>>>>>>> int i, rc;
>>>>>>>>>> + struct amba_device *adev;
>>>>>>>>>> struct etmv4_config *config = &drvdata->config;
>>>>>>>>>> struct device *etm_dev = &drvdata->csdev->dev;
>>>>>>>>>> + struct device *dev = drvdata->csdev->dev.parent;
>>>>>>>>>> +
>>>>>>>>>> + adev = container_of(dev, struct amba_device, dev);
>>>>>>>>>> + /*
>>>>>>>>>> + * If ETM device is HiSilicon ETM device, reduce the
>>>>>>>>>> + * core-commit to avoid ETM overflow.
>>>>>>>>>> + */
>>>>>>>>>> + if (adev->periphid == HISI_ETM_AMBA_ID_V1)
>>>>>>>> Please could you move this check to the function above ?
>>>>>>>> Ideally, it would be good to have something like :
>>>>>>>>
>>>>>>>> etm4_config_impdef_features();
>>>>>>>>
>>>>>>>> And :
>>>>>>>>
>>>>>>>> etm4_config_impdef_features(struct etm4_drvdata *drvdata)
>>>>>>>> {
>>>>>>>> etm4_cpu_hisilicon_config_core_commit(drvdata);
>>>>>>>> }
>>>>>>>>
>>>>>>> In addition to the above, Is it worth having this implementation
>>>>>>> defined code gated in the kernel configuration - like we do for core
>>>>>>> features sometimes?
>>>>>>> i,.e.
>>>>>>> CONFIG_ETM4X_IMPDEF_FEATURE (controls overall impdef support in the driver)
>>>>>>> and
>>>>>>> CONFIG_ETM4X_IMPDEF_HISILICON (depends on CONFIG_ETM4X_IMPDEF_FEATURE )
>>>>>>>
>>>>>>> This way we keep non ETM architectural code off platforms that cannot
>>>>>>> use it / test it.
>>>>>>>
>>>>>> That's a good idea - they do the same for CPU erratas.
>>>>>>
>>>>> Considering that users sometimes use the same set of code on different platforms, how about
>>>>> using both CONFIG andperiphid to keep core-commit code off the platforms that do not
>>>>> need it?
>>>>> i, .e.
>>>>> CONFIG_ETM4X_IMPDEF_FEATURE ( controls overall impdef support in the driver )
>>>>> periphid ( match impdef code with the target platform )
>>>>>
>>>>> This way we could keep the same set of code working on different platforms, and it could help to
>>>>> ensure compatibility.
>>>>
>>>> I'm not 100% sure of what you mean by "same set of code working on different
>>>> platforms"... Up to know the way we have been handling peripheral IDs has
>>>> worked quite well and I don't intend on changing it unless there is a really
>>>> good reason.
>>>>
>>> Hi Mathieu,
>>>
>>> Perhaps I didn't make it clear and here is the code to express what I mean:
>>>
>>> #ifdef CONFIG_ETM4X_IMPDEF_FEATURE
>>>
>>> #define HISI_HIP08_AMBA_ID 0x000b6d01
>>> #define HISI_HIP08_AMBA_MASK 0xfffff
>>> static void etm4_enable_arch_specific(struct etmv4_drvdata *drvdata)
>>> {
>>> struct device *dev = drvdata->csdev->dev.parent;
>>> struct amba_device *adev = container_of(dev, struct amba_device, dev);
>>
>> There is no guarantee that this is always an "AMBA" device, with this
>> patchset (which is still under review). Also, doing this check at
>> every time we enable/disable the ETM is not idea..
>>
>> May be we should add a concept of "features" and use a bit mask instead,
>> which can be set at probe time, where we do have this information.
>>
>> #define ETM4x_IMPDEF_HISILICON_CORE_COMMIT 0
>> #define ETM4x_IMPDEF_ARCH_N_FEATS 1
>>
>> struct etmv4_drvdata {
>>
>> ...
>> DECALRE_BITMAP(arch_features, ETM4x_IMPDEF_ARCH_FEAT_MAX);
>> }
>>
>> struct etm4x_arch_feature {
>> void (*callback)(struct etmv4_drdvata *, bool enable);
>> };
>>
>> static struct etm4x_arch_features[] = {
>> [ETM4x_IMPDEF_HISILICON_CORE_COMMIT] = {
>> .callback = etm4x_hisilicon_core_commit,
>> },
>> {}
>> };
>>
>> static void etm4_enable_arch_specific(struct etmv4_drvdata *drvdata)
>> {
>> for_each_set_bit(i, &drvdata->arch_features) {
>> struct etm4x_arch_feature *ftr = &etm4x_arch_features[i];
>>
>> if (ftr->callback)
>> ftr->callback(drvdata, true);
>> }
>> }
>>
>> etm4x_check_arch_features() {
>> if (hisilicon_etm4x_match_pid)
>> set_bit(drvdata->arch_features, ETM4x_IMPDEF_HISILICON_CORE_COMMIT);
>> }
>>
>> etm4x_probe() {
>>
>> etm4x_check_arch_features();
>> }
>>
>> Suzuki
>
> Hi Suzuki,
>
> Add a check in probe is a good idea, and perhaps we could also add CONFIG_ETM4X_IMPDEF_FEATURE
> here, like:
> etm4x_probe() {
> ...
> #ifdef CONFIG_ETM4X_IMPDEF_FEATURE
> etm4x_check_arch_features();
> #endif

Please make this unconditional and define the function conditionally, as below.


etm4x_check_arch_features();
> }

and :

#ifdef CONFIG_ETM4X_IMPDEF_FEATURE
static void etm4x_check_arch_features(struct etmv4_drvdata *drvdata, struct device *dev)
{

}

static void etm4x_enable_arch_specific(...)
{
.... do arch specific setup ...
}

#else
static inline void etm4x_check_arch_features(..)
{}
static void etm4x_enable_arch_specific(...)
{}
#endif


>
> Thanks
> Qi
>>
>> .
>>
>