From testing on sc7180-trogdor devices, reading the GMU registers
needs the GMU clocks to be enabled. Those clocks get turned on in
a6xx_gmu_resume(). Confusingly enough, that function is called as a
result of the runtime_pm of the GPU "struct device", not the GMU
"struct device".
Let's grab a reference to the correct device. Incidentally, this makes
us match the a5xx routine more closely.
This is easily shown to fix crashes that happen if we change the GPU's
pm_runtime usage to not use autosuspend. It's also believed to fix
some long tail GPU crashes even with autosuspend.
NOTE: the crashes I've seen were fixed by _only_ fixing
a6xx_gpu_busy(). However, I believe that the same arguments should be
made to a6xx_gmu_set_freq() so I've fixed that case too. To make that
fix clean, we've moved the pm runtime grabbing into the GPU file.
As a bonus fix with this change, we change the pm_runtime get
functions to check for <= 0 instead of ==. This handles the case where
pm_runtime is disabled.
Fixes: eadf79286a4b ("drm/msm: Check for powered down HW in the devfreq callbacks")
Signed-off-by: Douglas Anderson <[email protected]>
---
Changes in v2:
- Move the set_freq runtime pm grab to the GPU file.
- Use <= for the pm_runtime test, not ==.
drivers/gpu/drm/msm/adreno/a6xx_gmu.c | 9 ---------
drivers/gpu/drm/msm/adreno/a6xx_gpu.c | 13 +++++++++++--
2 files changed, 11 insertions(+), 11 deletions(-)
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
index 9f76f5b15759..2410815e77b4 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
@@ -125,17 +125,9 @@ void a6xx_gmu_set_freq(struct msm_gpu *gpu, struct dev_pm_opp *opp)
trace_msm_gmu_freq_change(gmu->freq, perf_index);
- /*
- * This can get called from devfreq while the hardware is idle. Don't
- * bring up the power if it isn't already active
- */
- if (pm_runtime_get_if_in_use(gmu->dev) == 0)
- return;
-
if (!gmu->legacy) {
a6xx_hfi_set_freq(gmu, perf_index);
dev_pm_opp_set_opp(&gpu->pdev->dev, opp);
- pm_runtime_put(gmu->dev);
return;
}
@@ -159,7 +151,6 @@ void a6xx_gmu_set_freq(struct msm_gpu *gpu, struct dev_pm_opp *opp)
dev_err(gmu->dev, "GMU set GPU frequency error: %d\n", ret);
dev_pm_opp_set_opp(&gpu->pdev->dev, opp);
- pm_runtime_put(gmu->dev);
}
unsigned long a6xx_gmu_get_freq(struct msm_gpu *gpu)
diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
index 42ed9a3c4905..54efd9b76ea6 100644
--- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
@@ -1659,7 +1659,7 @@ static u64 a6xx_gpu_busy(struct msm_gpu *gpu, unsigned long *out_sample_rate)
*out_sample_rate = 19200000;
/* Only read the gpu busy if the hardware is already active */
- if (pm_runtime_get_if_in_use(a6xx_gpu->gmu.dev) == 0)
+ if (pm_runtime_get_if_in_use(&gpu->pdev->dev) <= 0)
return 0;
busy_cycles = gmu_read64(&a6xx_gpu->gmu,
@@ -1667,7 +1667,7 @@ static u64 a6xx_gpu_busy(struct msm_gpu *gpu, unsigned long *out_sample_rate)
REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_H);
- pm_runtime_put(a6xx_gpu->gmu.dev);
+ pm_runtime_put(&gpu->pdev->dev);
return busy_cycles;
}
@@ -1677,9 +1677,18 @@ static void a6xx_gpu_set_freq(struct msm_gpu *gpu, struct dev_pm_opp *opp)
struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
+ /*
+ * This can get called from devfreq while the hardware is idle. Don't
+ * bring up the power if it isn't already active
+ */
+ if (pm_runtime_get_if_in_use(&gpu->pdev->dev) <= 0)
+ return;
+
mutex_lock(&a6xx_gpu->gmu.lock);
a6xx_gmu_set_freq(gpu, opp);
mutex_unlock(&a6xx_gpu->gmu.lock);
+
+ pm_runtime_put(&gpu->pdev->dev);
}
static struct msm_gem_address_space *
--
2.36.1.255.ge46751e96f-goog
On Thu, Jun 9, 2022 at 11:04 AM Akhil P Oommen <[email protected]> wrote:
>
> On 6/9/2022 10:17 PM, Douglas Anderson wrote:
> > >From testing on sc7180-trogdor devices, reading the GMU registers
> > needs the GMU clocks to be enabled. Those clocks get turned on in
> > a6xx_gmu_resume(). Confusingly enough, that function is called as a
> > result of the runtime_pm of the GPU "struct device", not the GMU
> > "struct device".
> >
> > Let's grab a reference to the correct device. Incidentally, this makes
> > us match the a5xx routine more closely.
> >
> > This is easily shown to fix crashes that happen if we change the GPU's
> > pm_runtime usage to not use autosuspend. It's also believed to fix
> > some long tail GPU crashes even with autosuspend.
> >
> > NOTE: the crashes I've seen were fixed by _only_ fixing
> > a6xx_gpu_busy(). However, I believe that the same arguments should be
> > made to a6xx_gmu_set_freq() so I've fixed that case too. To make that
> > fix clean, we've moved the pm runtime grabbing into the GPU file.
> >
> > As a bonus fix with this change, we change the pm_runtime get
> > functions to check for <= 0 instead of ==. This handles the case where
> > pm_runtime is disabled.
> >
> > Fixes: eadf79286a4b ("drm/msm: Check for powered down HW in the devfreq callbacks")
> > Signed-off-by: Douglas Anderson <[email protected]>
> > ---
> >
> > Changes in v2:
> > - Move the set_freq runtime pm grab to the GPU file.
> > - Use <= for the pm_runtime test, not ==.
> >
> > drivers/gpu/drm/msm/adreno/a6xx_gmu.c | 9 ---------
> > drivers/gpu/drm/msm/adreno/a6xx_gpu.c | 13 +++++++++++--
> > 2 files changed, 11 insertions(+), 11 deletions(-)
> >
> > diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
> > index 9f76f5b15759..2410815e77b4 100644
> > --- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
> > +++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
> > @@ -125,17 +125,9 @@ void a6xx_gmu_set_freq(struct msm_gpu *gpu, struct dev_pm_opp *opp)
> >
> > trace_msm_gmu_freq_change(gmu->freq, perf_index);
> >
> > - /*
> > - * This can get called from devfreq while the hardware is idle. Don't
> > - * bring up the power if it isn't already active
> > - */
> > - if (pm_runtime_get_if_in_use(gmu->dev) == 0)
> > - return;
> > -
> > if (!gmu->legacy) {
> > a6xx_hfi_set_freq(gmu, perf_index);
> > dev_pm_opp_set_opp(&gpu->pdev->dev, opp);
> > - pm_runtime_put(gmu->dev);
> > return;
> > }
> >
> > @@ -159,7 +151,6 @@ void a6xx_gmu_set_freq(struct msm_gpu *gpu, struct dev_pm_opp *opp)
> > dev_err(gmu->dev, "GMU set GPU frequency error: %d\n", ret);
> >
> > dev_pm_opp_set_opp(&gpu->pdev->dev, opp);
> > - pm_runtime_put(gmu->dev);
> > }
> >
> > unsigned long a6xx_gmu_get_freq(struct msm_gpu *gpu)
> > diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
> > index 42ed9a3c4905..54efd9b76ea6 100644
> > --- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
> > +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
> > @@ -1659,7 +1659,7 @@ static u64 a6xx_gpu_busy(struct msm_gpu *gpu, unsigned long *out_sample_rate)
> > *out_sample_rate = 19200000;
> >
> > /* Only read the gpu busy if the hardware is already active */
> > - if (pm_runtime_get_if_in_use(a6xx_gpu->gmu.dev) == 0)
> > + if (pm_runtime_get_if_in_use(&gpu->pdev->dev) <= 0)
> You are changing the behavior here when CONFIG_PM is not enabled.
I'd guess the odds of anything working with PM=n are low
BR,
-R
> -Akhil.
> > return 0;
> >
> > busy_cycles = gmu_read64(&a6xx_gpu->gmu,
> > @@ -1667,7 +1667,7 @@ static u64 a6xx_gpu_busy(struct msm_gpu *gpu, unsigned long *out_sample_rate)
> > REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_H);
> >
> >
> > - pm_runtime_put(a6xx_gpu->gmu.dev);
> > + pm_runtime_put(&gpu->pdev->dev);
> >
> > return busy_cycles;
> > }
> > @@ -1677,9 +1677,18 @@ static void a6xx_gpu_set_freq(struct msm_gpu *gpu, struct dev_pm_opp *opp)
> > struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
> > struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
> >
> > + /*
> > + * This can get called from devfreq while the hardware is idle. Don't
> > + * bring up the power if it isn't already active
> > + */
> > + if (pm_runtime_get_if_in_use(&gpu->pdev->dev) <= 0)
> > + return;
> > +
> > mutex_lock(&a6xx_gpu->gmu.lock);
> > a6xx_gmu_set_freq(gpu, opp);
> > mutex_unlock(&a6xx_gpu->gmu.lock);
> > +
> > + pm_runtime_put(&gpu->pdev->dev);
> > }
> >
> > static struct msm_gem_address_space *
>
On 6/9/2022 10:17 PM, Douglas Anderson wrote:
> >From testing on sc7180-trogdor devices, reading the GMU registers
> needs the GMU clocks to be enabled. Those clocks get turned on in
> a6xx_gmu_resume(). Confusingly enough, that function is called as a
> result of the runtime_pm of the GPU "struct device", not the GMU
> "struct device".
>
> Let's grab a reference to the correct device. Incidentally, this makes
> us match the a5xx routine more closely.
>
> This is easily shown to fix crashes that happen if we change the GPU's
> pm_runtime usage to not use autosuspend. It's also believed to fix
> some long tail GPU crashes even with autosuspend.
>
> NOTE: the crashes I've seen were fixed by _only_ fixing
> a6xx_gpu_busy(). However, I believe that the same arguments should be
> made to a6xx_gmu_set_freq() so I've fixed that case too. To make that
> fix clean, we've moved the pm runtime grabbing into the GPU file.
>
> As a bonus fix with this change, we change the pm_runtime get
> functions to check for <= 0 instead of ==. This handles the case where
> pm_runtime is disabled.
>
> Fixes: eadf79286a4b ("drm/msm: Check for powered down HW in the devfreq callbacks")
> Signed-off-by: Douglas Anderson <[email protected]>
> ---
>
> Changes in v2:
> - Move the set_freq runtime pm grab to the GPU file.
> - Use <= for the pm_runtime test, not ==.
>
> drivers/gpu/drm/msm/adreno/a6xx_gmu.c | 9 ---------
> drivers/gpu/drm/msm/adreno/a6xx_gpu.c | 13 +++++++++++--
> 2 files changed, 11 insertions(+), 11 deletions(-)
>
> diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
> index 9f76f5b15759..2410815e77b4 100644
> --- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
> +++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
> @@ -125,17 +125,9 @@ void a6xx_gmu_set_freq(struct msm_gpu *gpu, struct dev_pm_opp *opp)
>
> trace_msm_gmu_freq_change(gmu->freq, perf_index);
>
> - /*
> - * This can get called from devfreq while the hardware is idle. Don't
> - * bring up the power if it isn't already active
> - */
> - if (pm_runtime_get_if_in_use(gmu->dev) == 0)
> - return;
> -
> if (!gmu->legacy) {
> a6xx_hfi_set_freq(gmu, perf_index);
> dev_pm_opp_set_opp(&gpu->pdev->dev, opp);
> - pm_runtime_put(gmu->dev);
> return;
> }
>
> @@ -159,7 +151,6 @@ void a6xx_gmu_set_freq(struct msm_gpu *gpu, struct dev_pm_opp *opp)
> dev_err(gmu->dev, "GMU set GPU frequency error: %d\n", ret);
>
> dev_pm_opp_set_opp(&gpu->pdev->dev, opp);
> - pm_runtime_put(gmu->dev);
> }
>
> unsigned long a6xx_gmu_get_freq(struct msm_gpu *gpu)
> diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
> index 42ed9a3c4905..54efd9b76ea6 100644
> --- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
> +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
> @@ -1659,7 +1659,7 @@ static u64 a6xx_gpu_busy(struct msm_gpu *gpu, unsigned long *out_sample_rate)
> *out_sample_rate = 19200000;
>
> /* Only read the gpu busy if the hardware is already active */
> - if (pm_runtime_get_if_in_use(a6xx_gpu->gmu.dev) == 0)
> + if (pm_runtime_get_if_in_use(&gpu->pdev->dev) <= 0)
You are changing the behavior here when CONFIG_PM is not enabled.
-Akhil.
> return 0;
>
> busy_cycles = gmu_read64(&a6xx_gpu->gmu,
> @@ -1667,7 +1667,7 @@ static u64 a6xx_gpu_busy(struct msm_gpu *gpu, unsigned long *out_sample_rate)
> REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_H);
>
>
> - pm_runtime_put(a6xx_gpu->gmu.dev);
> + pm_runtime_put(&gpu->pdev->dev);
>
> return busy_cycles;
> }
> @@ -1677,9 +1677,18 @@ static void a6xx_gpu_set_freq(struct msm_gpu *gpu, struct dev_pm_opp *opp)
> struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
> struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
>
> + /*
> + * This can get called from devfreq while the hardware is idle. Don't
> + * bring up the power if it isn't already active
> + */
> + if (pm_runtime_get_if_in_use(&gpu->pdev->dev) <= 0)
> + return;
> +
> mutex_lock(&a6xx_gpu->gmu.lock);
> a6xx_gmu_set_freq(gpu, opp);
> mutex_unlock(&a6xx_gpu->gmu.lock);
> +
> + pm_runtime_put(&gpu->pdev->dev);
> }
>
> static struct msm_gem_address_space *
On 6/9/2022 11:42 PM, Rob Clark wrote:
> On Thu, Jun 9, 2022 at 11:04 AM Akhil P Oommen <[email protected]> wrote:
>> On 6/9/2022 10:17 PM, Douglas Anderson wrote:
>>> >From testing on sc7180-trogdor devices, reading the GMU registers
>>> needs the GMU clocks to be enabled. Those clocks get turned on in
>>> a6xx_gmu_resume(). Confusingly enough, that function is called as a
>>> result of the runtime_pm of the GPU "struct device", not the GMU
>>> "struct device".
>>>
>>> Let's grab a reference to the correct device. Incidentally, this makes
>>> us match the a5xx routine more closely.
>>>
>>> This is easily shown to fix crashes that happen if we change the GPU's
>>> pm_runtime usage to not use autosuspend. It's also believed to fix
>>> some long tail GPU crashes even with autosuspend.
>>>
>>> NOTE: the crashes I've seen were fixed by _only_ fixing
>>> a6xx_gpu_busy(). However, I believe that the same arguments should be
>>> made to a6xx_gmu_set_freq() so I've fixed that case too. To make that
>>> fix clean, we've moved the pm runtime grabbing into the GPU file.
>>>
>>> As a bonus fix with this change, we change the pm_runtime get
>>> functions to check for <= 0 instead of ==. This handles the case where
>>> pm_runtime is disabled.
>>>
>>> Fixes: eadf79286a4b ("drm/msm: Check for powered down HW in the devfreq callbacks")
>>> Signed-off-by: Douglas Anderson <[email protected]>
>>> ---
>>>
>>> Changes in v2:
>>> - Move the set_freq runtime pm grab to the GPU file.
>>> - Use <= for the pm_runtime test, not ==.
>>>
>>> drivers/gpu/drm/msm/adreno/a6xx_gmu.c | 9 ---------
>>> drivers/gpu/drm/msm/adreno/a6xx_gpu.c | 13 +++++++++++--
>>> 2 files changed, 11 insertions(+), 11 deletions(-)
>>>
>>> diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
>>> index 9f76f5b15759..2410815e77b4 100644
>>> --- a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
>>> +++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c
>>> @@ -125,17 +125,9 @@ void a6xx_gmu_set_freq(struct msm_gpu *gpu, struct dev_pm_opp *opp)
>>>
>>> trace_msm_gmu_freq_change(gmu->freq, perf_index);
>>>
>>> - /*
>>> - * This can get called from devfreq while the hardware is idle. Don't
>>> - * bring up the power if it isn't already active
>>> - */
>>> - if (pm_runtime_get_if_in_use(gmu->dev) == 0)
>>> - return;
>>> -
>>> if (!gmu->legacy) {
>>> a6xx_hfi_set_freq(gmu, perf_index);
>>> dev_pm_opp_set_opp(&gpu->pdev->dev, opp);
>>> - pm_runtime_put(gmu->dev);
>>> return;
>>> }
>>>
>>> @@ -159,7 +151,6 @@ void a6xx_gmu_set_freq(struct msm_gpu *gpu, struct dev_pm_opp *opp)
>>> dev_err(gmu->dev, "GMU set GPU frequency error: %d\n", ret);
>>>
>>> dev_pm_opp_set_opp(&gpu->pdev->dev, opp);
>>> - pm_runtime_put(gmu->dev);
>>> }
>>>
>>> unsigned long a6xx_gmu_get_freq(struct msm_gpu *gpu)
>>> diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
>>> index 42ed9a3c4905..54efd9b76ea6 100644
>>> --- a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
>>> +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c
>>> @@ -1659,7 +1659,7 @@ static u64 a6xx_gpu_busy(struct msm_gpu *gpu, unsigned long *out_sample_rate)
>>> *out_sample_rate = 19200000;
>>>
>>> /* Only read the gpu busy if the hardware is already active */
>>> - if (pm_runtime_get_if_in_use(a6xx_gpu->gmu.dev) == 0)
>>> + if (pm_runtime_get_if_in_use(&gpu->pdev->dev) <= 0)
>> You are changing the behavior here when CONFIG_PM is not enabled.
> I'd guess the odds of anything working with PM=n are low
fwiw, https://patchwork.freedesktop.org/patch/364077/#comment_672059
Akhil
>
> BR,
> -R
>
>> -Akhil.
>>> return 0;
>>>
>>> busy_cycles = gmu_read64(&a6xx_gpu->gmu,
>>> @@ -1667,7 +1667,7 @@ static u64 a6xx_gpu_busy(struct msm_gpu *gpu, unsigned long *out_sample_rate)
>>> REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_H);
>>>
>>>
>>> - pm_runtime_put(a6xx_gpu->gmu.dev);
>>> + pm_runtime_put(&gpu->pdev->dev);
>>>
>>> return busy_cycles;
>>> }
>>> @@ -1677,9 +1677,18 @@ static void a6xx_gpu_set_freq(struct msm_gpu *gpu, struct dev_pm_opp *opp)
>>> struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
>>> struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu);
>>>
>>> + /*
>>> + * This can get called from devfreq while the hardware is idle. Don't
>>> + * bring up the power if it isn't already active
>>> + */
>>> + if (pm_runtime_get_if_in_use(&gpu->pdev->dev) <= 0)
>>> + return;
>>> +
>>> mutex_lock(&a6xx_gpu->gmu.lock);
>>> a6xx_gmu_set_freq(gpu, opp);
>>> mutex_unlock(&a6xx_gpu->gmu.lock);
>>> +
>>> + pm_runtime_put(&gpu->pdev->dev);
>>> }
>>>
>>> static struct msm_gem_address_space *