2022-08-03 17:49:16

by Rob Clark

Subject: [PATCH 1/2] drm/msm: Move hangcheck timer restart

From: Rob Clark <[email protected]>

Don't directly restart the hangcheck timer from the timer handler, but
instead start it after the recover_worker replays remaining jobs.

If the kthread is blocked for other reasons, there is no point to
immediately restart the timer. Fixes a random symptom of the problem
fixed in the next patch.

Signed-off-by: Rob Clark <[email protected]>
---
drivers/gpu/drm/msm/msm_gpu.c | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c
index fba85f894314..8f9c48eabf7d 100644
--- a/drivers/gpu/drm/msm/msm_gpu.c
+++ b/drivers/gpu/drm/msm/msm_gpu.c
@@ -328,6 +328,7 @@ find_submit(struct msm_ringbuffer *ring, uint32_t fence)
}

static void retire_submits(struct msm_gpu *gpu);
+static void hangcheck_timer_reset(struct msm_gpu *gpu);

static void get_comm_cmdline(struct msm_gem_submit *submit, char **comm, char **cmd)
{
@@ -420,6 +421,8 @@ static void recover_worker(struct kthread_work *work)
}

if (msm_gpu_active(gpu)) {
+ bool restart_hangcheck = false;
+
/* retire completed submits, plus the one that hung: */
retire_submits(gpu);

@@ -436,10 +439,15 @@ static void recover_worker(struct kthread_work *work)
unsigned long flags;

spin_lock_irqsave(&ring->submit_lock, flags);
- list_for_each_entry(submit, &ring->submits, node)
+ list_for_each_entry(submit, &ring->submits, node) {
gpu->funcs->submit(gpu, submit);
+ restart_hangcheck = true;
+ }
spin_unlock_irqrestore(&ring->submit_lock, flags);
}
+
+ if (restart_hangcheck)
+ hangcheck_timer_reset(gpu);
}

mutex_unlock(&gpu->lock);
@@ -515,10 +523,6 @@ static void hangcheck_handler(struct timer_list *t)
kthread_queue_work(gpu->worker, &gpu->recover_work);
}

- /* if still more pending work, reset the hangcheck timer: */
- if (fence_after(ring->fctx->last_fence, ring->hangcheck_fence))
- hangcheck_timer_reset(gpu);
-
/* workaround for missing irq: */
msm_gpu_retire(gpu);
}
--
2.36.1

2022-08-03 20:16:53

by Akhil P Oommen

Subject: Re: [PATCH 1/2] drm/msm: Move hangcheck timer restart

On 8/3/2022 10:53 PM, Rob Clark wrote:
> From: Rob Clark <[email protected]>
>
> Don't directly restart the hangcheck timer from the timer handler, but
> instead start it after the recover_worker replays remaining jobs.
>
> If the kthread is blocked for other reasons, there is no point to
> immediately restart the timer. Fixes a random symptom of the problem
> fixed in the next patch.
>
> Signed-off-by: Rob Clark <[email protected]>
> ---
> drivers/gpu/drm/msm/msm_gpu.c | 14 +++++++++-----
> 1 file changed, 9 insertions(+), 5 deletions(-)
>
> diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c
> index fba85f894314..8f9c48eabf7d 100644
> --- a/drivers/gpu/drm/msm/msm_gpu.c
> +++ b/drivers/gpu/drm/msm/msm_gpu.c
> @@ -328,6 +328,7 @@ find_submit(struct msm_ringbuffer *ring, uint32_t fence)
> }
>
> static void retire_submits(struct msm_gpu *gpu);
> +static void hangcheck_timer_reset(struct msm_gpu *gpu);
>
> static void get_comm_cmdline(struct msm_gem_submit *submit, char **comm, char **cmd)
> {
> @@ -420,6 +421,8 @@ static void recover_worker(struct kthread_work *work)
> }
>
> if (msm_gpu_active(gpu)) {
> + bool restart_hangcheck = false;
> +
> /* retire completed submits, plus the one that hung: */
> retire_submits(gpu);
>
> @@ -436,10 +439,15 @@ static void recover_worker(struct kthread_work *work)
> unsigned long flags;
>
> spin_lock_irqsave(&ring->submit_lock, flags);
> - list_for_each_entry(submit, &ring->submits, node)
> + list_for_each_entry(submit, &ring->submits, node) {
> gpu->funcs->submit(gpu, submit);
> + restart_hangcheck = true;
> + }
> spin_unlock_irqrestore(&ring->submit_lock, flags);
> }
> +
> + if (restart_hangcheck)
> + hangcheck_timer_reset(gpu);
> }
>
> mutex_unlock(&gpu->lock);
> @@ -515,10 +523,6 @@ static void hangcheck_handler(struct timer_list *t)
> kthread_queue_work(gpu->worker, &gpu->recover_work);
> }
>
> - /* if still more pending work, reset the hangcheck timer: */
In the scenario mentioned here, shouldn't we restart the timer?

-Akhil.
> - if (fence_after(ring->fctx->last_fence, ring->hangcheck_fence))
> - hangcheck_timer_reset(gpu);
> -
> /* workaround for missing irq: */
> msm_gpu_retire(gpu);
> }
>

2022-08-03 20:44:45

by Rob Clark

Subject: Re: [PATCH 1/2] drm/msm: Move hangcheck timer restart

On Wed, Aug 3, 2022 at 12:52 PM Akhil P Oommen <[email protected]> wrote:
>
> On 8/3/2022 10:53 PM, Rob Clark wrote:
> > From: Rob Clark <[email protected]>
> >
> > Don't directly restart the hangcheck timer from the timer handler, but
> > instead start it after the recover_worker replays remaining jobs.
> >
> > If the kthread is blocked for other reasons, there is no point to
> > immediately restart the timer. Fixes a random symptom of the problem
> > fixed in the next patch.
> >
> > Signed-off-by: Rob Clark <[email protected]>
> > ---
> > drivers/gpu/drm/msm/msm_gpu.c | 14 +++++++++-----
> > 1 file changed, 9 insertions(+), 5 deletions(-)
> >
> > diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c
> > index fba85f894314..8f9c48eabf7d 100644
> > --- a/drivers/gpu/drm/msm/msm_gpu.c
> > +++ b/drivers/gpu/drm/msm/msm_gpu.c
> > @@ -328,6 +328,7 @@ find_submit(struct msm_ringbuffer *ring, uint32_t fence)
> > }
> >
> > static void retire_submits(struct msm_gpu *gpu);
> > +static void hangcheck_timer_reset(struct msm_gpu *gpu);
> >
> > static void get_comm_cmdline(struct msm_gem_submit *submit, char **comm, char **cmd)
> > {
> > @@ -420,6 +421,8 @@ static void recover_worker(struct kthread_work *work)
> > }
> >
> > if (msm_gpu_active(gpu)) {
> > + bool restart_hangcheck = false;
> > +
> > /* retire completed submits, plus the one that hung: */
> > retire_submits(gpu);
> >
> > @@ -436,10 +439,15 @@ static void recover_worker(struct kthread_work *work)
> > unsigned long flags;
> >
> > spin_lock_irqsave(&ring->submit_lock, flags);
> > - list_for_each_entry(submit, &ring->submits, node)
> > + list_for_each_entry(submit, &ring->submits, node) {
> > gpu->funcs->submit(gpu, submit);
> > + restart_hangcheck = true;
> > + }
> > spin_unlock_irqrestore(&ring->submit_lock, flags);
> > }
> > +
> > + if (restart_hangcheck)
> > + hangcheck_timer_reset(gpu);
> > }
> >
> > mutex_unlock(&gpu->lock);
> > @@ -515,10 +523,6 @@ static void hangcheck_handler(struct timer_list *t)
> > kthread_queue_work(gpu->worker, &gpu->recover_work);
> > }
> >
> > - /* if still more pending work, reset the hangcheck timer: */
> In the scenario mentioned here, shouldn't we restart the timer?

yeah, actually the case where we don't want to restart the timer is
*only* when we schedule recover_work..

BR,
-R

>
> -Akhil.
> > - if (fence_after(ring->fctx->last_fence, ring->hangcheck_fence))
> > - hangcheck_timer_reset(gpu);
> > -
> > /* workaround for missing irq: */
> > msm_gpu_retire(gpu);
> > }
> >
>

2022-08-04 08:41:43

by Akhil P Oommen

Subject: Re: [PATCH 1/2] drm/msm: Move hangcheck timer restart

On 8/4/2022 1:59 AM, Rob Clark wrote:
> On Wed, Aug 3, 2022 at 12:52 PM Akhil P Oommen <[email protected]> wrote:
>> On 8/3/2022 10:53 PM, Rob Clark wrote:
>>> From: Rob Clark <[email protected]>
>>>
>>> Don't directly restart the hangcheck timer from the timer handler, but
>>> instead start it after the recover_worker replays remaining jobs.
>>>
>>> If the kthread is blocked for other reasons, there is no point to
>>> immediately restart the timer. Fixes a random symptom of the problem
>>> fixed in the next patch.
>>>
>>> Signed-off-by: Rob Clark <[email protected]>
>>> ---
>>> drivers/gpu/drm/msm/msm_gpu.c | 14 +++++++++-----
>>> 1 file changed, 9 insertions(+), 5 deletions(-)
>>>
>>> diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c
>>> index fba85f894314..8f9c48eabf7d 100644
>>> --- a/drivers/gpu/drm/msm/msm_gpu.c
>>> +++ b/drivers/gpu/drm/msm/msm_gpu.c
>>> @@ -328,6 +328,7 @@ find_submit(struct msm_ringbuffer *ring, uint32_t fence)
>>> }
>>>
>>> static void retire_submits(struct msm_gpu *gpu);
>>> +static void hangcheck_timer_reset(struct msm_gpu *gpu);
>>>
>>> static void get_comm_cmdline(struct msm_gem_submit *submit, char **comm, char **cmd)
>>> {
>>> @@ -420,6 +421,8 @@ static void recover_worker(struct kthread_work *work)
>>> }
>>>
>>> if (msm_gpu_active(gpu)) {
>>> + bool restart_hangcheck = false;
>>> +
>>> /* retire completed submits, plus the one that hung: */
>>> retire_submits(gpu);
>>>
>>> @@ -436,10 +439,15 @@ static void recover_worker(struct kthread_work *work)
>>> unsigned long flags;
>>>
>>> spin_lock_irqsave(&ring->submit_lock, flags);
>>> - list_for_each_entry(submit, &ring->submits, node)
>>> + list_for_each_entry(submit, &ring->submits, node) {
>>> gpu->funcs->submit(gpu, submit);
>>> + restart_hangcheck = true;
>>> + }
>>> spin_unlock_irqrestore(&ring->submit_lock, flags);
>>> }
>>> +
>>> + if (restart_hangcheck)
>>> + hangcheck_timer_reset(gpu);
>>> }
>>>
>>> mutex_unlock(&gpu->lock);
>>> @@ -515,10 +523,6 @@ static void hangcheck_handler(struct timer_list *t)
>>> kthread_queue_work(gpu->worker, &gpu->recover_work);
>>> }
>>>
>>> - /* if still more pending work, reset the hangcheck timer: */
>> In the scenario mentioned here, shouldn't we restart the timer?
> yeah, actually the case where we don't want to restart the timer is
> *only* when we schedule recover_work..
>
> BR,
> -R
Not sure if your codebase is different but based on msm-next branch,
when "if (fence != ring->hangcheck_fence)" is true, we now skip
rescheduling the timer. I don't think that is what we want. There should
be a hangcheck timer running as long as there is an active submit,
unless we have scheduled a recover_work here.

-Akhil.
>
>> -Akhil.
>>> - if (fence_after(ring->fctx->last_fence, ring->hangcheck_fence))
>>> - hangcheck_timer_reset(gpu);
>>> -
>>> /* workaround for missing irq: */
>>> msm_gpu_retire(gpu);
>>> }
>>>

2022-08-04 18:07:12

by Rob Clark

Subject: Re: [PATCH 1/2] drm/msm: Move hangcheck timer restart

On Thu, Aug 4, 2022 at 12:53 AM Akhil P Oommen <[email protected]> wrote:
>
> On 8/4/2022 1:59 AM, Rob Clark wrote:
> > On Wed, Aug 3, 2022 at 12:52 PM Akhil P Oommen <[email protected]> wrote:
> >> On 8/3/2022 10:53 PM, Rob Clark wrote:
> >>> From: Rob Clark <[email protected]>
> >>>
> >>> Don't directly restart the hangcheck timer from the timer handler, but
> >>> instead start it after the recover_worker replays remaining jobs.
> >>>
> >>> If the kthread is blocked for other reasons, there is no point to
> >>> immediately restart the timer. Fixes a random symptom of the problem
> >>> fixed in the next patch.
> >>>
> >>> Signed-off-by: Rob Clark <[email protected]>
> >>> ---
> >>> drivers/gpu/drm/msm/msm_gpu.c | 14 +++++++++-----
> >>> 1 file changed, 9 insertions(+), 5 deletions(-)
> >>>
> >>> diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c
> >>> index fba85f894314..8f9c48eabf7d 100644
> >>> --- a/drivers/gpu/drm/msm/msm_gpu.c
> >>> +++ b/drivers/gpu/drm/msm/msm_gpu.c
> >>> @@ -328,6 +328,7 @@ find_submit(struct msm_ringbuffer *ring, uint32_t fence)
> >>> }
> >>>
> >>> static void retire_submits(struct msm_gpu *gpu);
> >>> +static void hangcheck_timer_reset(struct msm_gpu *gpu);
> >>>
> >>> static void get_comm_cmdline(struct msm_gem_submit *submit, char **comm, char **cmd)
> >>> {
> >>> @@ -420,6 +421,8 @@ static void recover_worker(struct kthread_work *work)
> >>> }
> >>>
> >>> if (msm_gpu_active(gpu)) {
> >>> + bool restart_hangcheck = false;
> >>> +
> >>> /* retire completed submits, plus the one that hung: */
> >>> retire_submits(gpu);
> >>>
> >>> @@ -436,10 +439,15 @@ static void recover_worker(struct kthread_work *work)
> >>> unsigned long flags;
> >>>
> >>> spin_lock_irqsave(&ring->submit_lock, flags);
> >>> - list_for_each_entry(submit, &ring->submits, node)
> >>> + list_for_each_entry(submit, &ring->submits, node) {
> >>> gpu->funcs->submit(gpu, submit);
> >>> + restart_hangcheck = true;
> >>> + }
> >>> spin_unlock_irqrestore(&ring->submit_lock, flags);
> >>> }
> >>> +
> >>> + if (restart_hangcheck)
> >>> + hangcheck_timer_reset(gpu);
> >>> }
> >>>
> >>> mutex_unlock(&gpu->lock);
> >>> @@ -515,10 +523,6 @@ static void hangcheck_handler(struct timer_list *t)
> >>> kthread_queue_work(gpu->worker, &gpu->recover_work);
> >>> }
> >>>
> >>> - /* if still more pending work, reset the hangcheck timer: */
> >> In the scenario mentioned here, shouldn't we restart the timer?
> > yeah, actually the case where we don't want to restart the timer is
> > *only* when we schedule recover_work..
> >
> > BR,
> > -R
> Not sure if your codebase is different but based on msm-next branch,
> when "if (fence != ring->hangcheck_fence)" is true, we now skip
> rescheduling the timer. I don't think that is what we want. There should
> be a hangcheck timer running as long as there is an active submit,
> unless we have scheduled a recover_work here.
>

right, v2 will change that to only skip rescheduling the timer in the
recover path

BR,
-R

> -Akhil.
> >
> >> -Akhil.
> >>> - if (fence_after(ring->fctx->last_fence, ring->hangcheck_fence))
> >>> - hangcheck_timer_reset(gpu);
> >>> -
> >>> /* workaround for missing irq: */
> >>> msm_gpu_retire(gpu);
> >>> }
> >>>
>