The workqueue may execute late even after remoteproc is stopped or
stopping, some resources (rpmsg device and endpoint) have been
released in rproc_stop_subdevices(), then rproc_vq_interrupt()
accessing these resources will cause kennel dump.
Call trace:
virtqueue_add_split+0x1ac/0x560
virtqueue_add_inbuf+0x4c/0x60
rpmsg_recv_done+0x15c/0x294
vring_interrupt+0x6c/0xa4
rproc_vq_interrupt+0x30/0x50
imx_dsp_rproc_vq_work+0x24/0x40 [imx_dsp_rproc]
process_one_work+0x1d0/0x354
worker_thread+0x13c/0x470
kthread+0x154/0x160
ret_from_fork+0x10/0x20
Add mutex protection in imx_dsp_rproc_vq_work(), if the state is
not running, then just skip calling rproc_vq_interrupt().
Also the flush workqueue operation can't be added in rproc stop
for the same reason. The call sequence is
rproc_shutdown
-> rproc_stop
->rproc_stop_subdevices
->rproc->ops->stop()
->imx_dsp_rproc_stop
->flush_work
-> rproc_vq_interrupt
The resource needed by rproc_vq_interrupt has been released in
rproc_stop_subdevices, so flush_work is not safe to be called in
imx_dsp_rproc_stop.
Fixes: ec0e5549f358 ("remoteproc: imx_dsp_rproc: Add remoteproc driver for DSP on i.MX")
Signed-off-by: Shengjiu Wang <[email protected]>
Reviewed-by: Peng Fan <[email protected]>
---
changes in v2:
- update commit message
drivers/remoteproc/imx_dsp_rproc.c | 12 +++++++++---
1 file changed, 9 insertions(+), 3 deletions(-)
diff --git a/drivers/remoteproc/imx_dsp_rproc.c b/drivers/remoteproc/imx_dsp_rproc.c
index 899aa8dd12f0..95da1cbefacf 100644
--- a/drivers/remoteproc/imx_dsp_rproc.c
+++ b/drivers/remoteproc/imx_dsp_rproc.c
@@ -347,9 +347,6 @@ static int imx_dsp_rproc_stop(struct rproc *rproc)
struct device *dev = rproc->dev.parent;
int ret = 0;
- /* Make sure work is finished */
- flush_work(&priv->rproc_work);
-
if (rproc->state == RPROC_CRASHED) {
priv->flags &= ~REMOTE_IS_READY;
return 0;
@@ -432,9 +429,18 @@ static void imx_dsp_rproc_vq_work(struct work_struct *work)
{
struct imx_dsp_rproc *priv = container_of(work, struct imx_dsp_rproc,
rproc_work);
+ struct rproc *rproc = priv->rproc;
+
+ mutex_lock(&rproc->lock);
+
+ if (rproc->state != RPROC_RUNNING)
+ goto unlock_mutex;
rproc_vq_interrupt(priv->rproc, 0);
rproc_vq_interrupt(priv->rproc, 1);
+
+unlock_mutex:
+ mutex_unlock(&rproc->lock);
}
/**
--
2.34.1
Hi Mathieu,
On 9/30/22 09:50, Shengjiu Wang wrote:
> The workqueue may execute late even after remoteproc is stopped or
> stopping, some resources (rpmsg device and endpoint) have been
> released in rproc_stop_subdevices(), then rproc_vq_interrupt()
> accessing these resources will cause kennel dump.
>
> Call trace:
> virtqueue_add_split+0x1ac/0x560
> virtqueue_add_inbuf+0x4c/0x60
> rpmsg_recv_done+0x15c/0x294
> vring_interrupt+0x6c/0xa4
> rproc_vq_interrupt+0x30/0x50
> imx_dsp_rproc_vq_work+0x24/0x40 [imx_dsp_rproc]
> process_one_work+0x1d0/0x354
> worker_thread+0x13c/0x470
> kthread+0x154/0x160
> ret_from_fork+0x10/0x20
>
> Add mutex protection in imx_dsp_rproc_vq_work(), if the state is
> not running, then just skip calling rproc_vq_interrupt().
>
> Also the flush workqueue operation can't be added in rproc stop
> for the same reason. The call sequence is
>
> rproc_shutdown
> -> rproc_stop
> ->rproc_stop_subdevices
> ->rproc->ops->stop()
> ->imx_dsp_rproc_stop
> ->flush_work
> -> rproc_vq_interrupt
>
> The resource needed by rproc_vq_interrupt has been released in
> rproc_stop_subdevices, so flush_work is not safe to be called in
> imx_dsp_rproc_stop.
>
> Fixes: ec0e5549f358 ("remoteproc: imx_dsp_rproc: Add remoteproc driver for DSP on i.MX")
> Signed-off-by: Shengjiu Wang <[email protected]>
> Reviewed-by: Peng Fan <[email protected]>
> ---
> changes in v2:
> - update commit message
>
> drivers/remoteproc/imx_dsp_rproc.c | 12 +++++++++---
> 1 file changed, 9 insertions(+), 3 deletions(-)
>
> diff --git a/drivers/remoteproc/imx_dsp_rproc.c b/drivers/remoteproc/imx_dsp_rproc.c
> index 899aa8dd12f0..95da1cbefacf 100644
> --- a/drivers/remoteproc/imx_dsp_rproc.c
> +++ b/drivers/remoteproc/imx_dsp_rproc.c
> @@ -347,9 +347,6 @@ static int imx_dsp_rproc_stop(struct rproc *rproc)
> struct device *dev = rproc->dev.parent;
> int ret = 0;
>
> - /* Make sure work is finished */
> - flush_work(&priv->rproc_work);
> -
> if (rproc->state == RPROC_CRASHED) {
> priv->flags &= ~REMOTE_IS_READY;
> return 0;
> @@ -432,9 +429,18 @@ static void imx_dsp_rproc_vq_work(struct work_struct *work)
> {
> struct imx_dsp_rproc *priv = container_of(work, struct imx_dsp_rproc,
> rproc_work);
> + struct rproc *rproc = priv->rproc;
> +
> + mutex_lock(&rproc->lock);
> +
> + if (rproc->state != RPROC_RUNNING)
> + goto unlock_mutex;
>
> rproc_vq_interrupt(priv->rproc, 0);
> rproc_vq_interrupt(priv->rproc, 1);
> +
> +unlock_mutex:
> + mutex_unlock(&rproc->lock);
> }
I guess that this fix the issue in case of the use of a work queue for the
notification.
As I reproduced the issue on stm32 only one time without fix,
difficult for me to test the fix...
I also have a look on some other drivers
It seems not applicable for some other drivers such as TI k3 R5[1]
that call virtio under interrupt context ( if I have well understood).
An alternative to try to fix that in a more generic way could be to call
rproc_remove_virtio_dev in the subdev->unprepare, instead of subdev->stop.
This should postpone the virtio unregister after the stop of the coprocessor.
Of course it is only a suggestion/area of improvement, this fix LGTM.
[1]https://elixir.bootlin.com/linux/v6.0/source/drivers/remoteproc/ti_k3_r5_remoteproc.c#L180
Regards,
Arnaud
>
> /**
On Fri, Sep 30, 2022 at 03:50:16PM +0800, Shengjiu Wang wrote:
> The workqueue may execute late even after remoteproc is stopped or
> stopping, some resources (rpmsg device and endpoint) have been
> released in rproc_stop_subdevices(), then rproc_vq_interrupt()
> accessing these resources will cause kennel dump.
>
> Call trace:
> virtqueue_add_split+0x1ac/0x560
> virtqueue_add_inbuf+0x4c/0x60
> rpmsg_recv_done+0x15c/0x294
> vring_interrupt+0x6c/0xa4
> rproc_vq_interrupt+0x30/0x50
> imx_dsp_rproc_vq_work+0x24/0x40 [imx_dsp_rproc]
> process_one_work+0x1d0/0x354
> worker_thread+0x13c/0x470
> kthread+0x154/0x160
> ret_from_fork+0x10/0x20
>
> Add mutex protection in imx_dsp_rproc_vq_work(), if the state is
> not running, then just skip calling rproc_vq_interrupt().
>
> Also the flush workqueue operation can't be added in rproc stop
> for the same reason. The call sequence is
>
> rproc_shutdown
> -> rproc_stop
> ->rproc_stop_subdevices
> ->rproc->ops->stop()
> ->imx_dsp_rproc_stop
> ->flush_work
> -> rproc_vq_interrupt
>
> The resource needed by rproc_vq_interrupt has been released in
> rproc_stop_subdevices, so flush_work is not safe to be called in
> imx_dsp_rproc_stop.
>
> Fixes: ec0e5549f358 ("remoteproc: imx_dsp_rproc: Add remoteproc driver for DSP on i.MX")
> Signed-off-by: Shengjiu Wang <[email protected]>
> Reviewed-by: Peng Fan <[email protected]>
> ---
> changes in v2:
> - update commit message
>
> drivers/remoteproc/imx_dsp_rproc.c | 12 +++++++++---
> 1 file changed, 9 insertions(+), 3 deletions(-)
Applied.
Thanks,
Mathieu
>
> diff --git a/drivers/remoteproc/imx_dsp_rproc.c b/drivers/remoteproc/imx_dsp_rproc.c
> index 899aa8dd12f0..95da1cbefacf 100644
> --- a/drivers/remoteproc/imx_dsp_rproc.c
> +++ b/drivers/remoteproc/imx_dsp_rproc.c
> @@ -347,9 +347,6 @@ static int imx_dsp_rproc_stop(struct rproc *rproc)
> struct device *dev = rproc->dev.parent;
> int ret = 0;
>
> - /* Make sure work is finished */
> - flush_work(&priv->rproc_work);
> -
> if (rproc->state == RPROC_CRASHED) {
> priv->flags &= ~REMOTE_IS_READY;
> return 0;
> @@ -432,9 +429,18 @@ static void imx_dsp_rproc_vq_work(struct work_struct *work)
> {
> struct imx_dsp_rproc *priv = container_of(work, struct imx_dsp_rproc,
> rproc_work);
> + struct rproc *rproc = priv->rproc;
> +
> + mutex_lock(&rproc->lock);
> +
> + if (rproc->state != RPROC_RUNNING)
> + goto unlock_mutex;
>
> rproc_vq_interrupt(priv->rproc, 0);
> rproc_vq_interrupt(priv->rproc, 1);
> +
> +unlock_mutex:
> + mutex_unlock(&rproc->lock);
> }
>
> /**
> --
> 2.34.1
>