During kexec on ARM device, we notice that device_shutdown() only calls
pm_runtime_force_suspend() while shutting down the GPU. This means the GPU
kthread is still running and further, there maybe active submits.
This causes all kinds of issues during a kexec reboot:
Warning from shutdown path:
[ 292.509662] WARNING: CPU: 0 PID: 6304 at [...] adreno_runtime_suspend+0x3c/0x44
[ 292.509863] Hardware name: Google Lazor (rev3 - 8) with LTE (DT)
[ 292.509872] pstate: 80400009 (Nzcv daif +PAN -UAO -TCO -DIT -SSBS BTYPE=--)
[ 292.509881] pc : adreno_runtime_suspend+0x3c/0x44
[ 292.509891] lr : pm_generic_runtime_suspend+0x30/0x44
[ 292.509905] sp : ffffffc014473bf0
[...]
[ 292.510043] Call trace:
[ 292.510051] adreno_runtime_suspend+0x3c/0x44
[ 292.510061] pm_generic_runtime_suspend+0x30/0x44
[ 292.510071] pm_runtime_force_suspend+0x54/0xc8
[ 292.510081] adreno_shutdown+0x1c/0x28
[ 292.510090] platform_shutdown+0x2c/0x38
[ 292.510104] device_shutdown+0x158/0x210
[ 292.510119] kernel_restart_prepare+0x40/0x4c
And here from GPU kthread, an SError OOPs:
[ 192.648789] el1h_64_error+0x7c/0x80
[ 192.648812] el1_interrupt+0x20/0x58
[ 192.648833] el1h_64_irq_handler+0x18/0x24
[ 192.648854] el1h_64_irq+0x7c/0x80
[ 192.648873] local_daif_inherit+0x10/0x18
[ 192.648900] el1h_64_sync_handler+0x48/0xb4
[ 192.648921] el1h_64_sync+0x7c/0x80
[ 192.648941] a6xx_gmu_set_oob+0xbc/0x1fc
[ 192.648968] a6xx_hw_init+0x44/0xe38
[ 192.648991] msm_gpu_hw_init+0x48/0x80
[ 192.649013] msm_gpu_submit+0x5c/0x1a8
[ 192.649034] msm_job_run+0xb0/0x11c
[ 192.649058] drm_sched_main+0x170/0x434
[ 192.649086] kthread+0x134/0x300
[ 192.649114] ret_from_fork+0x10/0x20
Fix by calling adreno_system_suspend() in the device_shutdown() path.
[ Applied Rob Clark feedback on fixing adreno_unbind() similarly, also
tested as above. ]
Cc: Rob Clark <[email protected]>
Cc: Steven Rostedt <[email protected]>
Cc: Ricardo Ribalda <[email protected]>
Cc: Ross Zwisler <[email protected]>
Signed-off-by: Joel Fernandes (Google) <[email protected]>
---
drivers/gpu/drm/msm/adreno/adreno_device.c | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/drivers/gpu/drm/msm/adreno/adreno_device.c b/drivers/gpu/drm/msm/adreno/adreno_device.c
index 628806423f7d..36f062c7582f 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_device.c
+++ b/drivers/gpu/drm/msm/adreno/adreno_device.c
@@ -551,13 +551,14 @@ static int adreno_bind(struct device *dev, struct device *master, void *data)
return 0;
}
+static int adreno_system_suspend(struct device *dev);
static void adreno_unbind(struct device *dev, struct device *master,
void *data)
{
struct msm_drm_private *priv = dev_get_drvdata(master);
struct msm_gpu *gpu = dev_to_gpu(dev);
- pm_runtime_force_suspend(dev);
+ WARN_ON_ONCE(adreno_system_suspend(dev));
gpu->funcs->destroy(gpu);
priv->gpu_pdev = NULL;
@@ -609,7 +610,7 @@ static int adreno_remove(struct platform_device *pdev)
static void adreno_shutdown(struct platform_device *pdev)
{
- pm_runtime_force_suspend(&pdev->dev);
+ WARN_ON_ONCE(adreno_system_suspend(&pdev->dev));
}
static const struct of_device_id dt_match[] = {
--
2.39.0.314.g84b9a713c41-goog
On Mon, 9 Jan 2023 at 23:25, Joel Fernandes (Google)
<[email protected]> wrote:
>
> During kexec on ARM device, we notice that device_shutdown() only calls
> pm_runtime_force_suspend() while shutting down the GPU. This means the GPU
> kthread is still running and further, there maybe active submits.
>
> This causes all kinds of issues during a kexec reboot:
>
> Warning from shutdown path:
>
> [ 292.509662] WARNING: CPU: 0 PID: 6304 at [...] adreno_runtime_suspend+0x3c/0x44
> [ 292.509863] Hardware name: Google Lazor (rev3 - 8) with LTE (DT)
> [ 292.509872] pstate: 80400009 (Nzcv daif +PAN -UAO -TCO -DIT -SSBS BTYPE=--)
> [ 292.509881] pc : adreno_runtime_suspend+0x3c/0x44
> [ 292.509891] lr : pm_generic_runtime_suspend+0x30/0x44
> [ 292.509905] sp : ffffffc014473bf0
> [...]
> [ 292.510043] Call trace:
> [ 292.510051] adreno_runtime_suspend+0x3c/0x44
> [ 292.510061] pm_generic_runtime_suspend+0x30/0x44
> [ 292.510071] pm_runtime_force_suspend+0x54/0xc8
> [ 292.510081] adreno_shutdown+0x1c/0x28
> [ 292.510090] platform_shutdown+0x2c/0x38
> [ 292.510104] device_shutdown+0x158/0x210
> [ 292.510119] kernel_restart_prepare+0x40/0x4c
>
> And here from GPU kthread, an SError OOPs:
>
> [ 192.648789] el1h_64_error+0x7c/0x80
> [ 192.648812] el1_interrupt+0x20/0x58
> [ 192.648833] el1h_64_irq_handler+0x18/0x24
> [ 192.648854] el1h_64_irq+0x7c/0x80
> [ 192.648873] local_daif_inherit+0x10/0x18
> [ 192.648900] el1h_64_sync_handler+0x48/0xb4
> [ 192.648921] el1h_64_sync+0x7c/0x80
> [ 192.648941] a6xx_gmu_set_oob+0xbc/0x1fc
> [ 192.648968] a6xx_hw_init+0x44/0xe38
> [ 192.648991] msm_gpu_hw_init+0x48/0x80
> [ 192.649013] msm_gpu_submit+0x5c/0x1a8
> [ 192.649034] msm_job_run+0xb0/0x11c
> [ 192.649058] drm_sched_main+0x170/0x434
> [ 192.649086] kthread+0x134/0x300
> [ 192.649114] ret_from_fork+0x10/0x20
>
> Fix by calling adreno_system_suspend() in the device_shutdown() path.
>
> [ Applied Rob Clark feedback on fixing adreno_unbind() similarly, also
> tested as above. ]
>
> Cc: Rob Clark <[email protected]>
> Cc: Steven Rostedt <[email protected]>
> Cc: Ricardo Ribalda <[email protected]>
> Cc: Ross Zwisler <[email protected]>
Reviewed-by: Ricardo Ribalda <[email protected]>
> Signed-off-by: Joel Fernandes (Google) <[email protected]>
> ---
> drivers/gpu/drm/msm/adreno/adreno_device.c | 5 +++--
> 1 file changed, 3 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/gpu/drm/msm/adreno/adreno_device.c b/drivers/gpu/drm/msm/adreno/adreno_device.c
> index 628806423f7d..36f062c7582f 100644
> --- a/drivers/gpu/drm/msm/adreno/adreno_device.c
> +++ b/drivers/gpu/drm/msm/adreno/adreno_device.c
> @@ -551,13 +551,14 @@ static int adreno_bind(struct device *dev, struct device *master, void *data)
> return 0;
> }
>
> +static int adreno_system_suspend(struct device *dev);
> static void adreno_unbind(struct device *dev, struct device *master,
> void *data)
> {
> struct msm_drm_private *priv = dev_get_drvdata(master);
> struct msm_gpu *gpu = dev_to_gpu(dev);
>
> - pm_runtime_force_suspend(dev);
> + WARN_ON_ONCE(adreno_system_suspend(dev));
> gpu->funcs->destroy(gpu);
>
> priv->gpu_pdev = NULL;
> @@ -609,7 +610,7 @@ static int adreno_remove(struct platform_device *pdev)
>
> static void adreno_shutdown(struct platform_device *pdev)
> {
> - pm_runtime_force_suspend(&pdev->dev);
> + WARN_ON_ONCE(adreno_system_suspend(&pdev->dev));
> }
>
> static const struct of_device_id dt_match[] = {
> --
> 2.39.0.314.g84b9a713c41-goog
>
--
Ricardo Ribalda
On Mon, Jan 9, 2023 at 2:25 PM Joel Fernandes (Google)
<[email protected]> wrote:
>
> During kexec on ARM device, we notice that device_shutdown() only calls
> pm_runtime_force_suspend() while shutting down the GPU. This means the GPU
> kthread is still running and further, there maybe active submits.
>
> This causes all kinds of issues during a kexec reboot:
>
> Warning from shutdown path:
>
> [ 292.509662] WARNING: CPU: 0 PID: 6304 at [...] adreno_runtime_suspend+0x3c/0x44
> [ 292.509863] Hardware name: Google Lazor (rev3 - 8) with LTE (DT)
> [ 292.509872] pstate: 80400009 (Nzcv daif +PAN -UAO -TCO -DIT -SSBS BTYPE=--)
> [ 292.509881] pc : adreno_runtime_suspend+0x3c/0x44
> [ 292.509891] lr : pm_generic_runtime_suspend+0x30/0x44
> [ 292.509905] sp : ffffffc014473bf0
> [...]
> [ 292.510043] Call trace:
> [ 292.510051] adreno_runtime_suspend+0x3c/0x44
> [ 292.510061] pm_generic_runtime_suspend+0x30/0x44
> [ 292.510071] pm_runtime_force_suspend+0x54/0xc8
> [ 292.510081] adreno_shutdown+0x1c/0x28
> [ 292.510090] platform_shutdown+0x2c/0x38
> [ 292.510104] device_shutdown+0x158/0x210
> [ 292.510119] kernel_restart_prepare+0x40/0x4c
>
> And here from GPU kthread, an SError OOPs:
>
> [ 192.648789] el1h_64_error+0x7c/0x80
> [ 192.648812] el1_interrupt+0x20/0x58
> [ 192.648833] el1h_64_irq_handler+0x18/0x24
> [ 192.648854] el1h_64_irq+0x7c/0x80
> [ 192.648873] local_daif_inherit+0x10/0x18
> [ 192.648900] el1h_64_sync_handler+0x48/0xb4
> [ 192.648921] el1h_64_sync+0x7c/0x80
> [ 192.648941] a6xx_gmu_set_oob+0xbc/0x1fc
> [ 192.648968] a6xx_hw_init+0x44/0xe38
> [ 192.648991] msm_gpu_hw_init+0x48/0x80
> [ 192.649013] msm_gpu_submit+0x5c/0x1a8
> [ 192.649034] msm_job_run+0xb0/0x11c
> [ 192.649058] drm_sched_main+0x170/0x434
> [ 192.649086] kthread+0x134/0x300
> [ 192.649114] ret_from_fork+0x10/0x20
>
> Fix by calling adreno_system_suspend() in the device_shutdown() path.
>
> [ Applied Rob Clark feedback on fixing adreno_unbind() similarly, also
> tested as above. ]
>
> Cc: Rob Clark <[email protected]>
> Cc: Steven Rostedt <[email protected]>
> Cc: Ricardo Ribalda <[email protected]>
> Cc: Ross Zwisler <[email protected]>
> Signed-off-by: Joel Fernandes (Google) <[email protected]>
Reviewed-by: Rob Clark <[email protected]>
> ---
> drivers/gpu/drm/msm/adreno/adreno_device.c | 5 +++--
> 1 file changed, 3 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/gpu/drm/msm/adreno/adreno_device.c b/drivers/gpu/drm/msm/adreno/adreno_device.c
> index 628806423f7d..36f062c7582f 100644
> --- a/drivers/gpu/drm/msm/adreno/adreno_device.c
> +++ b/drivers/gpu/drm/msm/adreno/adreno_device.c
> @@ -551,13 +551,14 @@ static int adreno_bind(struct device *dev, struct device *master, void *data)
> return 0;
> }
>
> +static int adreno_system_suspend(struct device *dev);
> static void adreno_unbind(struct device *dev, struct device *master,
> void *data)
> {
> struct msm_drm_private *priv = dev_get_drvdata(master);
> struct msm_gpu *gpu = dev_to_gpu(dev);
>
> - pm_runtime_force_suspend(dev);
> + WARN_ON_ONCE(adreno_system_suspend(dev));
> gpu->funcs->destroy(gpu);
>
> priv->gpu_pdev = NULL;
> @@ -609,7 +610,7 @@ static int adreno_remove(struct platform_device *pdev)
>
> static void adreno_shutdown(struct platform_device *pdev)
> {
> - pm_runtime_force_suspend(&pdev->dev);
> + WARN_ON_ONCE(adreno_system_suspend(&pdev->dev));
> }
>
> static const struct of_device_id dt_match[] = {
> --
> 2.39.0.314.g84b9a713c41-goog
>