iavf_reset_task() takes crit_lock at the beginning and holds
it during whole call. The function subsequently calls
iavf_init_interrupt_scheme() that grabs RTNL. Problem occurs
when userspace initiates during the reset task any ndo callback
that runs under RTNL like iavf_open() because some of that
functions tries to take crit_lock. This leads to classic A-B B-A
deadlock scenario.
To resolve this situation the device should be detached in
iavf_reset_task() prior taking crit_lock to avoid subsequent
ndos running under RTNL and reattach the device at the end.
Fixes: 62fe2a865e6d ("i40evf: add missing rtnl_lock() around i40evf_set_interrupt_capability")
Cc: Jacob Keller <[email protected]>
Cc: Patryk Piotrowski <[email protected]>
Tested-by: Vitaly Grinberg <[email protected]>
Signed-off-by: Ivan Vecera <[email protected]>
---
drivers/net/ethernet/intel/iavf/iavf_main.c | 22 +++++++++++++++------
1 file changed, 16 insertions(+), 6 deletions(-)
diff --git a/drivers/net/ethernet/intel/iavf/iavf_main.c b/drivers/net/ethernet/intel/iavf/iavf_main.c
index f39440ad5c50..ee8f911b57ea 100644
--- a/drivers/net/ethernet/intel/iavf/iavf_main.c
+++ b/drivers/net/ethernet/intel/iavf/iavf_main.c
@@ -2877,6 +2877,13 @@ static void iavf_reset_task(struct work_struct *work)
int i = 0, err;
bool running;
+ /*
+ * Detach interface to avoid subsequent NDO callbacks
+ */
+ rtnl_lock();
+ netif_device_detach(netdev);
+ rtnl_unlock();
+
/* When device is being removed it doesn't make sense to run the reset
* task, just return in such a case.
*/
@@ -2884,7 +2891,7 @@ static void iavf_reset_task(struct work_struct *work)
if (adapter->state != __IAVF_REMOVE)
queue_work(iavf_wq, &adapter->reset_task);
- return;
+ goto reset_finish;
}
while (!mutex_trylock(&adapter->client_lock))
@@ -2954,7 +2961,6 @@ static void iavf_reset_task(struct work_struct *work)
if (running) {
netif_carrier_off(netdev);
- netif_tx_stop_all_queues(netdev);
adapter->link_up = false;
iavf_napi_disable_all(adapter);
}
@@ -3081,10 +3087,8 @@ static void iavf_reset_task(struct work_struct *work)
adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED;
- mutex_unlock(&adapter->client_lock);
- mutex_unlock(&adapter->crit_lock);
+ goto reset_finish;
- return;
reset_err:
if (running) {
set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
@@ -3092,9 +3096,15 @@ static void iavf_reset_task(struct work_struct *work)
}
iavf_disable_vf(adapter);
+ dev_err(&adapter->pdev->dev, "failed to allocate resources during reinit\n");
+
+reset_finish:
mutex_unlock(&adapter->client_lock);
mutex_unlock(&adapter->crit_lock);
- dev_err(&adapter->pdev->dev, "failed to allocate resources during reinit\n");
+
+ rtnl_lock();
+ netif_device_attach(netdev);
+ rtnl_unlock();
}
/**
--
2.35.1
> -----Original Message-----
> From: ivecera <[email protected]>
> Sent: Thursday, August 18, 2022 6:56 PM
> To: [email protected]
> Cc: Keller, Jacob E <[email protected]>; Piotrowski, Patryk
> <[email protected]>; Vitaly Grinberg <[email protected]>;
> Brandeburg, Jesse <[email protected]>; Nguyen, Anthony L
> <[email protected]>; David S. Miller <[email protected]>; Eric
> Dumazet <[email protected]>; Jakub Kicinski <[email protected]>; Paolo
> Abeni <[email protected]>; Jeff Kirsher <[email protected]>;
> moderated list:INTEL ETHERNET DRIVERS <[email protected]>;
> open list <[email protected]>
> Subject: [PATCH net] iavf: Detach device during reset task
>
> iavf_reset_task() takes crit_lock at the beginning and holds it during whole call.
> The function subsequently calls
> iavf_init_interrupt_scheme() that grabs RTNL. Problem occurs when userspace
> initiates during the reset task any ndo callback that runs under RTNL like
> iavf_open() because some of that functions tries to take crit_lock. This leads to
> classic A-B B-A deadlock scenario.
>
> To resolve this situation the device should be detached in
> iavf_reset_task() prior taking crit_lock to avoid subsequent ndos running under
> RTNL and reattach the device at the end.
>
> Fixes: 62fe2a865e6d ("i40evf: add missing rtnl_lock() around
> i40evf_set_interrupt_capability")
> Cc: Jacob Keller <[email protected]>
> Cc: Patryk Piotrowski <[email protected]>
> Tested-by: Vitaly Grinberg <[email protected]>
> Signed-off-by: Ivan Vecera <[email protected]>
> ---
> drivers/net/ethernet/intel/iavf/iavf_main.c | 22 +++++++++++++++------
> 1 file changed, 16 insertions(+), 6 deletions(-)
>
> diff --git a/drivers/net/ethernet/intel/iavf/iavf_main.c
> b/drivers/net/ethernet/intel/iavf/iavf_main.c
> index f39440ad5c50..ee8f911b57ea 100644
> --- a/drivers/net/ethernet/intel/iavf/iavf_main.c
> +++ b/drivers/net/ethernet/intel/iavf/iavf_main.c
> @@ -2877,6 +2877,13 @@ static void iavf_reset_task(struct work_struct
> *work)
> int i = 0, err;
> bool running;
>
> + /*
> + * Detach interface to avoid subsequent NDO callbacks
> + */
nit:
The comment should start this way: /* Detach ...
> + rtnl_lock();
> + netif_device_detach(netdev);
> + rtnl_unlock();
> +
> /* When device is being removed it doesn't make sense to run the reset
> * task, just return in such a case.
> */
> @@ -2884,7 +2891,7 @@ static void iavf_reset_task(struct work_struct *work)
> if (adapter->state != __IAVF_REMOVE)
> queue_work(iavf_wq, &adapter->reset_task);
>
> - return;
> + goto reset_finish;
Correct me if I'm wrong.
In case when you fail to grab a crit_lock you'd jump to the reset_finish label and unlock the locks you didn't lock.
> }
>
> while (!mutex_trylock(&adapter->client_lock))
> @@ -2954,7 +2961,6 @@ static void iavf_reset_task(struct work_struct *work)
>
> if (running) {
> netif_carrier_off(netdev);
> - netif_tx_stop_all_queues(netdev);
> adapter->link_up = false;
> iavf_napi_disable_all(adapter);
> }
> @@ -3081,10 +3087,8 @@ static void iavf_reset_task(struct work_struct
> *work)
>
> adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED;
>
> - mutex_unlock(&adapter->client_lock);
> - mutex_unlock(&adapter->crit_lock);
> + goto reset_finish;
>
> - return;
> reset_err:
> if (running) {
> set_bit(__IAVF_VSI_DOWN, adapter->vsi.state); @@ -3092,9
> +3096,15 @@ static void iavf_reset_task(struct work_struct *work)
> }
> iavf_disable_vf(adapter);
>
> + dev_err(&adapter->pdev->dev, "failed to allocate resources during
> +reinit\n");
> +
> +reset_finish:
> mutex_unlock(&adapter->client_lock);
> mutex_unlock(&adapter->crit_lock);
> - dev_err(&adapter->pdev->dev, "failed to allocate resources during
> reinit\n");
> +
> + rtnl_lock();
> + netif_device_attach(netdev);
> + rtnl_unlock();
> }
>
> /**
> --
> 2.35.1
On Mon, 22 Aug 2022 17:42:44 +0000
"Laba, SlawomirX" <[email protected]> wrote:
> > -----Original Message-----
> > From: ivecera <[email protected]>
> > Sent: Thursday, August 18, 2022 6:56 PM
> > To: [email protected]
> > Cc: Keller, Jacob E <[email protected]>; Piotrowski, Patryk
> > <[email protected]>; Vitaly Grinberg <[email protected]>;
> > Brandeburg, Jesse <[email protected]>; Nguyen, Anthony L
> > <[email protected]>; David S. Miller <[email protected]>; Eric
> > Dumazet <[email protected]>; Jakub Kicinski <[email protected]>; Paolo
> > Abeni <[email protected]>; Jeff Kirsher <[email protected]>;
> > moderated list:INTEL ETHERNET DRIVERS <[email protected]>;
> > open list <[email protected]>
> > Subject: [PATCH net] iavf: Detach device during reset task
> >
> > iavf_reset_task() takes crit_lock at the beginning and holds it during whole call.
> > The function subsequently calls
> > iavf_init_interrupt_scheme() that grabs RTNL. Problem occurs when userspace
> > initiates during the reset task any ndo callback that runs under RTNL like
> > iavf_open() because some of that functions tries to take crit_lock. This leads to
> > classic A-B B-A deadlock scenario.
> >
> > To resolve this situation the device should be detached in
> > iavf_reset_task() prior taking crit_lock to avoid subsequent ndos running under
> > RTNL and reattach the device at the end.
> >
> > Fixes: 62fe2a865e6d ("i40evf: add missing rtnl_lock() around
> > i40evf_set_interrupt_capability")
> > Cc: Jacob Keller <[email protected]>
> > Cc: Patryk Piotrowski <[email protected]>
> > Tested-by: Vitaly Grinberg <[email protected]>
> > Signed-off-by: Ivan Vecera <[email protected]>
> > ---
> > drivers/net/ethernet/intel/iavf/iavf_main.c | 22 +++++++++++++++------
> > 1 file changed, 16 insertions(+), 6 deletions(-)
> >
> > diff --git a/drivers/net/ethernet/intel/iavf/iavf_main.c
> > b/drivers/net/ethernet/intel/iavf/iavf_main.c
> > index f39440ad5c50..ee8f911b57ea 100644
> > --- a/drivers/net/ethernet/intel/iavf/iavf_main.c
> > +++ b/drivers/net/ethernet/intel/iavf/iavf_main.c
> > @@ -2877,6 +2877,13 @@ static void iavf_reset_task(struct work_struct
> > *work)
> > int i = 0, err;
> > bool running;
> >
> > + /*
> > + * Detach interface to avoid subsequent NDO callbacks
> > + */
>
> nit:
> The comment should start this way: /* Detach ...
Will fix.
> > + rtnl_lock();
> > + netif_device_detach(netdev);
> > + rtnl_unlock();
> > +
> > /* When device is being removed it doesn't make sense to run the reset
> > * task, just return in such a case.
> > */
> > @@ -2884,7 +2891,7 @@ static void iavf_reset_task(struct work_struct *work)
> > if (adapter->state != __IAVF_REMOVE)
> > queue_work(iavf_wq, &adapter->reset_task);
> >
> > - return;
> > + goto reset_finish;
>
> Correct me if I'm wrong.
> In case when you fail to grab a crit_lock you'd jump to the reset_finish label and unlock the locks you didn't lock.
You're right.. I tried to dedup mutex unlocks after testing and missed this path.
Will fix by v2.
Thanks,
I.
> > }
> >
> > while (!mutex_trylock(&adapter->client_lock))
> > @@ -2954,7 +2961,6 @@ static void iavf_reset_task(struct work_struct *work)
> >
> > if (running) {
> > netif_carrier_off(netdev);
> > - netif_tx_stop_all_queues(netdev);
> > adapter->link_up = false;
> > iavf_napi_disable_all(adapter);
> > }
> > @@ -3081,10 +3087,8 @@ static void iavf_reset_task(struct work_struct
> > *work)
> >
> > adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED;
> >
> > - mutex_unlock(&adapter->client_lock);
> > - mutex_unlock(&adapter->crit_lock);
> > + goto reset_finish;
> >
> > - return;
> > reset_err:
> > if (running) {
> > set_bit(__IAVF_VSI_DOWN, adapter->vsi.state); @@ -3092,9
> > +3096,15 @@ static void iavf_reset_task(struct work_struct *work)
> > }
> > iavf_disable_vf(adapter);
> >
> > + dev_err(&adapter->pdev->dev, "failed to allocate resources during
> > +reinit\n");
> > +
> > +reset_finish:
> > mutex_unlock(&adapter->client_lock);
> > mutex_unlock(&adapter->crit_lock);
> > - dev_err(&adapter->pdev->dev, "failed to allocate resources during
> > reinit\n");
> > +
> > + rtnl_lock();
> > + netif_device_attach(netdev);
> > + rtnl_unlock();
> > }
> >
> > /**
> > --
> > 2.35.1
>