From: Lee Duncan <[email protected]>
If there is an error queueing an iscsi command in
iscsi_queuecommand(), for example if the transport fails
to take the command in sessuin->tt->xmit_task(), then
the error path can call iscsi_complete_task() without
first aquiring the back_lock as required. This can
lead to things like ITT pool can get corrupt, resulting
in duplicate ITTs being sent out.
The solution is to hold the back_lock around
iscsi_complete_task() calls, and to add a little commenting
to help others understand when back_lock must be held.
Signed-off-by: Lee Duncan <[email protected]>
---
drivers/scsi/libiscsi.c | 22 ++++++++++++++++++++--
1 file changed, 20 insertions(+), 2 deletions(-)
diff --git a/drivers/scsi/libiscsi.c b/drivers/scsi/libiscsi.c
index b8d325ce8754..ef7871e8c6bd 100644
--- a/drivers/scsi/libiscsi.c
+++ b/drivers/scsi/libiscsi.c
@@ -838,7 +838,7 @@ EXPORT_SYMBOL_GPL(iscsi_conn_send_pdu);
* @datalen: len of buffer
*
* iscsi_cmd_rsp sets up the scsi_cmnd fields based on the PDU and
- * then completes the command and task.
+ * then completes the command and task. called under back_lock
**/
static void iscsi_scsi_cmd_rsp(struct iscsi_conn *conn, struct iscsi_hdr *hdr,
struct iscsi_task *task, char *data,
@@ -941,6 +941,9 @@ static void iscsi_scsi_cmd_rsp(struct iscsi_conn *conn, struct iscsi_hdr *hdr,
* @conn: iscsi connection
* @hdr: iscsi pdu
* @task: scsi command task
+ *
+ * iscsi_data_in_rsp sets up the scsi_cmnd fields based on the data received
+ * then completes the command and task. called under back_lock
**/
static void
iscsi_data_in_rsp(struct iscsi_conn *conn, struct iscsi_hdr *hdr,
@@ -1025,6 +1028,16 @@ static int iscsi_send_nopout(struct iscsi_conn *conn, struct iscsi_nopin *rhdr)
return 0;
}
+/**
+ * iscsi_nop_out_rsp - SCSI NOP Response processing
+ * @task: scsi command task
+ * @nop: the nop structure
+ * @data: where to put the data
+ * @datalen: length of data
+ *
+ * iscsi_nop_out_rsp handles nop response from use or
+ * from user space. called under back_lock
+ **/
static int iscsi_nop_out_rsp(struct iscsi_task *task,
struct iscsi_nopin *nop, char *data, int datalen)
{
@@ -1791,7 +1804,9 @@ int iscsi_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *sc)
return 0;
prepd_reject:
+ spin_lock_bh(&session->back_lock);
iscsi_complete_task(task, ISCSI_TASK_REQUEUE_SCSIQ);
+ spin_unlock_bh(&session->back_lock);
reject:
spin_unlock_bh(&session->frwd_lock);
ISCSI_DBG_SESSION(session, "cmd 0x%x rejected (%d)\n",
@@ -1799,7 +1814,9 @@ int iscsi_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *sc)
return SCSI_MLQUEUE_TARGET_BUSY;
prepd_fault:
+ spin_lock_bh(&session->back_lock);
iscsi_complete_task(task, ISCSI_TASK_REQUEUE_SCSIQ);
+ spin_unlock_bh(&session->back_lock);
fault:
spin_unlock_bh(&session->frwd_lock);
ISCSI_DBG_SESSION(session, "iscsi: cmd 0x%x is not queued (%d)\n",
@@ -3121,8 +3138,9 @@ fail_mgmt_tasks(struct iscsi_session *session, struct iscsi_conn *conn)
state = ISCSI_TASK_ABRT_SESS_RECOV;
if (task->state == ISCSI_TASK_PENDING)
state = ISCSI_TASK_COMPLETED;
+ spin_lock_bh(&session->back_lock);
iscsi_complete_task(task, state);
-
+ spin_unlock_bh(&session->back_lock);
}
}
--
2.16.4
On Mon, Feb 25, 2019 at 09:41:30AM -0800, Lee Duncan wrote:
> From: Lee Duncan <[email protected]>
>
> If there is an error queueing an iscsi command in
> iscsi_queuecommand(), for example if the transport fails
> to take the command in sessuin->tt->xmit_task(), then
> the error path can call iscsi_complete_task() without
> first aquiring the back_lock as required. This can
> lead to things like ITT pool can get corrupt, resulting
> in duplicate ITTs being sent out.
>
> The solution is to hold the back_lock around
> iscsi_complete_task() calls, and to add a little commenting
> to help others understand when back_lock must be held.
>
> Signed-off-by: Lee Duncan <[email protected]>
> ---
Lee,
Quick question, can you confirm that you tested this with lockdep?
It seems right to me, it's just that we've been hit with lockdep
problems dealing with these locks before.
- Chris
> drivers/scsi/libiscsi.c | 22 ++++++++++++++++++++--
> 1 file changed, 20 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/scsi/libiscsi.c b/drivers/scsi/libiscsi.c
> index b8d325ce8754..ef7871e8c6bd 100644
> --- a/drivers/scsi/libiscsi.c
> +++ b/drivers/scsi/libiscsi.c
> @@ -838,7 +838,7 @@ EXPORT_SYMBOL_GPL(iscsi_conn_send_pdu);
> * @datalen: len of buffer
> *
> * iscsi_cmd_rsp sets up the scsi_cmnd fields based on the PDU and
> - * then completes the command and task.
> + * then completes the command and task. called under back_lock
> **/
> static void iscsi_scsi_cmd_rsp(struct iscsi_conn *conn, struct iscsi_hdr *hdr,
> struct iscsi_task *task, char *data,
> @@ -941,6 +941,9 @@ static void iscsi_scsi_cmd_rsp(struct iscsi_conn *conn, struct iscsi_hdr *hdr,
> * @conn: iscsi connection
> * @hdr: iscsi pdu
> * @task: scsi command task
> + *
> + * iscsi_data_in_rsp sets up the scsi_cmnd fields based on the data received
> + * then completes the command and task. called under back_lock
> **/
> static void
> iscsi_data_in_rsp(struct iscsi_conn *conn, struct iscsi_hdr *hdr,
> @@ -1025,6 +1028,16 @@ static int iscsi_send_nopout(struct iscsi_conn *conn, struct iscsi_nopin *rhdr)
> return 0;
> }
>
> +/**
> + * iscsi_nop_out_rsp - SCSI NOP Response processing
> + * @task: scsi command task
> + * @nop: the nop structure
> + * @data: where to put the data
> + * @datalen: length of data
> + *
> + * iscsi_nop_out_rsp handles nop response from use or
> + * from user space. called under back_lock
> + **/
> static int iscsi_nop_out_rsp(struct iscsi_task *task,
> struct iscsi_nopin *nop, char *data, int datalen)
> {
> @@ -1791,7 +1804,9 @@ int iscsi_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *sc)
> return 0;
>
> prepd_reject:
> + spin_lock_bh(&session->back_lock);
> iscsi_complete_task(task, ISCSI_TASK_REQUEUE_SCSIQ);
> + spin_unlock_bh(&session->back_lock);
> reject:
> spin_unlock_bh(&session->frwd_lock);
> ISCSI_DBG_SESSION(session, "cmd 0x%x rejected (%d)\n",
> @@ -1799,7 +1814,9 @@ int iscsi_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *sc)
> return SCSI_MLQUEUE_TARGET_BUSY;
>
> prepd_fault:
> + spin_lock_bh(&session->back_lock);
> iscsi_complete_task(task, ISCSI_TASK_REQUEUE_SCSIQ);
> + spin_unlock_bh(&session->back_lock);
> fault:
> spin_unlock_bh(&session->frwd_lock);
> ISCSI_DBG_SESSION(session, "iscsi: cmd 0x%x is not queued (%d)\n",
> @@ -3121,8 +3138,9 @@ fail_mgmt_tasks(struct iscsi_session *session, struct iscsi_conn *conn)
> state = ISCSI_TASK_ABRT_SESS_RECOV;
> if (task->state == ISCSI_TASK_PENDING)
> state = ISCSI_TASK_COMPLETED;
> + spin_lock_bh(&session->back_lock);
> iscsi_complete_task(task, state);
> -
> + spin_unlock_bh(&session->back_lock);
> }
> }
>
> --
> 2.16.4
>
On 3/6/19 10:23 AM, Chris Leech wrote:
> On Mon, Feb 25, 2019 at 09:41:30AM -0800, Lee Duncan wrote:
>> From: Lee Duncan <[email protected]>
>>
>> If there is an error queueing an iscsi command in
>> iscsi_queuecommand(), for example if the transport fails
>> to take the command in sessuin->tt->xmit_task(), then
>> the error path can call iscsi_complete_task() without
>> first aquiring the back_lock as required. This can
>> lead to things like ITT pool can get corrupt, resulting
>> in duplicate ITTs being sent out.
>>
>> The solution is to hold the back_lock around
>> iscsi_complete_task() calls, and to add a little commenting
>> to help others understand when back_lock must be held.
>>
>> Signed-off-by: Lee Duncan <[email protected]>
>> ---
>
> Lee,
>
> Quick question, can you confirm that you tested this with lockdep?
>
> It seems right to me, it's just that we've been hit with lockdep
> problems dealing with these locks before.
>
> - Chris
I'm glad you asked, to keep me honest -- I had not done it yet, because
it seemed obvious to me.
But I did check today, and did not find any deadlock nor leaks. I was
testing on a 5.0.0-1 kernel.
>
>> drivers/scsi/libiscsi.c | 22 ++++++++++++++++++++--
>> 1 file changed, 20 insertions(+), 2 deletions(-)
>>
>> diff --git a/drivers/scsi/libiscsi.c b/drivers/scsi/libiscsi.c
>> index b8d325ce8754..ef7871e8c6bd 100644
>> --- a/drivers/scsi/libiscsi.c
>> +++ b/drivers/scsi/libiscsi.c
>> @@ -838,7 +838,7 @@ EXPORT_SYMBOL_GPL(iscsi_conn_send_pdu);
>> * @datalen: len of buffer
>> *
>> * iscsi_cmd_rsp sets up the scsi_cmnd fields based on the PDU and
>> - * then completes the command and task.
>> + * then completes the command and task. called under back_lock
>> **/
>> static void iscsi_scsi_cmd_rsp(struct iscsi_conn *conn, struct iscsi_hdr *hdr,
>> struct iscsi_task *task, char *data,
>> @@ -941,6 +941,9 @@ static void iscsi_scsi_cmd_rsp(struct iscsi_conn *conn, struct iscsi_hdr *hdr,
>> * @conn: iscsi connection
>> * @hdr: iscsi pdu
>> * @task: scsi command task
>> + *
>> + * iscsi_data_in_rsp sets up the scsi_cmnd fields based on the data received
>> + * then completes the command and task. called under back_lock
>> **/
>> static void
>> iscsi_data_in_rsp(struct iscsi_conn *conn, struct iscsi_hdr *hdr,
>> @@ -1025,6 +1028,16 @@ static int iscsi_send_nopout(struct iscsi_conn *conn, struct iscsi_nopin *rhdr)
>> return 0;
>> }
>>
>> +/**
>> + * iscsi_nop_out_rsp - SCSI NOP Response processing
>> + * @task: scsi command task
>> + * @nop: the nop structure
>> + * @data: where to put the data
>> + * @datalen: length of data
>> + *
>> + * iscsi_nop_out_rsp handles nop response from use or
>> + * from user space. called under back_lock
>> + **/
>> static int iscsi_nop_out_rsp(struct iscsi_task *task,
>> struct iscsi_nopin *nop, char *data, int datalen)
>> {
>> @@ -1791,7 +1804,9 @@ int iscsi_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *sc)
>> return 0;
>>
>> prepd_reject:
>> + spin_lock_bh(&session->back_lock);
>> iscsi_complete_task(task, ISCSI_TASK_REQUEUE_SCSIQ);
>> + spin_unlock_bh(&session->back_lock);
>> reject:
>> spin_unlock_bh(&session->frwd_lock);
>> ISCSI_DBG_SESSION(session, "cmd 0x%x rejected (%d)\n",
>> @@ -1799,7 +1814,9 @@ int iscsi_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *sc)
>> return SCSI_MLQUEUE_TARGET_BUSY;
>>
>> prepd_fault:
>> + spin_lock_bh(&session->back_lock);
>> iscsi_complete_task(task, ISCSI_TASK_REQUEUE_SCSIQ);
>> + spin_unlock_bh(&session->back_lock);
>> fault:
>> spin_unlock_bh(&session->frwd_lock);
>> ISCSI_DBG_SESSION(session, "iscsi: cmd 0x%x is not queued (%d)\n",
>> @@ -3121,8 +3138,9 @@ fail_mgmt_tasks(struct iscsi_session *session, struct iscsi_conn *conn)
>> state = ISCSI_TASK_ABRT_SESS_RECOV;
>> if (task->state == ISCSI_TASK_PENDING)
>> state = ISCSI_TASK_COMPLETED;
>> + spin_lock_bh(&session->back_lock);
>> iscsi_complete_task(task, state);
>> -
>> + spin_unlock_bh(&session->back_lock);
>> }
>> }
>>
>> --
>> 2.16.4
>>
>
On Thu, Mar 07, 2019 at 05:37:50PM -0800, Lee Duncan wrote:
> On 3/6/19 10:23 AM, Chris Leech wrote:
> > On Mon, Feb 25, 2019 at 09:41:30AM -0800, Lee Duncan wrote:
> >> From: Lee Duncan <[email protected]>
> >>
> >> If there is an error queueing an iscsi command in
> >> iscsi_queuecommand(), for example if the transport fails
> >> to take the command in sessuin->tt->xmit_task(), then
> >> the error path can call iscsi_complete_task() without
> >> first aquiring the back_lock as required. This can
> >> lead to things like ITT pool can get corrupt, resulting
> >> in duplicate ITTs being sent out.
> >>
> >> The solution is to hold the back_lock around
> >> iscsi_complete_task() calls, and to add a little commenting
> >> to help others understand when back_lock must be held.
> >>
> >> Signed-off-by: Lee Duncan <[email protected]>
> >> ---
> >
> > Lee,
> >
> > Quick question, can you confirm that you tested this with lockdep?
> >
> > It seems right to me, it's just that we've been hit with lockdep
> > problems dealing with these locks before.
> >
> > - Chris
>
> I'm glad you asked, to keep me honest -- I had not done it yet, because
> it seemed obvious to me.
>
> But I did check today, and did not find any deadlock nor leaks. I was
> testing on a 5.0.0-1 kernel.
Thanks!
Acked-by: Chris Leech <[email protected]>
Lee,
> If there is an error queueing an iscsi command in
> iscsi_queuecommand(), for example if the transport fails to take the
> command in sessuin->tt->xmit_task(), then the error path can call
> iscsi_complete_task() without first aquiring the back_lock as
> required. This can lead to things like ITT pool can get corrupt,
> resulting in duplicate ITTs being sent out.
>
> The solution is to hold the back_lock around iscsi_complete_task()
> calls, and to add a little commenting to help others understand when
> back_lock must be held.
Applied to 5.1/scsi-queue, thanks!
--
Martin K. Petersen Oracle Linux Engineering