When there is a scsi command timed-out or failed, the scsi eh
tries a thorugh recovery, which is necessary for non-redundant
systems. However, the thorugh recovery usually takes much time,
which is not acceptable for misson critical systems. To improve
this latency, if we are working on a redundant system, we should
avoid the scsi eh for its long time failing recovery, and quick
failover to another path.
This set of patches is trying to implement above.
NOTE: the userland tools need to eusure the environment
restriction, which will be implemented later.
Thanks,
Ren
Ren Mingxin (5):
scsi: rename return code FAST_IO_FAIL to FAST_IO
FC transport: Add interface to specify fast io level for timed-out cmds
SAS transport: Add interface to specify fast io level for timed-out cmds
lpfc: Allow fast timed-out io recovery
mptfusion: Allow fast timed-out io recovery
drivers/message/fusion/mptscsih.c | 29 ++++++++-
drivers/scsi/lpfc/lpfc_scsi.c | 34 ++++++++++
drivers/scsi/scsi_error.c | 18 ++---
drivers/scsi/scsi_sas_internal.h | 4 -
drivers/scsi/scsi_transport_fc.c | 112 ++++++++++++++++++++++++++++++++++--
drivers/scsi/scsi_transport_iscsi.c | 6 -
drivers/scsi/scsi_transport_sas.c | 103 ++++++++++++++++++++++++++++++++-
include/scsi/scsi.h | 2
include/scsi/scsi_transport_fc.h | 11 +++
include/scsi/scsi_transport_sas.h | 8 ++
10 files changed, 303 insertions(+), 24 deletions(-)
This patch introduces new interfaces through sysfs for fc hosts
and rports to allow users to avoid the scsi_eh recovery actions
on different levels when scsi commands timed out, e.g.
/sys/devices/pci***/.../hostN/fc_host/hostN/fast_io_tmo_flags
/sys/devices/pci***/.../hostN/rport-X:Y-Z/fc_remote_ports/\
rport-X:Y-Z/fast_io_tmo_flags
This new added interface "fast_io_tmo_flags" is a 8-bit mask with
low 5-bit available up to now:
0x01 - Ignore aborting commands
0x02 - Ignore device resets
0x04 - Ignore target resets
0x08 - Ignore bus resets
0x10 - Ignore host resets
When scsi_eh unjams hosts, the corresponding bit fields will be
checked by LLDD to decide whether to ignore specified recovery
levels. Its value is zero by default, so it keeps existing
behavior, which is necessary for non-redundant systems.
This interface is mainly for redundant environments. To
redundant systems, they need a quick give up and failover,
instead of thorough recovery which usually takes much time.
The actions in LLDD/redundant configurations should be implemented
individually later.
Signed-off-by: Ren Mingxin <[email protected]>
---
drivers/scsi/scsi_transport_fc.c | 108 +++++++++++++++++++++++++++++++++++++-
include/scsi/scsi_transport_fc.h | 11 ++++
2 files changed, 117 insertions(+), 2 deletions(-)
diff --git a/drivers/scsi/scsi_transport_fc.c b/drivers/scsi/scsi_transport_fc.c
index 7b29e00..155a658 100644
--- a/drivers/scsi/scsi_transport_fc.c
+++ b/drivers/scsi/scsi_transport_fc.c
@@ -310,9 +310,9 @@ static void fc_scsi_scan_rport(struct work_struct *work);
* Increase these values if you add attributes
*/
#define FC_STARGET_NUM_ATTRS 3
-#define FC_RPORT_NUM_ATTRS 10
+#define FC_RPORT_NUM_ATTRS 11
#define FC_VPORT_NUM_ATTRS 9
-#define FC_HOST_NUM_ATTRS 29
+#define FC_HOST_NUM_ATTRS 30
struct fc_internal {
struct scsi_transport_template t;
@@ -995,6 +995,67 @@ store_fc_rport_fast_io_fail_tmo(struct device *dev,
static FC_DEVICE_ATTR(rport, fast_io_fail_tmo, S_IRUGO | S_IWUSR,
show_fc_rport_fast_io_fail_tmo, store_fc_rport_fast_io_fail_tmo);
+/*
+ * fast_io_tmo_flags attribute
+ */
+static ssize_t
+show_fc_rport_fast_io_tmo_flags(struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct fc_rport *rport = transport_class_to_rport(dev);
+
+ return sprintf(buf, "0x%02x\n", rport->fast_io_tmo_flags);
+}
+
+static int fc_str_to_fast_io_tmo_flags(const char *buf, u8 *val)
+{
+ char *cp;
+
+ *val = simple_strtoul(buf, &cp, 0) & 0xff;
+ if (cp == buf)
+ return -EINVAL;
+
+ return 0;
+}
+
+static int fc_rport_set_fast_io_tmo_flags(struct fc_rport *rport, u8 val)
+{
+ if ((rport->port_state == FC_PORTSTATE_BLOCKED) ||
+ (rport->port_state == FC_PORTSTATE_DELETED) ||
+ (rport->port_state == FC_PORTSTATE_NOTPRESENT))
+ return -EBUSY;
+
+ rport->fast_io_tmo_flags = val;
+
+ return 0;
+}
+
+static ssize_t
+store_fc_rport_fast_io_tmo_flags(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf,
+ size_t count)
+{
+ struct fc_rport *rport = transport_class_to_rport(dev);
+ u8 val;
+ int rc;
+
+ if (count < 1)
+ return -EINVAL;
+
+ rc = fc_str_to_fast_io_tmo_flags(buf, &val);
+ if (rc)
+ return rc;
+
+ rc = fc_rport_set_fast_io_tmo_flags(rport, val);
+ if (rc)
+ return rc;
+ return count;
+}
+static FC_DEVICE_ATTR(rport, fast_io_tmo_flags, S_IRUGO | S_IWUSR,
+ show_fc_rport_fast_io_tmo_flags, store_fc_rport_fast_io_tmo_flags);
+
/*
* FC SCSI Target Attribute Management
@@ -1679,6 +1740,47 @@ static FC_DEVICE_ATTR(host, dev_loss_tmo, S_IRUGO | S_IWUSR,
show_fc_host_dev_loss_tmo,
store_fc_private_host_dev_loss_tmo);
+static ssize_t
+show_fc_private_host_fast_io_tmo_flags (struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct Scsi_Host *shost = transport_class_to_shost(dev);
+
+ return sprintf(buf, "0x%02x\n", fc_host_fast_io_tmo_flags(shost));
+}
+
+static ssize_t
+store_fc_private_host_fast_io_tmo_flags(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf,
+ size_t count)
+{
+ struct Scsi_Host *shost = transport_class_to_shost(dev);
+ struct fc_host_attrs *fc_host = shost_to_fc_host(shost);
+ struct fc_rport *rport;
+ u8 val;
+ int rc;
+ unsigned long flags;
+
+ if (count < 1)
+ return -EINVAL;
+
+ rc = fc_str_to_fast_io_tmo_flags(buf, &val);
+ if (rc)
+ return rc;
+
+ fc_host_fast_io_tmo_flags(shost) = val;
+ spin_lock_irqsave(shost->host_lock, flags);
+ list_for_each_entry(rport, &fc_host->rports, peers)
+ fc_rport_set_fast_io_tmo_flags(rport, val);
+ spin_unlock_irqrestore(shost->host_lock, flags);
+ return count;
+}
+static FC_DEVICE_ATTR(host, fast_io_tmo_flags, S_IRUGO | S_IWUSR,
+ show_fc_private_host_fast_io_tmo_flags,
+ store_fc_private_host_fast_io_tmo_flags);
+
fc_private_host_rd_attr(npiv_vports_inuse, "%u\n", 20);
/*
@@ -2261,6 +2363,7 @@ fc_attach_transport(struct fc_function_template *ft)
/* Transport-managed attributes */
SETUP_PRIVATE_HOST_ATTRIBUTE_RW(dev_loss_tmo);
+ SETUP_PRIVATE_HOST_ATTRIBUTE_RW(fast_io_tmo_flags);
SETUP_PRIVATE_HOST_ATTRIBUTE_RW(tgtid_bind_type);
if (ft->issue_fc_host_lip)
SETUP_PRIVATE_HOST_ATTRIBUTE_RW(issue_lip);
@@ -2287,6 +2390,7 @@ fc_attach_transport(struct fc_function_template *ft)
SETUP_PRIVATE_RPORT_ATTRIBUTE_RD(port_state);
SETUP_PRIVATE_RPORT_ATTRIBUTE_RD(scsi_target_id);
SETUP_PRIVATE_RPORT_ATTRIBUTE_RW(fast_io_fail_tmo);
+ SETUP_PRIVATE_RPORT_ATTRIBUTE_RW(fast_io_tmo_flags);
BUG_ON(count > FC_RPORT_NUM_ATTRS);
diff --git a/include/scsi/scsi_transport_fc.h b/include/scsi/scsi_transport_fc.h
index b797e8f..8fd673c 100644
--- a/include/scsi/scsi_transport_fc.h
+++ b/include/scsi/scsi_transport_fc.h
@@ -336,6 +336,7 @@ struct fc_rport { /* aka fc_starget_attrs */
enum fc_port_state port_state; /* Will only be ONLINE or UNKNOWN */
u32 scsi_target_id;
u32 fast_io_fail_tmo;
+ u8 fast_io_tmo_flags;
/* exported data */
void *dd_data; /* Used for driver-specific storage */
@@ -354,6 +355,13 @@ struct fc_rport { /* aka fc_starget_attrs */
struct request_queue *rqst_q; /* bsg support */
} __attribute__((aligned(sizeof(unsigned long))));
+/* bit field values for struct fc_rport "fast_io_tmo_flags" field: */
+#define FC_RPORT_IGN_ABORT_CMDS 0x01
+#define FC_RPORT_IGN_DEVICE_RESET 0x02
+#define FC_RPORT_IGN_TARGET_RESET 0x04
+#define FC_RPORT_IGN_BUS_RESET 0x08
+#define FC_RPORT_IGN_HOST_RESET 0x10
+
/* bit field values for struct fc_rport "flags" field: */
#define FC_RPORT_DEVLOSS_PENDING 0x01
#define FC_RPORT_SCAN_PENDING 0x02
@@ -519,6 +527,7 @@ struct fc_host_attrs {
/* Private (Transport-managed) Attributes */
enum fc_tgtid_binding_type tgtid_bind_type;
+ u8 fast_io_tmo_flags;
/* internal data */
struct list_head rports;
@@ -616,6 +625,8 @@ struct fc_host_attrs {
(((struct fc_host_attrs *)(x)->shost_data)->devloss_work_q)
#define fc_host_dev_loss_tmo(x) \
(((struct fc_host_attrs *)(x)->shost_data)->dev_loss_tmo)
+#define fc_host_fast_io_tmo_flags(x) \
+ (((struct fc_host_attrs *)(x)->shost_data)->fast_io_tmo_flags)
struct fc_bsg_buffer {
--
1.7.1
The return code FAST_IO_FAIL was introduced for fast failed io
recovery. To use this code for fast timed-out io recovery as well,
we'd rename it to FAST_IO.
Signed-off-by: Ren Mingxin <[email protected]>
---
drivers/scsi/scsi_error.c | 18 +++++++++---------
drivers/scsi/scsi_transport_fc.c | 4 ++--
drivers/scsi/scsi_transport_iscsi.c | 6 +++---
include/scsi/scsi.h | 2 +-
4 files changed, 15 insertions(+), 15 deletions(-)
diff --git a/drivers/scsi/scsi_error.c b/drivers/scsi/scsi_error.c
index f43de1e..9e8e37a 100644
--- a/drivers/scsi/scsi_error.c
+++ b/drivers/scsi/scsi_error.c
@@ -1067,9 +1067,9 @@ static int scsi_eh_abort_cmds(struct list_head *work_q,
"0x%p\n", current->comm,
scmd));
rtn = scsi_try_to_abort_cmd(scmd->device->host->hostt, scmd);
- if (rtn == SUCCESS || rtn == FAST_IO_FAIL) {
+ if (rtn == SUCCESS || rtn == FAST_IO) {
scmd->eh_eflags &= ~SCSI_EH_CANCEL_CMD;
- if (rtn == FAST_IO_FAIL)
+ if (rtn == FAST_IO)
scsi_eh_finish_cmd(scmd, done_q);
else
list_move_tail(&scmd->eh_entry, &check_list);
@@ -1195,9 +1195,9 @@ static int scsi_eh_bus_device_reset(struct Scsi_Host *shost,
" 0x%p\n", current->comm,
sdev));
rtn = scsi_try_bus_device_reset(bdr_scmd);
- if (rtn == SUCCESS || rtn == FAST_IO_FAIL) {
+ if (rtn == SUCCESS || rtn == FAST_IO) {
if (!scsi_device_online(sdev) ||
- rtn == FAST_IO_FAIL ||
+ rtn == FAST_IO ||
!scsi_eh_tur(bdr_scmd)) {
list_for_each_entry_safe(scmd, next,
work_q, eh_entry) {
@@ -1248,7 +1248,7 @@ static int scsi_eh_target_reset(struct Scsi_Host *shost,
"to target %d\n",
current->comm, id));
rtn = scsi_try_target_reset(scmd);
- if (rtn != SUCCESS && rtn != FAST_IO_FAIL)
+ if (rtn != SUCCESS && rtn != FAST_IO)
SCSI_LOG_ERROR_RECOVERY(3, printk("%s: Target reset"
" failed target: "
"%d\n",
@@ -1259,7 +1259,7 @@ static int scsi_eh_target_reset(struct Scsi_Host *shost,
if (rtn == SUCCESS)
list_move_tail(&scmd->eh_entry, &check_list);
- else if (rtn == FAST_IO_FAIL)
+ else if (rtn == FAST_IO)
scsi_eh_finish_cmd(scmd, done_q);
else
/* push back on work queue for further processing */
@@ -1311,10 +1311,10 @@ static int scsi_eh_bus_reset(struct Scsi_Host *shost,
" %d\n", current->comm,
channel));
rtn = scsi_try_bus_reset(chan_scmd);
- if (rtn == SUCCESS || rtn == FAST_IO_FAIL) {
+ if (rtn == SUCCESS || rtn == FAST_IO) {
list_for_each_entry_safe(scmd, next, work_q, eh_entry) {
if (channel == scmd_channel(scmd)) {
- if (rtn == FAST_IO_FAIL)
+ if (rtn == FAST_IO)
scsi_eh_finish_cmd(scmd,
done_q);
else
@@ -1354,7 +1354,7 @@ static int scsi_eh_host_reset(struct list_head *work_q,
rtn = scsi_try_host_reset(scmd);
if (rtn == SUCCESS) {
list_splice_init(work_q, &check_list);
- } else if (rtn == FAST_IO_FAIL) {
+ } else if (rtn == FAST_IO) {
list_for_each_entry_safe(scmd, next, work_q, eh_entry) {
scsi_eh_finish_cmd(scmd, done_q);
}
diff --git a/drivers/scsi/scsi_transport_fc.c b/drivers/scsi/scsi_transport_fc.c
index e106c27..7b29e00 100644
--- a/drivers/scsi/scsi_transport_fc.c
+++ b/drivers/scsi/scsi_transport_fc.c
@@ -3301,7 +3301,7 @@ fc_scsi_scan_rport(struct work_struct *work)
* rports which would lead to offlined SCSI devices.
*
* Returns: 0 if the fc_rport left the state FC_PORTSTATE_BLOCKED.
- * FAST_IO_FAIL if the fast_io_fail_tmo fired, this should be
+ * FAST_IO if the fast_io_fail_tmo fired, this should be
* passed back to scsi_eh.
*/
int fc_block_scsi_eh(struct scsi_cmnd *cmnd)
@@ -3320,7 +3320,7 @@ int fc_block_scsi_eh(struct scsi_cmnd *cmnd)
spin_unlock_irqrestore(shost->host_lock, flags);
if (rport->flags & FC_RPORT_FAST_FAIL_TIMEDOUT)
- return FAST_IO_FAIL;
+ return FAST_IO;
return 0;
}
diff --git a/drivers/scsi/scsi_transport_iscsi.c b/drivers/scsi/scsi_transport_iscsi.c
index 133926b..8a88f45 100644
--- a/drivers/scsi/scsi_transport_iscsi.c
+++ b/drivers/scsi/scsi_transport_iscsi.c
@@ -1528,8 +1528,8 @@ static void iscsi_scan_session(struct work_struct *work)
*
* If the session is down this function will wait for the recovery
* timer to fire or for the session to be logged back in. If the
- * recovery timer fires then FAST_IO_FAIL is returned. The caller
- * should pass this error value to the scsi eh.
+ * recovery timer fires then FAST_IO is returned. The caller should
+ * pass this error value to the scsi eh.
*/
int iscsi_block_scsi_eh(struct scsi_cmnd *cmd)
{
@@ -1541,7 +1541,7 @@ int iscsi_block_scsi_eh(struct scsi_cmnd *cmd)
spin_lock_irqsave(&session->lock, flags);
while (session->state != ISCSI_SESSION_LOGGED_IN) {
if (session->state == ISCSI_SESSION_FREE) {
- ret = FAST_IO_FAIL;
+ ret = FAST_IO;
break;
}
spin_unlock_irqrestore(&session->lock, flags);
diff --git a/include/scsi/scsi.h b/include/scsi/scsi.h
index 66216c1..3e12e1e 100644
--- a/include/scsi/scsi.h
+++ b/include/scsi/scsi.h
@@ -480,7 +480,7 @@ static inline int scsi_is_wlun(unsigned int lun)
#define ADD_TO_MLQUEUE 0x2006
#define TIMEOUT_ERROR 0x2007
#define SCSI_RETURN_NOT_HANDLED 0x2008
-#define FAST_IO_FAIL 0x2009
+#define FAST_IO 0x2009
#define TARGET_ERROR 0x200A
/*
--
1.7.1
This patch implements fast timed-out io recovery in LLDD(lpfc) by
checking the corresponding bit fields specified in the new added
interface "fast_io_tmo_flags" and returning "FAST_IO" to avoid the
scsi_eh recovery actions on corresponding levels.
This is mainly for redundant configurations. To non-redundant
systems, the thorough recovery is necessary.
Furthermore, userland tools such as multipath-tools should ensure
that this policy is available only if there are more than one path
active, which will be implemented later.
Here is an example which can show the improvement of this patch:
before:
- takes about 3s to write 800MB normally
# dd if=/dev/zero of=/dev/mapper/mpathb bs=4k count=200000
200000+0 records in
200000+0 records out
819200000 bytes (819 MB) copied, 3.10581 s, 264 MB/s
- takes about 105s to write 800MB when I/Os timed out
# grep lpfc_template /proc/kallsyms
ffffffffa00f83a0 d lpfc_template [lpfc]
# insmod scsi_timeout.ko param=0xffffffffa00f83a0,2:0:0:1[*]
# dd if=/dev/zero of=/dev/mapper/mpathb bs=4k count=200000
200000+0 records in
200000+0 records out
819200000 bytes (819 MB) copied, 104.91 s, 7.8 MB/s
after:
- takes about 34s to write 800MB by using this patch when I/Os
timed out
# echo 0x1f > /sys/devices/pci0000:00/0000:00:03.0/\
0000:01:00.0/0000:02:01.0/0000:0a:00.0/\
0000:0b:01.0/0000:0d:00.0/host2/rport-2:0-2/\
fc_remote_ports/rport-2:0-2/fast_io_tmo_flags
# insmod scsi_timeout.ko param=0xffffffffa00f83a0,2:0:0:1
# dd if=/dev/zero of=/dev/mapper/mpathb bs=4k count=200000
200000+0 records in
200000+0 records out
819200000 bytes (819 MB) copied, 33.7718 s, 24.3 MB/s
* scsi_timeout.ko is a self-made module which wraps the scsi
queuecommand handler and ignores I/Os to the specified device
and any I/Os are not passed to LLDD.
Reference:
http://www.spinics.net/lists/linux-scsi/msg35091.html
So with this patch, we just spend time writing(about 3s) and
waiting through timeout(30s), and save about 71s in scsi eh.
Signed-off-by: Ren Mingxin <[email protected]>
---
drivers/scsi/lpfc/lpfc_scsi.c | 34 ++++++++++++++++++++++++++++++++--
1 files changed, 32 insertions(+), 2 deletions(-)
diff --git a/drivers/scsi/lpfc/lpfc_scsi.c b/drivers/scsi/lpfc/lpfc_scsi.c
index 8523b27..796893b 100644
--- a/drivers/scsi/lpfc/lpfc_scsi.c
+++ b/drivers/scsi/lpfc/lpfc_scsi.c
@@ -4798,6 +4798,7 @@ lpfc_abort_handler(struct scsi_cmnd *cmnd)
{
struct Scsi_Host *shost = cmnd->device->host;
struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;
+ struct fc_rport *rport = starget_to_rport(scsi_target(cmnd->device));
struct lpfc_hba *phba = vport->phba;
struct lpfc_iocbq *iocb;
struct lpfc_iocbq *abtsiocb;
@@ -4811,6 +4812,11 @@ lpfc_abort_handler(struct scsi_cmnd *cmnd)
if (status != 0 && status != SUCCESS)
return status;
+ if (rport->fast_io_tmo_flags & FC_RPORT_IGN_ABORT_CMDS) {
+ scsi_device_set_state(cmnd->device, SDEV_OFFLINE);
+ return FAST_IO;
+ }
+
spin_lock_irqsave(&phba->hbalock, flags);
/* driver queued commands are in process of being flushed */
if (phba->hba_flag & HBA_FCP_IOQ_FLUSH) {
@@ -5150,6 +5156,7 @@ lpfc_device_reset_handler(struct scsi_cmnd *cmnd)
{
struct Scsi_Host *shost = cmnd->device->host;
struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;
+ struct fc_rport *rport = starget_to_rport(scsi_target(cmnd->device));
struct lpfc_rport_data *rdata = cmnd->device->hostdata;
struct lpfc_nodelist *pnode;
unsigned tgt_id = cmnd->device->id;
@@ -5167,6 +5174,11 @@ lpfc_device_reset_handler(struct scsi_cmnd *cmnd)
if (status != 0 && status != SUCCESS)
return status;
+ if (rport->fast_io_tmo_flags & FC_RPORT_IGN_DEVICE_RESET) {
+ scsi_device_set_state(cmnd->device, SDEV_OFFLINE);
+ return FAST_IO;
+ }
+
status = lpfc_chk_tgt_mapped(vport, cmnd);
if (status == FAILED) {
lpfc_printf_vlog(vport, KERN_ERR, LOG_FCP,
@@ -5217,6 +5229,7 @@ lpfc_target_reset_handler(struct scsi_cmnd *cmnd)
{
struct Scsi_Host *shost = cmnd->device->host;
struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;
+ struct fc_rport *rport = starget_to_rport(scsi_target(cmnd->device));
struct lpfc_rport_data *rdata = cmnd->device->hostdata;
struct lpfc_nodelist *pnode;
unsigned tgt_id = cmnd->device->id;
@@ -5234,6 +5247,11 @@ lpfc_target_reset_handler(struct scsi_cmnd *cmnd)
if (status != 0 && status != SUCCESS)
return status;
+ if (rport->fast_io_tmo_flags & FC_RPORT_IGN_TARGET_RESET) {
+ scsi_device_set_state(cmnd->device, SDEV_OFFLINE);
+ return FAST_IO;
+ }
+
status = lpfc_chk_tgt_mapped(vport, cmnd);
if (status == FAILED) {
lpfc_printf_vlog(vport, KERN_ERR, LOG_FCP,
@@ -5284,6 +5302,7 @@ lpfc_bus_reset_handler(struct scsi_cmnd *cmnd)
{
struct Scsi_Host *shost = cmnd->device->host;
struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;
+ struct fc_rport *rport = starget_to_rport(scsi_target(cmnd->device));
struct lpfc_nodelist *ndlp = NULL;
struct lpfc_scsi_event_header scsi_event;
int match;
@@ -5302,6 +5321,11 @@ lpfc_bus_reset_handler(struct scsi_cmnd *cmnd)
if (status != 0 && status != SUCCESS)
return status;
+ if (rport->fast_io_tmo_flags & FC_RPORT_IGN_BUS_RESET) {
+ scsi_device_set_state(cmnd->device, SDEV_OFFLINE);
+ return FAST_IO;
+ }
+
/*
* Since the driver manages a single bus device, reset all
* targets known to the driver. Should any target reset
@@ -5373,11 +5397,17 @@ lpfc_bus_reset_handler(struct scsi_cmnd *cmnd)
static int
lpfc_host_reset_handler(struct scsi_cmnd *cmnd)
{
- struct Scsi_Host *shost = cmnd->device->host;
+ struct Scsi_Host *shost = cmnd->device->host;
struct lpfc_vport *vport = (struct lpfc_vport *) shost->hostdata;
- struct lpfc_hba *phba = vport->phba;
+ struct fc_rport *rport = starget_to_rport(scsi_target(cmnd->device));
+ struct lpfc_hba *phba = vport->phba;
int rc, ret = SUCCESS;
+ if (rport->fast_io_tmo_flags & FC_RPORT_IGN_HOST_RESET) {
+ scsi_device_set_state(cmnd->device, SDEV_OFFLINE);
+ return FAST_IO;
+ }
+
lpfc_printf_vlog(vport, KERN_ERR, LOG_FCP,
"3172 SCSI layer issued Host Reset Data:\n");
--
1.7.1
This patch implements fast timed-out io recovery in LLDD(mptfusion)
by checking the corresponding bit fields specified in the new added
interface "fast_io_tmo_flags" and returning "FAST_IO" to avoid the
scsi_eh recovery actions on corresponding levels.
This is mainly for redundant configurations. To non-redundant
systems, the thorough recovery is necessary.
Furthermore, userland tools such as mdadm should ensure that this
policy is available only if there are more than one mirrored
devices active, which will be implemented later.
NOTE: the device reset handler isn't implemented and the bus rest
handler isn't defined for mptsas_driver_template.
Here is an example which can show the improvement of this patch on
md-raid1 devices:
before:
- takes about 69s to write 8GB normally
# dd if=/dev/zero of=/dev/md0 bs=4k count=2000000
2000000+0 records in
2000000+0 records out
8192000000 bytes (8.2 GB) copied, 68.7898 s, 119 MB/s
- takes about 188s to write 8GB when I/Os timed out
# grep mptsas_driver_template /proc/kallsyms
ffffffffa00485c0 d mptsas_driver_template [mptsas]
# insmod scsi_timeout.ko param=0xffffffffa00485c0,1:0:1:0[*]
# dd if=/dev/zero of=/dev/md0 bs=4k count=2000000
2000000+0 records in
2000000+0 records out
8192000000 bytes (8.2 GB) copied, 187.857 s, 43.6 MB/s
after:
- takes about 129s to write 8GB by using this patch when I/Os
timed out
# echo 0x1f > /sys/devices/pci0000:00/0000:00:03.0/\
0000:01:00.0/0000:02:00.0/0000:03:00.0/\
0000:04:03.0/0000:08:00.0/host1/port-1:1/\
end_device-1:1/sas_device/end_device-1:1/\
fast_io_tmo_flags
# insmod scsi_timeout.ko param=0xffffffffa00485c0,1:0:1:0
# dd if=/dev/zero of=/dev/md127 bs=4k count=2000000
2000000+0 records in
2000000+0 records out
8192000000 bytes (8.2 GB) copied, 129.478 s, 63.3 MB/s
* scsi_timeout.ko is a self-made module which wraps the scsi
queuecommand handler and ignores I/Os to the specified device
and any I/Os are not passed to LLDD.
Reference:
http://www.spinics.net/lists/linux-scsi/msg35091.html
So with this patch, we just spend time writing(about 69s) and
waiting through timeout(60s), and save about 59s in scsi eh.
Signed-off-by: Ren Mingxin <[email protected]>
---
drivers/message/fusion/mptscsih.c | 29 +++++++++++++++++++++++++++--
1 files changed, 27 insertions(+), 2 deletions(-)
diff --git a/drivers/message/fusion/mptscsih.c b/drivers/message/fusion/mptscsih.c
index 727819c..47ef776 100644
--- a/drivers/message/fusion/mptscsih.c
+++ b/drivers/message/fusion/mptscsih.c
@@ -62,6 +62,7 @@
#include <scsi/scsi_host.h>
#include <scsi/scsi_tcq.h>
#include <scsi/scsi_dbg.h>
+#include <scsi/scsi_transport_sas.h>
#include "mptbase.h"
#include "mptscsih.h"
@@ -1698,6 +1699,12 @@ mptscsih_abort(struct scsi_cmnd * SCpnt)
int retval;
VirtDevice *vdevice;
MPT_ADAPTER *ioc;
+ struct sas_rphy *rphy = target_to_rphy(SCpnt->device->sdev_target);
+
+ if (rphy->fast_io_tmo_flags & SAS_RPHY_IGN_ABORT_CMDS) {
+ scsi_device_set_state(SCpnt->device, SDEV_OFFLINE);
+ return FAST_IO;
+ }
/* If we can't locate our host adapter structure, return FAILED status.
*/
@@ -1818,6 +1825,12 @@ mptscsih_dev_reset(struct scsi_cmnd * SCpnt)
int retval;
VirtDevice *vdevice;
MPT_ADAPTER *ioc;
+ struct sas_rphy *rphy = target_to_rphy(SCpnt->device->sdev_target);
+
+ if (rphy->fast_io_tmo_flags & SAS_RPHY_IGN_TARGET_RESET) {
+ scsi_device_set_state(SCpnt->device, SDEV_OFFLINE);
+ return FAST_IO;
+ }
/* If we can't locate our host adapter structure, return FAILED status.
*/
@@ -1878,6 +1891,12 @@ mptscsih_bus_reset(struct scsi_cmnd * SCpnt)
int retval;
VirtDevice *vdevice;
MPT_ADAPTER *ioc;
+ struct sas_rphy *rphy = target_to_rphy(SCpnt->device->sdev_target);
+
+ if (rphy->fast_io_tmo_flags & SAS_RPHY_IGN_BUS_RESET) {
+ scsi_device_set_state(SCpnt->device, SDEV_OFFLINE);
+ return FAST_IO;
+ }
/* If we can't locate our host adapter structure, return FAILED status.
*/
@@ -1924,10 +1943,16 @@ mptscsih_bus_reset(struct scsi_cmnd * SCpnt)
int
mptscsih_host_reset(struct scsi_cmnd *SCpnt)
{
- MPT_SCSI_HOST * hd;
- int status = SUCCESS;
+ MPT_SCSI_HOST *hd;
+ int status = SUCCESS;
MPT_ADAPTER *ioc;
int retval;
+ struct sas_rphy *rphy = target_to_rphy(SCpnt->device->sdev_target);
+
+ if (rphy->fast_io_tmo_flags & SAS_RPHY_IGN_HOST_RESET) {
+ scsi_device_set_state(SCpnt->device, SDEV_OFFLINE);
+ return FAST_IO;
+ }
/* If we can't locate the host to reset, then we failed. */
if ((hd = shost_priv(SCpnt->device->host)) == NULL){
--
1.7.1
This patch introduces new interfaces through sysfs for sas hosts
and rphys to allow users to avoid the scsi_eh recovery actions
on different levels when scsi commands timed out, e.g.
/sys/devices/pci***/.../hostN/sas_host/hostN/fast_io_tmo_flags
/sys/devices/pci***/.../hostN/port-X:Y/end_device-X:Y/\
sas_device/end_device-X:Y/fast_io_tmo_flags
This new added interface "fast_io_tmo_flags" is a 8-bit mask with
low 5-bit available up to now:
0x01 - Ignore aborting commands
0x02 - Ignore device resets
0x04 - Ignore target resets
0x08 - Ignore bus resets
0x10 - Ignore host resets
When scsi_eh unjams hosts, the corresponding bit fields will be
checked by LLDD to decide whether to ignore specified recovery
levels. Its value is zero by default, so it keeps existing
behavior, which is necessary for non-redundant systems.
This interface is mainly for redundant environments. To
redundant systems, they need a quick give up and failover,
instead of thorough recovery which usually takes much time.
The actions in LLDD/redundant configurations should be implemented
individually later.
Signed-off-by: Ren Mingxin <[email protected]>
---
drivers/scsi/scsi_sas_internal.h | 4 +-
drivers/scsi/scsi_transport_sas.c | 103 ++++++++++++++++++++++++++++++++++++-
include/scsi/scsi_transport_sas.h | 8 +++
3 files changed, 112 insertions(+), 3 deletions(-)
diff --git a/drivers/scsi/scsi_sas_internal.h b/drivers/scsi/scsi_sas_internal.h
index 6266a5d..8c7ab08 100644
--- a/drivers/scsi/scsi_sas_internal.h
+++ b/drivers/scsi/scsi_sas_internal.h
@@ -1,10 +1,10 @@
#ifndef _SCSI_SAS_INTERNAL_H
#define _SCSI_SAS_INTERNAL_H
-#define SAS_HOST_ATTRS 0
+#define SAS_HOST_ATTRS 1
#define SAS_PHY_ATTRS 17
#define SAS_PORT_ATTRS 1
-#define SAS_RPORT_ATTRS 7
+#define SAS_RPORT_ATTRS 8
#define SAS_END_DEV_ATTRS 5
#define SAS_EXPANDER_ATTRS 7
diff --git a/drivers/scsi/scsi_transport_sas.c b/drivers/scsi/scsi_transport_sas.c
index 1b68142..960f3e5 100644
--- a/drivers/scsi/scsi_transport_sas.c
+++ b/drivers/scsi/scsi_transport_sas.c
@@ -37,6 +37,7 @@
#include <scsi/scsi_host.h>
#include <scsi/scsi_transport.h>
#include <scsi/scsi_transport_sas.h>
+#include <scsi/scsi_cmnd.h>
#include "scsi_sas_internal.h"
struct sas_host_attrs {
@@ -46,6 +47,7 @@ struct sas_host_attrs {
u32 next_target_id;
u32 next_expander_id;
int next_port_id;
+ u8 fast_io_tmo_flags;
};
#define to_sas_host_attrs(host) ((struct sas_host_attrs *)(host)->shost_data)
@@ -277,6 +279,59 @@ static void sas_bsg_remove(struct Scsi_Host *shost, struct sas_rphy *rphy)
* SAS host attributes
*/
+static ssize_t
+show_sas_private_host_fast_io_tmo_flags(struct device *dev,
+ struct device_attribute *attr, char *buf)
+{
+ struct Scsi_Host *shost = dev_to_shost(dev);
+ struct sas_host_attrs *sas_host = to_sas_host_attrs(shost);
+
+ return sprintf(buf, "0x%02x\n", sas_host->fast_io_tmo_flags);
+}
+
+static int sas_str_to_fast_io_tmo_flags(const char *buf, u8 *val)
+{
+ char *cp;
+
+ *val = simple_strtoul(buf, &cp, 0) & 0xff;
+ if (cp == buf)
+ return -EINVAL;
+
+ return 0;
+}
+
+static ssize_t
+store_sas_private_host_fast_io_tmo_flags(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf,
+ size_t count)
+{
+ struct Scsi_Host *shost = dev_to_shost(dev);
+ struct sas_host_attrs *sas_host = to_sas_host_attrs(shost);
+ struct sas_rphy *rphy;
+ u8 val;
+ int rc;
+ unsigned long flags;
+
+ if (count < 1)
+ return -EINVAL;
+
+ rc = sas_str_to_fast_io_tmo_flags(buf, &val);
+ if (rc)
+ return rc;
+
+ sas_host->fast_io_tmo_flags = val;
+ spin_lock_irqsave(shost->host_lock, flags);
+ list_for_each_entry(rphy, &sas_host->rphy_list, list)
+ rphy->fast_io_tmo_flags = val;
+ spin_unlock_irqrestore(shost->host_lock, flags);
+ return count;
+}
+
+static SAS_DEVICE_ATTR(host, fast_io_tmo_flags, S_IRUGO | S_IWUSR,
+ show_sas_private_host_fast_io_tmo_flags,
+ store_sas_private_host_fast_io_tmo_flags);
+
static int sas_host_setup(struct transport_container *tc, struct device *dev,
struct device *cdev)
{
@@ -1267,6 +1322,38 @@ sas_rphy_simple_attr(identify.sas_address, sas_address, "0x%016llx\n",
unsigned long long);
sas_rphy_simple_attr(identify.phy_identifier, phy_identifier, "%d\n", u8);
+static ssize_t show_sas_rphy_fast_io_tmo_flags (struct device *dev,
+ struct device_attribute *attr,
+ char *buf)
+{
+ struct sas_rphy *rphy = transport_class_to_rphy(dev);
+
+ return sprintf(buf, "0x%02x\n", rphy->fast_io_tmo_flags);
+}
+
+static ssize_t store_sas_rphy_fast_io_tmo_flags(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf,
+ size_t count)
+{
+ struct sas_rphy *rphy = transport_class_to_rphy(dev);
+ u8 val;
+ int rc;
+
+ if (count < 1)
+ return -EINVAL;
+
+ rc = sas_str_to_fast_io_tmo_flags(buf, &val);
+ if (rc)
+ return rc;
+
+ rphy->fast_io_tmo_flags = val;
+ return count;
+}
+static SAS_DEVICE_ATTR(rphy, fast_io_tmo_flags, S_IRUGO | S_IWUSR,
+ show_sas_rphy_fast_io_tmo_flags,
+ store_sas_rphy_fast_io_tmo_flags);
+
/* only need 8 bytes of data plus header (4 or 8) */
#define BUF_SIZE 64
@@ -1751,9 +1838,18 @@ static int sas_user_scan(struct Scsi_Host *shost, uint channel,
if (test) \
count++
+#define SETUP_TEMPLATE_PRIVATE_RW(attrb, field, perm) \
+ SETUP_TEMPLATE_RW(attrb, field, perm, 1, 0, 0)
+
+#define SETUP_PRIVATE_HOST_ATTRIBUTE_RW(field) \
+ SETUP_TEMPLATE_PRIVATE_RW(host_attrs, field, S_IRUGO | S_IWUSR)
+
#define SETUP_RPORT_ATTRIBUTE(field) \
SETUP_TEMPLATE(rphy_attrs, field, S_IRUGO, 1)
+#define SETUP_PRIVATE_RPORT_ATTRIBUTE_RW(field) \
+ SETUP_TEMPLATE_PRIVATE_RW(rphy_attrs, field, S_IRUGO | S_IWUSR)
+
#define SETUP_OPTIONAL_RPORT_ATTRIBUTE(field, func) \
SETUP_TEMPLATE(rphy_attrs, field, S_IRUGO, i->f->func)
@@ -1762,7 +1858,7 @@ static int sas_user_scan(struct Scsi_Host *shost, uint channel,
#define SETUP_PHY_ATTRIBUTE_RW(field) \
SETUP_TEMPLATE_RW(phy_attrs, field, S_IRUGO | S_IWUSR, 1, \
- !i->f->set_phy_speed, S_IRUGO)
+ !i->f->set_phy_speed, S_IRUGO)
#define SETUP_OPTIONAL_PHY_ATTRIBUTE_RW(field, func) \
SETUP_TEMPLATE_RW(phy_attrs, field, S_IRUGO | S_IWUSR, 1, \
@@ -1836,6 +1932,10 @@ sas_attach_transport(struct sas_function_template *ft)
i->f = ft;
count = 0;
+ SETUP_PRIVATE_HOST_ATTRIBUTE_RW(host_fast_io_tmo_flags);
+ i->host_attrs[count] = NULL;
+
+ count = 0;
SETUP_PHY_ATTRIBUTE(initiator_port_protocols);
SETUP_PHY_ATTRIBUTE(target_port_protocols);
SETUP_PHY_ATTRIBUTE(device_type);
@@ -1867,6 +1967,7 @@ sas_attach_transport(struct sas_function_template *ft)
SETUP_RPORT_ATTRIBUTE(rphy_device_type);
SETUP_RPORT_ATTRIBUTE(rphy_sas_address);
SETUP_RPORT_ATTRIBUTE(rphy_phy_identifier);
+ SETUP_PRIVATE_RPORT_ATTRIBUTE_RW(rphy_fast_io_tmo_flags);
SETUP_OPTIONAL_RPORT_ATTRIBUTE(rphy_enclosure_identifier,
get_enclosure_identifier);
SETUP_OPTIONAL_RPORT_ATTRIBUTE(rphy_bay_identifier,
diff --git a/include/scsi/scsi_transport_sas.h b/include/scsi/scsi_transport_sas.h
index 0bd71e2..06e98b8 100644
--- a/include/scsi/scsi_transport_sas.h
+++ b/include/scsi/scsi_transport_sas.h
@@ -87,8 +87,16 @@ struct sas_rphy {
struct list_head list;
struct request_queue *q;
u32 scsi_target_id;
+ u8 fast_io_tmo_flags;
};
+/* bit field values for struct sas_rphy "fast_io_tmo_flags" field: */
+#define SAS_RPHY_IGN_ABORT_CMDS 0x01
+#define SAS_RPHY_IGN_DEVICE_RESET 0x02
+#define SAS_RPHY_IGN_TARGET_RESET 0x04
+#define SAS_RPHY_IGN_BUS_RESET 0x08
+#define SAS_RPHY_IGN_HOST_RESET 0x10
+
#define dev_to_rphy(d) \
container_of((d), struct sas_rphy, dev)
#define transport_class_to_rphy(dev) \
--
1.7.1
Ren,
Based on the discussion recently held at LSF 2013, we are reworking the
error recovery path to address all the issues you are mentioning. That
work contradicts these patches. So for now, these should be held off.
-- james s
On 5/20/2013 3:14 AM, Ren Mingxin wrote:
> When there is a scsi command timed-out or failed, the scsi eh
> tries a thorugh recovery, which is necessary for non-redundant
> systems. However, the thorugh recovery usually takes much time,
> which is not acceptable for misson critical systems. To improve
> this latency, if we are working on a redundant system, we should
> avoid the scsi eh for its long time failing recovery, and quick
> failover to another path.
>
> This set of patches is trying to implement above.
>
> NOTE: the userland tools need to eusure the environment
> restriction, which will be implemented later.
>
> Thanks,
> Ren
>
> Ren Mingxin (5):
> scsi: rename return code FAST_IO_FAIL to FAST_IO
> FC transport: Add interface to specify fast io level for timed-out cmds
> SAS transport: Add interface to specify fast io level for timed-out cmds
> lpfc: Allow fast timed-out io recovery
> mptfusion: Allow fast timed-out io recovery
>
> drivers/message/fusion/mptscsih.c | 29 ++++++++-
> drivers/scsi/lpfc/lpfc_scsi.c | 34 ++++++++++
> drivers/scsi/scsi_error.c | 18 ++---
> drivers/scsi/scsi_sas_internal.h | 4 -
> drivers/scsi/scsi_transport_fc.c | 112 ++++++++++++++++++++++++++++++++++--
> drivers/scsi/scsi_transport_iscsi.c | 6 -
> drivers/scsi/scsi_transport_sas.c | 103 ++++++++++++++++++++++++++++++++-
> include/scsi/scsi.h | 2
> include/scsi/scsi_transport_fc.h | 11 +++
> include/scsi/scsi_transport_sas.h | 8 ++
> 10 files changed, 303 insertions(+), 24 deletions(-)
>
>
Hi, James,
On 05/20/2013 11:53 PM, James Smart wrote:
> Based on the discussion recently held at LSF 2013, we are
> reworking the error recovery path to address all the issues
> you are mentioning. That work contradicts these patches.
> So for now, these should be held off.
Interesting. Can I have your general goal/idea briefly even
though via a reference? Will the URL below be one you will
refer to?
http://lwn.net/Articles/548500
And, could I know your current progress/schedule? Especially
when can we see your patches?
Much appreciated!
Thanks,
Ren
>
> On 5/20/2013 3:14 AM, Ren Mingxin wrote:
>> When there is a scsi command timed-out or failed, the scsi eh
>> tries a thorugh recovery, which is necessary for non-redundant
>> systems. However, the thorugh recovery usually takes much time,
>> which is not acceptable for misson critical systems. To improve
>> this latency, if we are working on a redundant system, we should
>> avoid the scsi eh for its long time failing recovery, and quick
>> failover to another path.
>>
>> This set of patches is trying to implement above.
>>
>> NOTE: the userland tools need to eusure the environment
>> restriction, which will be implemented later.
>>
>> Thanks,
>> Ren
>>
>> Ren Mingxin (5):
>> scsi: rename return code FAST_IO_FAIL to FAST_IO
>> FC transport: Add interface to specify fast io level for timed-out
>> cmds
>> SAS transport: Add interface to specify fast io level for
>> timed-out cmds
>> lpfc: Allow fast timed-out io recovery
>> mptfusion: Allow fast timed-out io recovery
>>
>> drivers/message/fusion/mptscsih.c | 29 ++++++++-
>> drivers/scsi/lpfc/lpfc_scsi.c | 34 ++++++++++
>> drivers/scsi/scsi_error.c | 18 ++---
>> drivers/scsi/scsi_sas_internal.h | 4 -
>> drivers/scsi/scsi_transport_fc.c | 112
>> ++++++++++++++++++++++++++++++++++--
>> drivers/scsi/scsi_transport_iscsi.c | 6 -
>> drivers/scsi/scsi_transport_sas.c | 103
>> ++++++++++++++++++++++++++++++++-
>> include/scsi/scsi.h | 2
>> include/scsi/scsi_transport_fc.h | 11 +++
>> include/scsi/scsi_transport_sas.h | 8 ++
>> 10 files changed, 303 insertions(+), 24 deletions(-)
yes - that was the session. Granted the posted notes were rather terse.
More of the ideas were presented in this recent email thread:
http://marc.info/?l=linux-scsi&m=136819142000596&w=2
In general - we're going to create a LLD library for error handling,
using paradigms in libsas, that:
- no longer stops the whole host on the 1st error and doesn't start
error handling till all outstanding io is finished/timedout
- sends per-io aborts immediately, and in parallel. LLD handlers will be
asynchronous.
- no lun/target will be stopped until i/o aborts start to fail.
- do smart handling of lun resets, target resets, bus resets, etc and
don't potentially do it for every i/o.
Several of these topics were touched on in the email thread.
the patches are being worked on now - hopefully to be posted as an RFC
within the next couple of weeks.
-- james s
On 5/22/2013 3:12 AM, Ren Mingxin wrote:
> Hi, James,
>
> On 05/20/2013 11:53 PM, James Smart wrote:
>> Based on the discussion recently held at LSF 2013, we are
>> reworking the error recovery path to address all the issues
>> you are mentioning. That work contradicts these patches.
>> So for now, these should be held off.
>
> Interesting. Can I have your general goal/idea briefly even
> though via a reference? Will the URL below be one you will
> refer to?
> http://lwn.net/Articles/548500
>
> And, could I know your current progress/schedule? Especially
> when can we see your patches?
>
> Much appreciated!
>
> Thanks,
> Ren
>
>>
>> On 5/20/2013 3:14 AM, Ren Mingxin wrote:
>>> When there is a scsi command timed-out or failed, the scsi eh
>>> tries a thorugh recovery, which is necessary for non-redundant
>>> systems. However, the thorugh recovery usually takes much time,
>>> which is not acceptable for misson critical systems. To improve
>>> this latency, if we are working on a redundant system, we should
>>> avoid the scsi eh for its long time failing recovery, and quick
>>> failover to another path.
>>>
>>> This set of patches is trying to implement above.
>>>
>>> NOTE: the userland tools need to eusure the environment
>>> restriction, which will be implemented later.
>>>
>>> Thanks,
>>> Ren
>>>
>>> Ren Mingxin (5):
>>> scsi: rename return code FAST_IO_FAIL to FAST_IO
>>> FC transport: Add interface to specify fast io level for
>>> timed-out cmds
>>> SAS transport: Add interface to specify fast io level for
>>> timed-out cmds
>>> lpfc: Allow fast timed-out io recovery
>>> mptfusion: Allow fast timed-out io recovery
>>>
>>> drivers/message/fusion/mptscsih.c | 29 ++++++++-
>>> drivers/scsi/lpfc/lpfc_scsi.c | 34 ++++++++++
>>> drivers/scsi/scsi_error.c | 18 ++---
>>> drivers/scsi/scsi_sas_internal.h | 4 -
>>> drivers/scsi/scsi_transport_fc.c | 112
>>> ++++++++++++++++++++++++++++++++++--
>>> drivers/scsi/scsi_transport_iscsi.c | 6 -
>>> drivers/scsi/scsi_transport_sas.c | 103
>>> ++++++++++++++++++++++++++++++++-
>>> include/scsi/scsi.h | 2
>>> include/scsi/scsi_transport_fc.h | 11 +++
>>> include/scsi/scsi_transport_sas.h | 8 ++
>>> 10 files changed, 303 insertions(+), 24 deletions(-)
> --
> To unsubscribe from this list: send the line "unsubscribe linux-scsi" in
> the body of a message to [email protected]
> More majordomo info at http://vger.kernel.org/majordomo-info.html
>