Return-Path: Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1752123AbbH2BpJ (ORCPT ); Fri, 28 Aug 2015 21:45:09 -0400 Received: from mail-la0-f53.google.com ([209.85.215.53]:35808 "EHLO mail-la0-f53.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1750896AbbH2BpF (ORCPT ); Fri, 28 Aug 2015 21:45:05 -0400 MIME-Version: 1.0 In-Reply-To: <1440774004-27923-7-git-send-email-aleksey.makarov@auriga.com> References: <1440774004-27923-1-git-send-email-aleksey.makarov@auriga.com> <1440774004-27923-7-git-send-email-aleksey.makarov@auriga.com> Date: Sat, 29 Aug 2015 04:45:03 +0300 Message-ID: Subject: Re: [PATCH net-next 7/8] net: thunderx: Support for upto 96 queues for a VF From: Alexey Klimov To: Aleksey Makarov Cc: netdev@vger.kernel.org, Robert Richter , David Daney , Sunil Goutham , Aleksey Makarov , Linux Kernel Mailing List , Robert Richter , Robert Richter , Sunil Goutham , linux-arm-kernel@lists.infradead.org Content-Type: text/plain; charset=UTF-8 Sender: linux-kernel-owner@vger.kernel.org List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Length: 44198 Lines: 1117 On Fri, Aug 28, 2015 at 5:59 PM, Aleksey Makarov wrote: > From: Sunil Goutham > > This patch adds support for handling multiple qsets assigned to a > single VF. There by increasing no of queues from earlier 8 to max > no of CPUs in the system i.e 48 queues on a single node and 96 on > dual node system. User doesn't have option to assign which Qsets/VFs > to be merged. Upon request from VF, PF assigns next free Qsets as > secondary qsets. To maintain current behavior no of queues is kept > to 8 by default which can be increased via ethtool. > > If user wants to unbind NICVF driver from a secondary Qset then it > should be done after tearing down primary VF's interface. > > Signed-off-by: Sunil Goutham > Signed-off-by: Aleksey Makarov > Signed-off-by: Robert Richter > --- > drivers/net/ethernet/cavium/thunder/nic.h | 42 ++++- > drivers/net/ethernet/cavium/thunder/nic_main.c | 173 +++++++++++++++-- > .../net/ethernet/cavium/thunder/nicvf_ethtool.c | 136 +++++++++---- > drivers/net/ethernet/cavium/thunder/nicvf_main.c | 210 +++++++++++++++++++-- > drivers/net/ethernet/cavium/thunder/nicvf_queues.c | 32 +++- > 5 files changed, 507 insertions(+), 86 deletions(-) > > diff --git a/drivers/net/ethernet/cavium/thunder/nic.h b/drivers/net/ethernet/cavium/thunder/nic.h > index 89b997e..35b2ee1 100644 > --- a/drivers/net/ethernet/cavium/thunder/nic.h > +++ b/drivers/net/ethernet/cavium/thunder/nic.h > @@ -258,13 +258,23 @@ struct nicvf_drv_stats { > }; > > struct nicvf { > + struct nicvf *pnicvf; > struct net_device *netdev; > struct pci_dev *pdev; > u8 vf_id; > u8 node; > - u8 tns_mode; > + u8 tns_mode:1; > + u8 sqs_mode:1; > u16 mtu; > struct queue_set *qs; > +#define MAX_SQS_PER_VF_SINGLE_NODE 5 > +#define MAX_SQS_PER_VF 11 > + u8 sqs_id; > + u8 sqs_count; /* Secondary Qset count */ > + struct nicvf *snicvf[MAX_SQS_PER_VF]; > + u8 rx_queues; > + u8 tx_queues; > + u8 max_queues; > void __iomem *reg_base; > bool link_up; > u8 duplex; > @@ -330,14 +340,19 @@ struct nicvf { > #define NIC_MBOX_MSG_RQ_SW_SYNC 0x0F /* Flush inflight pkts to RQ */ > #define NIC_MBOX_MSG_BGX_STATS 0x10 /* Get stats from BGX */ > #define NIC_MBOX_MSG_BGX_LINK_CHANGE 0x11 /* BGX:LMAC link status */ > -#define NIC_MBOX_MSG_CFG_DONE 0x12 /* VF configuration done */ > -#define NIC_MBOX_MSG_SHUTDOWN 0x13 /* VF is being shutdown */ > +#define NIC_MBOX_MSG_ALLOC_SQS 0x12 /* Allocate secondary Qset */ > +#define NIC_MBOX_MSG_NICVF_PTR 0x13 /* Send nicvf ptr to PF */ > +#define NIC_MBOX_MSG_PNICVF_PTR 0x14 /* Get primary qset nicvf ptr */ > +#define NIC_MBOX_MSG_SNICVF_PTR 0x15 /* Send sqet nicvf ptr to PVF */ > +#define NIC_MBOX_MSG_CFG_DONE 0xF0 /* VF configuration done */ > +#define NIC_MBOX_MSG_SHUTDOWN 0xF1 /* VF is being shutdown */ > > struct nic_cfg_msg { > u8 msg; > u8 vf_id; > - u8 tns_mode; > u8 node_id; > + u8 tns_mode:1; > + u8 sqs_mode:1; > u8 mac_addr[ETH_ALEN]; > }; > > @@ -345,6 +360,7 @@ struct nic_cfg_msg { > struct qs_cfg_msg { > u8 msg; > u8 num; > + u8 sqs_count; > u64 cfg; > }; > > @@ -361,6 +377,7 @@ struct sq_cfg_msg { > u8 msg; > u8 qs_num; > u8 sq_num; > + bool sqs_mode; > u64 cfg; > }; > > @@ -420,6 +437,21 @@ struct bgx_link_status { > u32 speed; > }; > > +/* Get Extra Qset IDs */ > +struct sqs_alloc { > + u8 msg; > + u8 vf_id; > + u8 qs_count; > +}; > + > +struct nicvf_ptr { > + u8 msg; > + u8 vf_id; > + bool sqs_mode; > + u8 sqs_id; > + u64 nicvf; > +}; > + > /* 128 bit shared memory between PF and each VF */ > union nic_mbx { > struct { u8 msg; } msg; > @@ -434,6 +466,8 @@ union nic_mbx { > struct rss_cfg_msg rss_cfg; > struct bgx_stats_msg bgx_stats; > struct bgx_link_status link_status; > + struct sqs_alloc sqs_alloc; > + struct nicvf_ptr nicvf; > }; > > #define NIC_NODE_ID_MASK 0x03 > diff --git a/drivers/net/ethernet/cavium/thunder/nic_main.c b/drivers/net/ethernet/cavium/thunder/nic_main.c > index 7dfec4a..51f3048 100644 > --- a/drivers/net/ethernet/cavium/thunder/nic_main.c > +++ b/drivers/net/ethernet/cavium/thunder/nic_main.c > @@ -28,6 +28,11 @@ struct nicpf { > u8 num_vf_en; /* No of VF enabled */ > bool vf_enabled[MAX_NUM_VFS_SUPPORTED]; > void __iomem *reg_base; /* Register start address */ > + u8 num_sqs_en; /* Secondary qsets enabled */ > + u64 nicvf[MAX_NUM_VFS_SUPPORTED]; > + u8 vf_sqs[MAX_NUM_VFS_SUPPORTED][MAX_SQS_PER_VF]; > + u8 pqs_vf[MAX_NUM_VFS_SUPPORTED]; > + bool sqs_used[MAX_NUM_VFS_SUPPORTED]; > struct pkind_cfg pkind; > #define NIC_SET_VF_LMAC_MAP(bgx, lmac) (((bgx & 0xF) << 4) | (lmac & 0xF)) > #define NIC_GET_BGX_FROM_VF_LMAC_MAP(map) ((map >> 4) & 0xF) > @@ -139,13 +144,15 @@ static void nic_mbx_send_ready(struct nicpf *nic, int vf) > > mbx.nic_cfg.tns_mode = NIC_TNS_BYPASS_MODE; > > - bgx_idx = NIC_GET_BGX_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vf]); > - lmac = NIC_GET_LMAC_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vf]); > - > - mac = bgx_get_lmac_mac(nic->node, bgx_idx, lmac); > - if (mac) > - ether_addr_copy((u8 *)&mbx.nic_cfg.mac_addr, mac); > + if (vf < MAX_LMAC) { > + bgx_idx = NIC_GET_BGX_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vf]); > + lmac = NIC_GET_LMAC_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vf]); > > + mac = bgx_get_lmac_mac(nic->node, bgx_idx, lmac); > + if (mac) > + ether_addr_copy((u8 *)&mbx.nic_cfg.mac_addr, mac); > + } > + mbx.nic_cfg.sqs_mode = (vf >= nic->num_vf_en) ? true : false; > mbx.nic_cfg.node_id = nic->node; > nic_send_msg_to_vf(nic, vf, &mbx); > } > @@ -433,6 +440,12 @@ static void nic_config_rss(struct nicpf *nic, struct rss_cfg_msg *cfg) > qset = cfg->vf_id; > > for (; rssi < (rssi_base + cfg->tbl_len); rssi++) { > + u8 svf = cfg->ind_tbl[idx] >> 3; > + > + if (svf) > + qset = nic->vf_sqs[cfg->vf_id][svf - 1]; > + else > + qset = cfg->vf_id; > nic_reg_write(nic, NIC_PF_RSSI_0_4097_RQ | (rssi << 3), > (qset << 3) | (cfg->ind_tbl[idx] & 0x7)); > idx++; > @@ -456,19 +469,31 @@ static void nic_config_rss(struct nicpf *nic, struct rss_cfg_msg *cfg) > * VNIC6-SQ0 -> TL4(528) -> TL3[132] -> TL2[33] -> TL1[1] -> BGX1 > * VNIC7-SQ0 -> TL4(536) -> TL3[134] -> TL2[33] -> TL1[1] -> BGX1 > */ > -static void nic_tx_channel_cfg(struct nicpf *nic, u8 vnic, u8 sq_idx) > +static void nic_tx_channel_cfg(struct nicpf *nic, u8 vnic, > + struct sq_cfg_msg *sq) > { > u32 bgx, lmac, chan; > u32 tl2, tl3, tl4; > u32 rr_quantum; > + u8 sq_idx = sq->sq_num; > + u8 pqs_vnic; > + > + if (sq->sqs_mode) > + pqs_vnic = nic->pqs_vf[vnic]; > + else > + pqs_vnic = vnic; > + > + bgx = NIC_GET_BGX_FROM_VF_LMAC_MAP(nic->vf_lmac_map[pqs_vnic]); > + lmac = NIC_GET_LMAC_FROM_VF_LMAC_MAP(nic->vf_lmac_map[pqs_vnic]); > > - bgx = NIC_GET_BGX_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vnic]); > - lmac = NIC_GET_LMAC_FROM_VF_LMAC_MAP(nic->vf_lmac_map[vnic]); > /* 24 bytes for FCS, IPG and preamble */ > rr_quantum = ((NIC_HW_MAX_FRS + 24) / 4); > > tl4 = (lmac * NIC_TL4_PER_LMAC) + (bgx * NIC_TL4_PER_BGX); > tl4 += sq_idx; > + if (sq->sqs_mode) > + tl4 += vnic * 8; > + > tl3 = tl4 / (NIC_MAX_TL4 / NIC_MAX_TL3); > nic_reg_write(nic, NIC_PF_QSET_0_127_SQ_0_7_CFG2 | > ((u64)vnic << NIC_QS_ID_SHIFT) | > @@ -489,6 +514,71 @@ static void nic_tx_channel_cfg(struct nicpf *nic, u8 vnic, u8 sq_idx) > nic_reg_write(nic, NIC_PF_TL2_0_63_PRI | (tl2 << 3), 0x00); > } > > +/* Send primary nicvf pointer to secondary QS's VF */ > +static void nic_send_pnicvf(struct nicpf *nic, int sqs) > +{ > + union nic_mbx mbx = {}; > + > + mbx.nicvf.msg = NIC_MBOX_MSG_PNICVF_PTR; > + mbx.nicvf.nicvf = nic->nicvf[nic->pqs_vf[sqs]]; > + nic_send_msg_to_vf(nic, sqs, &mbx); > +} > + > +/* Send SQS's nicvf pointer to primary QS's VF */ > +static void nic_send_snicvf(struct nicpf *nic, struct nicvf_ptr *nicvf) > +{ > + union nic_mbx mbx = {}; > + int sqs_id = nic->vf_sqs[nicvf->vf_id][nicvf->sqs_id]; > + > + mbx.nicvf.msg = NIC_MBOX_MSG_SNICVF_PTR; > + mbx.nicvf.sqs_id = nicvf->sqs_id; > + mbx.nicvf.nicvf = nic->nicvf[sqs_id]; > + nic_send_msg_to_vf(nic, nicvf->vf_id, &mbx); > +} > + > +/* Find next available Qset that can be assigned as a > + * secondary Qset to a VF. > + */ > +static int nic_nxt_avail_sqs(struct nicpf *nic) > +{ > + int sqs; > + > + for (sqs = 0; sqs < nic->num_sqs_en; sqs++) { > + if (!nic->sqs_used[sqs]) > + nic->sqs_used[sqs] = true; > + else > + continue; > + return sqs + nic->num_vf_en; > + } > + return -1; > +} > + > +/* Allocate additional Qsets for requested VF */ > +static void nic_alloc_sqs(struct nicpf *nic, struct sqs_alloc *sqs) > +{ > + union nic_mbx mbx = {}; > + int idx, alloc_qs = 0; > + int sqs_id; > + > + if (!nic->num_sqs_en) > + goto send_mbox; > + > + for (idx = 0; idx < sqs->qs_count; idx++) { > + sqs_id = nic_nxt_avail_sqs(nic); > + if (sqs_id < 0) > + break; > + nic->vf_sqs[sqs->vf_id][idx] = sqs_id; > + nic->pqs_vf[sqs_id] = sqs->vf_id; > + alloc_qs++; > + } > + > +send_mbox: > + mbx.sqs_alloc.msg = NIC_MBOX_MSG_ALLOC_SQS; > + mbx.sqs_alloc.vf_id = sqs->vf_id; > + mbx.sqs_alloc.qs_count = alloc_qs; > + nic_send_msg_to_vf(nic, sqs->vf_id, &mbx); > +} > + > /* Interrupt handler to handle mailbox messages from VFs */ > static void nic_handle_mbx_intr(struct nicpf *nic, int vf) > { > @@ -496,6 +586,7 @@ static void nic_handle_mbx_intr(struct nicpf *nic, int vf) > u64 *mbx_data; > u64 mbx_addr; > u64 reg_addr; > + u64 cfg; > int bgx, lmac; > int i; > int ret = 0; > @@ -516,15 +607,24 @@ static void nic_handle_mbx_intr(struct nicpf *nic, int vf) > switch (mbx.msg.msg) { > case NIC_MBOX_MSG_READY: > nic_mbx_send_ready(nic, vf); > - nic->link[vf] = 0; > - nic->duplex[vf] = 0; > - nic->speed[vf] = 0; > + if (vf < MAX_LMAC) { > + nic->link[vf] = 0; > + nic->duplex[vf] = 0; > + nic->speed[vf] = 0; > + } > ret = 1; > break; > case NIC_MBOX_MSG_QS_CFG: > reg_addr = NIC_PF_QSET_0_127_CFG | > (mbx.qs.num << NIC_QS_ID_SHIFT); > - nic_reg_write(nic, reg_addr, mbx.qs.cfg); > + cfg = mbx.qs.cfg; > + /* Check if its a secondary Qset */ > + if (vf >= nic->num_vf_en) { > + cfg = cfg & (~0x7FULL); > + /* Assign this Qset to primary Qset's VF */ > + cfg |= nic->pqs_vf[vf]; > + } > + nic_reg_write(nic, reg_addr, cfg); > break; > case NIC_MBOX_MSG_RQ_CFG: > reg_addr = NIC_PF_QSET_0_127_RQ_0_7_CFG | > @@ -552,9 +652,11 @@ static void nic_handle_mbx_intr(struct nicpf *nic, int vf) > (mbx.sq.qs_num << NIC_QS_ID_SHIFT) | > (mbx.sq.sq_num << NIC_Q_NUM_SHIFT); > nic_reg_write(nic, reg_addr, mbx.sq.cfg); > - nic_tx_channel_cfg(nic, mbx.qs.num, mbx.sq.sq_num); > + nic_tx_channel_cfg(nic, mbx.qs.num, &mbx.sq); > break; > case NIC_MBOX_MSG_SET_MAC: > + if (vf >= nic->num_vf_en) > + break; > lmac = mbx.mac.vf_id; > bgx = NIC_GET_BGX_FROM_VF_LMAC_MAP(nic->vf_lmac_map[lmac]); > lmac = NIC_GET_LMAC_FROM_VF_LMAC_MAP(nic->vf_lmac_map[lmac]); > @@ -581,7 +683,22 @@ static void nic_handle_mbx_intr(struct nicpf *nic, int vf) > case NIC_MBOX_MSG_SHUTDOWN: > /* First msg in VF teardown sequence */ > nic->vf_enabled[vf] = false; > + if (vf >= nic->num_vf_en) > + nic->sqs_used[vf - nic->num_vf_en] = false; > + nic->pqs_vf[vf] = 0; > + break; > + case NIC_MBOX_MSG_ALLOC_SQS: > + nic_alloc_sqs(nic, &mbx.sqs_alloc); > + goto unlock; > + case NIC_MBOX_MSG_NICVF_PTR: > + nic->nicvf[vf] = mbx.nicvf.nicvf; > break; > + case NIC_MBOX_MSG_PNICVF_PTR: > + nic_send_pnicvf(nic, vf); > + goto unlock; > + case NIC_MBOX_MSG_SNICVF_PTR: > + nic_send_snicvf(nic, &mbx.nicvf); > + goto unlock; > case NIC_MBOX_MSG_BGX_STATS: > nic_get_bgx_stats(nic, &mbx.bgx_stats); > goto unlock; > @@ -610,8 +727,7 @@ static void nic_mbx_intr_handler (struct nicpf *nic, int mbx) > if (intr & (1ULL << vf)) { > dev_dbg(&nic->pdev->dev, "Intr from VF %d\n", > vf + (mbx * vf_per_mbx_reg)); > - if ((vf + (mbx * vf_per_mbx_reg)) > nic->num_vf_en) > - break; > + > nic_handle_mbx_intr(nic, vf + (mbx * vf_per_mbx_reg)); > nic_clear_mbx_intr(nic, vf, mbx); > } > @@ -717,9 +833,24 @@ static void nic_unregister_interrupts(struct nicpf *nic) > nic_disable_msix(nic); > } > > +static int nic_num_sqs_en(struct nicpf *nic, int vf_en) > +{ > + int pos = 0, sqs_per_vf = MAX_SQS_PER_VF_SINGLE_NODE; Please check if you really need to initialize 'pos' by zero here. > + u16 total_vf; > + > + /* Check if its a multi-node environment */ > + if (nr_node_ids > 1) > + sqs_per_vf = MAX_SQS_PER_VF; > + > + pos = pci_find_ext_capability(nic->pdev, PCI_EXT_CAP_ID_SRIOV); > + pci_read_config_word(nic->pdev, (pos + PCI_SRIOV_TOTAL_VF), &total_vf); > + return min(total_vf - vf_en, vf_en * sqs_per_vf); > +} > + > static int nic_sriov_init(struct pci_dev *pdev, struct nicpf *nic) > { > int pos = 0; > + int vf_en; > int err; > u16 total_vf_cnt; > > @@ -736,16 +867,20 @@ static int nic_sriov_init(struct pci_dev *pdev, struct nicpf *nic) > if (!total_vf_cnt) > return 0; > > - err = pci_enable_sriov(pdev, nic->num_vf_en); > + vf_en = nic->num_vf_en; > + nic->num_sqs_en = nic_num_sqs_en(nic, nic->num_vf_en); > + vf_en += nic->num_sqs_en; > + > + err = pci_enable_sriov(pdev, vf_en); > if (err) { > dev_err(&pdev->dev, "SRIOV enable failed, num VF is %d\n", > - nic->num_vf_en); > + vf_en); > nic->num_vf_en = 0; > return err; > } > > dev_info(&pdev->dev, "SRIOV enabled, number of VF available %d\n", > - nic->num_vf_en); > + vf_en); > > nic->flags |= NIC_SRIOV_ENABLED; > return 0; > diff --git a/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c b/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c > index 1eec2cd..e4fa98a 100644 > --- a/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c > +++ b/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c > @@ -149,10 +149,33 @@ static void nicvf_set_msglevel(struct net_device *netdev, u32 lvl) > nic->msg_enable = lvl; > } > > +static void nicvf_get_qset_strings(struct nicvf *nic, u8 **data, int qset) > +{ > + int stats, qidx; > + int start_qidx = qset * MAX_RCV_QUEUES_PER_QS; > + > + for (qidx = 0; qidx < nic->qs->rq_cnt; qidx++) { > + for (stats = 0; stats < nicvf_n_queue_stats; stats++) { > + sprintf(*data, "rxq%d: %s", qidx + start_qidx, > + nicvf_queue_stats[stats].name); > + *data += ETH_GSTRING_LEN; > + } > + } > + > + for (qidx = 0; qidx < nic->qs->sq_cnt; qidx++) { > + for (stats = 0; stats < nicvf_n_queue_stats; stats++) { > + sprintf(*data, "txq%d: %s", qidx + start_qidx, > + nicvf_queue_stats[stats].name); > + *data += ETH_GSTRING_LEN; > + } > + } > +} > + > static void nicvf_get_strings(struct net_device *netdev, u32 sset, u8 *data) > { > struct nicvf *nic = netdev_priv(netdev); > - int stats, qidx; > + int stats; > + int sqs; > > if (sset != ETH_SS_STATS) > return; > @@ -167,20 +190,12 @@ static void nicvf_get_strings(struct net_device *netdev, u32 sset, u8 *data) > data += ETH_GSTRING_LEN; > } > > - for (qidx = 0; qidx < nic->qs->rq_cnt; qidx++) { > - for (stats = 0; stats < nicvf_n_queue_stats; stats++) { > - sprintf(data, "rxq%d: %s", qidx, > - nicvf_queue_stats[stats].name); > - data += ETH_GSTRING_LEN; > - } > - } > + nicvf_get_qset_strings(nic, &data, 0); > > - for (qidx = 0; qidx < nic->qs->sq_cnt; qidx++) { > - for (stats = 0; stats < nicvf_n_queue_stats; stats++) { > - sprintf(data, "txq%d: %s", qidx, > - nicvf_queue_stats[stats].name); > - data += ETH_GSTRING_LEN; > - } > + for (sqs = 0; sqs < nic->sqs_count; sqs++) { > + if (!nic->snicvf[sqs]) > + continue; > + nicvf_get_qset_strings(nic->snicvf[sqs], &data, sqs + 1); > } > > for (stats = 0; stats < BGX_RX_STATS_COUNT; stats++) { > @@ -197,21 +212,58 @@ static void nicvf_get_strings(struct net_device *netdev, u32 sset, u8 *data) > static int nicvf_get_sset_count(struct net_device *netdev, int sset) > { > struct nicvf *nic = netdev_priv(netdev); > + int qstats_count; > + int sqs; > > if (sset != ETH_SS_STATS) > return -EINVAL; > > + qstats_count = nicvf_n_queue_stats * > + (nic->qs->rq_cnt + nic->qs->sq_cnt); > + for (sqs = 0; sqs < nic->sqs_count; sqs++) { > + struct nicvf *snic; > + > + snic = nic->snicvf[sqs]; > + if (!snic) > + continue; > + qstats_count += nicvf_n_queue_stats * > + (snic->qs->rq_cnt + snic->qs->sq_cnt); > + } > + > return nicvf_n_hw_stats + nicvf_n_drv_stats + > - (nicvf_n_queue_stats * > - (nic->qs->rq_cnt + nic->qs->sq_cnt)) + > + qstats_count + > BGX_RX_STATS_COUNT + BGX_TX_STATS_COUNT; > } > > +static void nicvf_get_qset_stats(struct nicvf *nic, > + struct ethtool_stats *stats, u64 **data) > +{ > + int stat, qidx; > + > + if (!nic) > + return; > + > + for (qidx = 0; qidx < nic->qs->rq_cnt; qidx++) { > + nicvf_update_rq_stats(nic, qidx); > + for (stat = 0; stat < nicvf_n_queue_stats; stat++) > + *((*data)++) = ((u64 *)&nic->qs->rq[qidx].stats) > + [nicvf_queue_stats[stat].index]; > + } > + > + for (qidx = 0; qidx < nic->qs->sq_cnt; qidx++) { > + nicvf_update_sq_stats(nic, qidx); > + for (stat = 0; stat < nicvf_n_queue_stats; stat++) > + *((*data)++) = ((u64 *)&nic->qs->sq[qidx].stats) > + [nicvf_queue_stats[stat].index]; > + } > +} > + > static void nicvf_get_ethtool_stats(struct net_device *netdev, > struct ethtool_stats *stats, u64 *data) > { > struct nicvf *nic = netdev_priv(netdev); > - int stat, qidx; > + int stat; > + int sqs; > > nicvf_update_stats(nic); > > @@ -225,16 +277,12 @@ static void nicvf_get_ethtool_stats(struct net_device *netdev, > *(data++) = ((u64 *)&nic->drv_stats) > [nicvf_drv_stats[stat].index]; > > - for (qidx = 0; qidx < nic->qs->rq_cnt; qidx++) { > - for (stat = 0; stat < nicvf_n_queue_stats; stat++) > - *(data++) = ((u64 *)&nic->qs->rq[qidx].stats) > - [nicvf_queue_stats[stat].index]; > - } > + nicvf_get_qset_stats(nic, stats, &data); > > - for (qidx = 0; qidx < nic->qs->sq_cnt; qidx++) { > - for (stat = 0; stat < nicvf_n_queue_stats; stat++) > - *(data++) = ((u64 *)&nic->qs->sq[qidx].stats) > - [nicvf_queue_stats[stat].index]; > + for (sqs = 0; sqs < nic->sqs_count; sqs++) { > + if (!nic->snicvf[sqs]) > + continue; > + nicvf_get_qset_stats(nic->snicvf[sqs], stats, &data); > } > > for (stat = 0; stat < BGX_RX_STATS_COUNT; stat++) > @@ -393,7 +441,7 @@ static int nicvf_get_rxnfc(struct net_device *dev, > > switch (info->cmd) { > case ETHTOOL_GRXRINGS: > - info->data = nic->qs->rq_cnt; > + info->data = nic->rx_queues; > ret = 0; > break; > case ETHTOOL_GRXFH: > @@ -556,11 +604,11 @@ static void nicvf_get_channels(struct net_device *dev, > > memset(channel, 0, sizeof(*channel)); > > - channel->max_rx = MAX_RCV_QUEUES_PER_QS; > - channel->max_tx = MAX_SND_QUEUES_PER_QS; > + channel->max_rx = nic->max_queues; > + channel->max_tx = nic->max_queues; > > - channel->rx_count = nic->qs->rq_cnt; > - channel->tx_count = nic->qs->sq_cnt; > + channel->rx_count = nic->rx_queues; > + channel->tx_count = nic->tx_queues; > } > > /* Set no of Tx, Rx queues to be used */ > @@ -568,24 +616,36 @@ static int nicvf_set_channels(struct net_device *dev, > struct ethtool_channels *channel) > { > struct nicvf *nic = netdev_priv(dev); > - int err = 0; > bool if_up = netif_running(dev); > + int cqcount; > + int err = 0; Probably this initialization is not needed. > > if (!channel->rx_count || !channel->tx_count) > return -EINVAL; > - if (channel->rx_count > MAX_RCV_QUEUES_PER_QS) > + if (channel->rx_count > nic->max_queues) > return -EINVAL; > - if (channel->tx_count > MAX_SND_QUEUES_PER_QS) > + if (channel->tx_count > nic->max_queues) > return -EINVAL; > > if (if_up) > nicvf_stop(dev); > > - nic->qs->rq_cnt = channel->rx_count; > - nic->qs->sq_cnt = channel->tx_count; > + cqcount = max(channel->rx_count, channel->tx_count); > + > + if (cqcount > MAX_CMP_QUEUES_PER_QS) { > + nic->sqs_count = roundup(cqcount, MAX_CMP_QUEUES_PER_QS); > + nic->sqs_count = (nic->sqs_count / MAX_CMP_QUEUES_PER_QS) - 1; > + } else { > + nic->sqs_count = 0; > + } > + > + nic->qs->rq_cnt = min_t(u32, channel->rx_count, MAX_RCV_QUEUES_PER_QS); > + nic->qs->sq_cnt = min_t(u32, channel->tx_count, MAX_SND_QUEUES_PER_QS); > nic->qs->cq_cnt = max(nic->qs->rq_cnt, nic->qs->sq_cnt); > > - err = nicvf_set_real_num_queues(dev, nic->qs->sq_cnt, nic->qs->rq_cnt); > + nic->rx_queues = channel->rx_count; > + nic->tx_queues = channel->tx_count; > + err = nicvf_set_real_num_queues(dev, nic->tx_queues, nic->rx_queues); > if (err) > return err; > > @@ -593,7 +653,7 @@ static int nicvf_set_channels(struct net_device *dev, > nicvf_open(dev); > > netdev_info(dev, "Setting num Tx rings to %d, Rx rings to %d success\n", > - nic->qs->sq_cnt, nic->qs->rq_cnt); > + nic->tx_queues, nic->rx_queues); > > return err; > } > diff --git a/drivers/net/ethernet/cavium/thunder/nicvf_main.c b/drivers/net/ethernet/cavium/thunder/nicvf_main.c > index 2198f61..9a1091a 100644 > --- a/drivers/net/ethernet/cavium/thunder/nicvf_main.c > +++ b/drivers/net/ethernet/cavium/thunder/nicvf_main.c > @@ -51,6 +51,14 @@ module_param(cpi_alg, int, S_IRUGO); > MODULE_PARM_DESC(cpi_alg, > "PFC algorithm (0=none, 1=VLAN, 2=VLAN16, 3=IP Diffserv)"); > > +static inline u8 nicvf_netdev_qidx(struct nicvf *nic, u8 qidx) > +{ > + if (nic->sqs_mode) > + return qidx + ((nic->sqs_id + 1) * MAX_CMP_QUEUES_PER_QS); > + else > + return qidx; > +} > + > static inline void nicvf_set_rx_frame_cnt(struct nicvf *nic, > struct sk_buff *skb) > { > @@ -193,6 +201,7 @@ static void nicvf_handle_mbx_intr(struct nicvf *nic) > if (!nic->set_mac_pending) > ether_addr_copy(nic->netdev->dev_addr, > mbx.nic_cfg.mac_addr); > + nic->sqs_mode = mbx.nic_cfg.sqs_mode; > nic->link_up = false; > nic->duplex = 0; > nic->speed = 0; > @@ -230,6 +239,26 @@ static void nicvf_handle_mbx_intr(struct nicvf *nic) > netif_tx_stop_all_queues(nic->netdev); > } > break; > + case NIC_MBOX_MSG_ALLOC_SQS: > + nic->sqs_count = mbx.sqs_alloc.qs_count; > + nic->pf_acked = true; > + break; > + case NIC_MBOX_MSG_SNICVF_PTR: > + /* Primary VF: make note of secondary VF's pointer > + * to be used while packet transmission. > + */ > + nic->snicvf[mbx.nicvf.sqs_id] = > + (struct nicvf *)mbx.nicvf.nicvf; > + nic->pf_acked = true; > + break; > + case NIC_MBOX_MSG_PNICVF_PTR: > + /* Secondary VF/Qset: make note of primary VF's pointer > + * to be used while packet reception, to handover packet > + * to primary VF's netdev. > + */ > + nic->pnicvf = (struct nicvf *)mbx.nicvf.nicvf; > + nic->pf_acked = true; > + break; > default: > netdev_err(nic->netdev, > "Invalid message from PF, msg 0x%x\n", mbx.msg.msg); > @@ -338,11 +367,100 @@ static int nicvf_rss_init(struct nicvf *nic) > > for (idx = 0; idx < rss->rss_size; idx++) > rss->ind_tbl[idx] = ethtool_rxfh_indir_default(idx, > - nic->qs->rq_cnt); > + nic->rx_queues); > nicvf_config_rss(nic); > return 1; > } > > +/* Request PF to allocate additional Qsets */ > +static void nicvf_request_sqs(struct nicvf *nic) > +{ > + union nic_mbx mbx = {}; > + int sqs; > + int sqs_count = nic->sqs_count; > + int rx_queues = 0, tx_queues = 0; > + > + /* Only primary VF should request */ > + if (nic->sqs_mode || !nic->sqs_count) > + return; > + > + mbx.sqs_alloc.msg = NIC_MBOX_MSG_ALLOC_SQS; > + mbx.sqs_alloc.vf_id = nic->vf_id; > + mbx.sqs_alloc.qs_count = nic->sqs_count; > + if (nicvf_send_msg_to_pf(nic, &mbx)) { > + /* No response from PF */ > + nic->sqs_count = 0; > + return; > + } > + > + /* Return if no Secondary Qsets available */ > + if (!nic->sqs_count) > + return; > + > + if (nic->rx_queues > MAX_RCV_QUEUES_PER_QS) > + rx_queues = nic->rx_queues - MAX_RCV_QUEUES_PER_QS; > + if (nic->tx_queues > MAX_SND_QUEUES_PER_QS) > + tx_queues = nic->tx_queues - MAX_SND_QUEUES_PER_QS; > + > + /* Set no of Rx/Tx queues in each of the SQsets */ > + for (sqs = 0; sqs < nic->sqs_count; sqs++) { > + mbx.nicvf.msg = NIC_MBOX_MSG_SNICVF_PTR; > + mbx.nicvf.vf_id = nic->vf_id; > + mbx.nicvf.sqs_id = sqs; > + nicvf_send_msg_to_pf(nic, &mbx); > + > + nic->snicvf[sqs]->sqs_id = sqs; > + if (rx_queues > MAX_RCV_QUEUES_PER_QS) { > + nic->snicvf[sqs]->qs->rq_cnt = MAX_RCV_QUEUES_PER_QS; > + rx_queues -= MAX_RCV_QUEUES_PER_QS; > + } else { > + nic->snicvf[sqs]->qs->rq_cnt = rx_queues; > + rx_queues = 0; > + } > + > + if (tx_queues > MAX_SND_QUEUES_PER_QS) { > + nic->snicvf[sqs]->qs->sq_cnt = MAX_SND_QUEUES_PER_QS; > + tx_queues -= MAX_SND_QUEUES_PER_QS; > + } else { > + nic->snicvf[sqs]->qs->sq_cnt = tx_queues; > + tx_queues = 0; > + } > + > + nic->snicvf[sqs]->qs->cq_cnt = > + max(nic->snicvf[sqs]->qs->rq_cnt, nic->snicvf[sqs]->qs->sq_cnt); > + > + /* Initialize secondary Qset's queues and its interrupts */ > + nicvf_open(nic->snicvf[sqs]->netdev); > + } > + > + /* Update stack with actual Rx/Tx queue count allocated */ > + if (sqs_count != nic->sqs_count) > + nicvf_set_real_num_queues(nic->netdev, > + nic->tx_queues, nic->rx_queues); > +} > + > +/* Send this Qset's nicvf pointer to PF. > + * PF inturn sends primary VF's nicvf struct to secondary Qsets/VFs > + * so that packets received by these Qsets can use primary VF's netdev > + */ > +static void nicvf_send_vf_struct(struct nicvf *nic) > +{ > + union nic_mbx mbx = {}; > + > + mbx.nicvf.msg = NIC_MBOX_MSG_NICVF_PTR; > + mbx.nicvf.sqs_mode = nic->sqs_mode; > + mbx.nicvf.nicvf = (u64)nic; > + nicvf_send_msg_to_pf(nic, &mbx); > +} > + > +static void nicvf_get_primary_vf_struct(struct nicvf *nic) > +{ > + union nic_mbx mbx = {}; > + > + mbx.nicvf.msg = NIC_MBOX_MSG_PNICVF_PTR; > + nicvf_send_msg_to_pf(nic, &mbx); > +} > + > int nicvf_set_real_num_queues(struct net_device *netdev, > int tx_queues, int rx_queues) > { > @@ -453,6 +571,15 @@ static void nicvf_rcv_pkt_handler(struct net_device *netdev, > struct sk_buff *skb; > struct nicvf *nic = netdev_priv(netdev); > int err = 0; > + int rq_idx; > + > + rq_idx = nicvf_netdev_qidx(nic, cqe_rx->rq_idx); > + > + if (nic->sqs_mode) { > + /* Use primary VF's 'nicvf' struct */ > + nic = nic->pnicvf; > + netdev = nic->netdev; > + } > > /* Check for errors */ > err = nicvf_check_cqe_rx_errs(nic, cq, cqe_rx); > @@ -482,7 +609,7 @@ static void nicvf_rcv_pkt_handler(struct net_device *netdev, > > nicvf_set_rxhash(netdev, cqe_rx, skb); > > - skb_record_rx_queue(skb, cqe_rx->rq_idx); > + skb_record_rx_queue(skb, rq_idx); > if (netdev->hw_features & NETIF_F_RXCSUM) { > /* HW by default verifies TCP/UDP/SCTP checksums */ > skb->ip_summed = CHECKSUM_UNNECESSARY; > @@ -578,8 +705,11 @@ loop: > done: > /* Wakeup TXQ if its stopped earlier due to SQ full */ > if (tx_done) { > - txq = netdev_get_tx_queue(netdev, cq_idx); > - if (netif_tx_queue_stopped(txq)) { > + netdev = nic->pnicvf->netdev; > + txq = netdev_get_tx_queue(netdev, > + nicvf_netdev_qidx(nic, cq_idx)); > + nic = nic->pnicvf; > + if (netif_tx_queue_stopped(txq) && netif_carrier_ok(netdev)) { > netif_tx_start_queue(txq); > nic->drv_stats.txq_wake++; > if (netif_msg_tx_err(nic)) > @@ -893,7 +1023,6 @@ static netdev_tx_t nicvf_xmit(struct sk_buff *skb, struct net_device *netdev) > netdev_warn(netdev, > "%s: Transmit ring full, stopping SQ%d\n", > netdev->name, qid); > - > return NETDEV_TX_BUSY; > } > > @@ -926,6 +1055,17 @@ int nicvf_stop(struct net_device *netdev) > nicvf_send_msg_to_pf(nic, &mbx); > > netif_carrier_off(netdev); > + netif_tx_stop_all_queues(nic->netdev); > + > + /* Teardown secondary qsets first */ > + if (!nic->sqs_mode) { > + for (qidx = 0; qidx < nic->sqs_count; qidx++) { > + if (!nic->snicvf[qidx]) > + continue; > + nicvf_stop(nic->snicvf[qidx]->netdev); > + nic->snicvf[qidx] = NULL; > + } > + } > > /* Disable RBDR & QS error interrupts */ > for (qidx = 0; qidx < qs->rbdr_cnt; qidx++) { > @@ -973,6 +1113,10 @@ int nicvf_stop(struct net_device *netdev) > > nicvf_free_cq_poll(nic); > > + /* Clear multiqset info */ > + nic->pnicvf = nic; > + nic->sqs_count = 0; > + > return 0; > } > > @@ -1028,10 +1172,16 @@ int nicvf_open(struct net_device *netdev) > > /* Configure CPI alorithm */ > nic->cpi_alg = cpi_alg; > - nicvf_config_cpi(nic); > + if (!nic->sqs_mode) > + nicvf_config_cpi(nic); > + > + nicvf_request_sqs(nic); > + if (nic->sqs_mode) > + nicvf_get_primary_vf_struct(nic); > > /* Configure receive side scaling */ > - nicvf_rss_init(nic); > + if (!nic->sqs_mode) > + nicvf_rss_init(nic); > > err = nicvf_register_interrupts(nic); > if (err) > @@ -1282,8 +1432,7 @@ static int nicvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) > struct device *dev = &pdev->dev; > struct net_device *netdev; > struct nicvf *nic; > - struct queue_set *qs; > - int err; > + int err, qcount; > > err = pci_enable_device(pdev); > if (err) { > @@ -1309,9 +1458,17 @@ static int nicvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) > goto err_release_regions; > } > > - netdev = alloc_etherdev_mqs(sizeof(struct nicvf), > - MAX_RCV_QUEUES_PER_QS, > - MAX_SND_QUEUES_PER_QS); > + qcount = MAX_CMP_QUEUES_PER_QS; > + > + /* Restrict multiqset support only for host bound VFs */ > + if (pdev->is_virtfn) { > + /* Set max number of queues per VF */ > + qcount = roundup(num_online_cpus(), MAX_CMP_QUEUES_PER_QS); > + qcount = min(qcount, > + (MAX_SQS_PER_VF + 1) * MAX_CMP_QUEUES_PER_QS); > + } > + > + netdev = alloc_etherdev_mqs(sizeof(struct nicvf), qcount, qcount); > if (!netdev) { > err = -ENOMEM; > goto err_release_regions; > @@ -1324,6 +1481,8 @@ static int nicvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) > nic = netdev_priv(netdev); > nic->netdev = netdev; > nic->pdev = pdev; > + nic->pnicvf = nic; > + nic->max_queues = qcount; > > /* MAP VF's configuration registers */ > nic->reg_base = pcim_iomap(pdev, PCI_CFG_REG_BAR_NUM, 0); > @@ -1337,20 +1496,26 @@ static int nicvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) > if (err) > goto err_free_netdev; > > - qs = nic->qs; > - > - err = nicvf_set_real_num_queues(netdev, qs->sq_cnt, qs->rq_cnt); > - if (err) > - goto err_free_netdev; > - > /* Check if PF is alive and get MAC address for this VF */ > err = nicvf_register_misc_interrupt(nic); > if (err) > goto err_free_netdev; > > + nicvf_send_vf_struct(nic); > + > + /* Check if this VF is in QS only mode */ > + if (nic->sqs_mode) > + return 0; > + > + err = nicvf_set_real_num_queues(netdev, nic->tx_queues, nic->rx_queues); > + if (err) > + goto err_unregister_interrupts; > + > netdev->hw_features = (NETIF_F_RXCSUM | NETIF_F_IP_CSUM | NETIF_F_SG | > NETIF_F_TSO | NETIF_F_GRO | > - NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_RXHASH); > + NETIF_F_HW_VLAN_CTAG_RX); > + > + netdev->hw_features |= NETIF_F_RXHASH; > > netdev->features |= netdev->hw_features; > > @@ -1389,8 +1554,13 @@ static void nicvf_remove(struct pci_dev *pdev) > { > struct net_device *netdev = pci_get_drvdata(pdev); > struct nicvf *nic = netdev_priv(netdev); > + struct net_device *pnetdev = nic->pnicvf->netdev; > > - unregister_netdev(netdev); > + /* Check if this Qset is assigned to different VF. > + * If yes, clean primary and all secondary Qsets. > + */ > + if (pnetdev && (pnetdev->reg_state == NETREG_REGISTERED)) > + unregister_netdev(pnetdev); > nicvf_unregister_interrupts(nic); > pci_set_drvdata(pdev, NULL); > free_netdev(netdev); > diff --git a/drivers/net/ethernet/cavium/thunder/nicvf_queues.c b/drivers/net/ethernet/cavium/thunder/nicvf_queues.c > index b294d67..63a870a 100644 > --- a/drivers/net/ethernet/cavium/thunder/nicvf_queues.c > +++ b/drivers/net/ethernet/cavium/thunder/nicvf_queues.c > @@ -478,7 +478,7 @@ static void nicvf_reclaim_rbdr(struct nicvf *nic, > void nicvf_config_vlan_stripping(struct nicvf *nic, netdev_features_t features) > { > u64 rq_cfg; > - int sqs; > + int sqs = 0; > > rq_cfg = nicvf_queue_reg_read(nic, NIC_QSET_RQ_GEN_CFG, 0); > This change in nicvf_config_vlan_stripping() probably is not part of this patch. It fits better into 5/8. But please check if you really need this change at all. > @@ -621,6 +621,7 @@ static void nicvf_snd_queue_config(struct nicvf *nic, struct queue_set *qs, > mbx.sq.msg = NIC_MBOX_MSG_SQ_CFG; > mbx.sq.qs_num = qs->vnic_id; > mbx.sq.sq_num = qidx; > + mbx.sq.sqs_mode = nic->sqs_mode; > mbx.sq.cfg = (sq->cq_qs << 3) | sq->cq_idx; > nicvf_send_msg_to_pf(nic, &mbx); > > @@ -702,6 +703,7 @@ void nicvf_qset_config(struct nicvf *nic, bool enable) > /* Send a mailbox msg to PF to config Qset */ > mbx.qs.msg = NIC_MBOX_MSG_QS_CFG; > mbx.qs.num = qs->vnic_id; > + mbx.qs.sqs_count = nic->sqs_count; > > mbx.qs.cfg = 0; > qs_cfg = (struct qs_cfg *)&mbx.qs.cfg; > @@ -782,6 +784,10 @@ int nicvf_set_qset_resources(struct nicvf *nic) > qs->rbdr_len = RCV_BUF_COUNT; > qs->sq_len = SND_QUEUE_LEN; > qs->cq_len = CMP_QUEUE_LEN; > + > + nic->rx_queues = qs->rq_cnt; > + nic->tx_queues = qs->sq_cnt; > + > return 0; > } > > @@ -1025,7 +1031,7 @@ static inline void nicvf_sq_add_gather_subdesc(struct snd_queue *sq, int qentry, > * them to SQ for transfer > */ > static int nicvf_sq_append_tso(struct nicvf *nic, struct snd_queue *sq, > - int qentry, struct sk_buff *skb) > + int sq_num, int qentry, struct sk_buff *skb) > { > struct tso_t tso; > int seg_subdescs = 0, desc_cnt = 0; > @@ -1085,7 +1091,7 @@ static int nicvf_sq_append_tso(struct nicvf *nic, struct snd_queue *sq, > > /* Inform HW to xmit all TSO segments */ > nicvf_queue_reg_write(nic, NIC_QSET_SQ_0_7_DOOR, > - skb_get_queue_mapping(skb), desc_cnt); > + sq_num, desc_cnt); > nic->drv_stats.tx_tso++; > return 1; > } > @@ -1096,10 +1102,24 @@ int nicvf_sq_append_skb(struct nicvf *nic, struct sk_buff *skb) > int i, size; > int subdesc_cnt; > int sq_num, qentry; > - struct queue_set *qs = nic->qs; > + struct queue_set *qs; > struct snd_queue *sq; > > sq_num = skb_get_queue_mapping(skb); > + if (sq_num >= MAX_SND_QUEUES_PER_QS) { > + /* Get secondary Qset's SQ structure */ > + i = sq_num / MAX_SND_QUEUES_PER_QS; > + if (!nic->snicvf[i - 1]) { > + netdev_warn(nic->netdev, > + "Secondary Qset#%d's ptr not initialized\n", > + i - 1); > + return 1; > + } > + nic = (struct nicvf *)nic->snicvf[i - 1]; > + sq_num = sq_num % MAX_SND_QUEUES_PER_QS; > + } > + > + qs = nic->qs; > sq = &qs->sq[sq_num]; > > subdesc_cnt = nicvf_sq_subdesc_required(nic, skb); > @@ -1110,7 +1130,7 @@ int nicvf_sq_append_skb(struct nicvf *nic, struct sk_buff *skb) > > /* Check if its a TSO packet */ > if (skb_shinfo(skb)->gso_size) > - return nicvf_sq_append_tso(nic, sq, qentry, skb); > + return nicvf_sq_append_tso(nic, sq, sq_num, qentry, skb); > > /* Add SQ header subdesc */ > nicvf_sq_add_hdr_subdesc(sq, qentry, subdesc_cnt - 1, skb, skb->len); > @@ -1146,6 +1166,8 @@ doorbell: > return 1; > > append_fail: > + /* Use original PCI dev for debug log */ > + nic = nic->pnicvf; > netdev_dbg(nic->netdev, "Not enough SQ descriptors to xmit pkt\n"); > return 0; > } > -- > 2.5.0 > > > _______________________________________________ > linux-arm-kernel mailing list > linux-arm-kernel@lists.infradead.org > http://lists.infradead.org/mailman/listinfo/linux-arm-kernel -- Best regards, Klimov Alexey -- To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majordomo@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/